Skip to content

Commit 8ac6bdc

Browse files
buchgrcopybara-github
buchgr
authored and
copybara-github
committed
Refactor SimpleBlobStore#setActionResult signature to match that of
GrpcRemoteCache#setCachedActionResult This is another step towards merging GrpcRemoteCache and SimpleBlobStoreActionCache. Closes bazelbuild#9168. PiperOrigin-RevId: 263343360
1 parent 0fdb1f4 commit 8ac6bdc

21 files changed

Lines changed: 67 additions & 54 deletions

‎src/main/java/com/google/devtools/build/lib/remote/AbstractRemoteActionCache.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,8 @@
5252
import com.google.devtools.build.lib.remote.AbstractRemoteActionCache.ActionResultMetadata.DirectoryMetadata;
5353
import com.google.devtools.build.lib.remote.AbstractRemoteActionCache.ActionResultMetadata.FileMetadata;
5454
import com.google.devtools.build.lib.remote.AbstractRemoteActionCache.ActionResultMetadata.SymlinkMetadata;
55+
import com.google.devtools.build.lib.remote.common.SimpleBlobStore;
56+
import com.google.devtools.build.lib.remote.common.SimpleBlobStore.ActionKey;
5557
import com.google.devtools.build.lib.remote.options.RemoteOptions;
5658
import com.google.devtools.build.lib.remote.util.DigestUtil;
5759
import com.google.devtools.build.lib.remote.util.Utils;
@@ -115,7 +117,7 @@ public AbstractRemoteActionCache(RemoteOptions options, DigestUtil digestUtil) {
115117
* @throws IOException if the remote cache is unavailable.
116118
*/
117119
@Nullable
118-
abstract ActionResult getCachedActionResult(DigestUtil.ActionKey actionKey)
120+
abstract ActionResult getCachedActionResult(ActionKey actionKey)
119121
throws IOException, InterruptedException;
120122

121123
/**
@@ -125,7 +127,7 @@ abstract ActionResult getCachedActionResult(DigestUtil.ActionKey actionKey)
125127
* @throws ExecException if uploading any of the action outputs is not supported
126128
*/
127129
abstract void upload(
128-
DigestUtil.ActionKey actionKey,
130+
SimpleBlobStore.ActionKey actionKey,
129131
Action action,
130132
Command command,
131133
Path execRoot,
@@ -753,7 +755,7 @@ public void addFiles(Collection<Path> files) throws ExecException, IOException {
753755
* Adds an action and command protos to upload. They need to be uploaded as part of the action
754756
* result.
755757
*/
756-
public void addAction(DigestUtil.ActionKey actionKey, Action action, Command command) {
758+
public void addAction(SimpleBlobStore.ActionKey actionKey, Action action, Command command) {
757759
digestToBlobs.put(actionKey.getDigest(), action.toByteString());
758760
digestToBlobs.put(action.getCommandDigest(), command.toByteString());
759761
}

‎src/main/java/com/google/devtools/build/lib/remote/GrpcRemoteCache.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,10 +53,10 @@
5353
import com.google.devtools.build.lib.actions.ExecException;
5454
import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
5555
import com.google.devtools.build.lib.remote.RemoteRetrier.ProgressiveBackoff;
56+
import com.google.devtools.build.lib.remote.common.SimpleBlobStore.ActionKey;
5657
import com.google.devtools.build.lib.remote.merkletree.MerkleTree;
5758
import com.google.devtools.build.lib.remote.options.RemoteOptions;
5859
import com.google.devtools.build.lib.remote.util.DigestUtil;
59-
import com.google.devtools.build.lib.remote.util.DigestUtil.ActionKey;
6060
import com.google.devtools.build.lib.remote.util.TracingMetadataUtils;
6161
import com.google.devtools.build.lib.util.io.FileOutErr;
6262
import com.google.devtools.build.lib.vfs.Path;

‎src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,11 +45,11 @@
4545
import com.google.devtools.build.lib.profiler.Profiler;
4646
import com.google.devtools.build.lib.profiler.ProfilerTask;
4747
import com.google.devtools.build.lib.profiler.SilentCloseable;
48+
import com.google.devtools.build.lib.remote.common.SimpleBlobStore.ActionKey;
4849
import com.google.devtools.build.lib.remote.merkletree.MerkleTree;
4950
import com.google.devtools.build.lib.remote.options.RemoteOptions;
5051
import com.google.devtools.build.lib.remote.options.RemoteOutputsMode;
5152
import com.google.devtools.build.lib.remote.util.DigestUtil;
52-
import com.google.devtools.build.lib.remote.util.DigestUtil.ActionKey;
5353
import com.google.devtools.build.lib.remote.util.TracingMetadataUtils;
5454
import com.google.devtools.build.lib.remote.util.Utils;
5555
import com.google.devtools.build.lib.remote.util.Utils.InMemoryOutput;

‎src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,11 +56,11 @@
5656
import com.google.devtools.build.lib.profiler.Profiler;
5757
import com.google.devtools.build.lib.profiler.ProfilerTask;
5858
import com.google.devtools.build.lib.profiler.SilentCloseable;
59+
import com.google.devtools.build.lib.remote.common.SimpleBlobStore.ActionKey;
5960
import com.google.devtools.build.lib.remote.merkletree.MerkleTree;
6061
import com.google.devtools.build.lib.remote.options.RemoteOptions;
6162
import com.google.devtools.build.lib.remote.options.RemoteOutputsMode;
6263
import com.google.devtools.build.lib.remote.util.DigestUtil;
63-
import com.google.devtools.build.lib.remote.util.DigestUtil.ActionKey;
6464
import com.google.devtools.build.lib.remote.util.TracingMetadataUtils;
6565
import com.google.devtools.build.lib.remote.util.Utils;
6666
import com.google.devtools.build.lib.remote.util.Utils.InMemoryOutput;

‎src/main/java/com/google/devtools/build/lib/remote/SimpleBlobStoreActionCache.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,9 @@
3030
import com.google.devtools.build.lib.actions.ExecException;
3131
import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
3232
import com.google.devtools.build.lib.remote.common.SimpleBlobStore;
33+
import com.google.devtools.build.lib.remote.common.SimpleBlobStore.ActionKey;
3334
import com.google.devtools.build.lib.remote.options.RemoteOptions;
3435
import com.google.devtools.build.lib.remote.util.DigestUtil;
35-
import com.google.devtools.build.lib.remote.util.DigestUtil.ActionKey;
3636
import com.google.devtools.build.lib.remote.util.Utils;
3737
import com.google.devtools.build.lib.util.io.FileOutErr;
3838
import com.google.devtools.build.lib.vfs.Path;
@@ -78,7 +78,7 @@ public void downloadTree(Digest rootDigest, Path rootLocation)
7878

7979
@Override
8080
public void upload(
81-
DigestUtil.ActionKey actionKey,
81+
SimpleBlobStore.ActionKey actionKey,
8282
Action action,
8383
Command command,
8484
Path execRoot,
@@ -97,12 +97,12 @@ public void upload(
9797
getFromFuture(uploadFile(stdoutDigest, outErr.getOutputPath()));
9898
result.setStdoutDigest(stdoutDigest);
9999
}
100-
blobStore.putActionResult(actionKey.getDigest().getHash(), result.build().toByteArray());
100+
blobStore.putActionResult(actionKey, result.build());
101101
}
102102

103103
public void upload(
104104
ActionResult.Builder result,
105-
DigestUtil.ActionKey actionKey,
105+
SimpleBlobStore.ActionKey actionKey,
106106
Action action,
107107
Command command,
108108
Path execRoot,
@@ -170,7 +170,7 @@ private byte[] downloadActionResult(Digest digest) throws IOException, Interrupt
170170

171171
public void setCachedActionResult(ActionKey actionKey, ActionResult result)
172172
throws IOException, InterruptedException {
173-
blobStore.putActionResult(actionKey.getDigest().getHash(), result.toByteArray());
173+
blobStore.putActionResult(actionKey, result);
174174
}
175175

176176
@Override

‎src/main/java/com/google/devtools/build/lib/remote/blobstore/ConcurrentMapBlobStore.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
// limitations under the License.
1414
package com.google.devtools.build.lib.remote.blobstore;
1515

16+
import build.bazel.remote.execution.v2.ActionResult;
1617
import build.bazel.remote.execution.v2.Digest;
1718
import com.google.common.base.Preconditions;
1819
import com.google.common.io.ByteStreams;
@@ -69,8 +70,8 @@ public ListenableFuture<Boolean> getActionResult(String key, OutputStream out) {
6970
}
7071

7172
@Override
72-
public void putActionResult(String key, byte[] in) {
73-
map.put(ACTION_KEY_PREFIX + key, in);
73+
public void putActionResult(ActionKey actionKey, ActionResult actionResult) {
74+
map.put(ACTION_KEY_PREFIX + actionKey.getDigest().getHash(), actionResult.toByteArray());
7475
}
7576

7677
@Override

‎src/main/java/com/google/devtools/build/lib/remote/common/SimpleBlobStore.java‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414

1515
package com.google.devtools.build.lib.remote.common;
1616

17+
import build.bazel.remote.execution.v2.ActionResult;
1718
import build.bazel.remote.execution.v2.Digest;
1819
import com.google.common.util.concurrent.ListenableFuture;
1920
import com.google.devtools.build.lib.vfs.Path;
@@ -27,6 +28,23 @@
2728
* <p>Implementations must be thread-safe.
2829
*/
2930
public interface SimpleBlobStore {
31+
32+
/**
33+
* A special type of Digest that is used only as a remote action cache key. This is a separate
34+
* type in order to prevent accidentally using other Digests as action keys.
35+
*/
36+
final class ActionKey {
37+
private final Digest digest;
38+
39+
public Digest getDigest() {
40+
return digest;
41+
}
42+
43+
public ActionKey(Digest digest) {
44+
this.digest = digest;
45+
}
46+
}
47+
3048
/** Returns {@code true} if the provided {@code key} is stored in the CAS. */
3149
boolean contains(String key) throws IOException, InterruptedException;
3250

@@ -52,8 +70,9 @@ public interface SimpleBlobStore {
5270
*/
5371
ListenableFuture<Boolean> getActionResult(String actionKey, OutputStream out);
5472

55-
/** Uploads a bytearray BLOB (as {@code in}) indexed by {@code key} to the Action Cache. */
56-
void putActionResult(String actionKey, byte[] in) throws IOException, InterruptedException;
73+
/** Uploads an {@link ActionResult} keyed by the action hash to the action cache. */
74+
void putActionResult(ActionKey actionDigest, ActionResult actionResult)
75+
throws IOException, InterruptedException;
5776

5877
/** Close resources associated with the blob store. */
5978
void close();

‎src/main/java/com/google/devtools/build/lib/remote/disk/CombinedDiskHttpBlobStore.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
// limitations under the License.
1414
package com.google.devtools.build.lib.remote.disk;
1515

16+
import build.bazel.remote.execution.v2.ActionResult;
1617
import build.bazel.remote.execution.v2.Digest;
1718
import com.google.common.base.Preconditions;
1819
import com.google.common.util.concurrent.Futures;
@@ -55,9 +56,10 @@ public boolean containsActionResult(String key) {
5556
}
5657

5758
@Override
58-
public void putActionResult(String key, byte[] in) throws IOException, InterruptedException {
59-
diskCache.putActionResult(key, in);
60-
remoteCache.putActionResult(key, in);
59+
public void putActionResult(ActionKey actionKey, ActionResult actionResult)
60+
throws IOException, InterruptedException {
61+
diskCache.putActionResult(actionKey, actionResult);
62+
remoteCache.putActionResult(actionKey, actionResult);
6163
}
6264

6365
@Override

‎src/main/java/com/google/devtools/build/lib/remote/disk/OnDiskBlobStore.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
// limitations under the License.
1414
package com.google.devtools.build.lib.remote.disk;
1515

16+
import build.bazel.remote.execution.v2.ActionResult;
1617
import build.bazel.remote.execution.v2.Digest;
1718
import com.google.common.io.ByteStreams;
1819
import com.google.common.util.concurrent.Futures;
@@ -21,7 +22,6 @@
2122
import com.google.devtools.build.lib.remote.common.SimpleBlobStore;
2223
import com.google.devtools.build.lib.vfs.Path;
2324
import com.google.protobuf.ByteString;
24-
import java.io.ByteArrayInputStream;
2525
import java.io.IOException;
2626
import java.io.InputStream;
2727
import java.io.OutputStream;
@@ -69,8 +69,10 @@ public ListenableFuture<Boolean> getActionResult(String key, OutputStream out) {
6969
}
7070

7171
@Override
72-
public void putActionResult(String key, byte[] in) throws IOException {
73-
saveFile(getDiskKey(key, /* actionResult= */ true), new ByteArrayInputStream(in));
72+
public void putActionResult(ActionKey actionKey, ActionResult actionResult) throws IOException {
73+
try (InputStream data = actionResult.toByteString().newInput()) {
74+
saveFile(getDiskKey(actionKey.getDigest().getHash(), /* actionResult= */ true), data);
75+
}
7476
}
7577

7678
@Override

‎src/main/java/com/google/devtools/build/lib/remote/http/HttpBlobStore.java‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
// limitations under the License.
1414
package com.google.devtools.build.lib.remote.http;
1515

16+
import build.bazel.remote.execution.v2.ActionResult;
1617
import build.bazel.remote.execution.v2.Digest;
1718
import com.google.auth.Credentials;
1819
import com.google.common.collect.ImmutableList;
@@ -59,7 +60,6 @@
5960
import io.netty.util.concurrent.Future;
6061
import io.netty.util.concurrent.Promise;
6162
import io.netty.util.internal.PlatformDependent;
62-
import java.io.ByteArrayInputStream;
6363
import java.io.FileInputStream;
6464
import java.io.FilterInputStream;
6565
import java.io.IOException;
@@ -643,10 +643,11 @@ private boolean reset(InputStream in) throws IOException {
643643
}
644644

645645
@Override
646-
public void putActionResult(String actionKey, byte[] data)
646+
public void putActionResult(ActionKey actionKey, ActionResult actionResult)
647647
throws IOException, InterruptedException {
648-
try (InputStream in = new ByteArrayInputStream(data)) {
649-
uploadBlocking(actionKey, data.length, in, false);
648+
ByteString serialized = actionResult.toByteString();
649+
try (InputStream in = serialized.newInput()) {
650+
uploadBlocking(actionKey.getDigest().getHash(), serialized.size(), in, false);
650651
}
651652
}
652653

0 commit comments

Comments
 (0)