Skip to content

Commit 88412ce

Browse files
tylerwilliamscopybara-github
tylerwilliams
authored and
copybara-github
committed
Support new-style digest functions
Support new-style digest functions. This PR adds support for new-style digest functions to the remote execution library code. The remote-apis spec says: ``` // * `digest_function` is a lowercase string form of a `DigestFunction.Value` // enum, indicating which digest function was used to compute `hash`. If the // digest function used is one of MD5, MURMUR3, SHA1, SHA256, SHA384, SHA512, // or VSO, this component MUST be omitted. In that case the server SHOULD // infer the digest function using the length of the `hash` and the digest // functions announced in the server's capabilities. ``` This is a partial commit for bazelbuild#18658. Closes bazelbuild#18731. PiperOrigin-RevId: 543691155 Change-Id: If8c386d923db1b24dff6054c8ab3f783409b7f13
1 parent 5fa4718 commit 88412ce

File tree

4 files changed

+156
-37
lines changed

4 files changed

+156
-37
lines changed

‎src/main/java/com/google/devtools/build/lib/remote/ByteStreamUploader.java‎

Lines changed: 32 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import static java.util.concurrent.TimeUnit.SECONDS;
2222

2323
import build.bazel.remote.execution.v2.Digest;
24+
import build.bazel.remote.execution.v2.DigestFunction;
2425
import com.google.bytestream.ByteStreamGrpc;
2526
import com.google.bytestream.ByteStreamGrpc.ByteStreamFutureStub;
2627
import com.google.bytestream.ByteStreamGrpc.ByteStreamStub;
@@ -69,6 +70,7 @@ final class ByteStreamUploader {
6970
private final CallCredentialsProvider callCredentialsProvider;
7071
private final long callTimeoutSecs;
7172
private final RemoteRetrier retrier;
73+
private final DigestFunction.Value digestFunction;
7274

7375
@Nullable private final Semaphore openedFilePermits;
7476

@@ -89,14 +91,16 @@ final class ByteStreamUploader {
8991
CallCredentialsProvider callCredentialsProvider,
9092
long callTimeoutSecs,
9193
RemoteRetrier retrier,
92-
int maximumOpenFiles) {
94+
int maximumOpenFiles,
95+
DigestFunction.Value digestFunction) {
9396
checkArgument(callTimeoutSecs > 0, "callTimeoutSecs must be gt 0.");
9497
this.instanceName = instanceName;
9598
this.channel = channel;
9699
this.callCredentialsProvider = callCredentialsProvider;
97100
this.callTimeoutSecs = callTimeoutSecs;
98101
this.retrier = retrier;
99102
this.openedFilePermits = maximumOpenFiles != -1 ? new Semaphore(maximumOpenFiles) : null;
103+
this.digestFunction = digestFunction;
100104
}
101105

102106
@VisibleForTesting
@@ -175,11 +179,34 @@ public ListenableFuture<Void> uploadBlobAsync(
175179
MoreExecutors.directExecutor());
176180
}
177181

178-
private static String buildUploadResourceName(
182+
private boolean isOldStyleDigestFunction() {
183+
// Old-style digest functions (SHA256, etc) are distinguishable by the length
184+
// of their hash alone and do not require extra specification, but newer
185+
// digest functions (which may have the same length hashes as the older
186+
// functions!) must be explicitly specified in the upload resource name.
187+
return digestFunction.getNumber() <= 7;
188+
}
189+
190+
private String buildUploadResourceName(
179191
String instanceName, UUID uuid, Digest digest, boolean compressed) {
180-
String template =
181-
compressed ? "uploads/%s/compressed-blobs/zstd/%s/%d" : "uploads/%s/blobs/%s/%d";
182-
String resourceName = format(template, uuid, digest.getHash(), digest.getSizeBytes());
192+
193+
String resourceName;
194+
195+
if (isOldStyleDigestFunction()) {
196+
String template =
197+
compressed ? "uploads/%s/compressed-blobs/zstd/%s/%d" : "uploads/%s/blobs/%s/%d";
198+
resourceName = format(template, uuid, digest.getHash(), digest.getSizeBytes());
199+
} else {
200+
String template =
201+
compressed ? "uploads/%s/compressed-blobs/zstd/%s/%s/%d" : "uploads/%s/blobs/%s/%s/%d";
202+
resourceName =
203+
format(
204+
template,
205+
uuid,
206+
Ascii.toLowerCase(digestFunction.getValueDescriptor().getName()),
207+
digest.getHash(),
208+
digest.getSizeBytes());
209+
}
183210
if (!Strings.isNullOrEmpty(instanceName)) {
184211
resourceName = instanceName + "/" + resourceName;
185212
}

‎src/main/java/com/google/devtools/build/lib/remote/GrpcCacheClient.java‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import build.bazel.remote.execution.v2.ContentAddressableStorageGrpc;
2323
import build.bazel.remote.execution.v2.ContentAddressableStorageGrpc.ContentAddressableStorageFutureStub;
2424
import build.bazel.remote.execution.v2.Digest;
25+
import build.bazel.remote.execution.v2.DigestFunction;
2526
import build.bazel.remote.execution.v2.FindMissingBlobsRequest;
2627
import build.bazel.remote.execution.v2.FindMissingBlobsResponse;
2728
import build.bazel.remote.execution.v2.GetActionResultRequest;
@@ -107,7 +108,8 @@ public GrpcCacheClient(
107108
callCredentialsProvider,
108109
options.remoteTimeout.getSeconds(),
109110
retrier,
110-
options.maximumOpenFiles);
111+
options.maximumOpenFiles,
112+
digestUtil.getDigestFunction());
111113
maxMissingBlobsDigestsPerMessage = computeMaxMissingBlobsDigestsPerMessage();
112114
Preconditions.checkState(
113115
maxMissingBlobsDigestsPerMessage > 0, "Error: gRPC message size too small.");
@@ -352,12 +354,24 @@ private ListenableFuture<Void> downloadBlob(
352354
MoreExecutors.directExecutor());
353355
}
354356

355-
public static String getResourceName(String instanceName, Digest digest, boolean compressed) {
357+
private static boolean isOldStyleDigestFunction(DigestFunction.Value digestFunction) {
358+
// Old-style digest functions (SHA256, etc) are distinguishable by the length
359+
// of their hash alone and do not require extra specification, but newer
360+
// digest functions (which may have the same length hashes as the older
361+
// functions!) must be explicitly specified in the upload resource name.
362+
return digestFunction.getNumber() <= 7;
363+
}
364+
365+
public static String getResourceName(
366+
String instanceName, Digest digest, boolean compressed, DigestFunction.Value digestFunction) {
356367
String resourceName = "";
357368
if (!instanceName.isEmpty()) {
358369
resourceName += instanceName + "/";
359370
}
360371
resourceName += compressed ? "compressed-blobs/zstd/" : "blobs/";
372+
if (!isOldStyleDigestFunction(digestFunction)) {
373+
resourceName += Ascii.toLowerCase(digestFunction.getValueDescriptor().getName()) + "/";
374+
}
361375
return resourceName + DigestUtil.toString(digest);
362376
}
363377

@@ -369,7 +383,11 @@ private ListenableFuture<Long> requestRead(
369383
@Nullable Supplier<Digest> digestSupplier,
370384
Channel channel) {
371385
String resourceName =
372-
getResourceName(options.remoteInstanceName, digest, options.cacheCompression);
386+
getResourceName(
387+
options.remoteInstanceName,
388+
digest,
389+
options.cacheCompression,
390+
digestUtil.getDigestFunction());
373391
SettableFuture<Long> future = SettableFuture.create();
374392
OutputStream out;
375393
try {

0 commit comments

Comments
 (0)