Skip to content

Commit 4a2c36b

Browse files
coeuvrecopybara-github
coeuvre
authored and
copybara-github
committed
Add remote output service proto
Working towards #20933. Closes #21140. PiperOrigin-RevId: 613219486 Change-Id: Ie4e02e04e2dcfc444234fb164a73865edf8d9ba0
1 parent 45ad446 commit 4a2c36b

File tree

3 files changed

+455
-0
lines changed

3 files changed

+455
-0
lines changed

‎src/main/protobuf/BUILD‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -251,6 +251,42 @@ java_library_srcs(
251251
deps = [":remote_scrubbing_java_proto"],
252252
)
253253

254+
proto_library(
255+
name = "bazel_output_service_proto",
256+
srcs = ["bazel_output_service.proto"],
257+
deps = [
258+
"@com_google_protobuf//:any_proto",
259+
],
260+
)
261+
262+
java_proto_library(
263+
name = "bazel_output_service_java_proto",
264+
deps = [":bazel_output_service_proto"],
265+
)
266+
267+
java_library_srcs(
268+
name = "bazel_output_service_java_proto_srcs",
269+
deps = [":bazel_output_service_java_proto"],
270+
)
271+
272+
proto_library(
273+
name = "bazel_output_service_rev2_proto",
274+
srcs = ["bazel_output_service_rev2.proto"],
275+
deps = [
276+
"@remoteapis//:build_bazel_remote_execution_v2_remote_execution_proto",
277+
],
278+
)
279+
280+
java_proto_library(
281+
name = "bazel_output_service_rev2_java_proto",
282+
deps = [":bazel_output_service_rev2_proto"],
283+
)
284+
285+
java_library_srcs(
286+
name = "bazel_output_service_rev2_java_proto_srcs",
287+
deps = [":bazel_output_service_rev2_java_proto"],
288+
)
289+
254290
proto_library(
255291
name = "spawn_proto",
256292
srcs = ["spawn.proto"],

‎src/main/protobuf/bazel_output_service.proto‎

Lines changed: 349 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,349 @@
1+
// Copyright 2024 The Bazel Authors. All rights reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
//
15+
// This file contains the protocol buffer representation of a list of supported
16+
// flags for Bazel commands.
17+
syntax = "proto3";
18+
19+
package bazel_output_service;
20+
21+
import "google/protobuf/any.proto";
22+
import "google/rpc/status.proto";
23+
24+
option java_package = "com.google.devtools.build.lib.remote";
25+
option java_outer_classname = "BazelOutputServiceProto";
26+
option go_package = "bazeloutputservice";
27+
28+
// The Bazel Output Service may be used by users of the Remote Execution API to
29+
// construct a directory on the local system that contains all output files of a
30+
// build.
31+
//
32+
// Primitive implementations of this API may simply download files from the
33+
// Content Addressable Storage (CAS) and store them at their designated
34+
// location. Complex implementations may use a pseudo file system (e.g., FUSE)
35+
// to support deduplication, lazy loading and snapshotting.
36+
//
37+
// Details:
38+
// https://github.com/bazelbuild/proposals/blob/master/designs/2021-02-09-remote-output-service.md
39+
// https://groups.google.com/g/remote-execution-apis/c/qOSWWwBLPzo
40+
// https://groups.google.com/g/bazel-dev/c/lKzENsNd1Do
41+
// https://docs.google.com/document/d/1W6Tqq8cndssnDI0yzFSoj95oezRKcIhU57nwLHaN1qk/edit
42+
service BazelOutputService {
43+
// Clean all data associated with a single output path, so that the
44+
// next invocation of StartBuild() yields an empty output path. This
45+
// MAY be implemented in a way that's faster than removing all of the
46+
// files from the file system manually.
47+
rpc Clean(CleanRequest) returns (CleanResponse);
48+
49+
// Signal that a new build is about to start.
50+
//
51+
// Bazel uses this call to obtain a directory where outputs of the build may
52+
// be stored, called the output path. Based on the parameters provided, server
53+
// may provide an empty output path, or one that has contents from a previous
54+
// build of the same workspace.
55+
//
56+
// In case the output path contains data from a previous build, server is
57+
// responsible for calling ContentAddressableStorage.FindMissingBlobs() for
58+
// all of the objects that are stored remotely. This ensures that these
59+
// objects don't disappear from the Content Addressable Storage while the
60+
// build is running. Any files that are absent MUST be removed from the output
61+
// path and reported through InitialOutputPathContents.modified_path_prefixes,
62+
// unless the field has been omitted because it would have been too large.
63+
rpc StartBuild(StartBuildRequest) returns (StartBuildResponse);
64+
65+
// Stage build artifacts at the given paths with digests that are known to the
66+
// Content Addressable Storage. The file contents MAY be lazily downloaded
67+
// when they're accessed in the future, and are not guaranteed to have already
68+
// been downloaded upon return.
69+
rpc StageArtifacts(StageArtifactsRequest) returns (StageArtifactsResponse);
70+
71+
// Notify the server that a set of paths are not expected to be modified
72+
// further by Bazel or a local build action within the current build.
73+
//
74+
// For each of these paths, the server MAY decide to store a dirty bit,
75+
// initially unset. Subsequent modifications to the contents, or deletion of,
76+
// the file stored at that path, cause the dirty bit to be set. If the server
77+
// chooses to implement the dirty bit, any paths with the dirty bit set MUST
78+
// be reported back to Bazel in the next
79+
// InitialOutputPathContents.modified_path_prefixes for the same workspace.
80+
//
81+
// As an alternative to tracking modifications via a dirty bit, a server MAY
82+
// choose to freeze finalized paths, preventing further modifications to the
83+
// files stored there.
84+
rpc FinalizeArtifacts(FinalizeArtifactsRequest)
85+
returns (FinalizeArtifactsResponse);
86+
87+
// Signal that a build has been completed.
88+
rpc FinalizeBuild(FinalizeBuildRequest) returns (FinalizeBuildResponse);
89+
90+
// Obtain the status of one or more files, directories or symbolic
91+
// links that are stored in the output path.
92+
rpc BatchStat(BatchStatRequest) returns (BatchStatResponse);
93+
}
94+
95+
message CleanRequest {
96+
// The workspace identifier that was provided to
97+
// StartBuildRequest.output_base_id whose data needs to be removed.
98+
string output_base_id = 1;
99+
}
100+
101+
message CleanResponse {
102+
// Intentionally empty for future extensibility.
103+
}
104+
105+
message StartBuildRequest {
106+
// The version of the protocol Bazel currently uses. The service MUST return
107+
// an error if it doesn't recognize the version.
108+
//
109+
// A future Bazel version may introduce incompatible changes and increment
110+
// this version number. The incompatible change will be first made in the
111+
// development tree for the next major Bazel release, and the new version thus
112+
// introduced should be considered unstable until that major Bazel release
113+
// occurs. At that time, the new version becomes stable, and the old one is
114+
// immediately retired.
115+
//
116+
// In particular, version 1 must not be considered stable until Bazel 8.0.0 is
117+
// released.
118+
//
119+
// Current version: 1 (experimental).
120+
int32 version = 1;
121+
122+
// A client-chosen value for the output service to uniquely identify the
123+
// workspace the build is being started. This value must be set to ensure that
124+
// the remote output service is capable of managing builds for distinct
125+
// workspaces concurrently.
126+
//
127+
// Bazel sets this value to the MD5 sum of the absolute path of the output
128+
// base.
129+
//
130+
// Starting a build finalizes any previous build with the same output_base_id
131+
// that has not been finalized yet as if a FinalizeBuildRequest had been sent
132+
// with build_successful set to false.
133+
string output_base_id = 2;
134+
135+
// A client-chosen value that uniquely identifies this build. This value must
136+
// be provided to most other methods to ensure that operations are targeted
137+
// against the right output path. If the server receives a subsequent request
138+
// with a non-matching build_id, it SHOULD send back an error response.
139+
//
140+
// Bazel sets this value to --invocation_id.
141+
string build_id = 3;
142+
143+
// Additional arguments to pass depending on how Bazel communicate with the
144+
// Content Addressable Storage.
145+
//
146+
// In case of a REv2 CAS, the type is
147+
// [StartBuildArgs][bazel_output_service_rev2.StartBuildArgs].
148+
google.protobuf.Any args = 4;
149+
150+
// The absolute path at which the remote output service exposes its output
151+
// paths, as seen from the perspective of the client.
152+
//
153+
// This value needs to be provided by the client, because file system
154+
// namespace virtualization may cause this directory to appear at a location
155+
// that differs from the one used by the service.
156+
//
157+
// The purpose of this field is to ensure that the remote output service is
158+
// capable of expanding symbolic links containing absolute paths.
159+
//
160+
// If this is not set, or an empty string, the service must determine where to
161+
// expose its output path and return an absolute path in
162+
// StartBuildResponse.output_path_suffix.
163+
string output_path_prefix = 5;
164+
165+
// A map of paths on the system that will become symbolic links pointing to
166+
// locations inside the output path. Similar to output_path_prefix, this
167+
// option is used to ensure the remote output service is capable of expanding
168+
// symbolic links.
169+
//
170+
// Map keys are absolute paths, while map values are paths that are
171+
// relative to the output path.
172+
map<string, string> output_path_aliases = 6;
173+
}
174+
175+
message StartBuildResponse {
176+
// If set, the contents of the output path are almost entirely identical on
177+
// the results of a previous build. This information may be used by Bazel to
178+
// prevent unnecessary scanning of the file system.
179+
//
180+
// The server MUST leave this field unset in case the contents of the output
181+
// path are empty, not based on a previous build, if no tracking of this
182+
// information is performed, or if the number of changes made to the output
183+
// path is too large to be expressed.
184+
InitialOutputPathContents initial_output_path_contents = 1;
185+
186+
// A path that the client must append to StartBuildRequest.output_path_prefix
187+
// to obtain the full path at which outputs of the build are stored.
188+
//
189+
// Bazel replaces bazel-out/ with a symlink targeting this path.
190+
string output_path_suffix = 2;
191+
}
192+
193+
message InitialOutputPathContents {
194+
// The identifier of a previously finalized build whose results are stored in
195+
// the output path.
196+
string build_id = 1;
197+
198+
// Output path prefixes that have been deleted or modified since they were
199+
// finalized. Any path exactly matching or starting with one of these prefixes
200+
// may be assumed to have been modified or deleted.
201+
//
202+
// In the interest of performance, the server SHOULD only include path
203+
// prefixes that contain at least one of the paths that were finalized by the
204+
// previous build.
205+
repeated string modified_path_prefixes = 2;
206+
}
207+
208+
message StageArtifactsRequest {
209+
message Artifact {
210+
// path is relative to StartBuildResponse.output_path.
211+
string path = 1;
212+
// Describe how to stage the artifact.
213+
//
214+
// The concrete type of the locator depending on the CAS Bazel connects to.
215+
// In case of a REv2 CAS, the type is
216+
// [FileArtifactLocator][bazel_output_service_rev2.FileArtifactLocator].
217+
google.protobuf.Any locator = 2;
218+
}
219+
220+
// The identifier of the build. Server uses this to determine which output
221+
// path needs to be modified.
222+
string build_id = 1;
223+
224+
repeated Artifact artifacts = 2;
225+
}
226+
227+
message StageArtifactsResponse {
228+
message Response {
229+
// If the status has a code other than `OK`, it indicates that the artifact
230+
// could not be staged.
231+
//
232+
// Errors:
233+
// * `NOT_FOUND`: The requested Artifact is not in the CAS.
234+
google.rpc.Status status = 1;
235+
}
236+
237+
// The response for each of the requested artifacts, using the same order as
238+
// requested. This means that this list has the same length as
239+
// StageArtifactsRequest.artifacts.
240+
repeated Response responses = 1;
241+
}
242+
243+
message FinalizeArtifactsRequest {
244+
message Artifact {
245+
// path is relative to StartBuildResponse.output_path.
246+
string path = 1;
247+
// Expected digest for this path. This allows server to detect if the path
248+
// has been changed after Bazel finished creating the path and the
249+
// corresponding FinalizeArtifactsRequest is processed.
250+
//
251+
// The concrete type of the locator depending on the CAS Bazel connects to.
252+
// In case of a REv2 CAS, the type is
253+
// [FileArtifactLocator][bazel_output_service_rev2.FileArtifactLocator].
254+
google.protobuf.Any locator = 2;
255+
}
256+
257+
// The identifier of the build. Server uses this to determine which output
258+
// path needs to be modified.
259+
string build_id = 1;
260+
261+
repeated Artifact artifacts = 2;
262+
}
263+
264+
message FinalizeArtifactsResponse {
265+
// Intentionally empty for future extensibility.
266+
}
267+
268+
message FinalizeBuildRequest {
269+
// The identifier of the build that should be finalized.
270+
string build_id = 1;
271+
272+
// Whether the build completed successfully. The remote output service MAY,
273+
// for example, use this option to apply different retention policies that
274+
// take the outcome of the build into account.
275+
bool build_successful = 2;
276+
}
277+
278+
message FinalizeBuildResponse {
279+
// Intentionally empty for future extensibility.
280+
}
281+
282+
message BatchStatRequest {
283+
// The identifier of the build. The remote output service uses this to
284+
// determine which output path needs to be inspected.
285+
string build_id = 1;
286+
287+
// Paths whose status is to be obtained. The server MUST canonicalize each
288+
// path using lstat semantics, i.e., all components except the last must be
289+
// resolved if they are symlinks. If a symlink pointing to a location outside
290+
// of the output path is encountered at any point during the canonicalization
291+
// process, the server MAY use the information in
292+
// StartBuildRequest.output_path_aliases map to continue the canonicalization.
293+
//
294+
// Refer to Stat.type for how to handle a situation where canonicalization
295+
// fails due to a symlink pointing to a location outside of the output path.
296+
//
297+
// Path is relative to StartBuildResponse.output_path.
298+
repeated string paths = 2;
299+
}
300+
301+
message BatchStatResponse {
302+
message StatResponse {
303+
// The status of the path. If the path does not exist, this field MUST be
304+
// unset.
305+
Stat stat = 1;
306+
}
307+
308+
message Stat {
309+
message File {
310+
// The digest of the file.
311+
//
312+
// The server MAY leave this field unset if it is unable to compute the
313+
// digest.
314+
//
315+
// The concrete type of the digest depending on the CAS Bazel connects to.
316+
// In case of a REv2 CAS, the type is
317+
// [FileArtifactLocator][bazel_output_service_rev2.FileArtifactLocator].
318+
google.protobuf.Any locator = 1;
319+
}
320+
321+
message Symlink {
322+
// The target of the symbolic link.
323+
string target = 1;
324+
}
325+
326+
message Directory {}
327+
328+
// If the path cannot be canonicalized, the server MUST NOT set any of the
329+
// type fields.
330+
//
331+
// If the path resolves to a special file, the server MUST NOT set any of
332+
// the type fields.
333+
oneof type {
334+
// The path resolves to a regular file.
335+
File file = 1;
336+
337+
// The path resolves to a symbolic link.
338+
Symlink symlink = 2;
339+
340+
// The path resolves to a directory.
341+
Directory directory = 3;
342+
}
343+
}
344+
345+
// The status response for each of the requested paths, using the same
346+
// order as requested. This means that this list has the same length
347+
// as BatchStatRequest.paths.
348+
repeated StatResponse responses = 1;
349+
}

0 commit comments

Comments
 (0)