Skip to content

Commit 9d0fe54

Browse files
Google APIscopybara-github
Google APIs
authored and
copybara-github
committed
feat: new parameters in FlexTemplateRuntimeEnvironment
fix: added google.api.http annotations to RPCs Committer: @alexander-fenster PiperOrigin-RevId: 425662603
1 parent a092599 commit 9d0fe54

8 files changed

Lines changed: 319 additions & 175 deletions

File tree

‎google/dataflow/v1beta3/BUILD.bazel‎

Lines changed: 28 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -75,12 +75,15 @@ java_grpc_library(
7575
java_gapic_library(
7676
name = "dataflow_java_gapic",
7777
srcs = [":dataflow_proto_with_info"],
78+
gapic_yaml = None,
7879
grpc_service_config = "dataflow_grpc_service_config.json",
80+
service_yaml = "dataflow_v1beta3.yaml",
7981
test_deps = [
8082
":dataflow_java_grpc",
8183
],
8284
deps = [
8385
":dataflow_java_proto",
86+
"//google/api:api_java_proto",
8487
],
8588
)
8689

@@ -135,8 +138,8 @@ go_gapic_library(
135138
srcs = [":dataflow_proto_with_info"],
136139
grpc_service_config = "dataflow_grpc_service_config.json",
137140
importpath = "cloud.google.com/go/dataflow/apiv1beta3;dataflow",
138-
service_yaml = "dataflow_v1beta3.yaml",
139141
metadata = True,
142+
service_yaml = "dataflow_v1beta3.yaml",
140143
deps = [
141144
":dataflow_go_proto",
142145
"@io_bazel_rules_go//proto/wkt:duration_go_proto",
@@ -155,8 +158,8 @@ go_gapic_assembly_pkg(
155158
name = "gapi-cloud-dataflow-v1beta3-go",
156159
deps = [
157160
":dataflow_go_gapic",
158-
":dataflow_go_gapic_srcjar-test.srcjar",
159161
":dataflow_go_gapic_srcjar-metadata.srcjar",
162+
":dataflow_go_gapic_srcjar-test.srcjar",
160163
":dataflow_go_proto",
161164
],
162165
)
@@ -175,9 +178,9 @@ py_gapic_library(
175178
srcs = [":dataflow_proto"],
176179
grpc_service_config = "dataflow_grpc_service_config.json",
177180
opt_args = [
178-
"warehouse-package-name=google-cloud-dataflow-client",
179-
"python-gapic-namespace=google.cloud",
180181
"python-gapic-name=dataflow",
182+
"python-gapic-namespace=google.cloud",
183+
"warehouse-package-name=google-cloud-dataflow-client",
181184
],
182185
)
183186

@@ -265,8 +268,8 @@ nodejs_gapic_assembly_pkg(
265268
##############################################################################
266269
load(
267270
"@com_google_googleapis_imports//:imports.bzl",
268-
"ruby_gapic_assembly_pkg",
269271
"ruby_cloud_gapic_library",
272+
"ruby_gapic_assembly_pkg",
270273
"ruby_grpc_library",
271274
"ruby_proto_library",
272275
)
@@ -286,11 +289,11 @@ ruby_cloud_gapic_library(
286289
name = "dataflow_ruby_gapic",
287290
srcs = [":dataflow_proto_with_info"],
288291
extra_protoc_parameters = [
289-
"ruby-cloud-gem-name=google-cloud-dataflow-v1beta3",
290-
"ruby-cloud-env-prefix=DATAFLOW",
291-
"ruby-cloud-product-url=https://cloud.google.com/dataflow",
292292
"ruby-cloud-api-id=dataflow.googleapis.com",
293293
"ruby-cloud-api-shortname=dataflow",
294+
"ruby-cloud-env-prefix=DATAFLOW",
295+
"ruby-cloud-gem-name=google-cloud-dataflow-v1beta3",
296+
"ruby-cloud-product-url=https://cloud.google.com/dataflow",
294297
"ruby-cloud-service-override=JobsV1Beta3=Jobs;MessagesV1Beta3=Messages;MetricsV1Beta3=Metrics;SnapshotsV1Beta3=Snapshots",
295298
],
296299
grpc_service_config = "dataflow_grpc_service_config.json",
@@ -358,4 +361,20 @@ csharp_gapic_assembly_pkg(
358361
##############################################################################
359362
# C++
360363
##############################################################################
361-
# Put your C++ rules here
364+
load(
365+
"@com_google_googleapis_imports//:imports.bzl",
366+
"cc_grpc_library",
367+
"cc_proto_library",
368+
)
369+
370+
cc_proto_library(
371+
name = "dataflow_cc_proto",
372+
deps = [":dataflow_proto"],
373+
)
374+
375+
cc_grpc_library(
376+
name = "dataflow_cc_grpc",
377+
srcs = [":dataflow_proto"],
378+
grpc_only = True,
379+
deps = [":dataflow_cc_proto"],
380+
)

‎google/dataflow/v1beta3/environment.proto‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright 2021 Google LLC
1+
// Copyright 2022 Google LLC
22
//
33
// Licensed under the Apache License, Version 2.0 (the "License");
44
// you may not use this file except in compliance with the License.

‎google/dataflow/v1beta3/jobs.proto‎

Lines changed: 101 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright 2021 Google LLC
1+
// Copyright 2022 Google LLC
22
//
33
// Licensed under the Apache License, Version 2.0 (the "License");
44
// you may not use this file except in compliance with the License.
@@ -17,12 +17,12 @@ syntax = "proto3";
1717
package google.dataflow.v1beta3;
1818

1919
import "google/api/annotations.proto";
20+
import "google/api/client.proto";
2021
import "google/dataflow/v1beta3/environment.proto";
2122
import "google/dataflow/v1beta3/snapshots.proto";
2223
import "google/protobuf/duration.proto";
2324
import "google/protobuf/struct.proto";
2425
import "google/protobuf/timestamp.proto";
25-
import "google/api/client.proto";
2626

2727
option csharp_namespace = "Google.Cloud.Dataflow.V1Beta3";
2828
option go_package = "google.golang.org/genproto/googleapis/dataflow/v1beta3;dataflow";
@@ -50,6 +50,14 @@ service JobsV1Beta3 {
5050
// `projects.jobs.create` is not recommended, as your job will always start
5151
// in `us-central1`.
5252
rpc CreateJob(CreateJobRequest) returns (Job) {
53+
option (google.api.http) = {
54+
post: "/v1b3/projects/{project_id}/jobs"
55+
body: "job"
56+
additional_bindings {
57+
post: "/v1b3/projects/{project_id}/locations/{location}/jobs"
58+
body: "job"
59+
}
60+
};
5361
}
5462

5563
// Gets the state of the specified Cloud Dataflow job.
@@ -60,6 +68,12 @@ service JobsV1Beta3 {
6068
// `projects.jobs.get` is not recommended, as you can only get the state of
6169
// jobs that are running in `us-central1`.
6270
rpc GetJob(GetJobRequest) returns (Job) {
71+
option (google.api.http) = {
72+
get: "/v1b3/projects/{project_id}/jobs/{job_id}"
73+
additional_bindings {
74+
get: "/v1b3/projects/{project_id}/locations/{location}/jobs/{job_id}"
75+
}
76+
};
6377
}
6478

6579
// Updates the state of an existing Cloud Dataflow job.
@@ -70,6 +84,14 @@ service JobsV1Beta3 {
7084
// `projects.jobs.update` is not recommended, as you can only update the state
7185
// of jobs that are running in `us-central1`.
7286
rpc UpdateJob(UpdateJobRequest) returns (Job) {
87+
option (google.api.http) = {
88+
put: "/v1b3/projects/{project_id}/jobs/{job_id}"
89+
body: "job"
90+
additional_bindings {
91+
put: "/v1b3/projects/{project_id}/locations/{location}/jobs/{job_id}"
92+
body: "job"
93+
}
94+
};
7395
}
7496

7597
// List the jobs of a project.
@@ -81,10 +103,19 @@ service JobsV1Beta3 {
81103
// `projects.jobs.list` is not recommended, as you can only get the list of
82104
// jobs that are running in `us-central1`.
83105
rpc ListJobs(ListJobsRequest) returns (ListJobsResponse) {
106+
option (google.api.http) = {
107+
get: "/v1b3/projects/{project_id}/jobs"
108+
additional_bindings {
109+
get: "/v1b3/projects/{project_id}/locations/{location}/jobs"
110+
}
111+
};
84112
}
85113

86114
// List the jobs of a project across all regions.
87115
rpc AggregatedListJobs(ListJobsRequest) returns (ListJobsResponse) {
116+
option (google.api.http) = {
117+
get: "/v1b3/projects/{project_id}/jobs:aggregated"
118+
};
88119
}
89120

90121
// Check for existence of active jobs in the given project across all regions.
@@ -93,6 +124,14 @@ service JobsV1Beta3 {
93124

94125
// Snapshot the state of a streaming job.
95126
rpc SnapshotJob(SnapshotJobRequest) returns (Snapshot) {
127+
option (google.api.http) = {
128+
post: "/v1b3/projects/{project_id}/jobs/{job_id}:snapshot"
129+
body: "*"
130+
additional_bindings {
131+
post: "/v1b3/projects/{project_id}/locations/{location}/jobs/{job_id}:snapshot"
132+
body: "*"
133+
}
134+
};
96135
}
97136
}
98137

@@ -277,7 +316,7 @@ message FileIODetails {
277316
string file_pattern = 1;
278317
}
279318

280-
// Metadata for a Cloud BigTable connector used by the job.
319+
// Metadata for a Cloud Bigtable connector used by the job.
281320
message BigTableIODetails {
282321
// ProjectId accessed in the connection.
283322
string project_id = 1;
@@ -359,10 +398,10 @@ message JobMetadata {
359398
// Identification of a BigQuery source used in the Dataflow job.
360399
repeated BigQueryIODetails bigquery_details = 3;
361400

362-
// Identification of a Cloud BigTable source used in the Dataflow job.
401+
// Identification of a Cloud Bigtable source used in the Dataflow job.
363402
repeated BigTableIODetails big_table_details = 4;
364403

365-
// Identification of a PubSub source used in the Dataflow job.
404+
// Identification of a Pub/Sub source used in the Dataflow job.
366405
repeated PubSubIODetails pubsub_details = 5;
367406

368407
// Identification of a File source used in the Dataflow job.
@@ -398,27 +437,6 @@ message PipelineDescription {
398437
repeated DisplayData display_data = 3;
399438
}
400439

401-
// Description of the type, names/ids, and input/outputs for a transform.
402-
message TransformSummary {
403-
// Type of transform.
404-
KindType kind = 1;
405-
406-
// SDK generated id of this transform instance.
407-
string id = 2;
408-
409-
// User provided name for this transform instance.
410-
string name = 3;
411-
412-
// Transform-specific display data.
413-
repeated DisplayData display_data = 4;
414-
415-
// User names for all collection outputs to this transform.
416-
repeated string output_collection_name = 5;
417-
418-
// User names for all collection inputs to this transform.
419-
repeated string input_collection_name = 6;
420-
}
421-
422440
// Type of transform or stage operation.
423441
enum KindType {
424442
// Unrecognized transform type.
@@ -449,6 +467,27 @@ enum KindType {
449467
SHUFFLE_KIND = 8;
450468
}
451469

470+
// Description of the type, names/ids, and input/outputs for a transform.
471+
message TransformSummary {
472+
// Type of transform.
473+
KindType kind = 1;
474+
475+
// SDK generated id of this transform instance.
476+
string id = 2;
477+
478+
// User provided name for this transform instance.
479+
string name = 3;
480+
481+
// Transform-specific display data.
482+
repeated DisplayData display_data = 4;
483+
484+
// User names for all collection outputs to this transform.
485+
repeated string output_collection_name = 5;
486+
487+
// User names for all collection inputs to this transform.
488+
repeated string input_collection_name = 6;
489+
}
490+
452491
// Description of the composing transforms, names/ids, and input/outputs of a
453492
// stage of execution. Some composing transforms and sources may have been
454493
// generated by the Dataflow service during execution planning.
@@ -613,22 +652,6 @@ message Step {
613652
google.protobuf.Struct properties = 3;
614653
}
615654

616-
// Additional information about how a Cloud Dataflow job will be executed that
617-
// isn't contained in the submitted job.
618-
message JobExecutionInfo {
619-
// A mapping from each stage to the information about that stage.
620-
map<string, JobExecutionStageInfo> stages = 1;
621-
}
622-
623-
// Contains information about how a particular
624-
// [google.dataflow.v1beta3.Step][google.dataflow.v1beta3.Step] will be executed.
625-
message JobExecutionStageInfo {
626-
// The steps associated with the execution stage.
627-
// Note that stages may have several steps, and that a given step
628-
// might be run by more than one stage.
629-
repeated string step_name = 1;
630-
}
631-
632655
// Describes the overall state of a [google.dataflow.v1beta3.Job][google.dataflow.v1beta3.Job].
633656
enum JobState {
634657
// The job's run state isn't specified.
@@ -703,6 +726,42 @@ enum JobState {
703726
JOB_STATE_RESOURCE_CLEANING_UP = 12;
704727
}
705728

729+
// Additional information about how a Cloud Dataflow job will be executed that
730+
// isn't contained in the submitted job.
731+
message JobExecutionInfo {
732+
// A mapping from each stage to the information about that stage.
733+
map<string, JobExecutionStageInfo> stages = 1;
734+
}
735+
736+
// Contains information about how a particular
737+
// [google.dataflow.v1beta3.Step][google.dataflow.v1beta3.Step] will be executed.
738+
message JobExecutionStageInfo {
739+
// The steps associated with the execution stage.
740+
// Note that stages may have several steps, and that a given step
741+
// might be run by more than one stage.
742+
repeated string step_name = 1;
743+
}
744+
745+
// Selector for how much information is returned in Job responses.
746+
enum JobView {
747+
// The job view to return isn't specified, or is unknown.
748+
// Responses will contain at least the `JOB_VIEW_SUMMARY` information,
749+
// and may contain additional information.
750+
JOB_VIEW_UNKNOWN = 0;
751+
752+
// Request summary information only:
753+
// Project ID, Job ID, job name, job type, job status, start/end time,
754+
// and Cloud SDK version details.
755+
JOB_VIEW_SUMMARY = 1;
756+
757+
// Request all information available for this job.
758+
JOB_VIEW_ALL = 2;
759+
760+
// Request summary info and limited job description data for steps, labels and
761+
// environment.
762+
JOB_VIEW_DESCRIPTION = 3;
763+
}
764+
706765
// Request to create a Cloud Dataflow job.
707766
message CreateJobRequest {
708767
// The ID of the Cloud Platform project that the job belongs to.
@@ -740,26 +799,6 @@ message GetJobRequest {
740799
string location = 4;
741800
}
742801

743-
// Selector for how much information is returned in Job responses.
744-
enum JobView {
745-
// The job view to return isn't specified, or is unknown.
746-
// Responses will contain at least the `JOB_VIEW_SUMMARY` information,
747-
// and may contain additional information.
748-
JOB_VIEW_UNKNOWN = 0;
749-
750-
// Request summary information only:
751-
// Project ID, Job ID, job name, job type, job status, start/end time,
752-
// and Cloud SDK version details.
753-
JOB_VIEW_SUMMARY = 1;
754-
755-
// Request all information available for this job.
756-
JOB_VIEW_ALL = 2;
757-
758-
// Request summary info and limited job description data for steps, labels and
759-
// environment.
760-
JOB_VIEW_DESCRIPTION = 3;
761-
}
762-
763802
// Request to update a Cloud Dataflow job.
764803
message UpdateJobRequest {
765804
// The ID of the Cloud Platform project that the job belongs to.

0 commit comments

Comments
 (0)