Skip to content

Commit 5a395e4

Browse files
Google APIscopybara-github
Google APIs
authored and
copybara-github
committed
feat: add support for new Dataproc features
PiperOrigin-RevId: 510421528
1 parent e0d1801 commit 5a395e4

10 files changed

Lines changed: 491 additions & 222 deletions

‎google/cloud/dataproc/v1/BUILD.bazel‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,8 @@ go_gapic_library(
145145
transport = "grpc+rest",
146146
deps = [
147147
":dataproc_go_proto",
148+
"//google/cloud/location:location_go_proto",
149+
"//google/iam/v1:iam_go_proto",
148150
"//google/longrunning:longrunning_go_proto",
149151
"@com_google_cloud_go_longrunning//:go_default_library",
150152
"@com_google_cloud_go_longrunning//autogen:go_default_library",
@@ -186,6 +188,10 @@ py_gapic_library(
186188
rest_numeric_enums = True,
187189
service_yaml = "dataproc_v1.yaml",
188190
transport = "grpc+rest",
191+
deps = [
192+
"//google/iam/v1:iam_policy_py_proto",
193+
"//google/iam/v1:policy_py_proto",
194+
],
189195
)
190196

191197
py_test(

‎google/cloud/dataproc/v1/autoscaling_policies.proto‎

Lines changed: 40 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright 2021 Google LLC
1+
// Copyright 2022 Google LLC
22
//
33
// Licensed under the Apache License, Version 2.0 (the "License");
44
// you may not use this file except in compliance with the License.
@@ -36,10 +36,12 @@ option (google.api.resource_definition) = {
3636
// Dataproc API.
3737
service AutoscalingPolicyService {
3838
option (google.api.default_host) = "dataproc.googleapis.com";
39-
option (google.api.oauth_scopes) = "https://www.googleapis.com/auth/cloud-platform";
39+
option (google.api.oauth_scopes) =
40+
"https://www.googleapis.com/auth/cloud-platform";
4041

4142
// Creates new autoscaling policy.
42-
rpc CreateAutoscalingPolicy(CreateAutoscalingPolicyRequest) returns (AutoscalingPolicy) {
43+
rpc CreateAutoscalingPolicy(CreateAutoscalingPolicyRequest)
44+
returns (AutoscalingPolicy) {
4345
option (google.api.http) = {
4446
post: "/v1/{parent=projects/*/locations/*}/autoscalingPolicies"
4547
body: "policy"
@@ -55,7 +57,8 @@ service AutoscalingPolicyService {
5557
//
5658
// Disabled check for update_mask, because all updates will be full
5759
// replacements.
58-
rpc UpdateAutoscalingPolicy(UpdateAutoscalingPolicyRequest) returns (AutoscalingPolicy) {
60+
rpc UpdateAutoscalingPolicy(UpdateAutoscalingPolicyRequest)
61+
returns (AutoscalingPolicy) {
5962
option (google.api.http) = {
6063
put: "/v1/{policy.name=projects/*/locations/*/autoscalingPolicies/*}"
6164
body: "policy"
@@ -68,7 +71,8 @@ service AutoscalingPolicyService {
6871
}
6972

7073
// Retrieves autoscaling policy.
71-
rpc GetAutoscalingPolicy(GetAutoscalingPolicyRequest) returns (AutoscalingPolicy) {
74+
rpc GetAutoscalingPolicy(GetAutoscalingPolicyRequest)
75+
returns (AutoscalingPolicy) {
7276
option (google.api.http) = {
7377
get: "/v1/{name=projects/*/locations/*/autoscalingPolicies/*}"
7478
additional_bindings {
@@ -79,7 +83,8 @@ service AutoscalingPolicyService {
7983
}
8084

8185
// Lists autoscaling policies in the project.
82-
rpc ListAutoscalingPolicies(ListAutoscalingPoliciesRequest) returns (ListAutoscalingPoliciesResponse) {
86+
rpc ListAutoscalingPolicies(ListAutoscalingPoliciesRequest)
87+
returns (ListAutoscalingPoliciesResponse) {
8388
option (google.api.http) = {
8489
get: "/v1/{parent=projects/*/locations/*}/autoscalingPolicies"
8590
additional_bindings {
@@ -91,7 +96,8 @@ service AutoscalingPolicyService {
9196

9297
// Deletes an autoscaling policy. It is an error to delete an autoscaling
9398
// policy that is in use by one or more clusters.
94-
rpc DeleteAutoscalingPolicy(DeleteAutoscalingPolicyRequest) returns (google.protobuf.Empty) {
99+
rpc DeleteAutoscalingPolicy(DeleteAutoscalingPolicyRequest)
100+
returns (google.protobuf.Empty) {
95101
option (google.api.http) = {
96102
delete: "/v1/{name=projects/*/locations/*/autoscalingPolicies/*}"
97103
additional_bindings {
@@ -132,14 +138,17 @@ message AutoscalingPolicy {
132138

133139
// Autoscaling algorithm for policy.
134140
oneof algorithm {
135-
BasicAutoscalingAlgorithm basic_algorithm = 3 [(google.api.field_behavior) = REQUIRED];
141+
BasicAutoscalingAlgorithm basic_algorithm = 3
142+
[(google.api.field_behavior) = REQUIRED];
136143
}
137144

138145
// Required. Describes how the autoscaler will operate for primary workers.
139-
InstanceGroupAutoscalingPolicyConfig worker_config = 4 [(google.api.field_behavior) = REQUIRED];
146+
InstanceGroupAutoscalingPolicyConfig worker_config = 4
147+
[(google.api.field_behavior) = REQUIRED];
140148

141149
// Optional. Describes how the autoscaler will operate for secondary workers.
142-
InstanceGroupAutoscalingPolicyConfig secondary_worker_config = 5 [(google.api.field_behavior) = OPTIONAL];
150+
InstanceGroupAutoscalingPolicyConfig secondary_worker_config = 5
151+
[(google.api.field_behavior) = OPTIONAL];
143152

144153
// Optional. The labels to associate with this autoscaling policy.
145154
// Label **keys** must contain 1 to 63 characters, and must conform to
@@ -155,14 +164,16 @@ message AutoscalingPolicy {
155164
message BasicAutoscalingAlgorithm {
156165
oneof config {
157166
// Required. YARN autoscaling configuration.
158-
BasicYarnAutoscalingConfig yarn_config = 1 [(google.api.field_behavior) = REQUIRED];
167+
BasicYarnAutoscalingConfig yarn_config = 1
168+
[(google.api.field_behavior) = REQUIRED];
159169
}
160170

161171
// Optional. Duration between scaling events. A scaling period starts after
162172
// the update operation from the previous event has completed.
163173
//
164174
// Bounds: [2m, 1d]. Default: 2m.
165-
google.protobuf.Duration cooldown_period = 2 [(google.api.field_behavior) = OPTIONAL];
175+
google.protobuf.Duration cooldown_period = 2
176+
[(google.api.field_behavior) = OPTIONAL];
166177
}
167178

168179
// Basic autoscaling configurations for YARN.
@@ -173,22 +184,23 @@ message BasicYarnAutoscalingConfig {
173184
// downscaling operations.
174185
//
175186
// Bounds: [0s, 1d].
176-
google.protobuf.Duration graceful_decommission_timeout = 5 [(google.api.field_behavior) = REQUIRED];
177-
178-
// Required. Fraction of average YARN pending memory in the last cooldown period
179-
// for which to add workers. A scale-up factor of 1.0 will result in scaling
180-
// up so that there is no pending memory remaining after the update (more
181-
// aggressive scaling). A scale-up factor closer to 0 will result in a smaller
182-
// magnitude of scaling up (less aggressive scaling).
183-
// See [How autoscaling
187+
google.protobuf.Duration graceful_decommission_timeout = 5
188+
[(google.api.field_behavior) = REQUIRED];
189+
190+
// Required. Fraction of average YARN pending memory in the last cooldown
191+
// period for which to add workers. A scale-up factor of 1.0 will result in
192+
// scaling up so that there is no pending memory remaining after the update
193+
// (more aggressive scaling). A scale-up factor closer to 0 will result in a
194+
// smaller magnitude of scaling up (less aggressive scaling). See [How
195+
// autoscaling
184196
// works](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/autoscaling#how_autoscaling_works)
185197
// for more information.
186198
//
187199
// Bounds: [0.0, 1.0].
188200
double scale_up_factor = 1 [(google.api.field_behavior) = REQUIRED];
189201

190-
// Required. Fraction of average YARN pending memory in the last cooldown period
191-
// for which to remove workers. A scale-down factor of 1 will result in
202+
// Required. Fraction of average YARN pending memory in the last cooldown
203+
// period for which to remove workers. A scale-down factor of 1 will result in
192204
// scaling down so that there is no available memory remaining after the
193205
// update (more aggressive scaling). A scale-down factor of 0 disables
194206
// removing workers, which can be beneficial for autoscaling a single job.
@@ -206,7 +218,8 @@ message BasicYarnAutoscalingConfig {
206218
// on any recommended change.
207219
//
208220
// Bounds: [0.0, 1.0]. Default: 0.0.
209-
double scale_up_min_worker_fraction = 3 [(google.api.field_behavior) = OPTIONAL];
221+
double scale_up_min_worker_fraction = 3
222+
[(google.api.field_behavior) = OPTIONAL];
210223

211224
// Optional. Minimum scale-down threshold as a fraction of total cluster size
212225
// before scaling occurs. For example, in a 20-worker cluster, a threshold of
@@ -215,7 +228,8 @@ message BasicYarnAutoscalingConfig {
215228
// on any recommended change.
216229
//
217230
// Bounds: [0.0, 1.0]. Default: 0.0.
218-
double scale_down_min_worker_fraction = 4 [(google.api.field_behavior) = OPTIONAL];
231+
double scale_down_min_worker_fraction = 4
232+
[(google.api.field_behavior) = OPTIONAL];
219233
}
220234

221235
// Configuration for the size bounds of an instance group, including its
@@ -358,7 +372,8 @@ message ListAutoscalingPoliciesRequest {
358372
// A response to a request to list autoscaling policies in a project.
359373
message ListAutoscalingPoliciesResponse {
360374
// Output only. Autoscaling policies list.
361-
repeated AutoscalingPolicy policies = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
375+
repeated AutoscalingPolicy policies = 1
376+
[(google.api.field_behavior) = OUTPUT_ONLY];
362377

363378
// Output only. This token is included in the response if there are more
364379
// results to fetch.

0 commit comments

Comments
 (0)