Skip to content

Commit 02f62c8

Browse files
Google APIscopybara-github
Google APIs
authored and
copybara-github
committed
feat: add support for new Dataproc features
1. Allow flink and trino job support for workflow templates API 2. Add unreachable output field for LIST workflow template API 3. Add unreachable output field for LIST batch templates API 4. Add kms key input for create cluster API 5. Add FLINK metric source for Dataproc Metric Source PiperOrigin-RevId: 673000575
1 parent 32bc036 commit 02f62c8

3 files changed

Lines changed: 85 additions & 0 deletions

File tree

‎google/cloud/dataproc/v1/batches.proto‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,11 @@ message ListBatchesResponse {
170170
// A token, which can be sent as `page_token` to retrieve the next page.
171171
// If this field is omitted, there are no subsequent pages.
172172
string next_page_token = 2;
173+
174+
// Output only. List of Batches that could not be included in the response.
175+
// Attempting to get one of these resources may indicate why it was not
176+
// included in the list response.
177+
repeated string unreachable = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
173178
}
174179

175180
// A request to delete a batch workload.

‎google/cloud/dataproc/v1/clusters.proto‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -382,6 +382,34 @@ message EncryptionConfig {
382382
// Optional. The Cloud KMS key name to use for PD disk encryption for all
383383
// instances in the cluster.
384384
string gce_pd_kms_key_name = 1 [(google.api.field_behavior) = OPTIONAL];
385+
386+
// Optional. The Cloud KMS key resource name to use for cluster persistent
387+
// disk and job argument encryption. See [Use CMEK with cluster data]
388+
// (https://cloud.google.com//dataproc/docs/concepts/configuring-clusters/customer-managed-encryption#use_cmek_with_cluster_data)
389+
// for more information.
390+
//
391+
// When this key resource name is provided, the following job arguments of
392+
// the following job types submitted to the cluster are encrypted using CMEK:
393+
//
394+
// * [FlinkJob
395+
// args](https://cloud.google.com/dataproc/docs/reference/rest/v1/FlinkJob)
396+
// * [HadoopJob
397+
// args](https://cloud.google.com/dataproc/docs/reference/rest/v1/HadoopJob)
398+
// * [SparkJob
399+
// args](https://cloud.google.com/dataproc/docs/reference/rest/v1/SparkJob)
400+
// * [SparkRJob
401+
// args](https://cloud.google.com/dataproc/docs/reference/rest/v1/SparkRJob)
402+
// * [PySparkJob
403+
// args](https://cloud.google.com/dataproc/docs/reference/rest/v1/PySparkJob)
404+
// * [SparkSqlJob](https://cloud.google.com/dataproc/docs/reference/rest/v1/SparkSqlJob)
405+
// scriptVariables and queryList.queries
406+
// * [HiveJob](https://cloud.google.com/dataproc/docs/reference/rest/v1/HiveJob)
407+
// scriptVariables and queryList.queries
408+
// * [PigJob](https://cloud.google.com/dataproc/docs/reference/rest/v1/PigJob)
409+
// scriptVariables and queryList.queries
410+
// * [PrestoJob](https://cloud.google.com/dataproc/docs/reference/rest/v1/PrestoJob)
411+
// scriptVariables and queryList.queries
412+
string kms_key = 2 [(google.api.field_behavior) = OPTIONAL];
385413
}
386414

387415
// Common config settings for resources of Compute Engine cluster
@@ -1211,6 +1239,9 @@ message DataprocMetricConfig {
12111239

12121240
// hivemetastore metric source
12131241
HIVEMETASTORE = 7;
1242+
1243+
// flink metric source
1244+
FLINK = 8;
12141245
}
12151246

12161247
// A Dataproc custom metric.

‎google/cloud/dataproc/v1/workflow_templates.proto‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,39 @@ message WorkflowTemplate {
198198
history: ORIGINALLY_SINGLE_PATTERN
199199
};
200200

201+
// Encryption settings for encrypting workflow template job arguments.
202+
message EncryptionConfig {
203+
// Optional. The Cloud KMS key name to use for encrypting
204+
// workflow template job arguments.
205+
//
206+
// When this this key is provided, the following workflow template
207+
// [job arguments]
208+
// (https://cloud.google.com/dataproc/docs/concepts/workflows/use-workflows#adding_jobs_to_a_template),
209+
// if present, are
210+
// [CMEK
211+
// encrypted](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/customer-managed-encryption#use_cmek_with_workflow_template_data):
212+
//
213+
// * [FlinkJob
214+
// args](https://cloud.google.com/dataproc/docs/reference/rest/v1/FlinkJob)
215+
// * [HadoopJob
216+
// args](https://cloud.google.com/dataproc/docs/reference/rest/v1/HadoopJob)
217+
// * [SparkJob
218+
// args](https://cloud.google.com/dataproc/docs/reference/rest/v1/SparkJob)
219+
// * [SparkRJob
220+
// args](https://cloud.google.com/dataproc/docs/reference/rest/v1/SparkRJob)
221+
// * [PySparkJob
222+
// args](https://cloud.google.com/dataproc/docs/reference/rest/v1/PySparkJob)
223+
// * [SparkSqlJob](https://cloud.google.com/dataproc/docs/reference/rest/v1/SparkSqlJob)
224+
// scriptVariables and queryList.queries
225+
// * [HiveJob](https://cloud.google.com/dataproc/docs/reference/rest/v1/HiveJob)
226+
// scriptVariables and queryList.queries
227+
// * [PigJob](https://cloud.google.com/dataproc/docs/reference/rest/v1/PigJob)
228+
// scriptVariables and queryList.queries
229+
// * [PrestoJob](https://cloud.google.com/dataproc/docs/reference/rest/v1/PrestoJob)
230+
// scriptVariables and queryList.queries
231+
string kms_key = 1 [(google.api.field_behavior) = OPTIONAL];
232+
}
233+
201234
string id = 2 [(google.api.field_behavior) = REQUIRED];
202235

203236
// Output only. The resource name of the workflow template, as described
@@ -270,6 +303,11 @@ message WorkflowTemplate {
270303
// the cluster is deleted.
271304
google.protobuf.Duration dag_timeout = 10
272305
[(google.api.field_behavior) = OPTIONAL];
306+
307+
// Optional. Encryption settings for encrypting workflow template job
308+
// arguments.
309+
EncryptionConfig encryption_config = 11
310+
[(google.api.field_behavior) = OPTIONAL];
273311
}
274312

275313
// Specifies workflow execution target.
@@ -371,6 +409,12 @@ message OrderedJob {
371409

372410
// Optional. Job is a Presto job.
373411
PrestoJob presto_job = 12 [(google.api.field_behavior) = OPTIONAL];
412+
413+
// Optional. Job is a Trino job.
414+
TrinoJob trino_job = 13 [(google.api.field_behavior) = OPTIONAL];
415+
416+
// Optional. Job is a Flink job.
417+
FlinkJob flink_job = 14 [(google.api.field_behavior) = OPTIONAL];
374418
}
375419

376420
// Optional. The labels to associate with this job.
@@ -806,6 +850,11 @@ message ListWorkflowTemplatesResponse {
806850
// results to fetch. To fetch additional results, provide this value as the
807851
// page_token in a subsequent <code>ListWorkflowTemplatesRequest</code>.
808852
string next_page_token = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
853+
854+
// Output only. List of workflow templates that could not be included in the
855+
// response. Attempting to get one of these resources may indicate why it was
856+
// not included in the list response.
857+
repeated string unreachable = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
809858
}
810859

811860
// A request to delete a workflow template.

0 commit comments

Comments
 (0)