Skip to content

Commit b1161bf

Browse files
Google APIscopybara-github
Google APIs
authored and
copybara-github
committed
feat: Support adding a workflow action to execute a Data Preparation node
docs: Updated the formatting in some comments in multiple services PiperOrigin-RevId: 759511972
1 parent e82ef04 commit b1161bf

2 files changed

Lines changed: 241 additions & 20 deletions

File tree

‎google/cloud/dataform/v1beta1/dataform.proto‎

Lines changed: 198 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,10 @@ option (google.api.resource_definition) = {
4545
type: "cloudkms.googleapis.com/CryptoKeyVersion"
4646
pattern: "projects/{project}/locations/{location}/keyRings/{key_ring}/cryptoKeys/{crypto_key}/cryptoKeyVersions/{crypto_key_version}"
4747
};
48+
option (google.api.resource_definition) = {
49+
type: "aiplatform.googleapis.com/NotebookRuntimeTemplate"
50+
pattern: "projects/{project}/locations/{location}/notebookRuntimeTemplates/{notebook_runtime_template}"
51+
};
4852

4953
// Dataform is a service to develop, create, document, test, and update curated
5054
// tables in BigQuery.
@@ -55,6 +59,9 @@ service Dataform {
5559
"https://www.googleapis.com/auth/cloud-platform";
5660

5761
// Lists Repositories in a given project and location.
62+
//
63+
// **Note:** *This method can return repositories not shown in the [Dataform
64+
// UI](https://console.cloud.google.com/bigquery/dataform)*.
5865
rpc ListRepositories(ListRepositoriesRequest)
5966
returns (ListRepositoriesResponse) {
6067
option (google.api.http) = {
@@ -82,11 +89,10 @@ service Dataform {
8289

8390
// Updates a single Repository.
8491
//
85-
// **Note:** This method does not fully implement
86-
// [AIP-134](https://google.aip.dev/134); in particular:
87-
// - The wildcard entry (**\***) is treated as a bad request
88-
// - When the **field_mask** is omitted, instead of only updating the set
89-
// fields, the request is treated as a full update on all modifiable fields
92+
// **Note:** *This method does not fully implement
93+
// [AIP/134](https://google.aip.dev/134). The wildcard entry (\*) is treated
94+
// as a bad request, and when the `field_mask` is omitted, the request is
95+
// treated as a full update on all modifiable fields.*
9096
rpc UpdateRepository(UpdateRepositoryRequest) returns (Repository) {
9197
option (google.api.http) = {
9298
patch: "/v1beta1/{repository.name=projects/*/locations/*/repositories/*}"
@@ -358,11 +364,10 @@ service Dataform {
358364

359365
// Updates a single ReleaseConfig.
360366
//
361-
// **Note:** This method does not fully implement
362-
// [AIP-134](https://google.aip.dev/134); in particular:
363-
// - The wildcard entry (**\***) is treated as a bad request
364-
// - When the **field_mask** is omitted, instead of only updating the set
365-
// fields, the request is treated as a full update on all modifiable fields
367+
// **Note:** *This method does not fully implement
368+
// [AIP/134](https://google.aip.dev/134). The wildcard entry (\*) is treated
369+
// as a bad request, and when the `field_mask` is omitted, the request is
370+
// treated as a full update on all modifiable fields.*
366371
rpc UpdateReleaseConfig(UpdateReleaseConfigRequest) returns (ReleaseConfig) {
367372
option (google.api.http) = {
368373
patch: "/v1beta1/{release_config.name=projects/*/locations/*/repositories/*/releaseConfigs/*}"
@@ -446,11 +451,10 @@ service Dataform {
446451

447452
// Updates a single WorkflowConfig.
448453
//
449-
// **Note:** This method does not fully implement
450-
// [AIP-134](https://google.aip.dev/134); in particular:
451-
// - The wildcard entry (**\***) is treated as a bad request
452-
// - When the **field_mask** is omitted, instead of only updating the set
453-
// fields, the request is treated as a full update on all modifiable fields
454+
// **Note:** *This method does not fully implement
455+
// [AIP/134](https://google.aip.dev/134). The wildcard entry (\*) is treated
456+
// as a bad request, and when the `field_mask` is omitted, the request is
457+
// treated as a full update on all modifiable fields.*
454458
rpc UpdateWorkflowConfig(UpdateWorkflowConfigRequest)
455459
returns (WorkflowConfig) {
456460
option (google.api.http) = {
@@ -533,11 +537,10 @@ service Dataform {
533537

534538
// Update default config for a given project and location.
535539
//
536-
// **Note:** This method does not fully implement
537-
// [AIP-134](https://google.aip.dev/134); in particular:
538-
// - The wildcard entry (**\***) is treated as a bad request
539-
// - When the **field_mask** is omitted, instead of only updating the set
540-
// fields, the request is treated as a full update on all modifiable fields
540+
// **Note:** *This method does not fully implement
541+
// [AIP/134](https://google.aip.dev/134). The wildcard entry (\*) is treated
542+
// as a bad request, and when the `field_mask` is omitted, the request is
543+
// treated as a full update on all modifiable fields.*
541544
rpc UpdateConfig(UpdateConfigRequest) returns (Config) {
542545
option (google.api.http) = {
543546
patch: "/v1beta1/{config.name=projects/*/locations/*/config}"
@@ -1917,6 +1920,10 @@ message CodeCompilationConfig {
19171920
// Optional. The prefix that should be prepended to all table names.
19181921
string table_prefix = 7 [(google.api.field_behavior) = OPTIONAL];
19191922

1923+
// Optional. The prefix to prepend to built-in assertion names.
1924+
string builtin_assertion_name_prefix = 10
1925+
[(google.api.field_behavior) = OPTIONAL];
1926+
19201927
// Optional. The default notebook runtime options.
19211928
NotebookRuntimeOptions default_notebook_runtime_options = 9
19221929
[(google.api.field_behavior) = OPTIONAL];
@@ -1930,6 +1937,17 @@ message NotebookRuntimeOptions {
19301937
// Format: `gs://bucket-name`.
19311938
string gcs_output_bucket = 1 [(google.api.field_behavior) = OPTIONAL];
19321939
}
1940+
1941+
// Optional. The resource name of the [Colab runtime template]
1942+
// (https://cloud.google.com/colab/docs/runtimes), from which a runtime is
1943+
// created for notebook executions. If not specified, a runtime is created
1944+
// with Colab's default specifications.
1945+
string ai_platform_notebook_runtime_template = 2 [
1946+
(google.api.field_behavior) = OPTIONAL,
1947+
(google.api.resource_reference) = {
1948+
type: "aiplatform.googleapis.com/NotebookRuntimeTemplate"
1949+
}
1950+
];
19331951
}
19341952

19351953
// `ListCompilationResults` request message.
@@ -2213,6 +2231,81 @@ message CompilationResultAction {
22132231
repeated string tags = 4;
22142232
}
22152233

2234+
// Defines a compiled Data Preparation entity
2235+
message DataPreparation {
2236+
// Definition of a SQL Data Preparation
2237+
message SqlDefinition {
2238+
// The SQL query representing the data preparation steps. Formatted as a
2239+
// Pipe SQL query statement.
2240+
string query = 1;
2241+
2242+
// Error table configuration,
2243+
ErrorTable error_table = 2;
2244+
2245+
// Load configuration.
2246+
LoadConfig load = 3;
2247+
}
2248+
2249+
// Error table information, used to write error data into a BigQuery
2250+
// table.
2251+
message ErrorTable {
2252+
// Error Table target.
2253+
Target target = 1;
2254+
2255+
// Error table partition expiration in days. Only positive values are
2256+
// allowed.
2257+
int32 retention_days = 2;
2258+
}
2259+
2260+
// The definition for the data preparation.
2261+
oneof definition {
2262+
// The data preparation definition, stored as a YAML string.
2263+
string contents_yaml = 5;
2264+
2265+
// SQL definition for a Data Preparation. Contains a SQL query and
2266+
// additional context information.
2267+
SqlDefinition contents_sql = 6;
2268+
}
2269+
2270+
// A list of actions that this action depends on.
2271+
repeated Target dependency_targets = 1;
2272+
2273+
// Whether this action is disabled (i.e. should not be run).
2274+
bool disabled = 2;
2275+
2276+
// Arbitrary, user-defined tags on this action.
2277+
repeated string tags = 4;
2278+
}
2279+
2280+
// Simplified load configuration for actions
2281+
message LoadConfig {
2282+
// Load mode
2283+
oneof mode {
2284+
// Replace destination table
2285+
SimpleLoadMode replace = 1;
2286+
2287+
// Append into destination table
2288+
SimpleLoadMode append = 2;
2289+
2290+
// Insert records where the value exceeds the previous maximum value for a
2291+
// column in the destination table
2292+
IncrementalLoadMode maximum = 3;
2293+
2294+
// Insert records where the value of a column is not already present in
2295+
// the destination table
2296+
IncrementalLoadMode unique = 4;
2297+
}
2298+
}
2299+
2300+
// Simple load definition
2301+
message SimpleLoadMode {}
2302+
2303+
// Load definition for incremental load modes
2304+
message IncrementalLoadMode {
2305+
// Column name for incremental load modes
2306+
string column = 1;
2307+
}
2308+
22162309
// The compiled object.
22172310
oneof compiled_object {
22182311
// The database relation created/updated by this action.
@@ -2229,6 +2322,9 @@ message CompilationResultAction {
22292322

22302323
// The notebook executed by this action.
22312324
Notebook notebook = 8;
2325+
2326+
// The data preparation executed by this action.
2327+
DataPreparation data_preparation = 9;
22322328
}
22332329

22342330
// This action's identifier. Unique within the compilation result.
@@ -2352,6 +2448,9 @@ message WorkflowConfig {
23522448
repeated ScheduledExecutionRecord recent_scheduled_execution_records = 5
23532449
[(google.api.field_behavior) = OUTPUT_ONLY];
23542450

2451+
// Optional. Disables automatic creation of workflow invocations.
2452+
bool disabled = 8 [(google.api.field_behavior) = OPTIONAL];
2453+
23552454
// Output only. The timestamp of when the WorkflowConfig was created.
23562455
google.protobuf.Timestamp create_time = 9
23572456
[(google.api.field_behavior) = OUTPUT_ONLY];
@@ -2694,6 +2793,81 @@ message WorkflowInvocationAction {
26942793
string job_id = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
26952794
}
26962795

2796+
// Represents a workflow action that will run a Data Preparation.
2797+
message DataPreparationAction {
2798+
// Definition of a SQL Data Preparation
2799+
message ActionSqlDefinition {
2800+
// The SQL query representing the data preparation steps. Formatted as a
2801+
// Pipe SQL query statement.
2802+
string query = 1;
2803+
2804+
// Error table configuration,
2805+
ActionErrorTable error_table = 2;
2806+
2807+
// Load configuration.
2808+
ActionLoadConfig load_config = 3;
2809+
}
2810+
2811+
// Error table information, used to write error data into a BigQuery
2812+
// table.
2813+
message ActionErrorTable {
2814+
// Error Table target.
2815+
Target target = 1;
2816+
2817+
// Error table partition expiration in days. Only positive values are
2818+
// allowed.
2819+
int32 retention_days = 2;
2820+
}
2821+
2822+
// Simplified load configuration for actions
2823+
message ActionLoadConfig {
2824+
// Load mode
2825+
oneof mode {
2826+
// Replace destination table
2827+
ActionSimpleLoadMode replace = 1;
2828+
2829+
// Append into destination table
2830+
ActionSimpleLoadMode append = 2;
2831+
2832+
// Insert records where the value exceeds the previous maximum value for
2833+
// a column in the destination table
2834+
ActionIncrementalLoadMode maximum = 3;
2835+
2836+
// Insert records where the value of a column is not already present in
2837+
// the destination table
2838+
ActionIncrementalLoadMode unique = 4;
2839+
}
2840+
}
2841+
2842+
// Simple load definition
2843+
message ActionSimpleLoadMode {}
2844+
2845+
// Load definition for incremental load modes
2846+
message ActionIncrementalLoadMode {
2847+
// Column name for incremental load modes
2848+
string column = 1;
2849+
}
2850+
2851+
// The definition for the data preparation.
2852+
oneof definition {
2853+
// Output only. YAML representing the contents of the data preparation.
2854+
// Can be used to show the customer what the input was to their workflow.
2855+
string contents_yaml = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
2856+
2857+
// SQL definition for a Data Preparation. Contains a SQL query and
2858+
// additional context information.
2859+
ActionSqlDefinition contents_sql = 6;
2860+
}
2861+
2862+
// Output only. The generated BigQuery SQL script that will be executed. For
2863+
// reference only.
2864+
string generated_sql = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
2865+
2866+
// Output only. The ID of the BigQuery job that executed the SQL in
2867+
// sql_script. Only set once the job has started to run.
2868+
string job_id = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
2869+
}
2870+
26972871
// Represents the current state of a workflow invocation action.
26982872
enum State {
26992873
// The action has not yet been considered for invocation.
@@ -2729,6 +2903,10 @@ message WorkflowInvocationAction {
27292903
// Output only. The workflow action's notebook action details.
27302904
NotebookAction notebook_action = 8
27312905
[(google.api.field_behavior) = OUTPUT_ONLY];
2906+
2907+
// Output only. The workflow action's data preparation action details.
2908+
DataPreparationAction data_preparation_action = 9
2909+
[(google.api.field_behavior) = OUTPUT_ONLY];
27322910
}
27332911

27342912
// Output only. This action's identifier. Unique within the workflow

‎google/cloud/dataform/v1beta1/dataform_v1beta1.yaml‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,3 +87,46 @@ authentication:
8787
canonical_scopes: |-
8888
https://www.googleapis.com/auth/bigquery,
8989
https://www.googleapis.com/auth/cloud-platform
90+
91+
publishing:
92+
new_issue_uri: https://issuetracker.google.com/issues/new?component=994183
93+
documentation_uri: https://cloud.google.com/dataform/docs/overview
94+
api_short_name: dataform
95+
github_label: 'api: dataform'
96+
doc_tag_prefix: dataform
97+
organization: CLOUD
98+
library_settings:
99+
- version: google.cloud.dataform.v1beta1
100+
launch_stage: BETA
101+
java_settings:
102+
common:
103+
destinations:
104+
- PACKAGE_MANAGER
105+
cpp_settings:
106+
common:
107+
destinations:
108+
- PACKAGE_MANAGER
109+
php_settings:
110+
common:
111+
destinations:
112+
- PACKAGE_MANAGER
113+
python_settings:
114+
common:
115+
destinations:
116+
- PACKAGE_MANAGER
117+
node_settings:
118+
common:
119+
destinations:
120+
- PACKAGE_MANAGER
121+
dotnet_settings:
122+
common:
123+
destinations:
124+
- PACKAGE_MANAGER
125+
ruby_settings:
126+
common:
127+
destinations:
128+
- PACKAGE_MANAGER
129+
go_settings:
130+
common:
131+
destinations:
132+
- PACKAGE_MANAGER

0 commit comments

Comments
 (0)