Skip to content

Commit 2415cfc

Browse files
Google APIscopybara-github
Google APIs
authored and
copybara-github
committed
feat: Add custom BigQuery dataset location support in Auto Discovery
docs: update the Dataplex Catalog proto to remove the info about schema changes feat: DataTaxonomyService is now deprecated PiperOrigin-RevId: 735645981
1 parent 8a7b36d commit 2415cfc

8 files changed

Lines changed: 157 additions & 48 deletions

File tree

‎google/cloud/dataplex/v1/catalog.proto‎

Lines changed: 40 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -254,24 +254,14 @@ service CatalogService {
254254
}
255255

256256
// Gets an Entry.
257-
//
258-
// **Caution**: The BigQuery metadata that is stored in Dataplex Catalog is
259-
// changing. For more information, see [Changes to BigQuery metadata stored in
260-
// Dataplex
261-
// Catalog](https://cloud.google.com/dataplex/docs/biqquery-metadata-changes).
262257
rpc GetEntry(GetEntryRequest) returns (Entry) {
263258
option (google.api.http) = {
264259
get: "/v1/{name=projects/*/locations/*/entryGroups/*/entries/**}"
265260
};
266261
option (google.api.method_signature) = "name";
267262
}
268263

269-
// Looks up a single Entry by name using the permission on the source system.
270-
//
271-
// **Caution**: The BigQuery metadata that is stored in Dataplex Catalog is
272-
// changing. For more information, see [Changes to BigQuery metadata stored in
273-
// Dataplex
274-
// Catalog](https://cloud.google.com/dataplex/docs/biqquery-metadata-changes).
264+
// Looks up an entry by name using the permission on the source system.
275265
rpc LookupEntry(LookupEntryRequest) returns (Entry) {
276266
option (google.api.http) = {
277267
get: "/v1/{name=projects/*/locations/*}:lookupEntry"
@@ -1444,9 +1434,12 @@ message ImportItem {
14441434
// listed in the update mask, and regardless of whether a field is present
14451435
// in the `entry` object.
14461436
//
1447-
//
14481437
// The `update_mask` field is ignored when an entry is created or re-created.
14491438
//
1439+
// In an aspect-only metadata job (when entry sync mode is `NONE`), set this
1440+
// value to `aspects`.
1441+
//
1442+
//
14501443
// Dataplex also determines which entries and aspects to modify by comparing
14511444
// the values and timestamps that you provide in the metadata import file with
14521445
// the values and timestamps that exist in your project. For more information,
@@ -1460,18 +1453,18 @@ message ImportItem {
14601453
// aspect type and are attached directly to the entry.
14611454
// * `{aspect_type_reference}@{path}`: matches aspects that belong to the
14621455
// specified aspect type and path.
1463-
// * `<aspect_type_reference>@*` : matches aspects of the given type for all
1456+
// * `{aspect_type_reference}@*` : matches aspects of the given type for all
14641457
// paths.
14651458
// * `*@path` : matches aspects of all types on the given path.
1459+
//
14661460
// Replace `{aspect_type_reference}` with a reference to the aspect type, in
14671461
// the format
14681462
// `{project_id_or_number}.{location_id}.{aspect_type_id}`.
14691463
//
1470-
// If you leave this field empty, it is treated as specifying exactly those
1471-
// aspects that are present within the specified entry.
1472-
//
1473-
// In `FULL` entry sync mode, Dataplex implicitly adds the keys for all of the
1474-
// required aspects of an entry.
1464+
// In `FULL` entry sync mode, if you leave this field empty, it is treated as
1465+
// specifying exactly those aspects that are present within the specified
1466+
// entry. Dataplex implicitly adds the keys for all of the required aspects of
1467+
// an entry.
14751468
repeated string aspect_keys = 3;
14761469
}
14771470

@@ -1602,9 +1595,20 @@ message MetadataJob {
16021595
[(google.api.field_behavior) = OUTPUT_ONLY];
16031596
}
16041597

1605-
// Job specification for a metadata import job
1598+
// Job specification for a metadata import job.
1599+
//
1600+
// You can run the following kinds of metadata import jobs:
1601+
//
1602+
// * Full sync of entries with incremental import of their aspects.
1603+
// Supported for custom entries.
1604+
// * Incremental import of aspects only. Supported for aspects that belong
1605+
// to custom entries and system entries. For custom entries, you can modify
1606+
// both optional aspects and required aspects. For system entries, you can
1607+
// modify optional aspects.
16061608
message ImportJobSpec {
1607-
// Specifies how the entries and aspects in a metadata job are updated.
1609+
// Specifies how the entries and aspects in a metadata job are updated. For
1610+
// more information, see [Sync
1611+
// mode](https://cloud.google.com/dataplex/docs/import-metadata#sync-mode).
16081612
enum SyncMode {
16091613
// Sync mode unspecified.
16101614
SYNC_MODE_UNSPECIFIED = 0;
@@ -1613,16 +1617,22 @@ message MetadataJob {
16131617
// Dataplex but isn't included in the metadata import file, the resource
16141618
// is deleted when you run the metadata job. Use this mode to perform a
16151619
// full sync of the set of entries in the job scope.
1620+
//
1621+
// This sync mode is supported for entries.
16161622
FULL = 1;
16171623

1618-
// Only the entries and aspects that are explicitly included in the
1624+
// Only the resources that are explicitly included in the
16191625
// metadata import file are modified. Use this mode to modify a subset of
16201626
// resources while leaving unreferenced resources unchanged.
1627+
//
1628+
// This sync mode is supported for aspects.
16211629
INCREMENTAL = 2;
16221630

1623-
// If entry sync mode is NONE, then the entry-specific fields (apart from
1624-
// aspects) are not modified and the aspects are modified according to the
1625-
// aspect_sync_mode
1631+
// If entry sync mode is `NONE`, then aspects are modified according
1632+
// to the aspect sync mode. Other metadata that belongs to entries in the
1633+
// job's scope isn't modified.
1634+
//
1635+
// This sync mode is supported for entries.
16261636
NONE = 3;
16271637
}
16281638

@@ -1654,8 +1664,8 @@ message MetadataJob {
16541664
// Required. The entry group that is in scope for the import job,
16551665
// specified as a relative resource name in the format
16561666
// `projects/{project_number_or_id}/locations/{location_id}/entryGroups/{entry_group_id}`.
1657-
// Only entries that belong to the specified entry group are affected by
1658-
// the job.
1667+
// Only entries and aspects that belong to the specified entry group are
1668+
// affected by the job.
16591669
//
16601670
// Must contain exactly one element. The entry group and the job
16611671
// must be in the same location.
@@ -1669,7 +1679,8 @@ message MetadataJob {
16691679
// Required. The entry types that are in scope for the import job,
16701680
// specified as relative resource names in the format
16711681
// `projects/{project_number_or_id}/locations/{location_id}/entryTypes/{entry_type_id}`.
1672-
// The job modifies only the entries that belong to these entry types.
1682+
// The job modifies only the entries and aspects that belong to these
1683+
// entry types.
16731684
//
16741685
// If the metadata import file attempts to modify an entry whose type
16751686
// isn't included in this list, the import job is halted before modifying
@@ -1689,6 +1700,8 @@ message MetadataJob {
16891700
// `projects/{project_number_or_id}/locations/{location_id}/aspectTypes/{aspect_type_id}`.
16901701
// The job modifies only the aspects that belong to these aspect types.
16911702
//
1703+
// This field is required when creating an aspect-only import job.
1704+
//
16921705
// If the metadata import file attempts to modify an aspect whose type
16931706
// isn't included in this list, the import job is halted before modifying
16941707
// any entries or aspects.
@@ -1733,15 +1746,9 @@ message MetadataJob {
17331746
ImportJobScope scope = 2 [(google.api.field_behavior) = REQUIRED];
17341747

17351748
// Required. The sync mode for entries.
1736-
// Only `FULL` mode is supported for entries. All entries in the job's scope
1737-
// are modified. If an entry exists in Dataplex but isn't included in the
1738-
// metadata import file, the entry is deleted when you run the metadata job.
17391749
SyncMode entry_sync_mode = 3 [(google.api.field_behavior) = REQUIRED];
17401750

17411751
// Required. The sync mode for aspects.
1742-
// Only `INCREMENTAL` mode is supported for aspects. An aspect is modified
1743-
// only if the metadata import file includes a reference to the aspect in
1744-
// the `update_mask` field and the `aspect_keys` field.
17451752
SyncMode aspect_sync_mode = 4 [(google.api.field_behavior) = REQUIRED];
17461753

17471754
// Optional. The level of logs to write to Cloud Logging for this job.

‎google/cloud/dataplex/v1/data_discovery.proto‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,29 @@ message DataDiscoverySpec {
6565
type: "bigqueryconnection.googleapis.com/Connection"
6666
}
6767
];
68+
69+
// Optional. The location of the BigQuery dataset to publish BigLake
70+
// external or non-BigLake external tables to.
71+
// 1. If the Cloud Storage bucket is located in a multi-region bucket, then
72+
// BigQuery dataset can be in the same multi-region bucket or any single
73+
// region that is included in the same multi-region bucket. The datascan can
74+
// be created in any single region that is included in the same multi-region
75+
// bucket
76+
// 2. If the Cloud Storage bucket is located in a dual-region bucket, then
77+
// BigQuery dataset can be located in regions that are included in the
78+
// dual-region bucket, or in a multi-region that includes the dual-region.
79+
// The datascan can be created in any single region that is included in the
80+
// same dual-region bucket.
81+
// 3. If the Cloud Storage bucket is located in a single region, then
82+
// BigQuery dataset can be in the same single region or any multi-region
83+
// bucket that includes the same single region. The datascan will be created
84+
// in the same single region as the bucket.
85+
// 4. If the BigQuery dataset is in single region, it must be in the same
86+
// single region as the datascan.
87+
//
88+
// For supported values, refer to
89+
// https://cloud.google.com/bigquery/docs/locations#supported_locations.
90+
string location = 4 [(google.api.field_behavior) = OPTIONAL];
6891
}
6992

7093
// Configurations related to Cloud Storage as the data source.

‎google/cloud/dataplex/v1/data_profile.proto‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,8 +63,10 @@ message DataProfileSpec {
6363
float sampling_percent = 2 [(google.api.field_behavior) = OPTIONAL];
6464

6565
// Optional. A filter applied to all rows in a single DataScan job.
66-
// The filter needs to be a valid SQL expression for a WHERE clause in
67-
// BigQuery standard SQL syntax.
66+
// The filter needs to be a valid SQL expression for a [WHERE clause in
67+
// GoogleSQL
68+
// syntax](https://cloud.google.com/bigquery/docs/reference/standard-sql/query-syntax#where_clause).
69+
//
6870
// Example: col1 >= 0 AND col2 < 10
6971
string row_filter = 3 [(google.api.field_behavior) = OPTIONAL];
7072

‎google/cloud/dataplex/v1/data_quality.proto‎

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -106,8 +106,10 @@ message DataQualitySpec {
106106
float sampling_percent = 4 [(google.api.field_behavior) = OPTIONAL];
107107

108108
// Optional. A filter applied to all rows in a single DataScan job.
109-
// The filter needs to be a valid SQL expression for a WHERE clause in
110-
// BigQuery standard SQL syntax.
109+
// The filter needs to be a valid SQL expression for a [WHERE clause in
110+
// GoogleSQL
111+
// syntax](https://cloud.google.com/bigquery/docs/reference/standard-sql/query-syntax#where_clause).
112+
//
111113
// Example: col1 >= 0 AND col2 < 10
112114
string row_filter = 5 [(google.api.field_behavior) = OPTIONAL];
113115

@@ -350,8 +352,9 @@ message DataQualityRule {
350352

351353
// Evaluates whether each row passes the specified condition.
352354
//
353-
// The SQL expression needs to use BigQuery standard SQL syntax and should
354-
// produce a boolean value per row as the result.
355+
// The SQL expression needs to use [GoogleSQL
356+
// syntax](https://cloud.google.com/bigquery/docs/reference/standard-sql/query-syntax)
357+
// and should produce a boolean value per row as the result.
355358
//
356359
// Example: col1 >= 0 AND col2 < 10
357360
message RowConditionExpectation {
@@ -361,8 +364,9 @@ message DataQualityRule {
361364

362365
// Evaluates whether the provided expression is true.
363366
//
364-
// The SQL expression needs to use BigQuery standard SQL syntax and should
365-
// produce a scalar boolean result.
367+
// The SQL expression needs to use [GoogleSQL
368+
// syntax](https://cloud.google.com/bigquery/docs/reference/standard-sql/query-syntax)
369+
// and should produce a scalar boolean result.
366370
//
367371
// Example: MIN(col1) >= 0
368372
message TableConditionExpectation {
@@ -373,8 +377,9 @@ message DataQualityRule {
373377
// A SQL statement that is evaluated to return rows that match an invalid
374378
// state. If any rows are are returned, this rule fails.
375379
//
376-
// The SQL statement must use BigQuery standard SQL syntax, and must not
377-
// contain any semicolons.
380+
// The SQL statement must use [GoogleSQL
381+
// syntax](https://cloud.google.com/bigquery/docs/reference/standard-sql/query-syntax),
382+
// and must not contain any semicolons.
378383
//
379384
// You can use the data reference parameter `${data()}` to reference the
380385
// source table with all of its precondition filters applied. Examples of

0 commit comments

Comments
 (0)