Skip to content

Commit 99fd8be

Browse files
Google APIscopybara-github
Google APIs
authored and
copybara-github
committed
docs: update taxonomy display_name comment
feat: added Dataplex specific fields PiperOrigin-RevId: 440386238
1 parent d4ca15b commit 99fd8be

9 files changed

Lines changed: 243 additions & 6 deletions

‎google/cloud/datacatalog/v1/BUILD.bazel‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,9 @@ proto_library(
2525
"common.proto",
2626
"data_source.proto",
2727
"datacatalog.proto",
28+
"dataplex_spec.proto",
2829
"gcs_fileset_spec.proto",
30+
"physical_schema.proto",
2931
"policytagmanager.proto",
3032
"policytagmanagerserialization.proto",
3133
"schema.proto",

‎google/cloud/datacatalog/v1/common.proto‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,4 +48,7 @@ enum IntegratedSystem {
4848

4949
// Dataproc Metastore.
5050
DATAPROC_METASTORE = 3;
51+
52+
// Dataplex.
53+
DATAPLEX = 4;
5154
}

‎google/cloud/datacatalog/v1/data_source.proto‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,4 +48,40 @@ message DataSource {
4848
//
4949
// `//bigquery.googleapis.com/projects/{PROJECT_ID}/locations/{LOCATION}/datasets/{DATASET_ID}/tables/{TABLE_ID}`
5050
string resource = 2;
51+
52+
// Output only. Data Catalog entry name, if applicable.
53+
string source_entry = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
54+
55+
oneof properties {
56+
// Detailed properties of the underlying storage.
57+
StorageProperties storage_properties = 4;
58+
}
59+
}
60+
61+
// Details the properties of the underlying storage.
62+
message StorageProperties {
63+
// Patterns to identify a set of files for this fileset.
64+
//
65+
// Examples of a valid `file_pattern`:
66+
//
67+
// * `gs://bucket_name/dir/*`: matches all files in the `bucket_name/dir`
68+
// directory
69+
// * `gs://bucket_name/dir/**`: matches all files in the `bucket_name/dir`
70+
// and all subdirectories recursively
71+
// * `gs://bucket_name/file*`: matches files prefixed by `file` in
72+
// `bucket_name`
73+
// * `gs://bucket_name/??.txt`: matches files with two characters followed by
74+
// `.txt` in `bucket_name`
75+
// * `gs://bucket_name/[aeiou].txt`: matches files that contain a single
76+
// vowel character followed by `.txt` in
77+
// `bucket_name`
78+
// * `gs://bucket_name/[a-m].txt`: matches files that contain `a`, `b`, ...
79+
// or `m` followed by `.txt` in `bucket_name`
80+
// * `gs://bucket_name/a/*/b`: matches all files in `bucket_name` that match
81+
// the `a/*/b` pattern, such as `a/c/b`, `a/d/b`
82+
// * `gs://another_bucket/a.txt`: matches `gs://another_bucket/a.txt`
83+
repeated string file_pattern = 1;
84+
85+
// File type in MIME format, for example, `text/plain`.
86+
string file_type = 2;
5187
}

‎google/cloud/datacatalog/v1/datacatalog.proto‎

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import "google/api/resource.proto";
2323
import "google/cloud/datacatalog/v1/bigquery.proto";
2424
import "google/cloud/datacatalog/v1/common.proto";
2525
import "google/cloud/datacatalog/v1/data_source.proto";
26+
import "google/cloud/datacatalog/v1/dataplex_spec.proto";
2627
import "google/cloud/datacatalog/v1/gcs_fileset_spec.proto";
2728
import "google/cloud/datacatalog/v1/schema.proto";
2829
import "google/cloud/datacatalog/v1/search.proto";
@@ -34,7 +35,6 @@ import "google/iam/v1/iam_policy.proto";
3435
import "google/iam/v1/policy.proto";
3536
import "google/protobuf/empty.proto";
3637
import "google/protobuf/field_mask.proto";
37-
import "google/protobuf/timestamp.proto";
3838

3939
option cc_enable_arenas = true;
4040
option csharp_namespace = "Google.Cloud.DataCatalog.V1";
@@ -1065,6 +1065,10 @@ message Entry {
10651065
// Specification that applies to a user-defined function or procedure. Valid
10661066
// only for entries with the `ROUTINE` type.
10671067
RoutineSpec routine_spec = 28;
1068+
1069+
// Specification that applies to a fileset resource. Valid only
1070+
// for entries with the `FILESET` type.
1071+
FilesetSpec fileset_spec = 33;
10681072
}
10691073

10701074
// Display name of an entry.
@@ -1085,7 +1089,7 @@ message Entry {
10851089
// Default value is an empty string.
10861090
string description = 4;
10871091

1088-
// Business Context of the entry. Not supported for BigQuery datasets.
1092+
// Business Context of the entry. Not supported for BigQuery datasets
10891093
BusinessContext business_context = 37;
10901094

10911095
// Schema of the entry. An entry might not have any schema attached to it.
@@ -1133,6 +1137,18 @@ message DatabaseTableSpec {
11331137

11341138
// Type of this table.
11351139
TableType type = 1;
1140+
1141+
// Fields specific to a Dataplex table and present only in the Dataplex table
1142+
// entries.
1143+
DataplexTableSpec dataplex_table = 2;
1144+
}
1145+
1146+
// Specification that applies to a fileset. Valid only for entries with the
1147+
// 'FILESET' type.
1148+
message FilesetSpec {
1149+
// Fields specific to a Dataplex fileset and present only in the Dataplex
1150+
// fileset entries.
1151+
DataplexFilesetSpec dataplex_fileset = 1;
11361152
}
11371153

11381154
// Specification that applies to a data source connection. Valid only for
@@ -1241,8 +1257,8 @@ message Contacts {
12411257
// Designation of the person, for example, Data Steward.
12421258
string designation = 1;
12431259

1244-
// Email of the person in the format of `john.doe@example.com`,
1245-
// `<john.doe@example.com>`, or `John Doe<john.doe@example.com>`.
1260+
// Email of the person in the format of `john.doe@xyz`,
1261+
// `<john.doe@xyz>`, or `John Doe<john.doe@xyz>`.
12461262
string email = 2;
12471263
}
12481264

@@ -1681,6 +1697,12 @@ enum EntryType {
16811697
// Output only. Routine, for example, a BigQuery routine.
16821698
ROUTINE = 9;
16831699

1700+
// A Dataplex lake.
1701+
LAKE = 10;
1702+
1703+
// A Dataplex zone.
1704+
ZONE = 11;
1705+
16841706
// A service, for example, a Dataproc Metastore service.
16851707
SERVICE = 14;
16861708
}

‎google/cloud/datacatalog/v1/dataplex_spec.proto‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
// Copyright 2022 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
syntax = "proto3";
16+
17+
package google.cloud.datacatalog.v1;
18+
19+
import "google/cloud/datacatalog/v1/common.proto";
20+
import "google/cloud/datacatalog/v1/physical_schema.proto";
21+
22+
option cc_enable_arenas = true;
23+
option csharp_namespace = "Google.Cloud.DataCatalog.V1";
24+
option go_package = "google.golang.org/genproto/googleapis/cloud/datacatalog/v1;datacatalog";
25+
option java_multiple_files = true;
26+
option java_outer_classname = "DataplexSpecProto";
27+
option java_package = "com.google.cloud.datacatalog.v1";
28+
option php_namespace = "Google\\Cloud\\DataCatalog\\V1";
29+
option ruby_package = "Google::Cloud::DataCatalog::V1";
30+
31+
// Common Dataplex fields.
32+
message DataplexSpec {
33+
// Fully qualified resource name of an asset in Dataplex, to which the
34+
// underlying data source (Cloud Storage bucket or BigQuery dataset) of the
35+
// entity is attached.
36+
string asset = 1;
37+
38+
// Format of the data.
39+
PhysicalSchema data_format = 2;
40+
41+
// Compression format of the data, e.g., zip, gzip etc.
42+
string compression_format = 3;
43+
44+
// Project ID of the underlying Cloud Storage or BigQuery data. Note that
45+
// this may not be the same project as the correspondingly Dataplex lake /
46+
// zone / asset.
47+
string project_id = 4;
48+
}
49+
50+
// Entry specyfication for a Dataplex fileset.
51+
message DataplexFilesetSpec {
52+
// Common Dataplex fields.
53+
DataplexSpec dataplex_spec = 1;
54+
}
55+
56+
// Entry specification for a Dataplex table.
57+
message DataplexTableSpec {
58+
// List of external tables registered by Dataplex in other systems based on
59+
// the same underlying data.
60+
//
61+
// External tables allow to query this data in those systems.
62+
repeated DataplexExternalTable external_tables = 1;
63+
64+
// Common Dataplex fields.
65+
DataplexSpec dataplex_spec = 2;
66+
67+
// Indicates if the table schema is managed by the user or not.
68+
bool user_managed = 3;
69+
}
70+
71+
// External table registered by Dataplex.
72+
// Dataplex publishes data discovered from an asset into multiple other systems
73+
// (BigQuery, DPMS) in form of tables. We call them "external tables". External
74+
// tables are also synced into the Data Catalog.
75+
// This message contains pointers to
76+
// those external tables (fully qualified name, resource name et cetera) within
77+
// the Data Catalog.
78+
message DataplexExternalTable {
79+
// Service in which the external table is registered.
80+
IntegratedSystem system = 1;
81+
82+
// Fully qualified name (FQN) of the external table.
83+
string fully_qualified_name = 28;
84+
85+
// Google Cloud resource name of the external table.
86+
string google_cloud_resource = 3;
87+
88+
// Name of the Data Catalog entry representing the external table.
89+
string data_catalog_entry = 4;
90+
}

‎google/cloud/datacatalog/v1/physical_schema.proto‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
// Copyright 2022 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
syntax = "proto3";
16+
17+
package google.cloud.datacatalog.v1;
18+
19+
option cc_enable_arenas = true;
20+
option csharp_namespace = "Google.Cloud.DataCatalog.V1";
21+
option go_package = "google.golang.org/genproto/googleapis/cloud/datacatalog/v1;datacatalog";
22+
option java_multiple_files = true;
23+
option java_outer_classname = "PhysicalSchemaProto";
24+
option java_package = "com.google.cloud.datacatalog.v1";
25+
option php_namespace = "Google\\Cloud\\DataCatalog\\V1";
26+
option ruby_package = "Google::Cloud::DataCatalog::V1";
27+
28+
// Native schema used by a resource represented as an entry. Used by query
29+
// engines for deserializing and parsing source data.
30+
message PhysicalSchema {
31+
// Schema in Avro JSON format.
32+
message AvroSchema {
33+
// JSON source of the Avro schema.
34+
string text = 1;
35+
}
36+
37+
// Schema in Thrift format.
38+
message ThriftSchema {
39+
// Thrift IDL source of the schema.
40+
string text = 1;
41+
}
42+
43+
// Schema in protocol buffer format.
44+
message ProtobufSchema {
45+
// Protocol buffer source of the schema.
46+
string text = 1;
47+
}
48+
49+
// Marks a Parquet-encoded data source.
50+
message ParquetSchema {
51+
52+
}
53+
54+
// Marks an ORC-encoded data source.
55+
message OrcSchema {
56+
57+
}
58+
59+
// Marks a CSV-encoded data source.
60+
message CsvSchema {
61+
62+
}
63+
64+
oneof schema {
65+
// Schema in Avro JSON format.
66+
AvroSchema avro = 1;
67+
68+
// Schema in Thrift format.
69+
ThriftSchema thrift = 2;
70+
71+
// Schema in protocol buffer format.
72+
ProtobufSchema protobuf = 3;
73+
74+
// Marks a Parquet-encoded data source.
75+
ParquetSchema parquet = 4;
76+
77+
// Marks an ORC-encoded data source.
78+
OrcSchema orc = 5;
79+
80+
// Marks a CSV-encoded data source.
81+
CsvSchema csv = 6;
82+
}
83+
}

‎google/cloud/datacatalog/v1/policytagmanager.proto‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -229,6 +229,8 @@ message Taxonomy {
229229
// The name can't start or end with spaces, must contain only Unicode letters,
230230
// numbers, underscores, dashes, and spaces, and be at most 200 bytes long
231231
// when encoded in UTF-8.
232+
//
233+
// The taxonomy display name must be unique within an organization.
232234
string display_name = 2 [(google.api.field_behavior) = REQUIRED];
233235

234236
// Optional. Description of this taxonomy. If not set, defaults to empty.

‎google/cloud/datacatalog/v1/policytagmanagerserialization.proto‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ import "google/api/client.proto";
2121
import "google/api/field_behavior.proto";
2222
import "google/api/resource.proto";
2323
import "google/cloud/datacatalog/v1/policytagmanager.proto";
24-
import "google/iam/v1/policy.proto";
2524

2625
option cc_enable_arenas = true;
2726
option csharp_namespace = "Google.Cloud.DataCatalog.V1";

‎google/cloud/datacatalog/v1/tags.proto‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -164,7 +164,7 @@ message TagTemplate {
164164
// [ListTags][google.cloud.datacatalog.v1.ListTags] API response.
165165
//
166166
// Additionally, you can search for a public tag by value with a
167-
// simple search query instead of using a ``tag:`` predicate.
167+
// simple search query in addition to using a ``tag:`` predicate.
168168
bool is_publicly_readable = 5;
169169

170170
// Required. Map of tag template field IDs to the settings for the field.

0 commit comments

Comments
 (0)