feat:Enable Vertex AI Ingestion on DataPlex

Google APIs · copybara-github · commit f878578ebea3 · 2023-09-26T12:54:40.000-07:00
PiperOrigin-RevId: 568623212
diff --git a/google/cloud/datacatalog/v1/common.proto b/google/cloud/datacatalog/v1/common.proto
@@ -63,6 +63,9 @@ enum IntegratedSystem {
 
   // Looker
   LOOKER = 9;
+
+  // Vertex AI
+  VERTEX_AI = 10;
 }
 
 // This enum describes all the systems that manage
diff --git a/google/cloud/datacatalog/v1/datacatalog.proto b/google/cloud/datacatalog/v1/datacatalog.proto
@@ -1172,12 +1172,18 @@ message Entry {
     // only for entries with the `ROUTINE` type.
     RoutineSpec routine_spec = 28;
 
+    // Specification that applies to a dataset.
+    DatasetSpec dataset_spec = 32;
+
     // Specification that applies to a fileset resource. Valid only
     // for entries with the `FILESET` type.
     FilesetSpec fileset_spec = 33;
 
     // Specification that applies to a Service resource.
     ServiceSpec service_spec = 42;
+
+    // Model specification.
+    ModelSpec model_spec = 43;
   }
 
   // Display name of an entry.
@@ -1372,6 +1378,16 @@ message RoutineSpec {
   }
 }
 
+// Specification that applies to a dataset. Valid only for
+// entries with the `DATASET` type.
+message DatasetSpec {
+  // Fields specific to the source system.
+  oneof system_spec {
+    // Vertex AI Dataset specific fields
+    VertexDatasetSpec vertex_dataset_spec = 2;
+  }
+}
+
 // Specification that applies to
 // entries that are part `SQL_DATABASE` system
 // (user_specified_type)
@@ -1468,6 +1484,121 @@ message ServiceSpec {
   }
 }
 
+// Detail description of the source information of a Vertex model.
+message VertexModelSourceInfo {
+  // Source of the model.
+  enum ModelSourceType {
+    // Should not be used.
+    MODEL_SOURCE_TYPE_UNSPECIFIED = 0;
+
+    // The Model is uploaded by automl training pipeline.
+    AUTOML = 1;
+
+    // The Model is uploaded by user or custom training pipeline.
+    CUSTOM = 2;
+
+    // The Model is registered and sync'ed from BigQuery ML.
+    BQML = 3;
+
+    // The Model is saved or tuned from Model Garden.
+    MODEL_GARDEN = 4;
+  }
+
+  // Type of the model source.
+  ModelSourceType source_type = 1;
+
+  // If this Model is copy of another Model. If true then
+  // [source_type][google.cloud.datacatalog.v1.VertexModelSourceInfo.source_type]
+  // pertains to the original.
+  bool copy = 2;
+}
+
+// Specification for vertex model resources.
+message VertexModelSpec {
+  // The version ID of the model.
+  string version_id = 1;
+
+  // User provided version aliases so that a model version can be referenced via
+  // alias
+  repeated string version_aliases = 2;
+
+  // The description of this version.
+  string version_description = 3;
+
+  // Source of a Vertex model.
+  VertexModelSourceInfo vertex_model_source_info = 4;
+
+  // URI of the Docker image to be used as the custom container for serving
+  // predictions.
+  string container_image_uri = 5;
+}
+
+// Specification for vertex dataset resources.
+message VertexDatasetSpec {
+  // Type of data stored in the dataset.
+  enum DataType {
+    // Should not be used.
+    DATA_TYPE_UNSPECIFIED = 0;
+
+    // Structured data dataset.
+    TABLE = 1;
+
+    // Image dataset which supports ImageClassification, ImageObjectDetection
+    // and ImageSegmentation problems.
+    IMAGE = 2;
+
+    // Document dataset which supports TextClassification, TextExtraction and
+    // TextSentiment problems.
+    TEXT = 3;
+
+    // Video dataset which supports VideoClassification, VideoObjectTracking and
+    // VideoActionRecognition problems.
+    VIDEO = 4;
+
+    // Conversation dataset which supports conversation problems.
+    CONVERSATION = 5;
+
+    // TimeSeries dataset.
+    TIME_SERIES = 6;
+
+    // Document dataset which supports DocumentAnnotation problems.
+    DOCUMENT = 7;
+
+    // TextToSpeech dataset which supports TextToSpeech problems.
+    TEXT_TO_SPEECH = 8;
+
+    // Translation dataset which supports Translation problems.
+    TRANSLATION = 9;
+
+    // Store Vision dataset which is used for HITL integration.
+    STORE_VISION = 10;
+
+    // Enterprise Knowledge Graph dataset which is used for HITL labeling
+    // integration.
+    ENTERPRISE_KNOWLEDGE_GRAPH = 11;
+
+    // Text prompt dataset which supports Large Language Models.
+    TEXT_PROMPT = 12;
+  }
+
+  // The number of DataItems in this Dataset. Only apply for non-structured
+  // Dataset.
+  int64 data_item_count = 1;
+
+  // Type of the dataset.
+  DataType data_type = 2;
+}
+
+// Specification that applies to a model. Valid only for
+// entries with the `MODEL` type.
+message ModelSpec {
+  // System spec
+  oneof system_spec {
+    // Specification for vertex model resources.
+    VertexModelSpec vertex_model_spec = 1;
+  }
+}
+
 // Business Context of the entry.
 message BusinessContext {
   // Entry overview fields for rich text descriptions of entries.

Original file line number	Diff line number	Diff line change
`@@ -63,6 +63,9 @@ enum IntegratedSystem {`
`63`	`63`
`64`	`64`	`// Looker`
`65`	`65`	`LOOKER = 9;`
	`66`	`+`
	`67`	`+ // Vertex AI`
	`68`	`+ VERTEX_AI = 10;`
`66`	`69`	`}`
`67`	`70`
`68`	`71`	`// This enum describes all the systems that manage`