feat: Added DatabaseType field for the type of backing store used

Google APIs · copybara-github · commit 463cacaf4733 · 2022-03-21T12:08:52.000-07:00
feat: Added AVRO DatabaseDumpSpec for importing and exporting Avro files
feat: Added additional endTime field for MetadataImports
feat: Added RESTORING status on Backups
feat: Added support to record the services that are restoring the backup
feat: Added configuration for Dataplex integration
feat: Added support for IAM management for metadata resources
feat: Added AuxiliaryVersionConfig for configuring the auxiliary hive versions during creation or update of the DPMS instance
feat: Added NetworkConfig for exposing the DPMS endpoint in multiple subnetworks using PSC (this skips the need for VPC peering)
feat: Added EncryptionConfig which contains information used to configure the Dataproc Metastore service to encrypt customer data at rest (CMEK)
docs: formatting improvements

PiperOrigin-RevId: 436271267
diff --git a/google/cloud/metastore/v1alpha/BUILD.bazel b/google/cloud/metastore/v1alpha/BUILD.bazel
@@ -72,11 +72,13 @@ java_gapic_library(
     srcs = [":metastore_proto_with_info"],
     gapic_yaml = "metastore_gapic.yaml",
     grpc_service_config = "metastore_grpc_service_config.json",
+    service_yaml = "metastore_v1alpha.yaml",
     test_deps = [
         ":metastore_java_grpc",
     ],
     deps = [
         ":metastore_java_proto",
+        "//google/api:api_java_proto",
     ],
 )
 
@@ -203,7 +205,6 @@ php_grpc_library(
 php_gapic_library(
     name = "metastore_php_gapic",
     srcs = [":metastore_proto_with_info"],
-    gapic_yaml = "metastore_gapic.yaml",
     grpc_service_config = "metastore_grpc_service_config.json",
     service_yaml = "metastore_v1alpha.yaml",
     deps = [
@@ -340,4 +341,20 @@ csharp_gapic_assembly_pkg(
 ##############################################################################
 # C++
 ##############################################################################
-# Put your C++ rules here
+load(
+    "@com_google_googleapis_imports//:imports.bzl",
+    "cc_grpc_library",
+    "cc_proto_library",
+)
+
+cc_proto_library(
+    name = "metastore_cc_proto",
+    deps = [":metastore_proto"],
+)
+
+cc_grpc_library(
+    name = "metastore_cc_grpc",
+    srcs = [":metastore_proto"],
+    grpc_only = True,
+    deps = [":metastore_cc_proto"],
+)
diff --git a/google/cloud/metastore/v1alpha/metastore.proto b/google/cloud/metastore/v1alpha/metastore.proto
@@ -1,4 +1,4 @@
-// Copyright 2021 Google LLC
+// Copyright 2022 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -30,14 +30,23 @@ option go_package = "google.golang.org/genproto/googleapis/cloud/metastore/v1alp
 option java_multiple_files = true;
 option java_outer_classname = "MetastoreProto";
 option java_package = "com.google.cloud.metastore.v1alpha";
+option php_namespace = "Google\\Cloud\\Metastore\\V1alpha";
 option (google.api.resource_definition) = {
   type: "compute.googleapis.com/Network"
   pattern: "projects/{project}/global/networks/{network}"
 };
+option (google.api.resource_definition) = {
+  type: "compute.googleapis.com/Subnetwork"
+  pattern: "projects/{project}/regions/{region}/subnetworks/{subnetwork}"
+};
+option (google.api.resource_definition) = {
+  type: "dataplex.googleapis.com/Lake"
+  pattern: "projects/{project}/locations/{location}/lakes/{lake}"
+};
 
 // Configures and manages metastore services.
-// Metastore services are fully managed, highly available, auto-scaled,
-// auto-healing, OSS-native deployments of technical metadata management
+// Metastore services are fully managed, highly available, autoscaled,
+// autohealing, OSS-native deployments of technical metadata management
 // software. Each metastore service exposes a network endpoint through which
 // metadata queries are served. Metadata queries can originate from a variety
 // of sources, including Apache Hive, Apache Presto, and Apache Spark.
@@ -194,7 +203,7 @@ service DataprocMetastore {
     option (google.api.method_signature) = "name";
   }
 
-  // Creates a new Backup in a given project and location.
+  // Creates a new backup in a given project and location.
   rpc CreateBackup(CreateBackupRequest) returns (google.longrunning.Operation) {
     option (google.api.http) = {
       post: "/v1alpha/{parent=projects/*/locations/*/services/*}/backups"
@@ -288,6 +297,18 @@ message Service {
     STABLE = 2;
   }
 
+  // The backend database type for the metastore service.
+  enum DatabaseType {
+    // The DATABASE_TYPE is not set.
+    DATABASE_TYPE_UNSPECIFIED = 0;
+
+    // MySQL is used to persist the metastore data.
+    MYSQL = 1;
+
+    // Spanner is used to persist the metastore data.
+    SPANNER = 2;
+  }
+
   // Configuration properties specific to the underlying metastore service
   // technology (the software that serves metastore queries).
   oneof metastore_config {
@@ -347,6 +368,8 @@ message Service {
 
   // The one hour maintenance window of the metastore service. This specifies
   // when the service can be restarted for maintenance purposes in UTC time.
+  // Maintenance window is not needed for services with the SPANNER
+  // database type.
   MaintenanceWindow maintenance_window = 15;
 
   // Output only. The globally unique resource identifier of the metastore service.
@@ -358,12 +381,26 @@ message Service {
   // Immutable. The release channel of the service.
   // If unspecified, defaults to `STABLE`.
   ReleaseChannel release_channel = 19 [(google.api.field_behavior) = IMMUTABLE];
+
+  // Immutable. Information used to configure the Dataproc Metastore service to encrypt
+  // customer data at rest. Cannot be updated.
+  EncryptionConfig encryption_config = 20 [(google.api.field_behavior) = IMMUTABLE];
+
+  // Immutable. The configuration specifying the network settings for the
+  // Dataproc Metastore service.
+  NetworkConfig network_config = 21 [(google.api.field_behavior) = IMMUTABLE];
+
+  // Immutable. The database type that the Metastore service stores its data.
+  DatabaseType database_type = 22 [(google.api.field_behavior) = IMMUTABLE];
 }
 
 // Specifies how metastore metadata should be integrated with external services.
 message MetadataIntegration {
   // The integration config for the Data Catalog service.
   DataCatalogConfig data_catalog_config = 1;
+
+  // The integration config for the Dataplex service.
+  DataplexConfig dataplex_config = 2;
 }
 
 // Specifies how metastore metadata should be integrated with the Data Catalog
@@ -374,6 +411,25 @@ message DataCatalogConfig {
   bool enabled = 2;
 }
 
+// Specifies how metastore metadata should be integrated with the Dataplex
+// service.
+message DataplexConfig {
+  // A reference to the Lake resources that this metastore service is attached
+  // to. The key is the lake resource name. Example:
+  // `projects/{project_number}/locations/{location_id}/lakes/{lake_id}`.
+  map<string, Lake> lake_resources = 1;
+}
+
+// Represents a Lake resource
+message Lake {
+  // The Lake resource name.
+  // Example:
+  // `projects/{project_number}/locations/{location_id}/lakes/{lake_id}`
+  string name = 1 [(google.api.resource_reference) = {
+                     type: "dataplex.googleapis.com/Lake"
+                   }];
+}
+
 // Maintenance window. This specifies when Dataproc Metastore
 // may perform system maintenance operation to the service.
 message MaintenanceWindow {
@@ -387,12 +443,26 @@ message MaintenanceWindow {
 // Specifies configuration information specific to running Hive metastore
 // software as the metastore service.
 message HiveMetastoreConfig {
+  // Protocols available for serving the metastore service endpoint.
+  enum EndpointProtocol {
+    // The protocol is not set.
+    ENDPOINT_PROTOCOL_UNSPECIFIED = 0;
+
+    // Use the legacy Apache Thrift protocol for the metastore service endpoint.
+    THRIFT = 1;
+
+    // Use the modernized gRPC protocol for the metastore service endpoint.
+    GRPC = 2;
+  }
+
   // Immutable. The Hive metastore schema version.
   string version = 1 [(google.api.field_behavior) = IMMUTABLE];
 
   // A mapping of Hive metastore configuration key-value pairs to apply to the
   // Hive metastore (configured in `hive-site.xml`). The mappings
-  // override system defaults (some keys cannot be overridden).
+  // override system defaults (some keys cannot be overridden). These
+  // overrides are also applied to auxiliary versions and can be further
+  // customized in the auxiliary version's `AuxiliaryVersionConfig`.
   map<string, string> config_overrides = 2;
 
   // Information used to configure the Hive metastore service as a service
@@ -401,6 +471,20 @@ message HiveMetastoreConfig {
   // (`hive_metastore_config.kerberos_config`) in the request's `update_mask`
   // while omitting this field from the request's `service`.
   KerberosConfig kerberos_config = 3;
+
+  // The protocol to use for the metastore service endpoint. If unspecified,
+  // defaults to `THRIFT`.
+  EndpointProtocol endpoint_protocol = 4;
+
+  // A mapping of Hive metastore version to the auxiliary version
+  // configuration. When specified, a secondary Hive metastore service is
+  // created along with the primary service. All auxiliary versions must be less
+  // than the service's primary version. The key is the auxiliary service name
+  // and it must match the regular expression [a-z]([-a-z0-9]*[a-z0-9])?. This
+  // means that the first character must be a lowercase letter, and all the
+  // following characters must be hyphens, lowercase letters, or digits, except
+  // the last character, which cannot be a hyphen.
+  map<string, AuxiliaryVersionConfig> auxiliary_versions = 5;
 }
 
 // Configuration information for a Kerberos principal.
@@ -415,7 +499,7 @@ message KerberosConfig {
   string principal = 2;
 
   // A Cloud Storage URI that specifies the path to a
-  // krb5.conf file. It is of the form gs://{bucket_name}/path/to/krb5.conf,
+  // krb5.conf file. It is of the form `gs://{bucket_name}/path/to/krb5.conf`,
   // although the file does not need to be named krb5.conf explicitly.
   string krb5_config_gcs_uri = 3;
 }
@@ -431,6 +515,60 @@ message Secret {
   }
 }
 
+// Encryption settings for the service.
+message EncryptionConfig {
+  // The fully qualified customer provided Cloud KMS key name to use for
+  // customer data encryption, in the following form:
+  //
+  // `projects/{project_number}/locations/{location_id}/keyRings/{key_ring_id}/cryptoKeys/{crypto_key_id}`.
+  string kms_key = 1;
+}
+
+// Configuration information for the auxiliary service versions.
+message AuxiliaryVersionConfig {
+  // The Hive metastore version of the auxiliary service. It must be less
+  // than the primary Hive metastore service's version.
+  string version = 1;
+
+  // A mapping of Hive metastore configuration key-value pairs to apply to the
+  // auxiliary Hive metastore (configured in `hive-site.xml`) in addition to
+  // the primary version's overrides. If keys are present in both the auxiliary
+  // version's overrides and the primary version's overrides, the value from
+  // the auxiliary version's overrides takes precedence.
+  map<string, string> config_overrides = 2;
+
+  // Output only. The network configuration contains the endpoint URI(s) of the auxiliary
+  // Hive metastore service.
+  NetworkConfig network_config = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
+}
+
+// Network configuration for the Dataproc Metastore service.
+message NetworkConfig {
+  // Contains information of the customer's network configurations.
+  message Consumer {
+    oneof vpc_resource {
+      // The subnetwork of the customer project from which an IP address is
+      // reserved and used as the Dataproc Metastore service's
+      // endpoint. It is accessible to hosts in the subnet and to all
+      // hosts in a subnet in the same region and same network. There must
+      // be at least one IP address available in the subnet's primary range. The
+      // subnet is specified in the following form:
+      //
+      // `projects/{project_number}/regions/{region_id}/subnetworks/{subnetwork_id}
+      string subnetwork = 1 [(google.api.resource_reference) = {
+                               type: "compute.googleapis.com/Subnetwork"
+                             }];
+    }
+
+    // Output only. The URI of the endpoint used to access the metastore service.
+    string endpoint_uri = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
+  }
+
+  // Immutable. The consumer-side network configuration for the Dataproc Metastore
+  // instance.
+  repeated Consumer consumers = 1 [(google.api.field_behavior) = IMMUTABLE];
+}
+
 // The metadata management activities of the metastore service.
 message MetadataManagementActivity {
   // Output only. The latest metadata exports of the metastore service.
@@ -506,12 +644,15 @@ message MetadataImport {
   // The description of the metadata import.
   string description = 2;
 
-  // Output only. The time when the metadata import was created.
+  // Output only. The time when the metadata import was started.
   google.protobuf.Timestamp create_time = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
 
   // Output only. The time when the metadata import was last updated.
   google.protobuf.Timestamp update_time = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
 
+  // Output only. The time when the metadata import finished.
+  google.protobuf.Timestamp end_time = 7 [(google.api.field_behavior) = OUTPUT_ONLY];
+
   // Output only. The current state of the metadata import.
   State state = 5 [(google.api.field_behavior) = OUTPUT_ONLY];
 }
@@ -579,6 +720,9 @@ message Backup {
 
     // The backup failed.
     FAILED = 4;
+
+    // The backup is being restored.
+    RESTORING = 5;
   }
 
   // Immutable. The relative resource name of the backup, in the following form:
@@ -600,6 +744,9 @@ message Backup {
 
   // The description of the backup.
   string description = 6;
+
+  // Output only. Services that are restoring from the backup.
+  repeated string restoring_services = 7 [(google.api.field_behavior) = OUTPUT_ONLY];
 }
 
 // The details of a metadata restore operation.
@@ -622,7 +769,7 @@ message Restore {
     CANCELLED = 4;
   }
 
-  // The type of restore.
+  // The type of restore. If unspecified, defaults to `METADATA_ONLY`.
   enum RestoreType {
     // The restore type is unknown.
     RESTORE_TYPE_UNSPECIFIED = 0;
@@ -646,7 +793,7 @@ message Restore {
   // Output only. The relative resource name of the metastore service backup to restore
   // from, in the following form:
   //
-  // `projects/{project_id}/locations/{location_id}/services/{service_id}/backups/{backup_id}`
+  // `projects/{project_id}/locations/{location_id}/services/{service_id}/backups/{backup_id}`.
   string backup = 4 [
     (google.api.field_behavior) = OUTPUT_ONLY,
     (google.api.resource_reference) = {
@@ -894,7 +1041,7 @@ message CreateMetadataImportRequest {
   // Required. The relative resource name of the service in which to create a metastore
   // import, in the following form:
   //
-  // `projects/{project_number}/locations/{location_id}/services/{service_id}`
+  // `projects/{project_number}/locations/{location_id}/services/{service_id}`.
   string parent = 1 [
     (google.api.field_behavior) = REQUIRED,
     (google.api.resource_reference) = {
@@ -1029,7 +1176,7 @@ message CreateBackupRequest {
   // Required. The relative resource name of the service in which to create a backup
   // of the following form:
   //
-  // `projects/{project_number}/locations/{location_id}/services/{service_id}`
+  // `projects/{project_number}/locations/{location_id}/services/{service_id}`.
   string parent = 1 [
     (google.api.field_behavior) = REQUIRED,
     (google.api.resource_reference) = {
@@ -1105,7 +1252,7 @@ message ExportMetadataRequest {
   // Required. The relative resource name of the metastore service to run export, in the
   // following form:
   //
-  // `projects/{project_id}/locations/{location_id}/services/{service_id}`
+  // `projects/{project_id}/locations/{location_id}/services/{service_id}`.
   string service = 1 [
     (google.api.field_behavior) = REQUIRED,
     (google.api.resource_reference) = {
@@ -1136,7 +1283,7 @@ message RestoreServiceRequest {
   // Required. The relative resource name of the metastore service to run restore, in the
   // following form:
   //
-  // `projects/{project_id}/locations/{location_id}/services/{service_id}`
+  // `projects/{project_id}/locations/{location_id}/services/{service_id}`.
   string service = 1 [
     (google.api.field_behavior) = REQUIRED,
     (google.api.resource_reference) = {
@@ -1147,7 +1294,7 @@ message RestoreServiceRequest {
   // Required. The relative resource name of the metastore service backup to restore
   // from, in the following form:
   //
-  // `projects/{project_id}/locations/{location_id}/services/{service_id}/backups/{backup_id}`
+  // `projects/{project_id}/locations/{location_id}/services/{service_id}/backups/{backup_id}`.
   string backup = 2 [
     (google.api.field_behavior) = REQUIRED,
     (google.api.resource_reference) = {
@@ -1227,6 +1374,9 @@ message DatabaseDumpSpec {
 
     // Database dump is a MySQL dump file.
     MYSQL = 1;
+
+    // Database dump contains Avro files.
+    AVRO = 2;
   }
 
 
diff --git a/google/cloud/metastore/v1alpha/metastore_v1alpha.yaml b/google/cloud/metastore/v1alpha/metastore_v1alpha.yaml
