Skip to content

Commit 83e5198

Browse files
Google APIscopybara-github
Google APIs
authored and
copybara-github
committed
feat: add HDFS configuration
feat: add GCS Managed Folders feat: add S3 Managed Private Network feat: add S3 Cloudfront Domain PiperOrigin-RevId: 662684810
1 parent e813a85 commit 83e5198

2 files changed

Lines changed: 81 additions & 49 deletions

File tree

‎google/storagetransfer/v1/storagetransfer_v1.yaml‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -50,13 +50,6 @@ documentation:
5050
Lists transfer operations. Operations are ordered by their creation
5151
time in reverse chronological order.
5252
53-
backend:
54-
rules:
55-
- selector: 'google.longrunning.Operations.*'
56-
deadline: 30.0
57-
- selector: 'google.storagetransfer.v1.StorageTransferService.*'
58-
deadline: 30.0
59-
6053
http:
6154
rules:
6255
- selector: google.longrunning.Operations.CancelOperation

‎google/storagetransfer/v1/transfer_types.proto‎

Lines changed: 81 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,21 @@ message GcsData {
205205
// The root path value must meet
206206
// [Object Name Requirements](/storage/docs/naming#objectnames).
207207
string path = 3;
208+
209+
// Preview. Enables the transfer of managed folders between Cloud Storage
210+
// buckets. Set this option on the gcs_data_source.
211+
//
212+
// If set to true:
213+
//
214+
// - Managed folders in the source bucket are transferred to the
215+
// destination bucket.
216+
// - Managed folders in the destination bucket are overwritten. Other
217+
// OVERWRITE options are not supported.
218+
//
219+
// See
220+
// [Transfer Cloud Storage managed
221+
// folders](/storage-transfer/docs/managed-folders).
222+
bool managed_folder_transfer_enabled = 4;
208223
}
209224

210225
// An AwsS3Data resource can be a data source, but not a data sink.
@@ -242,26 +257,46 @@ message AwsS3Data {
242257
// this project.
243258
string role_arn = 4;
244259

260+
// Optional. The CloudFront distribution domain name pointing to this bucket,
261+
// to use when fetching.
262+
//
263+
// See
264+
// [Transfer from S3 via
265+
// CloudFront](https://cloud.google.com/storage-transfer/docs/s3-cloudfront)
266+
// for more information.
267+
//
268+
// Format: `https://{id}.cloudfront.net` or any valid custom domain. Must
269+
// begin with `https://`.
270+
string cloudfront_domain = 6 [(google.api.field_behavior) = OPTIONAL];
271+
245272
// Optional. The Resource name of a secret in Secret Manager.
246273
//
247-
// The Azure SAS token must be stored in Secret Manager in JSON format:
248-
// <pre>{
249-
// "sas_token" : "<var>SAS_TOKEN</var>"
250-
// }</pre>
274+
// AWS credentials must be stored in Secret Manager in JSON format:
275+
//
276+
// {
277+
// "access_key_id": "ACCESS_KEY_ID",
278+
// "secret_access_key": "SECRET_ACCESS_KEY"
279+
// }
280+
//
251281
// [GoogleServiceAccount][google.storagetransfer.v1.GoogleServiceAccount] must
252282
// be granted `roles/secretmanager.secretAccessor` for the resource.
253283
//
254-
// See [Configure access to a source: Microsoft Azure Blob Storage]
255-
// (https://cloud.google.com/storage-transfer/docs/source-microsoft-azure#secret_manager)
284+
// See [Configure access to a source: Amazon S3]
285+
// (https://cloud.google.com/storage-transfer/docs/source-amazon-s3#secret_manager)
256286
// for more information.
257287
//
258-
// If `credentials_secret` is specified, do not specify [azure_credentials][].
259-
//
260-
// This feature is in
261-
// [preview](https://cloud.google.com/terms/service-terms#1).
288+
// If `credentials_secret` is specified, do not specify
289+
// [role_arn][google.storagetransfer.v1.AwsS3Data.role_arn] or
290+
// [aws_access_key][google.storagetransfer.v1.AwsS3Data.aws_access_key].
262291
//
263292
// Format: `projects/{project_number}/secrets/{secret_name}`
264293
string credentials_secret = 7 [(google.api.field_behavior) = OPTIONAL];
294+
295+
oneof private_network {
296+
// Egress bytes over a Google-managed private network.
297+
// This network is shared between other users of Storage Transfer Service.
298+
bool managed_private_network = 8;
299+
}
265300
}
266301

267302
// An AzureBlobStorageData resource can be a data source, but not a data sink.
@@ -298,9 +333,11 @@ message AzureBlobStorageData {
298333
// Optional. The Resource name of a secret in Secret Manager.
299334
//
300335
// The Azure SAS token must be stored in Secret Manager in JSON format:
301-
// <pre>{
302-
// "sas_token" : "<var>SAS_TOKEN</var>"
303-
// }</pre>
336+
//
337+
// {
338+
// "sas_token" : "SAS_TOKEN"
339+
// }
340+
//
304341
// [GoogleServiceAccount][google.storagetransfer.v1.GoogleServiceAccount] must
305342
// be granted `roles/secretmanager.secretAccessor` for the resource.
306343
//
@@ -311,9 +348,6 @@ message AzureBlobStorageData {
311348
// If `credentials_secret` is specified, do not specify
312349
// [azure_credentials][google.storagetransfer.v1.AzureBlobStorageData.azure_credentials].
313350
//
314-
// This feature is in
315-
// [preview](https://cloud.google.com/terms/service-terms#1).
316-
//
317351
// Format: `projects/{project_number}/secrets/{secret_name}`
318352
string credentials_secret = 7 [(google.api.field_behavior) = OPTIONAL];
319353
}
@@ -371,6 +405,15 @@ message PosixFilesystem {
371405
string root_directory = 1;
372406
}
373407

408+
// An HdfsData resource specifies a path within an HDFS entity (e.g. a cluster).
409+
// All cluster-specific settings, such as namenodes and ports, are configured on
410+
// the transfer agents servicing requests, so HdfsData only contains the root
411+
// path to the data in our transfer.
412+
message HdfsData {
413+
// Root path to transfer files.
414+
string path = 1;
415+
}
416+
374417
// An AwsS3CompatibleData resource.
375418
message AwsS3CompatibleData {
376419
// Required. Specifies the name of the bucket.
@@ -473,7 +516,7 @@ message S3CompatibleMetadata {
473516
ListApi list_api = 4;
474517
}
475518

476-
// Represents an On-Premises Agent pool.
519+
// Represents an agent pool.
477520
message AgentPool {
478521
option (google.api.resource) = {
479522
type: "storagetransfer.googleapis.com/agentPools"
@@ -485,8 +528,8 @@ message AgentPool {
485528
// Default value. This value is unused.
486529
STATE_UNSPECIFIED = 0;
487530

488-
// This is an initialization state. During this stage, the resources such as
489-
// Pub/Sub topics are allocated for the AgentPool.
531+
// This is an initialization state. During this stage, resources are
532+
// allocated for the AgentPool.
490533
CREATING = 1;
491534

492535
// Determines that the AgentPool is created for use. At this state, Agents
@@ -602,6 +645,9 @@ message TransferSpec {
602645

603646
// An AWS S3 compatible data source.
604647
AwsS3CompatibleData aws_s3_compatible_data_source = 19;
648+
649+
// An HDFS cluster data source.
650+
HdfsData hdfs_data_source = 20;
605651
}
606652

607653
oneof intermediate_data_location {
@@ -768,10 +814,10 @@ message MetadataOptions {
768814
// Do not preserve the `timeCreated` metadata from the source object.
769815
TIME_CREATED_SKIP = 1;
770816

771-
// Preserves the source object's `timeCreated` metadata in the `customTime`
772-
// field in the destination object. Note that any value stored in the
773-
// source object's `customTime` field will not be propagated to the
774-
// destination object.
817+
// Preserves the source object's `timeCreated` or `lastModified` metadata in
818+
// the `customTime` field in the destination object. Note that any value
819+
// stored in the source object's `customTime` field will not be propagated
820+
// to the destination object.
775821
TIME_CREATED_PRESERVE_AS_CUSTOM_TIME = 2;
776822
}
777823

@@ -818,9 +864,10 @@ message MetadataOptions {
818864
KmsKey kms_key = 8;
819865

820866
// Specifies how each object's `timeCreated` metadata is preserved for
821-
// transfers between Google Cloud Storage buckets. If unspecified, the
822-
// default behavior is the same as
867+
// transfers. If unspecified, the default behavior is the same as
823868
// [TIME_CREATED_SKIP][google.storagetransfer.v1.MetadataOptions.TimeCreated.TIME_CREATED_SKIP].
869+
// This behavior is supported for transfers to Cloud Storage buckets from
870+
// Cloud Storage, Amazon S3, S3-compatible storage, and Azure sources.
824871
TimeCreated time_created = 9;
825872
}
826873

@@ -986,8 +1033,7 @@ message TransferJob {
9861033
// Transfer specification.
9871034
TransferSpec transfer_spec = 4;
9881035

989-
// Notification configuration. This is not supported for transfers involving
990-
// PosixFilesystem.
1036+
// Notification configuration.
9911037
NotificationConfig notification_config = 11;
9921038

9931039
// Logging configuration.
@@ -1219,16 +1265,10 @@ message NotificationConfig {
12191265

12201266
// Specifies the logging behavior for transfer operations.
12211267
//
1222-
// For cloud-to-cloud transfers, logs are sent to Cloud Logging. See
1268+
// Logs can be sent to Cloud Logging for all transfer types. See
12231269
// [Read transfer
12241270
// logs](https://cloud.google.com/storage-transfer/docs/read-transfer-logs) for
12251271
// details.
1226-
//
1227-
// For transfers to or from a POSIX file system, logs are stored in the
1228-
// Cloud Storage bucket that is the source or sink of the transfer.
1229-
// See [Managing Transfer for on-premises jobs]
1230-
// (https://cloud.google.com/storage-transfer/docs/managing-on-prem-jobs#viewing-logs)
1231-
// for details.
12321272
message LoggingConfig {
12331273
// Loggable actions.
12341274
enum LoggableAction {
@@ -1260,19 +1300,18 @@ message LoggingConfig {
12601300
}
12611301

12621302
// Specifies the actions to be logged. If empty, no logs are generated.
1263-
// Not supported for transfers with PosixFilesystem data sources; use
1264-
// [enable_onprem_gcs_transfer_logs][google.storagetransfer.v1.LoggingConfig.enable_onprem_gcs_transfer_logs]
1265-
// instead.
12661303
repeated LoggableAction log_actions = 1;
12671304

12681305
// States in which `log_actions` are logged. If empty, no logs are generated.
1269-
// Not supported for transfers with PosixFilesystem data sources; use
1270-
// [enable_onprem_gcs_transfer_logs][google.storagetransfer.v1.LoggingConfig.enable_onprem_gcs_transfer_logs]
1271-
// instead.
12721306
repeated LoggableActionState log_action_states = 2;
12731307

1274-
// For transfers with a PosixFilesystem source, this option enables the Cloud
1275-
// Storage transfer logs for this transfer.
1308+
// For PosixFilesystem transfers, enables
1309+
// [file system transfer
1310+
// logs](https://cloud.google.com/storage-transfer/docs/on-prem-transfer-log-format)
1311+
// instead of, or in addition to, Cloud Logging.
1312+
//
1313+
// This option ignores [LoggableAction] and [LoggableActionState]. If these
1314+
// are set, Cloud Logging will also be enabled for this transfer.
12761315
bool enable_onprem_gcs_transfer_logs = 3;
12771316
}
12781317

0 commit comments

Comments
 (0)