Skip to content

Commit b4da4fd

Browse files
Google APIscopybara-github
Google APIs
authored and
copybara-github
committed
feat: add BigQuery Storage Write API v1
Committer: @yirutang PiperOrigin-RevId: 397350004
1 parent ddf396f commit b4da4fd

7 files changed

Lines changed: 730 additions & 53 deletions

File tree

‎google/cloud/bigquery/storage/v1/BUILD.bazel‎

Lines changed: 56 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -12,15 +12,21 @@ proto_library(
1212
srcs = [
1313
"arrow.proto",
1414
"avro.proto",
15+
"protobuf.proto",
1516
"storage.proto",
1617
"stream.proto",
18+
"table.proto",
1719
],
1820
deps = [
1921
"//google/api:annotations_proto",
2022
"//google/api:client_proto",
2123
"//google/api:field_behavior_proto",
2224
"//google/api:resource_proto",
25+
"//google/rpc:status_proto",
26+
"@com_google_protobuf//:descriptor_proto",
27+
"@com_google_protobuf//:empty_proto",
2328
"@com_google_protobuf//:timestamp_proto",
29+
"@com_google_protobuf//:wrappers_proto",
2430
],
2531
)
2632

@@ -105,6 +111,7 @@ go_proto_library(
105111
protos = [":storage_proto"],
106112
deps = [
107113
"//google/api:annotations_go_proto",
114+
"//google/rpc:status_go_proto",
108115
],
109116
)
110117

@@ -236,53 +243,55 @@ nodejs_gapic_assembly_pkg(
236243
##############################################################################
237244
# Ruby
238245
##############################################################################
239-
load(
240-
"@com_google_googleapis_imports//:imports.bzl",
241-
"ruby_cloud_gapic_library",
242-
"ruby_gapic_assembly_pkg",
243-
"ruby_grpc_library",
244-
"ruby_proto_library",
245-
)
246-
247-
ruby_proto_library(
248-
name = "storage_ruby_proto",
249-
deps = [":storage_proto"],
250-
)
251-
252-
ruby_grpc_library(
253-
name = "storage_ruby_grpc",
254-
srcs = [":storage_proto"],
255-
deps = [":storage_ruby_proto"],
256-
)
257-
258-
ruby_cloud_gapic_library(
259-
name = "storage_ruby_gapic",
260-
srcs = [":storage_proto_with_info"],
261-
extra_protoc_parameters = [
262-
"ruby-cloud-gem-name=google-cloud-bigquery-storage-v1",
263-
"ruby-cloud-env-prefix=BIGQUERY_STORAGE",
264-
"ruby-cloud-product-url=https://cloud.google.com/bigquery/docs/reference/storage",
265-
"ruby-cloud-api-id=bigquerystorage.googleapis.com",
266-
"ruby-cloud-api-shortname=bigquerystorage",
267-
],
268-
grpc_service_config = "bigquerystorage_grpc_service_config.json",
269-
ruby_cloud_description = "The BigQuery Storage API provides fast access to BigQuery managed storage.",
270-
ruby_cloud_title = "BigQuery Storage V1",
271-
deps = [
272-
":storage_ruby_grpc",
273-
":storage_ruby_proto",
274-
],
275-
)
276-
277-
# Open Source Packages
278-
ruby_gapic_assembly_pkg(
279-
name = "google-cloud-bigquery-storage-v1-ruby",
280-
deps = [
281-
":storage_ruby_gapic",
282-
":storage_ruby_grpc",
283-
":storage_ruby_proto",
284-
],
285-
)
246+
# disabled due to https://github.com/protocolbuffers/protobuf/issues/7992
247+
#
248+
# load(
249+
# "@com_google_googleapis_imports//:imports.bzl",
250+
# "ruby_cloud_gapic_library",
251+
# "ruby_gapic_assembly_pkg",
252+
# "ruby_grpc_library",
253+
# "ruby_proto_library",
254+
# )
255+
256+
# ruby_proto_library(
257+
# name = "storage_ruby_proto",
258+
# deps = [":storage_proto"],
259+
# )
260+
261+
# ruby_grpc_library(
262+
# name = "storage_ruby_grpc",
263+
# srcs = [":storage_proto"],
264+
# deps = [":storage_ruby_proto"],
265+
# )
266+
267+
# ruby_cloud_gapic_library(
268+
# name = "storage_ruby_gapic",
269+
# srcs = [":storage_proto_with_info"],
270+
# extra_protoc_parameters = [
271+
# "ruby-cloud-gem-name=google-cloud-bigquery-storage-v1",
272+
# "ruby-cloud-env-prefix=BIGQUERY_STORAGE",
273+
# "ruby-cloud-product-url=https://cloud.google.com/bigquery/docs/reference/storage",
274+
# "ruby-cloud-api-id=bigquerystorage.googleapis.com",
275+
# "ruby-cloud-api-shortname=bigquerystorage",
276+
# ],
277+
# grpc_service_config = "bigquerystorage_grpc_service_config.json",
278+
# ruby_cloud_description = "The BigQuery Storage API provides fast access to BigQuery managed storage.",
279+
# ruby_cloud_title = "BigQuery Storage V1",
280+
# deps = [
281+
# ":storage_ruby_grpc",
282+
# ":storage_ruby_proto",
283+
# ],
284+
# )
285+
286+
# # Open Source Packages
287+
# ruby_gapic_assembly_pkg(
288+
# name = "google-cloud-bigquery-storage-v1-ruby",
289+
# deps = [
290+
# ":storage_ruby_gapic",
291+
# ":storage_ruby_grpc",
292+
# ":storage_ruby_proto",
293+
# ],
294+
# )
286295

287296
##############################################################################
288297
# C#

‎google/cloud/bigquery/storage/v1/bigquerystorage_grpc_service_config.json‎

Lines changed: 35 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,12 +34,27 @@
3434
"UNAVAILABLE"
3535
]
3636
}
37-
},
38-
{
37+
}, {
3938
"name": [
4039
{
41-
"service": "google.cloud.bigquery.storage.v1.BigQueryRead",
42-
"method": "SplitReadStream"
40+
"service": "google.cloud.bigquery.storage.v1.BigQueryWrite",
41+
"method": "CreateWriteStream"
42+
},
43+
{
44+
"service": "google.cloud.bigquery.storage.v1.BigQueryWrite",
45+
"method": "BatchCommitWriteStreams"
46+
},
47+
{
48+
"service": "google.cloud.bigquery.storage.v1.BigQueryWrite",
49+
"method": "FinalizeWriteStream"
50+
},
51+
{
52+
"service": "google.cloud.bigquery.storage.v1.BigQueryWrite",
53+
"method": "GetWriteStream"
54+
},
55+
{
56+
"service": "google.cloud.bigquery.storage.v1.BigQueryWrite",
57+
"method": "FlushRows"
4358
}
4459
],
4560
"timeout": "600s",
@@ -52,6 +67,22 @@
5267
"UNAVAILABLE"
5368
]
5469
}
70+
}, {
71+
"name": [
72+
{
73+
"service": "google.cloud.bigquery.storage.v1.BigQueryWrite",
74+
"method": "AppendRows"
75+
}
76+
],
77+
"timeout": "86400s",
78+
"retryPolicy": {
79+
"initialBackoff": "0.100s",
80+
"maxBackoff": "60s",
81+
"backoffMultiplier": 1.3,
82+
"retryableStatusCodes": [
83+
"UNAVAILABLE"
84+
]
85+
}
5586
}
5687
]
5788
}

‎google/cloud/bigquery/storage/v1/bigquerystorage_v1.yaml‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ title: BigQuery Storage API
55

66
apis:
77
- name: google.cloud.bigquery.storage.v1.BigQueryRead
8+
- name: google.cloud.bigquery.storage.v1.BigQueryWrite
89

910
backend:
1011
rules:
@@ -14,12 +15,29 @@ backend:
1415
deadline: 21600.0
1516
- selector: google.cloud.bigquery.storage.v1.BigQueryRead.SplitReadStream
1617
deadline: 120.0
18+
- selector: google.cloud.bigquery.storage.v1.BigQueryWrite.CreateWriteStram
19+
deadline: 120.0
20+
- selector: google.cloud.bigquery.storage.v1.BigQueryWrite.FinalizeWriteStream
21+
deadline: 120.0
22+
- selector: google.cloud.bigquery.storage.v1.BigQueryWrite.BatchCommitWriteStreams
23+
deadline: 120.0
24+
- selector: google.cloud.bigquery.storage.v1.BigQueryWrite.GetWriteStream
25+
deadline: 120.0
26+
- selector: google.cloud.bigquery.storage.v1.BigQueryWrite.FlushRows
27+
deadline: 120.0
28+
- selector: google.cloud.bigquery.storage.v1.BigQueryWrite.AppendRows
29+
deadline: 86400.0
1730

1831
authentication:
1932
rules:
2033
- selector: 'google.cloud.bigquery.storage.v1.BigQueryRead.*'
2134
oauth:
2235
canonical_scopes: |-
2336
https://www.googleapis.com/auth/bigquery,
24-
https://www.googleapis.com/auth/bigquery.readonly,
37+
https://www.googleapis.com/auth/cloud-platform
38+
- selector: 'google.cloud.bigquery.storage.v1.BigQueryWrite.*'
39+
oauth:
40+
canonical_scopes: |-
41+
https://www.googleapis.com/auth/bigquery,
42+
https://www.googleapis.com/auth/bigquery.insertdata,
2543
https://www.googleapis.com/auth/cloud-platform

‎google/cloud/bigquery/storage/v1/protobuf.proto‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
// Copyright 2021 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
syntax = "proto3";
16+
17+
package google.cloud.bigquery.storage.v1;
18+
19+
import "google/protobuf/descriptor.proto";
20+
21+
option csharp_namespace = "Google.Cloud.BigQuery.Storage.V1";
22+
option go_package = "google.golang.org/genproto/googleapis/cloud/bigquery/storage/v1;storage";
23+
option java_multiple_files = true;
24+
option java_outer_classname = "ProtoBufProto";
25+
option java_package = "com.google.cloud.bigquery.storage.v1";
26+
option php_namespace = "Google\\Cloud\\BigQuery\\Storage\\V1";
27+
28+
// ProtoSchema describes the schema of the serialized protocol buffer data rows.
29+
message ProtoSchema {
30+
// Descriptor for input message. The provided descriptor must be self
31+
// contained, such that data rows sent can be fully decoded using only the
32+
// single descriptor. For data rows that are compositions of multiple
33+
// independent messages, this means the descriptor may need to be transformed
34+
// to only use nested types:
35+
// https://developers.google.com/protocol-buffers/docs/proto#nested
36+
//
37+
// For additional information for how proto types and values map onto BigQuery
38+
// see: https://cloud.google.com/bigquery/docs/write-api#data_type_conversions
39+
google.protobuf.DescriptorProto proto_descriptor = 1;
40+
}
41+
42+
message ProtoRows {
43+
// A sequence of rows serialized as a Protocol Buffer.
44+
//
45+
// See https://developers.google.com/protocol-buffers/docs/overview for more
46+
// information on deserializing this field.
47+
repeated bytes serialized_rows = 1;
48+
}

0 commit comments

Comments
 (0)