Skip to content

Commit f73cfc7

Browse files
gene-telligentgene-telligent
committed
Implementing BigQuery: LoadJobConfiguration to support schemaUpdateOptions (GitHub Issue #1566)
1 parent e61ca31 commit f73cfc7

8 files changed

Lines changed: 160 additions & 5 deletions

File tree

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/JobInfo.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,22 @@ public enum WriteDisposition {
8888
WRITE_EMPTY
8989
}
9090

91+
/**
92+
* Specifies options relating to allowing the schema of the destination table to be updated as a side effect of the
93+
* load or query job.
94+
*/
95+
public enum SchemaUpdateOption {
96+
/**
97+
* Allow adding a nullable field to the schema.
98+
*/
99+
ALLOW_FIELD_ADDITION,
100+
101+
/**
102+
* Allow relaxing a required field in the original schema to nullable.
103+
*/
104+
ALLOW_FIELD_RELAXATION
105+
}
106+
91107
/**
92108
* A builder for {@code JobInfo} objects.
93109
*/

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadConfiguration.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818

1919
import com.google.cloud.bigquery.JobInfo.CreateDisposition;
2020
import com.google.cloud.bigquery.JobInfo.WriteDisposition;
21+
import com.google.cloud.bigquery.JobInfo.SchemaUpdateOption;
2122

2223
import java.util.List;
2324

@@ -164,6 +165,14 @@ interface Builder {
164165
*/
165166
Builder setProjectionFields(List<String> projectionFields);
166167

168+
/**
169+
* [Experimental] Sets options allowing the schema of the destination table to be updated as a side effect of the
170+
* load job. Schema update options are supported in two cases: when writeDisposition is WRITE_APPEND; when
171+
* writeDisposition is WRITE_TRUNCATE and the destination table is a partition of a table, specified by partition
172+
* decorators. For normal tables, WRITE_TRUNCATE will always overwrite the schema.
173+
*/
174+
Builder setSchemaUpdateOptions(List<SchemaUpdateOption> schemaUpdateOptions);
175+
167176
LoadConfiguration build();
168177
}
169178

@@ -289,6 +298,14 @@ interface Builder {
289298
*/
290299
List<String> getProjectionFields();
291300

301+
/**
302+
* [Experimental] Returns options allowing the schema of the destination table to be updated as a side effect of the
303+
* load job. Schema update options are supported in two cases: when writeDisposition is WRITE_APPEND; when
304+
* writeDisposition is WRITE_TRUNCATE and the destination table is a partition of a table, specified by partition
305+
* decorators. For normal tables, WRITE_TRUNCATE will always overwrite the schema.
306+
*/
307+
List<SchemaUpdateOption> getSchemaUpdateOptions();
308+
292309
/**
293310
* Returns a builder for the load configuration object.
294311
*/

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java‎

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ public final class LoadJobConfiguration extends JobConfiguration implements Load
4444
private final Schema schema;
4545
private final Boolean ignoreUnknownValues;
4646
private final List<String> projectionFields;
47+
private final List<JobInfo.SchemaUpdateOption> schemaUpdateOptions;
4748

4849
public static final class Builder
4950
extends JobConfiguration.Builder<LoadJobConfiguration, Builder>
@@ -58,6 +59,7 @@ public static final class Builder
5859
private Schema schema;
5960
private Boolean ignoreUnknownValues;
6061
private List<String> projectionFields;
62+
private List<JobInfo.SchemaUpdateOption> schemaUpdateOptions;
6163

6264
private Builder() {
6365
super(Type.LOAD);
@@ -74,6 +76,7 @@ private Builder(LoadJobConfiguration loadConfiguration) {
7476
this.ignoreUnknownValues = loadConfiguration.ignoreUnknownValues;
7577
this.projectionFields = loadConfiguration.projectionFields;
7678
this.sourceUris = loadConfiguration.sourceUris;
79+
this.schemaUpdateOptions = loadConfiguration.schemaUpdateOptions;
7780
}
7881

7982
private Builder(com.google.api.services.bigquery.model.JobConfiguration configurationPb) {
@@ -121,6 +124,13 @@ private Builder(com.google.api.services.bigquery.model.JobConfiguration configur
121124
if (loadConfigurationPb.getSourceUris() != null) {
122125
this.sourceUris = ImmutableList.copyOf(configurationPb.getLoad().getSourceUris());
123126
}
127+
if (loadConfigurationPb.getSchemaUpdateOptions() != null) {
128+
ImmutableList.Builder<JobInfo.SchemaUpdateOption> schemaUpdateOptionsBuilder = new ImmutableList.Builder<>();
129+
for (String rawSchemaUpdateOption : loadConfigurationPb.getSchemaUpdateOptions()) {
130+
schemaUpdateOptionsBuilder.add(JobInfo.SchemaUpdateOption.valueOf(rawSchemaUpdateOption));
131+
}
132+
this.schemaUpdateOptions = schemaUpdateOptionsBuilder.build();
133+
}
124134
}
125135

126136
@Override
@@ -240,6 +250,13 @@ public Builder setSourceUris(List<String> sourceUris) {
240250
return this;
241251
}
242252

253+
@Override
254+
public Builder setSchemaUpdateOptions(List<JobInfo.SchemaUpdateOption> schemaUpdateOptions) {
255+
this.schemaUpdateOptions =
256+
schemaUpdateOptions != null ? ImmutableList.copyOf(schemaUpdateOptions) : null;
257+
return this;
258+
}
259+
243260
@Override
244261
public LoadJobConfiguration build() {
245262
return new LoadJobConfiguration(this);
@@ -257,6 +274,7 @@ private LoadJobConfiguration(Builder builder) {
257274
this.schema = builder.schema;
258275
this.ignoreUnknownValues = builder.ignoreUnknownValues;
259276
this.projectionFields = builder.projectionFields;
277+
this.schemaUpdateOptions = builder.schemaUpdateOptions;
260278
}
261279

262280
@Override
@@ -371,6 +389,11 @@ public List<String> getSourceUris() {
371389
return sourceUris;
372390
}
373391

392+
@Override
393+
public List<JobInfo.SchemaUpdateOption> getSchemaUpdateOptions() {
394+
return schemaUpdateOptions;
395+
}
396+
374397
@Override
375398
public Builder toBuilder() {
376399
return new Builder(this);
@@ -387,7 +410,8 @@ ToStringHelper toStringHelper() {
387410
.add("schema", schema)
388411
.add("ignoreUnknownValue", ignoreUnknownValues)
389412
.add("projectionFields", projectionFields)
390-
.add("sourceUris", sourceUris);
413+
.add("sourceUris", sourceUris)
414+
.add("schemaUpdateOptions", schemaUpdateOptions);
391415
}
392416

393417
@Override
@@ -441,6 +465,13 @@ com.google.api.services.bigquery.model.JobConfiguration toPb() {
441465
if (sourceUris != null) {
442466
loadConfigurationPb.setSourceUris(ImmutableList.copyOf(sourceUris));
443467
}
468+
if (schemaUpdateOptions != null) {
469+
ImmutableList.Builder<String> schemaUpdateOptionsBuilder = new ImmutableList.Builder<>();
470+
for (JobInfo.SchemaUpdateOption schemaUpdateOption : schemaUpdateOptions) {
471+
schemaUpdateOptionsBuilder.add(schemaUpdateOption.name());
472+
}
473+
loadConfigurationPb.setSchemaUpdateOptions(schemaUpdateOptionsBuilder.build());
474+
}
444475
return new com.google.api.services.bigquery.model.JobConfiguration()
445476
.setLoad(loadConfigurationPb);
446477
}

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryJobConfiguration.java‎

Lines changed: 44 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import com.google.api.services.bigquery.model.JobConfigurationQuery;
2222
import com.google.cloud.bigquery.JobInfo.CreateDisposition;
2323
import com.google.cloud.bigquery.JobInfo.WriteDisposition;
24+
import com.google.cloud.bigquery.JobInfo.SchemaUpdateOption;
2425
import com.google.common.base.MoreObjects.ToStringHelper;
2526
import com.google.common.collect.ImmutableList;
2627
import com.google.common.collect.ImmutableMap;
@@ -52,6 +53,7 @@ public final class QueryJobConfiguration extends JobConfiguration {
5253
private final Boolean flattenResults;
5354
private final Boolean dryRun;
5455
private final Boolean useLegacySql;
56+
private final List<SchemaUpdateOption> schemaUpdateOptions;
5557

5658
/**
5759
* Priority levels for a query. If not specified the priority is assumed to be
@@ -89,6 +91,7 @@ public static final class Builder
8991
private Boolean flattenResults;
9092
private Boolean dryRun;
9193
private Boolean useLegacySql;
94+
private List<SchemaUpdateOption> schemaUpdateOptions;
9295

9396
private Builder() {
9497
super(Type.QUERY);
@@ -109,6 +112,7 @@ private Builder(QueryJobConfiguration jobConfiguration) {
109112
this.flattenResults = jobConfiguration.flattenResults;
110113
this.dryRun = jobConfiguration.dryRun;
111114
this.useLegacySql = jobConfiguration.useLegacySql;
115+
this.schemaUpdateOptions = jobConfiguration.schemaUpdateOptions;
112116
}
113117

114118
private Builder(com.google.api.services.bigquery.model.JobConfiguration configurationPb) {
@@ -146,6 +150,13 @@ private Builder(com.google.api.services.bigquery.model.JobConfiguration configur
146150
writeDisposition =
147151
WriteDisposition.valueOf(queryConfigurationPb.getWriteDisposition());
148152
}
153+
if (queryConfigurationPb.getSchemaUpdateOptions() != null) {
154+
ImmutableList.Builder<JobInfo.SchemaUpdateOption> schemaUpdateOptionsBuilder = new ImmutableList.Builder<>();
155+
for (String rawSchemaUpdateOption : queryConfigurationPb.getSchemaUpdateOptions()) {
156+
schemaUpdateOptionsBuilder.add(JobInfo.SchemaUpdateOption.valueOf(rawSchemaUpdateOption));
157+
}
158+
this.schemaUpdateOptions = schemaUpdateOptionsBuilder.build();
159+
}
149160
}
150161

151162
/**
@@ -459,6 +470,18 @@ public Builder setUseLegacySql(Boolean useLegacySql) {
459470
return this;
460471
}
461472

473+
474+
/**
475+
* [Experimental] Sets options allowing the schema of the destination table to be updated as a side effect of the
476+
* query job. Schema update options are supported in two cases: when writeDisposition is WRITE_APPEND; when
477+
* writeDisposition is WRITE_TRUNCATE and the destination table is a partition of a table, specified by partition
478+
* decorators. For normal tables, WRITE_TRUNCATE will always overwrite the schema.
479+
*/
480+
public Builder setSchemaUpdateOptions(List<SchemaUpdateOption> schemaUpdateOptions) {
481+
this.schemaUpdateOptions = schemaUpdateOptions;
482+
return this;
483+
}
484+
462485
public QueryJobConfiguration build() {
463486
return new QueryJobConfiguration(this);
464487
}
@@ -480,6 +503,7 @@ private QueryJobConfiguration(Builder builder) {
480503
builder.tableDefinitions != null ? ImmutableMap.copyOf(builder.tableDefinitions) : null;
481504
this.dryRun = builder.dryRun;
482505
this.useLegacySql = builder.useLegacySql;
506+
this.schemaUpdateOptions = builder.schemaUpdateOptions;
483507
}
484508

485509
/**
@@ -682,6 +706,16 @@ public Boolean useLegacySql() {
682706
return useLegacySql;
683707
}
684708

709+
/**
710+
* [Experimental] Returns options allowing the schema of the destination table to be updated as a side effect of the
711+
* query job. Schema update options are supported in two cases: when writeDisposition is WRITE_APPEND; when
712+
* writeDisposition is WRITE_TRUNCATE and the destination table is a partition of a table, specified by partition
713+
* decorators. For normal tables, WRITE_TRUNCATE will always overwrite the schema.
714+
*/
715+
public List<SchemaUpdateOption> getSchemaUpdateOptions() {
716+
return schemaUpdateOptions;
717+
}
718+
685719
@Override
686720
public Builder toBuilder() {
687721
return new Builder(this);
@@ -702,7 +736,8 @@ ToStringHelper toStringHelper() {
702736
.add("createDisposition", createDisposition)
703737
.add("writeDisposition", writeDisposition)
704738
.add("dryRun", dryRun)
705-
.add("useLegacySql", useLegacySql);
739+
.add("useLegacySql", useLegacySql)
740+
.add("schemaUpdateOptions", schemaUpdateOptions);
706741
}
707742

708743
@Override
@@ -716,7 +751,7 @@ public boolean equals(Object obj) {
716751
public int hashCode() {
717752
return Objects.hash(baseHashCode(), allowLargeResults, createDisposition, destinationTable,
718753
defaultDataset, flattenResults, priority, query, tableDefinitions, useQueryCache,
719-
userDefinedFunctions, writeDisposition, dryRun, useLegacySql);
754+
userDefinedFunctions, writeDisposition, dryRun, useLegacySql, schemaUpdateOptions);
720755
}
721756

722757
@Override
@@ -773,6 +808,13 @@ com.google.api.services.bigquery.model.JobConfiguration toPb() {
773808
if (useLegacySql != null) {
774809
queryConfigurationPb.setUseLegacySql(useLegacySql);
775810
}
811+
if (schemaUpdateOptions != null) {
812+
ImmutableList.Builder<String> schemaUpdateOptionsBuilder = new ImmutableList.Builder<>();
813+
for (JobInfo.SchemaUpdateOption schemaUpdateOption : schemaUpdateOptions) {
814+
schemaUpdateOptionsBuilder.add(schemaUpdateOption.name());
815+
}
816+
queryConfigurationPb.setSchemaUpdateOptions(schemaUpdateOptionsBuilder.build());
817+
}
776818
return configurationPb.setQuery(queryConfigurationPb);
777819
}
778820

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/WriteChannelConfiguration.java‎

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import com.google.api.services.bigquery.model.JobConfigurationLoad;
2222
import com.google.cloud.bigquery.JobInfo.CreateDisposition;
2323
import com.google.cloud.bigquery.JobInfo.WriteDisposition;
24+
import com.google.cloud.bigquery.JobInfo.SchemaUpdateOption;
2425
import com.google.common.base.MoreObjects;
2526
import com.google.common.collect.ImmutableList;
2627
import com.google.common.primitives.Ints;
@@ -46,6 +47,7 @@ public final class WriteChannelConfiguration implements LoadConfiguration, Seria
4647
private final Schema schema;
4748
private final Boolean ignoreUnknownValues;
4849
private final List<String> projectionFields;
50+
private final List<SchemaUpdateOption> schemaUpdateOptions;
4951

5052
public static final class Builder implements LoadConfiguration.Builder {
5153

@@ -57,6 +59,7 @@ public static final class Builder implements LoadConfiguration.Builder {
5759
private Schema schema;
5860
private Boolean ignoreUnknownValues;
5961
private List<String> projectionFields;
62+
private List<SchemaUpdateOption> schemaUpdateOptions;
6063

6164
private Builder() {}
6265

@@ -69,6 +72,7 @@ private Builder(WriteChannelConfiguration writeChannelConfiguration) {
6972
this.schema = writeChannelConfiguration.schema;
7073
this.ignoreUnknownValues = writeChannelConfiguration.ignoreUnknownValues;
7174
this.projectionFields = writeChannelConfiguration.projectionFields;
75+
this.schemaUpdateOptions = writeChannelConfiguration.schemaUpdateOptions;
7276
}
7377

7478
private Builder(com.google.api.services.bigquery.model.JobConfiguration configurationPb) {
@@ -111,6 +115,13 @@ private Builder(com.google.api.services.bigquery.model.JobConfiguration configur
111115
}
112116
this.ignoreUnknownValues = loadConfigurationPb.getIgnoreUnknownValues();
113117
this.projectionFields = loadConfigurationPb.getProjectionFields();
118+
if (loadConfigurationPb.getSchemaUpdateOptions() != null) {
119+
ImmutableList.Builder<JobInfo.SchemaUpdateOption> schemaUpdateOptionsBuilder = new ImmutableList.Builder<>();
120+
for (String rawSchemaUpdateOption : loadConfigurationPb.getSchemaUpdateOptions()) {
121+
schemaUpdateOptionsBuilder.add(JobInfo.SchemaUpdateOption.valueOf(rawSchemaUpdateOption));
122+
}
123+
this.schemaUpdateOptions = schemaUpdateOptionsBuilder.build();
124+
}
114125
}
115126

116127
@Override
@@ -210,6 +221,13 @@ public Builder setProjectionFields(List<String> projectionFields) {
210221
return this;
211222
}
212223

224+
@Override
225+
public LoadConfiguration.Builder setSchemaUpdateOptions(List<SchemaUpdateOption> schemaUpdateOptions) {
226+
this.schemaUpdateOptions =
227+
schemaUpdateOptions != null ? ImmutableList.copyOf(schemaUpdateOptions) : null;
228+
return this;
229+
}
230+
213231
@Override
214232
public WriteChannelConfiguration build() {
215233
return new WriteChannelConfiguration(this);
@@ -225,6 +243,7 @@ protected WriteChannelConfiguration(Builder builder) {
225243
this.schema = builder.schema;
226244
this.ignoreUnknownValues = builder.ignoreUnknownValues;
227245
this.projectionFields = builder.projectionFields;
246+
this.schemaUpdateOptions = builder.schemaUpdateOptions;
228247
}
229248

230249
@Override
@@ -320,6 +339,11 @@ public List<String> getProjectionFields() {
320339
return projectionFields;
321340
}
322341

342+
@Override
343+
public List<SchemaUpdateOption> getSchemaUpdateOptions() {
344+
return schemaUpdateOptions;
345+
}
346+
323347
@Override
324348
public Builder toBuilder() {
325349
return new Builder(this);
@@ -334,7 +358,8 @@ MoreObjects.ToStringHelper toStringHelper() {
334358
.add("maxBadRecords", maxBadRecords)
335359
.add("schema", schema)
336360
.add("ignoreUnknownValue", ignoreUnknownValues)
337-
.add("projectionFields", projectionFields);
361+
.add("projectionFields", projectionFields)
362+
.add("schemaUpdateOptions", schemaUpdateOptions);
338363
}
339364

340365
@Override
@@ -352,7 +377,7 @@ public boolean equals(Object obj) {
352377
@Override
353378
public int hashCode() {
354379
return Objects.hash(destinationTable, createDisposition, writeDisposition, formatOptions,
355-
maxBadRecords, schema, ignoreUnknownValues, projectionFields);
380+
maxBadRecords, schema, ignoreUnknownValues, projectionFields, schemaUpdateOptions);
356381
}
357382

358383
WriteChannelConfiguration setProjectId(String projectId) {
@@ -389,6 +414,13 @@ com.google.api.services.bigquery.model.JobConfiguration toPb() {
389414
loadConfigurationPb.setMaxBadRecords(maxBadRecords);
390415
loadConfigurationPb.setIgnoreUnknownValues(ignoreUnknownValues);
391416
loadConfigurationPb.setProjectionFields(projectionFields);
417+
if (schemaUpdateOptions != null) {
418+
ImmutableList.Builder<String> schemaUpdateOptionsBuilder = new ImmutableList.Builder<>();
419+
for (JobInfo.SchemaUpdateOption schemaUpdateOption : schemaUpdateOptions) {
420+
schemaUpdateOptionsBuilder.add(schemaUpdateOption.name());
421+
}
422+
loadConfigurationPb.setSchemaUpdateOptions(schemaUpdateOptionsBuilder.build());
423+
}
392424
return new com.google.api.services.bigquery.model.JobConfiguration()
393425
.setLoad(loadConfigurationPb);
394426
}

0 commit comments

Comments
 (0)