Skip to content

Commit 1865c47

Browse files
gene-telligentgene-telligent
committed
Implementing BigQuery: LoadJobConfiguration to support schemaUpdateOptions (GitHub Issue #1566)
1 parent 95b70eb commit 1865c47

9 files changed

Lines changed: 169 additions & 7 deletions

File tree

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/JobInfo.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,22 @@ public enum WriteDisposition {
8888
WRITE_EMPTY
8989
}
9090

91+
/**
92+
* Specifies options relating to allowing the schema of the destination table to be updated as a side effect of the
93+
* load or query job.
94+
*/
95+
public enum SchemaUpdateOption {
96+
/**
97+
* Allow adding a nullable field to the schema.
98+
*/
99+
ALLOW_FIELD_ADDITION,
100+
101+
/**
102+
* Allow relaxing a required field in the original schema to nullable.
103+
*/
104+
ALLOW_FIELD_RELAXATION
105+
}
106+
91107
/**
92108
* A builder for {@code JobInfo} objects.
93109
*/

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadConfiguration.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818

1919
import com.google.cloud.bigquery.JobInfo.CreateDisposition;
2020
import com.google.cloud.bigquery.JobInfo.WriteDisposition;
21+
import com.google.cloud.bigquery.JobInfo.SchemaUpdateOption;
2122

2223
import java.util.List;
2324

@@ -92,6 +93,14 @@ interface Builder {
9293
*/
9394
Builder setIgnoreUnknownValues(Boolean ignoreUnknownValues);
9495

96+
/**
97+
* [Experimental] Sets options allowing the schema of the destination table to be updated as a side effect of the
98+
* load job. Schema update options are supported in two cases: when writeDisposition is WRITE_APPEND; when
99+
* writeDisposition is WRITE_TRUNCATE and the destination table is a partition of a table, specified by partition
100+
* decorators. For normal tables, WRITE_TRUNCATE will always overwrite the schema.
101+
*/
102+
Builder setSchemaUpdateOptions(List<SchemaUpdateOption> schemaUpdateOptions);
103+
95104
LoadConfiguration build();
96105
}
97106

@@ -160,6 +169,14 @@ interface Builder {
160169
*/
161170
DatastoreBackupOptions getDatastoreBackupOptions();
162171

172+
/**
173+
* [Experimental] Returns options allowing the schema of the destination table to be updated as a side effect of the
174+
* load job. Schema update options are supported in two cases: when writeDisposition is WRITE_APPEND; when
175+
* writeDisposition is WRITE_TRUNCATE and the destination table is a partition of a table, specified by partition
176+
* decorators. For normal tables, WRITE_TRUNCATE will always overwrite the schema.
177+
*/
178+
List<SchemaUpdateOption> getSchemaUpdateOptions();
179+
163180
/**
164181
* Returns a builder for the load configuration object.
165182
*/

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java‎

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ public final class LoadJobConfiguration extends JobConfiguration implements Load
4343
private final Integer maxBadRecords;
4444
private final Schema schema;
4545
private final Boolean ignoreUnknownValues;
46+
private final List<JobInfo.SchemaUpdateOption> schemaUpdateOptions;
4647

4748
public static final class Builder
4849
extends JobConfiguration.Builder<LoadJobConfiguration, Builder>
@@ -57,6 +58,7 @@ public static final class Builder
5758
private Schema schema;
5859
private Boolean ignoreUnknownValues;
5960
private List<String> projectionFields;
61+
private List<JobInfo.SchemaUpdateOption> schemaUpdateOptions;
6062

6163
private Builder() {
6264
super(Type.LOAD);
@@ -72,6 +74,7 @@ private Builder(LoadJobConfiguration loadConfiguration) {
7274
this.schema = loadConfiguration.schema;
7375
this.ignoreUnknownValues = loadConfiguration.ignoreUnknownValues;
7476
this.sourceUris = loadConfiguration.sourceUris;
77+
this.schemaUpdateOptions = loadConfiguration.schemaUpdateOptions;
7578
}
7679

7780
private Builder(com.google.api.services.bigquery.model.JobConfiguration configurationPb) {
@@ -119,6 +122,13 @@ private Builder(com.google.api.services.bigquery.model.JobConfiguration configur
119122
if (loadConfigurationPb.getSourceUris() != null) {
120123
this.sourceUris = ImmutableList.copyOf(configurationPb.getLoad().getSourceUris());
121124
}
125+
if (loadConfigurationPb.getSchemaUpdateOptions() != null) {
126+
ImmutableList.Builder<JobInfo.SchemaUpdateOption> schemaUpdateOptionsBuilder = new ImmutableList.Builder<>();
127+
for (String rawSchemaUpdateOption : loadConfigurationPb.getSchemaUpdateOptions()) {
128+
schemaUpdateOptionsBuilder.add(JobInfo.SchemaUpdateOption.valueOf(rawSchemaUpdateOption));
129+
}
130+
this.schemaUpdateOptions = schemaUpdateOptionsBuilder.build();
131+
}
122132
}
123133

124134

@@ -180,6 +190,13 @@ public Builder setSourceUris(List<String> sourceUris) {
180190
return this;
181191
}
182192

193+
@Override
194+
public Builder setSchemaUpdateOptions(List<JobInfo.SchemaUpdateOption> schemaUpdateOptions) {
195+
this.schemaUpdateOptions =
196+
schemaUpdateOptions != null ? ImmutableList.copyOf(schemaUpdateOptions) : null;
197+
return this;
198+
}
199+
183200
@Override
184201
public LoadJobConfiguration build() {
185202
return new LoadJobConfiguration(this);
@@ -196,6 +213,7 @@ private LoadJobConfiguration(Builder builder) {
196213
this.maxBadRecords = builder.maxBadRecords;
197214
this.schema = builder.schema;
198215
this.ignoreUnknownValues = builder.ignoreUnknownValues;
216+
this.schemaUpdateOptions = builder.schemaUpdateOptions;
199217
}
200218

201219

@@ -260,6 +278,11 @@ public List<String> getSourceUris() {
260278
return sourceUris;
261279
}
262280

281+
@Override
282+
public List<JobInfo.SchemaUpdateOption> getSchemaUpdateOptions() {
283+
return schemaUpdateOptions;
284+
}
285+
263286
@Override
264287
public Builder toBuilder() {
265288
return new Builder(this);
@@ -275,7 +298,8 @@ ToStringHelper toStringHelper() {
275298
.add("maxBadRecords", maxBadRecords)
276299
.add("schema", schema)
277300
.add("ignoreUnknownValue", ignoreUnknownValues)
278-
.add("sourceUris", sourceUris);
301+
.add("sourceUris", sourceUris)
302+
.add("schemaUpdateOptions", schemaUpdateOptions);
279303
}
280304

281305
@Override
@@ -332,6 +356,13 @@ com.google.api.services.bigquery.model.JobConfiguration toPb() {
332356
if (sourceUris != null) {
333357
loadConfigurationPb.setSourceUris(ImmutableList.copyOf(sourceUris));
334358
}
359+
if (schemaUpdateOptions != null) {
360+
ImmutableList.Builder<String> schemaUpdateOptionsBuilder = new ImmutableList.Builder<>();
361+
for (JobInfo.SchemaUpdateOption schemaUpdateOption : schemaUpdateOptions) {
362+
schemaUpdateOptionsBuilder.add(schemaUpdateOption.name());
363+
}
364+
loadConfigurationPb.setSchemaUpdateOptions(schemaUpdateOptionsBuilder.build());
365+
}
335366
return new com.google.api.services.bigquery.model.JobConfiguration()
336367
.setLoad(loadConfigurationPb);
337368
}

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryJobConfiguration.java‎

Lines changed: 44 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import com.google.api.services.bigquery.model.JobConfigurationQuery;
2222
import com.google.cloud.bigquery.JobInfo.CreateDisposition;
2323
import com.google.cloud.bigquery.JobInfo.WriteDisposition;
24+
import com.google.cloud.bigquery.JobInfo.SchemaUpdateOption;
2425
import com.google.common.base.MoreObjects.ToStringHelper;
2526
import com.google.common.collect.ImmutableList;
2627
import com.google.common.collect.ImmutableMap;
@@ -52,6 +53,7 @@ public final class QueryJobConfiguration extends JobConfiguration {
5253
private final Boolean flattenResults;
5354
private final Boolean dryRun;
5455
private final Boolean useLegacySql;
56+
private final List<SchemaUpdateOption> schemaUpdateOptions;
5557

5658
/**
5759
* Priority levels for a query. If not specified the priority is assumed to be
@@ -89,6 +91,7 @@ public static final class Builder
8991
private Boolean flattenResults;
9092
private Boolean dryRun;
9193
private Boolean useLegacySql;
94+
private List<SchemaUpdateOption> schemaUpdateOptions;
9295

9396
private Builder() {
9497
super(Type.QUERY);
@@ -109,6 +112,7 @@ private Builder(QueryJobConfiguration jobConfiguration) {
109112
this.flattenResults = jobConfiguration.flattenResults;
110113
this.dryRun = jobConfiguration.dryRun;
111114
this.useLegacySql = jobConfiguration.useLegacySql;
115+
this.schemaUpdateOptions = jobConfiguration.schemaUpdateOptions;
112116
}
113117

114118
private Builder(com.google.api.services.bigquery.model.JobConfiguration configurationPb) {
@@ -146,6 +150,13 @@ private Builder(com.google.api.services.bigquery.model.JobConfiguration configur
146150
writeDisposition =
147151
WriteDisposition.valueOf(queryConfigurationPb.getWriteDisposition());
148152
}
153+
if (queryConfigurationPb.getSchemaUpdateOptions() != null) {
154+
ImmutableList.Builder<JobInfo.SchemaUpdateOption> schemaUpdateOptionsBuilder = new ImmutableList.Builder<>();
155+
for (String rawSchemaUpdateOption : queryConfigurationPb.getSchemaUpdateOptions()) {
156+
schemaUpdateOptionsBuilder.add(JobInfo.SchemaUpdateOption.valueOf(rawSchemaUpdateOption));
157+
}
158+
this.schemaUpdateOptions = schemaUpdateOptionsBuilder.build();
159+
}
149160
}
150161

151162

@@ -325,6 +336,18 @@ public Builder setUseLegacySql(Boolean useLegacySql) {
325336
return this;
326337
}
327338

339+
340+
/**
341+
* [Experimental] Sets options allowing the schema of the destination table to be updated as a side effect of the
342+
* query job. Schema update options are supported in two cases: when writeDisposition is WRITE_APPEND; when
343+
* writeDisposition is WRITE_TRUNCATE and the destination table is a partition of a table, specified by partition
344+
* decorators. For normal tables, WRITE_TRUNCATE will always overwrite the schema.
345+
*/
346+
public Builder setSchemaUpdateOptions(List<SchemaUpdateOption> schemaUpdateOptions) {
347+
this.schemaUpdateOptions = schemaUpdateOptions;
348+
return this;
349+
}
350+
328351
public QueryJobConfiguration build() {
329352
return new QueryJobConfiguration(this);
330353
}
@@ -346,6 +369,7 @@ private QueryJobConfiguration(Builder builder) {
346369
builder.tableDefinitions != null ? ImmutableMap.copyOf(builder.tableDefinitions) : null;
347370
this.dryRun = builder.dryRun;
348371
this.useLegacySql = builder.useLegacySql;
372+
this.schemaUpdateOptions = builder.schemaUpdateOptions;
349373
}
350374

351375
/**
@@ -479,6 +503,16 @@ public Boolean useLegacySql() {
479503
return useLegacySql;
480504
}
481505

506+
/**
507+
* [Experimental] Returns options allowing the schema of the destination table to be updated as a side effect of the
508+
* query job. Schema update options are supported in two cases: when writeDisposition is WRITE_APPEND; when
509+
* writeDisposition is WRITE_TRUNCATE and the destination table is a partition of a table, specified by partition
510+
* decorators. For normal tables, WRITE_TRUNCATE will always overwrite the schema.
511+
*/
512+
public List<SchemaUpdateOption> getSchemaUpdateOptions() {
513+
return schemaUpdateOptions;
514+
}
515+
482516
@Override
483517
public Builder toBuilder() {
484518
return new Builder(this);
@@ -499,7 +533,8 @@ ToStringHelper toStringHelper() {
499533
.add("createDisposition", createDisposition)
500534
.add("writeDisposition", writeDisposition)
501535
.add("dryRun", dryRun)
502-
.add("useLegacySql", useLegacySql);
536+
.add("useLegacySql", useLegacySql)
537+
.add("schemaUpdateOptions", schemaUpdateOptions);
503538
}
504539

505540
@Override
@@ -513,7 +548,7 @@ public boolean equals(Object obj) {
513548
public int hashCode() {
514549
return Objects.hash(baseHashCode(), allowLargeResults, createDisposition, destinationTable,
515550
defaultDataset, flattenResults, priority, query, tableDefinitions, useQueryCache,
516-
userDefinedFunctions, writeDisposition, dryRun, useLegacySql);
551+
userDefinedFunctions, writeDisposition, dryRun, useLegacySql, schemaUpdateOptions);
517552
}
518553

519554
@Override
@@ -570,6 +605,13 @@ com.google.api.services.bigquery.model.JobConfiguration toPb() {
570605
if (useLegacySql != null) {
571606
queryConfigurationPb.setUseLegacySql(useLegacySql);
572607
}
608+
if (schemaUpdateOptions != null) {
609+
ImmutableList.Builder<String> schemaUpdateOptionsBuilder = new ImmutableList.Builder<>();
610+
for (JobInfo.SchemaUpdateOption schemaUpdateOption : schemaUpdateOptions) {
611+
schemaUpdateOptionsBuilder.add(schemaUpdateOption.name());
612+
}
613+
queryConfigurationPb.setSchemaUpdateOptions(schemaUpdateOptionsBuilder.build());
614+
}
573615
return configurationPb.setQuery(queryConfigurationPb);
574616
}
575617

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/WriteChannelConfiguration.java‎

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import com.google.api.services.bigquery.model.JobConfigurationLoad;
2222
import com.google.cloud.bigquery.JobInfo.CreateDisposition;
2323
import com.google.cloud.bigquery.JobInfo.WriteDisposition;
24+
import com.google.cloud.bigquery.JobInfo.SchemaUpdateOption;
2425
import com.google.common.base.MoreObjects;
2526
import com.google.common.collect.ImmutableList;
2627
import com.google.common.primitives.Ints;
@@ -45,6 +46,7 @@ public final class WriteChannelConfiguration implements LoadConfiguration, Seria
4546
private final Integer maxBadRecords;
4647
private final Schema schema;
4748
private final Boolean ignoreUnknownValues;
49+
private final List<SchemaUpdateOption> schemaUpdateOptions;
4850

4951
public static final class Builder implements LoadConfiguration.Builder {
5052

@@ -55,6 +57,7 @@ public static final class Builder implements LoadConfiguration.Builder {
5557
private Integer maxBadRecords;
5658
private Schema schema;
5759
private Boolean ignoreUnknownValues;
60+
private List<SchemaUpdateOption> schemaUpdateOptions;
5861

5962
private Builder() {}
6063

@@ -66,6 +69,7 @@ private Builder(WriteChannelConfiguration writeChannelConfiguration) {
6669
this.maxBadRecords = writeChannelConfiguration.maxBadRecords;
6770
this.schema = writeChannelConfiguration.schema;
6871
this.ignoreUnknownValues = writeChannelConfiguration.ignoreUnknownValues;
72+
this.schemaUpdateOptions = writeChannelConfiguration.schemaUpdateOptions;
6973
}
7074

7175
private Builder(com.google.api.services.bigquery.model.JobConfiguration configurationPb) {
@@ -112,6 +116,13 @@ private Builder(com.google.api.services.bigquery.model.JobConfiguration configur
112116
.setProjectionFields(loadConfigurationPb.getProjectionFields())
113117
.build();
114118
}
119+
if (loadConfigurationPb.getSchemaUpdateOptions() != null) {
120+
ImmutableList.Builder<JobInfo.SchemaUpdateOption> schemaUpdateOptionsBuilder = new ImmutableList.Builder<>();
121+
for (String rawSchemaUpdateOption : loadConfigurationPb.getSchemaUpdateOptions()) {
122+
schemaUpdateOptionsBuilder.add(JobInfo.SchemaUpdateOption.valueOf(rawSchemaUpdateOption));
123+
}
124+
this.schemaUpdateOptions = schemaUpdateOptionsBuilder.build();
125+
}
115126
}
116127

117128

@@ -163,6 +174,13 @@ public Builder setIgnoreUnknownValues(Boolean ignoreUnknownValues) {
163174
return this;
164175
}
165176

177+
@Override
178+
public Builder setSchemaUpdateOptions(List<SchemaUpdateOption> schemaUpdateOptions) {
179+
this.schemaUpdateOptions =
180+
schemaUpdateOptions != null ? ImmutableList.copyOf(schemaUpdateOptions) : null;
181+
return this;
182+
}
183+
166184
@Override
167185
public WriteChannelConfiguration build() {
168186
return new WriteChannelConfiguration(this);
@@ -177,6 +195,7 @@ protected WriteChannelConfiguration(Builder builder) {
177195
this.maxBadRecords = builder.maxBadRecords;
178196
this.schema = builder.schema;
179197
this.ignoreUnknownValues = builder.ignoreUnknownValues;
198+
this.schemaUpdateOptions = builder.schemaUpdateOptions;
180199
}
181200

182201

@@ -233,6 +252,11 @@ public DatastoreBackupOptions getDatastoreBackupOptions() {
233252
(DatastoreBackupOptions) formatOptions : null;
234253
}
235254

255+
@Override
256+
public List<SchemaUpdateOption> getSchemaUpdateOptions() {
257+
return schemaUpdateOptions;
258+
}
259+
236260
@Override
237261
public Builder toBuilder() {
238262
return new Builder(this);
@@ -246,7 +270,8 @@ MoreObjects.ToStringHelper toStringHelper() {
246270
.add("formatOptions", formatOptions)
247271
.add("maxBadRecords", maxBadRecords)
248272
.add("schema", schema)
249-
.add("ignoreUnknownValue", ignoreUnknownValues);
273+
.add("ignoreUnknownValue", ignoreUnknownValues)
274+
.add("schemaUpdateOptions", schemaUpdateOptions);
250275
}
251276

252277
@Override
@@ -264,7 +289,7 @@ public boolean equals(Object obj) {
264289
@Override
265290
public int hashCode() {
266291
return Objects.hash(destinationTable, createDisposition, writeDisposition, formatOptions,
267-
maxBadRecords, schema, ignoreUnknownValues);
292+
maxBadRecords, schema, ignoreUnknownValues, schemaUpdateOptions);
268293
}
269294

270295
WriteChannelConfiguration setProjectId(String projectId) {
@@ -304,6 +329,13 @@ com.google.api.services.bigquery.model.JobConfiguration toPb() {
304329
DatastoreBackupOptions backupOptions = getDatastoreBackupOptions();
305330
loadConfigurationPb.setProjectionFields(backupOptions.getProjectionFields());
306331
}
332+
if (schemaUpdateOptions != null) {
333+
ImmutableList.Builder<String> schemaUpdateOptionsBuilder = new ImmutableList.Builder<>();
334+
for (JobInfo.SchemaUpdateOption schemaUpdateOption : schemaUpdateOptions) {
335+
schemaUpdateOptionsBuilder.add(schemaUpdateOption.name());
336+
}
337+
loadConfigurationPb.setSchemaUpdateOptions(schemaUpdateOptionsBuilder.build());
338+
}
307339
return new com.google.api.services.bigquery.model.JobConfiguration()
308340
.setLoad(loadConfigurationPb);
309341
}

0 commit comments

Comments
 (0)