Skip to content

Commit 3339b37

Browse files
Praful Makanisduskis
Praful Makani
authored and
sduskis
committed
---
yaml --- r: 19637 b: refs/heads/autosynth-firestore c: e0d4cdd h: refs/heads/master i: 19635: bb7e3bb
1 parent c3e1be4 commit 3339b37

6 files changed

Lines changed: 106 additions & 4 deletions

File tree

‎[refs]‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,7 @@ refs/heads/autosynth-containeranalysis: 0f377286f60df2b6fd37c183bf170c68dc014cfb
131131
refs/heads/autosynth-datastore: 9acd400b484d6691a080c9152a331d88d24fefc1
132132
refs/heads/autosynth-dialogflow: 7dbc2c1ea714328ccfa4f33645045f017ff080e7
133133
refs/heads/autosynth-errorreporting: 1101a04e8be074802c35332d5fcf8297f61cae32
134-
refs/heads/autosynth-firestore: 685f585b5cd4514f41a1ab157f9d2c1ea848c92c
134+
refs/heads/autosynth-firestore: e0d4cdd7e6b914551b9afdabc240a01044bad7b0
135135
refs/heads/autosynth-iot: f16fc4ba85bbb37eacecb64a2a7336f685813f6c
136136
refs/heads/autosynth-kms: 4fff8f3cf6c830f7706ac4ef272f036bb87fbaf8
137137
refs/heads/autosynth-language: 6262e2eb76944f01972c887b3e684aaf65ec999a

‎branches/autosynth-firestore/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadConfiguration.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,13 @@ interface Builder {
118118
/** Sets the clustering specification for the destination table. */
119119
Builder setClustering(Clustering clustering);
120120

121+
/**
122+
* If FormatOptions is set to AVRO, you can interpret logical types into their corresponding
123+
* types (such as TIMESTAMP) instead of only using their raw types (such as INTEGER). The value
124+
* may be {@code null}.
125+
*/
126+
Builder setUseAvroLogicalTypes(Boolean useAvroLogicalTypes);
127+
121128
LoadConfiguration build();
122129
}
123130

@@ -204,6 +211,9 @@ interface Builder {
204211
/** Returns the clustering specification for the definition table. */
205212
Clustering getClustering();
206213

214+
/** Returns True/False. Indicates whether the logical type is interpreted. */
215+
Boolean getUseAvroLogicalTypes();
216+
207217
/** Returns a builder for the load configuration object. */
208218
Builder toBuilder();
209219
}

‎branches/autosynth-firestore/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ public final class LoadJobConfiguration extends JobConfiguration implements Load
4949
private final Boolean autodetect;
5050
private final TimePartitioning timePartitioning;
5151
private final Clustering clustering;
52+
private final Boolean useAvroLogicalTypes;
5253

5354
public static final class Builder extends JobConfiguration.Builder<LoadJobConfiguration, Builder>
5455
implements LoadConfiguration.Builder {
@@ -68,6 +69,7 @@ public static final class Builder extends JobConfiguration.Builder<LoadJobConfig
6869
private Boolean autodetect;
6970
private TimePartitioning timePartitioning;
7071
private Clustering clustering;
72+
private Boolean useAvroLogicalTypes;
7173

7274
private Builder() {
7375
super(Type.LOAD);
@@ -90,6 +92,7 @@ private Builder(LoadJobConfiguration loadConfiguration) {
9092
loadConfiguration.destinationEncryptionConfiguration;
9193
this.timePartitioning = loadConfiguration.timePartitioning;
9294
this.clustering = loadConfiguration.clustering;
95+
this.useAvroLogicalTypes = loadConfiguration.useAvroLogicalTypes;
9396
}
9497

9598
private Builder(com.google.api.services.bigquery.model.JobConfiguration configurationPb) {
@@ -156,6 +159,7 @@ private Builder(com.google.api.services.bigquery.model.JobConfiguration configur
156159
this.clustering = Clustering.fromPb(loadConfigurationPb.getClustering());
157160
}
158161
this.autodetect = loadConfigurationPb.getAutodetect();
162+
this.useAvroLogicalTypes = loadConfigurationPb.getUseAvroLogicalTypes();
159163
if (loadConfigurationPb.getDestinationEncryptionConfiguration() != null) {
160164
this.destinationEncryptionConfiguration =
161165
new EncryptionConfiguration.Builder(
@@ -231,6 +235,12 @@ public Builder setClustering(Clustering clustering) {
231235
return this;
232236
}
233237

238+
@Override
239+
public Builder setUseAvroLogicalTypes(Boolean useAvroLogicalTypes) {
240+
this.useAvroLogicalTypes = useAvroLogicalTypes;
241+
return this;
242+
}
243+
234244
/**
235245
* Sets the fully-qualified URIs that point to source data in Google Cloud Storage (e.g.
236246
* gs://bucket/path). Each URI can contain one '*' wildcard character and it must come after the
@@ -275,6 +285,7 @@ private LoadJobConfiguration(Builder builder) {
275285
this.destinationEncryptionConfiguration = builder.destinationEncryptionConfiguration;
276286
this.timePartitioning = builder.timePartitioning;
277287
this.clustering = builder.clustering;
288+
this.useAvroLogicalTypes = builder.useAvroLogicalTypes;
278289
}
279290

280291
@Override
@@ -357,6 +368,11 @@ public Clustering getClustering() {
357368
return clustering;
358369
}
359370

371+
@Override
372+
public Boolean getUseAvroLogicalTypes() {
373+
return useAvroLogicalTypes;
374+
}
375+
360376
@Override
361377
public List<JobInfo.SchemaUpdateOption> getSchemaUpdateOptions() {
362378
return schemaUpdateOptions;
@@ -383,7 +399,8 @@ ToStringHelper toStringHelper() {
383399
.add("schemaUpdateOptions", schemaUpdateOptions)
384400
.add("autodetect", autodetect)
385401
.add("timePartitioning", timePartitioning)
386-
.add("clustering", clustering);
402+
.add("clustering", clustering)
403+
.add("useAvroLogicalTypes", useAvroLogicalTypes);
387404
}
388405

389406
@Override
@@ -464,6 +481,7 @@ com.google.api.services.bigquery.model.JobConfiguration toPb() {
464481
if (clustering != null) {
465482
loadConfigurationPb.setClustering(clustering.toPb());
466483
}
484+
loadConfigurationPb.setUseAvroLogicalTypes(useAvroLogicalTypes);
467485
return new com.google.api.services.bigquery.model.JobConfiguration()
468486
.setLoad(loadConfigurationPb);
469487
}

‎branches/autosynth-firestore/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/WriteChannelConfiguration.java‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ public final class WriteChannelConfiguration implements LoadConfiguration, Seria
5252
private final EncryptionConfiguration destinationEncryptionConfiguration;
5353
private final TimePartitioning timePartitioning;
5454
private final Clustering clustering;
55+
private final Boolean useAvroLogicalTypes;
5556

5657
public static final class Builder implements LoadConfiguration.Builder {
5758

@@ -68,6 +69,7 @@ public static final class Builder implements LoadConfiguration.Builder {
6869
private EncryptionConfiguration destinationEncryptionConfiguration;
6970
private TimePartitioning timePartitioning;
7071
private Clustering clustering;
72+
private Boolean useAvroLogicalTypes;
7173

7274
private Builder() {}
7375

@@ -86,6 +88,7 @@ private Builder(WriteChannelConfiguration writeChannelConfiguration) {
8688
writeChannelConfiguration.destinationEncryptionConfiguration;
8789
this.timePartitioning = writeChannelConfiguration.timePartitioning;
8890
this.clustering = writeChannelConfiguration.clustering;
91+
this.useAvroLogicalTypes = writeChannelConfiguration.useAvroLogicalTypes;
8992
}
9093

9194
private Builder(com.google.api.services.bigquery.model.JobConfiguration configurationPb) {
@@ -158,6 +161,7 @@ private Builder(com.google.api.services.bigquery.model.JobConfiguration configur
158161
if (loadConfigurationPb.getClustering() != null) {
159162
this.clustering = Clustering.fromPb(loadConfigurationPb.getClustering());
160163
}
164+
this.useAvroLogicalTypes = loadConfigurationPb.getUseAvroLogicalTypes();
161165
}
162166

163167
@Override
@@ -240,6 +244,12 @@ public Builder setClustering(Clustering clustering) {
240244
return this;
241245
}
242246

247+
@Override
248+
public Builder setUseAvroLogicalTypes(Boolean useAvroLogicalTypes) {
249+
this.useAvroLogicalTypes = useAvroLogicalTypes;
250+
return this;
251+
}
252+
243253
@Override
244254
public WriteChannelConfiguration build() {
245255
return new WriteChannelConfiguration(this);
@@ -260,6 +270,7 @@ protected WriteChannelConfiguration(Builder builder) {
260270
this.destinationEncryptionConfiguration = builder.destinationEncryptionConfiguration;
261271
this.timePartitioning = builder.timePartitioning;
262272
this.clustering = builder.clustering;
273+
this.useAvroLogicalTypes = builder.useAvroLogicalTypes;
263274
}
264275

265276
@Override
@@ -339,6 +350,11 @@ public Clustering getClustering() {
339350
return clustering;
340351
}
341352

353+
@Override
354+
public Boolean getUseAvroLogicalTypes() {
355+
return useAvroLogicalTypes;
356+
}
357+
342358
@Override
343359
public Builder toBuilder() {
344360
return new Builder(this);
@@ -358,7 +374,8 @@ MoreObjects.ToStringHelper toStringHelper() {
358374
.add("schemaUpdateOptions", schemaUpdateOptions)
359375
.add("autodetect", autodetect)
360376
.add("timePartitioning", timePartitioning)
361-
.add("clustering", clustering);
377+
.add("clustering", clustering)
378+
.add("useAvroLogicalTypes", useAvroLogicalTypes);
362379
}
363380

364381
@Override
@@ -387,7 +404,8 @@ public int hashCode() {
387404
schemaUpdateOptions,
388405
autodetect,
389406
timePartitioning,
390-
clustering);
407+
clustering,
408+
useAvroLogicalTypes);
391409
}
392410

393411
WriteChannelConfiguration setProjectId(String projectId) {
@@ -452,6 +470,7 @@ com.google.api.services.bigquery.model.JobConfiguration toPb() {
452470
if (clustering != null) {
453471
loadConfigurationPb.setClustering(clustering.toPb());
454472
}
473+
loadConfigurationPb.setUseAvroLogicalTypes(useAvroLogicalTypes);
455474
return new com.google.api.services.bigquery.model.JobConfiguration()
456475
.setLoad(loadConfigurationPb);
457476
}

‎branches/autosynth-firestore/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ public class LoadJobConfigurationTest {
5252
ImmutableList.of(SchemaUpdateOption.ALLOW_FIELD_ADDITION);
5353
private static final Schema TABLE_SCHEMA = Schema.of(FIELD_SCHEMA);
5454
private static final Boolean AUTODETECT = true;
55+
private static final Boolean USERAVROLOGICALTYPES = true;
5556
private static final EncryptionConfiguration JOB_ENCRYPTION_CONFIGURATION =
5657
EncryptionConfiguration.newBuilder().setKmsKeyName("KMS_KEY_1").build();
5758
private static final TimePartitioning TIME_PARTITIONING = TimePartitioning.of(Type.DAY);
@@ -86,6 +87,21 @@ public class LoadJobConfigurationTest {
8687
.setSchemaUpdateOptions(SCHEMA_UPDATE_OPTIONS)
8788
.setAutodetect(AUTODETECT)
8889
.build();
90+
private static final LoadJobConfiguration LOAD_CONFIGURATION_AVRO =
91+
LoadJobConfiguration.newBuilder(TABLE_ID, SOURCE_URIS)
92+
.setCreateDisposition(CREATE_DISPOSITION)
93+
.setWriteDisposition(WRITE_DISPOSITION)
94+
.setFormatOptions(FormatOptions.avro())
95+
.setIgnoreUnknownValues(IGNORE_UNKNOWN_VALUES)
96+
.setMaxBadRecords(MAX_BAD_RECORDS)
97+
.setSchema(TABLE_SCHEMA)
98+
.setSchemaUpdateOptions(SCHEMA_UPDATE_OPTIONS)
99+
.setAutodetect(AUTODETECT)
100+
.setDestinationEncryptionConfiguration(JOB_ENCRYPTION_CONFIGURATION)
101+
.setTimePartitioning(TIME_PARTITIONING)
102+
.setClustering(CLUSTERING)
103+
.setUseAvroLogicalTypes(USERAVROLOGICALTYPES)
104+
.build();
89105

90106
@Test
91107
public void testToBuilder() {
@@ -109,6 +125,17 @@ public void testToBuilder() {
109125
assertEquals("newTable", configurationBackup.getDestinationTable().getTable());
110126
configurationBackup = configurationBackup.toBuilder().setDestinationTable(TABLE_ID).build();
111127
compareLoadJobConfiguration(LOAD_CONFIGURATION_BACKUP, configurationBackup);
128+
129+
compareLoadJobConfiguration(
130+
LOAD_CONFIGURATION_AVRO, LOAD_CONFIGURATION_AVRO.toBuilder().build());
131+
LoadJobConfiguration configurationAvro =
132+
LOAD_CONFIGURATION_AVRO
133+
.toBuilder()
134+
.setDestinationTable(TableId.of("dataset", "newTable"))
135+
.build();
136+
assertEquals("newTable", configurationAvro.getDestinationTable().getTable());
137+
configurationAvro = configurationAvro.toBuilder().setDestinationTable(TABLE_ID).build();
138+
compareLoadJobConfiguration(LOAD_CONFIGURATION_AVRO, configurationAvro);
112139
}
113140

114141
@Test
@@ -188,5 +215,6 @@ private void compareLoadJobConfiguration(
188215
value.getDestinationEncryptionConfiguration());
189216
assertEquals(expected.getTimePartitioning(), value.getTimePartitioning());
190217
assertEquals(expected.getClustering(), value.getClustering());
218+
assertEquals(expected.getUseAvroLogicalTypes(), value.getUseAvroLogicalTypes());
191219
}
192220
}

‎branches/autosynth-firestore/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/WriteChannelConfigurationTest.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ public class WriteChannelConfigurationTest {
4949
.build();
5050
private static final Schema TABLE_SCHEMA = Schema.of(FIELD_SCHEMA);
5151
private static final Boolean AUTODETECT = true;
52+
private static final Boolean USERAVROLOGICALTYPES = true;
5253
private static final List<JobInfo.SchemaUpdateOption> SCHEMA_UPDATE_OPTIONS =
5354
ImmutableList.of(JobInfo.SchemaUpdateOption.ALLOW_FIELD_ADDITION);
5455
private static final TimePartitioning TIME_PARTITIONING = TimePartitioning.of(Type.DAY);
@@ -84,6 +85,21 @@ public class WriteChannelConfigurationTest {
8485
.setSchemaUpdateOptions(SCHEMA_UPDATE_OPTIONS)
8586
.setAutodetect(AUTODETECT)
8687
.build();
88+
private static final WriteChannelConfiguration LOAD_CONFIGURATION_AVRO =
89+
WriteChannelConfiguration.newBuilder(TABLE_ID)
90+
.setCreateDisposition(CREATE_DISPOSITION)
91+
.setWriteDisposition(WRITE_DISPOSITION)
92+
.setNullMarker(NULL_MARKER)
93+
.setFormatOptions(FormatOptions.avro())
94+
.setIgnoreUnknownValues(IGNORE_UNKNOWN_VALUES)
95+
.setMaxBadRecords(MAX_BAD_RECORDS)
96+
.setSchema(TABLE_SCHEMA)
97+
.setSchemaUpdateOptions(SCHEMA_UPDATE_OPTIONS)
98+
.setAutodetect(AUTODETECT)
99+
.setTimePartitioning(TIME_PARTITIONING)
100+
.setClustering(CLUSTERING)
101+
.setUseAvroLogicalTypes(USERAVROLOGICALTYPES)
102+
.build();
87103

88104
@Test
89105
public void testToBuilder() {
@@ -96,6 +112,16 @@ public void testToBuilder() {
96112
assertEquals("newTable", configuration.getDestinationTable().getTable());
97113
configuration = configuration.toBuilder().setDestinationTable(TABLE_ID).build();
98114
compareLoadConfiguration(LOAD_CONFIGURATION_CSV, configuration);
115+
116+
compareLoadConfiguration(LOAD_CONFIGURATION_AVRO, LOAD_CONFIGURATION_AVRO.toBuilder().build());
117+
WriteChannelConfiguration configurationAvro =
118+
LOAD_CONFIGURATION_AVRO
119+
.toBuilder()
120+
.setDestinationTable(TableId.of("dataset", "newTable"))
121+
.build();
122+
assertEquals("newTable", configurationAvro.getDestinationTable().getTable());
123+
configurationAvro = configurationAvro.toBuilder().setDestinationTable(TABLE_ID).build();
124+
compareLoadConfiguration(LOAD_CONFIGURATION_AVRO, configurationAvro);
99125
}
100126

101127
@Test
@@ -191,5 +217,6 @@ private void compareLoadConfiguration(
191217
assertEquals(expected.getAutodetect(), value.getAutodetect());
192218
assertEquals(expected.getTimePartitioning(), value.getTimePartitioning());
193219
assertEquals(expected.getClustering(), value.getClustering());
220+
assertEquals(expected.getUseAvroLogicalTypes(), value.getUseAvroLogicalTypes());
194221
}
195222
}

0 commit comments

Comments
 (0)