Skip to content
This repository was archived by the owner on Mar 23, 2026. It is now read-only.

Commit 8c488e6

Browse files
Neenu1995gcf-owl-bot[bot]
Neenu1995
and
gcf-owl-bot[bot]
authored
feat: add reference file schema option for federated formats (#2269)
* feat: add reference file schema option for federated formats * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * chore: fix clirr check * chore: add assertion to tests * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * chore: add create external table tests * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * chore: delete table for external table after testing * comment * cleanup * chore: remove enforced login from library code * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
1 parent 4dd963b commit 8c488e6

4 files changed

Lines changed: 257 additions & 1 deletion

File tree

‎google-cloud-bigquery/clirr-ignored-differences.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,4 +14,9 @@
1414
<method>com.google.api.services.bigquery.model.GetQueryResultsResponse getQueryResultsWithRowLimit(java.lang.String, java.lang.String, java.lang.String, java.lang.Integer)</method>
1515
<justification>getQueryResultsWithRowLimit is just used by ConnectionImpl at the moment so it should be fine to update the signature instead of writing an overloaded method</justification>
1616
</difference>
17+
<difference>
18+
<differenceType>7013</differenceType>
19+
<className>com/google/cloud/bigquery/ExternalTableDefinition*</className>
20+
<method>*ReferenceFileSchemaUri(*)</method>
21+
</difference>
1722
</differences>

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,14 @@ public Builder setHivePartitioningOptions(HivePartitioningOptions hivePartitioni
157157
return setHivePartitioningOptionsInner(hivePartitioningOptions);
158158
};
159159

160+
/**
161+
* When creating an external table, the user can provide a reference file with the table schema.
162+
* This is enabled for the following formats: AVRO, PARQUET, ORC.
163+
*
164+
* @param referenceFileSchemaUri or {@code null} for none
165+
*/
166+
public abstract Builder setReferenceFileSchemaUri(String referenceFileSchemaUri);
167+
160168
abstract Builder setHivePartitioningOptionsInner(
161169
HivePartitioningOptions hivePartitioningOptions);
162170

@@ -250,6 +258,9 @@ public <F extends FormatOptions> F getFormatOptions() {
250258
@Nullable
251259
public abstract Boolean getAutodetect();
252260

261+
@Nullable
262+
public abstract String getReferenceFileSchemaUri();
263+
253264
/**
254265
* [Experimental] Returns the HivePartitioningOptions when the data layout follows Hive
255266
* partitioning convention
@@ -317,6 +328,10 @@ com.google.api.services.bigquery.model.ExternalDataConfiguration toExternalDataC
317328
if (getAutodetect() != null) {
318329
externalConfigurationPb.setAutodetect(getAutodetect());
319330
}
331+
if (getReferenceFileSchemaUri() != null) {
332+
externalConfigurationPb.setReferenceFileSchemaUri(getReferenceFileSchemaUri());
333+
}
334+
320335
if (getHivePartitioningOptions() != null) {
321336
externalConfigurationPb.setHivePartitioningOptions(getHivePartitioningOptions().toPb());
322337
}
@@ -486,6 +501,9 @@ static ExternalTableDefinition fromPb(Table tablePb) {
486501
builder.setHivePartitioningOptions(
487502
HivePartitioningOptions.fromPb(externalDataConfiguration.getHivePartitioningOptions()));
488503
}
504+
if (externalDataConfiguration.getReferenceFileSchemaUri() != null) {
505+
builder.setReferenceFileSchemaUri(externalDataConfiguration.getReferenceFileSchemaUri());
506+
}
489507
}
490508
return builder.build();
491509
}
@@ -538,10 +556,14 @@ static ExternalTableDefinition fromExternalDataConfiguration(
538556
if (externalDataConfiguration.getAutodetect() != null) {
539557
builder.setAutodetect(externalDataConfiguration.getAutodetect());
540558
}
559+
if (externalDataConfiguration.getReferenceFileSchemaUri() != null) {
560+
builder.setReferenceFileSchemaUri(externalDataConfiguration.getReferenceFileSchemaUri());
561+
}
541562
if (externalDataConfiguration.getHivePartitioningOptions() != null) {
542563
builder.setHivePartitioningOptions(
543564
HivePartitioningOptions.fromPb(externalDataConfiguration.getHivePartitioningOptions()));
544565
}
566+
545567
return builder.build();
546568
}
547569
}

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java‎

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ public final class LoadJobConfiguration extends JobConfiguration implements Load
5656
private final Long jobTimeoutMs;
5757
private final RangePartitioning rangePartitioning;
5858
private final HivePartitioningOptions hivePartitioningOptions;
59+
private final String referenceFileSchemaUri;
5960

6061
public static final class Builder extends JobConfiguration.Builder<LoadJobConfiguration, Builder>
6162
implements LoadConfiguration.Builder {
@@ -81,6 +82,7 @@ public static final class Builder extends JobConfiguration.Builder<LoadJobConfig
8182
private Long jobTimeoutMs;
8283
private RangePartitioning rangePartitioning;
8384
private HivePartitioningOptions hivePartitioningOptions;
85+
private String referenceFileSchemaUri;
8486

8587
private Builder() {
8688
super(Type.LOAD);
@@ -109,6 +111,7 @@ private Builder(LoadJobConfiguration loadConfiguration) {
109111
this.jobTimeoutMs = loadConfiguration.jobTimeoutMs;
110112
this.rangePartitioning = loadConfiguration.rangePartitioning;
111113
this.hivePartitioningOptions = loadConfiguration.hivePartitioningOptions;
114+
this.referenceFileSchemaUri = loadConfiguration.referenceFileSchemaUri;
112115
}
113116

114117
private Builder(com.google.api.services.bigquery.model.JobConfiguration configurationPb) {
@@ -199,6 +202,9 @@ private Builder(com.google.api.services.bigquery.model.JobConfiguration configur
199202
this.hivePartitioningOptions =
200203
HivePartitioningOptions.fromPb(loadConfigurationPb.getHivePartitioningOptions());
201204
}
205+
if (loadConfigurationPb.getReferenceFileSchemaUri() != null) {
206+
this.referenceFileSchemaUri = loadConfigurationPb.getReferenceFileSchemaUri();
207+
}
202208
}
203209

204210
@Override
@@ -351,6 +357,17 @@ public Builder setHivePartitioningOptions(HivePartitioningOptions hivePartitioni
351357
return this;
352358
}
353359

360+
/**
361+
* When creating an external table, the user can provide a reference file with the table schema.
362+
* This is enabled for the following formats: AVRO, PARQUET, ORC.
363+
*
364+
* @param referenceFileSchemaUri or {@code null} for none
365+
*/
366+
public Builder setReferenceFileSchemaUri(String referenceFileSchemaUri) {
367+
this.referenceFileSchemaUri = referenceFileSchemaUri;
368+
return this;
369+
}
370+
354371
@Override
355372
public LoadJobConfiguration build() {
356373
return new LoadJobConfiguration(this);
@@ -379,6 +396,7 @@ private LoadJobConfiguration(Builder builder) {
379396
this.jobTimeoutMs = builder.jobTimeoutMs;
380397
this.rangePartitioning = builder.rangePartitioning;
381398
this.hivePartitioningOptions = builder.hivePartitioningOptions;
399+
this.referenceFileSchemaUri = builder.referenceFileSchemaUri;
382400
}
383401

384402
@Override
@@ -498,6 +516,10 @@ public HivePartitioningOptions getHivePartitioningOptions() {
498516
return hivePartitioningOptions;
499517
}
500518

519+
public String getReferenceFileSchemaUri() {
520+
return referenceFileSchemaUri;
521+
}
522+
501523
@Override
502524
public Builder toBuilder() {
503525
return new Builder(this);
@@ -525,7 +547,8 @@ ToStringHelper toStringHelper() {
525547
.add("labels", labels)
526548
.add("jobTimeoutMs", jobTimeoutMs)
527549
.add("rangePartitioning", rangePartitioning)
528-
.add("hivePartitioningOptions", hivePartitioningOptions);
550+
.add("hivePartitioningOptions", hivePartitioningOptions)
551+
.add("referenceFileSchemaUri", referenceFileSchemaUri);
529552
}
530553

531554
@Override
@@ -628,6 +651,10 @@ com.google.api.services.bigquery.model.JobConfiguration toPb() {
628651
if (hivePartitioningOptions != null) {
629652
loadConfigurationPb.setHivePartitioningOptions(hivePartitioningOptions.toPb());
630653
}
654+
if (referenceFileSchemaUri != null) {
655+
loadConfigurationPb.setReferenceFileSchemaUri(referenceFileSchemaUri);
656+
}
657+
631658
jobConfiguration.setLoad(loadConfigurationPb);
632659
return jobConfiguration;
633660
}

‎google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java‎

Lines changed: 202 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@
6565
import com.google.cloud.bigquery.ExternalTableDefinition;
6666
import com.google.cloud.bigquery.ExtractJobConfiguration;
6767
import com.google.cloud.bigquery.Field;
68+
import com.google.cloud.bigquery.Field.Mode;
6869
import com.google.cloud.bigquery.FieldList;
6970
import com.google.cloud.bigquery.FieldValue;
7071
import com.google.cloud.bigquery.FieldValue.Attribute;
@@ -4586,4 +4587,205 @@ public void testPreserveAsciiControlCharacters()
45864587
assertEquals("\u0000", row.get(0).getStringValue());
45874588
assertTrue(bigquery.delete(tableId));
45884589
}
4590+
4591+
@Test
4592+
public void testReferenceFileSchemaUriForAvro() {
4593+
try {
4594+
String destinationTableName = "test_reference_file_schema_avro";
4595+
TableId tableId = TableId.of(DATASET, destinationTableName);
4596+
Schema expectedSchema =
4597+
Schema.of(
4598+
Field.newBuilder("username", StandardSQLTypeName.STRING)
4599+
.setMode(Mode.NULLABLE)
4600+
.build(),
4601+
Field.newBuilder("tweet", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(),
4602+
Field.newBuilder("timestamp", StandardSQLTypeName.STRING)
4603+
.setMode(Mode.NULLABLE)
4604+
.build(),
4605+
Field.newBuilder("likes", StandardSQLTypeName.INT64).setMode(Mode.NULLABLE).build());
4606+
4607+
// By default, the table should have c-twitter schema because it is lexicographically last.
4608+
// a-twitter schema (username, tweet, timestamp, likes)
4609+
// b-twitter schema (username, tweet, timestamp)
4610+
// c-twitter schema (username, tweet)
4611+
List<String> SOURCE_URIS =
4612+
ImmutableList.of(
4613+
"gs://"
4614+
+ CLOUD_SAMPLES_DATA
4615+
+ "/bigquery/federated-formats-reference-file-schema/a-twitter.avro",
4616+
"gs://"
4617+
+ CLOUD_SAMPLES_DATA
4618+
+ "/bigquery/federated-formats-reference-file-schema/b-twitter.avro",
4619+
"gs://"
4620+
+ CLOUD_SAMPLES_DATA
4621+
+ "/bigquery/federated-formats-reference-file-schema/c-twitter.avro");
4622+
4623+
// Because referenceFileSchemaUri is set as a-twitter, the table will have a-twitter schema
4624+
String referenceFileSchema =
4625+
"gs://"
4626+
+ CLOUD_SAMPLES_DATA
4627+
+ "/bigquery/federated-formats-reference-file-schema/a-twitter.avro";
4628+
4629+
LoadJobConfiguration loadJobConfiguration =
4630+
LoadJobConfiguration.newBuilder(tableId, SOURCE_URIS, FormatOptions.avro())
4631+
.setReferenceFileSchemaUri(referenceFileSchema)
4632+
.build();
4633+
4634+
Job job = bigquery.create(JobInfo.of(loadJobConfiguration));
4635+
// Blocks until this load table job completes its execution, either failing or succeeding.
4636+
job = job.waitFor();
4637+
assertEquals(true, job.isDone());
4638+
4639+
LoadJobConfiguration actualLoadJobConfiguration = job.getConfiguration();
4640+
Table generatedTable = bigquery.getTable(actualLoadJobConfiguration.getDestinationTable());
4641+
4642+
assertEquals(expectedSchema, generatedTable.getDefinition().getSchema());
4643+
// clean up after test to avoid conflict with other tests
4644+
boolean success = bigquery.delete(tableId);
4645+
assertEquals(true, success);
4646+
} catch (BigQueryException | InterruptedException e) {
4647+
System.out.println("Column not added during load append \n" + e.toString());
4648+
}
4649+
}
4650+
4651+
@Test
4652+
public void testReferenceFileSchemaUriForParquet() {
4653+
try {
4654+
String destinationTableName = "test_reference_file_schema_parquet";
4655+
TableId tableId = TableId.of(DATASET, destinationTableName);
4656+
Schema expectedSchema =
4657+
Schema.of(
4658+
Field.newBuilder("username", StandardSQLTypeName.STRING)
4659+
.setMode(Mode.NULLABLE)
4660+
.build(),
4661+
Field.newBuilder("tweet", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(),
4662+
Field.newBuilder("timestamp", StandardSQLTypeName.STRING)
4663+
.setMode(Mode.NULLABLE)
4664+
.build(),
4665+
Field.newBuilder("likes", StandardSQLTypeName.INT64).setMode(Mode.NULLABLE).build());
4666+
4667+
// By default, the table should have c-twitter schema because it is lexicographically last.
4668+
// a-twitter schema (username, tweet, timestamp, likes)
4669+
// b-twitter schema (username, tweet, timestamp)
4670+
// c-twitter schema (username, tweet)
4671+
List<String> SOURCE_URIS =
4672+
ImmutableList.of(
4673+
"gs://"
4674+
+ CLOUD_SAMPLES_DATA
4675+
+ "/bigquery/federated-formats-reference-file-schema/a-twitter.parquet",
4676+
"gs://"
4677+
+ CLOUD_SAMPLES_DATA
4678+
+ "/bigquery/federated-formats-reference-file-schema/b-twitter.parquet",
4679+
"gs://"
4680+
+ CLOUD_SAMPLES_DATA
4681+
+ "/bigquery/federated-formats-reference-file-schema/c-twitter.parquet");
4682+
4683+
// Because referenceFileSchemaUri is set as a-twitter, the table will have a-twitter schema
4684+
String referenceFileSchema =
4685+
"gs://"
4686+
+ CLOUD_SAMPLES_DATA
4687+
+ "/bigquery/federated-formats-reference-file-schema/a-twitter.parquet";
4688+
4689+
LoadJobConfiguration loadJobConfiguration =
4690+
LoadJobConfiguration.newBuilder(tableId, SOURCE_URIS, FormatOptions.parquet())
4691+
.setReferenceFileSchemaUri(referenceFileSchema)
4692+
.build();
4693+
4694+
Job job = bigquery.create(JobInfo.of(loadJobConfiguration));
4695+
// Blocks until this load table job completes its execution, either failing or succeeding.
4696+
job = job.waitFor();
4697+
assertEquals(true, job.isDone());
4698+
LoadJobConfiguration actualLoadJobConfiguration = job.getConfiguration();
4699+
Table generatedTable = bigquery.getTable(actualLoadJobConfiguration.getDestinationTable());
4700+
4701+
assertEquals(expectedSchema, generatedTable.getDefinition().getSchema());
4702+
// clean up after test to avoid conflict with other tests
4703+
boolean success = bigquery.delete(tableId);
4704+
assertEquals(true, success);
4705+
} catch (BigQueryException | InterruptedException e) {
4706+
System.out.println("Column not added during load append \n" + e.toString());
4707+
}
4708+
}
4709+
4710+
@Test
4711+
public void testCreateExternalTableWithReferenceFileSchemaAvro() {
4712+
String destinationTableName = "test_create_external_table_reference_file_schema_avro";
4713+
TableId tableId = TableId.of(DATASET, destinationTableName);
4714+
Schema expectedSchema =
4715+
Schema.of(
4716+
Field.newBuilder("username", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(),
4717+
Field.newBuilder("tweet", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(),
4718+
Field.newBuilder("timestamp", StandardSQLTypeName.STRING)
4719+
.setMode(Mode.NULLABLE)
4720+
.build(),
4721+
Field.newBuilder("likes", StandardSQLTypeName.INT64).setMode(Mode.NULLABLE).build());
4722+
String CLOUD_SAMPLES_DATA = "cloud-samples-data";
4723+
4724+
// By default, the table should have c-twitter schema because it is lexicographically last.
4725+
// a-twitter schema (username, tweet, timestamp, likes)
4726+
// b-twitter schema (username, tweet, timestamp)
4727+
// c-twitter schema (username, tweet)
4728+
String SOURCE_URI =
4729+
"gs://" + CLOUD_SAMPLES_DATA + "/bigquery/federated-formats-reference-file-schema/*.avro";
4730+
4731+
// Because referenceFileSchemaUri is set as a-twitter, the table will have a-twitter schema
4732+
String referenceFileSchema =
4733+
"gs://"
4734+
+ CLOUD_SAMPLES_DATA
4735+
+ "/bigquery/federated-formats-reference-file-schema/a-twitter.avro";
4736+
4737+
ExternalTableDefinition externalTableDefinition =
4738+
ExternalTableDefinition.newBuilder(SOURCE_URI, FormatOptions.avro())
4739+
.setReferenceFileSchemaUri(referenceFileSchema)
4740+
.build();
4741+
TableInfo tableInfo = TableInfo.of(tableId, externalTableDefinition);
4742+
Table createdTable = bigquery.create(tableInfo);
4743+
Table generatedTable = bigquery.getTable(createdTable.getTableId());
4744+
assertEquals(expectedSchema, generatedTable.getDefinition().getSchema());
4745+
// clean up after test to avoid conflict with other tests
4746+
boolean success = bigquery.delete(tableId);
4747+
assertEquals(true, success);
4748+
}
4749+
4750+
@Test
4751+
public void testCreateExternalTableWithReferenceFileSchemaParquet() {
4752+
String destinationTableName = "test_create_external_table_reference_file_schema_parquet";
4753+
TableId tableId = TableId.of(DATASET, destinationTableName);
4754+
Schema expectedSchema =
4755+
Schema.of(
4756+
Field.newBuilder("username", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(),
4757+
Field.newBuilder("tweet", StandardSQLTypeName.STRING).setMode(Mode.NULLABLE).build(),
4758+
Field.newBuilder("timestamp", StandardSQLTypeName.STRING)
4759+
.setMode(Mode.NULLABLE)
4760+
.build(),
4761+
Field.newBuilder("likes", StandardSQLTypeName.INT64).setMode(Mode.NULLABLE).build());
4762+
String CLOUD_SAMPLES_DATA = "cloud-samples-data";
4763+
4764+
// By default, the table should have c-twitter schema because it is lexicographically last.
4765+
// a-twitter schema (username, tweet, timestamp, likes)
4766+
// b-twitter schema (username, tweet, timestamp)
4767+
// c-twitter schema (username, tweet)
4768+
String SOURCE_URI =
4769+
"gs://"
4770+
+ CLOUD_SAMPLES_DATA
4771+
+ "/bigquery/federated-formats-reference-file-schema/*.parquet";
4772+
4773+
// Because referenceFileSchemaUri is set as a-twitter, the table will have a-twitter schema
4774+
String referenceFileSchema =
4775+
"gs://"
4776+
+ CLOUD_SAMPLES_DATA
4777+
+ "/bigquery/federated-formats-reference-file-schema/a-twitter.parquet";
4778+
4779+
ExternalTableDefinition externalTableDefinition =
4780+
ExternalTableDefinition.newBuilder(SOURCE_URI, FormatOptions.parquet())
4781+
.setReferenceFileSchemaUri(referenceFileSchema)
4782+
.build();
4783+
TableInfo tableInfo = TableInfo.of(tableId, externalTableDefinition);
4784+
Table createdTable = bigquery.create(tableInfo);
4785+
Table generatedTable = bigquery.getTable(createdTable.getTableId());
4786+
assertEquals(expectedSchema, generatedTable.getDefinition().getSchema());
4787+
// clean up after test to avoid conflict with other tests
4788+
boolean success = bigquery.delete(tableId);
4789+
assertEquals(true, success);
4790+
}
45894791
}

0 commit comments

Comments
 (0)