Skip to content
This repository was archived by the owner on Mar 23, 2026. It is now read-only.

Commit 4165e55

Browse files
davidrabinowitzdavidrabinowitz
authored
feat: add field in HivePartitioningOptions (#2678)
Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly: - [ v] Make sure to open an issue as a [bug/issue](https://togithub.com/googleapis/java-bigquery/issues/new/choose) before writing your code! That way we can discuss the change, evaluate designs, and agree on the general idea - [ v] Ensure the tests and linter pass - [ v] Code coverage does not decrease (if any source code was changed) - [v ] Appropriate docs were updated (if necessary) Fixes #2677 ☕️
1 parent e1326c8 commit 4165e55

4 files changed

Lines changed: 79 additions & 4 deletions

File tree

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/HivePartitioningOptions.java‎

Lines changed: 30 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
package com.google.cloud.bigquery;
1818

1919
import com.google.common.base.MoreObjects;
20+
import java.util.List;
2021
import java.util.Objects;
2122

2223
/** HivePartitioningOptions currently supported types include: AVRO, CSV, JSON, ORC and Parquet. */
@@ -25,19 +26,22 @@ public final class HivePartitioningOptions {
2526
private final String mode;
2627
private final Boolean requirePartitionFilter;
2728
private final String sourceUriPrefix;
29+
private final List<String> fields;
2830

2931
public static final class Builder {
3032

3133
private String mode;
3234
private Boolean requirePartitionFilter;
3335
private String sourceUriPrefix;
36+
private List<String> fields;
3437

3538
private Builder() {}
3639

3740
private Builder(HivePartitioningOptions options) {
3841
this.mode = options.mode;
3942
this.requirePartitionFilter = options.requirePartitionFilter;
4043
this.sourceUriPrefix = options.sourceUriPrefix;
44+
this.fields = options.fields;
4145
}
4246

4347
/**
@@ -78,6 +82,19 @@ public Builder setSourceUriPrefix(String sourceUriPrefix) {
7882
return this;
7983
}
8084

85+
/**
86+
* [Output-only] For permanent external tables, this field is populated with the hive partition
87+
* keys in the order they were inferred.
88+
*
89+
* <p>The types of the partition keys can be deduced by checking the table schema (which will
90+
* include the partition keys). Not every API will populate this field in the output. For
91+
* example, Tables.Get will populate it, but Tables.List will not contain this field.
92+
*/
93+
public Builder setFields(List<String> fields) {
94+
this.fields = fields;
95+
return this;
96+
}
97+
8198
/** Creates a {@link HivePartitioningOptions} object. */
8299
public HivePartitioningOptions build() {
83100
return new HivePartitioningOptions(this);
@@ -88,6 +105,7 @@ private HivePartitioningOptions(Builder builder) {
88105
this.mode = builder.mode;
89106
this.requirePartitionFilter = builder.requirePartitionFilter;
90107
this.sourceUriPrefix = builder.sourceUriPrefix;
108+
this.fields = builder.fields;
91109
}
92110

93111
/* Returns the mode of hive partitioning */
@@ -108,6 +126,11 @@ public String getSourceUriPrefix() {
108126
return sourceUriPrefix;
109127
}
110128

129+
/* Returns the fields of hive partitioning */
130+
public List<String> getFields() {
131+
return fields;
132+
}
133+
111134
/** Returns a builder for the {@link HivePartitioningOptions} object. */
112135
public Builder toBuilder() {
113136
return new Builder(this);
@@ -135,13 +158,13 @@ public boolean equals(Object obj) {
135158
&& Objects.equals(mode, ((HivePartitioningOptions) obj).getMode())
136159
&& Objects.equals(
137160
requirePartitionFilter, ((HivePartitioningOptions) obj).getRequirePartitionFilter())
138-
&& Objects.equals(
139-
sourceUriPrefix, ((HivePartitioningOptions) obj).getSourceUriPrefix());
161+
&& Objects.equals(sourceUriPrefix, ((HivePartitioningOptions) obj).getSourceUriPrefix())
162+
&& Objects.equals(fields, ((HivePartitioningOptions) obj).getFields());
140163
}
141164

142165
@Override
143166
public int hashCode() {
144-
return Objects.hash(mode, sourceUriPrefix);
167+
return Objects.hash(mode, sourceUriPrefix, fields);
145168
}
146169

147170
com.google.api.services.bigquery.model.HivePartitioningOptions toPb() {
@@ -150,6 +173,7 @@ com.google.api.services.bigquery.model.HivePartitioningOptions toPb() {
150173
options.setMode(mode);
151174
options.setRequirePartitionFilter(requirePartitionFilter);
152175
options.setSourceUriPrefix(sourceUriPrefix);
176+
options.setFields(fields);
153177
return options;
154178
}
155179

@@ -165,6 +189,9 @@ static HivePartitioningOptions fromPb(
165189
if (options.getSourceUriPrefix() != null) {
166190
builder.setSourceUriPrefix(options.getSourceUriPrefix());
167191
}
192+
if (options.getFields() != null) {
193+
builder.setFields(options.getFields());
194+
}
168195
return builder.build();
169196
}
170197
}

‎google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/HivePartitioningOptionsTest.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,18 +18,22 @@
1818

1919
import static com.google.common.truth.Truth.assertThat;
2020

21+
import java.util.Arrays;
22+
import java.util.List;
2123
import org.junit.Test;
2224

2325
public class HivePartitioningOptionsTest {
2426

2527
private static final String MODE = "STRING";
2628
private static final String SOURCE_URI_PREFIX = "gs://bucket/path_to_table";
2729
private static final Boolean REQUIRE_PARTITION_FILTER = true;
30+
private static final List<String> FIELDS = Arrays.asList("FIELD1", "FIELD2");
2831
private static final HivePartitioningOptions HIVE_PARTITIONING_OPTIONS =
2932
HivePartitioningOptions.newBuilder()
3033
.setMode(MODE)
3134
.setRequirePartitionFilter(REQUIRE_PARTITION_FILTER)
3235
.setSourceUriPrefix(SOURCE_URI_PREFIX)
36+
.setFields(FIELDS)
3337
.build();
3438

3539
@Test

‎google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5233,4 +5233,48 @@ public void testCloneTableCopyJob() throws InterruptedException {
52335233
assertTrue(remoteTable.delete());
52345234
assertTrue(cloneTable.delete());
52355235
}
5236+
5237+
@Test
5238+
public void testHivePartitioningOptionsFieldsFieldExistence() throws InterruptedException {
5239+
String tableName = "hive_partitioned_external_table";
5240+
5241+
// Create data on GCS
5242+
String sourceDirectory = "bigquery/hive-partitioning-table/example";
5243+
BlobInfo blobInfo = BlobInfo.newBuilder(BUCKET, sourceDirectory + "/key=foo/data.json").build();
5244+
assertNotNull(
5245+
"Failed to upload JSON to GCS",
5246+
storage.create(blobInfo, "{\"name\":\"bar\"}".getBytes(StandardCharsets.UTF_8)));
5247+
String sourceUri = "gs://" + BUCKET + "/" + sourceDirectory + "/*";
5248+
String sourceUriPrefix = "gs://" + BUCKET + "/" + sourceDirectory + "/";
5249+
5250+
// Create the external table
5251+
HivePartitioningOptions hivePartitioningOptions =
5252+
HivePartitioningOptions.newBuilder()
5253+
.setMode("AUTO")
5254+
.setRequirePartitionFilter(true)
5255+
.setSourceUriPrefix(sourceUriPrefix)
5256+
.build();
5257+
5258+
TableId tableId = TableId.of(DATASET, tableName);
5259+
ExternalTableDefinition customTable =
5260+
ExternalTableDefinition.newBuilder(sourceUri, FormatOptions.json())
5261+
.setAutodetect(true)
5262+
.setHivePartitioningOptions(hivePartitioningOptions)
5263+
.build();
5264+
bigquery.create(TableInfo.of(tableId, customTable));
5265+
5266+
// Validate the existence of the field HivePartitioningOptions.fields
5267+
Table table = bigquery.getTable(tableId);
5268+
assertThat(table).isNotNull();
5269+
HivePartitioningOptions options =
5270+
((ExternalTableDefinition) table.getDefinition()).getHivePartitioningOptions();
5271+
List<String> fields = options.getFields();
5272+
assertThat(fields).isNotNull();
5273+
assertThat(fields).hasSize(1);
5274+
assertThat(fields).contains("key");
5275+
5276+
// Clean up
5277+
assertTrue(table.delete());
5278+
assertTrue(storage.delete(blobInfo.getBlobId()));
5279+
}
52365280
}

‎pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@
5454
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
5555
<github.global.server>github</github.global.server>
5656
<site.installationModule>google-cloud-bigquery-parent</site.installationModule>
57-
<google-api-services-bigquery.version>v2-rev20230408-2.0.0</google-api-services-bigquery.version>
57+
<google-api-services-bigquery.version>v2-rev20230422-2.0.0</google-api-services-bigquery.version>
5858
<google.cloud.shared-dependencies.version>3.8.0</google.cloud.shared-dependencies.version>
5959
<arrow.version>11.0.0</arrow.version>
6060
</properties>

0 commit comments

Comments
 (0)