Skip to content

Commit 4d9b8fa

Browse files
tcoffee-googletcoffee-google
committed
Add null marker to BigQuery job load configuration
Incorporates update to BigQuery API v2. Addresses #2061.
1 parent 7a43064 commit 4d9b8fa

5 files changed

Lines changed: 76 additions & 2 deletions

File tree

‎google-cloud-bigquery/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@
3535
<dependency>
3636
<groupId>com.google.apis</groupId>
3737
<artifactId>google-api-services-bigquery</artifactId>
38-
<version>v2-rev330-1.22.0</version>
38+
<version>v2-rev347-1.22.0</version>
3939
<scope>compile</scope>
4040
<exclusions>
4141
<exclusion>

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadConfiguration.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,17 @@ interface Builder {
6969
Builder setFormatOptions(FormatOptions formatOptions);
7070

7171

72+
/**
73+
* Sets the string that represents a null value in a CSV file. For example, if you specify "\N",
74+
* BigQuery interprets "\N" as a null value when loading a CSV file. The default value is the
75+
* empty string. If you set this property to a custom value, BigQuery throws an error if an
76+
* empty string is present for all data types except for {@code STRING} and {@code BYTE}. For
77+
* {@code STRING} and {@code BYTE} columns, BigQuery interprets the empty string as an empty
78+
* value.
79+
*/
80+
Builder setNullMarker(String nullMarker);
81+
82+
7283
/**
7384
* Sets the maximum number of bad records that BigQuery can ignore when running the job. If the
7485
* number of bad records exceeds this value, an invalid error is returned in the job result.
@@ -134,6 +145,15 @@ interface Builder {
134145
WriteDisposition getWriteDisposition();
135146

136147

148+
/**
149+
* Returns the string that represents a null value in a CSV file.
150+
*
151+
* @see <a href="https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.load.nullMarker">
152+
* Null Marker</a>
153+
*/
154+
String getNullMarker();
155+
156+
137157
/**
138158
* Returns additional properties used to parse CSV data (used when {@link #getFormat()} is set
139159
* to CSV). Returns {@code null} if not set.

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ public final class LoadJobConfiguration extends JobConfiguration implements Load
4040
private final JobInfo.CreateDisposition createDisposition;
4141
private final JobInfo.WriteDisposition writeDisposition;
4242
private final FormatOptions formatOptions;
43+
private final String nullMarker;
4344
private final Integer maxBadRecords;
4445
private final Schema schema;
4546
private final Boolean ignoreUnknownValues;
@@ -55,6 +56,7 @@ public static final class Builder
5556
private JobInfo.CreateDisposition createDisposition;
5657
private JobInfo.WriteDisposition writeDisposition;
5758
private FormatOptions formatOptions;
59+
private String nullMarker;
5860
private Integer maxBadRecords;
5961
private Schema schema;
6062
private Boolean ignoreUnknownValues;
@@ -72,6 +74,7 @@ private Builder(LoadJobConfiguration loadConfiguration) {
7274
this.createDisposition = loadConfiguration.createDisposition;
7375
this.writeDisposition = loadConfiguration.writeDisposition;
7476
this.formatOptions = loadConfiguration.formatOptions;
77+
this.nullMarker = loadConfiguration.nullMarker;
7578
this.maxBadRecords = loadConfiguration.maxBadRecords;
7679
this.schema = loadConfiguration.schema;
7780
this.ignoreUnknownValues = loadConfiguration.ignoreUnknownValues;
@@ -95,6 +98,9 @@ private Builder(com.google.api.services.bigquery.model.JobConfiguration configur
9598
if (loadConfigurationPb.getSourceFormat() != null) {
9699
this.formatOptions = FormatOptions.of(loadConfigurationPb.getSourceFormat());
97100
}
101+
if (loadConfigurationPb.getNullMarker() != null) {
102+
this.nullMarker = loadConfigurationPb.getNullMarker();
103+
}
98104
if (loadConfigurationPb.getAllowJaggedRows() != null
99105
|| loadConfigurationPb.getAllowQuotedNewlines() != null
100106
|| loadConfigurationPb.getEncoding() != null
@@ -164,6 +170,13 @@ public Builder setFormatOptions(FormatOptions formatOptions) {
164170
}
165171

166172

173+
@Override
174+
public Builder setNullMarker(String nullMarker) {
175+
this.nullMarker = nullMarker;
176+
return this;
177+
}
178+
179+
167180
@Override
168181
public Builder setMaxBadRecords(Integer maxBadRecords) {
169182
this.maxBadRecords = maxBadRecords;
@@ -219,6 +232,7 @@ private LoadJobConfiguration(Builder builder) {
219232
this.createDisposition = builder.createDisposition;
220233
this.writeDisposition = builder.writeDisposition;
221234
this.formatOptions = builder.formatOptions;
235+
this.nullMarker = builder.nullMarker;
222236
this.maxBadRecords = builder.maxBadRecords;
223237
this.schema = builder.schema;
224238
this.ignoreUnknownValues = builder.ignoreUnknownValues;
@@ -245,6 +259,12 @@ public JobInfo.WriteDisposition getWriteDisposition() {
245259
}
246260

247261

262+
@Override
263+
public String getNullMarker() {
264+
return nullMarker;
265+
}
266+
267+
248268
@Override
249269
public CsvOptions getCsvOptions() {
250270
return formatOptions instanceof CsvOptions ? (CsvOptions) formatOptions : null;
@@ -309,6 +329,7 @@ ToStringHelper toStringHelper() {
309329
.add("createDisposition", createDisposition)
310330
.add("writeDisposition", writeDisposition)
311331
.add("formatOptions", formatOptions)
332+
.add("nullMarker", nullMarker)
312333
.add("maxBadRecords", maxBadRecords)
313334
.add("schema", schema)
314335
.add("ignoreUnknownValue", ignoreUnknownValues)
@@ -344,6 +365,9 @@ com.google.api.services.bigquery.model.JobConfiguration toPb() {
344365
if (writeDisposition != null) {
345366
loadConfigurationPb.setWriteDisposition(writeDisposition.toString());
346367
}
368+
if (nullMarker != null) {
369+
loadConfigurationPb.setNullMarker(nullMarker);
370+
}
347371
if (getCsvOptions() != null) {
348372
CsvOptions csvOptions = getCsvOptions();
349373
loadConfigurationPb.setFieldDelimiter(csvOptions.getFieldDelimiter())

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/WriteChannelConfiguration.java‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ public final class WriteChannelConfiguration implements LoadConfiguration, Seria
4343
private final CreateDisposition createDisposition;
4444
private final WriteDisposition writeDisposition;
4545
private final FormatOptions formatOptions;
46+
private final String nullMarker;
4647
private final Integer maxBadRecords;
4748
private final Schema schema;
4849
private final Boolean ignoreUnknownValues;
@@ -55,6 +56,7 @@ public static final class Builder implements LoadConfiguration.Builder {
5556
private CreateDisposition createDisposition;
5657
private WriteDisposition writeDisposition;
5758
private FormatOptions formatOptions;
59+
private String nullMarker;
5860
private Integer maxBadRecords;
5961
private Schema schema;
6062
private Boolean ignoreUnknownValues;
@@ -68,6 +70,7 @@ private Builder(WriteChannelConfiguration writeChannelConfiguration) {
6870
this.createDisposition = writeChannelConfiguration.createDisposition;
6971
this.writeDisposition = writeChannelConfiguration.writeDisposition;
7072
this.formatOptions = writeChannelConfiguration.formatOptions;
73+
this.nullMarker = writeChannelConfiguration.nullMarker;
7174
this.maxBadRecords = writeChannelConfiguration.maxBadRecords;
7275
this.schema = writeChannelConfiguration.schema;
7376
this.ignoreUnknownValues = writeChannelConfiguration.ignoreUnknownValues;
@@ -88,6 +91,9 @@ private Builder(com.google.api.services.bigquery.model.JobConfiguration configur
8891
if (loadConfigurationPb.getSourceFormat() != null) {
8992
this.formatOptions = FormatOptions.of(loadConfigurationPb.getSourceFormat());
9093
}
94+
if (loadConfigurationPb.getNullMarker() != null) {
95+
this.nullMarker = loadConfigurationPb.getNullMarker();
96+
}
9197
if (loadConfigurationPb.getAllowJaggedRows() != null
9298
|| loadConfigurationPb.getAllowQuotedNewlines() != null
9399
|| loadConfigurationPb.getEncoding() != null
@@ -158,6 +164,13 @@ public Builder setFormatOptions(FormatOptions formatOptions) {
158164
}
159165

160166

167+
@Override
168+
public Builder setNullMarker(String nullMarker) {
169+
this.nullMarker = nullMarker;
170+
return this;
171+
}
172+
173+
161174
@Override
162175
public Builder setMaxBadRecords(Integer maxBadRecords) {
163176
this.maxBadRecords = maxBadRecords;
@@ -202,6 +215,7 @@ protected WriteChannelConfiguration(Builder builder) {
202215
this.createDisposition = builder.createDisposition;
203216
this.writeDisposition = builder.writeDisposition;
204217
this.formatOptions = builder.formatOptions;
218+
this.nullMarker = builder.nullMarker;
205219
this.maxBadRecords = builder.maxBadRecords;
206220
this.schema = builder.schema;
207221
this.ignoreUnknownValues = builder.ignoreUnknownValues;
@@ -228,6 +242,12 @@ public WriteDisposition getWriteDisposition() {
228242
}
229243

230244

245+
@Override
246+
public String getNullMarker() {
247+
return nullMarker;
248+
}
249+
250+
231251
@Override
232252
public CsvOptions getCsvOptions() {
233253
return formatOptions instanceof CsvOptions ? (CsvOptions) formatOptions : null;
@@ -284,6 +304,7 @@ MoreObjects.ToStringHelper toStringHelper() {
284304
.add("createDisposition", createDisposition)
285305
.add("writeDisposition", writeDisposition)
286306
.add("formatOptions", formatOptions)
307+
.add("nullMarker", nullMarker)
287308
.add("maxBadRecords", maxBadRecords)
288309
.add("schema", schema)
289310
.add("ignoreUnknownValue", ignoreUnknownValues)
@@ -306,7 +327,7 @@ public boolean equals(Object obj) {
306327
@Override
307328
public int hashCode() {
308329
return Objects.hash(destinationTable, createDisposition, writeDisposition, formatOptions,
309-
maxBadRecords, schema, ignoreUnknownValues, schemaUpdateOptions, autodetect);
330+
nullMarker, maxBadRecords, schema, ignoreUnknownValues, schemaUpdateOptions, autodetect);
310331
}
311332

312333
WriteChannelConfiguration setProjectId(String projectId) {
@@ -322,6 +343,9 @@ com.google.api.services.bigquery.model.JobConfiguration toPb() {
322343
if (writeDisposition != null) {
323344
loadConfigurationPb.setWriteDisposition(writeDisposition.toString());
324345
}
346+
if (nullMarker != null) {
347+
loadConfigurationPb.setNullMarker(nullMarker);
348+
}
325349
if (getCsvOptions() != null) {
326350
CsvOptions csvOptions = getCsvOptions();
327351
loadConfigurationPb.setFieldDelimiter(csvOptions.getFieldDelimiter())

‎google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/WriteChannelConfigurationTest.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ public class WriteChannelConfigurationTest {
3838
private static final TableId TABLE_ID = TableId.of("dataset", "table");
3939
private static final CreateDisposition CREATE_DISPOSITION = CreateDisposition.CREATE_IF_NEEDED;
4040
private static final WriteDisposition WRITE_DISPOSITION = WriteDisposition.WRITE_APPEND;
41+
private static final String NULL_MARKER = "\\N";
4142
private static final Integer MAX_BAD_RECORDS = 42;
4243
private static final String FORMAT = "CSV";
4344
private static final Boolean IGNORE_UNKNOWN_VALUES = true;
@@ -53,6 +54,7 @@ public class WriteChannelConfigurationTest {
5354
WriteChannelConfiguration.newBuilder(TABLE_ID)
5455
.setCreateDisposition(CREATE_DISPOSITION)
5556
.setWriteDisposition(WRITE_DISPOSITION)
57+
.setNullMarker(NULL_MARKER)
5658
.setFormatOptions(CSV_OPTIONS)
5759
.setIgnoreUnknownValues(IGNORE_UNKNOWN_VALUES)
5860
.setMaxBadRecords(MAX_BAD_RECORDS)
@@ -108,6 +110,7 @@ public void testBuilder() {
108110
assertEquals(TABLE_ID, LOAD_CONFIGURATION_CSV.getDestinationTable());
109111
assertEquals(CREATE_DISPOSITION, LOAD_CONFIGURATION_CSV.getCreateDisposition());
110112
assertEquals(WRITE_DISPOSITION, LOAD_CONFIGURATION_CSV.getWriteDisposition());
113+
assertEquals(NULL_MARKER, LOAD_CONFIGURATION_CSV.getNullMarker());
111114
assertEquals(CSV_OPTIONS, LOAD_CONFIGURATION_CSV.getCsvOptions());
112115
assertEquals(FORMAT, LOAD_CONFIGURATION_CSV.getFormat());
113116
assertEquals(IGNORE_UNKNOWN_VALUES, LOAD_CONFIGURATION_CSV.ignoreUnknownValues());
@@ -120,6 +123,7 @@ public void testBuilder() {
120123
WriteChannelConfiguration.newBuilder(TABLE_ID, CSV_OPTIONS)
121124
.setCreateDisposition(CREATE_DISPOSITION)
122125
.setWriteDisposition(WRITE_DISPOSITION)
126+
.setNullMarker(NULL_MARKER)
123127
.setIgnoreUnknownValues(IGNORE_UNKNOWN_VALUES)
124128
.setMaxBadRecords(MAX_BAD_RECORDS)
125129
.setSchemaUpdateOptions(SCHEMA_UPDATE_OPTIONS)
@@ -129,6 +133,7 @@ public void testBuilder() {
129133
assertEquals(TABLE_ID, loadConfigurationCSV.getDestinationTable());
130134
assertEquals(CREATE_DISPOSITION, loadConfigurationCSV.getCreateDisposition());
131135
assertEquals(WRITE_DISPOSITION, loadConfigurationCSV.getWriteDisposition());
136+
assertEquals(NULL_MARKER, loadConfigurationCSV.getNullMarker());
132137
assertEquals(CSV_OPTIONS, loadConfigurationCSV.getCsvOptions());
133138
assertEquals(FORMAT, loadConfigurationCSV.getFormat());
134139
assertEquals(IGNORE_UNKNOWN_VALUES, loadConfigurationCSV.ignoreUnknownValues());
@@ -158,6 +163,7 @@ private void compareLoadConfiguration(WriteChannelConfiguration expected,
158163
assertEquals(expected.getDestinationTable(), value.getDestinationTable());
159164
assertEquals(expected.getCreateDisposition(), value.getCreateDisposition());
160165
assertEquals(expected.getWriteDisposition(), value.getWriteDisposition());
166+
assertEquals(expected.getNullMarker(), value.getNullMarker());
161167
assertEquals(expected.getCsvOptions(), value.getCsvOptions());
162168
assertEquals(expected.getFormat(), value.getFormat());
163169
assertEquals(expected.ignoreUnknownValues(), value.ignoreUnknownValues());

0 commit comments

Comments
 (0)