Skip to content

Commit e8f9472

Browse files
shinfanshinfan
committed
Add DatastoreBackupOptions class for BigQuery
1 parent 619f1d8 commit e8f9472

8 files changed

Lines changed: 188 additions & 64 deletions

File tree

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/DatastoreBackupOptions.java‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
/*
2+
* Copyright 2017 Google Inc. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.bigquery;
18+
19+
import com.google.common.base.MoreObjects;
20+
import java.util.List;
21+
import java.util.Objects;
22+
23+
/**
24+
* Google BigQuery options for Cloud Datastore backup.
25+
*/
26+
public final class DatastoreBackupOptions extends FormatOptions {
27+
private List<String> projectionFields;
28+
29+
public static final class Builder {
30+
private List<String> projectionFields;
31+
32+
private Builder() {}
33+
34+
private Builder(DatastoreBackupOptions options) {
35+
projectionFields = options.projectionFields;
36+
}
37+
38+
/**
39+
* Sets which entity properties to load into BigQuery from a Cloud Datastore backup. Property
40+
* names are case sensitive and must be top-level properties.
41+
* If no properties are specified, BigQuery loads all properties. If any named property isn't
42+
* found in the Cloud Datastore backup, an invalid error is returned in the job result.
43+
*/
44+
Builder setProjectionFields(List<String> projectionFields) {
45+
this.projectionFields = projectionFields;
46+
return this;
47+
}
48+
49+
/**
50+
* Creates a {@code DatastoreBackupOptions} object.
51+
*/
52+
public DatastoreBackupOptions build() {
53+
return new DatastoreBackupOptions(this);
54+
}
55+
}
56+
57+
private DatastoreBackupOptions(Builder builder) {
58+
super(FormatOptions.DATASTORE_BACKUP);
59+
this.projectionFields = builder.projectionFields;
60+
}
61+
62+
/**
63+
* Returns the value of which entity properties to load into BigQuery from a Cloud Datastore
64+
* backup.
65+
*/
66+
public List<String> getProjectionFields() {
67+
return projectionFields;
68+
}
69+
70+
/**
71+
* Returns a builder for the {@code DatastoreBackupOptions} object.
72+
*/
73+
public Builder toBuilder() {
74+
return new Builder(this);
75+
}
76+
77+
/**
78+
* Returns a builder for a {@code DatastoreBackupOptions} object.
79+
*/
80+
public static Builder newBuilder() {
81+
return new Builder();
82+
}
83+
84+
@Override
85+
public String toString() {
86+
return MoreObjects.toStringHelper(this)
87+
.add("projectionFields", projectionFields)
88+
.toString();
89+
}
90+
91+
@Override
92+
public int hashCode() {
93+
return Objects.hash(getType(), projectionFields);
94+
}
95+
96+
@Override
97+
public boolean equals(Object obj) {
98+
return obj == this
99+
|| obj instanceof DatastoreBackupOptions
100+
&& Objects.equals(projectionFields, ((DatastoreBackupOptions) obj).getProjectionFields());
101+
}
102+
}

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadConfiguration.java‎

Lines changed: 2 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -92,16 +92,6 @@ interface Builder {
9292
*/
9393
Builder setIgnoreUnknownValues(Boolean ignoreUnknownValues);
9494

95-
96-
/**
97-
* Sets which entity properties to load into BigQuery from a Cloud Datastore backup. This field
98-
* is only used if the source format is set to {@code DATASTORE_BACKUP}. Property names are case
99-
* sensitive and must be top-level properties. If no properties are specified, BigQuery loads
100-
* all properties. If any named property isn't found in the Cloud Datastore backup, an invalid
101-
* error is returned in the job result.
102-
*/
103-
Builder setProjectionFields(List<String> projectionFields);
104-
10595
LoadConfiguration build();
10696
}
10797

@@ -166,13 +156,9 @@ interface Builder {
166156

167157

168158
/**
169-
* Returns which entity properties to load into BigQuery from a Cloud Datastore backup. This field
170-
* is only used if the source format is set to {@code DATASTORE_BACKUP}. Property names are case
171-
* sensitive and must be top-level properties. If no properties are specified, BigQuery loads
172-
* all properties. If any named property isn't found in the Cloud Datastore backup, an invalid
173-
* error is returned in the job result.
159+
* Returns additional options used to load from a Cloud datastore backup.
174160
*/
175-
List<String> getProjectionFields();
161+
DatastoreBackupOptions getDatastoreBackupOptions();
176162

177163
/**
178164
* Returns a builder for the load configuration object.

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java‎

Lines changed: 10 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,6 @@ public final class LoadJobConfiguration extends JobConfiguration implements Load
4343
private final Integer maxBadRecords;
4444
private final Schema schema;
4545
private final Boolean ignoreUnknownValues;
46-
private final List<String> projectionFields;
4746

4847
public static final class Builder
4948
extends JobConfiguration.Builder<LoadJobConfiguration, Builder>
@@ -72,7 +71,6 @@ private Builder(LoadJobConfiguration loadConfiguration) {
7271
this.maxBadRecords = loadConfiguration.maxBadRecords;
7372
this.schema = loadConfiguration.schema;
7473
this.ignoreUnknownValues = loadConfiguration.ignoreUnknownValues;
75-
this.projectionFields = loadConfiguration.projectionFields;
7674
this.sourceUris = loadConfiguration.sourceUris;
7775
}
7876

@@ -172,15 +170,6 @@ public Builder setIgnoreUnknownValues(Boolean ignoreUnknownValues) {
172170
return this;
173171
}
174172

175-
176-
@Override
177-
public Builder setProjectionFields(List<String> projectionFields) {
178-
this.projectionFields =
179-
projectionFields != null ? ImmutableList.copyOf(projectionFields) : null;
180-
return this;
181-
}
182-
183-
184173
/**
185174
* Sets the fully-qualified URIs that point to source data in Google Cloud Storage (e.g.
186175
* gs://bucket/path). Each URI can contain one '*' wildcard character and it must come after the
@@ -207,7 +196,6 @@ private LoadJobConfiguration(Builder builder) {
207196
this.maxBadRecords = builder.maxBadRecords;
208197
this.schema = builder.schema;
209198
this.ignoreUnknownValues = builder.ignoreUnknownValues;
210-
this.projectionFields = builder.projectionFields;
211199
}
212200

213201

@@ -234,6 +222,12 @@ public CsvOptions getCsvOptions() {
234222
return formatOptions instanceof CsvOptions ? (CsvOptions) formatOptions : null;
235223
}
236224

225+
@Override
226+
public DatastoreBackupOptions getDatastoreBackupOptions() {
227+
return formatOptions instanceof DatastoreBackupOptions ?
228+
(DatastoreBackupOptions) formatOptions : null;
229+
}
230+
237231

238232
@Override
239233
public String getFormat() {
@@ -257,13 +251,6 @@ public Boolean ignoreUnknownValues() {
257251
return ignoreUnknownValues;
258252
}
259253

260-
261-
@Override
262-
public List<String> getProjectionFields() {
263-
return projectionFields;
264-
}
265-
266-
267254
/**
268255
* Returns the fully-qualified URIs that point to source data in Google Cloud Storage (e.g.
269256
* gs://bucket/path). Each URI can contain one '*' wildcard character and it must come after the
@@ -288,7 +275,6 @@ ToStringHelper toStringHelper() {
288275
.add("maxBadRecords", maxBadRecords)
289276
.add("schema", schema)
290277
.add("ignoreUnknownValue", ignoreUnknownValues)
291-
.add("projectionFields", projectionFields)
292278
.add("sourceUris", sourceUris);
293279
}
294280

@@ -339,7 +325,10 @@ com.google.api.services.bigquery.model.JobConfiguration toPb() {
339325
}
340326
loadConfigurationPb.setMaxBadRecords(maxBadRecords);
341327
loadConfigurationPb.setIgnoreUnknownValues(ignoreUnknownValues);
342-
loadConfigurationPb.setProjectionFields(projectionFields);
328+
if (getDatastoreBackupOptions() != null) {
329+
DatastoreBackupOptions backOptions = getDatastoreBackupOptions();
330+
loadConfigurationPb.setProjectionFields(backOptions.getProjectionFields());
331+
}
343332
if (sourceUris != null) {
344333
loadConfigurationPb.setSourceUris(ImmutableList.copyOf(sourceUris));
345334
}

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/WriteChannelConfiguration.java‎

Lines changed: 9 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,6 @@ public final class WriteChannelConfiguration implements LoadConfiguration, Seria
4545
private final Integer maxBadRecords;
4646
private final Schema schema;
4747
private final Boolean ignoreUnknownValues;
48-
private final List<String> projectionFields;
4948

5049
public static final class Builder implements LoadConfiguration.Builder {
5150

@@ -68,7 +67,6 @@ private Builder(WriteChannelConfiguration writeChannelConfiguration) {
6867
this.maxBadRecords = writeChannelConfiguration.maxBadRecords;
6968
this.schema = writeChannelConfiguration.schema;
7069
this.ignoreUnknownValues = writeChannelConfiguration.ignoreUnknownValues;
71-
this.projectionFields = writeChannelConfiguration.projectionFields;
7270
}
7371

7472
private Builder(com.google.api.services.bigquery.model.JobConfiguration configurationPb) {
@@ -162,14 +160,6 @@ public Builder setIgnoreUnknownValues(Boolean ignoreUnknownValues) {
162160
return this;
163161
}
164162

165-
166-
@Override
167-
public Builder setProjectionFields(List<String> projectionFields) {
168-
this.projectionFields =
169-
projectionFields != null ? ImmutableList.copyOf(projectionFields) : null;
170-
return this;
171-
}
172-
173163
@Override
174164
public WriteChannelConfiguration build() {
175165
return new WriteChannelConfiguration(this);
@@ -184,7 +174,6 @@ protected WriteChannelConfiguration(Builder builder) {
184174
this.maxBadRecords = builder.maxBadRecords;
185175
this.schema = builder.schema;
186176
this.ignoreUnknownValues = builder.ignoreUnknownValues;
187-
this.projectionFields = builder.projectionFields;
188177
}
189178

190179

@@ -236,8 +225,9 @@ public Boolean ignoreUnknownValues() {
236225

237226

238227
@Override
239-
public List<String> getProjectionFields() {
240-
return projectionFields;
228+
public DatastoreBackupOptions getDatastoreBackupOptions() {
229+
return formatOptions instanceof DatastoreBackupOptions ?
230+
(DatastoreBackupOptions) formatOptions : null;
241231
}
242232

243233
@Override
@@ -253,8 +243,7 @@ MoreObjects.ToStringHelper toStringHelper() {
253243
.add("formatOptions", formatOptions)
254244
.add("maxBadRecords", maxBadRecords)
255245
.add("schema", schema)
256-
.add("ignoreUnknownValue", ignoreUnknownValues)
257-
.add("projectionFields", projectionFields);
246+
.add("ignoreUnknownValue", ignoreUnknownValues);
258247
}
259248

260249
@Override
@@ -272,7 +261,7 @@ public boolean equals(Object obj) {
272261
@Override
273262
public int hashCode() {
274263
return Objects.hash(destinationTable, createDisposition, writeDisposition, formatOptions,
275-
maxBadRecords, schema, ignoreUnknownValues, projectionFields);
264+
maxBadRecords, schema, ignoreUnknownValues);
276265
}
277266

278267
WriteChannelConfiguration setProjectId(String projectId) {
@@ -308,7 +297,10 @@ com.google.api.services.bigquery.model.JobConfiguration toPb() {
308297
}
309298
loadConfigurationPb.setMaxBadRecords(maxBadRecords);
310299
loadConfigurationPb.setIgnoreUnknownValues(ignoreUnknownValues);
311-
loadConfigurationPb.setProjectionFields(projectionFields);
300+
if (getDatastoreBackupOptions() != null) {
301+
DatastoreBackupOptions backupOptions = getDatastoreBackupOptions();
302+
loadConfigurationPb.setProjectionFields(backupOptions.getProjectionFields());
303+
}
312304
return new com.google.api.services.bigquery.model.JobConfiguration()
313305
.setLoad(loadConfigurationPb);
314306
}

‎google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/DatastoreBackupOptionsTest.java‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
/*
2+
* Copyright 2017 Google Inc. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.bigquery;
18+
19+
import static org.junit.Assert.assertEquals;
20+
21+
import com.google.common.collect.ImmutableList;
22+
import org.junit.Test;
23+
24+
import java.util.List;
25+
26+
public class DatastoreBackupOptionsTest {
27+
28+
private static final List<String> PROJECTION_FIELDS = ImmutableList.of("field1", "field2");
29+
private static final DatastoreBackupOptions BACKUP_OPTIONS = DatastoreBackupOptions.newBuilder()
30+
.setProjectionFields(PROJECTION_FIELDS)
31+
.build();
32+
33+
@Test
34+
public void testToBuilder() {
35+
compareDatastoreBackupOptions(BACKUP_OPTIONS, BACKUP_OPTIONS.toBuilder().build());
36+
List<String> fields = ImmutableList.of("field1", "field2");
37+
DatastoreBackupOptions backupOptions = BACKUP_OPTIONS.toBuilder()
38+
.setProjectionFields(fields)
39+
.build();
40+
assertEquals(fields, backupOptions.getProjectionFields());
41+
backupOptions = backupOptions.toBuilder().setProjectionFields(PROJECTION_FIELDS).build();
42+
compareDatastoreBackupOptions(BACKUP_OPTIONS, backupOptions);
43+
}
44+
45+
@Test
46+
public void testToBuilderIncomplete() {
47+
DatastoreBackupOptions backupOptions =
48+
DatastoreBackupOptions.newBuilder().setProjectionFields(PROJECTION_FIELDS).build();
49+
assertEquals(backupOptions, backupOptions.toBuilder().build());
50+
}
51+
52+
@Test
53+
public void testBuilder() {
54+
assertEquals(FormatOptions.DATASTORE_BACKUP, BACKUP_OPTIONS.getType());
55+
assertEquals(PROJECTION_FIELDS, BACKUP_OPTIONS.getProjectionFields());
56+
}
57+
58+
private void compareDatastoreBackupOptions(
59+
DatastoreBackupOptions expected, DatastoreBackupOptions value) {
60+
assertEquals(expected, value);
61+
assertEquals(expected.getProjectionFields(), value.getProjectionFields());
62+
}
63+
}

‎google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/JobInfoTest.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,6 @@ public class JobInfoTest {
132132
.setFormatOptions(CSV_OPTIONS)
133133
.setIgnoreUnknownValues(IGNORE_UNKNOWN_VALUES)
134134
.setMaxBadRecords(MAX_BAD_RECORDS)
135-
.setProjectionFields(PROJECTION_FIELDS)
136135
.setSchema(TABLE_SCHEMA)
137136
.build();
138137
private static final String QUERY = "BigQuery SQL";

0 commit comments

Comments
 (0)