Skip to content

Commit b4bddff

Browse files
aozarovaozarov
committed
Merge pull request #584 from mziccard/bigquery-hierachies
Remove JobInfo hierarchy, add JobConfiguration hierarchy
2 parents 44a1533 + 73250ea commit b4bddff

45 files changed

Lines changed: 2296 additions & 1635 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎README.md‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ import com.google.gcloud.bigquery.BigQuery;
130130
import com.google.gcloud.bigquery.BigQueryOptions;
131131
import com.google.gcloud.bigquery.Field;
132132
import com.google.gcloud.bigquery.JobStatus;
133-
import com.google.gcloud.bigquery.LoadJobInfo;
133+
import com.google.gcloud.bigquery.JobInfo;
134134
import com.google.gcloud.bigquery.Schema;
135135
import com.google.gcloud.bigquery.TableId;
136136
import com.google.gcloud.bigquery.TableInfo;
@@ -144,7 +144,8 @@ if (info == null) {
144144
bigquery.create(TableInfo.of(tableId, Schema.of(integerField)));
145145
} else {
146146
System.out.println("Loading data into table " + tableId);
147-
LoadJobInfo loadJob = LoadJobInfo.of(tableId, "gs://bucket/path");
147+
LoadJobConfiguration configuration = LoadJobConfiguration.of(tableId, "gs://bucket/path");
148+
JobInfo loadJob = JobInfo.of(configuration);
148149
loadJob = bigquery.create(loadJob);
149150
while (loadJob.status().state() != JobStatus.State.DONE) {
150151
Thread.sleep(1000L);

‎gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/BaseTableInfo.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -394,6 +394,10 @@ protected final boolean baseEquals(BaseTableInfo tableInfo) {
394394
return Objects.equals(toPb(), tableInfo.toPb());
395395
}
396396

397+
BaseTableInfo setProjectId(String projectId) {
398+
return toBuilder().tableId(tableId().setProjectId(projectId)).build();
399+
}
400+
397401
Table toPb() {
398402
Table tablePb = new Table();
399403
tablePb.setTableReference(tableId.toPb());

‎gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/BigQuery.java‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -371,8 +371,9 @@ public static JobListOption startPageToken(String pageToken) {
371371
* is not provided all job's fields are returned. {@code JobOption.fields()} can be used to
372372
* specify only the fields of interest. {@link JobInfo#jobId()}, {@link JobStatus#state()},
373373
* {@link JobStatus#error()} as well as type-specific configuration (e.g.
374-
* {@link QueryJobInfo#query()} for Query Jobs) are always returned, even if not specified.
375-
* {@link JobField#SELF_LINK} and {@link JobField#ETAG} can not be selected when listing jobs.
374+
* {@link QueryJobConfiguration#query()} for Query Jobs) are always returned, even if not
375+
* specified. {@link JobField#SELF_LINK} and {@link JobField#ETAG} can not be selected when
376+
* listing jobs.
376377
*/
377378
public static JobListOption fields(JobField... fields) {
378379
String selector = JobField.selector(fields);
@@ -397,8 +398,8 @@ private JobOption(BigQueryRpc.Option option, Object value) {
397398
* Returns an option to specify the job's fields to be returned by the RPC call. If this option
398399
* is not provided all job's fields are returned. {@code JobOption.fields()} can be used to
399400
* specify only the fields of interest. {@link JobInfo#jobId()} as well as type-specific
400-
* configuration (e.g. {@link QueryJobInfo#query()} for Query Jobs) are always returned, even if
401-
* not specified.
401+
* configuration (e.g. {@link QueryJobConfiguration#query()} for Query Jobs) are always
402+
* returned, even if not specified.
402403
*/
403404
public static JobOption fields(JobField... fields) {
404405
return new JobOption(BigQueryRpc.Option.FIELDS, JobField.selector(fields));
@@ -470,7 +471,7 @@ public static QueryResultsOption maxWaitTime(long maxWaitTime) {
470471
*
471472
* @throws BigQueryException upon failure
472473
*/
473-
<T extends JobInfo> T create(T job, JobOption... options) throws BigQueryException;
474+
JobInfo create(JobInfo job, JobOption... options) throws BigQueryException;
474475

475476
/**
476477
* Returns the requested dataset or {@code null} if not found.
@@ -611,14 +612,14 @@ Page<List<FieldValue>> listTableData(TableId tableId, TableDataListOption... opt
611612
*
612613
* @throws BigQueryException upon failure
613614
*/
614-
<T extends JobInfo> T getJob(String jobId, JobOption... options) throws BigQueryException;
615+
JobInfo getJob(String jobId, JobOption... options) throws BigQueryException;
615616

616617
/**
617618
* Returns the requested job or {@code null} if not found.
618619
*
619620
* @throws BigQueryException upon failure
620621
*/
621-
<T extends JobInfo> T getJob(JobId jobId, JobOption... options) throws BigQueryException;
622+
JobInfo getJob(JobId jobId, JobOption... options) throws BigQueryException;
622623

623624
/**
624625
* Lists the jobs.
@@ -665,9 +666,9 @@ Page<List<FieldValue>> listTableData(TableId tableId, TableDataListOption... opt
665666

666667
/**
667668
* Returns a channel to write data to be inserted into a BigQuery table. Data format and other
668-
* options can be configured using the {@link LoadConfiguration} parameter.
669+
* options can be configured using the {@link WriteChannelConfiguration} parameter.
669670
*
670671
* @throws BigQueryException upon failure
671672
*/
672-
TableDataWriteChannel writer(LoadConfiguration loadConfiguration);
673+
TableDataWriteChannel writer(WriteChannelConfiguration writeChannelConfiguration);
673674
}

‎gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/BigQueryException.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,6 @@
1616

1717
package com.google.gcloud.bigquery;
1818

19-
import com.google.api.client.googleapis.json.GoogleJsonError;
20-
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
2119
import com.google.common.collect.ImmutableSet;
2220
import com.google.gcloud.BaseServiceException;
2321
import com.google.gcloud.RetryHelper.RetryHelperException;

‎gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/BigQueryImpl.java‎

Lines changed: 13 additions & 102 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525
import com.google.api.services.bigquery.model.Table;
2626
import com.google.api.services.bigquery.model.TableDataInsertAllRequest;
2727
import com.google.api.services.bigquery.model.TableDataInsertAllRequest.Rows;
28-
import com.google.api.services.bigquery.model.TableReference;
2928
import com.google.api.services.bigquery.model.TableRow;
3029
import com.google.common.base.Function;
3130
import com.google.common.collect.ImmutableList;
@@ -159,7 +158,7 @@ public QueryResult nextPage() {
159158
@Override
160159
public DatasetInfo create(DatasetInfo dataset, DatasetOption... options)
161160
throws BigQueryException {
162-
final Dataset datasetPb = setProjectId(dataset).toPb();
161+
final Dataset datasetPb = dataset.setProjectId(options().projectId()).toPb();
163162
final Map<BigQueryRpc.Option, ?> optionsMap = optionMap(options);
164163
try {
165164
return DatasetInfo.fromPb(runWithRetries(new Callable<Dataset>() {
@@ -176,7 +175,7 @@ public Dataset call() {
176175
@Override
177176
public <T extends BaseTableInfo> T create(T table, TableOption... options)
178177
throws BigQueryException {
179-
final Table tablePb = setProjectId(table).toPb();
178+
final Table tablePb = table.setProjectId(options().projectId()).toPb();
180179
final Map<BigQueryRpc.Option, ?> optionsMap = optionMap(options);
181180
try {
182181
return BaseTableInfo.fromPb(runWithRetries(new Callable<Table>() {
@@ -191,8 +190,8 @@ public Table call() {
191190
}
192191

193192
@Override
194-
public <T extends JobInfo> T create(T job, JobOption... options) throws BigQueryException {
195-
final Job jobPb = setProjectId(job).toPb();
193+
public JobInfo create(JobInfo job, JobOption... options) throws BigQueryException {
194+
final Job jobPb = job.setProjectId(options().projectId()).toPb();
196195
final Map<BigQueryRpc.Option, ?> optionsMap = optionMap(options);
197196
try {
198197
return JobInfo.fromPb(runWithRetries(new Callable<Job>() {
@@ -295,7 +294,7 @@ public Boolean call() {
295294
@Override
296295
public DatasetInfo update(DatasetInfo dataset, DatasetOption... options)
297296
throws BigQueryException {
298-
final Dataset datasetPb = setProjectId(dataset).toPb();
297+
final Dataset datasetPb = dataset.setProjectId(options().projectId()).toPb();
299298
final Map<BigQueryRpc.Option, ?> optionsMap = optionMap(options);
300299
try {
301300
return DatasetInfo.fromPb(runWithRetries(new Callable<Dataset>() {
@@ -312,7 +311,7 @@ public Dataset call() {
312311
@Override
313312
public <T extends BaseTableInfo> T update(T table, TableOption... options)
314313
throws BigQueryException {
315-
final Table tablePb = setProjectId(table).toPb();
314+
final Table tablePb = table.setProjectId(options().projectId()).toPb();
316315
final Map<BigQueryRpc.Option, ?> optionsMap = optionMap(options);
317316
try {
318317
return BaseTableInfo.fromPb(runWithRetries(new Callable<Table>() {
@@ -442,12 +441,12 @@ public List<FieldValue> apply(TableRow rowPb) {
442441
}
443442

444443
@Override
445-
public <T extends JobInfo> T getJob(String jobId, JobOption... options) throws BigQueryException {
444+
public JobInfo getJob(String jobId, JobOption... options) throws BigQueryException {
446445
return getJob(JobId.of(jobId), options);
447446
}
448447

449448
@Override
450-
public <T extends JobInfo> T getJob(final JobId jobId, JobOption... options)
449+
public JobInfo getJob(final JobId jobId, JobOption... options)
451450
throws BigQueryException {
452451
final Map<BigQueryRpc.Option, ?> optionsMap = optionMap(options);
453452
try {
@@ -457,7 +456,7 @@ public Job call() {
457456
return bigQueryRpc.getJob(jobId.job(), optionsMap);
458457
}
459458
}, options().retryParams(), EXCEPTION_HANDLER);
460-
return answer == null ? null : JobInfo.<T>fromPb(answer);
459+
return answer == null ? null : JobInfo.fromPb(answer);
461460
} catch (RetryHelper.RetryHelperException e) {
462461
throw BigQueryException.translateAndThrow(e);
463462
}
@@ -508,7 +507,7 @@ public QueryResponse query(final QueryRequest request) throws BigQueryException
508507
runWithRetries(new Callable<com.google.api.services.bigquery.model.QueryResponse>() {
509508
@Override
510509
public com.google.api.services.bigquery.model.QueryResponse call() {
511-
return bigQueryRpc.query(setProjectId(request).toPb());
510+
return bigQueryRpc.query(request.setProjectId(options().projectId()).toPb());
512511
}
513512
}, options().retryParams(), EXCEPTION_HANDLER);
514513
QueryResponse.Builder builder = QueryResponse.builder();
@@ -596,8 +595,9 @@ private static QueryResult.Builder transformQueryResults(JobId jobId, List<Table
596595
.results(transformTableData(rowsPb));
597596
}
598597

599-
public TableDataWriteChannel writer(LoadConfiguration loadConfiguration) {
600-
return new TableDataWriteChannel(options(), setProjectId(loadConfiguration));
598+
public TableDataWriteChannel writer(WriteChannelConfiguration writeChannelConfiguration) {
599+
return new TableDataWriteChannel(options(),
600+
writeChannelConfiguration.setProjectId(options().projectId()));
601601
}
602602

603603
private Map<BigQueryRpc.Option, ?> optionMap(Option... options) {
@@ -608,93 +608,4 @@ public TableDataWriteChannel writer(LoadConfiguration loadConfiguration) {
608608
}
609609
return optionMap;
610610
}
611-
612-
private DatasetInfo setProjectId(DatasetInfo dataset) {
613-
DatasetInfo.Builder datasetBuilder = dataset.toBuilder();
614-
datasetBuilder.datasetId(setProjectId(dataset.datasetId()));
615-
if (dataset.acl() != null) {
616-
List<Acl> acls = Lists.newArrayListWithCapacity(dataset.acl().size());
617-
for (Acl acl : dataset.acl()) {
618-
if (acl.entity().type() == Acl.Entity.Type.VIEW) {
619-
Dataset.Access accessPb = acl.toPb();
620-
TableReference viewReferencePb = accessPb.getView();
621-
if (viewReferencePb.getProjectId() == null) {
622-
viewReferencePb.setProjectId(options().projectId());
623-
}
624-
acls.add(Acl.of(new Acl.View(TableId.fromPb(viewReferencePb))));
625-
} else {
626-
acls.add(acl);
627-
}
628-
}
629-
datasetBuilder.acl(acls);
630-
}
631-
return datasetBuilder.build();
632-
}
633-
634-
private DatasetId setProjectId(DatasetId dataset) {
635-
return dataset.project() != null ? dataset
636-
: DatasetId.of(options().projectId(), dataset.dataset());
637-
}
638-
639-
private BaseTableInfo setProjectId(BaseTableInfo table) {
640-
return table.toBuilder().tableId(setProjectId(table.tableId())).build();
641-
}
642-
643-
private TableId setProjectId(TableId table) {
644-
return table.project() != null ? table
645-
: TableId.of(options().projectId(), table.dataset(), table.table());
646-
}
647-
648-
private JobInfo setProjectId(JobInfo job) {
649-
if (job instanceof CopyJobInfo) {
650-
CopyJobInfo copyJob = (CopyJobInfo) job;
651-
CopyJobInfo.Builder copyBuilder = copyJob.toBuilder();
652-
copyBuilder.destinationTable(setProjectId(copyJob.destinationTable()));
653-
copyBuilder.sourceTables(
654-
Lists.transform(copyJob.sourceTables(), new Function<TableId, TableId>() {
655-
@Override
656-
public TableId apply(TableId tableId) {
657-
return setProjectId(tableId);
658-
}
659-
}));
660-
return copyBuilder.build();
661-
}
662-
if (job instanceof QueryJobInfo) {
663-
QueryJobInfo queryJob = (QueryJobInfo) job;
664-
QueryJobInfo.Builder queryBuilder = queryJob.toBuilder();
665-
if (queryJob.destinationTable() != null) {
666-
queryBuilder.destinationTable(setProjectId(queryJob.destinationTable()));
667-
}
668-
if (queryJob.defaultDataset() != null) {
669-
queryBuilder.defaultDataset(setProjectId(queryJob.defaultDataset()));
670-
}
671-
return queryBuilder.build();
672-
}
673-
if (job instanceof ExtractJobInfo) {
674-
ExtractJobInfo extractJob = (ExtractJobInfo) job;
675-
ExtractJobInfo.Builder extractBuilder = extractJob.toBuilder();
676-
extractBuilder.sourceTable(setProjectId(extractJob.sourceTable()));
677-
return extractBuilder.build();
678-
}
679-
if (job instanceof LoadJobInfo) {
680-
LoadJobInfo loadJob = (LoadJobInfo) job;
681-
LoadJobInfo.Builder loadBuilder = loadJob.toBuilder();
682-
return loadBuilder.configuration(setProjectId(loadJob.configuration())).build();
683-
}
684-
return job;
685-
}
686-
687-
private QueryRequest setProjectId(QueryRequest request) {
688-
QueryRequest.Builder builder = request.toBuilder();
689-
if (request.defaultDataset() != null) {
690-
builder.defaultDataset(setProjectId(request.defaultDataset()));
691-
}
692-
return builder.build();
693-
}
694-
695-
private LoadConfiguration setProjectId(LoadConfiguration configuration) {
696-
LoadConfiguration.Builder builder = configuration.toBuilder();
697-
builder.destinationTable(setProjectId(configuration.destinationTable()));
698-
return builder.build();
699-
}
700611
}

0 commit comments

Comments
 (0)