Skip to content

Commit f1a0d3b

Browse files
shollymanshollyman
authored
Merge pull request #3253 from shollyman/morestuff
BigQuery: add missing query statistics.
2 parents d3fe98c + ddb1ce4 commit f1a0d3b

2 files changed

Lines changed: 253 additions & 1 deletion

File tree

‎google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/JobStatistics.java‎

Lines changed: 212 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,13 @@
1616

1717
package com.google.cloud.bigquery;
1818

19+
import com.google.api.core.ApiFunction;
1920
import com.google.api.services.bigquery.model.JobConfiguration;
2021
import com.google.api.services.bigquery.model.JobStatistics2;
2122
import com.google.api.services.bigquery.model.JobStatistics3;
2223
import com.google.api.services.bigquery.model.JobStatistics4;
24+
import com.google.cloud.StringEnumType;
25+
import com.google.cloud.StringEnumValue;
2326
import com.google.common.base.MoreObjects;
2427
import com.google.common.base.MoreObjects.ToStringHelper;
2528
import com.google.common.collect.Lists;
@@ -311,6 +314,7 @@ static LoadStatistics fromPb(com.google.api.services.bigquery.model.JobStatistic
311314
}
312315
}
313316

317+
314318
/**
315319
* A Google BigQuery Query Job statistics.
316320
*/
@@ -320,18 +324,92 @@ public static class QueryStatistics extends JobStatistics {
320324

321325
private final Integer billingTier;
322326
private final Boolean cacheHit;
327+
private final String ddlOperationPerformed;
328+
private final TableId ddlTargetTable;
329+
private final Long estimatedBytesProcessed;
330+
private final Long numDmlAffectedRows;
331+
private final List<TableId> referencedTables;
332+
private final StatementType statementType;
323333
private final Long totalBytesBilled;
324334
private final Long totalBytesProcessed;
335+
private final Long totalPartitionsProcessed;
336+
private final Long totalSlotMs;
325337
private final List<QueryStage> queryPlan;
326338
private final List<TimelineSample> timeline;
327339
private final Schema schema;
328340

341+
342+
/**
343+
* StatementType represents possible types of SQL statements reported as part of the
344+
* QueryStatistics of a BigQuery job.
345+
*/
346+
public static final class StatementType extends StringEnumValue {
347+
private static final long serialVersionUID = 818920627219751204L;
348+
349+
private static final ApiFunction<String, StatementType> CONSTRUCTOR =
350+
new ApiFunction<String, StatementType>() {
351+
@Override
352+
public StatementType apply(String constant) {
353+
return new StatementType(constant);
354+
}
355+
};
356+
357+
private static final StringEnumType<StatementType> type = new StringEnumType(
358+
StatementType.class,
359+
CONSTRUCTOR);
360+
361+
public static final StatementType SELECT = type.createAndRegister("SELECT");
362+
public static final StatementType UPDATE = type.createAndRegister("UPDATE");
363+
public static final StatementType INSERT = type.createAndRegister("INSERT");
364+
public static final StatementType DELETE = type.createAndRegister("DELETE");
365+
public static final StatementType CREATE_TABLE = type.createAndRegister("CREATE_TABLE");
366+
public static final StatementType CREATE_TABLE_AS_SELECT = type.createAndRegister("CREATE_TABLE_AS_SELECT");
367+
public static final StatementType CREATE_VIEW = type.createAndRegister("CREATE_VIEW");
368+
public static final StatementType DROP_TABLE = type.createAndRegister("DROP_TABLE");
369+
public static final StatementType DROP_VIEW = type.createAndRegister("DROP_VIEW");
370+
public static final StatementType MERGE = type.createAndRegister("MERGE");
371+
372+
private StatementType(String constant) {
373+
super(constant);
374+
}
375+
376+
/**
377+
* Get the StatementType for the given String constant, and throw an exception if the constant is
378+
* not recognized.
379+
*/
380+
public static StatementType valueOfStrict(String constant) {
381+
return type.valueOfStrict(constant);
382+
}
383+
384+
/**
385+
* Get the State for the given String constant, and allow unrecognized values.
386+
*/
387+
public static StatementType valueOf(String constant) {
388+
return type.valueOf(constant);
389+
}
390+
391+
/**
392+
* Return the known values for State.
393+
*/
394+
public static StatementType[] values() {
395+
return type.values();
396+
}
397+
}
398+
329399
static final class Builder extends JobStatistics.Builder<QueryStatistics, Builder> {
330400

331401
private Integer billingTier;
332402
private Boolean cacheHit;
403+
private String ddlOperationPerformed;
404+
private TableId ddlTargetTable;
405+
private Long estimatedBytesProcessed;
406+
private Long numDmlAffectedRows;
407+
private List<TableId> referencedTables;
408+
private StatementType statementType;
333409
private Long totalBytesBilled;
334410
private Long totalBytesProcessed;
411+
private Long totalPartitionsProcessed;
412+
private Long totalSlotMs;
335413
private List<QueryStage> queryPlan;
336414
private List<TimelineSample> timeline;
337415
private Schema schema;
@@ -343,8 +421,25 @@ private Builder(com.google.api.services.bigquery.model.JobStatistics statisticsP
343421
if (statisticsPb.getQuery() != null) {
344422
this.billingTier = statisticsPb.getQuery().getBillingTier();
345423
this.cacheHit = statisticsPb.getQuery().getCacheHit();
424+
this.ddlOperationPerformed = statisticsPb.getQuery().getDdlOperationPerformed();
425+
if (statisticsPb.getQuery().getDdlTargetTable() != null) {
426+
this.ddlTargetTable = TableId.fromPb(statisticsPb.getQuery().getDdlTargetTable());
427+
}
428+
this.estimatedBytesProcessed = statisticsPb.getQuery().getEstimatedBytesProcessed();
429+
this.numDmlAffectedRows = statisticsPb.getQuery().getNumDmlAffectedRows();
346430
this.totalBytesBilled = statisticsPb.getQuery().getTotalBytesBilled();
347431
this.totalBytesProcessed = statisticsPb.getQuery().getTotalBytesProcessed();
432+
this.totalPartitionsProcessed = statisticsPb.getQuery().getTotalPartitionsProcessed();
433+
this.totalSlotMs = statisticsPb.getQuery().getTotalSlotMs();
434+
if (statisticsPb.getQuery().getStatementType() != null) {
435+
this.statementType = StatementType.valueOf(statisticsPb.getQuery().getStatementType());
436+
}
437+
438+
if (statisticsPb.getQuery().getReferencedTables() != null) {
439+
this.referencedTables =
440+
Lists.transform(
441+
statisticsPb.getQuery().getReferencedTables(), TableId.FROM_PB_FUNCTION);
442+
}
348443
if (statisticsPb.getQuery().getQueryPlan() != null) {
349444
this.queryPlan =
350445
Lists.transform(
@@ -371,6 +466,41 @@ Builder setCacheHit(Boolean cacheHit) {
371466
return self();
372467
}
373468

469+
Builder setDDLOperationPerformed(String ddlOperationPerformed) {
470+
this.ddlOperationPerformed = ddlOperationPerformed;
471+
return self();
472+
}
473+
474+
Builder setDDLTargetTable(TableId ddlTargetTable) {
475+
this.ddlTargetTable = ddlTargetTable;
476+
return self();
477+
}
478+
479+
Builder setEstimatedBytesProcessed(Long estimatedBytesProcessed) {
480+
this.estimatedBytesProcessed = estimatedBytesProcessed;
481+
return self();
482+
}
483+
484+
Builder setNumDmlAffectedRows(Long numDmlAffectedRows) {
485+
this.numDmlAffectedRows = numDmlAffectedRows;
486+
return self();
487+
}
488+
489+
Builder setReferenceTables(List<TableId> referencedTables) {
490+
this.referencedTables = referencedTables;
491+
return self();
492+
}
493+
494+
Builder setStatementType(StatementType statementType) {
495+
this.statementType = statementType;
496+
return self();
497+
}
498+
499+
Builder setStatementType(String strStatementType) {
500+
this.statementType = StatementType.valueOf(strStatementType);
501+
return self();
502+
}
503+
374504
Builder setTotalBytesBilled(Long totalBytesBilled) {
375505
this.totalBytesBilled = totalBytesBilled;
376506
return self();
@@ -381,6 +511,16 @@ Builder setTotalBytesProcessed(Long totalBytesProcessed) {
381511
return self();
382512
}
383513

514+
Builder setTotalPartitionsProcessed(Long totalPartitionsProcessed) {
515+
this.totalPartitionsProcessed = totalPartitionsProcessed;
516+
return self();
517+
}
518+
519+
Builder setTotalSlotMs(Long totalSlotMs) {
520+
this.totalSlotMs = totalSlotMs;
521+
return self();
522+
}
523+
384524
Builder setQueryPlan(List<QueryStage> queryPlan) {
385525
this.queryPlan = queryPlan;
386526
return self();
@@ -406,8 +546,16 @@ private QueryStatistics(Builder builder) {
406546
super(builder);
407547
this.billingTier = builder.billingTier;
408548
this.cacheHit = builder.cacheHit;
549+
this.ddlOperationPerformed = builder.ddlOperationPerformed;
550+
this.ddlTargetTable = builder.ddlTargetTable;
551+
this.estimatedBytesProcessed = builder.estimatedBytesProcessed;
552+
this.numDmlAffectedRows = builder.numDmlAffectedRows;
553+
this.referencedTables = builder.referencedTables;
554+
this.statementType = builder.statementType;
409555
this.totalBytesBilled = builder.totalBytesBilled;
410556
this.totalBytesProcessed = builder.totalBytesProcessed;
557+
this.totalPartitionsProcessed = builder.totalPartitionsProcessed;
558+
this.totalSlotMs = builder.totalSlotMs;
411559
this.queryPlan = builder.queryPlan;
412560
this.timeline = builder.timeline;
413561
this.schema = builder.schema;
@@ -432,6 +580,47 @@ public Boolean getCacheHit() {
432580
return cacheHit;
433581
}
434582

583+
/**
584+
* [BETA] For DDL queries, returns the operation applied to the DDL target table.
585+
*/
586+
public String getDdlOperationPerformed() { return ddlOperationPerformed; }
587+
588+
/**
589+
* [BETA] For DDL queries, returns the TableID of the targeted table.
590+
*/
591+
public TableId getDdlTargetTable() { return ddlTargetTable; }
592+
593+
/**
594+
* The original estimate of bytes processed for the job.
595+
*/
596+
public Long getEstimatedBytesProcessed() { return estimatedBytesProcessed; }
597+
598+
/**
599+
* The number of rows affected by a DML statement.
600+
* Present only for DML statements INSERT, UPDATE or DELETE.
601+
*/
602+
public Long getNumDmlAffectedRows() { return numDmlAffectedRows; }
603+
604+
/**
605+
* Referenced tables for the job.
606+
* Queries that reference more than 50 tables will not have a complete list.
607+
*/
608+
public List<TableId> getReferencedTables() { return referencedTables; }
609+
610+
/**
611+
* [BETA] The type of query statement, if valid.
612+
* Possible values include:
613+
* SELECT
614+
* INSERT
615+
* UPDATE
616+
* DELETE
617+
* CREATE_TABLE
618+
* CREATE_TABLE_AS_SELECT
619+
* DROP_TABLE
620+
* CREATE_VIEW
621+
* DROP_VIEW
622+
*/
623+
public StatementType getStatementType() { return statementType; }
435624

436625
/**
437626
* Returns the total number of bytes billed for the job.
@@ -448,6 +637,15 @@ public Long getTotalBytesProcessed() {
448637
return totalBytesProcessed;
449638
}
450639

640+
/**
641+
* Total number of partitions processed from all partitioned tables referenced in the job.
642+
*/
643+
public Long getTotalPartitionsProcessed() { return totalPartitionsProcessed; }
644+
645+
/**
646+
* Returns the slot-milliseconds consumed by the query.
647+
*/
648+
public Long getTotalSlotMs() { return totalSlotMs; }
451649

452650
/**
453651
* Returns the query plan as a list of stages or {@code null} if a query plan is not available.
@@ -509,8 +707,22 @@ com.google.api.services.bigquery.model.JobStatistics toPb() {
509707
JobStatistics2 queryStatisticsPb = new JobStatistics2();
510708
queryStatisticsPb.setBillingTier(billingTier);
511709
queryStatisticsPb.setCacheHit(cacheHit);
710+
queryStatisticsPb.setDdlOperationPerformed(ddlOperationPerformed);
711+
queryStatisticsPb.setEstimatedBytesProcessed(estimatedBytesProcessed);
512712
queryStatisticsPb.setTotalBytesBilled(totalBytesBilled);
513713
queryStatisticsPb.setTotalBytesProcessed(totalBytesProcessed);
714+
queryStatisticsPb.setTotalPartitionsProcessed(totalPartitionsProcessed);
715+
queryStatisticsPb.setTotalSlotMs(totalSlotMs);
716+
if (ddlTargetTable != null) {
717+
queryStatisticsPb.setDdlTargetTable(ddlTargetTable.toPb());
718+
}
719+
720+
if (referencedTables != null) {
721+
queryStatisticsPb.setReferencedTables(Lists.transform(referencedTables, TableId.TO_PB_FUNCTION));
722+
}
723+
if (statementType != null) {
724+
queryStatisticsPb.setStatementType(statementType.toString());
725+
}
514726
if (queryPlan != null) {
515727
queryStatisticsPb.setQueryPlan(Lists.transform(queryPlan, QueryStage.TO_PB_FUNCTION));
516728
}

‎google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/JobStatisticsTest.java‎

Lines changed: 41 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,12 +32,22 @@ public class JobStatisticsTest {
3232

3333
private static final Integer BILLING_TIER = 42;
3434
private static final Boolean CACHE_HIT = true;
35+
private static final String DDL_OPERATION_PERFORMED = "SKIP";
36+
private static final TableId DDL_TARGET_TABLE = TableId.of("foo", "bar", "baz");
37+
private static final Long ESTIMATE_BYTES_PROCESSED = 101L;
38+
private static final Long NUM_DML_AFFECTED_ROWS = 88L;
39+
private static final QueryStatistics.StatementType STATEMENT_TYPE = QueryStatistics.StatementType.SELECT;
3540
private static final Long TOTAL_BYTES_BILLED = 24L;
3641
private static final Long TOTAL_BYTES_PROCESSED = 42L;
42+
private static final Long TOTAL_PARTITION_PROCESSED = 63L;
43+
private static final Long TOTAL_SLOT_MS = 10202L;
3744
private static final Long INPUT_BYTES = 1L;
3845
private static final Long INPUT_FILES = 2L;
3946
private static final Long OUTPUT_BYTES = 3L;
4047
private static final Long OUTPUT_ROWS = 4L;
48+
private static final List<TableId> REFERENCED_TABLES = ImmutableList.of(
49+
TableId.of("foo", "bar", "table1"),
50+
TableId.of("foo","bar","table2"));
4151
private static final List<Long> FILE_COUNT = ImmutableList.of(1L, 2L, 3L);
4252
private static final Long CREATION_TIME = 10L;
4353
private static final Long END_TIME = 20L;
@@ -112,8 +122,16 @@ public class JobStatisticsTest {
112122
.setStartTime(START_TIME)
113123
.setBillingTier(BILLING_TIER)
114124
.setCacheHit(CACHE_HIT)
125+
.setDDLOperationPerformed(DDL_OPERATION_PERFORMED)
126+
.setDDLTargetTable(DDL_TARGET_TABLE)
127+
.setEstimatedBytesProcessed(ESTIMATE_BYTES_PROCESSED)
128+
.setNumDmlAffectedRows(NUM_DML_AFFECTED_ROWS)
129+
.setReferenceTables(REFERENCED_TABLES)
130+
.setStatementType(STATEMENT_TYPE)
115131
.setTotalBytesBilled(TOTAL_BYTES_BILLED)
116132
.setTotalBytesProcessed(TOTAL_BYTES_PROCESSED)
133+
.setTotalPartitionsProcessed(TOTAL_PARTITION_PROCESSED)
134+
.setTotalSlotMs(TOTAL_SLOT_MS)
117135
.setQueryPlan(QUERY_PLAN)
118136
.setTimeline(TIMELINE)
119137
.setSchema(SCHEMA)
@@ -146,9 +164,16 @@ public void testBuilder() {
146164
assertEquals(END_TIME, QUERY_STATISTICS.getEndTime());
147165
assertEquals(BILLING_TIER, QUERY_STATISTICS.getBillingTier());
148166
assertEquals(CACHE_HIT, QUERY_STATISTICS.getCacheHit());
167+
assertEquals(DDL_OPERATION_PERFORMED, QUERY_STATISTICS.getDdlOperationPerformed());
168+
assertEquals(DDL_TARGET_TABLE, QUERY_STATISTICS.getDdlTargetTable());
169+
assertEquals(ESTIMATE_BYTES_PROCESSED, QUERY_STATISTICS.getEstimatedBytesProcessed());
170+
assertEquals(NUM_DML_AFFECTED_ROWS, QUERY_STATISTICS.getNumDmlAffectedRows());
171+
assertEquals(REFERENCED_TABLES, QUERY_STATISTICS.getReferencedTables());
172+
assertEquals(STATEMENT_TYPE, QUERY_STATISTICS.getStatementType());
149173
assertEquals(TOTAL_BYTES_BILLED, QUERY_STATISTICS.getTotalBytesBilled());
150174
assertEquals(TOTAL_BYTES_PROCESSED, QUERY_STATISTICS.getTotalBytesProcessed());
151-
assertEquals(TOTAL_BYTES_PROCESSED, QUERY_STATISTICS.getTotalBytesProcessed());
175+
assertEquals(TOTAL_PARTITION_PROCESSED, QUERY_STATISTICS.getTotalPartitionsProcessed());
176+
assertEquals(TOTAL_SLOT_MS, QUERY_STATISTICS.getTotalSlotMs());
152177
assertEquals(QUERY_PLAN, QUERY_STATISTICS.getQueryPlan());
153178
assertEquals(TIMELINE, QUERY_STATISTICS.getTimeline());
154179

@@ -165,8 +190,15 @@ public void testBuilder() {
165190
assertEquals(END_TIME, QUERY_STATISTICS_INCOMPLETE.getEndTime());
166191
assertEquals(BILLING_TIER, QUERY_STATISTICS_INCOMPLETE.getBillingTier());
167192
assertEquals(CACHE_HIT, QUERY_STATISTICS_INCOMPLETE.getCacheHit());
193+
assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getDdlOperationPerformed());
194+
assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getDdlTargetTable());
195+
assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getEstimatedBytesProcessed());
196+
assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getNumDmlAffectedRows());
168197
assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getTotalBytesBilled());
169198
assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getTotalBytesProcessed());
199+
assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getTotalPartitionsProcessed());
200+
assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getTotalSlotMs());
201+
assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getReferencedTables());
170202
assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getQueryPlan());
171203
}
172204

@@ -235,10 +267,18 @@ private void compareQueryStatistics(QueryStatistics expected, QueryStatistics va
235267
compareStatistics(expected, value);
236268
assertEquals(expected.getBillingTier(), value.getBillingTier());
237269
assertEquals(expected.getCacheHit(), value.getCacheHit());
270+
assertEquals(expected.getDdlOperationPerformed(), value.getDdlOperationPerformed());
271+
assertEquals(expected.getDdlTargetTable(), value.getDdlTargetTable());
272+
assertEquals(expected.getEstimatedBytesProcessed(), value.getEstimatedBytesProcessed());
238273
assertEquals(expected.getTotalBytesBilled(), value.getTotalBytesBilled());
239274
assertEquals(expected.getTotalBytesProcessed(), value.getTotalBytesProcessed());
275+
assertEquals(expected.getTotalPartitionsProcessed(), value.getTotalPartitionsProcessed());
276+
assertEquals(expected.getTotalSlotMs(), value.getTotalSlotMs());
240277
assertEquals(expected.getQueryPlan(), value.getQueryPlan());
278+
assertEquals(expected.getReferencedTables(), value.getReferencedTables());
241279
assertEquals(expected.getSchema(), value.getSchema());
280+
assertEquals(expected.getStatementType(), value.getStatementType());
281+
assertEquals(expected.getTimeline(), value.getTimeline());
242282
}
243283

244284
private void compareStatistics(JobStatistics expected, JobStatistics value) {

0 commit comments

Comments
 (0)