Skip to content

Commit 82bcff3

Browse files
shollymanshollyman
committed
BigQuery: add missing query statistics.
1 parent 27c90f1 commit 82bcff3

2 files changed

Lines changed: 162 additions & 1 deletion

File tree

‎google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/JobStatistics.java‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -320,8 +320,16 @@ public static class QueryStatistics extends JobStatistics {
320320

321321
private final Integer billingTier;
322322
private final Boolean cacheHit;
323+
private final String ddlOperationPerformed;
324+
private final TableId ddlTargetTable;
325+
private final Long estimatedBytesProcessed;
326+
private final Long numDmlAffectedRows;
327+
private final List<TableId> referencedTables;
328+
private final String statementType;
323329
private final Long totalBytesBilled;
324330
private final Long totalBytesProcessed;
331+
private final Long totalPartitionsProcessed;
332+
private final Long totalSlotMs;
325333
private final List<QueryStage> queryPlan;
326334
private final List<TimelineSample> timeline;
327335
private final Schema schema;
@@ -330,8 +338,16 @@ static final class Builder extends JobStatistics.Builder<QueryStatistics, Builde
330338

331339
private Integer billingTier;
332340
private Boolean cacheHit;
341+
private String ddlOperationPerformed;
342+
private TableId ddlTargetTable;
343+
private Long estimatedBytesProcessed;
344+
private Long numDmlAffectedRows;
345+
private List<TableId> referencedTables;
346+
private String statementType;
333347
private Long totalBytesBilled;
334348
private Long totalBytesProcessed;
349+
private Long totalPartitionsProcessed;
350+
private Long totalSlotMs;
335351
private List<QueryStage> queryPlan;
336352
private List<TimelineSample> timeline;
337353
private Schema schema;
@@ -343,8 +359,22 @@ private Builder(com.google.api.services.bigquery.model.JobStatistics statisticsP
343359
if (statisticsPb.getQuery() != null) {
344360
this.billingTier = statisticsPb.getQuery().getBillingTier();
345361
this.cacheHit = statisticsPb.getQuery().getCacheHit();
362+
this.ddlOperationPerformed = statisticsPb.getQuery().getDdlOperationPerformed();
363+
if (statisticsPb.getQuery().getDdlTargetTable() != null) {
364+
this.ddlTargetTable = TableId.fromPb(statisticsPb.getQuery().getDdlTargetTable());
365+
}
366+
this.estimatedBytesProcessed = statisticsPb.getQuery().getEstimatedBytesProcessed();
367+
this.numDmlAffectedRows = statisticsPb.getQuery().getNumDmlAffectedRows();
368+
this.statementType = statisticsPb.getQuery().getStatementType();
346369
this.totalBytesBilled = statisticsPb.getQuery().getTotalBytesBilled();
347370
this.totalBytesProcessed = statisticsPb.getQuery().getTotalBytesProcessed();
371+
this.totalPartitionsProcessed = statisticsPb.getQuery().getTotalPartitionsProcessed();
372+
this.totalSlotMs = statisticsPb.getQuery().getTotalSlotMs();
373+
if (statisticsPb.getQuery().getReferencedTables() != null) {
374+
this.referencedTables =
375+
Lists.transform(
376+
statisticsPb.getQuery().getReferencedTables(), TableId.FROM_PB_FUNCTION);
377+
}
348378
if (statisticsPb.getQuery().getQueryPlan() != null) {
349379
this.queryPlan =
350380
Lists.transform(
@@ -371,6 +401,36 @@ Builder setCacheHit(Boolean cacheHit) {
371401
return self();
372402
}
373403

404+
Builder setDDLOperationPerformed(String ddlOperationPerformed) {
405+
this.ddlOperationPerformed = ddlOperationPerformed;
406+
return self();
407+
}
408+
409+
Builder setDDLTargetTable(TableId ddlTargetTable) {
410+
this.ddlTargetTable = ddlTargetTable;
411+
return self();
412+
}
413+
414+
Builder setEstimatedBytesProcessed(Long estimatedBytesProcessed) {
415+
this.estimatedBytesProcessed = estimatedBytesProcessed;
416+
return self();
417+
}
418+
419+
Builder setNumDmlAffectedRows(Long numDmlAffectedRows) {
420+
this.numDmlAffectedRows = numDmlAffectedRows;
421+
return self();
422+
}
423+
424+
Builder setReferenceTables(List<TableId> referencedTables) {
425+
this.referencedTables = referencedTables;
426+
return self();
427+
}
428+
429+
Builder setStatementType(String statementType) {
430+
this.statementType = statementType;
431+
return self();
432+
}
433+
374434
Builder setTotalBytesBilled(Long totalBytesBilled) {
375435
this.totalBytesBilled = totalBytesBilled;
376436
return self();
@@ -381,6 +441,16 @@ Builder setTotalBytesProcessed(Long totalBytesProcessed) {
381441
return self();
382442
}
383443

444+
Builder setTotalPartitionsProcessed(Long totalPartitionsProcessed) {
445+
this.totalPartitionsProcessed = totalPartitionsProcessed;
446+
return self();
447+
}
448+
449+
Builder setTotalSlotMs(Long totalSlotMs) {
450+
this.totalSlotMs = totalSlotMs;
451+
return self();
452+
}
453+
384454
Builder setQueryPlan(List<QueryStage> queryPlan) {
385455
this.queryPlan = queryPlan;
386456
return self();
@@ -406,8 +476,16 @@ private QueryStatistics(Builder builder) {
406476
super(builder);
407477
this.billingTier = builder.billingTier;
408478
this.cacheHit = builder.cacheHit;
479+
this.ddlOperationPerformed = builder.ddlOperationPerformed;
480+
this.ddlTargetTable = builder.ddlTargetTable;
481+
this.estimatedBytesProcessed = builder.estimatedBytesProcessed;
482+
this.numDmlAffectedRows = builder.numDmlAffectedRows;
483+
this.referencedTables = builder.referencedTables;
484+
this.statementType = builder.statementType;
409485
this.totalBytesBilled = builder.totalBytesBilled;
410486
this.totalBytesProcessed = builder.totalBytesProcessed;
487+
this.totalPartitionsProcessed = builder.totalPartitionsProcessed;
488+
this.totalSlotMs = builder.totalSlotMs;
411489
this.queryPlan = builder.queryPlan;
412490
this.timeline = builder.timeline;
413491
this.schema = builder.schema;
@@ -432,6 +510,47 @@ public Boolean getCacheHit() {
432510
return cacheHit;
433511
}
434512

513+
/**
514+
* [BETA] For DDL queries, returns the operation applied to the DDL target table.
515+
*/
516+
public String getDdlOperationPerformed() { return ddlOperationPerformed; }
517+
518+
/**
519+
* [BETA] For DDL queries, returns the TableID of the targeted table.
520+
*/
521+
public TableId getDdlTargetTable() { return ddlTargetTable; }
522+
523+
/**
524+
* The original estimate of bytes processed for the job.
525+
*/
526+
public Long getEstimatedBytesProcessed() { return estimatedBytesProcessed; }
527+
528+
/**
529+
* The number of rows affected by a DML statement.
530+
* Present only for DML statements INSERT, UPDATE or DELETE.
531+
*/
532+
public Long getNumDmlAffectedRows() { return numDmlAffectedRows; }
533+
534+
/**
535+
* Referenced tables for the job.
536+
* Queries that reference more than 50 tables will not have a complete list.
537+
*/
538+
public List<TableId> getReferencedTables() { return referencedTables; }
539+
540+
/**
541+
* [BETA] The type of query statement, if valid.
542+
* Possible values include:
543+
* SELECT
544+
* INSERT
545+
* UPDATE
546+
* DELETE
547+
* CREATE_TABLE
548+
* CREATE_TABLE_AS_SELECT
549+
* DROP_TABLE
550+
* CREATE_VIEW
551+
* DROP_VIEW
552+
*/
553+
public String getStatementType() { return statementType; }
435554

436555
/**
437556
* Returns the total number of bytes billed for the job.
@@ -448,6 +567,15 @@ public Long getTotalBytesProcessed() {
448567
return totalBytesProcessed;
449568
}
450569

570+
/**
571+
* Total number of partitions processed from all partitioned tables referenced in the job.
572+
*/
573+
public Long getTotalPartitionsProcessed() { return totalPartitionsProcessed; }
574+
575+
/**
576+
* Returns the slot-milliseconds consumed by the query.
577+
*/
578+
public Long getTotalSlotMs() { return totalSlotMs; }
451579

452580
/**
453581
* Returns the query plan as a list of stages or {@code null} if a query plan is not available.

‎google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/JobStatisticsTest.java‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,12 +32,22 @@ public class JobStatisticsTest {
3232

3333
private static final Integer BILLING_TIER = 42;
3434
private static final Boolean CACHE_HIT = true;
35+
private static final String DDL_OPERATION_PERFORMED = "SKIP";
36+
private static final TableId DDL_TARGET_TABLE = TableId.of("foo", "bar", "baz");
37+
private static final Long ESTIMATE_BYTES_PROCESSED = 101L;
38+
private static final Long NUM_DML_AFFECTED_ROWS = 88L;
39+
private static final String STATEMENT_TYPE = "SELECT";
3540
private static final Long TOTAL_BYTES_BILLED = 24L;
3641
private static final Long TOTAL_BYTES_PROCESSED = 42L;
42+
private static final Long TOTAL_PARTITION_PROCESSED = 63L;
43+
private static final Long TOTAL_SLOT_MS = 10202L;
3744
private static final Long INPUT_BYTES = 1L;
3845
private static final Long INPUT_FILES = 2L;
3946
private static final Long OUTPUT_BYTES = 3L;
4047
private static final Long OUTPUT_ROWS = 4L;
48+
private static final List<TableId> REFERENCED_TABLES = ImmutableList.of(
49+
TableId.of("foo", "bar", "table1"),
50+
TableId.of("foo","bar","table2"));
4151
private static final List<Long> FILE_COUNT = ImmutableList.of(1L, 2L, 3L);
4252
private static final Long CREATION_TIME = 10L;
4353
private static final Long END_TIME = 20L;
@@ -112,8 +122,16 @@ public class JobStatisticsTest {
112122
.setStartTime(START_TIME)
113123
.setBillingTier(BILLING_TIER)
114124
.setCacheHit(CACHE_HIT)
125+
.setDDLOperationPerformed(DDL_OPERATION_PERFORMED)
126+
.setDDLTargetTable(DDL_TARGET_TABLE)
127+
.setEstimatedBytesProcessed(ESTIMATE_BYTES_PROCESSED)
128+
.setNumDmlAffectedRows(NUM_DML_AFFECTED_ROWS)
129+
.setReferenceTables(REFERENCED_TABLES)
130+
.setStatementType(STATEMENT_TYPE)
115131
.setTotalBytesBilled(TOTAL_BYTES_BILLED)
116132
.setTotalBytesProcessed(TOTAL_BYTES_PROCESSED)
133+
.setTotalPartitionsProcessed(TOTAL_PARTITION_PROCESSED)
134+
.setTotalSlotMs(TOTAL_SLOT_MS)
117135
.setQueryPlan(QUERY_PLAN)
118136
.setTimeline(TIMELINE)
119137
.setSchema(SCHEMA)
@@ -146,9 +164,16 @@ public void testBuilder() {
146164
assertEquals(END_TIME, QUERY_STATISTICS.getEndTime());
147165
assertEquals(BILLING_TIER, QUERY_STATISTICS.getBillingTier());
148166
assertEquals(CACHE_HIT, QUERY_STATISTICS.getCacheHit());
167+
assertEquals(DDL_OPERATION_PERFORMED, QUERY_STATISTICS.getDdlOperationPerformed());
168+
assertEquals(DDL_TARGET_TABLE, QUERY_STATISTICS.getDdlTargetTable());
169+
assertEquals(ESTIMATE_BYTES_PROCESSED, QUERY_STATISTICS.getEstimatedBytesProcessed());
170+
assertEquals(NUM_DML_AFFECTED_ROWS, QUERY_STATISTICS.getNumDmlAffectedRows());
171+
assertEquals(REFERENCED_TABLES, QUERY_STATISTICS.getReferencedTables());
172+
assertEquals(STATEMENT_TYPE, QUERY_STATISTICS.getStatementType());
149173
assertEquals(TOTAL_BYTES_BILLED, QUERY_STATISTICS.getTotalBytesBilled());
150174
assertEquals(TOTAL_BYTES_PROCESSED, QUERY_STATISTICS.getTotalBytesProcessed());
151-
assertEquals(TOTAL_BYTES_PROCESSED, QUERY_STATISTICS.getTotalBytesProcessed());
175+
assertEquals(TOTAL_PARTITION_PROCESSED, QUERY_STATISTICS.getTotalPartitionsProcessed());
176+
assertEquals(TOTAL_SLOT_MS, QUERY_STATISTICS.getTotalSlotMs());
152177
assertEquals(QUERY_PLAN, QUERY_STATISTICS.getQueryPlan());
153178
assertEquals(TIMELINE, QUERY_STATISTICS.getTimeline());
154179

@@ -165,8 +190,16 @@ public void testBuilder() {
165190
assertEquals(END_TIME, QUERY_STATISTICS_INCOMPLETE.getEndTime());
166191
assertEquals(BILLING_TIER, QUERY_STATISTICS_INCOMPLETE.getBillingTier());
167192
assertEquals(CACHE_HIT, QUERY_STATISTICS_INCOMPLETE.getCacheHit());
193+
assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getDdlOperationPerformed());
194+
assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getDdlTargetTable());
195+
assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getEstimatedBytesProcessed());
196+
assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getNumDmlAffectedRows());
197+
assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getStatementType());
168198
assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getTotalBytesBilled());
169199
assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getTotalBytesProcessed());
200+
assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getTotalPartitionsProcessed());
201+
assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getTotalSlotMs());
202+
assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getReferencedTables());
170203
assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getQueryPlan());
171204
}
172205

0 commit comments

Comments
 (0)