Skip to content
This repository was archived by the owner on Mar 23, 2026. It is now read-only.

Commit 75ea095

Browse files
PhongChuongPhongChuong
authored
feat: Add totalSlotMs to JobStatistics (#3250)
* feat: Add totalSlotMs to JobStatistics * fix: testQuery IT flakiness * Fix query statistics test
1 parent 0b0b414 commit 75ea095

3 files changed

Lines changed: 51 additions & 20 deletions

File tree

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/JobStatistics.java‎

Lines changed: 23 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@
4040
/** A Google BigQuery Job statistics. */
4141
public abstract class JobStatistics implements Serializable {
4242

43-
private static final long serialVersionUID = 1433024714741660399L;
43+
private static final long serialVersionUID = 1433024714741660400L;
4444

4545
private final Long creationTime;
4646
private final Long endTime;
@@ -51,6 +51,7 @@ public abstract class JobStatistics implements Serializable {
5151
private final List<ReservationUsage> reservationUsage;
5252
private final TransactionInfo transactionInfo;
5353
private final SessionInfo sessionInfo;
54+
private final Long totalSlotMs;
5455

5556
/** A Google BigQuery Copy Job statistics. */
5657
public static class CopyStatistics extends JobStatistics {
@@ -390,7 +391,7 @@ static LoadStatistics fromPb(com.google.api.services.bigquery.model.JobStatistic
390391
/** A Google BigQuery Query Job statistics. */
391392
public static class QueryStatistics extends JobStatistics {
392393

393-
private static final long serialVersionUID = 7539354109226732353L;
394+
private static final long serialVersionUID = 7539354109226732354L;
394395

395396
private final BiEngineStats biEngineStats;
396397
private final Integer billingTier;
@@ -407,7 +408,6 @@ public static class QueryStatistics extends JobStatistics {
407408
private final Long totalBytesBilled;
408409
private final Long totalBytesProcessed;
409410
private final Long totalPartitionsProcessed;
410-
private final Long totalSlotMs;
411411
private final List<QueryStage> queryPlan;
412412
private final List<TimelineSample> timeline;
413413
private final Schema schema;
@@ -567,7 +567,6 @@ static final class Builder extends JobStatistics.Builder<QueryStatistics, Builde
567567
private Long totalBytesBilled;
568568
private Long totalBytesProcessed;
569569
private Long totalPartitionsProcessed;
570-
private Long totalSlotMs;
571570
private List<QueryStage> queryPlan;
572571
private List<TimelineSample> timeline;
573572
private Schema schema;
@@ -599,7 +598,6 @@ private Builder(com.google.api.services.bigquery.model.JobStatistics statisticsP
599598
this.totalBytesBilled = statisticsPb.getQuery().getTotalBytesBilled();
600599
this.totalBytesProcessed = statisticsPb.getQuery().getTotalBytesProcessed();
601600
this.totalPartitionsProcessed = statisticsPb.getQuery().getTotalPartitionsProcessed();
602-
this.totalSlotMs = statisticsPb.getQuery().getTotalSlotMs();
603601
if (statisticsPb.getQuery().getStatementType() != null) {
604602
this.statementType = StatementType.valueOf(statisticsPb.getQuery().getStatementType());
605603
}
@@ -719,11 +717,6 @@ Builder setTotalPartitionsProcessed(Long totalPartitionsProcessed) {
719717
return self();
720718
}
721719

722-
Builder setTotalSlotMs(Long totalSlotMs) {
723-
this.totalSlotMs = totalSlotMs;
724-
return self();
725-
}
726-
727720
Builder setQueryPlan(List<QueryStage> queryPlan) {
728721
this.queryPlan = queryPlan;
729722
return self();
@@ -777,7 +770,6 @@ private QueryStatistics(Builder builder) {
777770
this.totalBytesBilled = builder.totalBytesBilled;
778771
this.totalBytesProcessed = builder.totalBytesProcessed;
779772
this.totalPartitionsProcessed = builder.totalPartitionsProcessed;
780-
this.totalSlotMs = builder.totalSlotMs;
781773
this.queryPlan = builder.queryPlan;
782774
this.timeline = builder.timeline;
783775
this.schema = builder.schema;
@@ -874,11 +866,6 @@ public Long getTotalPartitionsProcessed() {
874866
return totalPartitionsProcessed;
875867
}
876868

877-
/** Returns the slot-milliseconds consumed by the query. */
878-
public Long getTotalSlotMs() {
879-
return totalSlotMs;
880-
}
881-
882869
/**
883870
* Returns the query plan as a list of stages or {@code null} if a query plan is not available.
884871
* Each stage involves a number of steps that read from data sources, perform a series of
@@ -984,7 +971,6 @@ com.google.api.services.bigquery.model.JobStatistics toPb() {
984971
queryStatisticsPb.setTotalBytesBilled(totalBytesBilled);
985972
queryStatisticsPb.setTotalBytesProcessed(totalBytesProcessed);
986973
queryStatisticsPb.setTotalPartitionsProcessed(totalPartitionsProcessed);
987-
queryStatisticsPb.setTotalSlotMs(totalSlotMs);
988974
if (ddlTargetTable != null) {
989975
queryStatisticsPb.setDdlTargetTable(ddlTargetTable.toPb());
990976
}
@@ -1589,6 +1575,7 @@ abstract static class Builder<T extends JobStatistics, B extends Builder<T, B>>
15891575
private List<ReservationUsage> reservationUsage;
15901576
private TransactionInfo transactionInfo;
15911577
private SessionInfo sessionInfo;
1578+
private Long totalSlotMs;
15921579

15931580
protected Builder() {}
15941581

@@ -1598,6 +1585,9 @@ protected Builder(com.google.api.services.bigquery.model.JobStatistics statistic
15981585
this.startTime = statisticsPb.getStartTime();
15991586
this.numChildJobs = statisticsPb.getNumChildJobs();
16001587
this.parentJobId = statisticsPb.getParentJobId();
1588+
if (statisticsPb.getTotalSlotMs() != null) {
1589+
this.totalSlotMs = statisticsPb.getTotalSlotMs();
1590+
}
16011591
if (statisticsPb.getScriptStatistics() != null) {
16021592
this.scriptStatistics = ScriptStatistics.fromPb(statisticsPb.getScriptStatistics());
16031593
}
@@ -1633,6 +1623,11 @@ B setStartTime(Long startTime) {
16331623
return self();
16341624
}
16351625

1626+
B setTotalSlotMs(Long totalSlotMs) {
1627+
this.totalSlotMs = totalSlotMs;
1628+
return self();
1629+
}
1630+
16361631
abstract T build();
16371632
}
16381633

@@ -1646,6 +1641,7 @@ protected JobStatistics(Builder builder) {
16461641
this.reservationUsage = builder.reservationUsage;
16471642
this.transactionInfo = builder.transactionInfo;
16481643
this.sessionInfo = builder.sessionInfo;
1644+
this.totalSlotMs = builder.totalSlotMs;
16491645
}
16501646

16511647
/** Returns the creation time of the job in milliseconds since epoch. */
@@ -1699,6 +1695,11 @@ public SessionInfo getSessionInfo() {
16991695
return sessionInfo;
17001696
}
17011697

1698+
/** Returns the slot-milliseconds for the job. */
1699+
public Long getTotalSlotMs() {
1700+
return totalSlotMs;
1701+
}
1702+
17021703
ToStringHelper toStringHelper() {
17031704
return MoreObjects.toStringHelper(this)
17041705
.add("creationTime", creationTime)
@@ -1709,7 +1710,8 @@ ToStringHelper toStringHelper() {
17091710
.add("scriptStatistics", scriptStatistics)
17101711
.add("reservationUsage", reservationUsage)
17111712
.add("transactionInfo", transactionInfo)
1712-
.add("sessionInfo", sessionInfo);
1713+
.add("sessionInfo", sessionInfo)
1714+
.add("totalSlotMs", totalSlotMs);
17131715
}
17141716

17151717
@Override
@@ -1727,7 +1729,8 @@ final int baseHashCode() {
17271729
scriptStatistics,
17281730
reservationUsage,
17291731
transactionInfo,
1730-
sessionInfo);
1732+
sessionInfo,
1733+
totalSlotMs);
17311734
}
17321735

17331736
final boolean baseEquals(JobStatistics jobStatistics) {
@@ -1742,6 +1745,7 @@ com.google.api.services.bigquery.model.JobStatistics toPb() {
17421745
statistics.setStartTime(startTime);
17431746
statistics.setNumChildJobs(numChildJobs);
17441747
statistics.setParentJobId(parentJobId);
1748+
statistics.setTotalSlotMs(totalSlotMs);
17451749
if (scriptStatistics != null) {
17461750
statistics.setScriptStatistics(scriptStatistics.toPb());
17471751
}

‎google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/JobStatisticsTest.java‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,7 @@ public class JobStatisticsTest {
102102
.setStartTime(START_TIME)
103103
.setCopiedRows(COPIED_ROW)
104104
.setCopiedLogicalBytes(COPIED_LOGICAL_BYTES)
105+
.setTotalSlotMs(TOTAL_SLOT_MS)
105106
.build();
106107
private static final ExtractStatistics EXTRACT_STATISTICS =
107108
ExtractStatistics.newBuilder()
@@ -110,6 +111,7 @@ public class JobStatisticsTest {
110111
.setStartTime(START_TIME)
111112
.setDestinationUriFileCounts(FILE_COUNT)
112113
.setInputBytes(INPUT_BYTES)
114+
.setTotalSlotMs(TOTAL_SLOT_MS)
113115
.build();
114116
private static final LoadStatistics LOAD_STATISTICS =
115117
LoadStatistics.newBuilder()
@@ -121,6 +123,7 @@ public class JobStatisticsTest {
121123
.setOutputBytes(OUTPUT_BYTES)
122124
.setOutputRows(OUTPUT_ROWS)
123125
.setBadRecords(BAD_RECORDS)
126+
.setTotalSlotMs(TOTAL_SLOT_MS)
124127
.build();
125128
private static final LoadStatistics LOAD_STATISTICS_INCOMPLETE =
126129
LoadStatistics.newBuilder()
@@ -130,6 +133,7 @@ public class JobStatisticsTest {
130133
.setInputBytes(INPUT_BYTES)
131134
.setInputFiles(INPUT_FILES)
132135
.setBadRecords(BAD_RECORDS)
136+
.setTotalSlotMs(TOTAL_SLOT_MS)
133137
.build();
134138
private static final List<String> SUBSTEPS1 = ImmutableList.of("substep1", "substep2");
135139
private static final List<String> SUBSTEPS2 = ImmutableList.of("substep3", "substep4");
@@ -272,18 +276,21 @@ public void testBuilder() {
272276
assertEquals(CREATION_TIME, EXTRACT_STATISTICS.getCreationTime());
273277
assertEquals(START_TIME, EXTRACT_STATISTICS.getStartTime());
274278
assertEquals(END_TIME, EXTRACT_STATISTICS.getEndTime());
279+
assertEquals(TOTAL_SLOT_MS, EXTRACT_STATISTICS.getTotalSlotMs());
275280
assertEquals(FILE_COUNT, EXTRACT_STATISTICS.getDestinationUriFileCounts());
276281
assertEquals(INPUT_BYTES, EXTRACT_STATISTICS.getInputBytes());
277282

278283
assertEquals(CREATION_TIME, COPY_STATISTICS.getCreationTime());
279284
assertEquals(START_TIME, COPY_STATISTICS.getStartTime());
280285
assertEquals(END_TIME, COPY_STATISTICS.getEndTime());
286+
assertEquals(TOTAL_SLOT_MS, COPY_STATISTICS.getTotalSlotMs());
281287
assertEquals(COPIED_LOGICAL_BYTES, COPY_STATISTICS.getCopiedLogicalBytes());
282288
assertEquals(COPIED_ROW, COPY_STATISTICS.getCopiedRows());
283289

284290
assertEquals(CREATION_TIME, LOAD_STATISTICS.getCreationTime());
285291
assertEquals(START_TIME, LOAD_STATISTICS.getStartTime());
286292
assertEquals(END_TIME, LOAD_STATISTICS.getEndTime());
293+
assertEquals(TOTAL_SLOT_MS, LOAD_STATISTICS.getTotalSlotMs());
287294
assertEquals(INPUT_BYTES, LOAD_STATISTICS.getInputBytes());
288295
assertEquals(INPUT_FILES, LOAD_STATISTICS.getInputFiles());
289296
assertEquals(OUTPUT_BYTES, LOAD_STATISTICS.getOutputBytes());
@@ -293,6 +300,7 @@ public void testBuilder() {
293300
assertEquals(CREATION_TIME, QUERY_STATISTICS.getCreationTime());
294301
assertEquals(START_TIME, QUERY_STATISTICS.getStartTime());
295302
assertEquals(END_TIME, QUERY_STATISTICS.getEndTime());
303+
assertEquals(TOTAL_SLOT_MS, QUERY_STATISTICS.getTotalSlotMs());
296304
assertEquals(BI_ENGINE_STATS, QUERY_STATISTICS.getBiEngineStats());
297305
assertEquals(BILLING_TIER, QUERY_STATISTICS.getBillingTier());
298306
assertEquals(CACHE_HIT, QUERY_STATISTICS.getCacheHit());
@@ -308,7 +316,6 @@ public void testBuilder() {
308316
assertEquals(TOTAL_BYTES_BILLED, QUERY_STATISTICS.getTotalBytesBilled());
309317
assertEquals(TOTAL_BYTES_PROCESSED, QUERY_STATISTICS.getTotalBytesProcessed());
310318
assertEquals(TOTAL_PARTITION_PROCESSED, QUERY_STATISTICS.getTotalPartitionsProcessed());
311-
assertEquals(TOTAL_SLOT_MS, QUERY_STATISTICS.getTotalSlotMs());
312319
assertEquals(QUERY_PLAN, QUERY_STATISTICS.getQueryPlan());
313320
assertEquals(TIMELINE, QUERY_STATISTICS.getTimeline());
314321

@@ -472,6 +479,7 @@ private void compareStatistics(JobStatistics expected, JobStatistics value) {
472479
assertEquals(expected.getNumChildJobs(), value.getNumChildJobs());
473480
assertEquals(expected.getParentJobId(), value.getParentJobId());
474481
assertEquals(expected.getScriptStatistics(), value.getScriptStatistics());
482+
assertEquals(expected.getTotalSlotMs(), value.getTotalSlotMs());
475483
}
476484

477485
private void compareScriptStatistics(ScriptStatistics expected, ScriptStatistics value) {

‎google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3160,6 +3160,23 @@ public void testQuery() throws InterruptedException {
31603160
assertNotNull(statistics.getQueryPlan());
31613161
}
31623162

3163+
@Test
3164+
public void testQueryStatistics() throws InterruptedException {
3165+
// Use CURRENT_TIMESTAMP to avoid potential caching.
3166+
String query = "SELECT CURRENT_TIMESTAMP() AS ts";
3167+
QueryJobConfiguration config =
3168+
QueryJobConfiguration.newBuilder(query)
3169+
.setDefaultDataset(DatasetId.of(DATASET))
3170+
.setUseQueryCache(false)
3171+
.build();
3172+
Job job = bigquery.create(JobInfo.of(JobId.of(), config));
3173+
job = job.waitFor();
3174+
3175+
JobStatistics.QueryStatistics statistics = job.getStatistics();
3176+
assertNotNull(statistics.getQueryPlan());
3177+
assertThat(statistics.getTotalSlotMs()).isGreaterThan(0L);
3178+
}
3179+
31633180
@Test
31643181
public void testExecuteSelectDefaultConnectionSettings() throws SQLException {
31653182
// Use the default connection settings
@@ -4429,6 +4446,7 @@ public void testLoadSessionSupport() throws InterruptedException {
44294446

44304447
Job loadJob = bigquery.getJob(job.getJobId());
44314448
JobStatistics.LoadStatistics statistics = loadJob.getStatistics();
4449+
assertThat(statistics.getTotalSlotMs()).isGreaterThan(0L);
44324450
String sessionId = statistics.getSessionInfo().getSessionId();
44334451
assertNotNull(sessionId);
44344452

@@ -5678,6 +5696,7 @@ public void testExtractJob() throws InterruptedException, TimeoutException {
56785696
assertEquals(1L, extractStatistics.getDestinationUriFileCounts().size());
56795697
assertEquals(
56805698
loadStatistics.getOutputBytes().longValue(), extractStatistics.getInputBytes().longValue());
5699+
assertThat(extractStatistics.getTotalSlotMs()).isGreaterThan(0L);
56815700

56825701
String extractedCsv =
56835702
new String(storage.readAllBytes(BUCKET, EXTRACT_FILE), StandardCharsets.UTF_8);

0 commit comments

Comments
 (0)