Skip to content

Commit 16a25a3

Browse files
shollymanpongad
shollyman
authored and
pongad
committed
BigQuery: Add query timeline to statistics information. (#3201)
1 parent c8e5d55 commit 16a25a3

5 files changed

Lines changed: 222 additions & 1 deletion

File tree

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/JobStatistics.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -323,6 +323,7 @@ public static class QueryStatistics extends JobStatistics {
323323
private final Long totalBytesBilled;
324324
private final Long totalBytesProcessed;
325325
private final List<QueryStage> queryPlan;
326+
private final List<TimelineSample> timeline;
326327
private final Schema schema;
327328

328329
static final class Builder extends JobStatistics.Builder<QueryStatistics, Builder> {
@@ -332,6 +333,7 @@ static final class Builder extends JobStatistics.Builder<QueryStatistics, Builde
332333
private Long totalBytesBilled;
333334
private Long totalBytesProcessed;
334335
private List<QueryStage> queryPlan;
336+
private List<TimelineSample> timeline;
335337
private Schema schema;
336338

337339
private Builder() {}
@@ -348,6 +350,11 @@ private Builder(com.google.api.services.bigquery.model.JobStatistics statisticsP
348350
Lists.transform(
349351
statisticsPb.getQuery().getQueryPlan(), QueryStage.FROM_PB_FUNCTION);
350352
}
353+
if (statisticsPb.getQuery().getTimeline() != null) {
354+
this.timeline =
355+
Lists.transform(
356+
statisticsPb.getQuery().getTimeline(), TimelineSample.FROM_PB_FUNCTION);
357+
}
351358
if (statisticsPb.getQuery().getSchema() != null) {
352359
this.schema = Schema.fromPb(statisticsPb.getQuery().getSchema());
353360
}
@@ -379,6 +386,11 @@ Builder setQueryPlan(List<QueryStage> queryPlan) {
379386
return self();
380387
}
381388

389+
Builder setTimeline(List<TimelineSample> timeline) {
390+
this.timeline = timeline;
391+
return self();
392+
}
393+
382394
Builder setSchema(Schema schema) {
383395
this.schema = schema;
384396
return self();
@@ -397,6 +409,7 @@ private QueryStatistics(Builder builder) {
397409
this.totalBytesBilled = builder.totalBytesBilled;
398410
this.totalBytesProcessed = builder.totalBytesProcessed;
399411
this.queryPlan = builder.queryPlan;
412+
this.timeline = builder.timeline;
400413
this.schema = builder.schema;
401414
}
402415

@@ -448,6 +461,14 @@ public List<QueryStage> getQueryPlan() {
448461
return queryPlan;
449462
}
450463

464+
/**
465+
* Return the timeline for the query, as a list of timeline samples. Each sample provides
466+
* information about the overall progress of the query. Information includes time of the
467+
* sample, progress reporting on active, completed, and
468+
* pending units of work, as well as the cumulative estimation of slot-milliseconds consumed
469+
* by the query.
470+
*/
471+
public List<TimelineSample> getTimeline() { return timeline; }
451472

452473
/**
453474
* Returns the schema for the query result. Present only for successful dry run of
@@ -465,6 +486,7 @@ ToStringHelper toStringHelper() {
465486
.add("totalBytesBilled", totalBytesBilled)
466487
.add("totalBytesProcessed", totalBytesProcessed)
467488
.add("queryPlan", queryPlan)
489+
.add("timeline", timeline)
468490
.add("schema", schema);
469491
}
470492

@@ -492,6 +514,9 @@ com.google.api.services.bigquery.model.JobStatistics toPb() {
492514
if (queryPlan != null) {
493515
queryStatisticsPb.setQueryPlan(Lists.transform(queryPlan, QueryStage.TO_PB_FUNCTION));
494516
}
517+
if (timeline != null) {
518+
queryStatisticsPb.setTimeline(Lists.transform(timeline, TimelineSample.TO_PB_FUNCTION));
519+
}
495520
if (schema != null) {
496521
queryStatisticsPb.setSchema(schema.toPb());
497522
}

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/TimelineSample.java‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
/*
2+
* Copyright 2018 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.google.cloud.bigquery;
17+
18+
import com.google.api.services.bigquery.model.QueryTimelineSample;
19+
import com.google.common.base.Function;
20+
import com.google.auto.value.AutoValue;
21+
import javax.annotation.Nullable;
22+
23+
/**
24+
* A specific timeline sample. This instruments work progress at a given point in time, providing
25+
* information about work units active/pending/completed as well as cumulative slot-milliseconds.
26+
*/
27+
@AutoValue
28+
public abstract class TimelineSample {
29+
30+
private static final long serialVersionUID = 1L;
31+
32+
@AutoValue.Builder
33+
public abstract static class Builder {
34+
35+
public abstract Builder setElapsedMs(Long elapsedMs);
36+
37+
public abstract Builder setActiveUnits(Long activeUnits);
38+
39+
public abstract Builder setCompletedUnits(Long completedUnits);
40+
41+
public abstract Builder setPendingUnits(Long pendingUnits);
42+
43+
public abstract Builder setSlotMillis(Long slotMillis);
44+
45+
public abstract TimelineSample build();
46+
47+
}
48+
49+
/**
50+
* Returns the sample time as milliseconds elapsed since the start of query execution.
51+
*/
52+
@Nullable
53+
public abstract Long getElapsedMs();
54+
55+
/**
56+
* Returns the total number of work units currently being processed.
57+
*/
58+
@Nullable
59+
public abstract Long getActiveUnits();
60+
61+
/**
62+
* Returns the total number of work units completed by this query.
63+
*/
64+
@Nullable
65+
public abstract Long getCompletedUnits();
66+
67+
/**
68+
* Returns the number of work units remaining for the currently active stages.
69+
*/
70+
@Nullable
71+
public abstract Long getPendingUnits();
72+
73+
/**
74+
* Returns the cumulative slot-milliseconds consumed by the query.
75+
*/
76+
@Nullable
77+
public abstract Long getSlotMillis();
78+
79+
/**
80+
* return a builder for the {@TimelineSample} object.
81+
*/
82+
public abstract Builder toBuilder();
83+
84+
static Builder newBuilder() {
85+
return new AutoValue_TimelineSample.Builder();
86+
}
87+
88+
static TimelineSample fromPb(QueryTimelineSample sample) {
89+
Builder builder = newBuilder();
90+
builder.setElapsedMs(sample.getElapsedMs());
91+
builder.setActiveUnits(sample.getActiveUnits());
92+
builder.setCompletedUnits(sample.getCompletedUnits());
93+
builder.setPendingUnits(sample.getPendingUnits());
94+
builder.setSlotMillis(sample.getTotalSlotMs());
95+
return builder.build();
96+
}
97+
98+
QueryTimelineSample toPb() {
99+
QueryTimelineSample sample = new QueryTimelineSample()
100+
.setElapsedMs(getElapsedMs())
101+
.setActiveUnits(getActiveUnits())
102+
.setCompletedUnits(getCompletedUnits())
103+
.setPendingUnits(getPendingUnits())
104+
.setTotalSlotMs(getSlotMillis());
105+
return sample;
106+
}
107+
108+
static final Function<QueryTimelineSample, TimelineSample> FROM_PB_FUNCTION =
109+
new Function<QueryTimelineSample, TimelineSample>() {
110+
@Override
111+
public TimelineSample apply(QueryTimelineSample pb) {
112+
return TimelineSample.fromPb(pb);
113+
}
114+
};
115+
116+
static final Function<TimelineSample, QueryTimelineSample> TO_PB_FUNCTION =
117+
new Function<TimelineSample, QueryTimelineSample>() {
118+
@Override
119+
public QueryTimelineSample apply(TimelineSample sample) {
120+
return sample.toPb();
121+
}
122+
};
123+
124+
}

‎google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/JobStatisticsTest.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,22 @@ public class JobStatisticsTest {
8888
.setWriteRatioAvg(9.9)
8989
.setWriteRatioMax(10.10)
9090
.build();
91+
92+
private static final TimelineSample TIMELINE_SAMPLE1 = TimelineSample.newBuilder()
93+
.setElapsedMs(1001L)
94+
.setActiveUnits(100L)
95+
.setCompletedUnits(200L)
96+
.setPendingUnits(50L)
97+
.setSlotMillis(12345L)
98+
.build();
99+
private static final TimelineSample TIMELINE_SAMPLE2 = TimelineSample.newBuilder()
100+
.setElapsedMs(2002L)
101+
.setActiveUnits(48L)
102+
.setCompletedUnits(302L)
103+
.setPendingUnits(0L)
104+
.setSlotMillis(23456L)
105+
.build();
106+
private static final List<TimelineSample> TIMELINE = ImmutableList.of(TIMELINE_SAMPLE1, TIMELINE_SAMPLE2);
91107
private static final List<QueryStage> QUERY_PLAN = ImmutableList.of(QUERY_STAGE);
92108
private static final Schema SCHEMA = Schema.of(Field.of("column", LegacySQLTypeName.DATETIME));
93109
private static final QueryStatistics QUERY_STATISTICS = QueryStatistics.newBuilder()
@@ -99,6 +115,7 @@ public class JobStatisticsTest {
99115
.setTotalBytesBilled(TOTAL_BYTES_BILLED)
100116
.setTotalBytesProcessed(TOTAL_BYTES_PROCESSED)
101117
.setQueryPlan(QUERY_PLAN)
118+
.setTimeline(TIMELINE)
102119
.setSchema(SCHEMA)
103120
.build();
104121
private static final QueryStatistics QUERY_STATISTICS_INCOMPLETE = QueryStatistics.newBuilder()
@@ -133,6 +150,7 @@ public void testBuilder() {
133150
assertEquals(TOTAL_BYTES_PROCESSED, QUERY_STATISTICS.getTotalBytesProcessed());
134151
assertEquals(TOTAL_BYTES_PROCESSED, QUERY_STATISTICS.getTotalBytesProcessed());
135152
assertEquals(QUERY_PLAN, QUERY_STATISTICS.getQueryPlan());
153+
assertEquals(TIMELINE, QUERY_STATISTICS.getTimeline());
136154

137155
assertEquals(CREATION_TIME, LOAD_STATISTICS_INCOMPLETE.getCreationTime());
138156
assertEquals(START_TIME, LOAD_STATISTICS_INCOMPLETE.getStartTime());

‎google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/TimelineSampleTest.java‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
/*
2+
* Copyright 2018 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.google.cloud.bigquery;
17+
18+
import static org.junit.Assert.assertEquals;
19+
20+
import org.junit.Test;
21+
22+
public class TimelineSampleTest {
23+
24+
private static final Long ELAPSED_MS = 1001L;
25+
private static final Long ACTIVE_UNITS = 500L;
26+
private static final Long COMPLETED_UNITS = 843L;
27+
private static final Long PENDING_UNITS = 14L;
28+
private static final Long SLOT_MILLIS = 1220012L;
29+
30+
private static final TimelineSample TIMELINE_SAMPLE = TimelineSample.newBuilder()
31+
.setElapsedMs(ELAPSED_MS)
32+
.setActiveUnits(ACTIVE_UNITS)
33+
.setCompletedUnits(COMPLETED_UNITS)
34+
.setPendingUnits(PENDING_UNITS)
35+
.setSlotMillis(SLOT_MILLIS)
36+
.build();
37+
38+
39+
40+
@Test
41+
public void testTimelineSampleBuilder() {
42+
assertEquals(ELAPSED_MS, TIMELINE_SAMPLE.getElapsedMs());
43+
assertEquals(ACTIVE_UNITS, TIMELINE_SAMPLE.getActiveUnits());
44+
assertEquals(COMPLETED_UNITS, TIMELINE_SAMPLE.getCompletedUnits());
45+
assertEquals(PENDING_UNITS, TIMELINE_SAMPLE.getPendingUnits());
46+
assertEquals(SLOT_MILLIS, TIMELINE_SAMPLE.getSlotMillis());
47+
}
48+
49+
@Test
50+
public void TestEquals() {
51+
assertEquals(TIMELINE_SAMPLE, TIMELINE_SAMPLE);
52+
}
53+
54+
}

‎pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -181,7 +181,7 @@
181181
<dependency>
182182
<groupId>com.google.apis</groupId>
183183
<artifactId>google-api-services-bigquery</artifactId>
184-
<version>v2-rev377-1.23.0</version>
184+
<version>v2-rev383-1.23.0</version>
185185
</dependency>
186186
<dependency>
187187
<groupId>com.google.apis</groupId>

0 commit comments

Comments
 (0)