Skip to content

Commit 2adca7a

Browse files
pongadpongad
authored
---
yaml --- r: 8451 b: refs/heads/master c: ce99df1 h: refs/heads/master i: 8449: 7a03ce4 8447: bd370ef
1 parent f8e0159 commit 2adca7a

5 files changed

Lines changed: 131 additions & 1 deletion

File tree

‎[refs]‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
---
2-
refs/heads/master: 96e07c0537fab956d11aa68ca88f505e53435296
2+
refs/heads/master: ce99df1a1ef7a52720d6e6736eb1443b8dccece3
33
refs/heads/travis: 47e4fee4fd5af9b2a8ce46f23c72ec95f9b195b2
44
refs/heads/gh-pages: 3e16a39145437096333db5811e5c0292719c1823
55
refs/tags/0.0.9: 22f1839238f66c39e67ed4dfdcd273b1ae2e8444

‎trunk/google-cloud-bigquery/pom.xml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,15 @@
7777
</resource>
7878
</resources>
7979
<plugins>
80+
<plugin>
81+
<!-- Allow script to run, so we can run benchmarks. -->
82+
<groupId>org.codehaus.mojo</groupId>
83+
<artifactId>exec-maven-plugin</artifactId>
84+
<configuration>
85+
<cleanupDaemonThreads>false</cleanupDaemonThreads>
86+
<skip>false</skip>
87+
</configuration>
88+
</plugin>
8089
<plugin>
8190
<artifactId>maven-antrun-plugin</artifactId>
8291
<version>1.4</version>
@@ -96,6 +105,25 @@
96105
</execution>
97106
</executions>
98107
</plugin>
108+
<plugin>
109+
<groupId>org.codehaus.mojo</groupId>
110+
<artifactId>build-helper-maven-plugin</artifactId>
111+
<version>1.7</version>
112+
<executions>
113+
<execution>
114+
<id>add-source</id>
115+
<phase>generate-sources</phase>
116+
<goals>
117+
<goal>add-source</goal>
118+
</goals>
119+
<configuration>
120+
<sources>
121+
<source>src/benchmark/java</source>
122+
</sources>
123+
</configuration>
124+
</execution>
125+
</executions>
126+
</plugin>
99127
</plugins>
100128
</build>
101129
</project>

‎trunk/google-cloud-bigquery/src/benchmark/java/com/google/cloud/bigquery/benchmark/Benchmark.java‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
/*
2+
* Copyright 2017 Google Inc. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.bigquery.benchmark;
18+
19+
import com.google.api.client.json.jackson.JacksonFactory;
20+
import com.google.cloud.bigquery.BigQuery;
21+
import com.google.cloud.bigquery.BigQueryOptions;
22+
import com.google.cloud.bigquery.FieldValue;
23+
import com.google.cloud.bigquery.QueryJobConfiguration;
24+
import com.google.cloud.bigquery.QueryResponse;
25+
import java.io.FileInputStream;
26+
import java.util.List;
27+
import org.threeten.bp.Clock;
28+
import org.threeten.bp.Duration;
29+
import org.threeten.bp.Instant;
30+
31+
public class Benchmark {
32+
33+
private static final double NS_PER_SECOND = 1000 * 1000 * 1000;
34+
35+
private Benchmark() {}
36+
37+
public static void main(String[] args) throws Exception {
38+
if (args.length < 1) {
39+
System.out.println("need path to queries.json");
40+
return;
41+
}
42+
String[] requests =
43+
new JacksonFactory()
44+
.createJsonParser(new FileInputStream(args[0]))
45+
.parseAndClose(String[].class);
46+
47+
Clock clock = Clock.systemUTC();
48+
BigQuery bq = BigQueryOptions.getDefaultInstance().getService();
49+
50+
for (String request : requests) {
51+
if (request.isEmpty()) {
52+
continue;
53+
}
54+
55+
Instant start = clock.instant();
56+
QueryResponse queryResponse =
57+
bq.query(QueryJobConfiguration.newBuilder(request).setUseLegacySql(false).build());
58+
59+
int rows = 0;
60+
int cols = 0;
61+
Duration firstByte = null;
62+
for (List<FieldValue> row : queryResponse.getResult().iterateAll()) {
63+
rows++;
64+
if (cols == 0) {
65+
cols = row.size();
66+
firstByte = Duration.between(start, clock.instant());
67+
} else if (cols != row.size()) {
68+
throw new IllegalStateException(
69+
String.format("expected %d cols, found %d", cols, row.size()));
70+
}
71+
}
72+
Duration total = Duration.between(start, clock.instant());
73+
74+
double firstByteSec = (double) (firstByte.getNano()) / NS_PER_SECOND + firstByte.getSeconds();
75+
double totalSec = (double) (total.getNano()) / NS_PER_SECOND + total.getSeconds();
76+
77+
System.out.println(
78+
String.format(
79+
"query \"%s\": read %d rows, %d cols, first byte %f sec, total %f sec",
80+
request, rows, cols, firstByteSec, totalSec));
81+
}
82+
}
83+
}

‎trunk/google-cloud-bigquery/src/benchmark/java/com/google/cloud/bigquery/benchmark/README.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
# BigQuery Benchmark
2+
This directory contains benchmarks for BigQuery client.
3+
4+
## Usage
5+
From the `google-cloud-bigquery` directory, run
6+
`mvn compile exec:java -Dexec.mainClass=com.google.cloud.bigquery.benchmark.Benchmark -Dexec.args="src/main/java/com/google/cloud/bigquery/benchmark/queries.json"`
7+
8+
BigQuery service caches requests so the benchmark should be run
9+
at least twice, disregarding the first result.

‎trunk/google-cloud-bigquery/src/benchmark/java/com/google/cloud/bigquery/benchmark/queries.json‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
[
2+
"SELECT * FROM `nyc-tlc.yellow.trips` LIMIT 10000",
3+
"SELECT * FROM `nyc-tlc.yellow.trips` LIMIT 100000",
4+
"SELECT * FROM `nyc-tlc.yellow.trips` LIMIT 1000000",
5+
"SELECT title FROM `bigquery-public-data.samples.wikipedia` ORDER BY title LIMIT 1000",
6+
"SELECT title, id, timestamp, contributor_ip FROM `bigquery-public-data.samples.wikipedia` WHERE title like 'Blo%' ORDER BY id",
7+
"SELECT * FROM `bigquery-public-data.baseball.games_post_wide` ORDER BY gameId",
8+
"SELECT * FROM `bigquery-public-data.samples.github_nested` WHERE repository.has_downloads ORDER BY repository.created_at LIMIT 10000",
9+
"SELECT repo_name, path FROM `bigquery-public-data.github_repos.files` WHERE path LIKE '%.java' ORDER BY id LIMIT 1000000"
10+
]

0 commit comments

Comments
 (0)