Skip to content

Commit 764385c

Browse files
babupe-gbabupe-g
committed
Interpreter modification, License, doc changes
1 parent d85abd2 commit 764385c

File tree

4 files changed

+88
-42
lines changed

4 files changed

+88
-42
lines changed

‎bigquery/src/main/java/org/apache/zeppelin/bigquery/bigQueryInterpreter.java‎

Lines changed: 56 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
import com.google.api.services.bigquery.model.GetQueryResultsResponse;
4141
import com.google.api.services.bigquery.model.QueryRequest;
4242
import com.google.api.services.bigquery.model.QueryResponse;
43+
import com.google.api.services.bigquery.model.JobCancelResponse;
4344
import com.google.gson.Gson;
4445

4546
import java.io.IOException;
@@ -76,12 +77,14 @@
7677
* BigQuery interpreter for Zeppelin.
7778
*
7879
* <ul>
79-
* <li>{@code bigquery.project_id} - Project ID in GCP</li>
80+
* <li>{@code zeppelin.bigquery.project_id} - Project ID in GCP</li>
81+
* <li>{@code zeppelin.bigquery.wait_time} - Query Timeout in ms</li>
82+
* <li>{@code zeppelin.bigquery.max_no_of_rows} - Max Result size</li>
8083
* </ul>
8184
*
8285
* <p>
8386
* How to use: <br/>
84-
* {@code %bqsql.sql<br/>
87+
* {@code %bigquery.sql<br/>
8588
* {@code
8689
* SELECT departure_airport,count(case when departure_delay>0 then 1 else 0 end) as no_of_delays
8790
* FROM [bigquery-samples:airline_ontime_data.flights]
@@ -103,22 +106,12 @@ public class bigQueryInterpreter extends Interpreter {
103106
//Mutex created to create the singleton in thread-safe fashion.
104107
private static Object serviceLock = new Object();
105108

106-
static final String PROJECT_ID = "bqsql.project_id";
107-
static final String DEFAULT_PROJECT_ID = "";
108-
static final String WAIT_TIME = "bqsql.query_wait_time";
109-
static final String DEFAULT_WAIT_TIME = "5000";
110-
111-
// Registering BigQuery Interpreter and defining attributes
112-
static {
113-
Interpreter.register(
114-
"sql",
115-
"bqsql",
116-
bigQueryInterpreter.class.getName(),
117-
new InterpreterPropertyBuilder()
118-
.add(PROJECT_ID, DEFAULT_PROJECT_ID, "Google Project ID")
119-
.add(WAIT_TIME, DEFAULT_WAIT_TIME, "Query timeout in Milliseconds")
120-
.build());
121-
}
109+
static final String PROJECT_ID = "zeppelin.bigquery.project_id";
110+
static final String WAIT_TIME = "zeppelin.bigquery.wait_time";
111+
static final String MAX_ROWS = "zeppelin.bigquery.max_no_of_rows";
112+
113+
private static String jobId = null;
114+
private static String projectId = null;
122115

123116
private static final List NO_COMPLETION = new ArrayList<>();
124117
private Exception exceptionOnConnect;
@@ -174,19 +167,24 @@ private static Bigquery createAuthorizedClient() throws IOException {
174167
public static String printRows(final GetQueryResultsResponse response) {
175168
StringBuilder msg = null;
176169
msg = new StringBuilder();
177-
for (TableFieldSchema schem: response.getSchema().getFields()) {
178-
msg.append(schem.getName());
179-
msg.append(TAB);
180-
}
181-
msg.append(NEWLINE);
182-
for (TableRow row : response.getRows()) {
183-
for (TableCell field : row.getF()) {
184-
msg.append(field.getV().toString());
170+
try {
171+
for (TableFieldSchema schem: response.getSchema().getFields()) {
172+
msg.append(schem.getName());
185173
msg.append(TAB);
186-
}
174+
}
187175
msg.append(NEWLINE);
176+
for (TableRow row : response.getRows()) {
177+
for (TableCell field : row.getF()) {
178+
msg.append(field.getV().toString());
179+
msg.append(TAB);
180+
}
181+
msg.append(NEWLINE);
182+
}
183+
return msg.toString();
184+
}
185+
catch ( NullPointerException ex ) {
186+
throw new NullPointerException("SQL Execution returned an error!");
188187
}
189-
return msg.toString();
190188
}
191189

192190
//Function to poll a job for completion. Future use
@@ -250,24 +248,32 @@ private InterpreterResult executeSql(String sql) {
250248
finalmessage = new StringBuilder("%table ");
251249
String projId = getProperty(PROJECT_ID);
252250
long wTime = Long.parseLong(getProperty(WAIT_TIME));
253-
Iterator<GetQueryResultsResponse> pages = run(sql, projId, wTime);
254-
while (pages.hasNext()) {
255-
finalmessage.append(printRows(pages.next()));
251+
long maxRows = Long.parseLong(getProperty(MAX_ROWS));
252+
Iterator<GetQueryResultsResponse> pages = run(sql, projId, wTime, maxRows);
253+
try {
254+
while (pages.hasNext()) {
255+
finalmessage.append(printRows(pages.next()));
256+
}
257+
return new InterpreterResult(Code.SUCCESS, finalmessage.toString());
258+
}
259+
catch ( NullPointerException ex ) {
260+
return new InterpreterResult(Code.ERROR, ex.getMessage());
256261
}
257-
return new InterpreterResult(Code.SUCCESS, finalmessage.toString());
258262
}
259263

260264
//Function to run the SQL on bigQuery service
261265
public static Iterator<GetQueryResultsResponse> run(final String queryString,
262-
final String projId, final long wTime) {
266+
final String projId, final long wTime, final long maxRows) {
263267
try {
264268
QueryResponse query = service.jobs().query(
265269
projId,
266-
new QueryRequest().setTimeoutMs(wTime).setQuery(queryString))
270+
new QueryRequest().setTimeoutMs(wTime).setQuery(queryString).setMaxResults(maxRows))
267271
.execute();
272+
jobId = query.getJobReference().getJobId();
273+
projectId = query.getJobReference().getProjectId();
268274
GetQueryResults getRequest = service.jobs().getQueryResults(
269-
query.getJobReference().getProjectId(),
270-
query.getJobReference().getJobId());
275+
projectId,
276+
jobId);
271277
return getPages(getRequest);
272278
}
273279
catch (IOException e) {
@@ -309,10 +315,21 @@ public int getProgress(InterpreterContext context) {
309315
@Override
310316
public void cancel(InterpreterContext context) {
311317

312-
logger.info("Cancel current query statement.");
318+
logger.info("Trying to Cancel current query statement.");
313319

314-
if (service != null) {
315-
service = null;
320+
if (service != null && jobId != null && projectId != null) {
321+
try {
322+
Bigquery.Jobs.Cancel request = service.jobs().cancel(projectId, jobId);
323+
JobCancelResponse response = request.execute();
324+
jobId = null;
325+
logger.info("Query Execution cancelled");
326+
}
327+
catch (IOException ex) {
328+
logger.error("Could not cancel the SQL execution");
329+
}
330+
}
331+
else {
332+
logger.info("Query Execution was already cancelled");
316333
}
317334
}
318335

‎docs/_includes/themes/zeppelin/_navigation.html‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@
6666
<li><a href="{{BASE_PATH}}/interpreter/scalding.html">Scalding</a></li>
6767
<li><a href="{{BASE_PATH}}/interpreter/shell.html">Shell</a></li>
6868
<li><a href="{{BASE_PATH}}/interpreter/spark.html">Spark</a></li>
69+
<li><a href="{{BASE_PATH}}/interpreter/bigquery.html">BigQuery</a></li>
6970
</ul>
7071
</li>
7172
<li>
@@ -113,4 +114,4 @@
113114
</nav><!--/.navbar-collapse -->
114115
</div>
115116
</div>
116-
117+

‎docs/interpreter/bigquery.md‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,31 @@ group: interpreter
1212
## Overview
1313
[BigQuery](https://cloud.google.com/bigquery/what-is-bigquery) is a highly scalable no-ops data warehouse in the Google Cloud Platform. Querying massive datasets can be time consuming and expensive without the right hardware and infrastructure. Google BigQuery solves this problem by enabling super-fast SQL queries against append-only tables using the processing power of Google's infrastructure. Simply move your data into BigQuery and let us handle the hard work. You can control access to both the project and your data based on your business needs, such as giving others the ability to view or query your data.
1414

15+
## Configuration
16+
<table class="table-configuration">
17+
<tr>
18+
<th>Name</th>
19+
<th>Default Value</th>
20+
<th>Description</th>
21+
</tr>
22+
<tr>
23+
<td>zeppelin.bigquery.project_id</td>
24+
<td> </td>
25+
<td>Google Project Id</td>
26+
</tr>
27+
<tr>
28+
<td>zeppelin.bigquery.wait_time</td>
29+
<td>5000</td>
30+
<td>Query Timeout in Milliseconds</td>
31+
</tr>
32+
<tr>
33+
<td>zeppelin.bigquery.max_no_of_rows</td>
34+
<td>100000</td>
35+
<td>Max result set size</td>
36+
</tr>
37+
</table>
38+
39+
1540
## BigQuery API
1641
Zeppelin is built against BigQuery API version v2-rev265-1.21.0.
1742

‎zeppelin-distribution/src/bin_license/LICENSE‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ The following components are provided under Apache License.
7474
(Apache 2.0) json-flattener (com.github.wnameless:json-flattener:0.1.6 - https://github.com/wnameless/json-flattener)
7575
(Apache 2.0) Spatial4J (com.spatial4j:spatial4j:0.4.1 - https://github.com/spatial4j/spatial4j)
7676
(Apache 2.0) T-Digest (com.tdunning:t-digest:3.0 - https://github.com/tdunning/t-digest)
77-
(Apache 2.0) Netty (io.netty:netty:3.8.0.Final - http://netty.io/)
77+
(Apache 2.0) Netty (io.netty:netty:3.10.5.Final - http://netty.io/)
7878
(Apache 2.0) Lucene Common Analyzers (org.apache.lucene:lucene-analyzers-common:5.3.1 - http://lucene.apache.org/lucene-parent/lucene-analyzers-common)
7979
(Apache 2.0) Lucene Memory (org.apache.lucene:lucene-backward-codecs:5.3.1 - http://lucene.apache.org/lucene-parent/lucene-backward-codecs)
8080
(Apache 2.0) Lucene Core (org.apache.lucene:lucene-core:5.3.1 - http://lucene.apache.org/lucene-parent/lucene-core)
@@ -101,8 +101,12 @@ The following components are provided under Apache License.
101101
(Apache 2.0) Alluxio Underfs Local (org.alluxio:alluxio-underfs-local:1.0.0 - http://alluxio.org)
102102
(Apache 2.0) Microsoft Azure Storage Library for Java (com.microsoft.azure:azure-storage:4.0.0 - https://github.com/Azure/azure-storage-java)
103103
(Apache 2.0) Roboto Font (https://github.com/google/roboto/)
104+
<<<<<<< HEAD
104105
(Apache 2.0) stream (com.clearspring.analytics:stream:2.7.0) - https://github.com/addthis/stream-lib/blob/v2.7.0/LICENSE.txt
105106
(Apache 2.0) io.dropwizard.metrics:3.1.2 - https://github.com/dropwizard/metrics/blob/v3.1.2/LICENSE
107+
=======
108+
(Apache 2.0) Google BigQuery API for Java (com.google.api.services.bigquery:v2-rev265-1.21.0 - https://cloud.google.com/bigquery/)
109+
>>>>>>> Interpreter modification, License, doc changes
106110

107111

108112
========================================================================
@@ -172,7 +176,6 @@ The following components are provided under the BSD-style License.
172176

173177
(New BSD License) JGit (org.eclipse.jgit:org.eclipse.jgit:jar:4.1.1.201511131810-r - https://eclipse.org/jgit/)
174178
(New BSD License) Kryo (com.esotericsoftware.kryo:kryo:3.0.3 - http://code.google.com/p/kryo/)
175-
(New BSD License) leveldbjni (org.fusesource.leveldbjni:leveldbjni-all:1.8) - https://github.com/fusesource/leveldbjni/blob/leveldbjni-1.8/license.txt
176179
(New BSD License) MinLog (com.esotericsoftware.minlog:minlog:1.3 - http://code.google.com/p/minlog/)
177180
(New BSD License) ReflectASM (com.esotericsoftware.reflectasm:reflectasm:1.07 - http://code.google.com/p/reflectasm/)
178181
(BSD-like) Scala Library (org.scala-lang:scala-library:2.11.7 - http://www.scala-lang.org/)

0 commit comments

Comments
 (0)