Skip to content
This repository was archived by the owner on Feb 24, 2026. It is now read-only.

Commit fd1ecf2

Browse files
stephaniewang526gcf-owl-bot[bot]
stephaniewang526
and
gcf-owl-bot[bot]
authored
docs(tutorials): add new tutorials directory and JsonWriterDefaultStream tutorial (#1498)
* docs(tutorials): add new tutorials directory and JsonWriterDefaultStream tutorial porting over tutorial from #1497 * exclude the templated files from owlbot * rename className * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
1 parent 0d414f8 commit fd1ecf2

7 files changed

Lines changed: 223 additions & 1 deletion

File tree

‎.github/workflows/approve-readme.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
on:
22
pull_request:
3+
paths-ignore:
4+
- 'tutorials/**'
35
name: auto-merge-readme
46
jobs:
57
approve:

‎.github/workflows/auto-release.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
on:
22
pull_request:
3+
paths-ignore:
4+
- 'tutorials/**'
35
name: auto-release
46
jobs:
57
approve:

‎.github/workflows/ci.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
on:
22
push:
3+
paths-ignore:
4+
- 'tutorials/**'
35
branches:
46
- main
57
pull_request:

‎.github/workflows/samples.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
on:
22
pull_request:
3+
paths-ignore:
4+
- 'tutorials/**'
35
name: samples
46
jobs:
57
checkstyle:

‎owlbot.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,10 @@
2727
".kokoro/build.sh",
2828
".kokoro/nightly/samples.cfg",
2929
".kokoro/presubmit/samples.cfg",
30-
".kokoro/dependencies.sh"
30+
".kokoro/dependencies.sh",
31+
".github/workflows/approve-readme.yaml",
32+
".github/workflows/auto-release.yaml",
33+
".github/workflows/ci.yaml",
34+
".github/workflows/samples.yaml"
3135
]
3236
)

‎tutorials/JsonWriterDefaultStream/JsonWriterDefaultStream.java‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
/*
2+
* Copyright 2022 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.example.bigquerystorage;
18+
19+
import com.google.api.core.ApiFuture;
20+
import com.google.cloud.bigquery.BigQuery;
21+
import com.google.cloud.bigquery.BigQueryOptions;
22+
import com.google.cloud.bigquery.Field;
23+
import com.google.cloud.bigquery.Schema;
24+
import com.google.cloud.bigquery.StandardSQLTypeName;
25+
import com.google.cloud.bigquery.StandardTableDefinition;
26+
import com.google.cloud.bigquery.Table;
27+
import com.google.cloud.bigquery.TableId;
28+
import com.google.cloud.bigquery.TableInfo;
29+
import com.google.cloud.bigquery.storage.v1.AppendRowsResponse;
30+
import com.google.cloud.bigquery.storage.v1.JsonStreamWriter;
31+
import com.google.cloud.bigquery.storage.v1.TableName;
32+
import com.google.cloud.bigquery.storage.v1.TableSchema;
33+
import com.google.protobuf.Descriptors.DescriptorValidationException;
34+
import java.io.BufferedReader;
35+
import java.io.FileReader;
36+
import java.io.IOException;
37+
import java.util.concurrent.ExecutionException;
38+
import org.json.JSONArray;
39+
import org.json.JSONObject;
40+
41+
public class JsonWriterDefaultStream {
42+
43+
public static void main(String[] args) throws Exception {
44+
if (args.length < 4) {
45+
System.out.println("Arguments: project, dataset, table, source_file");
46+
return;
47+
}
48+
49+
String projectId = args[0];
50+
String datasetName = args[1];
51+
String tableName = args[2];
52+
String dataFile = args[3];
53+
createDestinationTable(projectId, datasetName, tableName);
54+
writeToDefaultStream(projectId, datasetName, tableName, dataFile);
55+
}
56+
57+
// createDestinationTable: Creates the destination table for streaming with the desired schema.
58+
public static void createDestinationTable(
59+
String projectId, String datasetName, String tableName) {
60+
BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService();
61+
// Create a schema that matches the source data.
62+
Schema schema =
63+
Schema.of(
64+
Field.of("commit", StandardSQLTypeName.STRING),
65+
Field.newBuilder("parent", StandardSQLTypeName.STRING)
66+
.setMode(Field.Mode.REPEATED)
67+
.build(),
68+
Field.of("author", StandardSQLTypeName.STRING),
69+
Field.of("committer", StandardSQLTypeName.STRING),
70+
Field.of("time_sec", StandardSQLTypeName.INT64),
71+
Field.of("subject", StandardSQLTypeName.STRING),
72+
Field.of("message", StandardSQLTypeName.STRING),
73+
Field.of("repo_name", StandardSQLTypeName.STRING));
74+
75+
// Create a table that uses this schema.
76+
TableId tableId = TableId.of(projectId, datasetName, tableName);
77+
Table table = bigquery.getTable(tableId);
78+
if (table == null) {
79+
TableInfo tableInfo =
80+
TableInfo.newBuilder(tableId, StandardTableDefinition.of(schema)).build();
81+
bigquery.create(tableInfo);
82+
}
83+
}
84+
85+
// writeToDefaultStream: Writes records from the source file to the destination table.
86+
public static void writeToDefaultStream(
87+
String projectId, String datasetName, String tableName, String dataFile)
88+
throws DescriptorValidationException, InterruptedException, IOException {
89+
90+
BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService();
91+
92+
// Get the schema of the destination table and convert to the equivalent BigQueryStorage type.
93+
Table table = bigquery.getTable(datasetName, tableName);
94+
Schema schema = table.getDefinition().getSchema();
95+
TableSchema tableSchema = BqToBqStorageSchemaConverter.convertTableSchema(schema);
96+
97+
// Use the JSON stream writer to send records in JSON format.
98+
TableName parentTable = TableName.of(projectId, datasetName, tableName);
99+
try (JsonStreamWriter writer =
100+
JsonStreamWriter.newBuilder(parentTable.toString(), tableSchema).build()) {
101+
// Read JSON data from the source file and send it to the Write API.
102+
BufferedReader reader = new BufferedReader(new FileReader(dataFile));
103+
String line = reader.readLine();
104+
while (line != null) {
105+
// As a best practice, send batches of records, instead of single records at a time.
106+
JSONArray jsonArr = new JSONArray();
107+
for (int i = 0; i < 100; i++) {
108+
JSONObject record = new JSONObject(line);
109+
jsonArr.put(record);
110+
line = reader.readLine();
111+
if (line == null) {
112+
break;
113+
}
114+
} // batch
115+
ApiFuture<AppendRowsResponse> future = writer.append(jsonArr);
116+
AppendRowsResponse response = future.get();
117+
}
118+
System.out.println("Appended records successfully.");
119+
} catch (ExecutionException e) {
120+
// If the wrapped exception is a StatusRuntimeException, check the state of the operation.
121+
// If the state is INTERNAL, CANCELLED, or ABORTED, you can retry. For more information, see:
122+
// https://grpc.github.io/grpc-java/javadoc/io/grpc/StatusRuntimeException.html
123+
System.out.println("Failed to append records. \n" + e.toString());
124+
}
125+
}
126+
}

‎tutorials/JsonWriterDefaultStream/pom.xml‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
<project>
2+
<modelVersion>4.0.0</modelVersion>
3+
<groupId>com.google.cloud</groupId>
4+
<artifactId>google-cloud-bigquerystorage</artifactId>
5+
6+
<parent>
7+
<groupId>com.google.cloud.samples</groupId>
8+
<artifactId>shared-configuration</artifactId>
9+
<version>1.2.0</version>
10+
</parent>
11+
12+
<properties>
13+
<maven.compiler.target>1.8</maven.compiler.target>
14+
<maven.compiler.source>1.8</maven.compiler.source>
15+
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
16+
</properties>
17+
18+
<dependencies>
19+
<dependency>
20+
<groupId>com.google.cloud</groupId>
21+
<artifactId>google-cloud-bigquerystorage</artifactId>
22+
<version>2.8.2</version>
23+
</dependency>
24+
<dependency>
25+
<groupId>com.google.cloud</groupId>
26+
<artifactId>google-cloud-bigquery</artifactId>
27+
<version>2.6.0</version>
28+
</dependency>
29+
<dependency>
30+
<groupId>org.apache.avro</groupId>
31+
<artifactId>avro</artifactId>
32+
<version>1.10.2</version>
33+
</dependency>
34+
<dependency>
35+
<groupId>org.apache.arrow</groupId>
36+
<artifactId>arrow-vector</artifactId>
37+
<version>6.0.1</version>
38+
</dependency>
39+
<dependency>
40+
<groupId>org.apache.arrow</groupId>
41+
<artifactId>arrow-memory-netty</artifactId>
42+
<version>6.0.1</version>
43+
</dependency>
44+
</dependencies>
45+
46+
<build>
47+
<plugins>
48+
<plugin>
49+
<groupId>org.codehaus.mojo</groupId>
50+
<artifactId>build-helper-maven-plugin</artifactId>
51+
<version>3.0.0</version>
52+
<executions>
53+
<execution>
54+
<phase>generate-sources</phase>
55+
<goals>
56+
<goal>add-source</goal>
57+
</goals>
58+
<configuration>
59+
<sources>
60+
<source>src/main/java</source>
61+
<source>../snippets/src/main/java/com/example/bigquerystorage</source>
62+
</sources>
63+
</configuration>
64+
</execution>
65+
</executions>
66+
</plugin>
67+
<plugin>
68+
<groupId>org.codehaus.mojo</groupId>
69+
<artifactId>exec-maven-plugin</artifactId>
70+
<version>3.0.0</version>
71+
<executions>
72+
<execution>
73+
<goals>
74+
<goal>exec</goal>
75+
</goals>
76+
</execution>
77+
</executions>
78+
<configuration>
79+
<executable>maven</executable>
80+
</configuration>
81+
</plugin>
82+
</plugins>
83+
</build>
84+
</project>

0 commit comments

Comments
 (0)