Skip to content

Commit a1299c7

Browse files
mziccardmziccard
committed
Merge pull request #502 from mziccard/bigquery
Merge master into bigquery and add BigQuery to README and TESTING
2 parents a0f5f1b + 7b1b0dc commit a1299c7

64 files changed

Lines changed: 4202 additions & 236 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎README.md‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ This client supports the following Google Cloud Platform services:
1414

1515
- [Google Cloud Datastore] (#google-cloud-datastore)
1616
- [Google Cloud Storage] (#google-cloud-storage)
17+
- [Google Cloud Resource Manager] (#google-cloud-resource-manager)
18+
- [Google Cloud BigQuery] (#google-cloud-bigquery)
1719

1820
> Note: This client is a work-in-progress, and may occasionally
1921
> make backwards-incompatible changes.
@@ -182,6 +184,82 @@ if (blob == null) {
182184
}
183185
```
184186
187+
Google Cloud Resource Manager
188+
----------------------
189+
190+
- [API Documentation][resourcemanager-api]
191+
- [Official Documentation][cloud-resourcemanager-docs]
192+
193+
#### Preview
194+
195+
Here is a code snippet showing a simple usage example. Note that you must supply Google SDK credentials for this service, not other forms of authentication listed in the [Authentication section](#authentication).
196+
197+
```java
198+
import com.google.gcloud.resourcemanager.ProjectInfo;
199+
import com.google.gcloud.resourcemanager.ResourceManager;
200+
import com.google.gcloud.resourcemanager.ResourceManagerOptions;
201+
202+
import java.util.Iterator;
203+
204+
ResourceManager resourceManager = ResourceManagerOptions.defaultInstance().service();
205+
ProjectInfo myProject = resourceManager.get("some-project-id"); // Use an existing project's ID
206+
ProjectInfo newProjectInfo = resourceManager.replace(myProject.toBuilder()
207+
.addLabel("launch-status", "in-development").build());
208+
System.out.println("Updated the labels of project " + newProjectInfo.projectId()
209+
+ " to be " + newProjectInfo.labels());
210+
// List all the projects you have permission to view.
211+
Iterator<ProjectInfo> projectIterator = resourceManager.list().iterateAll();
212+
System.out.println("Projects I can view:");
213+
while (projectIterator.hasNext()) {
214+
System.out.println(projectIterator.next().projectId());
215+
}
216+
```
217+
218+
Google Cloud BigQuery
219+
----------------------
220+
221+
- [API Documentation][bigquery-api]
222+
- [Official Documentation][cloud-bigquery-docs]
223+
224+
#### Preview
225+
226+
Here is a code snippet showing a simple usage example from within Compute/App Engine. Note that you
227+
must [supply credentials](#authentication) and a project ID if running this snippet elsewhere.
228+
229+
```java
230+
import com.google.gcloud.bigquery.BaseTableInfo;
231+
import com.google.gcloud.bigquery.BigQuery;
232+
import com.google.gcloud.bigquery.BigQueryOptions;
233+
import com.google.gcloud.bigquery.Field;
234+
import com.google.gcloud.bigquery.JobStatus;
235+
import com.google.gcloud.bigquery.LoadJobInfo;
236+
import com.google.gcloud.bigquery.Schema;
237+
import com.google.gcloud.bigquery.TableId;
238+
import com.google.gcloud.bigquery.TableInfo;
239+
240+
BigQuery bigquery = BigQueryOptions.defaultInstance().service();
241+
TableId tableId = TableId.of("dataset", "table");
242+
BaseTableInfo info = bigquery.getTable(tableId);
243+
if (info == null) {
244+
System.out.println("Creating table " + tableId);
245+
Field integerField = Field.of("fieldName", Field.Type.integer());
246+
bigquery.create(TableInfo.of(tableId, Schema.of(integerField)));
247+
} else {
248+
System.out.println("Loading data into table " + tableId);
249+
LoadJobInfo loadJob = LoadJobInfo.of(tableId, "gs://bucket/path");
250+
loadJob = bigquery.create(loadJob);
251+
while (loadJob.status().state() != JobStatus.State.DONE) {
252+
Thread.sleep(1000L);
253+
loadJob = bigquery.getJob(loadJob.jobId());
254+
}
255+
if (loadJob.status().error() != null) {
256+
System.out.println("Job completed with errors");
257+
} else {
258+
System.out.println("Job succeeded");
259+
}
260+
}
261+
```
262+
185263
Troubleshooting
186264
---------------
187265
@@ -241,3 +319,10 @@ Apache 2.0 - See [LICENSE] for more information.
241319
[cloud-storage-create-bucket]: https://cloud.google.com/storage/docs/cloud-console#_creatingbuckets
242320
[cloud-storage-activation]: https://cloud.google.com/storage/docs/signup
243321
[storage-api]: http://googlecloudplatform.github.io/gcloud-java/apidocs/index.html?com/google/gcloud/storage/package-summary.html
322+
323+
[resourcemanager-api]:http://googlecloudplatform.github.io/gcloud-java/apidocs/index.html?com/google/gcloud/resourcemanager/package-summary.html
324+
[cloud-resourcemanager-docs]:https://cloud.google.com/resource-manager/
325+
326+
[cloud-bigquery]: https://cloud.google.com/bigquery/
327+
[cloud-bigquery-docs]: https://cloud.google.com/bigquery/docs/overview
328+
[bigquery-api]: http://googlecloudplatform.github.io/gcloud-java/apidocs/index.html?com/google/gcloud/bigquery/package-summary.html

‎TESTING.md‎

Lines changed: 70 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,11 @@
11
## gcloud-java tools for testing
22

3-
This library provides tools to help write tests for code that uses gcloud-java services.
3+
This library provides tools to help write tests for code that uses the following gcloud-java services:
4+
5+
- [Datastore] (#testing-code-that-uses-datastore)
6+
- [Storage] (#testing-code-that-uses-storage)
7+
- [Resource Manager] (#testing-code-that-uses-resource-manager)
8+
- [BigQuery] (#testing-code-that-uses-bigquery)
49

510
### Testing code that uses Datastore
611

@@ -51,7 +56,8 @@ Currently, there isn't an emulator for Google Cloud Storage, so an alternative i
5156
3. Create a `RemoteGcsHelper` object using your project ID and JSON key.
5257
Here is an example that uses the `RemoteGcsHelper` to create a bucket.
5358
```java
54-
RemoteGcsHelper gcsHelper = RemoteGcsHelper.create(PROJECT_ID, "/path/to/my/JSON/key.json");
59+
RemoteGcsHelper gcsHelper =
60+
RemoteGcsHelper.create(PROJECT_ID, new FileInputStream("/path/to/my/JSON/key.json"));
5561
Storage storage = gcsHelper.options().service();
5662
String bucket = RemoteGcsHelper.generateBucketName();
5763
storage.create(BucketInfo.of(bucket));
@@ -65,5 +71,67 @@ Here is an example that clears the bucket created in Step 3 with a timeout of 5
6571
RemoteGcsHelper.forceDelete(storage, bucket, 5, TimeUnit.SECONDS);
6672
```
6773

74+
### Testing code that uses Resource Manager
75+
76+
#### On your machine
77+
78+
You can test against a temporary local Resource Manager by following these steps:
79+
80+
1. Before running your testing code, start the Resource Manager emulator `LocalResourceManagerHelper`. This can be done as follows:
81+
82+
```java
83+
import com.google.gcloud.resourcemanager.testing.LocalResourceManagerHelper;
84+
85+
LocalResourceManagerHelper helper = LocalResourceManagerHelper.create();
86+
helper.start();
87+
```
88+
89+
This will spawn a server thread that listens to `localhost` at an ephemeral port for Resource Manager requests.
90+
91+
2. In your program, create and use a Resource Manager service object whose host is set to `localhost` at the appropriate port. For example:
92+
93+
```java
94+
ResourceManager resourceManager = LocalResourceManagerHelper.options().service();
95+
```
96+
97+
3. Run your tests.
98+
99+
4. Stop the Resource Manager emulator.
100+
101+
```java
102+
helper.stop();
103+
```
104+
105+
This method will block until the server thread has been terminated.
106+
107+
### Testing code that uses BigQuery
108+
109+
Currently, there isn't an emulator for Google BigQuery, so an alternative is to create a test
110+
project. `RemoteBigQueryHelper` contains convenience methods to make setting up and cleaning up the
111+
test project easier. To use this class, follow the steps below:
112+
113+
1. Create a test Google Cloud project.
114+
115+
2. Download a [JSON service account credentials file][create-service-account] from the Google
116+
Developer's Console.
117+
118+
3. Create a `RemoteBigQueryHelper` object using your project ID and JSON key.
119+
Here is an example that uses the `RemoteBigQueryHelper` to create a dataset.
120+
```java
121+
RemoteBigQueryHelper bigqueryHelper =
122+
RemoteBigQueryHelper.create(PROJECT_ID, new FileInputStream("/path/to/my/JSON/key.json"));
123+
BigQuery bigquery = bigqueryHelper.options().service();
124+
String dataset = RemoteBigQueryHelper.generateDatasetName();
125+
bigquery.create(DatasetInfo.builder(dataset).build());
126+
```
127+
128+
4. Run your tests.
129+
130+
5. Clean up the test project by using `forceDelete` to clear any datasets used.
131+
Here is an example that clears the dataset created in Step 3.
132+
```java
133+
RemoteBigQueryHelper.forceDelete(bigquery, dataset);
134+
```
68135

69136
[cloud-platform-storage-authentication]:https://cloud.google.com/storage/docs/authentication?hl=en#service_accounts
137+
[create-service-account]:https://developers.google.com/identity/protocols/OAuth2ServiceAccount#creatinganaccount

‎gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BaseDatastoreBatchWriter.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -199,7 +199,7 @@ protected DatastoreException newInvalidRequest(String msg, Object... params) {
199199
return DatastoreException.throwInvalidRequest(String.format(msg, params));
200200
}
201201

202-
protected DatastoreV1.Mutation.Builder toMutationPb() {
202+
DatastoreV1.Mutation.Builder toMutationPb() {
203203
DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder();
204204
for (FullEntity<IncompleteKey> entity : toAddAutoId()) {
205205
mutationPb.addInsertAutoId(entity.toPb());

‎gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BaseEntity.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ private B self() {
9090
}
9191

9292
@SuppressWarnings("unchecked")
93-
protected B fill(DatastoreV1.Entity entityPb) {
93+
B fill(DatastoreV1.Entity entityPb) {
9494
Map<String, Value<?>> copiedProperties = Maps.newHashMap();
9595
for (DatastoreV1.Property property : entityPb.getPropertyList()) {
9696
copiedProperties.put(property.getName(), Value.fromPb(property.getValue()));
@@ -375,7 +375,7 @@ ImmutableSortedMap<String, Value<?>> properties() {
375375
}
376376

377377
@Override
378-
protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
378+
Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
379379
Builder<?, ?> builder = emptyBuilder();
380380
builder.fill(DatastoreV1.Entity.parseFrom(bytesPb));
381381
return builder.build();
@@ -384,7 +384,7 @@ protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
384384
protected abstract Builder<?, ?> emptyBuilder();
385385

386386
@Override
387-
protected final DatastoreV1.Entity toPb() {
387+
final DatastoreV1.Entity toPb() {
388388
DatastoreV1.Entity.Builder entityPb = DatastoreV1.Entity.newBuilder();
389389
for (Map.Entry<String, Value<?>> entry : properties.entrySet()) {
390390
DatastoreV1.Property.Builder propertyPb = DatastoreV1.Property.newBuilder();

‎gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BaseKey.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,7 @@ public boolean equals(Object obj) {
172172
}
173173

174174
@Override
175-
protected DatastoreV1.Key toPb() {
175+
DatastoreV1.Key toPb() {
176176
DatastoreV1.Key.Builder keyPb = DatastoreV1.Key.newBuilder();
177177
DatastoreV1.PartitionId.Builder partitionIdPb = DatastoreV1.PartitionId.newBuilder();
178178
if (projectId != null) {

‎gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Blob.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -147,12 +147,12 @@ public static Blob copyFrom(InputStream input) throws IOException {
147147
}
148148

149149
@Override
150-
protected Value toPb() {
150+
Value toPb() {
151151
return DatastoreV1.Value.newBuilder().setBlobValue(byteString).build();
152152
}
153153

154154
@Override
155-
protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
155+
Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
156156
return new Blob(DatastoreV1.Value.parseFrom(bytesPb).getBlobValue());
157157
}
158158
}

‎gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Cursor.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -102,12 +102,12 @@ public static Cursor copyFrom(byte[] bytes) {
102102
}
103103

104104
@Override
105-
protected Value toPb() {
105+
Value toPb() {
106106
return DatastoreV1.Value.newBuilder().setBlobValue(byteString).build();
107107
}
108108

109109
@Override
110-
protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
110+
Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
111111
return fromPb(DatastoreV1.Value.parseFrom(bytesPb));
112112
}
113113

‎gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DateTime.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -98,12 +98,12 @@ public static DateTime copyFrom(Calendar calendar) {
9898
}
9999

100100
@Override
101-
protected Value toPb() {
101+
Value toPb() {
102102
return DatastoreV1.Value.newBuilder().setIntegerValue(timestampMicroseconds).build();
103103
}
104104

105105
@Override
106-
protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
106+
Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
107107
return new DateTime(DatastoreV1.Value.parseFrom(bytesPb).getIntegerValue());
108108
}
109109
}

‎gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/GqlQuery.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ public boolean equals(Object obj) {
126126
}
127127

128128
@Override
129-
protected DatastoreV1.GqlQueryArg toPb() {
129+
DatastoreV1.GqlQueryArg toPb() {
130130
DatastoreV1.GqlQueryArg.Builder argPb = DatastoreV1.GqlQueryArg.newBuilder();
131131
if (name != null) {
132132
argPb.setName(name);
@@ -141,7 +141,7 @@ protected DatastoreV1.GqlQueryArg toPb() {
141141
}
142142

143143
@Override
144-
protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
144+
Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
145145
return fromPb(DatastoreV1.GqlQueryArg.parseFrom(bytesPb));
146146
}
147147

@@ -370,7 +370,7 @@ public boolean equals(Object obj) {
370370
}
371371

372372
@Override
373-
protected DatastoreV1.GqlQuery toPb() {
373+
DatastoreV1.GqlQuery toPb() {
374374
DatastoreV1.GqlQuery.Builder queryPb = DatastoreV1.GqlQuery.newBuilder();
375375
queryPb.setQueryString(queryString);
376376
queryPb.setAllowLiteral(allowLiteral);
@@ -384,18 +384,18 @@ protected DatastoreV1.GqlQuery toPb() {
384384
}
385385

386386
@Override
387-
protected void populatePb(DatastoreV1.RunQueryRequest.Builder requestPb) {
387+
void populatePb(DatastoreV1.RunQueryRequest.Builder requestPb) {
388388
requestPb.setGqlQuery(toPb());
389389
}
390390

391391
@Override
392-
protected GqlQuery<V> nextQuery(DatastoreV1.QueryResultBatch responsePb) {
392+
GqlQuery<V> nextQuery(DatastoreV1.QueryResultBatch responsePb) {
393393
// See issue #17
394394
throw new UnsupportedOperationException("paging for this query is not implemented yet");
395395
}
396396

397397
@Override
398-
protected Object fromPb(ResultType<V> resultType, String namespace, byte[] bytesPb)
398+
Object fromPb(ResultType<V> resultType, String namespace, byte[] bytesPb)
399399
throws InvalidProtocolBufferException {
400400
return fromPb(resultType, namespace, DatastoreV1.GqlQuery.parseFrom(bytesPb));
401401
}

‎gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/IncompleteKey.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ public IncompleteKey build() {
5454
}
5555

5656
@Override
57-
protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
57+
Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
5858
return fromPb(DatastoreV1.Key.parseFrom(bytesPb));
5959
}
6060

0 commit comments

Comments
 (0)