Skip to content

Commit b7102bb

Browse files
igorbernstein2garrettjonesgoogle
igorbernstein2
authored and
garrettjonesgoogle
committed
Bigtable: 19 - Implement integration tests (#2997)
1 parent 7499e96 commit b7102bb

17 files changed

Lines changed: 1111 additions & 19 deletions

File tree

‎.circleci/config.yml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,19 @@ jobs:
7979
- run:
8080
name: Run integration tests for google-cloud-bigquery
8181
command: ./utilities/verify_single_it.sh google-cloud-bigquery
82+
83+
bigtable_it:
84+
working_directory: ~/googleapis
85+
<<: *anchor_docker
86+
<<: *anchor_auth_vars
87+
steps:
88+
- checkout
89+
- run:
90+
<<: *anchor_run_decrypt
91+
- run:
92+
name: Run integration tests for google-cloud-bigtable
93+
command: ./utilities/verify_single_it.sh google-cloud-bigtable -Dbigtable.env=prod -Dbigtable.table=projects/gcloud-devel/instances/google-cloud-bigtable/tables/integration-tests
94+
8295
compute_it:
8396
working_directory: ~/googleapis
8497
<<: *anchor_docker
@@ -220,6 +233,10 @@ workflows:
220233
filters:
221234
branches:
222235
only: master
236+
- bigtable_it:
237+
filters:
238+
branches:
239+
only: master
223240
- compute_it:
224241
filters:
225242
branches:

‎TESTING.md‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
This library provides tools to help write tests for code that uses the following google-cloud services:
44

55
- [BigQuery](#testing-code-that-uses-bigquery)
6+
- [Bigtable](#testing-code-that-uses-bigtable)
67
- [Compute](#testing-code-that-uses-compute)
78
- [Datastore](#testing-code-that-uses-datastore)
89
- [DNS](#testing-code-that-uses-dns)
@@ -41,6 +42,29 @@ Here is an example that clears the dataset created in Step 3.
4142
RemoteBigQueryHelper.forceDelete(bigquery, dataset);
4243
```
4344

45+
### Testing code that uses Bigtable
46+
47+
Bigtable integration tests can either be run against an emulator or a real Bigtable table. The
48+
target environment can be selected via the `bigtable.env` system property. By default it is set to
49+
`emulator` and the other option is `prod`.
50+
51+
To use the `emulator` environment, please install the gcloud sdk and use it to install the
52+
`cbtemulator` via `gcloud components install bigtable`.
53+
54+
To use the `prod` environment:
55+
1. Set up the target table using `google-cloud-bigtable/scripts/setup-test-table.sh`
56+
2. Download the [JSON service account credentials file][create-service-account] from the Google
57+
Developer's Console.
58+
3. Set the environment variable `GOOGLE_APPLICATION_CREDENTIALS` to the path of the credentials file
59+
4. Set the system property `bigtable.env=prod` and `bigtable.table` to the full table name you
60+
created earlier. Example:
61+
```shell
62+
mvn verify -am -pl google-cloud-bigtable \
63+
-Dbigtable.env=prod \
64+
-Dbigtable.table=projects/my-project/instances/my-instance/tables/my-table
65+
```
66+
67+
4468
### Testing code that uses Compute
4569

4670
Currently, there isn't an emulator for Google Compute, so an alternative is to create a test

‎google-cloud-bigtable/pom.xml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,24 @@
127127
</execution>
128128
</executions>
129129
</plugin>
130+
<plugin>
131+
<groupId>org.apache.maven.plugins</groupId>
132+
<artifactId>maven-failsafe-plugin</artifactId>
133+
<version>2.19.1</version>
134+
<executions>
135+
<execution>
136+
<goals>
137+
<goal>integration-test</goal>
138+
<goal>verify</goal>
139+
</goals>
140+
</execution>
141+
</executions>
142+
<configuration>
143+
<parallel>classes</parallel>
144+
<perCoreThreadCount>true</perCoreThreadCount>
145+
<threadCount>2</threadCount>
146+
</configuration>
147+
</plugin>
130148
<plugin>
131149
<groupId>org.apache.maven.plugins</groupId>
132150
<artifactId>maven-javadoc-plugin</artifactId>

‎google-cloud-bigtable/scripts/setup-test-table.sh‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
#!/usr/bin/env bash
2+
3+
# Set up a table to use for integration tests.
4+
5+
set -x
6+
7+
# Format: projects/<project-id>/instances/<instance-id>/tables/<table-id>
8+
TABLE_NAME=$1
9+
10+
if [[ ${TABLE_NAME} =~ projects\/([^/]+)\/instances\/([^/]+)\/tables\/([^/]+) ]]; then
11+
PROJECT_ID=${BASH_REMATCH[1]}
12+
INSTANCE_ID=${BASH_REMATCH[2]}
13+
TABLE_ID=${BASH_REMATCH[3]}
14+
else
15+
echo "Invalid table name: $TABLE_NAME" 1>&2
16+
exit 1
17+
fi
18+
19+
cbt -project $PROJECT_ID -instance $INSTANCE_ID createtable $TABLE_ID
20+
cbt -project $PROJECT_ID -instance $INSTANCE_ID createfamily $TABLE_ID cf
21+
cbt -project $PROJECT_ID -instance $INSTANCE_ID setgcpolicy $TABLE_ID cf maxversions=1

‎google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/InstanceName.java‎

Lines changed: 7 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -58,26 +58,20 @@ private InstanceName(Builder builder) {
5858
}
5959

6060
public static InstanceName of(String project, String instance) {
61-
return newBuilder()
62-
.setProject(project)
63-
.setInstance(instance)
64-
.build();
61+
return newBuilder().setProject(project).setInstance(instance).build();
6562
}
6663

6764
public static String format(String project, String instance) {
68-
return newBuilder()
69-
.setProject(project)
70-
.setInstance(instance)
71-
.build()
72-
.toString();
65+
return newBuilder().setProject(project).setInstance(instance).build().toString();
7366
}
7467

7568
public static InstanceName parse(String formattedString) {
7669
if (formattedString.isEmpty()) {
7770
return null;
7871
}
7972
Map<String, String> matchMap =
80-
PATH_TEMPLATE.validatedMatch(formattedString, "InstanceName.parse: formattedString not in valid format");
73+
PATH_TEMPLATE.validatedMatch(
74+
formattedString, "InstanceName.parse: formattedString not in valid format");
8175
return of(matchMap.get("project"), matchMap.get("instance"));
8276
}
8377

@@ -123,9 +117,7 @@ public String getFieldValue(String fieldName) {
123117
return getFieldValuesMap().get(fieldName);
124118
}
125119

126-
/**
127-
* @deprecated This method is only present to satisfy the ResourceName interface.
128-
*/
120+
/** @deprecated This method is only present to satisfy the ResourceName interface. */
129121
@Deprecated
130122
public ResourceNameType getType() {
131123
throw new UnsupportedOperationException("InstanceName.getType() not supported");
@@ -160,8 +152,7 @@ public Builder setInstance(String instance) {
160152
return this;
161153
}
162154

163-
private Builder() {
164-
}
155+
private Builder() {}
165156

166157
private Builder(InstanceName instanceName) {
167158
project = instanceName.project;
@@ -180,8 +171,7 @@ public boolean equals(Object o) {
180171
}
181172
if (o instanceof InstanceName) {
182173
InstanceName that = (InstanceName) o;
183-
return (this.project.equals(that.project))
184-
&& (this.instance.equals(that.instance));
174+
return (this.project.equals(that.project)) && (this.instance.equals(that.instance));
185175
}
186176
return false;
187177
}
@@ -196,4 +186,3 @@ public int hashCode() {
196186
return h;
197187
}
198188
}
199-

‎google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BulkMutationBatcherIT.java‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
/*
2+
* Copyright 2018 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.google.cloud.bigtable.data.v2.it;
17+
18+
import com.google.api.gax.rpc.ServerStream;
19+
import com.google.cloud.bigtable.data.v2.BigtableDataClient;
20+
import com.google.cloud.bigtable.data.v2.it.env.TestEnvRule;
21+
import com.google.cloud.bigtable.data.v2.models.BulkMutationBatcher;
22+
import com.google.cloud.bigtable.data.v2.models.Query;
23+
import com.google.cloud.bigtable.data.v2.models.Row;
24+
import com.google.cloud.bigtable.data.v2.models.RowCell;
25+
import com.google.cloud.bigtable.data.v2.models.RowMutation;
26+
import com.google.common.collect.Lists;
27+
import com.google.common.truth.Truth;
28+
import com.google.protobuf.ByteString;
29+
import java.util.List;
30+
import org.junit.ClassRule;
31+
import org.junit.Test;
32+
import org.junit.runner.RunWith;
33+
import org.junit.runners.JUnit4;
34+
35+
@RunWith(JUnit4.class)
36+
public class BulkMutationBatcherIT {
37+
@ClassRule public static TestEnvRule testEnvRule = new TestEnvRule();
38+
39+
@Test
40+
public void test() throws Exception {
41+
BigtableDataClient client = testEnvRule.env().getDataClient();
42+
String tableId = testEnvRule.env().getTableName().getTable();
43+
String family = testEnvRule.env().getFamilyId();
44+
String rowPrefix = testEnvRule.env().getRowPrefix();
45+
46+
try (BulkMutationBatcher batcher = client.newBulkMutationBatcher()) {
47+
for (int i = 0; i < 10; i++) {
48+
batcher.add(
49+
RowMutation.create(tableId, rowPrefix + "-" + i).setCell(family, "q", 10_000, "value"));
50+
}
51+
}
52+
53+
List<Row> expectedRows = Lists.newArrayList();
54+
for (int i = 0; i < 10; i++) {
55+
expectedRows.add(
56+
Row.create(
57+
ByteString.copyFromUtf8(rowPrefix + "-" + i),
58+
Lists.newArrayList(
59+
RowCell.create(
60+
family,
61+
ByteString.copyFromUtf8("q"),
62+
10_000,
63+
Lists.<String>newArrayList(),
64+
ByteString.copyFromUtf8("value")))));
65+
}
66+
ServerStream<Row> actualRows = client.readRows(Query.create(tableId).prefix(rowPrefix));
67+
68+
Truth.assertThat(actualRows).containsExactlyElementsIn(expectedRows);
69+
}
70+
}

‎google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/CheckAndMutateIT.java‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
/*
2+
* Copyright 2018 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.google.cloud.bigtable.data.v2.it;
17+
18+
import static com.google.cloud.bigtable.data.v2.models.Filters.FILTERS;
19+
import static com.google.common.truth.Truth.assertThat;
20+
21+
import com.google.cloud.bigtable.data.v2.it.env.TestEnvRule;
22+
import com.google.cloud.bigtable.data.v2.models.ConditionalRowMutation;
23+
import com.google.cloud.bigtable.data.v2.models.Mutation;
24+
import com.google.cloud.bigtable.data.v2.models.Query;
25+
import com.google.cloud.bigtable.data.v2.models.Row;
26+
import com.google.cloud.bigtable.data.v2.models.RowMutation;
27+
import com.google.protobuf.ByteString;
28+
import java.util.concurrent.TimeUnit;
29+
import org.junit.ClassRule;
30+
import org.junit.Test;
31+
import org.junit.runner.RunWith;
32+
import org.junit.runners.JUnit4;
33+
34+
@RunWith(JUnit4.class)
35+
public class CheckAndMutateIT {
36+
@ClassRule public static TestEnvRule testEnvRule = new TestEnvRule();
37+
38+
@Test
39+
public void test() throws Exception {
40+
String tableId = testEnvRule.env().getTableName().getTable();
41+
String rowKey = testEnvRule.env().getRowPrefix();
42+
String familyId = testEnvRule.env().getFamilyId();
43+
44+
testEnvRule
45+
.env()
46+
.getDataClient()
47+
.mutateRowCallable()
48+
.call(
49+
RowMutation.create(tableId, rowKey)
50+
.setCell(familyId, "q1", "val1")
51+
.setCell(familyId, "q2", "val2"));
52+
53+
testEnvRule
54+
.env()
55+
.getDataClient()
56+
.checkAndMutateRowAsync(
57+
ConditionalRowMutation.create(tableId, rowKey)
58+
.condition(FILTERS.qualifier().exactMatch("q1"))
59+
.then(Mutation.create().setCell(familyId, "q3", "q1")))
60+
.get(1, TimeUnit.MINUTES);
61+
62+
Row row =
63+
testEnvRule
64+
.env()
65+
.getDataClient()
66+
.readRowsCallable()
67+
.first()
68+
.call(Query.create(tableId).rowKey(rowKey));
69+
70+
assertThat(row.getCells()).hasSize(3);
71+
assertThat(row.getCells().get(2).getValue()).isEqualTo(ByteString.copyFromUtf8("q1"));
72+
}
73+
}

‎google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/MutateRowIT.java‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
/*
2+
* Copyright 2018 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.google.cloud.bigtable.data.v2.it;
17+
18+
import static com.google.common.truth.Truth.assertThat;
19+
20+
import com.google.cloud.bigtable.data.v2.it.env.TestEnvRule;
21+
import com.google.cloud.bigtable.data.v2.models.Query;
22+
import com.google.cloud.bigtable.data.v2.models.Row;
23+
import com.google.cloud.bigtable.data.v2.models.RowMutation;
24+
import com.google.protobuf.ByteString;
25+
import java.util.concurrent.TimeUnit;
26+
import org.junit.ClassRule;
27+
import org.junit.Test;
28+
import org.junit.runner.RunWith;
29+
import org.junit.runners.JUnit4;
30+
31+
@RunWith(JUnit4.class)
32+
public class MutateRowIT {
33+
@ClassRule public static TestEnvRule testEnvRule = new TestEnvRule();
34+
35+
@Test
36+
public void test() throws Exception {
37+
String rowKey = testEnvRule.env().getRowPrefix() + "testA";
38+
String familyId = testEnvRule.env().getFamilyId();
39+
40+
testEnvRule
41+
.env()
42+
.getDataClient()
43+
.mutateRowAsync(
44+
RowMutation.create(testEnvRule.env().getTableName().getTable(), rowKey)
45+
.setCell(familyId, "q", "myVal")
46+
.setCell(familyId, "q2", "myVal2")
47+
.setCell(familyId, "q3", "myVal3"))
48+
.get(1, TimeUnit.MINUTES);
49+
50+
testEnvRule
51+
.env()
52+
.getDataClient()
53+
.mutateRowAsync(
54+
RowMutation.create(testEnvRule.env().getTableName().getTable(), rowKey)
55+
.deleteCells(familyId, "q2"))
56+
.get(1, TimeUnit.MINUTES);
57+
58+
Row row =
59+
testEnvRule
60+
.env()
61+
.getDataClient()
62+
.readRowsCallable()
63+
.first()
64+
.call(Query.create(testEnvRule.env().getTableName().getTable()).rowKey(rowKey));
65+
66+
assertThat(row.getCells()).hasSize(2);
67+
assertThat(row.getCells().get(0).getValue()).isEqualTo(ByteString.copyFromUtf8("myVal"));
68+
assertThat(row.getCells().get(1).getValue()).isEqualTo(ByteString.copyFromUtf8("myVal3"));
69+
}
70+
}

0 commit comments

Comments
 (0)