Skip to content

Commit 2e502db

Browse files
igorbernstein2garrettjonesgoogle
igorbernstein2
authored and
garrettjonesgoogle
committed
---
yaml --- r: 8721 b: refs/heads/master c: 937e49d h: refs/heads/master i: 8719: 14f300a
1 parent 9b35b24 commit 2e502db

8 files changed

Lines changed: 243 additions & 13 deletions

File tree

‎[refs]‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
---
2-
refs/heads/master: d8071372dabaca311cf7ab28dadb914470130dba
2+
refs/heads/master: 937e49d7159c9327255e167f6339c51bb7372c37
33
refs/heads/travis: 47e4fee4fd5af9b2a8ce46f23c72ec95f9b195b2
44
refs/heads/gh-pages: 6daca92127d91b7c2c99490080ecf8a13fa94cde
55
refs/tags/0.0.9: 22f1839238f66c39e67ed4dfdcd273b1ae2e8444

‎trunk/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataClient.java‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,16 +15,20 @@
1515
*/
1616
package com.google.cloud.bigtable.data.v2;
1717

18+
import com.google.api.core.ApiFuture;
1819
import com.google.api.core.InternalApi;
1920
import com.google.api.gax.rpc.ResponseObserver;
2021
import com.google.api.gax.rpc.ServerStream;
2122
import com.google.api.gax.rpc.ServerStreamingCallable;
23+
import com.google.api.gax.rpc.UnaryCallable;
2224
import com.google.bigtable.admin.v2.InstanceName;
2325
import com.google.cloud.bigtable.data.v2.stub.EnhancedBigtableStub;
26+
import com.google.cloud.bigtable.data.v2.wrappers.KeyOffset;
2427
import com.google.cloud.bigtable.data.v2.wrappers.Query;
2528
import com.google.cloud.bigtable.data.v2.wrappers.Row;
2629
import com.google.cloud.bigtable.data.v2.wrappers.RowAdapter;
2730
import java.io.IOException;
31+
import java.util.List;
2832

2933
/**
3034
* Client for reading from and writing to existing Bigtable tables.
@@ -247,6 +251,47 @@ public <RowT> ServerStreamingCallable<Query, RowT> readRowsCallable(RowAdapter<R
247251
return stub.createReadRowsCallable(rowAdapter);
248252
}
249253

254+
/**
255+
* Convenience method to asynchronously return a sample of row keys in the table. The returned row
256+
* keys will delimit contiguous sections of the table of approximately equal size, which can be
257+
* used to break up the data for distributed tasks like mapreduces.
258+
*
259+
* <p>Sample code:
260+
*
261+
* <pre>{@code
262+
* InstanceName instanceName = InstanceName.of("[PROJECT]", "[INSTANCE]");
263+
* try (BigtableClient bigtableClient = BigtableClient.create(instanceName)) {
264+
* ApiFuture<List<KeyOffset>> keyOffsets = bigtableClient.sampleRowKeys("[TABLE]");
265+
* }
266+
* }</pre>
267+
*/
268+
public ApiFuture<List<KeyOffset>> sampleRowKeys(String tableId) {
269+
return sampleRowKeysCallable().futureCall(tableId);
270+
}
271+
272+
/**
273+
* Returns a sample of row keys in the table. The returned row keys will delimit contiguous
274+
* sections of the table of approximately equal size, which can be used to break up the data for
275+
* distributed tasks like mapreduces. The returned callable object allows for customization of api
276+
* invocation.
277+
*
278+
* <p>Sample code:
279+
*
280+
* <pre>{@code
281+
* InstanceName instanceName = InstanceName.of("[PROJECT]", "[INSTANCE]");
282+
* try (BigtableClient bigtableClient = BigtableClient.create(instanceName)) {
283+
* // Synchronous invocation
284+
* List<KeyOffset> keyOffsets = bigtableClient.sampleRowKeysCallable().call("[TABLE]");
285+
*
286+
* // Asynchronous invocation
287+
* ApiFuture<List<KeyOffset>> keyOffsets = bigtableClient.sampleRowKeysCallable().futureCall("[TABLE]");
288+
* }
289+
* }</pre>
290+
*/
291+
public UnaryCallable<String, List<KeyOffset>> sampleRowKeysCallable() {
292+
return stub.sampleRowKeysCallable();
293+
}
294+
250295
/** Close the clients and releases all associated resources. */
251296
@Override
252297
public void close() throws Exception {

‎trunk/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,14 @@
1717

1818
import com.google.api.gax.rpc.ClientSettings;
1919
import com.google.api.gax.rpc.ServerStreamingCallSettings;
20+
import com.google.api.gax.rpc.UnaryCallSettings;
2021
import com.google.bigtable.admin.v2.InstanceName;
2122
import com.google.cloud.bigtable.data.v2.stub.EnhancedBigtableStubSettings;
23+
import com.google.cloud.bigtable.data.v2.wrappers.KeyOffset;
2224
import com.google.cloud.bigtable.data.v2.wrappers.Query;
2325
import com.google.cloud.bigtable.data.v2.wrappers.Row;
2426
import java.io.IOException;
27+
import java.util.List;
2528
import javax.annotation.Nonnull;
2629

2730
/**
@@ -75,6 +78,11 @@ public ServerStreamingCallSettings<Query, Row> readRowsSettings() {
7578
return getTypedStubSettings().readRowsSettings();
7679
}
7780

81+
/** Returns the object with the settings used for calls to sampleRowKeys. */
82+
public UnaryCallSettings<String, List<KeyOffset>> sampleRowKeysSettings() {
83+
return getTypedStubSettings().sampleRowKeysSettings();
84+
}
85+
7886
@SuppressWarnings("unchecked")
7987
EnhancedBigtableStubSettings getTypedStubSettings() {
8088
return (EnhancedBigtableStubSettings) getStubSettings();
@@ -140,6 +148,11 @@ public ServerStreamingCallSettings.Builder<Query, Row> readRowsSettings() {
140148
return getTypedStubSettings().readRowsSettings();
141149
}
142150

151+
/** Returns the builder for the settings used for calls to SampleRowKeysSettings. */
152+
public UnaryCallSettings.Builder<String, List<KeyOffset>> sampleRowKeysSettings() {
153+
return getTypedStubSettings().sampleRowKeysSettings();
154+
}
155+
143156
@SuppressWarnings("unchecked")
144157
private EnhancedBigtableStubSettings.Builder getTypedStubSettings() {
145158
return (EnhancedBigtableStubSettings.Builder) getStubSettings();

‎trunk/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,17 +15,21 @@
1515
*/
1616
package com.google.cloud.bigtable.data.v2.stub;
1717

18+
import com.google.api.core.ApiFuture;
1819
import com.google.api.core.InternalApi;
1920
import com.google.api.gax.rpc.ApiCallContext;
2021
import com.google.api.gax.rpc.ClientContext;
2122
import com.google.api.gax.rpc.ResponseObserver;
2223
import com.google.api.gax.rpc.ServerStreamingCallable;
24+
import com.google.api.gax.rpc.UnaryCallable;
2325
import com.google.cloud.bigtable.data.v2.internal.RequestContext;
2426
import com.google.cloud.bigtable.data.v2.wrappers.DefaultRowAdapter;
27+
import com.google.cloud.bigtable.data.v2.wrappers.KeyOffset;
2528
import com.google.cloud.bigtable.data.v2.wrappers.Query;
2629
import com.google.cloud.bigtable.data.v2.wrappers.Row;
2730
import com.google.cloud.bigtable.data.v2.wrappers.RowAdapter;
2831
import java.io.IOException;
32+
import java.util.List;
2933

3034
/**
3135
* The core client that converts method calls to RPCs.
@@ -47,6 +51,7 @@ public class EnhancedBigtableStub implements AutoCloseable {
4751
private final RequestContext requestContext;
4852

4953
private final ServerStreamingCallable<Query, Row> readRowsCallable;
54+
private final UnaryCallable<String, List<KeyOffset>> sampleRowKeysCallable;
5055

5156
public static EnhancedBigtableStub create(EnhancedBigtableStubSettings settings)
5257
throws IOException {
@@ -74,6 +79,7 @@ public static EnhancedBigtableStub create(EnhancedBigtableStubSettings settings)
7479
RequestContext.create(settings.getInstanceName(), settings.getAppProfileId());
7580

7681
readRowsCallable = createReadRowsCallable(new DefaultRowAdapter());
82+
sampleRowKeysCallable = createSampleRowKeysCallable();
7783
}
7884

7985
// <editor-fold desc="Callable creators">
@@ -87,12 +93,36 @@ public void call(
8793
}
8894
};
8995
}
96+
97+
/**
98+
* Creates a callable chain to handle SampleRowKeys RPcs. The chain will:
99+
*
100+
* <ul>
101+
* <li>Convert a table id to a {@link com.google.bigtable.v2.SampleRowKeysRequest}.
102+
* <li>Dispatch the request to the GAPIC's {@link BigtableStub#sampleRowKeysCallable()}.
103+
* <li>Spool responses into a list.
104+
* <li>Retry on failure.
105+
* <li>Convert the responses into {@link KeyOffset}s.
106+
* </ul>
107+
*/
108+
private UnaryCallable<String, List<KeyOffset>> createSampleRowKeysCallable() {
109+
return new UnaryCallable<String, List<KeyOffset>>() {
110+
@Override
111+
public ApiFuture<List<KeyOffset>> futureCall(String request, ApiCallContext context) {
112+
throw new UnsupportedOperationException("todo");
113+
}
114+
};
115+
}
90116
// </editor-fold>
91117

92118
// <editor-fold desc="Callable accessors">
93119
public ServerStreamingCallable<Query, Row> readRowsCallable() {
94120
return readRowsCallable;
95121
}
122+
123+
public UnaryCallable<String, List<KeyOffset>> sampleRowKeysCallable() {
124+
return sampleRowKeysCallable;
125+
}
96126
// </editor-fold>
97127

98128
@Override

‎trunk/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubSettings.java‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,11 @@
2323
import com.google.api.gax.rpc.StubSettings;
2424
import com.google.api.gax.rpc.UnaryCallSettings;
2525
import com.google.bigtable.admin.v2.InstanceName;
26+
import com.google.cloud.bigtable.data.v2.wrappers.KeyOffset;
2627
import com.google.cloud.bigtable.data.v2.wrappers.Query;
2728
import com.google.cloud.bigtable.data.v2.wrappers.Row;
2829
import com.google.common.base.Preconditions;
30+
import java.util.List;
2931
import javax.annotation.Nonnull;
3032
import org.threeten.bp.Duration;
3133

@@ -69,6 +71,7 @@ public class EnhancedBigtableStubSettings extends StubSettings<EnhancedBigtableS
6971
private final String appProfileId;
7072

7173
private final ServerStreamingCallSettings<Query, Row> readRowsSettings;
74+
private final UnaryCallSettings<String, List<KeyOffset>> sampleRowKeysSettings;
7275

7376
private EnhancedBigtableStubSettings(Builder builder) {
7477
super(builder);
@@ -77,6 +80,7 @@ private EnhancedBigtableStubSettings(Builder builder) {
7780

7881
// Per method settings.
7982
readRowsSettings = builder.readRowsSettings.build();
83+
sampleRowKeysSettings = builder.sampleRowKeysSettings.build();
8084
}
8185

8286
/** Create a new builder. */
@@ -99,6 +103,11 @@ public ServerStreamingCallSettings<Query, Row> readRowsSettings() {
99103
return readRowsSettings;
100104
}
101105

106+
/** Returns the object with the settings used for calls to SampleRowKeys. */
107+
public UnaryCallSettings<String, List<KeyOffset>> sampleRowKeysSettings() {
108+
return sampleRowKeysSettings;
109+
}
110+
102111
/** Returns a builder containing all the values of this settings class. */
103112
public Builder toBuilder() {
104113
return new Builder(this);
@@ -110,6 +119,7 @@ public static class Builder extends StubSettings.Builder<EnhancedBigtableStubSet
110119
private String appProfileId;
111120

112121
private final ServerStreamingCallSettings.Builder<Query, Row> readRowsSettings;
122+
private final UnaryCallSettings.Builder<String, List<KeyOffset>> sampleRowKeysSettings;
113123

114124
/**
115125
* Initializes a new Builder with sane defaults for all settings.
@@ -152,6 +162,22 @@ private Builder() {
152162
.setRpcTimeoutMultiplier(1)
153163
.setMaxRpcTimeout(Duration.ofSeconds(20))
154164
.build());
165+
166+
sampleRowKeysSettings = UnaryCallSettings.newUnaryCallSettingsBuilder();
167+
/* TODO: copy retryCodes & retrySettings from baseSettings.sampleRowKeysSettings once it exists in GAPIC */
168+
sampleRowKeysSettings
169+
.setRetryableCodes(Code.DEADLINE_EXCEEDED, Code.UNAVAILABLE, Code.ABORTED)
170+
.setRetrySettings(
171+
RetrySettings.newBuilder()
172+
.setMaxAttempts(10)
173+
.setTotalTimeout(Duration.ofMinutes(1))
174+
.setInitialRetryDelay(Duration.ofMillis(100))
175+
.setRetryDelayMultiplier(1.3)
176+
.setMaxRetryDelay(Duration.ofMinutes(1))
177+
.setInitialRpcTimeout(Duration.ofSeconds(20))
178+
.setRpcTimeoutMultiplier(1)
179+
.setMaxRpcTimeout(Duration.ofSeconds(20))
180+
.build());
155181
}
156182

157183
private Builder(EnhancedBigtableStubSettings settings) {
@@ -161,6 +187,7 @@ private Builder(EnhancedBigtableStubSettings settings) {
161187

162188
// Per method settings.
163189
readRowsSettings = settings.readRowsSettings.toBuilder();
190+
sampleRowKeysSettings = settings.sampleRowKeysSettings.toBuilder();
164191
}
165192

166193
// <editor-fold desc="Private Helpers">
@@ -213,6 +240,11 @@ public ServerStreamingCallSettings.Builder<Query, Row> readRowsSettings() {
213240
return readRowsSettings;
214241
}
215242

243+
/** Returns the builder for the settings used for calls to SampleRowKeysSettings. */
244+
public UnaryCallSettings.Builder<String, List<KeyOffset>> sampleRowKeysSettings() {
245+
return sampleRowKeysSettings;
246+
}
247+
216248
@SuppressWarnings("unchecked")
217249
public EnhancedBigtableStubSettings build() {
218250
Preconditions.checkState(instanceName != null, "InstanceName must be set");

‎trunk/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/wrappers/KeyOffset.java‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
/*
2+
* Copyright 2018 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.google.cloud.bigtable.data.v2.wrappers;
17+
18+
import com.google.api.core.InternalApi;
19+
import com.google.auto.value.AutoValue;
20+
import com.google.protobuf.ByteString;
21+
22+
/** Represents the offset of a row key in a table. */
23+
@AutoValue
24+
public abstract class KeyOffset {
25+
@InternalApi
26+
public static KeyOffset create(ByteString key, long offsetBytes) {
27+
return new AutoValue_KeyOffset(key, offsetBytes);
28+
}
29+
30+
/**
31+
* A row key value that is returned as part of the response of {@link
32+
* com.google.cloud.bigtable.data.v2.BigtableDataClient#sampleRowKeys(String)}. The key represents
33+
* end boundary of one of the contiguous sections in a list of approximately equal sized sections.
34+
*
35+
* <p>Note that row key may not have ever been written to or read from, and users should therefore
36+
* not make any assumptions about the row key structure that are specific to their use case.
37+
*/
38+
public abstract ByteString geyKey();
39+
40+
/**
41+
* Approximate total storage space used by all rows in the table which precede {@link #geyKey()}.
42+
* Buffering the contents of all rows between two subsequent samples would require space roughly
43+
* equal to the difference in their {@link #getOffsetBytes()} fields.
44+
*/
45+
public abstract long getOffsetBytes();
46+
}

‎trunk/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/BigtableDataClientTest.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,12 @@
1919

2020
import com.google.api.gax.rpc.ResponseObserver;
2121
import com.google.api.gax.rpc.ServerStreamingCallable;
22+
import com.google.api.gax.rpc.UnaryCallable;
2223
import com.google.cloud.bigtable.data.v2.stub.EnhancedBigtableStub;
24+
import com.google.cloud.bigtable.data.v2.wrappers.KeyOffset;
2325
import com.google.cloud.bigtable.data.v2.wrappers.Query;
2426
import com.google.cloud.bigtable.data.v2.wrappers.Row;
27+
import java.util.List;
2528
import org.junit.Before;
2629
import org.junit.Test;
2730
import org.junit.runner.RunWith;
@@ -33,12 +36,14 @@
3336
public class BigtableDataClientTest {
3437
@Mock private EnhancedBigtableStub mockStub;
3538
@Mock private ServerStreamingCallable<Query, Row> mockReadRowsCallable;
39+
@Mock private UnaryCallable<String, List<KeyOffset>> mockSampleRowKeysCallable;
3640

3741
private BigtableDataClient bigtableDataClient;
3842

3943
@Before
4044
public void setUp() {
4145
Mockito.when(mockStub.readRowsCallable()).thenReturn(mockReadRowsCallable);
46+
Mockito.when(mockStub.sampleRowKeysCallable()).thenReturn(mockSampleRowKeysCallable);
4247
bigtableDataClient = new BigtableDataClient(mockStub);
4348
}
4449

@@ -70,4 +75,15 @@ public void proxyReadRowsAsyncTest() {
7075

7176
Mockito.verify(mockReadRowsCallable).call(query, mockObserver);
7277
}
78+
79+
@Test
80+
public void proxySampleRowKeysCallableTest() {
81+
assertThat(bigtableDataClient.sampleRowKeysCallable()).isSameAs(mockSampleRowKeysCallable);
82+
}
83+
84+
@Test
85+
public void proxySampleRowKeysTest() {
86+
bigtableDataClient.sampleRowKeys("fake-table");
87+
Mockito.verify(mockSampleRowKeysCallable).futureCall("fake-table");
88+
}
7389
}

0 commit comments

Comments
 (0)