Skip to content

Commit 15a1f8f

Browse files
igorbernstein2igorbernstein2
authored
---
yaml --- r: 15325 b: refs/heads/autosynth-bigtable c: 41a04ee h: refs/heads/master i: 15323: e380520
1 parent e7ee53a commit 15a1f8f

7 files changed

Lines changed: 202 additions & 33 deletions

File tree

‎[refs]‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ refs/heads/autosynth-asset: cd8251de8c40e239ad24dcf9ed93ea2708a3eed5
126126
refs/heads/autosynth-automl: cced2f56bbef0499609073edbca6253e1df5e535
127127
refs/heads/autosynth-bigquerydatatransfer: a6667617707608b1dbfb02d59c22b5152f208da7
128128
refs/heads/autosynth-bigquerystorage: 14ab055598b943ae3f33f484e9fb1653355d08e7
129-
refs/heads/autosynth-bigtable: 9ef0c0edbf0d4cc7c581fc13741fd5d6bb77cae1
129+
refs/heads/autosynth-bigtable: 41a04eea905c4c0b55e8daa0acb94d258835abf6
130130
refs/heads/autosynth-bigtable-admin: 6379a2bc712f2736c83de0e009b4d26da4fa82ca
131131
refs/heads/autosynth-containeranalysis: 781fdb430a60f9a6491f116e31e4e10118157bdb
132132
refs/heads/autosynth-datastore: af1fb76aa3eee02fe6f31f8fa1c72a4f048d149b

‎branches/autosynth-bigtable/google-cloud-clients/google-cloud-bigtable/README.md‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,66 @@ try {
130130
}
131131
```
132132

133+
## Opencensus Tracing
134+
135+
Cloud Bigtable client supports [Opencensus Tracing](https://opencensus.io/tracing/),
136+
which gives insight into the client internals and aids in debugging production issues.
137+
By default, the functionality is disabled. To enable, you need to add a couple of
138+
dependencies and configure an exporter. For example to enable tracing using
139+
[Google Stackdriver](https://cloud.google.com/trace/docs/):
140+
141+
[//]: # (TODO: figure out how to keep opencensus version in sync with pom.xml)
142+
143+
If you are using Maven, add this to your pom.xml file
144+
```xml
145+
<dependency>
146+
<groupId>io.opencensus</groupId>
147+
<artifactId>opencensus-impl</artifactId>
148+
<version>0.18.0</version>
149+
</dependency>
150+
<dependency>
151+
<groupId>io.opencensus</groupId>
152+
<artifactId>opencensus-exporter-trace-stackdriver</artifactId>
153+
<version>0.18.0</version>
154+
</dependency>
155+
```
156+
If you are using Gradle, add this to your dependencies
157+
```Groovy
158+
compile 'io.opencensus:opencensus-impl:0.18.0'
159+
compile 'io.opencensus:opencensus-exporter-trace-stackdriver:0.18.0'
160+
```
161+
If you are using SBT, add this to your dependencies
162+
```Scala
163+
libraryDependencies += "io.opencensus" % "opencensus-impl" % "0.18.0"
164+
libraryDependencies += "io.opencensus" % "opencensus-exporter-trace-stackdriver" % "0.18.0"
165+
```
166+
167+
Then at the start of your application configure the exporter:
168+
169+
```java
170+
import io.opencensus.exporter.trace.stackdriver.StackdriverTraceConfiguration;
171+
import io.opencensus.exporter.trace.stackdriver.StackdriverTraceExporter;
172+
173+
StackdriverTraceExporter.createAndRegister(
174+
StackdriverTraceConfiguration.builder()
175+
.setProjectId("YOUR-PROJECT_ID")
176+
.build());
177+
```
178+
179+
By default traces are [sampled](https://opencensus.io/tracing/sampling) at a rate of about 1/10,000.
180+
You can configure a higher rate by updating the active tracing params:
181+
182+
```java
183+
import io.opencensus.trace.Tracing;
184+
import io.opencensus.trace.samplers.Samplers;
185+
186+
Tracing.getTraceConfig().updateActiveTraceParams(
187+
Tracing.getTraceConfig().getActiveTraceParams().toBuilder()
188+
.setSampler(Samplers.probabilitySampler(0.01))
189+
.build()
190+
);
191+
```
192+
133193
## Troubleshooting
134194

135195
To get help, follow the instructions in the [shared Troubleshooting

‎branches/autosynth-bigtable/google-cloud-clients/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java‎

Lines changed: 74 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,18 @@
1818
import com.google.api.core.InternalApi;
1919
import com.google.api.gax.retrying.ExponentialRetryAlgorithm;
2020
import com.google.api.gax.retrying.RetryAlgorithm;
21-
import com.google.api.gax.retrying.RetryingExecutor;
21+
import com.google.api.gax.retrying.RetryingExecutorWithContext;
2222
import com.google.api.gax.retrying.ScheduledRetryingExecutor;
2323
import com.google.api.gax.rpc.BatchingCallSettings;
2424
import com.google.api.gax.rpc.Callables;
2525
import com.google.api.gax.rpc.ClientContext;
2626
import com.google.api.gax.rpc.ServerStreamingCallSettings;
2727
import com.google.api.gax.rpc.ServerStreamingCallable;
2828
import com.google.api.gax.rpc.UnaryCallable;
29+
import com.google.api.gax.tracing.SpanName;
30+
import com.google.api.gax.tracing.TracedBatchingCallable;
31+
import com.google.api.gax.tracing.TracedServerStreamingCallable;
32+
import com.google.api.gax.tracing.TracedUnaryCallable;
2933
import com.google.bigtable.v2.MutateRowsRequest;
3034
import com.google.bigtable.v2.ReadRowsRequest;
3135
import com.google.bigtable.v2.SampleRowKeysRequest;
@@ -50,6 +54,7 @@
5054
import com.google.cloud.bigtable.data.v2.stub.readrows.ReadRowsUserCallable;
5155
import com.google.cloud.bigtable.data.v2.stub.readrows.RowMergingCallable;
5256
import com.google.cloud.bigtable.gaxx.retrying.ApiResultRetryAlgorithm;
57+
import com.google.cloud.bigtable.gaxx.tracing.WrappedTracerFactory;
5358
import java.io.IOException;
5459
import java.util.List;
5560
import org.threeten.bp.Duration;
@@ -68,6 +73,9 @@
6873
*/
6974
@InternalApi
7075
public class EnhancedBigtableStub implements AutoCloseable {
76+
private static final String TRACING_OUTER_CLIENT_NAME = "Bigtable";
77+
private static final String TRACING_INNER_CLIENT_NAME = "BaseBigtable";
78+
7179
private final EnhancedBigtableStubSettings settings;
7280
private final GrpcBigtableStub stub;
7381
private final ClientContext clientContext;
@@ -92,7 +100,10 @@ public static EnhancedBigtableStub create(EnhancedBigtableStubSettings settings)
92100
.setCredentialsProvider(settings.getCredentialsProvider())
93101
.setHeaderProvider(settings.getHeaderProvider())
94102
.setStreamWatchdogProvider(settings.getStreamWatchdogProvider())
95-
.setStreamWatchdogCheckInterval(settings.getStreamWatchdogCheckInterval());
103+
.setStreamWatchdogCheckInterval(settings.getStreamWatchdogCheckInterval())
104+
// Force the base stub to use a different TracerFactory
105+
.setTracerFactory(
106+
new WrappedTracerFactory(settings.getTracerFactory(), TRACING_INNER_CLIENT_NAME));
96107

97108
// ReadRow retries are handled in the overlay: disable retries in the base layer (but make
98109
// sure to preserve the exception callable settings).
@@ -140,6 +151,9 @@ public static EnhancedBigtableStub create(EnhancedBigtableStubSettings settings)
140151
ClientContext clientContext = ClientContext.create(baseSettings);
141152
GrpcBigtableStub stub = new GrpcBigtableStub(baseSettings, clientContext);
142153

154+
// Make sure to keep the original tracer factory for the outer client.
155+
clientContext = clientContext.toBuilder().setTracerFactory(settings.getTracerFactory()).build();
156+
143157
return new EnhancedBigtableStub(settings, clientContext, stub);
144158
}
145159

@@ -247,15 +261,8 @@ private <RowT> ServerStreamingCallable<Query, RowT> createReadRowsCallable(
247261
FilterMarkerRowsCallable<RowT> filtering =
248262
new FilterMarkerRowsCallable<>(retrying2, rowAdapter);
249263

250-
ServerStreamingCallable<ReadRowsRequest, RowT> withContext =
251-
filtering.withDefaultCallContext(clientContext.getDefaultCallContext());
252-
253-
// NOTE: Ideally `withDefaultCallContext` should be the outer-most callable, however the
254-
// ReadRowsUserCallable overrides the first() method. This override would be lost if
255-
// ReadRowsUserCallable is wrapped by another callable. At some point in the future,
256-
// gax-java should allow preserving these kind of overrides through callable chains, at which
257-
// point this should be re-ordered.
258-
return new ReadRowsUserCallable<>(withContext, requestContext);
264+
return createUserFacingServerStreamingCallable(
265+
"ReadRows", new ReadRowsUserCallable<>(filtering, requestContext));
259266
}
260267

261268
/**
@@ -276,10 +283,8 @@ private UnaryCallable<String, List<KeyOffset>> createSampleRowKeysCallable() {
276283
UnaryCallable<SampleRowKeysRequest, List<SampleRowKeysResponse>> retryable =
277284
Callables.retrying(spoolable, settings.sampleRowKeysSettings(), clientContext);
278285

279-
UnaryCallable<String, List<KeyOffset>> userFacing =
280-
new SampleRowKeysCallable(retryable, requestContext);
281-
282-
return userFacing.withDefaultCallContext(clientContext.getDefaultCallContext());
286+
return createUserFacingUnaryCallable(
287+
"SampleRowKeys", new SampleRowKeysCallable(retryable, requestContext));
283288
}
284289

285290
/**
@@ -290,9 +295,8 @@ private UnaryCallable<String, List<KeyOffset>> createSampleRowKeysCallable() {
290295
* </ul>
291296
*/
292297
private UnaryCallable<RowMutation, Void> createMutateRowCallable() {
293-
MutateRowCallable userFacing = new MutateRowCallable(stub.mutateRowCallable(), requestContext);
294-
295-
return userFacing.withDefaultCallContext(clientContext.getDefaultCallContext());
298+
return createUserFacingUnaryCallable(
299+
"MutateRow", new MutateRowCallable(stub.mutateRowCallable(), requestContext));
296300
}
297301

298302
/**
@@ -311,7 +315,9 @@ private UnaryCallable<RowMutation, Void> createMutateRowCallable() {
311315
*/
312316
private UnaryCallable<BulkMutation, Void> createBulkMutateRowsCallable() {
313317
UnaryCallable<MutateRowsRequest, Void> baseCallable = createMutateRowsBaseCallable();
314-
return new BulkMutateRowsUserFacingCallable(baseCallable, requestContext);
318+
319+
return createUserFacingUnaryCallable(
320+
"BulkMutateRows", new BulkMutateRowsUserFacingCallable(baseCallable, requestContext));
315321
}
316322

317323
/**
@@ -338,8 +344,17 @@ private UnaryCallable<RowMutation, Void> createBulkMutateRowsBatchingCallable()
338344
BatchingCallSettings.newBuilder(new MutateRowsBatchingDescriptor())
339345
.setBatchingSettings(settings.bulkMutateRowsSettings().getBatchingSettings());
340346

347+
// This is a special case, the tracing starts after the batching, so we can't use
348+
// createUserFacingUnaryCallable
349+
TracedBatchingCallable<MutateRowsRequest, Void> traced =
350+
new TracedBatchingCallable<>(
351+
baseCallable,
352+
clientContext.getTracerFactory(),
353+
SpanName.of(TRACING_OUTER_CLIENT_NAME, "BulkMutateRows"),
354+
batchingCallSettings.getBatchingDescriptor());
355+
341356
UnaryCallable<MutateRowsRequest, Void> batching =
342-
Callables.batching(baseCallable, batchingCallSettings.build(), clientContext);
357+
Callables.batching(traced, batchingCallSettings.build(), clientContext);
343358

344359
MutateRowsUserFacingCallable userFacing =
345360
new MutateRowsUserFacingCallable(batching, requestContext);
@@ -359,7 +374,7 @@ private UnaryCallable<MutateRowsRequest, Void> createMutateRowsBaseCallable() {
359374
new ApiResultRetryAlgorithm<Void>(),
360375
new ExponentialRetryAlgorithm(
361376
settings.bulkMutateRowsSettings().getRetrySettings(), clientContext.getClock()));
362-
RetryingExecutor<Void> retryingExecutor =
377+
RetryingExecutorWithContext<Void> retryingExecutor =
363378
new ScheduledRetryingExecutor<>(retryAlgorithm, clientContext.getExecutor());
364379

365380
return new MutateRowsRetryingCallable(
@@ -378,10 +393,9 @@ private UnaryCallable<MutateRowsRequest, Void> createMutateRowsBaseCallable() {
378393
* </ul>
379394
*/
380395
private UnaryCallable<ConditionalRowMutation, Boolean> createCheckAndMutateRowCallable() {
381-
CheckAndMutateRowCallable userFacing =
382-
new CheckAndMutateRowCallable(stub.checkAndMutateRowCallable(), requestContext);
383-
384-
return userFacing.withDefaultCallContext(clientContext.getDefaultCallContext());
396+
return createUserFacingUnaryCallable(
397+
"CheckAndMutateRow",
398+
new CheckAndMutateRowCallable(stub.checkAndMutateRowCallable(), requestContext));
385399
}
386400

387401
/**
@@ -394,10 +408,42 @@ private UnaryCallable<ConditionalRowMutation, Boolean> createCheckAndMutateRowCa
394408
* </ul>
395409
*/
396410
private UnaryCallable<ReadModifyWriteRow, Row> createReadModifyWriteRowCallable() {
397-
ReadModifyWriteRowCallable userFacing =
398-
new ReadModifyWriteRowCallable(stub.readModifyWriteRowCallable(), requestContext);
411+
return createUserFacingUnaryCallable(
412+
"ReadModifyWriteRow",
413+
new ReadModifyWriteRowCallable(stub.readModifyWriteRowCallable(), requestContext));
414+
}
399415

400-
return userFacing.withDefaultCallContext(clientContext.getDefaultCallContext());
416+
/**
417+
* Wraps a callable chain in a user presentable callable that will inject the default call context
418+
* and trace the call.
419+
*/
420+
private <RequestT, ResponseT> UnaryCallable<RequestT, ResponseT> createUserFacingUnaryCallable(
421+
String methodName, UnaryCallable<RequestT, ResponseT> inner) {
422+
423+
UnaryCallable<RequestT, ResponseT> traced =
424+
new TracedUnaryCallable<>(
425+
inner,
426+
clientContext.getTracerFactory(),
427+
SpanName.of(TRACING_OUTER_CLIENT_NAME, methodName));
428+
429+
return traced.withDefaultCallContext(clientContext.getDefaultCallContext());
430+
}
431+
432+
/**
433+
* Wraps a callable chain in a user presentable callable that will inject the default call context
434+
* and trace the call.
435+
*/
436+
private <RequestT, ResponseT>
437+
ServerStreamingCallable<RequestT, ResponseT> createUserFacingServerStreamingCallable(
438+
String methodName, ServerStreamingCallable<RequestT, ResponseT> inner) {
439+
440+
ServerStreamingCallable<RequestT, ResponseT> traced =
441+
new TracedServerStreamingCallable<>(
442+
inner,
443+
clientContext.getTracerFactory(),
444+
SpanName.of(TRACING_OUTER_CLIENT_NAME, methodName));
445+
446+
return traced.withDefaultCallContext(clientContext.getDefaultCallContext());
401447
}
402448
// </editor-fold>
403449

‎branches/autosynth-bigtable/google-cloud-clients/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubSettings.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,9 @@
1919
import com.google.api.gax.batching.BatchingSettings;
2020
import com.google.api.gax.batching.FlowControlSettings;
2121
import com.google.api.gax.batching.FlowController.LimitExceededBehavior;
22+
import com.google.api.gax.core.GaxProperties;
2223
import com.google.api.gax.core.GoogleCredentialsProvider;
24+
import com.google.api.gax.grpc.GaxGrpcProperties;
2325
import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider;
2426
import com.google.api.gax.retrying.RetrySettings;
2527
import com.google.api.gax.rpc.BatchingCallSettings;
@@ -28,6 +30,7 @@
2830
import com.google.api.gax.rpc.StubSettings;
2931
import com.google.api.gax.rpc.TransportChannelProvider;
3032
import com.google.api.gax.rpc.UnaryCallSettings;
33+
import com.google.api.gax.tracing.OpencensusTracerFactory;
3134
import com.google.cloud.bigtable.data.v2.internal.DummyBatchingDescriptor;
3235
import com.google.cloud.bigtable.data.v2.models.ConditionalRowMutation;
3336
import com.google.cloud.bigtable.data.v2.models.KeyOffset;
@@ -37,6 +40,7 @@
3740
import com.google.cloud.bigtable.data.v2.models.RowMutation;
3841
import com.google.common.base.Preconditions;
3942
import com.google.common.collect.ImmutableList;
43+
import com.google.common.collect.ImmutableMap;
4044
import com.google.common.collect.ImmutableSet;
4145
import java.util.List;
4246
import java.util.Set;
@@ -268,6 +272,13 @@ private Builder() {
268272
setStreamWatchdogCheckInterval(baseDefaults.getStreamWatchdogCheckInterval());
269273
setStreamWatchdogProvider(baseDefaults.getStreamWatchdogProvider());
270274

275+
setTracerFactory(
276+
new OpencensusTracerFactory(
277+
ImmutableMap.of(
278+
"gax", GaxGrpcProperties.getGaxGrpcVersion(),
279+
"grpc", GaxGrpcProperties.getGrpcVersion(),
280+
"gapic", GaxProperties.getLibraryVersion(EnhancedBigtableStubSettings.class))));
281+
271282
// Per-method settings using baseSettings for defaults.
272283
readRowsSettings = ServerStreamingCallSettings.newBuilder();
273284
readRowsSettings

‎branches/autosynth-bigtable/google-cloud-clients/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/mutaterows/MutateRowsAttemptCallable.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,10 @@ public Void call() {
179179
return null;
180180
}
181181

182+
callContext
183+
.getTracer()
184+
.attemptStarted(externalFuture.getAttemptSettings().getOverallAttemptCount());
185+
182186
// Make the actual call
183187
ApiFuture<List<MutateRowsResponse>> innerFuture =
184188
innerCallable.futureCall(currentRequest, currentCallContext);

‎branches/autosynth-bigtable/google-cloud-clients/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/mutaterows/MutateRowsRetryingCallable.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
package com.google.cloud.bigtable.data.v2.stub.mutaterows;
1717

1818
import com.google.api.core.InternalApi;
19-
import com.google.api.gax.retrying.RetryingExecutor;
19+
import com.google.api.gax.retrying.RetryingExecutorWithContext;
2020
import com.google.api.gax.retrying.RetryingFuture;
2121
import com.google.api.gax.rpc.ApiCallContext;
2222
import com.google.api.gax.rpc.ServerStreamingCallable;
@@ -42,13 +42,13 @@
4242
public class MutateRowsRetryingCallable extends UnaryCallable<MutateRowsRequest, Void> {
4343
private final ApiCallContext callContextPrototype;
4444
private final ServerStreamingCallable<MutateRowsRequest, MutateRowsResponse> callable;
45-
private final RetryingExecutor<Void> executor;
45+
private final RetryingExecutorWithContext<Void> executor;
4646
private final ImmutableSet<Code> retryCodes;
4747

4848
public MutateRowsRetryingCallable(
4949
@Nonnull ApiCallContext callContextPrototype,
5050
@Nonnull ServerStreamingCallable<MutateRowsRequest, MutateRowsResponse> callable,
51-
@Nonnull RetryingExecutor<Void> executor,
51+
@Nonnull RetryingExecutorWithContext<Void> executor,
5252
@Nonnull Set<StatusCode.Code> retryCodes) {
5353
this.callContextPrototype = Preconditions.checkNotNull(callContextPrototype);
5454
this.callable = Preconditions.checkNotNull(callable);
@@ -62,7 +62,7 @@ public RetryingFuture<Void> futureCall(MutateRowsRequest request, ApiCallContext
6262
MutateRowsAttemptCallable retryCallable =
6363
new MutateRowsAttemptCallable(callable.all(), request, context, retryCodes);
6464

65-
RetryingFuture<Void> retryingFuture = executor.createFuture(retryCallable);
65+
RetryingFuture<Void> retryingFuture = executor.createFuture(retryCallable, context);
6666
retryCallable.setExternalFuture(retryingFuture);
6767
retryCallable.call();
6868

0 commit comments

Comments
 (0)