Skip to content
This repository was archived by the owner on Mar 4, 2026. It is now read-only.

Commit 646e6ea

Browse files
lszinvsurbhigarg92
lszinv
and
surbhigarg92
authored
Feat: Operation, Attempt, and GFE metrics (#2328)
* feat: Operation and Attempt metrics Added operation and attempt metrics emissions - Operation Latency - Operation Count - Attempt Latency - Attempt Count - GFE Latency - GFE Connectivity Error Count Metrics are feature gated to the env var `SPANNER_DISABLE_BUILTIN_METRICS` being set to `true` - Metrics are disabled when emulator is used * Addressed Review Comments - Disabled exporting of metrics if already exported within last 30s - Added View definitions for bucket boundaries for histograms - Fixed issue with CreateTracer reusing previously defined operation. - Moved Client Name and UID setting to exporter - Improved tracer associations with attempts by using google request id instead of the method * Added requestId to interceptor fetching of Tracer * Addressed review comments - Added periodic tracer cleanup to MetricsTracerFactory - Changed RequestStream operation completion logic to better handle error cases * Added tests for tracer pruning * Addressed cleanup intervals and running logic * testing system test cleanup * Added better cleanup to tests - Disabled metrics for unit tests * Added Env var setting changes to owlbot to persist between generations * disable metrics for unit tests - Disabled Metrics from unit tests by setting the disable ENV var in command definition - Added MetricsTracerFactory cleanup to spanner.close() * reverted package and fixed metrics permission - Disabled metrics from inside tests instead * unref the MetricsTracerFactory Interval --------- Co-authored-by: surbhigarg92 <[email protected]>
1 parent 68cc27c commit 646e6ea

18 files changed

+1911
-408
lines changed

‎observability-test/spanner.ts‎

Lines changed: 55 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import * as mockDatabaseAdmin from '../test/mockserver/mockdatabaseadmin';
2727
import * as sinon from 'sinon';
2828
import {Row} from '../src/partial-result-stream';
2929
import {END_TO_END_TRACING_HEADER} from '../src/common';
30+
import {MetricsTracerFactory} from '../src/metrics/metrics-tracer-factory';
3031
const {
3132
AlwaysOnSampler,
3233
NodeTracerProvider,
@@ -52,6 +53,23 @@ const {ObservabilityOptions} = require('../src/instrument');
5253
const selectSql = 'SELECT 1';
5354
const updateSql = 'UPDATE FOO SET BAR=1 WHERE BAZ=2';
5455

56+
async function disableMetrics(sandbox?: sinon.SinonSandbox) {
57+
if (sandbox) {
58+
if (
59+
Object.prototype.hasOwnProperty.call(
60+
process.env,
61+
'SPANNER_DISABLE_BUILTIN_METRICS',
62+
)
63+
) {
64+
sandbox.replace(process.env, 'SPANNER_DISABLE_BUILTIN_METRICS', 'true');
65+
} else {
66+
sandbox.define(process.env, 'SPANNER_DISABLE_BUILTIN_METRICS', 'true');
67+
}
68+
}
69+
await MetricsTracerFactory.resetInstance();
70+
MetricsTracerFactory.enabled = false;
71+
}
72+
5573
/** A simple result set for SELECT 1. */
5674
function createSelect1ResultSet(): protobuf.ResultSet {
5775
const fields = [
@@ -79,6 +97,7 @@ interface setupResults {
7997

8098
async function setup(
8199
observabilityOptions?: typeof ObservabilityOptions,
100+
sandbox?: sinon.SinonSandbox,
82101
): Promise<setupResults> {
83102
const server = new grpc.Server();
84103

@@ -109,6 +128,7 @@ async function setup(
109128
mock.StatementResult.updateCount(1),
110129
);
111130

131+
await disableMetrics(sandbox);
112132
const spanner = new Spanner({
113133
projectId: 'observability-project-id',
114134
servicePath: 'localhost',
@@ -125,6 +145,7 @@ async function setup(
125145
}
126146

127147
describe('EndToEnd', async () => {
148+
const sandbox = sinon.createSandbox();
128149
const contextManager = new AsyncHooksContextManager();
129150
setGlobalContextManager(contextManager);
130151
afterEach(() => {
@@ -139,10 +160,13 @@ describe('EndToEnd', async () => {
139160
spanProcessors: [new SimpleSpanProcessor(traceExporter)],
140161
});
141162

142-
const setupResult = await setup({
143-
tracerProvider: tracerProvider,
144-
enableExtendedTracing: false,
145-
});
163+
const setupResult = await setup(
164+
{
165+
tracerProvider: tracerProvider,
166+
enableExtendedTracing: false,
167+
},
168+
sandbox,
169+
);
146170

147171
const server = setupResult.server;
148172
const spannerMock = setupResult.spannerMock;
@@ -152,6 +176,7 @@ describe('EndToEnd', async () => {
152176
after(async () => {
153177
spanner.close();
154178
await server.tryShutdown(() => {});
179+
sandbox.restore();
155180
});
156181

157182
afterEach(async () => {
@@ -397,6 +422,16 @@ describe('EndToEnd', async () => {
397422
});
398423

399424
describe('ObservabilityOptions injection and propagation', async () => {
425+
let sandbox;
426+
427+
beforeEach(() => {
428+
sandbox = sinon.createSandbox();
429+
});
430+
431+
afterEach(() => {
432+
sandbox.restore();
433+
});
434+
400435
it('Passed into Spanner, Instance and Database', async () => {
401436
const traceExporter = new InMemorySpanExporter();
402437
const tracerProvider = new NodeTracerProvider({
@@ -410,7 +445,7 @@ describe('ObservabilityOptions injection and propagation', async () => {
410445
enableExtendedTracing: true,
411446
};
412447

413-
const setupResult = await setup(observabilityOptions);
448+
const setupResult = await setup(observabilityOptions, sandbox);
414449
const spanner = setupResult.spanner;
415450
const server = setupResult.server;
416451
const spannerMock = setupResult.spannerMock;
@@ -421,6 +456,7 @@ describe('ObservabilityOptions injection and propagation', async () => {
421456
spannerMock.resetRequests();
422457
spanner.close();
423458
server.tryShutdown(() => {});
459+
sandbox.restore();
424460
});
425461

426462
// Ensure that the same observability configuration is set on the Spanner client.
@@ -470,7 +506,7 @@ describe('ObservabilityOptions injection and propagation', async () => {
470506
tracerProvider: tracerProvider,
471507
enableExtendedTracing: true,
472508
};
473-
const setupResult = await setup(observabilityOptions);
509+
const setupResult = await setup(observabilityOptions, sandbox);
474510
const spanner = setupResult.spanner;
475511
const server = setupResult.server;
476512
const spannerMock = setupResult.spannerMock;
@@ -746,7 +782,7 @@ describe('ObservabilityOptions injection and propagation', async () => {
746782
tracerProvider: injectedTracerProvider,
747783
enableExtendedTracing: true,
748784
};
749-
const setupResult = await setup(observabilityOptions);
785+
const setupResult = await setup(observabilityOptions, sandbox);
750786
const spanner = setupResult.spanner;
751787
const server = setupResult.server;
752788
const spannerMock = setupResult.spannerMock;
@@ -832,8 +868,10 @@ describe('E2E traces with async/await', async () => {
832868
let traceExporter: typeof InMemorySpanExporter;
833869
let provider: typeof NodeTracerProvider;
834870
let observabilityOptions: typeof ObservabilityOptions;
871+
let sandbox;
835872

836873
beforeEach(async () => {
874+
sandbox = sinon.createSandbox();
837875
traceExporter = new InMemorySpanExporter();
838876
provider = new NodeTracerProvider({
839877
sampler: new AlwaysOnSampler(),
@@ -845,7 +883,7 @@ describe('E2E traces with async/await', async () => {
845883
tracerProvider: provider,
846884
enableExtendedTracing: true,
847885
};
848-
const setupResult = await setup(observabilityOptions);
886+
const setupResult = await setup(observabilityOptions, sandbox);
849887
spanner = setupResult.spanner;
850888
server = setupResult.server;
851889
spannerMock = setupResult.spannerMock;
@@ -857,6 +895,7 @@ describe('E2E traces with async/await', async () => {
857895
spannerMock.resetRequests();
858896
spanner.close();
859897
server.tryShutdown(() => {});
898+
sandbox.restore();
860899
});
861900

862901
function assertAsyncAwaitExpectations() {
@@ -1021,6 +1060,7 @@ describe('Negative cases', async () => {
10211060
let traceExporter: typeof InMemorySpanExporter;
10221061
let provider: typeof NodeTracerProvider;
10231062
let observabilityOptions: typeof ObservabilityOptions;
1063+
let sandbox;
10241064

10251065
const selectSql1p = 'SELECT 1p';
10261066
const messageBadSelect1p = `Missing whitespace between literal and alias [at 1:9]
@@ -1032,6 +1072,7 @@ SELECT 1p
10321072
'Failed to insert row with primary key ({pk#SingerId:1}) due to previously existing row';
10331073

10341074
beforeEach(async () => {
1075+
sandbox = sinon.createSandbox();
10351076
traceExporter = new InMemorySpanExporter();
10361077
provider = new NodeTracerProvider({
10371078
sampler: new AlwaysOnSampler(),
@@ -1043,7 +1084,7 @@ SELECT 1p
10431084
tracerProvider: provider,
10441085
enableExtendedTracing: true,
10451086
};
1046-
const setupResult = await setup(observabilityOptions);
1087+
const setupResult = await setup(observabilityOptions, sandbox);
10471088
spanner = setupResult.spanner;
10481089
server = setupResult.server;
10491090
spannerMock = setupResult.spannerMock;
@@ -1073,6 +1114,7 @@ SELECT 1p
10731114
spannerMock.resetRequests();
10741115
spanner.close();
10751116
server.tryShutdown(() => {});
1117+
sandbox.restore();
10761118
});
10771119

10781120
function assertRunBadSyntaxExpectations() {
@@ -1930,17 +1972,19 @@ describe('Traces for ExecuteStream broken stream retries', () => {
19301972
});
19311973

19321974
describe('End to end tracing headers', () => {
1975+
let sandbox;
19331976
let server: grpc.Server;
19341977
let spanner: Spanner;
19351978
let spannerMock: mock.MockSpanner;
19361979
let observabilityOptions: typeof ObservabilityOptions;
19371980

19381981
beforeEach(async () => {
1982+
sandbox = sinon.createSandbox();
19391983
observabilityOptions = {
19401984
enableEndToEndTracing: true,
19411985
};
19421986

1943-
const setupResult = await setup(observabilityOptions);
1987+
const setupResult = await setup(observabilityOptions, sandbox);
19441988
spanner = setupResult.spanner;
19451989
server = setupResult.server;
19461990
spannerMock = setupResult.spannerMock;
@@ -1950,6 +1994,7 @@ describe('End to end tracing headers', () => {
19501994
spannerMock.resetRequests();
19511995
spanner.close();
19521996
server.tryShutdown(() => {});
1997+
sandbox.restore();
19531998
});
19541999

19552000
it('run', done => {

0 commit comments

Comments
 (0)