Skip to content

Commit 89db27e

Browse files
authored
agentless via json intake (#7632)
agentless via json intake switch to batching by trace, and add a stress test script simplified export lint codeowners fix Co-authored-by: bryan.english <[email protected]>
1 parent d7bee55 commit 89db27e

File tree

14 files changed

+1460
-0
lines changed

14 files changed

+1460
-0
lines changed

CODEOWNERS

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -199,10 +199,13 @@
199199
/packages/dd-trace/test/dogstatsd.spec.js @DataDog/lang-platform-js
200200
/packages/dd-trace/test/encode/0.4.spec.js @DataDog/lang-platform-js
201201
/packages/dd-trace/test/encode/0.5.spec.js @DataDog/lang-platform-js
202+
/packages/dd-trace/test/encode/agentless-json.spec.js @DataDog/lang-platform-js
202203
/packages/dd-trace/test/encode/span-stats.spec.js @DataDog/lang-platform-js
203204
/packages/dd-trace/test/exporter.spec.js @DataDog/lang-platform-js
204205
/packages/dd-trace/test/exporters/agent/exporter.spec.js @DataDog/lang-platform-js
205206
/packages/dd-trace/test/exporters/agent/writer.spec.js @DataDog/lang-platform-js
207+
/packages/dd-trace/test/exporters/agentless/exporter.spec.js @DataDog/lang-platform-js
208+
/packages/dd-trace/test/exporters/agentless/writer.spec.js @DataDog/lang-platform-js
206209
/packages/dd-trace/test/exporters/common/buffering-exporter.spec.js @DataDog/lang-platform-js
207210
/packages/dd-trace/test/exporters/common/docker.spec.js @DataDog/lang-platform-js
208211
/packages/dd-trace/test/exporters/common/request.spec.js @DataDog/lang-platform-js

ext/exporters.d.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
declare const exporters: {
22
LOG: 'log',
33
AGENT: 'agent',
4+
AGENTLESS: 'agentless',
45
DATADOG: 'datadog',
56
AGENT_PROXY: 'agent_proxy',
67
JEST_WORKER: 'jest_worker',

ext/exporters.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
module.exports = {
33
LOG: 'log',
44
AGENT: 'agent',
5+
AGENTLESS: 'agentless',
56
DATADOG: 'datadog',
67
AGENT_PROXY: 'agent_proxy',
78
JEST_WORKER: 'jest_worker',

packages/dd-trace/src/config/index.js

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1103,6 +1103,21 @@ class Config {
11031103
? new URL(DD_CIVISIBILITY_AGENTLESS_URL)
11041104
: getAgentUrl(this.#getTraceAgentUrl(), this.#optionsArg)
11051105

1106+
// Experimental agentless APM span intake
1107+
// When enabled, sends spans directly to Datadog intake without an agent
1108+
const agentlessEnabled = isTrue(getEnv('_DD_APM_TRACING_AGENTLESS_ENABLED'))
1109+
if (agentlessEnabled) {
1110+
setString(calc, 'experimental.exporter', 'agentless')
1111+
// Disable rate limiting - server-side sampling will be used
1112+
calc['sampler.rateLimit'] = -1
1113+
// Disable client-side stats computation
1114+
setBoolean(calc, 'stats.enabled', false)
1115+
// Enable hostname reporting
1116+
setBoolean(calc, 'reportHostname', true)
1117+
// Clear sampling rules - server-side sampling handles this
1118+
calc['sampler.rules'] = []
1119+
}
1120+
11061121
if (this.#isCiVisibility()) {
11071122
setBoolean(calc, 'isEarlyFlakeDetectionEnabled',
11081123
getEnv('DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED') ?? true)

packages/dd-trace/src/config/supported-configurations.json

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,14 @@
2525
"default": null
2626
}
2727
],
28+
"_DD_APM_TRACING_AGENTLESS_ENABLED": [
29+
{
30+
"implementation": "A",
31+
"type": "boolean",
32+
"default": "false",
33+
"description": "Experimental: Enable agentless APM span intake. When enabled, spans are sent directly to Datadog intake without an agent."
34+
}
35+
],
2836
"DD_AGENT_HOST": [
2937
{
3038
"implementation": "E",
Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
'use strict'
2+
3+
const log = require('../log')
4+
const { truncateSpan, normalizeSpan } = require('./tags-processors')
5+
6+
/**
7+
* Formats a span for JSON encoding.
8+
* @param {object} span - The span to format
9+
* @returns {object} The formatted span
10+
*/
11+
function formatSpan (span) {
12+
span = normalizeSpan(truncateSpan(span, false))
13+
14+
if (span.span_events) {
15+
span.meta.events = JSON.stringify(span.span_events)
16+
delete span.span_events
17+
}
18+
19+
return span
20+
}
21+
22+
/**
23+
* Converts a span to JSON-serializable format.
24+
* IDs are converted to lowercase hex strings. Start time is converted from
25+
* nanoseconds to seconds for the intake format.
26+
* @param {object} span - The formatted span
27+
* @returns {object} JSON-serializable span object
28+
*/
29+
function spanToJSON (span) {
30+
const result = {
31+
trace_id: span.trace_id.toString(16).toLowerCase(),
32+
span_id: span.span_id.toString(16).toLowerCase(),
33+
parent_id: span.parent_id.toString(16).toLowerCase(),
34+
name: span.name,
35+
resource: span.resource,
36+
service: span.service,
37+
error: span.error,
38+
start: Math.floor(span.start / 1e9),
39+
duration: span.duration,
40+
meta: span.meta,
41+
metrics: span.metrics,
42+
}
43+
44+
if (span.type) {
45+
result.type = span.type
46+
}
47+
48+
if (span.meta_struct) {
49+
result.meta_struct = span.meta_struct
50+
}
51+
52+
if (span.links && span.links.length > 0) {
53+
result.links = span.links
54+
}
55+
56+
return result
57+
}
58+
59+
/**
60+
* JSON encoder for agentless span intake.
61+
* Encodes a single trace as JSON with the payload format: {"spans": [...]}
62+
*
63+
* This encoder handles one trace at a time since each trace must be sent as a
64+
* separate request to the intake. -- bengl
65+
*/
66+
class AgentlessJSONEncoder {
67+
constructor () {
68+
this._reset()
69+
}
70+
71+
/**
72+
* Returns the number of spans encoded.
73+
* @returns {number}
74+
*/
75+
count () {
76+
return this._spanCount
77+
}
78+
79+
/**
80+
* Encodes a trace (array of spans) into the buffer.
81+
* @param {object[]} trace - Array of spans to encode
82+
*/
83+
encode (trace) {
84+
for (const span of trace) {
85+
try {
86+
const formattedSpan = formatSpan(span)
87+
const jsonSpan = spanToJSON(formattedSpan)
88+
89+
this._spans.push(jsonSpan)
90+
this._spanCount++
91+
} catch (err) {
92+
log.error(
93+
'Failed to encode span (name: %s, service: %s). Span will be dropped. Error: %s\n%s',
94+
span?.name || 'unknown',
95+
span?.service || 'unknown',
96+
err.message,
97+
err.stack
98+
)
99+
}
100+
}
101+
}
102+
103+
/**
104+
* Creates the JSON payload for the encoded trace.
105+
* @returns {Buffer} JSON payload as a buffer, or empty buffer if no spans
106+
*/
107+
makePayload () {
108+
if (this._spans.length === 0) {
109+
this._reset()
110+
return Buffer.alloc(0)
111+
}
112+
113+
try {
114+
const payload = JSON.stringify({ spans: this._spans })
115+
this._reset()
116+
return Buffer.from(payload, 'utf8')
117+
} catch (err) {
118+
log.error(
119+
'Failed to encode trace as JSON (%d spans). Trace will be dropped. Error: %s',
120+
this._spans.length,
121+
err.message
122+
)
123+
this._reset()
124+
return Buffer.alloc(0)
125+
}
126+
}
127+
128+
/**
129+
* Resets the encoder state.
130+
*/
131+
reset () {
132+
this._reset()
133+
}
134+
135+
_reset () {
136+
this._spans = []
137+
this._spanCount = 0
138+
}
139+
}
140+
141+
module.exports = { AgentlessJSONEncoder }

packages/dd-trace/src/exporter.js

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ module.exports = function getExporter (name) {
1111
return require('./exporters/log')
1212
case exporters.AGENT:
1313
return require('./exporters/agent')
14+
case exporters.AGENTLESS:
15+
return require('./exporters/agentless')
1416
case exporters.DATADOG:
1517
return require('./ci-visibility/exporters/agentless')
1618
case exporters.AGENT_PROXY:
Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
'use strict'
2+
3+
const { URL } = require('node:url')
4+
5+
const log = require('../../log')
6+
const Writer = require('./writer')
7+
8+
/**
9+
* Agentless exporter for APM span intake.
10+
* Sends spans directly to the Datadog intake without requiring a local agent.
11+
*
12+
* Each trace is sent immediately as a separate request. The intake only accepts one trace
13+
* per request - requests with spans from different traces return HTTP 200 but silently
14+
* drop all spans. By flushing immediately after each export (which contains one trace),
15+
* we avoid this limitation entirely. -- bengl
16+
*/
17+
class AgentlessExporter {
18+
/**
19+
* @param {object} config - Configuration object
20+
* @param {string} [config.site='datadoghq.com'] - The Datadog site
21+
* @param {string} [config.url] - Override intake URL
22+
*/
23+
constructor (config) {
24+
this._config = config
25+
const { site = 'datadoghq.com', url } = config
26+
27+
try {
28+
this._url = url ? new URL(url) : new URL(`https://public-trace-http-intake.logs.${site}`)
29+
} catch (err) {
30+
log.error(
31+
'Invalid URL configuration for agentless exporter. url=%s, site=%s. Error: %s',
32+
url || 'not set',
33+
site,
34+
err.message
35+
)
36+
this._url = null
37+
}
38+
39+
this._writer = new Writer({
40+
url: this._url,
41+
site,
42+
})
43+
44+
const ddTrace = globalThis[Symbol.for('dd-trace')]
45+
if (ddTrace?.beforeExitHandlers) {
46+
ddTrace.beforeExitHandlers.add(this.flush.bind(this))
47+
} else {
48+
log.error('dd-trace global not properly initialized. beforeExit handler not registered for agentless exporter.')
49+
}
50+
}
51+
52+
/**
53+
* Sets the intake URL.
54+
* @param {string} urlString - The new intake URL
55+
* @returns {boolean} True if URL was set successfully
56+
*/
57+
setUrl (urlString) {
58+
try {
59+
const url = new URL(urlString)
60+
this._url = url
61+
this._writer.setUrl(url)
62+
return true
63+
} catch {
64+
log.error('Invalid URL for agentless exporter: %s. Using previous URL: %s', urlString, this._url?.href || 'none')
65+
return false
66+
}
67+
}
68+
69+
/**
70+
* Exports a trace to the intake. Flushes immediately since each trace must be
71+
* sent as a separate request.
72+
* @param {object[]} spans - Array of spans (all from the same trace)
73+
*/
74+
export (spans) {
75+
this._writer.append(spans)
76+
this._writer.flush()
77+
}
78+
79+
/**
80+
* Flushes any pending spans. With immediate flush per trace, this is mainly
81+
* used for the beforeExit handler to ensure nothing is left unsent.
82+
* @param {Function} [done] - Callback when flush is complete
83+
*/
84+
flush (done = () => {}) {
85+
this._writer.flush(done)
86+
}
87+
}
88+
89+
module.exports = AgentlessExporter

0 commit comments

Comments
 (0)