Skip to content

Commit 81dff08

Browse files
cbasitodxcbasitodx
authored
[test optimization] Update extraction of ci.job.url in github actions (#7685)
1 parent 46ef22d commit 81dff08

File tree

8 files changed

+230
-4
lines changed

8 files changed

+230
-4
lines changed

‎.github/CODEOWNERS‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,8 @@
8787
/packages/dd-trace/test/plugins/util/fixtures/github_event_payload_malformed.json @DataDog/ci-app-libraries
8888
/packages/dd-trace/test/plugins/util/fixtures/github_event_payload.json @DataDog/ci-app-libraries
8989
/packages/dd-trace/test/plugins/util/fixtures/istanbul-map-fixture.json @DataDog/ci-app-libraries
90+
/packages/dd-trace/test/plugins/util/fixtures/runner/ @DataDog/ci-app-libraries
91+
/packages/dd-trace/test/plugins/util/fixtures/runner_empty/ @DataDog/ci-app-libraries
9092

9193
/packages/datadog-instrumentations/src/jest.js @DataDog/ci-app-libraries
9294
/packages/datadog-instrumentations/src/mocha/ @DataDog/ci-app-libraries

‎packages/dd-trace/src/plugins/util/ci.js‎

Lines changed: 95 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
'use strict'
22

3-
const { readFileSync } = require('fs')
3+
const { readFileSync, readdirSync, existsSync } = require('fs')
4+
const path = require('path')
45
const { getEnvironmentVariable, getEnvironmentVariables, getValueFromEnvSources } = require('../../config/helper')
56
const {
67
GIT_BRANCH,
@@ -102,8 +103,92 @@ function getGitHubEventPayload () {
102103
return JSON.parse(readFileSync(path, 'utf8'))
103104
}
104105

106+
function getJobIDFromDiagFile (runnerTemp) {
107+
if (!runnerTemp || !existsSync(runnerTemp)) { return null }
108+
109+
// RUNNER_TEMP usually looks like:
110+
// Linux/mac hosted: /home/runner/work/_temp
111+
// Windows hosted: C:\actions-runner\_work\_temp
112+
// Self-hosted (unix): /opt/actions-runner/_work/_temp
113+
114+
const workDir = path.dirname(runnerTemp) // .../work or .../_work
115+
const runnerRoot = path.dirname(workDir) // /home/runner/ (runner root)
116+
117+
const dirs = [
118+
path.join(runnerRoot, 'cached', '_diag'),
119+
path.join(runnerRoot, '_diag'),
120+
path.join(runnerRoot, 'actions-runner', 'cached', '_diag'),
121+
path.join(runnerRoot, 'actions-runner', '_diag'),
122+
]
123+
124+
const isWin = process.platform === 'win32'
125+
126+
// Hardcoded fallbacks
127+
if (isWin) {
128+
dirs.push(
129+
'C:/actions-runner/cached/_diag',
130+
'C:/actions-runner/_diag',
131+
)
132+
} else {
133+
dirs.push(
134+
'/home/runner/actions-runner/cached/_diag',
135+
'/home/runner/actions-runner/_diag',
136+
'/opt/actions-runner/_diag',
137+
)
138+
}
139+
140+
// Remove duplicates
141+
const possibleDiagsPaths = [...new Set(dirs)]
142+
143+
// This will hold the names of the worker log files that (potentially) contain the Job ID
144+
let workerLogFiles = []
145+
146+
// This will hold the chosen diagnostics path (between the ones that are contemplated in possibleDiagsPath)
147+
let chosenDiagPath = ''
148+
149+
for (const diagPath of possibleDiagsPaths) {
150+
try {
151+
// Obtain a list of fs.Dirent objects of the files in diagPath
152+
const files = readdirSync(diagPath, { withFileTypes: true })
153+
154+
// Check if there are valid potential log files
155+
const potentialLogs = files
156+
.filter((file) => file.isFile() && file.name.startsWith('Worker_'))
157+
.map((file) => file.name)
158+
159+
if (potentialLogs.length > 0) {
160+
chosenDiagPath = diagPath
161+
workerLogFiles = potentialLogs
162+
break // No need to keep looking for more log files
163+
}
164+
} catch (error) {
165+
// If the directory was not found, just look in the next one
166+
if (error instanceof Error && 'code' in error && error.code === 'ENOENT') {
167+
continue
168+
}
169+
170+
// Any other kind of error must force a return
171+
return null
172+
}
173+
}
174+
175+
// Get the job ID via regex
176+
for (const logFile of workerLogFiles) {
177+
const filePath = path.posix.join(chosenDiagPath, logFile)
178+
const content = readFileSync(filePath, 'utf8')
179+
180+
const match = content.match(/"job":\s*{[\s\S]*?"v"\s*:\s*(\d+)(?:\.0)?/)
181+
182+
// match[1] is the captured group with the display name
183+
if (match && match[1]) { return match[1] }
184+
}
185+
186+
return null
187+
}
188+
105189
module.exports = {
106190
normalizeRef,
191+
getJobIDFromDiagFile,
107192
getCIMetadata () {
108193
const env = getEnvironmentVariables()
109194

@@ -281,6 +366,8 @@ module.exports = {
281366
GITHUB_RUN_ATTEMPT,
282367
GITHUB_JOB,
283368
GITHUB_BASE_REF,
369+
RUNNER_TEMP,
370+
JOB_CHECK_RUN_ID,
284371
} = env
285372

286373
const repositoryURL = `${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git`
@@ -290,7 +377,12 @@ module.exports = {
290377
pipelineURL = `${pipelineURL}/attempts/${GITHUB_RUN_ATTEMPT}`
291378
}
292379

293-
const jobUrl = `${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/commit/${GITHUB_SHA}/checks`
380+
// Build the job url extracting the job ID. If extraction fails, job url is constructed as a generalized url
381+
const GITHUB_JOB_ID = JOB_CHECK_RUN_ID ?? getJobIDFromDiagFile(RUNNER_TEMP)
382+
const jobUrl =
383+
GITHUB_JOB_ID === null
384+
? `${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/commit/${GITHUB_SHA}/checks`
385+
: `${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/job/${GITHUB_JOB_ID}`
294386

295387
const ref = GITHUB_HEAD_REF || GITHUB_REF || ''
296388
const refKey = ref.includes('tags/') ? GIT_TAG : GIT_BRANCH
@@ -315,7 +407,7 @@ module.exports = {
315407
GITHUB_RUN_ID,
316408
GITHUB_RUN_ATTEMPT,
317409
}),
318-
[CI_JOB_ID]: GITHUB_JOB,
410+
[CI_JOB_ID]: GITHUB_JOB_ID ?? GITHUB_JOB,
319411
}
320412
if (GITHUB_BASE_REF) { // `pull_request` or `pull_request_target` event
321413
tags[GIT_PULL_REQUEST_BASE_BRANCH] = GITHUB_BASE_REF

‎packages/dd-trace/test/plugins/util/fixtures/.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# Some fixtures are log files and logs are ignored in the global `.gitignore`
2+
!*.log

‎packages/dd-trace/test/plugins/util/fixtures/runner/actions-runner/_diag/Worker_20240115-102345-12345.log‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
[2024-01-15 10:23:45Z INFO WorkerMessageServer] THIS IS A MOCK DIAGNOSTICS FILE CREATED BY CLAUDE FOR TESTING getJobIDFromDiagFile FUNCTION
2+
[2024-01-15 10:23:45Z INFO WorkerMessageServer] Worker starting, pid: 12345
3+
[2024-01-15 10:23:45Z INFO WorkerMessageServer] Connecting to runner process
4+
[2024-01-15 10:23:45Z INFO WorkerMessageServer] Connection established
5+
[2024-01-15 10:23:46Z INFO Worker] Processing JobRequest message
6+
[2024-01-15 10:23:46Z INFO Worker] Received job message:
7+
{
8+
"messageType": "PipelineAgentJobRequest",
9+
"plan": {
10+
"scopeIdentifier": "3a7e8b2d-1f4c-4e9a-b6d3-9c2f1a8e5d7b",
11+
"planType": "Build",
12+
"planId": "f2c4a6e8-3b5d-4f7a-9c1e-2d4f6a8c0e2a",
13+
"version": "1",
14+
"artifactUri": "https://pipelines.actions.githubusercontent.com/abc123XYZ",
15+
"requestedForId": "1234567"
16+
},
17+
"timeline": {
18+
"id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890",
19+
"changeId": 1,
20+
"location": null
21+
},
22+
"jobId": "d4e5f6a7-b8c9-0123-def4-567890abcdef",
23+
"jobDisplayName": "build-and-test",
24+
"jobName": "build-and-test",
25+
"requestId": 9876543210,
26+
"lockedUntil": "2024-01-15T11:23:46.000Z",
27+
"resources": {
28+
"endpoints": [
29+
{
30+
"data": {
31+
"downloadUrl": "https://codeload.github.com/"
32+
},
33+
"name": "GITHUB_TOKEN",
34+
"url": "https://api.github.com/",
35+
"authorization": {
36+
"scheme": "OAuth",
37+
"parameters": {
38+
"accessToken": "***"
39+
}
40+
},
41+
"isShared": false,
42+
"isReady": true
43+
}
44+
],
45+
"files": [],
46+
"repositories": [
47+
{
48+
"alias": "self",
49+
"id": "repo-id-abc123",
50+
"type": "GitHub",
51+
"name": "my-org/my-repo",
52+
"url": "https://github.com/my-org/my-repo",
53+
"version": "abc123def456",
54+
"ref": "refs/heads/main"
55+
}
56+
],
57+
"containers": []
58+
},
59+
"variables": {
60+
"system.github.job": { "value": "build-and-test", "isSecret": false },
61+
"GITHUB_RUN_ID": { "value": "7654321", "isSecret": false },
62+
"GITHUB_RUN_NUMBER": { "value": "42", "isSecret": false },
63+
"GITHUB_WORKFLOW": { "value": "CI", "isSecret": false }
64+
},
65+
"steps": [
66+
{
67+
"type": "Task",
68+
"id": "step-id-0001",
69+
"name": "actions/checkout",
70+
"displayName": "Checkout repository",
71+
"enabled": true,
72+
"continueOnError": false,
73+
"condition": "succeeded()",
74+
"timeoutInMinutes": 0,
75+
"inputs": {
76+
"repository": "my-org/my-repo",
77+
"ref": "refs/heads/main",
78+
"token": "***"
79+
},
80+
"environment": {},
81+
"retryCountOnTaskFailure": 0
82+
}
83+
],
84+
"contextData": {},
85+
"workspace": { "clean": null },
86+
"mask": [
87+
{ "type": "regex", "value": "\\*\\*\\*" }
88+
],
89+
"oidcToken": null,
90+
"finaStrategy": null,
91+
"job": {
92+
"v": 9876543210,
93+
"name": "build-and-test",
94+
"id": "d4e5f6a7-b8c9-0123-def4-567890abcdef",
95+
"attempt": 1,
96+
"workflowName": "CI",
97+
"headBranch": "refs/heads/main",
98+
"headSha": "abc123def456abc123def456abc123def456abc1"
99+
}
100+
}
101+
[2024-01-15 10:23:46Z INFO Worker] Job assignment accepted, jobId: d4e5f6a7-b8c9-0123-def4-567890abcdef
102+
[2024-01-15 10:23:46Z INFO JobRunner] Starting job execution
103+
[2024-01-15 10:23:47Z INFO StepRunner] Running step: Checkout repository
104+
[2024-01-15 10:24:01Z INFO StepRunner] Step completed: Checkout repository (result: Succeeded)
105+
[2024-01-15 10:24:01Z INFO JobRunner] All steps completed
106+
[2024-01-15 10:24:02Z INFO Worker] Job completed, result: Succeeded
107+
[2024-01-15 10:24:02Z INFO WorkerMessageServer] Sending job result to runner process
108+
[2024-01-15 10:24:02Z INFO WorkerMessageServer] Worker shutting down

‎packages/dd-trace/test/plugins/util/fixtures/runner/work/_temp/donoterase.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
This file exists to simulate the structure of github actions runner
2+
It is here just so github can keep track of the whole directory structure

‎packages/dd-trace/test/plugins/util/fixtures/runner_empty/actions-runner/_diag/Worker_empty.log‎

Whitespace-only changes.

‎packages/dd-trace/test/plugins/util/fixtures/runner_empty/work/_temp/donoterase.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
This file exists to simulate the structure of github actions runner
2+
It is here just so github can keep track of the whole directory structure

‎packages/dd-trace/test/plugins/util/test-environment.spec.js‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ require('../../setup/core')
1313

1414
const cachedExecStub = sinon.stub().returns('')
1515

16-
const { getCIMetadata } = require('../../../src/plugins/util/ci')
16+
const { getCIMetadata, getJobIDFromDiagFile } = require('../../../src/plugins/util/ci')
1717
const {
1818
CI_ENV_VARS,
1919
CI_NODE_LABELS,
@@ -112,3 +112,21 @@ describe('test environment data', () => {
112112
})
113113
})
114114
})
115+
116+
describe('test getJobIDFromDiagFile function', () => {
117+
const TEST_HOME = path.join(__dirname, 'fixtures')
118+
119+
const runnerTempPaths = [
120+
{ runnerTemp: path.join(TEST_HOME, '/runner/work/_temp'), expected: '9876543210' },
121+
{ runnerTemp: null, expected: null },
122+
{ runnerTemp: undefined, expected: null },
123+
{ runnerTemp: path.join(TEST_HOME, Math.random().toString(36).slice(2, 10)), expected: null },
124+
{ runnerTemp: path.join(TEST_HOME, '/runner_empty/work/_temp'), expected: null },
125+
]
126+
127+
for (const { runnerTemp, expected } of runnerTempPaths) {
128+
it(`returns ${expected} for runnerTemp: ${runnerTemp}`, () => {
129+
assert.strictEqual(getJobIDFromDiagFile(runnerTemp), expected)
130+
})
131+
}
132+
})

0 commit comments

Comments
 (0)