Skip to content

Commit 9f09a75

Browse files
CelianRCelianR
authored
[acix-24] Notify job status as infra failures using job_failure (#24215)
* Reapply "[gitlab-use-module] Use gitlab python module instead of raw http requests (#24070)" (#24651) This reverts commit 3215c0d. * [gitlab-use-module] Fixed pipeline not refreshed * [gitlab-use-module] Fixed pipeline not refreshed * [filter-out-non-script-failures] Remade changes * [filter-out-non-script-failures] Fixed test data * [filter-out-non-script-failures] Added unit tests * [filter-out-non-script-failures] Cleaned code * [filter-out-non-script-failures] Removed duplicated code
1 parent 9e59192 commit 9f09a75

4 files changed

Lines changed: 96 additions & 3 deletions

File tree

‎tasks/libs/pipeline/data.py‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ def get_failed_jobs(project_name: str, pipeline_id: str) -> FailedJobs:
3333
# Check the final job in the list: it contains the current status of the job
3434
# This excludes jobs that were retried and succeeded
3535
trace = str(repo.jobs.get(job.id, lazy=True).trace(), 'utf-8')
36-
failure_type, failure_reason = get_job_failure_context(trace)
36+
failure_type, failure_reason = get_job_failure_context(job, trace)
3737
final_status = ProjectJob(
3838
repo.manager,
3939
attrs={
@@ -120,15 +120,21 @@ def get_failed_jobs(project_name: str, pipeline_id: str) -> FailedJobs:
120120
]
121121

122122

123-
def get_job_failure_context(job_log):
123+
def get_job_failure_context(job: ProjectJob, job_log: str):
124124
"""
125125
Parses job logs (provided as a string), and returns the type of failure (infra or job) as well
126126
as the precise reason why the job failed.
127127
"""
128128

129+
infra_failure_reasons = FailedJobReason.get_infra_failure_mapping().keys()
130+
131+
if job.failure_reason in infra_failure_reasons:
132+
return FailedJobType.INFRA_FAILURE, FailedJobReason.from_gitlab_job_failure_reason(job.failure_reason)
133+
129134
for regex, type in infra_failure_logs:
130135
if regex.search(job_log):
131136
return FailedJobType.INFRA_FAILURE, type
137+
132138
return FailedJobType.JOB_FAILURE, FailedJobReason.FAILED_JOB_SCRIPT
133139

134140

‎tasks/libs/types/types.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,22 @@ class FailedJobReason(Enum):
5454
EC2_SPOT = 8
5555
E2E_INFRA_FAILURE = 9
5656

57+
@staticmethod
58+
def get_infra_failure_mapping():
59+
return {
60+
'runner_system_failure': FailedJobReason.RUNNER,
61+
'stuck_or_timeout_failure': FailedJobReason.GITLAB,
62+
'unknown_failure': FailedJobReason.GITLAB,
63+
'api_failure': FailedJobReason.GITLAB,
64+
'scheduler_failure': FailedJobReason.GITLAB,
65+
'stale_schedule': FailedJobReason.GITLAB,
66+
'data_integrity_failure': FailedJobReason.GITLAB,
67+
}
68+
69+
@staticmethod
70+
def from_gitlab_job_failure_reason(failure_reason: str):
71+
return FailedJobReason.get_infra_failure_mapping().get(failure_reason, FailedJobReason.GITLAB)
72+
5773

5874
class FailedJobs:
5975
def __init__(self):

‎tasks/unit-tests/pipeline_lib_tests.py‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,11 @@
11
import unittest
2+
from unittest.mock import MagicMock
3+
4+
from gitlab.v4.objects import ProjectJob
25

36
from tasks.libs.pipeline import notifications
7+
from tasks.libs.pipeline.data import get_job_failure_context
8+
from tasks.libs.types.types import FailedJobReason, FailedJobType
49

510

611
class TestLoadAndValidate(unittest.TestCase):
@@ -12,3 +17,69 @@ def test_files_loaded_correctly(self):
1217
# Assert that a couple of expected entries are there, including one that uses DEFAULT_SLACK_PROJECT
1318
self.assertEqual(notifications.GITHUB_SLACK_MAP['@datadog/agent-all'], "#datadog-agent-pipelines")
1419
self.assertEqual(notifications.GITHUB_SLACK_MAP['@datadog/agent-ci-experience'], "#agent-developer-experience")
20+
21+
22+
class TestFailedJobs(unittest.TestCase):
23+
def test_infra_failure(self):
24+
job = ProjectJob(
25+
MagicMock(),
26+
attrs={
27+
"name": "test",
28+
"id": 618,
29+
"stage": "test",
30+
"status": "failed",
31+
"tag_list": [],
32+
"allow_failure": False,
33+
"web_url": "https://sometest.test",
34+
"retry_summary": ["failed"],
35+
"failure_reason": "runner_system_failure",
36+
},
37+
)
38+
log = 'Empty log'
39+
40+
fail_type, _fail_reason = get_job_failure_context(job, log)
41+
42+
self.assertEqual(fail_type, FailedJobType.INFRA_FAILURE)
43+
44+
def test_infra_failure_log(self):
45+
job = ProjectJob(
46+
MagicMock(),
47+
attrs={
48+
"name": "test",
49+
"id": 618,
50+
"stage": "test",
51+
"status": "failed",
52+
"tag_list": [],
53+
"allow_failure": False,
54+
"web_url": "https://sometest.test",
55+
"retry_summary": ["failed"],
56+
"failure_reason": "script_failure",
57+
},
58+
)
59+
log = 'Some test...\nE2E INTERNAL ERROR\n...\n'
60+
61+
fail_type, fail_reason = get_job_failure_context(job, log)
62+
63+
self.assertEqual(fail_type, FailedJobType.INFRA_FAILURE)
64+
self.assertEqual(fail_reason, FailedJobReason.E2E_INFRA_FAILURE)
65+
66+
def test_non_infra_failure_log(self):
67+
job = ProjectJob(
68+
MagicMock(),
69+
attrs={
70+
"name": "test",
71+
"id": 618,
72+
"stage": "test",
73+
"status": "failed",
74+
"tag_list": [],
75+
"allow_failure": False,
76+
"web_url": "https://sometest.test",
77+
"retry_summary": ["failed"],
78+
"failure_reason": "script_failure",
79+
},
80+
)
81+
log = '...E\nTraceback...\n'
82+
83+
fail_type, _fail_reason = get_job_failure_context(job, log)
84+
85+
self.assertEqual(fail_type, FailedJobType.JOB_FAILURE)

‎tasks/unit-tests/testdata/jobs.json‎

Lines changed: 1 addition & 1 deletion
Large diffs are not rendered by default.

0 commit comments

Comments
 (0)