Skip to content

Commit 36ff528

Browse files
bric3bric3
authored
Muzzle assertions as junit report, upload to Test Optimization (#10591)
chore: Emit a proper JUnit report for muzzle assertions chore: Collect muzzle reports chore: Upload muzzle reports to CI app chore: Proper use of task inputs for the MuzzleEndTask fix: MuzzleEndTask test, also collect buildSrc tests in ci app fix: job_base_name normalization chore: Don't prepend test suite name by muzzle Co-authored-by: brice.dutheil <[email protected]>
1 parent 6cefaf6 commit 36ff528

7 files changed

Lines changed: 320 additions & 35 deletions

File tree

‎.gitlab-ci.yml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -450,12 +450,16 @@ test_published_artifacts:
450450
- source .gitlab/gitlab-utils.sh
451451
- gitlab_section_start "collect-reports" "Collecting reports"
452452
- .gitlab/collect_reports.sh --destination ./check_reports --move
453+
- .gitlab/collect_results.sh
453454
- gitlab_section_end "collect-reports"
454455
artifacts:
455456
when: always
456457
paths:
457458
- ./check_reports
459+
- ./results
458460
- '.gradle/daemon/*/*.out.log'
461+
reports:
462+
junit: results/*.xml
459463
retry:
460464
max: 2
461465
when:
@@ -516,15 +520,21 @@ muzzle:
516520
after_script:
517521
- *container_info
518522
- *cgroup_info
523+
- *set_datadog_api_keys
519524
- source .gitlab/gitlab-utils.sh
520525
- gitlab_section_start "collect-reports" "Collecting reports"
521526
- .gitlab/collect_reports.sh
527+
- .gitlab/collect_results.sh
528+
- .gitlab/upload_ciapp.sh $CACHE_TYPE
522529
- gitlab_section_end "collect-reports"
523530
artifacts:
524531
when: always
525532
paths:
526533
- ./reports
534+
- ./results
527535
- '.gradle/daemon/*/*.out.log'
536+
reports:
537+
junit: results/*.xml
528538

529539
muzzle-dep-report:
530540
extends: .gradle_build

‎.gitlab/collect_results.sh‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -67,15 +67,17 @@ do
6767
# E.g. for the example path: tomcat-5.5_forkedTest_TEST-TomcatServletV1ForkedTest.xml
6868
AGGREGATED_FILE_NAME=$(echo "$RESULT_XML_FILE" | rev | cut -d "/" -f 1,2,5 | rev | tr "/" "_")
6969
echo -n " as $AGGREGATED_FILE_NAME"
70-
cp "$RESULT_XML_FILE" "$TEST_RESULTS_DIR/$AGGREGATED_FILE_NAME"
70+
TARGET_DIR="$TEST_RESULTS_DIR"
71+
mkdir -p "$TARGET_DIR"
72+
cp "$RESULT_XML_FILE" "$TARGET_DIR/$AGGREGATED_FILE_NAME"
7173
# Insert file attribute to testcase XML nodes
7274
get_source_file
73-
sed -i "/<testcase/ s|\(time=\"[^\"]*\"\)|\1 file=\"$file_path\"|g" "$TEST_RESULTS_DIR/$AGGREGATED_FILE_NAME"
75+
sed -i "/<testcase/ s|\(time=\"[^\"]*\"\)|\1 file=\"$file_path\"|g" "$TARGET_DIR/$AGGREGATED_FILE_NAME"
7476
# Replace Java Object hashCode by marker in testcase XML nodes to get stable test names
75-
sed -i '/<testcase/ s/@[0-9a-f]\{5,\}/@HASHCODE/g' "$TEST_RESULTS_DIR/$AGGREGATED_FILE_NAME"
77+
sed -i '/<testcase/ s/@[0-9a-f]\{5,\}/@HASHCODE/g' "$TARGET_DIR/$AGGREGATED_FILE_NAME"
7678
# Replace random port numbers by marker in testcase XML nodes to get stable test names
77-
sed -i '/<testcase/ s/localhost:[0-9]\{2,5\}/localhost:PORT/g' "$TEST_RESULTS_DIR/$AGGREGATED_FILE_NAME"
78-
if cmp -s "$RESULT_XML_FILE" "$TEST_RESULTS_DIR/$AGGREGATED_FILE_NAME"; then
79+
sed -i '/<testcase/ s/localhost:[0-9]\{2,5\}/localhost:PORT/g' "$TARGET_DIR/$AGGREGATED_FILE_NAME"
80+
if cmp -s "$RESULT_XML_FILE" "$TARGET_DIR/$AGGREGATED_FILE_NAME"; then
7981
echo ""
8082
else
8183
echo -n " (non-stable test names detected)"

‎.gitlab/upload_ciapp.sh‎

Lines changed: 29 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,27 @@
11
#!/usr/bin/env bash
22
SERVICE_NAME="dd-trace-java"
33
CACHE_TYPE=$1
4-
TEST_JVM=$2
4+
TEST_JVM=${2:-}
55

66
# CI_JOB_NAME, CI_NODE_INDEX, and CI_NODE_TOTAL are read from GitLab CI environment
77

88
# JAVA_???_HOME are set in the base image for each used JDK https://github.com/DataDog/dd-trace-java-docker-build/blob/master/Dockerfile#L86
9-
JAVA_HOME="JAVA_${TEST_JVM}_HOME"
10-
JAVA_BIN="${!JAVA_HOME}/bin/java"
11-
if [ ! -x "$JAVA_BIN" ]; then
12-
JAVA_BIN=$(which java)
9+
JAVA_PROPS=""
10+
if [ -n "$TEST_JVM" ]; then
11+
JAVA_BIN=""
12+
if [[ "$TEST_JVM" =~ ^[A-Za-z0-9_]+$ ]]; then
13+
JAVA_HOME_VAR="JAVA_${TEST_JVM}_HOME"
14+
JAVA_HOME_VALUE="${!JAVA_HOME_VAR}"
15+
if [ -n "$JAVA_HOME_VALUE" ] && [ -x "$JAVA_HOME_VALUE/bin/java" ]; then
16+
JAVA_BIN="$JAVA_HOME_VALUE/bin/java"
17+
fi
18+
fi
19+
if [ -z "$JAVA_BIN" ]; then
20+
JAVA_BIN="$(command -v java)"
21+
fi
22+
JAVA_PROPS=$($JAVA_BIN -XshowSettings:properties -version 2>&1)
1323
fi
1424

15-
# Extract Java properties from the JVM used to run the tests
16-
JAVA_PROPS=$($JAVA_BIN -XshowSettings:properties -version 2>&1)
1725
java_prop() {
1826
local PROP_NAME=$1
1927
echo "$JAVA_PROPS" | grep "$PROP_NAME" | head -n1 | cut -d'=' -f2 | xargs
@@ -27,11 +35,23 @@ junit_upload() {
2735
# Build custom tags array directly from arguments
2836
local custom_tags_args=()
2937

30-
# Extract job base name from CI_JOB_NAME (strip matrix suffix)
38+
# Extract job base name from CI_JOB_NAME.
39+
# Handles:
40+
# - matrix suffix format: "job-name: [value, 1/6]" -> "job-name"
41+
# - split suffix format: "job-name 1/6" -> "job-name"
3142
local job_base_name="${CI_JOB_NAME%%:*}"
43+
job_base_name="$(echo "$job_base_name" | sed -E 's/[[:space:]]+[0-9]+\/[0-9]+$//')"
3244

3345
# Add custom test configuration tags
34-
custom_tags_args+=(--tags "test.configuration.jvm:${TEST_JVM}")
46+
if [ -n "$TEST_JVM" ]; then
47+
custom_tags_args+=(--tags "test.configuration.jvm:${TEST_JVM}")
48+
custom_tags_args+=(--tags "runtime.name:$(java_prop java.runtime.name)")
49+
custom_tags_args+=(--tags "runtime.vendor:$(java_prop java.vendor)")
50+
custom_tags_args+=(--tags "runtime.version:$(java_prop java.version)")
51+
custom_tags_args+=(--tags "os.architecture:$(java_prop os.arch)")
52+
custom_tags_args+=(--tags "os.platform:$(java_prop os.name)")
53+
custom_tags_args+=(--tags "os.version:$(java_prop os.version)")
54+
fi
3555
if [ -n "$CI_NODE_INDEX" ] && [ -n "$CI_NODE_TOTAL" ]; then
3656
custom_tags_args+=(--tags "test.configuration.split:${CI_NODE_INDEX}/${CI_NODE_TOTAL}")
3757
fi
@@ -43,12 +63,6 @@ junit_upload() {
4363
datadog-ci junit upload --service $SERVICE_NAME \
4464
--logs \
4565
--tags "test.traits:{\"category\":[\"$CACHE_TYPE\"]}" \
46-
--tags "runtime.name:$(java_prop java.runtime.name)" \
47-
--tags "runtime.vendor:$(java_prop java.vendor)" \
48-
--tags "runtime.version:$(java_prop java.version)" \
49-
--tags "os.architecture:$(java_prop os.arch)" \
50-
--tags "os.platform:$(java_prop os.name)" \
51-
--tags "os.version:$(java_prop os.version)" \
5266
--tags "git.repository_url:https://github.com/DataDog/dd-trace-java" \
5367
"${custom_tags_args[@]}" \
5468
./results

‎buildSrc/src/main/kotlin/datadog/gradle/plugin/muzzle/MuzzlePlugin.kt‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,17 +120,20 @@ class MuzzlePlugin : Plugin<Project> {
120120
project.afterEvaluate {
121121
// use runAfter to set up task finalizers in version order
122122
var runAfter: TaskProvider<MuzzleTask> = muzzleTask
123+
val muzzleReportTasks = mutableListOf<TaskProvider<MuzzleTask>>()
123124

124125
project.extensions.getByType<MuzzleExtension>().directives.forEach { directive ->
125126
project.logger.debug("configuring {}", directive)
126127

127128
if (directive.isCoreJdk) {
128129
runAfter = addMuzzleTask(directive, null, project, runAfter, muzzleBootstrap, muzzleTooling)
130+
muzzleReportTasks.add(runAfter)
129131
} else {
130132
val range = resolveVersionRange(directive, system, session)
131133

132134
muzzleDirectiveToArtifacts(directive, range).forEach {
133135
runAfter = addMuzzleTask(directive, it, project, runAfter, muzzleBootstrap, muzzleTooling)
136+
muzzleReportTasks.add(runAfter)
134137
}
135138

136139
if (directive.assertInverse) {
@@ -139,15 +142,21 @@ class MuzzlePlugin : Plugin<Project> {
139142

140143
muzzleDirectiveToArtifacts(inverseDirective, inverseRange).forEach {
141144
runAfter = addMuzzleTask(inverseDirective, it, project, runAfter, muzzleBootstrap, muzzleTooling)
145+
muzzleReportTasks.add(runAfter)
142146
}
143147
}
144148
}
145149
}
146150
project.logger.info("configured $directive")
147151
}
148152

153+
if (muzzleReportTasks.isEmpty() && !project.extensions.getByType<MuzzleExtension>().directives.any { it.assertPass }) {
154+
muzzleReportTasks.add(muzzleTask)
155+
}
156+
149157
val timingTask = project.tasks.register<MuzzleEndTask>("muzzle-end") {
150158
startTimeMs.set(startTime)
159+
muzzleResultFiles.from(muzzleReportTasks.map { it.flatMap { task -> task.result } })
151160
}
152161
// last muzzle task to run
153162
runAfter.configure {

‎buildSrc/src/main/kotlin/datadog/gradle/plugin/muzzle/tasks/MuzzleEndTask.kt‎

Lines changed: 140 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,39 +1,167 @@
11
package datadog.gradle.plugin.muzzle.tasks
22

33
import datadog.gradle.plugin.muzzle.pathSlug
4+
import org.gradle.api.file.ConfigurableFileCollection
45
import org.gradle.api.provider.Property
56
import org.gradle.api.tasks.Input
7+
import org.gradle.api.tasks.InputFiles
68
import org.gradle.api.tasks.OutputFile
9+
import org.gradle.api.tasks.PathSensitive
10+
import org.gradle.api.tasks.PathSensitivity
711
import org.gradle.api.tasks.TaskAction
12+
import java.io.File
13+
import java.io.StringWriter
14+
import javax.xml.stream.XMLOutputFactory
815

916
abstract class MuzzleEndTask : AbstractMuzzleTask() {
1017
@get:Input
1118
abstract val startTimeMs: Property<Long>
1219

20+
@get:InputFiles
21+
@get:PathSensitive(PathSensitivity.RELATIVE)
22+
abstract val muzzleResultFiles: ConfigurableFileCollection
23+
24+
@get:OutputFile
25+
val resultsFile = project
26+
.layout
27+
.buildDirectory
28+
.file("test-results/muzzle/TEST-muzzle-${project.pathSlug}.xml")
29+
1330
@get:OutputFile
14-
val resultsFile = project.rootProject
31+
val legacyResultsFile = project.rootProject
1532
.layout
1633
.buildDirectory
1734
.file("${MUZZLE_TEST_RESULTS}/${project.pathSlug}_muzzle/results.xml")
1835

1936
@TaskAction
2037
fun generatesResultFile() {
38+
val report = buildJUnitReport()
39+
writeReportFile(project.file(resultsFile), renderReportXml(report), "muzzle junit")
40+
writeReportFile(project.file(legacyResultsFile), renderLegacyReportXml(report.durationSeconds), "muzzle legacy")
41+
}
42+
43+
private fun buildJUnitReport(): MuzzleJUnitReport {
2144
val endTimeMs = System.currentTimeMillis()
2245
val seconds = (endTimeMs - startTimeMs.get()).toDouble() / 1000.0
23-
with(project.file(resultsFile)) {
24-
parentFile.mkdirs()
25-
writeText(
26-
"""
27-
<?xml version="1.0" encoding="UTF-8"?>
28-
<testsuite name="$name" tests="1" id="0" time="$seconds">
29-
<testcase name="$name" time="$seconds"/>
30-
</testsuite>
31-
""".trimIndent()
32-
)
33-
project.logger.info("Wrote muzzle results report to\n $this")
46+
val testCases = muzzleResultFiles.files
47+
.sortedBy { it.name }
48+
.map { resultFile ->
49+
val taskName = resultFile.name.removeSuffix(".txt")
50+
when {
51+
!resultFile.exists() -> {
52+
MuzzleJUnitCase(
53+
name = taskName,
54+
failureMessage = "Muzzle result file missing",
55+
failureText = "Expected ${resultFile.path}"
56+
)
57+
}
58+
59+
resultFile.readText() == "PASSING" -> MuzzleJUnitCase(name = taskName)
60+
else -> {
61+
MuzzleJUnitCase(
62+
name = taskName,
63+
failureMessage = "Muzzle validation failed",
64+
failureText = resultFile.readText()
65+
)
66+
}
67+
}
68+
}
69+
return MuzzleJUnitReport(
70+
suiteName = project.path,
71+
module = project.path,
72+
className = "muzzle.${project.pathSlug}",
73+
durationSeconds = seconds,
74+
testCases = testCases
75+
)
76+
}
77+
78+
private fun renderReportXml(report: MuzzleJUnitReport): String {
79+
val output = StringWriter()
80+
val xmlWriter = XMLOutputFactory.newInstance().createXMLStreamWriter(output)
81+
with(xmlWriter) {
82+
try {
83+
writeStartDocument("UTF-8", "1.0")
84+
writeCharacters("\n")
85+
writeStartElement("testsuite")
86+
writeAttribute("name", report.suiteName)
87+
writeAttribute("tests", report.testCases.size.toString())
88+
writeAttribute("failures", report.failures.toString())
89+
writeAttribute("errors", "0")
90+
writeAttribute("skipped", "0")
91+
writeAttribute("time", report.durationSeconds.toString())
92+
writeCharacters("\n")
93+
94+
writeStartElement("properties")
95+
writeCharacters("\n")
96+
writeEmptyElement("property")
97+
writeAttribute("name", "category")
98+
writeAttribute("value", "muzzle")
99+
writeCharacters("\n")
100+
writeEmptyElement("property")
101+
writeAttribute("name", "module")
102+
writeAttribute("value", report.module)
103+
writeCharacters("\n")
104+
writeEndElement()
105+
writeCharacters("\n")
106+
107+
report.testCases.forEach { testCase ->
108+
writeStartElement("testcase")
109+
writeAttribute("classname", report.className)
110+
writeAttribute("name", testCase.name)
111+
writeAttribute("time", "0")
112+
if (testCase.failureMessage != null) {
113+
writeCharacters("\n")
114+
writeStartElement("failure")
115+
writeAttribute("message", testCase.failureMessage)
116+
writeCharacters(testCase.failureText ?: "")
117+
writeEndElement()
118+
writeCharacters("\n")
119+
}
120+
writeEndElement()
121+
writeCharacters("\n")
122+
}
123+
writeEndElement()
124+
writeEndDocument()
125+
flush()
126+
} finally {
127+
close()
128+
}
34129
}
130+
return output.toString()
131+
}
132+
133+
private fun writeReportFile(file: File, xml: String, label: String) {
134+
file.parentFile.mkdirs()
135+
file.writeText(xml)
136+
project.logger.info("Wrote $label report to\n $file")
35137
}
36138

139+
private fun renderLegacyReportXml(durationSeconds: Double): String {
140+
return """
141+
<?xml version="1.0" encoding="UTF-8"?>
142+
<testsuite name="$name" tests="1" id="0" time="$durationSeconds">
143+
<testcase name="$name" time="$durationSeconds"/>
144+
</testsuite>
145+
""".trimIndent()
146+
}
147+
148+
private data class MuzzleJUnitReport(
149+
val suiteName: String,
150+
val module: String,
151+
val className: String,
152+
val durationSeconds: Double,
153+
val testCases: List<MuzzleJUnitCase>
154+
) {
155+
val failures: Int
156+
get() = testCases.count { it.failureMessage != null }
157+
}
158+
159+
private data class MuzzleJUnitCase(
160+
val name: String,
161+
val failureMessage: String? = null,
162+
val failureText: String? = null
163+
)
164+
37165
companion object {
38166
private const val MUZZLE_TEST_RESULTS = "muzzle-test-results"
39167
}

‎buildSrc/src/main/kotlin/datadog/gradle/plugin/muzzle/tasks/MuzzleTask.kt‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -66,10 +66,8 @@ abstract class MuzzleTask @Inject constructor(
6666
@get:Optional
6767
val muzzleDirective: Property<MuzzleDirective> = objects.property()
6868

69-
// This output is only used to make the task cacheable, this is not exposed
7069
@get:OutputFile
71-
@get:Optional
72-
protected val result: RegularFileProperty = objects.fileProperty().convention(
70+
val result: RegularFileProperty = objects.fileProperty().convention(
7371
project.layout.buildDirectory.file("reports/$name.txt")
7472
)
7573

0 commit comments

Comments
 (0)