Skip to content

Commit ae6b9ab

Browse files
darcciodarccio
authored
Merge branch 'main' into dario.castane/er/version-stats
2 parents cb4d501 + 87fcf6f commit ae6b9ab

59 files changed

Lines changed: 2561 additions & 476 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.gitlab-ci.yml‎

Lines changed: 5 additions & 109 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
stages:
2+
- ci-image
23
- docker-images-reliability-env
34
- generate
4-
- benchmarks
5+
- microbenchmarks
56
- macrobenchmarks
67
# These benchmarks are planned to replace the legacy macrobenchmarks long term
78
- go-go-prof-app-parallel
@@ -11,118 +12,12 @@ stages:
1112
- test-apps
1213

1314
variables:
14-
# This base image is created here: https://gitlab.ddbuild.io/DataDog/apm-reliability/benchmarking-platform/-/jobs/1577883208
15-
BASE_CI_IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/benchmarking-platform:dd-trace-go-106683130
1615
INDEX_FILE: index.txt
17-
BENCHMARK_TARGETS: "BenchmarkStartRequestSpan|BenchmarkHttpServeTrace|BenchmarkTracerAddSpans|BenchmarkStartSpan|BenchmarkSingleSpanRetention|BenchmarkOTelApiWithCustomTags|BenchmarkInjectW3C|BenchmarkExtractW3C|BenchmarkPartialFlushing|BenchmarkConfig|BenchmarkStartSpanConfig|BenchmarkGraphQL|BenchmarkSampleWAFContext|BenchmarkCaptureStackTrace|BenchmarkSetTagString|BenchmarkSetTagStringPtr|BenchmarkSetTagMetric|BenchmarkSetTagStringer|BenchmarkSerializeSpanLinksInMeta|BenchmarkLogs|BenchmarkParallelLogs|BenchmarkMetrics|BenchmarkParallelMetrics|BenchmarkPayloadVersions|BenchmarkOTLPTraceWriterAdd|BenchmarkOTLPTraceWriterFlush|BenchmarkOTLPProtoMarshal|BenchmarkOTLPProtoSize|BenchmarkOTLPTraceWriterConcurrent"
1816

1917
workflow:
2018
auto_cancel:
2119
on_new_commit: interruptible
2220

23-
.benchmark-rules: &benchmark-rules
24-
- if: '$CI_COMMIT_TAG =~ /^.*\/v\d+\.\d+\.\d+(?:-.*\.\d+)?$/'
25-
when: never
26-
- if: '$CI_COMMIT_BRANCH =~ /^graphite-base\/.*$/'
27-
when: never
28-
- if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-.*$/'
29-
when: never
30-
- if: '$CI_COMMIT_BRANCH == "main"'
31-
when: always
32-
interruptible: false
33-
- when: always
34-
interruptible: true
35-
36-
# In order to run benchmarks in parallel, we generate a matrix of test names based on the BENCHMARK_TARGETS variable.
37-
# This will be used in tandem with bp-runner in benchmarks.yml.
38-
# This will allow us to spin up a child job in GitLab CI that handles running all of the benchmarks in parallel.
39-
generate_matrix:
40-
stage: generate
41-
image: $BASE_CI_IMAGE
42-
tags: ["arch:amd64"]
43-
rules: *benchmark-rules
44-
script: |
45-
echo "=== Debug: Environment ==="
46-
echo "Image: ${BASE_CI_IMAGE}"
47-
go version
48-
go env GOTOOLCHAIN GOWORK
49-
echo "=== End Debug ==="
50-
cd .gitlab
51-
GOWORK=off go run generate_config.go
52-
mv generated_benchmark_matrix.yml ../generated_benchmark_matrix.yml
53-
artifacts:
54-
paths:
55-
- generated_benchmark_matrix.yml
56-
expire_in: 1 hour # Artifact is temporary, needed only for the current pipeline
57-
58-
trigger_child_pipeline:
59-
stage: benchmarks
60-
trigger:
61-
include:
62-
- artifact: generated_benchmark_matrix.yml
63-
job: generate_matrix
64-
strategy: depend
65-
needs:
66-
- generate_matrix
67-
rules: *benchmark-rules
68-
variables:
69-
PARENT_PIPELINE_ID: $CI_PIPELINE_ID
70-
71-
check-big-regressions:
72-
stage: benchmarks
73-
needs:
74-
- job: trigger_child_pipeline
75-
rules: *benchmark-rules
76-
when: on_success
77-
tags:
78-
- "arch:amd64"
79-
image: $BASE_CI_IMAGE
80-
script:
81-
- |
82-
pwd
83-
export ARTIFACTS_DIR="$(pwd)/reports" && (mkdir "${ARTIFACTS_DIR}" || :)
84-
git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/"
85-
git clone --branch dd-trace-go https://github.com/DataDog/benchmarking-platform platform && cd platform
86-
./steps/download-child-pipeline-artifacts.sh
87-
pushd "${ARTIFACTS_DIR}/"
88-
pwd
89-
bp-runner ../platform/bp-runner.fail-on-regression.yml --debug
90-
artifacts:
91-
name: "artifacts"
92-
when: always
93-
paths:
94-
- reports/
95-
expire_in: "30 days"
96-
97-
analyze-benchmark-results:
98-
stage: benchmarks
99-
needs:
100-
- job: trigger_child_pipeline
101-
rules: *benchmark-rules
102-
when: always
103-
tags:
104-
- "arch:amd64"
105-
image: $BASE_CI_IMAGE
106-
script:
107-
- |
108-
mkdir -p reports
109-
export ARTIFACTS_DIR="$(pwd)/reports"
110-
git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/"
111-
git clone --branch dd-trace-go https://github.com/DataDog/benchmarking-platform platform && cd platform
112-
./steps/download-child-pipeline-artifacts.sh
113-
echo "All benchmark artifacts collected:"
114-
ls -la "${ARTIFACTS_DIR}/"
115-
./steps/analyze-results.sh
116-
./steps/post-pr-comment.sh
117-
118-
artifacts:
119-
name: "reports"
120-
when: always
121-
paths:
122-
- reports/
123-
expire_in: 3 months
124-
125-
12621
# Config Registry CI Jobs
12722
validate_supported_configurations_v2_local_file:
12823
stage: config-validation
@@ -146,8 +41,9 @@ update_central_configurations_version_range_v2:
14641
LANGUAGE_NAME: "golang"
14742

14843
include:
149-
- local: ".gitlab/macrobenchmarks.yml"
150-
- local: ".gitlab/test-apps.yml"
44+
- local: ".gitlab/benchmarks/macro/gitlab-ci.yml"
45+
- local: ".gitlab/benchmarks/micro/gitlab-ci.yml"
46+
- local: ".gitlab/benchmarks/test-apps/test-apps.yml"
15147
- local: ".gitlab/docker-images-reliability-env.yml"
15248
- local: ".gitlab/configuration-central-validation.yml"
15349
- project: 'DataDog/apm-reliability/apm-sdks-benchmarks'

‎.gitlab/benchmarks.yml‎

Lines changed: 0 additions & 30 deletions
This file was deleted.

‎.gitlab/benchmarks/README.md‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
# Benchmarks
2+
3+
This directory contains the configuration for the performance benchmarks that
4+
run on the [Datadog Benchmarking Platform](https://datadoghq.atlassian.net/wiki/spaces/APMINT/pages/2419261562/Benchmarking+Platform)
5+
via GitLab CI. Benchmarks are split into three complementary suites plus the
6+
shared performance-gate configuration.
7+
8+
## Layout
9+
10+
- **`micro/`** — Go microbenchmarks (`go test -bench`) executed on every PR.
11+
Each PR is compared against a baseline (`main`) and the pipeline fails on
12+
regressions above the configured thresholds. See `micro/README.md` for how
13+
to add, run, or mark benchmarks as flaky, and how to run the pipeline
14+
locally with `bp-runner`.
15+
- **`macro/`** — End-to-end macrobenchmarks that build the
16+
[`go-prof-app`](https://gitlab.ddbuild.io/DataDog/benchmarking-platform)
17+
SUT with the candidate `dd-trace-go` and exercise it under realistic load
18+
scenarios (`io-bound`, `cpu-bound`, `cgo-cpu-bound`, etc.). Runs
19+
automatically on `main` and release branches, and on-demand (manual) on
20+
feature branches. Used to detect regressions in tracer, profiler, AppSec,
21+
and service-extensions overhead.
22+
- **`test-apps/`** — Long-running test applications (e.g. `unit-of-work`)
23+
used to capture profiles and validate behavior over extended runs
24+
(~10+ minutes). Triggered manually; useful for investigating issues that
25+
short benchmarks cannot reproduce.
26+
- **`pr-gate.thresholds.yml`** — Regression thresholds enforced by the
27+
`pr-performance-gates` job for microbenchmark PR runs.
28+
- **`pre-release-gate.slos.yml`** — SLOs evaluated before releases to gate
29+
promotion based on macrobenchmark results.
30+
31+
## External: `go-go-prof-app-parallel`
32+
33+
In addition to the suites under this directory, the top-level
34+
`.gitlab-ci.yml` includes a shared pipeline definition from the
35+
[`DataDog/apm-reliability/apm-sdks-benchmarks`](https://gitlab.ddbuild.io/DataDog/apm-reliability/apm-sdks-benchmarks)
36+
project:
37+
38+
```yaml
39+
- project: 'DataDog/apm-reliability/apm-sdks-benchmarks'
40+
file: '.gitlab/ci-go-go-prof-app-parallel.yml'
41+
ref: 'main'
42+
```
43+
44+
This contributes the `go-go-prof-app-parallel` and
45+
`go-go-prof-app-parallel-slo` stages, which run a next-generation,
46+
parallelized variant of the `go-prof-app` macrobenchmarks. The job
47+
definitions, SUT image, and SLOs are maintained centrally in
48+
`apm-sdks-benchmarks` (shared across APM SDK repositories) rather than
49+
in this repository. These benchmarks are intended to replace the legacy
50+
`macro/` suite long term; until that migration completes, both suites
51+
run side by side. To change thresholds, scenarios, or runner
52+
configuration for this suite, open a PR against
53+
`apm-sdks-benchmarks`.
54+
55+
## How it fits together
56+
57+
1. On every PR, `micro/gitlab-ci.yml` runs the microbenchmarks suite and
58+
compares results against `main` using `pr-gate.thresholds.yml`.
59+
2. On `main` and release branches, `macro/gitlab-ci.yml` runs the
60+
macrobenchmarks suite; results are evaluated against
61+
`pre-release-gate.slos.yml` before a release is cut.
62+
3. `test-apps/test-apps.yml` provides manual jobs for deeper investigation
63+
when micro/macro signals are not sufficient.
64+
4. The `go-go-prof-app-parallel` and `go-go-prof-app-parallel-slo` stages run the next-generation parallel `go-prof-app`
65+
macrobenchmarks alongside the legacy `macro/` suite.
66+
67+
Each suite pins its own CI image and runs on dedicated
68+
bare-metal GitLab runners to keep results stable:
69+
70+
- **Microbenchmarks** use `MICROBENCHMARKS_CI_IMAGE` and **test-apps** use `TESTAPPS_BENCHMARKS_CI_IMAGE`:
71+
`registry.ddbuild.io/ci/benchmarking-platform:dd-trace-go-<job-id>`,
72+
built from `micro/container/` (a `golang` base plus `bp-runner` tooling
73+
installed via `bp-install`).
74+
- **Macrobenchmarks** use `MACROBENCHMARKS_CI_IMAGE`:
75+
`486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/benchmarking-platform:go-go-prof-app-and-serviceextensions-and-haproxy-<rev>`,
76+
which bundles the `go-prof-app` SUT, the Envoy service-extensions and
77+
HAProxy SPOA harnesses.
78+
79+
For details on running, extending, or debugging a specific suite, see the
80+
README inside each subdirectory.

.gitlab/macrobenchmarks.yml renamed to .gitlab/benchmarks/macro/gitlab-ci.yml

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
variables:
2-
BENCHMARKS_CI_IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/benchmarking-platform:go-go-prof-app-and-serviceextensions-and-haproxy-0003
2+
MACROBENCHMARKS_CI_IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/benchmarking-platform:go-go-prof-app-and-serviceextensions-and-haproxy-0003
33

44
.on-graphite-never: &on-graphite-never
55
if: $CI_COMMIT_BRANCH =~ /^graphite-base\/.*$/
@@ -40,7 +40,7 @@ variables:
4040
timeout: 1h
4141
rules: *macrobenchmarks-rules
4242
# If you have a problem with Gitlab cache, see Troubleshooting section in Benchmarking Platform docs
43-
image: $BENCHMARKS_CI_IMAGE
43+
image: $MACROBENCHMARKS_CI_IMAGE
4444
script:
4545
- git clone --branch go/go-prof-app https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform platform && cd platform
4646
- "./generate-run-plan-and-run-benchmarks.sh"
@@ -256,7 +256,7 @@ check-slo-breaches:
256256
variables:
257257
DDOCTOSTS_POLICY: "gitlab.github-access.slo-change-tracking.contents-read"
258258
ARTIFACTS_DIR: "platform/artifacts"
259-
SLO_FILE: ".gitlab/bp-runner.fail-on-breach.yml"
259+
SLO_FILE: ".gitlab/benchmarks/pre-release-gate.slos.yml"
260260
artifacts:
261261
name: "artifacts"
262262
when: always
@@ -289,7 +289,7 @@ notify-slo-breaches:
289289
timeout: 1h
290290
rules: *macrobenchmarks-rules
291291
# If you have a problem with Gitlab cache, see Troubleshooting section in Benchmarking Platform docs
292-
image: $BENCHMARKS_CI_IMAGE
292+
image: $MACROBENCHMARKS_CI_IMAGE
293293
script:
294294
- git clone --branch go/go-prof-app https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform platform && cd platform
295295
- bp-runner bp-runner.envoy_serviceextension.yml --debug
@@ -385,7 +385,7 @@ se-no-tracer-no-ext_proc:
385385
timeout: 1h
386386
rules: *macrobenchmarks-rules
387387
# If you have a problem with Gitlab cache, see Troubleshooting section in Benchmarking Platform docs
388-
image: $BENCHMARKS_CI_IMAGE
388+
image: $MACROBENCHMARKS_CI_IMAGE
389389
script:
390390
- git clone --branch go/go-prof-app https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform platform && cd platform
391391
- bp-runner bp-runner.haproxy_spoa.yml --debug

‎.gitlab/benchmarks/micro/.env.example‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
BENCHMARKS="BenchmarkStartRequestSpan|BenchmarkHttpServeTrace|BenchmarkTracerAddSpans"
2+
CPUS_PER_BENCHMARK=1
3+
REPETITIONS=1

‎.gitlab/benchmarks/micro/.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
artifacts
2+
.env

0 commit comments

Comments
 (0)