Auto-detect pipeline ID and commit SHA for local e2e test runs (#47575)

pducolin · web-flow · commit 88d4c44f7b49 · 2026-03-12T11:01:18.000Z
### What does this PR do?

When running `dda inv new-e2e-tests.run` locally (not in CI), automatically look up the most recent GitLab pipeline for the current HEAD commit SHA and set `E2E_PIPELINE_ID` / `E2E_COMMIT_SHA` — so tests use the correct build artifacts without needing the user to set those env vars manually.

Two changes in `tasks/new_e2e_tests.py`:

**New helper `_find_pipeline_for_commit_sha`**
- Queries the GitLab API for pipelines matching the current HEAD commit SHA
- For each candidate pipeline, fetches all jobs and verifies that the `packaging` and `deploy_packages` stages are both present and have all jobs in `success` or `skipped` state
- Returns the first pipeline that passes, or `None` with a diagnostic message to stderr

**Updated `run` task (local path only)**
- Inside the existing `if not running_in_ci():` block, if `E2E_PIPELINE_ID` is not already in the environment, calls the new helper and sets `E2E_PIPELINE_ID` + `E2E_COMMIT_SHA` in `env_vars`
- If no qualifying pipeline is found, prints a yellow warning and continues — the task still runs, preserving previous behaviour for cases where no pipeline exists (e.g. a brand-new branch)

### Motivation

Previously, developers had to manually find the right pipeline ID and set `E2E_PIPELINE_ID`/`E2E_COMMIT_SHA` before running e2e tests locally. This was error-prone and tedious. The auto-detection removes that friction while ensuring only pipelines with completed packaging artifacts are used.

### Describe how you validated your changes

- Code review: verified the GitLab `pipelines.list(sha=...)` and `pipeline.jobs.list(get_all=True)` API usage matches the python-gitlab client interface already used elsewhere in the codebase (`_find_recent_successful_pipeline`)
- The change is gated behind `not running_in_ci()` and `"E2E_PIPELINE_ID" not in os.environ`, so it is a no-op in CI and when users have already set the env var manually
- Tried on this branch, with unready pipeline I had the following message, continued with default behavior
```
Pipeline 101237294 skipped: jobs not passed: packaging/agent_suse-arm64-a7-fips (running), packaging/agent_rpm-arm64-a7-fips (running), packaging/agent_suse-arm64-a7 (running), packaging/agent_rpm-arm64-a7 (running), packaging/agent_oci (created), packaging/agent_deb-arm64-a7-fips (running), packaging/agent_deb-arm64-a7 (running), deploy_packages/deploy_ddot_oci (manual), deploy_packages/deploy_installer_oci (manual), deploy_packages/deploy_agent_oci (created), deploy_packages/qa_installer_script_linux (manual)
No pipeline with passing packaging and deploy_packages stages found for commit 1dfa6245, skipping pipeline auto-detection
```

Once the pipeline is ready, got

```
Pipeline ID: 101251448
```

### Additional Notes

- Requires a `GITLAB_TOKEN` env var with `read_api` scope for the GitLab API calls; if absent, a warning is printed and the task continues without auto-detection
- `skipped` jobs are counted as passing (rules may exclude certain platforms/architectures)

Co-authored-by: paola.ducolin &lt;paola.ducolin@datadoghq.com&gt;
diff --git a/tasks/new_e2e_tests.py b/tasks/new_e2e_tests.py
@@ -331,6 +331,33 @@ def run(
             parsed_params["ddagent:imagePullRegistry"] += ",376334461865.dkr.ecr.us-east-1.amazonaws.com"
             parsed_params["ddagent:imagePullUsername"] += ",AWS"
 
+        # Auto-detect pipeline ID and commit SHA for local runs if not already set
+        if "E2E_PIPELINE_ID" not in os.environ:
+            print(
+                color_message(
+                    "E2E_PIPELINE_ID is not set. The E2E job you are running may require build and packaging "
+                    "jobs to have completed in the pipeline (e.g. container images, deb/rpm packages, OCI deploys). "
+                    "Check the `needs:` of your target job in .gitlab/test/e2e/e2e.yml and ensure those jobs "
+                    "have run on your branch before triggering the E2E job.",
+                    "yellow",
+                )
+            )
+            commit_sha = get_commit_sha(ctx)
+            short_commit_sha = get_commit_sha(ctx, short=True)
+            print(color_message(f"Auto-detecting pipeline for commit {short_commit_sha}...", "blue"))
+            pipeline_id = _find_pipeline_for_commit_sha(ctx, commit_sha)
+            if pipeline_id:
+                print(color_message(f"Auto-detected pipeline {pipeline_id} for commit {short_commit_sha}", "blue"))
+                env_vars["E2E_PIPELINE_ID"] = pipeline_id
+                env_vars["E2E_COMMIT_SHA"] = short_commit_sha
+            else:
+                print(
+                    color_message(
+                        f"No pipeline with passing packaging and deploy_packages stages found for commit {short_commit_sha}, skipping pipeline auto-detection",
+                        "yellow",
+                    )
+                )
+
     for param in configparams:
         parts = param.split("=", 1)
         if len(parts) != 2:
@@ -1168,6 +1195,66 @@ def _find_recent_successful_pipeline(ctx: Context, branch: str | None = None) ->
         return None
 
 
+def _find_pipeline_for_commit_sha(ctx: Context, commit_sha: str) -> str | None:
+    """
+    Find the most recent pipeline for a given commit SHA where stages
+    'packaging' and 'deploy_packages' have all jobs successfully completed (success or skipped).
+    Searches by branch ref and matches on SHA, as GitLab's sha filter is unreliable.
+    Returns pipeline_id or None if not found.
+    """
+    required_stages = {"packaging", "deploy_packages"}
+
+    try:
+        token = os.environ.get('GITLAB_TOKEN')
+        repo = get_gitlab_repo(token=token)
+
+        branch = get_current_branch(ctx)
+        pipelines = repo.pipelines.list(ref=branch, per_page=20, order_by='updated_at', get_all=False)
+        for pipeline in pipelines:
+            if not pipeline.sha.startswith(commit_sha) and not commit_sha.startswith(pipeline.sha):
+                continue
+            jobs = pipeline.jobs.list(get_all=True)
+
+            stage_jobs: dict[str, list] = {}
+            for job in jobs:
+                if job.stage in required_stages:
+                    stage_jobs.setdefault(job.stage, []).append(job)
+
+            # All required stages must be present
+            if not required_stages.issubset(stage_jobs.keys()):
+                missing = required_stages - stage_jobs.keys()
+                print(
+                    f"Pipeline {pipeline.id} skipped: missing stages {missing}",
+                    file=sys.stderr,
+                )
+                continue
+
+            # All jobs in required stages must have passed (success, skipped, or manual/not triggered)
+            failed_jobs = [
+                f"{job.stage}/{job.name} ({job.status})"
+                for stage in required_stages
+                for job in stage_jobs[stage]
+                if job.status not in ("success", "skipped", "manual")
+            ]
+            if failed_jobs:
+                print(
+                    f"Pipeline {pipeline.id} skipped: jobs not passed: {', '.join(failed_jobs)}",
+                    file=sys.stderr,
+                )
+                continue
+
+            return str(pipeline.id)
+
+        return None
+    except Exception as e:
+        print(f"Warning: Could not query GitLab for pipelines for commit {commit_sha[:8]}: {e}")
+        if 'GITLAB_TOKEN' not in os.environ:
+            print(
+                "No GITLAB_TOKEN environment variable found, set it with a GitLab Personal Access Token (read_api scope)"
+            )
+        return None
+
+
 def _find_local_msi_build(pkg: str | None = None) -> str | None:
     """
     Find a local MSI build in the omnibus/pkg directory.
