Skip to content

Commit 1ee18da

Browse files
authored
Merge branch 'main' into optimize-projections-reuse-schema
2 parents 148979f + af752e5 commit 1ee18da

9 files changed

Lines changed: 997 additions & 43 deletions

File tree

‎.github/workflows/dev.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ jobs:
8888
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
8989
with:
9090
node-version: "24"
91-
- run: node --test ci/scripts/test_fork_ci_status.cjs
91+
- run: node --test ci/scripts/test_fork_ci_status.cjs ci/scripts/test_cancel_merge_queue_runs.cjs
9292

9393
typos:
9494
name: Spell Check with Typos
Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
name: Merge queue cleanup
19+
20+
on:
21+
# New groups trigger an early scan. Ref deletion can lag this event; the
22+
# sweep also catches groups removed without a replacement or missed earlier.
23+
merge_group:
24+
types: [checks_requested]
25+
schedule:
26+
- cron: "*/15 * * * *"
27+
workflow_dispatch:
28+
inputs:
29+
dry_run:
30+
description: "Report obsolete runs without cancelling them"
31+
type: boolean
32+
default: true
33+
34+
permissions:
35+
contents: read
36+
actions: write
37+
38+
# Merge-group scans attach checks to commits that can land on main, so each
39+
# group SHA gets its own group: a replaced pending scan would show as cancelled.
40+
# Scheduled and manual scans share one group per mode, so a dry run cannot
41+
# replace a pending cancellation scan. Do not interrupt running scans.
42+
concurrency:
43+
group: merge-queue-cleanup-${{ github.event_name == 'merge_group' && github.event.merge_group.head_sha || (vars.MERGE_QUEUE_CLEANUP_ENABLED != 'true' || (github.event_name == 'workflow_dispatch' && inputs.dry_run)) }}
44+
cancel-in-progress: false
45+
46+
jobs:
47+
cleanup:
48+
if: github.repository == 'apache/datafusion'
49+
runs-on: ubuntu-slim
50+
timeout-minutes: 10
51+
steps:
52+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
53+
with:
54+
ref: ${{ github.event.repository.default_branch }}
55+
persist-credentials: false
56+
- name: Cancel obsolete merge-group runs
57+
# During the PR that introduces this workflow, main has no script yet.
58+
# Execute main's copy of the script.
59+
if: hashFiles('ci/scripts/cancel_merge_queue_runs.cjs') != ''
60+
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
61+
env:
62+
# Observe candidates for at least a week before enabling cancellation.
63+
DRY_RUN: ${{ vars.MERGE_QUEUE_CLEANUP_ENABLED != 'true' || (github.event_name == 'workflow_dispatch' && inputs.dry_run) }}
64+
with:
65+
script: |
66+
const { run } = require('./ci/scripts/cancel_merge_queue_runs.cjs');
67+
const dryRun = process.env.DRY_RUN !== 'false';
68+
const results = await run({ github, context, core, dryRun });
69+
await core.summary
70+
.addHeading(dryRun ? 'Would cancel merge-group runs' : 'Requested merge-group cancellations')
71+
.addTable([
72+
['Run', 'SHA', 'Attempt', 'Observed at (UTC)', 'Reason'].map(data => ({ data, header: true })),
73+
...results.map(({ id, sha, attempt, observed_at, reason }) => [String(id), sha, String(attempt), observed_at, reason]),
74+
])
75+
.write();
Lines changed: 238 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,238 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
// Called on each new merge group and on a schedule. The queue's exact Git
19+
// refs, rather than PR state or failing checks, determine which runs are stale.
20+
const ACTIVE = new Set([
21+
"queued",
22+
"in_progress",
23+
"waiting",
24+
"pending",
25+
"requested",
26+
]);
27+
const CLEANUP_PATH = ".github/workflows/merge_queue_cleanup.yml";
28+
const QUEUE_REF = /^gh-readonly-queue\/(.+)\/pr-[1-9][0-9]*-[0-9a-f]{40}$/;
29+
30+
function eligible(run, repo, currentRunId) {
31+
return (
32+
run.id !== currentRunId &&
33+
run.event === "merge_group" &&
34+
run.run_attempt === 1 &&
35+
ACTIVE.has(run.status) &&
36+
run.path?.split("@")[0] !== CLEANUP_PATH &&
37+
run.repository?.full_name?.toLowerCase() === repo.toLowerCase() &&
38+
run.head_repository?.full_name?.toLowerCase() === repo.toLowerCase() &&
39+
QUEUE_REF.test(run.head_branch) &&
40+
/^[0-9a-f]{40}$/.test(run.head_sha)
41+
);
42+
}
43+
44+
async function refSha(github, repo, branch) {
45+
try {
46+
const { data } = await github.rest.git.getRef({
47+
...repo,
48+
ref: `heads/${branch}`,
49+
});
50+
// Only a missing ref or a known, different commit proves obsolescence.
51+
// Unexpected responses and permission/server errors must not cancel CI.
52+
if (
53+
data.object?.type !== "commit" ||
54+
!/^[0-9a-f]{40}$/.test(data.object.sha)
55+
) {
56+
throw new Error(`Invalid Git ref response for ${branch}`);
57+
}
58+
return data.object.sha;
59+
} catch (error) {
60+
if (error.status === 404) return null;
61+
throw error;
62+
}
63+
}
64+
65+
// Filtered Actions listings are capped at 1,000 results even with pagination.
66+
// Active merge-group runs stay far below the cap, so a full listing is an error.
67+
async function listRuns(github, repo, status) {
68+
const params = { ...repo, event: "merge_group", status, per_page: 100 };
69+
const runs = [];
70+
for (let page = 1; ; page++) {
71+
const { data } = await github.rest.actions.listWorkflowRunsForRepo({
72+
...params,
73+
page,
74+
});
75+
if (
76+
!Number.isInteger(data.total_count) ||
77+
data.total_count < 0 ||
78+
!Array.isArray(data.workflow_runs)
79+
) {
80+
throw new Error("Invalid workflow run listing");
81+
}
82+
if (data.total_count >= 1000) {
83+
throw new Error(
84+
`${data.total_count} ${status} runs exceed the listing cap`
85+
);
86+
}
87+
runs.push(...data.workflow_runs);
88+
if (data.workflow_runs.length < 100) return runs;
89+
}
90+
}
91+
92+
function reportError(core, context, message) {
93+
core.warning(message);
94+
// Maintenance errors must not mark a merge-group commit as failing CI.
95+
// Scheduled/manual scans still expose broken cleanup as workflow failures.
96+
if (context.eventName !== "merge_group") core.setFailed(message);
97+
}
98+
99+
async function run({ github, context, core, dryRun = true }) {
100+
const repo = { owner: context.repo.owner, repo: context.repo.repo };
101+
const fullName = `${repo.owner}/${repo.repo}`;
102+
if (fullName !== "apache/datafusion") return [];
103+
104+
// Finish pagination before cancelling: otherwise mutations shrink the active
105+
// result sets and can cause later pages to skip runs.
106+
const listed = new Map();
107+
try {
108+
for (const status of ACTIVE) {
109+
const runs = await listRuns(github, repo, status);
110+
for (const candidate of runs) listed.set(candidate.id, candidate);
111+
}
112+
} catch (error) {
113+
reportError(
114+
core,
115+
context,
116+
`Cannot list merge-group runs: ${error.message}`
117+
);
118+
return [];
119+
}
120+
121+
const refs = new Map();
122+
const comparisons = new Map();
123+
const results = [];
124+
const errors = [];
125+
for (const candidate of listed.values()) {
126+
if (!eligible(candidate, fullName, context.runId)) continue;
127+
try {
128+
// Share the initial lookup among workflows for the same merge group.
129+
// Cache failures too: an outage must not multiply requests for this ref.
130+
if (!refs.has(candidate.head_branch)) {
131+
refs.set(
132+
candidate.head_branch,
133+
refSha(github, repo, candidate.head_branch)
134+
);
135+
}
136+
if ((await refs.get(candidate.head_branch)) === candidate.head_sha) {
137+
core.info(
138+
`Kept active group ${JSON.stringify({
139+
id: candidate.id,
140+
sha: candidate.head_sha,
141+
branch: candidate.head_branch,
142+
observed_at: new Date().toISOString(),
143+
trigger: context.eventName,
144+
})}`
145+
);
146+
continue;
147+
}
148+
149+
const { data: current } = await github.rest.actions.getWorkflowRun({
150+
...repo,
151+
run_id: candidate.id,
152+
});
153+
// A completed run or a newly requested rerun is no longer this candidate.
154+
if (
155+
!eligible(current, fullName, context.runId) ||
156+
current.head_branch !== candidate.head_branch ||
157+
current.head_sha !== candidate.head_sha ||
158+
current.run_attempt !== candidate.run_attempt
159+
)
160+
continue;
161+
162+
const head = await refSha(github, repo, current.head_branch);
163+
if (head === current.head_sha) continue;
164+
// A successful merge removes the queue ref too. Preserve checks on
165+
// commits already in the target branch, including optional checks.
166+
// Workflows of one group share the comparison, failures included.
167+
const base = QUEUE_REF.exec(current.head_branch)[1];
168+
const basehead = `${base}...${current.head_sha}`;
169+
if (!comparisons.has(basehead)) {
170+
comparisons.set(
171+
basehead,
172+
github.rest.repos.compareCommitsWithBasehead({
173+
...repo,
174+
basehead,
175+
per_page: 1,
176+
})
177+
);
178+
}
179+
const { data: comparison } = await comparisons.get(basehead);
180+
if (["identical", "behind"].includes(comparison.status)) continue;
181+
if (!["ahead", "diverged"].includes(comparison.status)) {
182+
throw new Error(`Invalid comparison status for ${current.head_sha}`);
183+
}
184+
const reason = head === null ? "queue ref removed" : "queue ref replaced";
185+
const result = {
186+
id: current.id,
187+
sha: current.head_sha,
188+
branch: current.head_branch,
189+
attempt: current.run_attempt,
190+
reason,
191+
observed_at: new Date().toISOString(),
192+
trigger: context.eventName,
193+
};
194+
const description = JSON.stringify(result);
195+
if (dryRun) {
196+
core.info(`Would cancel ${description}`);
197+
} else {
198+
try {
199+
await github.rest.actions.cancelWorkflowRun({
200+
...repo,
201+
run_id: current.id,
202+
});
203+
} catch (error) {
204+
// A 409 can mean a completion race, but also an active run that
205+
// GitHub cannot cancel. Only suppress it after confirming the race.
206+
if (error.status === 409) {
207+
const { data: latest } = await github.rest.actions.getWorkflowRun({
208+
...repo,
209+
run_id: current.id,
210+
});
211+
if (
212+
latest.status === "completed" ||
213+
latest.run_attempt !== current.run_attempt
214+
)
215+
continue;
216+
}
217+
throw error;
218+
}
219+
core.info(`Requested cancellation of ${description}`);
220+
}
221+
results.push(result);
222+
} catch (error) {
223+
// One failed read/cancellation must not leave other obsolete groups running.
224+
const message = `Run ${candidate.id}: ${error.message}`;
225+
core.warning(message);
226+
errors.push(message);
227+
}
228+
}
229+
if (errors.length)
230+
reportError(
231+
core,
232+
context,
233+
`${errors.length} merge queue cleanup errors; see warnings.`
234+
);
235+
return results;
236+
}
237+
238+
module.exports = { run };

0 commit comments

Comments
 (0)