Skip to content

Commit 80c8ec3

Browse files
authored
chore: Improve network retry configuration for maven and artifact upload (#5782)
1 parent df632c3 commit 80c8ec3

13 files changed

Lines changed: 465 additions & 19 deletions

File tree

‎.github/actions/java-test/action.yaml‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,10 @@ runs:
136136
MAVEN_OPTS="-Xmx4G -Xms2G -DwildcardSuites=$MAVEN_SUITES -XX:+UnlockDiagnosticVMOptions -XX:+ShowMessageBoxOnError -XX:+HeapDumpOnOutOfMemoryError -XX:ErrorFile=./hs_err_pid%p.log" SPARK_HOME=`pwd` ./mvnw -B -Prelease install ${{ inputs.maven_opts }}
137137
- name: Upload crash logs
138138
if: failure()
139-
uses: actions/upload-artifact@v6
139+
# These three stay on the plain action rather than
140+
# ../upload-artifact-retry: a local action calling another local action
141+
# is untested in this repo, and these only run on already-failing jobs.
142+
uses: actions/upload-artifact@v7
140143
with:
141144
name: crash-logs-${{ inputs.artifact_name }}
142145
path: "**/hs_err_pid*.log"
@@ -155,14 +158,14 @@ runs:
155158
find . -name 'unit-tests.log'
156159
- name: Upload unit-tests.log
157160
if: failure()
158-
uses: actions/upload-artifact@v6
161+
uses: actions/upload-artifact@v7
159162
with:
160163
name: unit-tests-${{ inputs.artifact_name }}
161164
path: "**/target/unit-tests.log"
162165
if-no-files-found: ignore
163166
- name: Upload test results
164167
if: ${{ !cancelled() && inputs.upload-test-reports == 'true' }}
165-
uses: actions/upload-artifact@v6
168+
uses: actions/upload-artifact@v7
166169
with:
167170
name: java-test-reports-${{ inputs.artifact_name }}
168171
path: "**/target/surefire-reports/*.txt"
Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
name: "Upload Artifact (with retry)"
19+
description: >
20+
Drop-in replacement for actions/upload-artifact that retries the upload three
21+
times. The artifact client only retries 429/500/502/503/504, so a
22+
FinalizeArtifact answered "(403) Forbidden: Error from intermediary" fails the
23+
step even though the content uploaded fine. There is no input to widen that
24+
list, Actions has no built-in step retry, and third-party retry wrappers are
25+
not on the Apache allowed-actions list. See .github/workflows/README.md.
26+
27+
Retries force overwrite, which deletes the newest artifact carrying the name
28+
before re-uploading. `name` must therefore identify exactly one producer in
29+
the run: a reusable workflow ci.yml calls more than once has to qualify the
30+
name with its version inputs. dev/ci/check-ci-config.py enforces that.
31+
32+
# Inputs mirror actions/upload-artifact@v7 one for one. The boolean defaults
33+
# have to be real 'true'/'false' strings because core.getBooleanInput() throws
34+
# on an empty value; the numeric ones default to '' so "unset" round-trips.
35+
inputs:
36+
name:
37+
description: 'Artifact name'
38+
required: false
39+
default: 'artifact'
40+
path:
41+
description: 'A file, directory or wildcard pattern that describes what to upload'
42+
required: true
43+
if-no-files-found:
44+
description: "Behavior if no files are found: warn, error or ignore"
45+
required: false
46+
default: 'warn'
47+
retention-days:
48+
description: 'Days before the artifact expires (empty means repository default)'
49+
required: false
50+
default: ''
51+
compression-level:
52+
description: 'Zlib compression level 0-9 (empty means the action default)'
53+
required: false
54+
default: ''
55+
overwrite:
56+
description: 'Delete an existing artifact with the same name before uploading'
57+
required: false
58+
default: 'false'
59+
include-hidden-files:
60+
description: 'Include hidden files in the artifact'
61+
required: false
62+
default: 'false'
63+
archive:
64+
description: 'Zip the content before uploading'
65+
required: false
66+
default: 'true'
67+
68+
# `||` yields the first non-empty operand, so this picks whichever attempt ran.
69+
outputs:
70+
artifact-id:
71+
description: 'ID of the uploaded artifact'
72+
value: ${{ steps.attempt-1.outputs.artifact-id || steps.attempt-2.outputs.artifact-id || steps.attempt-3.outputs.artifact-id }}
73+
artifact-url:
74+
description: 'Download URL of the uploaded artifact'
75+
value: ${{ steps.attempt-1.outputs.artifact-url || steps.attempt-2.outputs.artifact-url || steps.attempt-3.outputs.artifact-url }}
76+
artifact-digest:
77+
description: 'SHA-256 digest of the uploaded artifact'
78+
value: ${{ steps.attempt-1.outputs.artifact-digest || steps.attempt-2.outputs.artifact-digest || steps.attempt-3.outputs.artifact-digest }}
79+
80+
runs:
81+
using: "composite"
82+
steps:
83+
# continue-on-error keeps a failed attempt from failing the job while still
84+
# recording outcome == 'failure', which the later attempts gate on. The last
85+
# attempt omits it so a genuinely broken upload still fails loudly.
86+
- name: Upload ${{ inputs.name }} (attempt 1 of 3)
87+
id: attempt-1
88+
uses: actions/upload-artifact@v7
89+
continue-on-error: true
90+
with:
91+
name: ${{ inputs.name }}
92+
path: ${{ inputs.path }}
93+
if-no-files-found: ${{ inputs.if-no-files-found }}
94+
retention-days: ${{ inputs.retention-days }}
95+
compression-level: ${{ inputs.compression-level }}
96+
overwrite: ${{ inputs.overwrite }}
97+
include-hidden-files: ${{ inputs.include-hidden-files }}
98+
archive: ${{ inputs.archive }}
99+
100+
- name: Wait before retrying ${{ inputs.name }}
101+
if: ${{ steps.attempt-1.outcome == 'failure' }}
102+
shell: bash
103+
run: |
104+
echo "::warning::Upload of '${{ inputs.name }}' failed; retrying in 15s (attempt 2 of 3)."
105+
sleep 15
106+
107+
# Retries force overwrite: attempt 1 may have created the server-side record
108+
# before failing, and CreateArtifact rejects a duplicate name. The delete is
109+
# best effort inside upload-artifact, so it no-ops when nothing exists, and
110+
# the per-producer naming rule keeps the record it does find our own.
111+
- name: Upload ${{ inputs.name }} (attempt 2 of 3)
112+
id: attempt-2
113+
if: ${{ steps.attempt-1.outcome == 'failure' }}
114+
uses: actions/upload-artifact@v7
115+
continue-on-error: true
116+
with:
117+
name: ${{ inputs.name }}
118+
path: ${{ inputs.path }}
119+
if-no-files-found: ${{ inputs.if-no-files-found }}
120+
retention-days: ${{ inputs.retention-days }}
121+
compression-level: ${{ inputs.compression-level }}
122+
overwrite: 'true'
123+
include-hidden-files: ${{ inputs.include-hidden-files }}
124+
archive: ${{ inputs.archive }}
125+
126+
- name: Wait before final retry of ${{ inputs.name }}
127+
if: ${{ steps.attempt-1.outcome == 'failure' && steps.attempt-2.outcome == 'failure' }}
128+
shell: bash
129+
run: |
130+
echo "::warning::Upload of '${{ inputs.name }}' failed again; retrying in 45s (attempt 3 of 3)."
131+
sleep 45
132+
133+
- name: Upload ${{ inputs.name }} (attempt 3 of 3)
134+
id: attempt-3
135+
if: ${{ steps.attempt-1.outcome == 'failure' && steps.attempt-2.outcome == 'failure' }}
136+
uses: actions/upload-artifact@v7
137+
with:
138+
name: ${{ inputs.name }}
139+
path: ${{ inputs.path }}
140+
if-no-files-found: ${{ inputs.if-no-files-found }}
141+
retention-days: ${{ inputs.retention-days }}
142+
compression-level: ${{ inputs.compression-level }}
143+
overwrite: 'true'
144+
include-hidden-files: ${{ inputs.include-hidden-files }}
145+
archive: ${{ inputs.archive }}

‎.github/workflows/README.md‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,66 @@ dict at the top of `dev/ci/compute-changes.py`. The `changes` job in
130130
`needs.changes.outputs.<name>`. When adding a new test suite or moving
131131
sources, update the relevant filter entry there.
132132

133+
A file that a job reads but that no filter lists is silent: the job skips,
134+
and the edit merges with only `preflight` having looked at it. The shared
135+
build inputs (`mvnw`, `.mvn/**`, the local composite actions) are pinned by
136+
a routing table in `dev/ci/check-ci-config.py`, which `preflight` runs.
137+
138+
## Artifact names must be unique per producer
139+
140+
Artifact names are scoped to the workflow **run**, not to the calling
141+
workflow. `ci.yml` calls `spark_sql_test_reusable.yml` once per Spark
142+
version and `iceberg_spark_test_reusable.yml` once per Iceberg version, all
143+
inside the same run, so an unqualified name like `native-lib-linux` would be
144+
claimed by several producers at once. That breaks two things:
145+
146+
- `download-artifact` resolves a name to the highest matching artifact ID.
147+
Nothing ties it to the producer the consumer declared in `needs`.
148+
- `upload-artifact` with `overwrite: true` deletes the newest record with
149+
that name before uploading, which can be a sibling's finished artifact.
150+
The retry wrapper below forces `overwrite` on attempts 2 and 3.
151+
152+
So every artifact published by a reusable workflow that `ci.yml` calls more
153+
than once carries its version inputs, e.g.
154+
`native-lib-spark-4.1.3-jdk17`. `dev/ci/check-ci-config.py` enforces this,
155+
and also that every `download-artifact` name is produced by an upload in the
156+
same workflow.
157+
158+
## Retrying flaky network operations
159+
160+
**Maven.** `.mvn/maven.config` tunes the Maven Resolver HTTP transport: six
161+
retries instead of three, `408/429/500/502/503/504` retryable instead of only
162+
`429/503`, a 30s connect timeout and a 10 minute socket read timeout. The
163+
wrapper pins `maven.multiModuleProjectDirectory` to the directory holding
164+
`.mvn`, so one file covers every `mvnw` invocation in CI (including
165+
`cd spark && ../mvnw ...`) with no per-workflow wiring.
166+
167+
The Wagon transport (`-Dmaven.resolver.transport=wagon`, `maven.wagon.http.*`)
168+
was evaluated and rejected: it is deprecated in Resolver 1.9 and removed in
169+
Maven 4, its retry knobs mirror the native transport's, and its
170+
service-unavailable retry strategy defaults to `none`, so adopting it would
171+
first have to buy back the `429/503` retry we already get. The one thing it
172+
can still do that the native transport cannot is shrink HttpClient's
173+
non-retryable exception list (`retryHandler.class=default` plus
174+
`retryHandler.nonRetryableClasses=...`), the only way to retry a connect or
175+
read timeout. We size those timeouts not to fire instead.
176+
177+
**Artifact upload.** `actions/upload-artifact` fails the job when
178+
`FinalizeArtifact` returns `(403) Forbidden: Error from intermediary`, even
179+
though the content already uploaded. Its client only retries
180+
`429/500/502/503/504`, exposes no input to widen that, and Actions has no
181+
built-in step retry. Use `./.github/actions/upload-artifact-retry` instead for
182+
any artifact a later job consumes: same inputs and outputs, three attempts,
183+
15s then 45s backoff. Attempts 2 and 3 force `overwrite: true`, so the name
184+
must belong to exactly one producer in the run (see above). The diagnostic
185+
uploads inside `./.github/actions/java-test` stay on the plain action, since a
186+
local action calling another local action is untested here and those run only
187+
on already-failing jobs.
188+
189+
**Maven wrapper bootstrap.** `./.github/actions/java-test` retries
190+
`./mvnw --version` with exponential backoff, so a failed download of the Maven
191+
distribution does not surface as a test failure.
192+
133193
## Branch protection
134194

135195
Required-check names changed when these workflows were consolidated. The

‎.github/workflows/ci.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,9 @@ jobs:
9494
- name: Check Iceberg shard inventory validation
9595
run: python3 dev/ci/test-iceberg-shards.py
9696

97+
- name: Check CI config invariants
98+
run: python3 dev/ci/check-ci-config.py
99+
97100
- name: Install actionlint
98101
run: |
99102
curl -sSfL https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash | bash

‎.github/workflows/docs.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ jobs:
2626
build-docs:
2727
name: Build docs
2828
if: ${{ startsWith(github.repository, 'apache/') }}
29-
runs-on: ubuntu-latest
29+
runs-on: ubuntu-24.04
3030
steps:
3131
- name: Checkout docs sources
3232
uses: actions/checkout@v7

‎.github/workflows/iceberg_spark_test_reusable.yml‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -110,9 +110,13 @@ jobs:
110110
key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }}
111111

112112
- name: Upload native library
113-
uses: actions/upload-artifact@v7
113+
uses: ./.github/actions/upload-artifact-retry
114114
with:
115-
name: native-lib-iceberg
115+
# Version-qualified: ci.yml calls this workflow once per Iceberg
116+
# version inside a single run, and artifact names are scoped to the
117+
# run, not to the calling workflow. See "Artifact names must be
118+
# unique per producer" in .github/workflows/README.md.
119+
name: native-lib-iceberg-${{ inputs.iceberg-full }}-spark-${{ inputs.spark-full }}-jdk${{ inputs.java }}
116120
path: native/target/ci/libcomet.so
117121
retention-days: 1
118122

@@ -137,7 +141,7 @@ jobs:
137141
- name: Download native library
138142
uses: actions/download-artifact@v8
139143
with:
140-
name: native-lib-iceberg
144+
name: native-lib-iceberg-${{ inputs.iceberg-full }}-spark-${{ inputs.spark-full }}-jdk${{ inputs.java }}
141145
path: native/target/release/
142146
- name: Build Comet
143147
run: |
@@ -203,7 +207,7 @@ jobs:
203207
- name: Download native library
204208
uses: actions/download-artifact@v8
205209
with:
206-
name: native-lib-iceberg
210+
name: native-lib-iceberg-${{ inputs.iceberg-full }}-spark-${{ inputs.spark-full }}-jdk${{ inputs.java }}
207211
path: native/target/release/
208212
- name: Build Comet
209213
run: |
@@ -238,7 +242,7 @@ jobs:
238242
- name: Download native library
239243
uses: actions/download-artifact@v8
240244
with:
241-
name: native-lib-iceberg
245+
name: native-lib-iceberg-${{ inputs.iceberg-full }}-spark-${{ inputs.spark-full }}-jdk${{ inputs.java }}
242246
path: native/target/release/
243247
- name: Build Comet
244248
run: |

‎.github/workflows/pr_build_linux.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -255,7 +255,7 @@ jobs:
255255
RUSTFLAGS: "-Ctarget-cpu=x86-64-v3 -Clink-arg=-fuse-ld=bfd"
256256

257257
- name: Upload native library
258-
uses: actions/upload-artifact@v7
258+
uses: ./.github/actions/upload-artifact-retry
259259
with:
260260
name: native-lib-linux
261261
path: native/target/ci/libcomet.so

‎.github/workflows/pr_build_macos.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ jobs:
3131
# Fast lint check - gates all other jobs (runs on Linux for cost efficiency)
3232
lint:
3333
name: Lint
34-
runs-on: ubuntu-latest
34+
runs-on: ubuntu-24.04
3535
container:
3636
image: amd64/rust
3737
steps:
@@ -79,7 +79,7 @@ jobs:
7979
RUSTFLAGS: "-Ctarget-cpu=apple-m1"
8080

8181
- name: Upload native library
82-
uses: actions/upload-artifact@v7
82+
uses: ./.github/actions/upload-artifact-retry
8383
with:
8484
name: native-lib-macos
8585
path: native/target/ci/libcomet.dylib

‎.github/workflows/pyarrow_udf_test.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ env:
6464
jobs:
6565
pyarrow-udf:
6666
name: PyArrow UDF (${{ matrix.name }}, JDK 17, Python 3.11)
67-
runs-on: ubuntu-latest
67+
runs-on: ubuntu-24.04
6868
strategy:
6969
fail-fast: false
7070
matrix:

0 commit comments

Comments
 (0)