Skip to content

Commit dd6caba

Browse files
Merge branch 'main' into fix/apache#69262
2 parents 4ce58a9 + e1155ed commit dd6caba

106 files changed

Lines changed: 3715 additions & 1644 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.
Lines changed: 177 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,177 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
#
18+
# Runs the Java SDK reviewer checklist ("Verifying a release" in
19+
# java-sdk/README.md) automatically, minus the GPG signature (only the release
20+
# manager's key can satisfy that).
21+
#
22+
# * on a pull request touching java-sdk/ -> verify HEAD can produce a source
23+
# release that is self-contained and builds from scratch.
24+
# * on a `java-sdk/**` tag push -> the same check against the tagged commit
25+
# (release readiness gate).
26+
# * on manual dispatch -> verify the ACTUAL staged artifacts: the source
27+
# package in dist/dev and the closed Nexus staging repo. Run this after
28+
# staging, before sending the [VOTE] email.
29+
#
30+
# The environment setup intentionally mirror what a human verifying the release
31+
# would do. An (unpinned) latest Gradle is downloaded to create the Gradle
32+
# wrapper from, and a more modern JDK is used to run things.
33+
34+
---
35+
name: Java SDK release verification
36+
37+
on: # yamllint disable-line rule:truthy
38+
pull_request:
39+
paths:
40+
- "java-sdk/**"
41+
- ".github/workflows/java-sdk-release-verify.yml"
42+
push:
43+
tags:
44+
- "java-sdk/**"
45+
workflow_dispatch:
46+
inputs:
47+
tag:
48+
description: "Java SDK release tag (starts with 'java-sdk/') to verify"
49+
required: true
50+
type: string
51+
nexus_repo_id:
52+
description: "Nexus staging repo id (the NNNN in orgapacheairflow-NNNN)"
53+
required: true
54+
type: string
55+
56+
permissions:
57+
contents: read
58+
59+
concurrency:
60+
group: java-sdk-release-verify-${{ github.event_name }}-${{ github.ref }}-${{ inputs.tag }}
61+
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
62+
63+
env:
64+
JAVA_VERSION: "21"
65+
66+
jobs:
67+
source-release:
68+
name: Source release builds cleanly
69+
if: github.event_name != 'workflow_dispatch'
70+
runs-on: ubuntu-latest
71+
steps:
72+
- name: Checkout
73+
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
74+
with:
75+
ref: ${{ github.event_name == 'push' && github.ref || github.sha }}
76+
fetch-depth: 0
77+
persist-credentials: false
78+
- name: Set up JDK ${{ env.JAVA_VERSION }}
79+
uses: actions/setup-java@1bcf9fb12cf4aa7d266a90ae39939e61372fe520 # v5.4.0
80+
with:
81+
distribution: temurin
82+
java-version: ${{ env.JAVA_VERSION }}
83+
- name: Use an isolated Gradle home
84+
run: echo "GRADLE_USER_HOME=${RUNNER_TEMP}/gradle-home" >> "$GITHUB_ENV"
85+
- name: Resolve the ref to build the source release from
86+
run: |
87+
set -euo pipefail
88+
if [ "${{ github.event_name }}" = "push" ]; then
89+
echo "RELEASE_REF=${GITHUB_REF_NAME}" >> "$GITHUB_ENV"
90+
else # A PR HEAD.
91+
echo "RELEASE_REF=$(git rev-parse HEAD)" >> "$GITHUB_ENV"
92+
fi
93+
- name: Build the source tarball from the ref (project's wrapper)
94+
working-directory: java-sdk
95+
run: ./gradlew --no-daemon sourceTarball checksumSourceTarball -PgitRef="${RELEASE_REF}"
96+
- name: Install the latest Gradle to bootstrap a new wrapper for the source extracted from the tarball
97+
run: |
98+
set -euo pipefail
99+
meta=$(curl -fsSL https://services.gradle.org/versions/current)
100+
ver=$(echo "$meta" | jq -r .version)
101+
url=$(echo "$meta" | jq -r .downloadUrl)
102+
sum=$(curl -fsSL "$(echo "$meta" | jq -r .checksumUrl)")
103+
curl -fsSL -o /tmp/gradle.zip "$url"
104+
echo "${sum} /tmp/gradle.zip" | sha256sum -c -
105+
unzip -q -d "$HOME/gradle" /tmp/gradle.zip
106+
echo "$HOME/gradle/gradle-${ver}/bin" >> "$GITHUB_PATH"
107+
- name: Verify the source tarball
108+
run: |
109+
set -euo pipefail
110+
tarball=$(ls java-sdk/build/distributions/apache-airflow-java-sdk-*-src.tar.gz)
111+
java-sdk/scripts/ci/verify-source-release.sh \
112+
--tarball "${tarball}" \
113+
--sha512 "${tarball}.sha512" \
114+
--tag "${RELEASE_REF}"
115+
116+
artifact:
117+
name: Staged artifacts pass the reviewer checklist
118+
if: github.event_name == 'workflow_dispatch'
119+
runs-on: ubuntu-latest
120+
steps:
121+
- name: Checkout tag
122+
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
123+
with:
124+
ref: ${{ inputs.tag }}
125+
fetch-depth: 0
126+
persist-credentials: false
127+
- name: Set up JDK ${{ env.JAVA_VERSION }}
128+
uses: actions/setup-java@1bcf9fb12cf4aa7d266a90ae39939e61372fe520 # v5.4.0
129+
with:
130+
distribution: temurin
131+
java-version: ${{ env.JAVA_VERSION }}
132+
- name: Use an isolated Gradle home
133+
run: echo "GRADLE_USER_HOME=${RUNNER_TEMP}/gradle-home" >> "$GITHUB_ENV"
134+
- name: Download the staged source package from dist/dev
135+
env:
136+
TAG: ${{ inputs.tag }}
137+
run: |
138+
set -euo pipefail
139+
rcdir="${TAG#java-sdk/}" # <version>-rc<N>
140+
version="${rcdir%-rc*}" # <version>
141+
base="https://dist.apache.org/repos/dist/dev/airflow/java-sdk/${rcdir}"
142+
file="apache-airflow-java-sdk-${version}-src.tar.gz"
143+
mkdir -p staged
144+
curl -fsSL -o "staged/${file}" "${base}/${file}"
145+
curl -fsSL -o "staged/${file}.sha512" "${base}/${file}.sha512"
146+
- name: Install the latest Gradle to bootstrap a new wrapper for the source extracted from the tarball
147+
run: |
148+
set -euo pipefail
149+
meta=$(curl -fsSL https://services.gradle.org/versions/current)
150+
ver=$(echo "$meta" | jq -r .version)
151+
url=$(echo "$meta" | jq -r .downloadUrl)
152+
sum=$(curl -fsSL "$(echo "$meta" | jq -r .checksumUrl)")
153+
curl -fsSL -o /tmp/gradle.zip "$url"
154+
echo "${sum} /tmp/gradle.zip" | sha256sum -c -
155+
unzip -q -d "$HOME/gradle" /tmp/gradle.zip
156+
echo "$HOME/gradle/gradle-${ver}/bin" >> "$GITHUB_PATH"
157+
- name: Verify the staged source package
158+
env:
159+
TAG: ${{ inputs.tag }}
160+
run: |
161+
set -euo pipefail
162+
rcdir="${TAG#java-sdk/}"; version="${rcdir%-rc*}"
163+
file="apache-airflow-java-sdk-${version}-src.tar.gz"
164+
java-sdk/scripts/ci/verify-source-release.sh \
165+
--tarball "staged/${file}" \
166+
--sha512 "staged/${file}.sha512" \
167+
--tag "${TAG}"
168+
- name: Smoke-test the staged convenience binaries
169+
env:
170+
TAG: ${{ inputs.tag }}
171+
NEXUS_REPO_ID: ${{ inputs.nexus_repo_id }}
172+
run: |
173+
set -euo pipefail
174+
rcdir="${TAG#java-sdk/}"; version="${rcdir%-rc*}"
175+
java-sdk/scripts/ci/smoke-test-staged-binaries.sh \
176+
--nexus-repo-id "${NEXUS_REPO_ID}" \
177+
--version "${version}"

‎Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ ARG PYTHON_LTO="true"
7373
# Also use `force pip` label on your PR to swap all places we use `uv` to `pip`
7474
ARG AIRFLOW_PIP_VERSION=26.1.2
7575
# ARG AIRFLOW_PIP_VERSION="git+https://github.com/pypa/pip.git@main"
76-
ARG AIRFLOW_UV_VERSION=0.11.25
76+
ARG AIRFLOW_UV_VERSION=0.11.26
7777
ARG AIRFLOW_USE_UV="false"
7878
ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow"
7979
ARG AIRFLOW_IMAGE_README_URL="https://raw.githubusercontent.com/apache/airflow/main/docs/docker-stack/README.md"

‎Dockerfile.ci‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1720,7 +1720,7 @@ ENV DEV_APT_COMMAND=${DEV_APT_COMMAND} \
17201720

17211721
ARG AIRFLOW_PYTHON_VERSION="3.13.14"
17221722
ENV AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION}
1723-
ENV GOLANG_MAJOR_MINOR_VERSION="1.26.4"
1723+
ENV GOLANG_MAJOR_MINOR_VERSION="1.26.5"
17241724
ENV TEMURIN_VERSION="11"
17251725
ENV RUSTUP_HOME="/usr/local/rustup"
17261726
ENV CARGO_HOME="/usr/local/cargo"
@@ -1879,8 +1879,8 @@ COPY --from=scripts common.sh install_packaging_tools.sh install_additional_depe
18791879
# Also use `force pip` label on your PR to swap all places we use `uv` to `pip`
18801880
ARG AIRFLOW_PIP_VERSION=26.1.2
18811881
# ARG AIRFLOW_PIP_VERSION="git+https://github.com/pypa/pip.git@main"
1882-
ARG AIRFLOW_UV_VERSION=0.11.25
1883-
ARG AIRFLOW_PREK_VERSION="0.4.5"
1882+
ARG AIRFLOW_UV_VERSION=0.11.26
1883+
ARG AIRFLOW_PREK_VERSION="0.4.6"
18841884

18851885
# UV_LINK_MODE=copy is needed since we are using cache mounted from the host
18861886
ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \

‎airflow-core/docs/core-concepts/resumable-tasks.rst‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -143,8 +143,9 @@ operations (HTTP requests, database queries, file reads) within a single task ex
143143
blocking the event loop while waiting for each one.
144144

145145
The worker slot is held for the full duration of the task. Async tasks are not designed for
146-
long external waits or crash recovery; they are designed for high-throughput I/O work that
147-
completes within a single execution.
146+
long external waits; they are designed for high-throughput I/O work that completes within a
147+
single execution. To survive a worker crash, an async task can checkpoint its progress to
148+
:doc:`task state store <task-state-store>` with ``aget``/``aset`` and resume from the checkpoint on retry.
148149

149150
For guidance on when to use async tasks versus deferrable operators, see
150151
:doc:`task-sdk:deferred-vs-async-operators`.

‎airflow-core/docs/core-concepts/task-state-store.rst‎

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@
2222
intra
2323
Intra
2424
checkpointing
25+
Checkpointing
26+
accessors
2527

2628
Task State Store
2729
================
@@ -30,7 +32,7 @@ Task State Store
3032

3133
Task store is a persistent key/value store scoped to a single task instance (``dag_id`` + ``run_id`` + ``task_id`` + ``map_index``). It survives worker crashes and task retries within the same Dag run, making it suitable for storing external job IDs, intra-task checkpoints, and progress metadata.
3234

33-
Data persisted via task state store is accessed through the task context via ``context["task_state_store"]`` and exposes four methods: ``get``, ``set``, ``delete``, and ``clear``.
35+
Data persisted via task state store is accessed through the task context via ``context["task_state_store"]`` and exposes the synchronous methods ``get``, ``set``, ``delete``, and ``clear``, plus the async counterparts ``aget``, ``aset``, ``adelete``, and ``aclear`` for use inside ``async`` tasks.
3436

3537

3638
Accessing task state store
@@ -133,6 +135,19 @@ Deletes *all* task state store keys for this task instance.
133135
134136
task_state_store.clear()
135137
138+
``aget``, ``aset``, ``adelete``, ``aclear``
139+
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
140+
141+
Async counterparts of ``get``, ``set``, ``delete``, and ``clear`` for use inside ``async`` tasks. They take the same arguments and behave identically to their synchronous siblings, but ``await`` the round-trip to the API server instead of blocking the event loop, so a coroutine can checkpoint its progress without stalling other concurrent work.
142+
143+
.. code-block:: python
144+
145+
value = await task_state_store.aget("job_id", default="123456789")
146+
await task_state_store.aset("job_id", job_id, retention=NEVER_EXPIRE)
147+
await task_state_store.adelete("job_id")
148+
await task_state_store.aclear()
149+
150+
Calling the synchronous ``get``/``set``/``delete``/``clear`` from inside an ``async`` task blocks the event loop and defeats the concurrency the coroutine was written for. Use the ``a``-prefixed methods there instead.
136151

137152
Some Example Use Cases
138153
----------------------
@@ -197,6 +212,30 @@ For tasks that process paginated or batched data, store the last-completed offse
197212
198213
On a retry, the task reads ``last_page`` and skips pages that were already processed.
199214

215+
Checkpointing from an async task
216+
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
217+
218+
An ``async`` task gains concurrency by awaiting I/O instead of blocking on it. To keep that benefit while checkpointing, use the async accessors ``aget``/``aset`` so the task store round-trip does not stall the event loop. This is the same paginated-ingest pattern as above, written for a coroutine that paginates through an external API.
219+
220+
.. code-block:: python
221+
222+
from airflow.sdk import DAG, task
223+
224+
with DAG("async_paginated_ingest", schedule="@daily"):
225+
226+
@task
227+
async def ingest_pages(**context):
228+
task_state_store = context["task_state_store"]
229+
last_page = await task_state_store.aget("last_page")
230+
231+
start_page = last_page + 1 if last_page is not None else 1
232+
233+
for page in range(start_page, total_pages + 1):
234+
await fetch_and_load(page)
235+
await task_state_store.aset("last_page", page)
236+
237+
After a crash the task resumes from the stored ``last_page`` instead of restarting at page 1, without ever blocking the event loop.
238+
200239
Progress metadata
201240
~~~~~~~~~~~~~~~~~
202241

‎airflow-core/docs/faq.rst‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -420,6 +420,23 @@ Additionally, you can catch these issues earlier in your development workflow by
420420
dynamic values in Dag and Task constructors as part of static linting. See
421421
:ref:`best_practices/code_quality_and_linting` for how to set up ruff with Airflow-specific rules.
422422

423+
.. _faq:duplicate-dag-id-warning:
424+
425+
Why do I see a "duplicate dag id" warning?
426+
------------------------------------------
427+
428+
The Dag processor recognizes each Dag by its ``dag_id``, not by its file path. If two files define a
429+
Dag with the same ``dag_id``, only one of them is shown in the UI at a time — whichever file was parsed
430+
most recently — and Airflow records a **Dag warning** naming the other, overwritten file.
431+
432+
This is expected in one common case: you renamed or moved a Dag file. The old file's row is not removed
433+
from the database until it goes unobserved for
434+
:ref:`stale_dag_threshold <config:dag_processor__stale_dag_threshold>`, so the warning briefly
435+
appears for both the old and new paths and then clears on its own once the old file stops being parsed.
436+
437+
If the warning does not clear, two files with the same ``dag_id`` genuinely coexist. Which one "wins" is
438+
not guaranteed to stay stable across parses, so rename one of them to give each Dag a unique ``dag_id``.
439+
423440

424441
Dag construction
425442
^^^^^^^^^^^^^^^^

‎airflow-core/docs/migrations-ref.rst‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,9 @@ Here's the list of all the Database Migrations that are executed via when you ru
3939
+-------------------------+------------------+-------------------+--------------------------------------------------------------+
4040
| Revision ID | Revises ID | Airflow Version | Description |
4141
+=========================+==================+===================+==============================================================+
42-
| ``436dc127462c`` (head) | ``5a5d3253e946`` | ``3.4.0`` | Drop span_status column. |
42+
| ``c4e7a1f9b2d0`` (head) | ``436dc127462c`` | ``3.4.0`` | Add index on asset.uri. |
43+
+-------------------------+------------------+-------------------+--------------------------------------------------------------+
44+
| ``436dc127462c`` | ``5a5d3253e946`` | ``3.4.0`` | Drop span_status column. |
4345
+-------------------------+------------------+-------------------+--------------------------------------------------------------+
4446
| ``5a5d3253e946`` | ``d2f4e1b3c5a7`` | ``3.4.0`` | Add GIN index on asset_event.extra for PostgreSQL. |
4547
+-------------------------+------------------+-------------------+--------------------------------------------------------------+
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Warn in the UI when the same dag_id is registered from multiple files.

0 commit comments

Comments
 (0)