@@ -68,6 +68,8 @@ github:
           - "cargo check datafusion-functions features"
           - "cargo check datafusion-spark features"
           - "cargo test (amd64)"
+          - "cargo test 'extended_tests' (amd64)"
+          - "cargo test hash collisions (amd64)"
           - "cargo test datafusion-cli (amd64)"
           - "cargo examples (amd64)"
           - "cargo test doc (amd64)"
@@ -76,6 +78,7 @@ github:
           - "verify benchmark results (amd64)"
           - "Run sqllogictest with Postgres runner"
           - "Run sqllogictest in Substrait round-trip mode"
+          - "Run sqllogictests with the sqlite test suite"
           - "cargo test (macos-aarch64)"
           - "Verify Vendored Code"
           - "Check cargo fmt"
 
@@ -15,37 +15,32 @@
 # specific language governing permissions and limitations
 # under the License.
 
+# Each test command below is implemented with `cargo xtask` so developers can
+# run the same suite locally. Append `--explain` to print the underlying
+# `cargo test` invocation without running it. Job setup, the clean working tree
+# check, and `cargo clean` remain in this workflow.
+#
+# See `xtask/README.md` for details about DataFusion's `xtask` tooling.
+# Track progress at https://github.com/apache/datafusion/issues/24487.
+
 name: Datafusion extended tests
 
 concurrency:
   group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
   cancel-in-progress: true
 
-# https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#running-your-pull_request-workflow-when-a-pull-request-merges
-#
-# These jobs are not run as part of PR checks as they are time-consuming
-# and should not fail often.
-#
-# We still run them as they provide important coverage to ensure correctness
-# in the (very rare) event of a hash failure or sqlite library query failure.
+# Run these required checks in the merge queue before merging into main.
+# On PR updates, skip the jobs so their required checks allow queue entry
+# without running the expensive suites. Do not add pull_request path filters:
+# a skipped workflow would leave required checks pending.
 on:
+  pull_request:
+  merge_group:
   push:
     branches:
-      - main
-      # support extended test suite for release candidate branches,
-      # it is not expected to have many changes in these branches,
-      # so running extended tests is not a burden
+      # Release branches retain post-push coverage. main is covered by the
+      # merge queue, so do not repeat the tests after merging.
       - 'branch-*'
-  # Also run for changes to some critical areas that are most likely
-  # to trigger errors in extended tests
-  pull_request:
-    branches: [ '**' ]
-    paths:
-      - 'datafusion/physical*/**/*.rs'
-      - 'datafusion/expr*/**/*.rs'
-      - 'datafusion/optimizer/**/*.rs'
-      - 'datafusion/sql/**/*.rs'
-      - 'datafusion-testing'
   workflow_dispatch:
     inputs:
       pr_head_sha:
@@ -60,13 +55,15 @@ jobs:
   # Run extended tests (with feature 'extended_tests')
   linux-test-extended:
     name: cargo test 'extended_tests' (amd64)
+    # Long-running job: run in the merge queue, on release pushes, or manually.
+    if: github.event_name != 'pull_request'
     runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=32,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
     # note: do not use amd/rust container to preserve disk space
     steps:
       - uses: runs-on/action@efac073ea2507ec18797de3a81704201ade11d9d  # v2.3.1
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
-          ref: ${{ github.event.inputs.pr_head_sha }} # will be empty if triggered by push
+          ref: ${{ github.event.inputs.pr_head_sha }} # otherwise use the event's commit, including the merge group
           submodules: true
           fetch-depth: 1
       - name: Free Disk Space (Ubuntu)
@@ -86,22 +83,7 @@ jobs:
         run: |
           df -h
       - name: Run tests (excluding doctests)
-        env:
-          RUST_BACKTRACE: 1
-          # Run more random scenarios of the spill pool fuzzer than the
-          # default suite does (about a minute).
-          DATAFUSION_SPILL_POOL_FUZZ_ITERATIONS: 1000
-        run: |
-          cargo test \
-            --profile ci \
-            --exclude datafusion-examples \
-            --exclude datafusion-benchmarks \
-            --exclude datafusion-cli \
-            --workspace \
-            --lib \
-            --tests \
-            --bins \
-            --features avro,json,backtrace,extended_tests,recursive_protection,parquet_encryption
+        run: cargo xtask ci step test extended
       - parallel:
           - name: Verify Working Directory Clean
             run: git diff --exit-code
@@ -111,14 +93,16 @@ jobs:
   # Check answers are correct when hash values collide
   hash-collisions:
     name: cargo test hash collisions (amd64)
+    # Long-running job: run in the merge queue, on release pushes, or manually.
+    if: github.event_name != 'pull_request'
     runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
     container:
       image: amd64/rust
     steps:
       - uses: runs-on/action@efac073ea2507ec18797de3a81704201ade11d9d  # v2.3.1
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
-          ref: ${{ github.event.inputs.pr_head_sha }} # will be empty if triggered by push
+          ref: ${{ github.event.inputs.pr_head_sha }} # otherwise use the event's commit, including the merge group
           submodules: true
           fetch-depth: 1
       - name: Setup Rust toolchain
@@ -127,12 +111,14 @@ jobs:
           rust-version: stable
       - name: Run tests
         run: |
+          cargo xtask ci step test hash-collisions
           cd datafusion
-          cargo test  --profile ci --exclude datafusion-examples --exclude datafusion-benchmarks --exclude datafusion-sqllogictest --exclude datafusion-cli --workspace --lib --tests --features=force_hash_collisions,avro
           cargo clean
 
   sqllogictest-sqlite:
     name: "Run sqllogictests with the sqlite test suite"
+    # Long-running job: run in the merge queue, on release pushes, or manually.
+    if: github.event_name != 'pull_request'
     runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=32,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
     container:
       image: amd64/rust
@@ -141,13 +127,12 @@ jobs:
       - parallel:
           - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
             with:
-              ref: ${{ github.event.inputs.pr_head_sha }} # will be empty if triggered by push
+              ref: ${{ github.event.inputs.pr_head_sha }} # otherwise use the event's commit, including the merge group
               submodules: true
               fetch-depth: 1
           # Don't use setup-builder to avoid configuring RUST_BACKTRACE which is expensive
           - name: Install protobuf compiler
             run: |
               apt-get update && apt-get install -y protobuf-compiler
       - name: Run sqllogictest
-        run: |
-          cargo test --features backtrace,parquet_encryption --profile ci-optimized --test sqllogictests -- --include-sqlite
+        run: cargo xtask ci step test sqlite
@@ -73,7 +73,7 @@ jobs:
       - name: Rust Dependency Cache
         uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6  # v2.9.2
         with:
-          shared-key: "amd-ci-check" # this job uses it's own cache becase check has a separate cache and we need it to be fast as it blocks other jobs
+          shared-key: "amd-ci-check" # Keep cargo check artifacts separate from test builds.
           save-if: ${{ github.ref_name == 'main' }}
       - name: Prepare cargo build
         # `--locked` asserts that Cargo.lock is up to date with the manifest.
@@ -95,6 +95,11 @@ jobs:
         uses: ./.github/actions/setup-builder
         with:
           rust-version: stable
+      - name: Rust Dependency Cache
+        uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6  # v2.9.2
+        with:
+          save-if: false # The workspace and functions checks save this cache on main.
+          shared-key: "amd-ci-check"
       - name: Check datafusion-common (default features)
         run: cargo xtask ci step check datafusion-common default
       #
@@ -158,6 +163,11 @@ jobs:
         uses: ./.github/actions/setup-builder
         with:
           rust-version: stable
+      - name: Rust Dependency Cache
+        uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6  # v2.9.2
+        with:
+          save-if: false # The workspace and functions checks save this cache on main.
+          shared-key: "amd-ci-check"
       - name: Check datafusion-proto (default features)
         run: cargo xtask ci step check datafusion-proto default
       #
@@ -189,6 +199,11 @@ jobs:
         uses: ./.github/actions/setup-builder
         with:
           rust-version: stable
+      - name: Rust Dependency Cache
+        uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6  # v2.9.2
+        with:
+          save-if: false # The workspace and functions checks save this cache on main.
+          shared-key: "amd-ci-check"
       - name: Check datafusion-ffi (no-default-features)
         run: cargo xtask ci step check datafusion-ffi no-default
 
@@ -275,6 +290,12 @@ jobs:
         uses: ./.github/actions/setup-builder
         with:
           rust-version: stable
+      - name: Rust Dependency Cache
+        uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6  # v2.9.2
+        with:
+          # Warm GitHub's cache when linux-build-lib saves to Runs-On.
+          save-if: ${{ github.ref_name == 'main' }}
+          shared-key: "amd-ci-check"
       - name: Check datafusion-functions (default features)
         run: cargo xtask ci step check datafusion-functions default
       #
@@ -314,6 +335,11 @@ jobs:
         uses: ./.github/actions/setup-builder
         with:
           rust-version: stable
+      - name: Rust Dependency Cache
+        uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6  # v2.9.2
+        with:
+          save-if: false # The workspace and functions checks save this cache on main.
+          shared-key: "amd-ci-check"
       - name: Check datafusion-spark (default features)
         run: cargo check --profile ci --all-targets -p datafusion-spark
       #
@@ -401,12 +427,13 @@ jobs:
       # with transient Docker transport errors. The tests retry the pull
       # themselves, so a failure here is only a warning.
       #
-      # MINIO_IMAGE must match the image used by the `minio` module of the
-      # `testcontainers-modules` crate. The `minio_image_matches_ci_prepull`
-      # test in `datafusion-cli/tests/cli_integration.rs` fails if it drifts.
+      # MINIO_IMAGE must match what the tests start: the tag comes from the
+      # `minio` module of `testcontainers-modules`, the registry from
+      # MINIO_IMAGE_NAME in `datafusion-cli/tests/cli_integration.rs`. The
+      # `minio_image_matches_ci_prepull` test fails if either drifts.
       - name: Pre-pull MinIO image
         env:
-          MINIO_IMAGE: minio/minio:RELEASE.2025-02-28T09-55-16Z
+          MINIO_IMAGE: quay.io/minio/minio:RELEASE.2025-02-28T09-55-16Z
         run: |
           ci/scripts/retry timeout 120 docker pull "$MINIO_IMAGE" \
             || echo "::warning::Could not pre-pull $MINIO_IMAGE, the tests will pull it themselves"