Skip to content

Commit 517133c

Browse files
viiryaIsaac
andcommitted
Merge remote-tracking branch 'upstream/master' into SPARK-59832-rle-truncated
Co-authored-by: Isaac <[email protected]>
2 parents 1a2868d + 1dd8171 commit 517133c

324 files changed

Lines changed: 29342 additions & 4354 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.
Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#
19+
20+
name: 'Setup Build Environment'
21+
description: >-
22+
Cache SBT/Maven launcher artifacts, optionally cache or restore Coursier and
23+
the Maven local repository, optionally free disk space, and install a Zulu
24+
JDK. Callers that use a local composite action must add a bare
25+
'actions/checkout' step first so the latest action definition is present
26+
(same bootstrap pattern as checkout-and-sync).
27+
There is a default cache limit of 10 GB.
28+
inputs:
29+
java-version:
30+
description: >-
31+
JDK version passed to actions/setup-java (Zulu distribution). Empty skips
32+
the install step (rare; used when the caller installs Java conditionally).
33+
required: false
34+
default: ''
35+
coursier:
36+
description: >-
37+
How to handle the Coursier cache: 'cache' (save+restore), 'restore'
38+
(restore only), or 'none'.
39+
required: false
40+
default: 'restore'
41+
cache-maven-local:
42+
description: 'Cache ~/.m2/repository'
43+
required: false
44+
default: 'false'
45+
free-disk-space:
46+
description: >-
47+
Free disk space before installing Java. 'host' runs
48+
dev/free_disk_space, 'container' runs dev/free_disk_space_container,
49+
'none' skips.
50+
required: false
51+
default: 'none'
52+
53+
runs:
54+
using: 'composite'
55+
steps:
56+
# Key families are shared across workflows:
57+
# SBT: build-<os>-<hash(build files)>
58+
# Coursier: coursier-<os>-<hash(**/pom.xml, **/plugins.sbt)>
59+
# Maven: maven-<os>-java<ver>-<hash(**/pom.xml)>
60+
- name: Cache SBT and Maven
61+
uses: actions/cache@v6
62+
with:
63+
path: |
64+
build/apache-maven-*
65+
build/*.jar
66+
~/.sbt
67+
key: >-
68+
build-${{ runner.os }}-${{ hashFiles(
69+
'**/pom.xml',
70+
'project/build.properties',
71+
'build/mvn',
72+
'build/sbt',
73+
'build/sbt-launch-lib.bash',
74+
'build/spark-build-info'
75+
) }}
76+
restore-keys: |
77+
build-${{ runner.os }}-
78+
79+
- name: Cache Coursier local repository
80+
if: inputs.coursier == 'cache'
81+
uses: actions/cache@v6
82+
with:
83+
path: ~/.cache/coursier
84+
key: >-
85+
coursier-${{ runner.os }}-${{ hashFiles(
86+
'**/pom.xml',
87+
'**/plugins.sbt'
88+
) }}
89+
restore-keys: |
90+
coursier-${{ runner.os }}-
91+
92+
- name: Restore Coursier local repository
93+
if: inputs.coursier == 'restore'
94+
uses: actions/cache/restore@v6
95+
with:
96+
path: ~/.cache/coursier
97+
key: >-
98+
coursier-${{ runner.os }}-${{ hashFiles(
99+
'**/pom.xml',
100+
'**/plugins.sbt'
101+
) }}
102+
restore-keys: |
103+
coursier-${{ runner.os }}-
104+
105+
- name: Cache Maven local repository
106+
if: inputs.cache-maven-local == 'true'
107+
uses: actions/cache@v6
108+
with:
109+
path: ~/.m2/repository
110+
key: >-
111+
maven-${{ runner.os }}-java${{ inputs.java-version }}-${{ hashFiles(
112+
'**/pom.xml'
113+
) }}
114+
restore-keys: |
115+
maven-${{ runner.os }}-java${{ inputs.java-version }}-
116+
117+
- name: Free up disk space
118+
if: inputs.free-disk-space == 'host'
119+
continue-on-error: true
120+
shell: bash
121+
run: timeout --kill-after=30s 10m ./dev/free_disk_space
122+
123+
- name: Free up disk space
124+
if: inputs.free-disk-space == 'container'
125+
continue-on-error: true
126+
shell: bash
127+
run: timeout --kill-after=30s 10m ./dev/free_disk_space_container
128+
129+
- name: Install Java ${{ inputs.java-version }}
130+
if: inputs.java-version != ''
131+
uses: actions/setup-java@v6
132+
with:
133+
distribution: zulu
134+
java-version: ${{ inputs.java-version }}

‎.github/workflows/benchmark.yml‎

Lines changed: 8 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -93,26 +93,13 @@ jobs:
9393
# In order to get diff files
9494
with:
9595
fetch-depth: 0
96-
- name: Cache SBT and Maven
97-
uses: actions/cache@v5
96+
- name: Setup build environment
97+
uses: ./.github/actions/setup-build-env
9898
with:
99-
path: |
100-
build/apache-maven-*
101-
build/*.jar
102-
~/.sbt
103-
key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }}
104-
restore-keys: |
105-
build-
106-
- name: Cache Coursier local repository
107-
uses: actions/cache@v5
108-
with:
109-
path: ~/.cache/coursier
110-
key: benchmark-coursier-${{ inputs.jdk }}-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }}
111-
restore-keys: |
112-
benchmark-coursier-${{ inputs.jdk }}
99+
coursier: cache
113100
- name: Cache TPC-DS generated data
114101
id: cache-tpcds-sf-1
115-
uses: actions/cache@v5
102+
uses: actions/cache@v6
116103
with:
117104
path: |
118105
./tpcds-sf-1
@@ -181,32 +168,15 @@ jobs:
181168
exit 1
182169
fi
183170
fi
184-
- name: Cache SBT and Maven
185-
uses: actions/cache@v5
186-
with:
187-
path: |
188-
build/apache-maven-*
189-
build/*.jar
190-
~/.sbt
191-
key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }}
192-
restore-keys: |
193-
build-
194-
- name: Cache Coursier local repository
195-
uses: actions/cache@v5
196-
with:
197-
path: ~/.cache/coursier
198-
key: benchmark-coursier-${{ inputs.jdk }}-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }}
199-
restore-keys: |
200-
benchmark-coursier-${{ inputs.jdk }}
201-
- name: Install Java ${{ inputs.jdk }}
202-
uses: actions/setup-java@v6
171+
- name: Setup build environment
172+
uses: ./.github/actions/setup-build-env
203173
with:
204-
distribution: zulu
205174
java-version: ${{ inputs.jdk }}
175+
coursier: cache
206176
- name: Cache TPC-DS generated data
207177
if: inputs.generate-tpcds
208178
id: cache-tpcds-sf-1
209-
uses: actions/cache@v5
179+
uses: actions/cache@v6
210180
with:
211181
path: |
212182
./tpcds-sf-1
@@ -254,4 +224,3 @@ jobs:
254224
with:
255225
name: benchmark-results-${{ inputs.jdk }}-${{ inputs.scala }}-${{ matrix.split }}
256226
path: target/benchmark-results-${{ inputs.jdk }}-${{ inputs.scala }}.tar
257-

0 commit comments

Comments
 (0)