Skip to content

Commit fa62325

Browse files
andreydevyatkinandreydevyatkin
committed
# This is a combination of 2 commits.
# This is the 1st commit message: added beam_PostCommit_Java_ValidatesRunner jobs to GitHub Actions # This is the commit message #2: Don't improperly filter newly-added elements that overlap with a delete.
1 parent f86d9e2 commit fa62325

97 files changed

Lines changed: 6414 additions & 790 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/gh-actions-self-hosted-runners/arc/environments/beam.env‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,9 +34,9 @@ main_runner = {
3434
runner_image = "us-central1-docker.pkg.dev/apache-beam-testing/beam-github-actions/beam-arc-runner:60d397ecfbd2b10a1929615c70d500eb71a2c053"
3535
machine_type = "e2-standard-16"
3636
min_node_count = "1"
37-
max_node_count = "16"
37+
max_node_count = "24"
3838
min_replicas = "1"
39-
max_replicas = "128"
39+
max_replicas = "200"
4040
webhook_scaling = true
4141
disk_size_gb = 200
4242
requests = {

‎.github/workflows/README.md‎

Lines changed: 36 additions & 3 deletions
Large diffs are not rendered by default.

‎.github/workflows/beam_PostCommit_Go_VR_Flink.yml‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
name: PostCommit Go VR Flink
17+
18+
on:
19+
issue_comment:
20+
types: [created]
21+
schedule:
22+
- cron: '0 */6 * * *'
23+
workflow_dispatch:
24+
25+
#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
26+
permissions:
27+
actions: write
28+
pull-requests: read
29+
checks: read
30+
contents: read
31+
deployments: read
32+
id-token: none
33+
issues: read
34+
discussions: read
35+
packages: read
36+
pages: read
37+
repository-projects: read
38+
security-events: read
39+
statuses: read
40+
41+
# This allows a subsequently queued workflow run to interrupt previous runs
42+
concurrency:
43+
group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.body || github.event.sender.login }}'
44+
cancel-in-progress: true
45+
46+
env:
47+
GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
48+
GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
49+
GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
50+
51+
jobs:
52+
beam_PostCommit_Go_VR_Flink:
53+
if: |
54+
github.event_name == 'workflow_dispatch' ||
55+
github.event_name == 'schedule' ||
56+
github.event.comment.body == 'Run Go Flink ValidatesRunner'
57+
runs-on: [self-hosted, ubuntu-20.04, main]
58+
timeout-minutes: 100
59+
name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
60+
strategy:
61+
matrix:
62+
job_name: ["beam_PostCommit_Go_VR_Flink"]
63+
job_phrase: ["Run Go Flink ValidatesRunner"]
64+
steps:
65+
- uses: actions/checkout@v3
66+
- name: Setup repository
67+
uses: ./.github/actions/setup-action
68+
with:
69+
comment_phrase: ${{ matrix.job_phrase }}
70+
github_token: ${{ secrets.GITHUB_TOKEN }}
71+
github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
72+
- name: run Go Flink ValidatesRunner script
73+
env:
74+
CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}}
75+
uses: ./.github/actions/gradle-command-self-hosted-action
76+
with:
77+
gradle-command: :sdks:go:test:flinkValidatesRunner

‎.github/workflows/beam_PostCommit_Go_VR_Spark.yml‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
name: PostCommit Go VR Spark
17+
18+
on:
19+
issue_comment:
20+
types: [created]
21+
schedule:
22+
- cron: '0 */6 * * *'
23+
workflow_dispatch:
24+
25+
#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
26+
permissions:
27+
actions: write
28+
pull-requests: read
29+
checks: read
30+
contents: read
31+
deployments: read
32+
id-token: none
33+
issues: read
34+
discussions: read
35+
packages: read
36+
pages: read
37+
repository-projects: read
38+
security-events: read
39+
statuses: read
40+
41+
# This allows a subsequently queued workflow run to interrupt previous runs
42+
concurrency:
43+
group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.body || github.event.sender.login}}'
44+
cancel-in-progress: true
45+
46+
env:
47+
GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
48+
GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
49+
GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
50+
51+
jobs:
52+
beam_PostCommit_Go_VR_Spark:
53+
if: |
54+
github.event_name == 'workflow_dispatch' ||
55+
github.event_name == 'schedule' ||
56+
github.event.comment.body == 'Run Go Spark ValidatesRunner'
57+
runs-on: [self-hosted, ubuntu-20.04, main]
58+
timeout-minutes: 100
59+
name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
60+
strategy:
61+
matrix:
62+
job_name: ["beam_PostCommit_Go_VR_Spark"]
63+
job_phrase: ["Run Go Spark ValidatesRunner"]
64+
steps:
65+
- uses: actions/checkout@v3
66+
- name: Setup repository
67+
uses: ./.github/actions/setup-action
68+
with:
69+
comment_phrase: ${{ matrix.job_phrase }}
70+
github_token: ${{ secrets.GITHUB_TOKEN }}
71+
github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
72+
- name: run Go Spark ValidatesRunner script
73+
env:
74+
CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}}
75+
uses: ./.github/actions/gradle-command-self-hosted-action
76+
with:
77+
gradle-command: :sdks:go:test:sparkValidatesRunner
78+

‎.github/workflows/beam_PostCommit_Java_Avro_Versions.yml‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
name: PostCommit Java Avro Versions
19+
20+
on:
21+
issue_comment:
22+
types: [created]
23+
schedule:
24+
- cron: '0 */6 * * *'
25+
workflow_dispatch:
26+
27+
# This allows a subsequently queued workflow run to interrupt previous runs
28+
concurrency:
29+
group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.body || github.event.sender.login}}'
30+
cancel-in-progress: true
31+
32+
#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
33+
permissions:
34+
actions: write
35+
pull-requests: read
36+
checks: read
37+
contents: read
38+
deployments: read
39+
id-token: none
40+
issues: read
41+
discussions: read
42+
packages: read
43+
pages: read
44+
repository-projects: read
45+
security-events: read
46+
statuses: read
47+
48+
jobs:
49+
beam_PostCommit_Java_Avro_Versions:
50+
name: ${{matrix.job_name}} (${{matrix.job_phrase}})
51+
runs-on: [self-hosted, ubuntu-20.04, main]
52+
timeout-minutes: 240
53+
strategy:
54+
matrix:
55+
job_name: [beam_PostCommit_Java_Avro_Versions]
56+
job_phrase: [Run Java Avro Versions PostCommit]
57+
if: |
58+
github.event_name == 'workflow_dispatch' ||
59+
github.event_name == 'schedule' ||
60+
github.event.comment.body == 'Run Java Avro Versions PostCommit'
61+
steps:
62+
- uses: actions/checkout@v3
63+
- name: Setup repository
64+
uses: ./.github/actions/setup-action
65+
with:
66+
comment_phrase: ${{ matrix.job_phrase }}
67+
github_token: ${{ secrets.GITHUB_TOKEN }}
68+
github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
69+
- name: run PostCommit Java Avro Versions script
70+
uses: ./.github/actions/gradle-command-self-hosted-action
71+
with:
72+
gradle-command: :javaAvroVersionsTest
73+
- name: Upload test report
74+
uses: actions/upload-artifact@v3
75+
with:
76+
name: java-code-coverage-report
77+
path: "**/build/test-results/**/*.xml"

‎.github/workflows/beam_PostCommit_Java_DataflowV1.yml‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
name: PostCommit Java Dataflow V1
19+
20+
on:
21+
issue_comment:
22+
types: [created]
23+
schedule:
24+
- cron: '0 */6 * * *'
25+
workflow_dispatch:
26+
27+
# This allows a subsequently queued workflow run to interrupt previous runs
28+
concurrency:
29+
group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.body || github.event.sender.login}}'
30+
cancel-in-progress: true
31+
32+
#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
33+
permissions:
34+
actions: write
35+
pull-requests: read
36+
checks: read
37+
contents: read
38+
deployments: read
39+
id-token: none
40+
issues: read
41+
discussions: read
42+
packages: read
43+
pages: read
44+
repository-projects: read
45+
security-events: read
46+
statuses: read
47+
48+
env:
49+
GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
50+
GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
51+
GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
52+
53+
jobs:
54+
beam_PostCommit_Java_DataflowV1:
55+
name: ${{matrix.job_name}} (${{matrix.job_phrase}})
56+
runs-on: [self-hosted, ubuntu-20.04, main]
57+
timeout-minutes: 240
58+
strategy:
59+
matrix:
60+
job_name: [beam_PostCommit_Java_DataflowV1]
61+
job_phrase: [Run PostCommit_Java_Dataflow]
62+
if: |
63+
github.event_name == 'workflow_dispatch' ||
64+
github.event_name == 'schedule' ||
65+
github.event.comment.body == 'Run PostCommit_Java_Dataflow'
66+
steps:
67+
- uses: actions/checkout@v3
68+
- name: Setup repository
69+
uses: ./.github/actions/setup-action
70+
with:
71+
comment_phrase: ${{ matrix.job_phrase }}
72+
github_token: ${{ secrets.GITHUB_TOKEN }}
73+
github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
74+
- name: Set up Java
75+
uses: actions/[email protected]
76+
with:
77+
distribution: 'temurin'
78+
java-version: |
79+
11
80+
8
81+
- name: run PostCommit Java Dataflow V1 script
82+
uses: ./.github/actions/gradle-command-self-hosted-action
83+
with:
84+
gradle-command: :runners:google-cloud-dataflow-java:postCommit
85+
- name: Upload test report
86+
uses: actions/upload-artifact@v3
87+
with:
88+
name: java-code-coverage-report
89+
path: "**/build/test-results/**/*.xml"

0 commit comments

Comments
 (0)