Skip to content

Commit d8e2fa7

Browse files
pabloempabloem
authored
Merge branch 'master' into satybald/fix-avro-load-issues
2 parents 1512b98 + 6390bcd commit d8e2fa7

1,000 files changed

Lines changed: 34492 additions & 12325 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/build_playground_backend.yml‎

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,9 @@ jobs:
3333
steps:
3434
- name: Check out the repo
3535
uses: actions/checkout@v2
36-
- name: Set up JDK 1.8
37-
uses: joschi/setup-jdk@v2
36+
- uses: actions/setup-java@v2
3837
with:
38+
distribution: 'zulu'
3939
java-version: '8'
4040
- uses: actions/setup-go@v2
4141
with:
@@ -64,12 +64,28 @@ jobs:
6464
- name: Login to Docker Registry
6565
run: cat /tmp/gcp_access.json | docker login -u _json_key --password-stdin https://${{ secrets.REGISTRY_NAME }}
6666
if: startsWith(github.ref, 'ref/tags/')
67-
- name: Preapre Build
67+
- name: Preapre Build Java
6868
run: ./gradlew playground:backend:containers:java:dockerPush -Pdocker-repository-root='${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository' -Pbase-image='apache/beam_java8_sdk:${{ env.BEAM_VERSION }}'
6969
if: startsWith(github.ref, 'ref/tags/')
70-
- name: Deploy Backend Application
70+
- name: Preapre Build Python
71+
run: ./gradlew playground:backend:containers:python:dockerPush -Pdocker-repository-root='${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository' -Pbase-image='apache/beam_java8_sdk:${{ env.BEAM_VERSION }}'
72+
- name: Preapre Build Go
73+
run: ./gradlew playground:backend:containers:go:dockerPush -Pdocker-repository-root='${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository' -Pbase-image='apache/beam_java8_sdk:${{ env.BEAM_VERSION }}'
74+
if: startsWith(github.ref, 'ref/tags/')
75+
- name: Deploy Backend Application Java
7176
env:
7277
GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json
7378
run: terraform init && terraform apply -auto-approve -var="project_id=${{ secrets.PROJECT_ID }}" -var="docker_registry_address=${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository"
7479
working-directory: playground/terraform/applications/backend
7580
if: startsWith(github.ref, 'ref/tags/')
81+
- name: Deploy Backend Application Python
82+
env:
83+
GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json
84+
run: terraform init && terraform apply -auto-approve -var="project_id=${{ secrets.PROJECT_ID }}" -var="docker_registry_address=${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository"
85+
working-directory: playground/terraform/applications/backend-python
86+
- name: Deploy Backend Application Go
87+
env:
88+
GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json
89+
run: terraform init && terraform apply -auto-approve -var="project_id=${{ secrets.PROJECT_ID }}" -var="docker_registry_address=${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository"
90+
working-directory: playground/terraform/applications/backend-go
91+
if: startsWith(github.ref, 'ref/tags/')

‎.github/workflows/build_playground_frontend.yml‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -30,17 +30,23 @@ jobs:
3030
GO_VERSION: 1.17.0
3131
BEAM_VERSION: 2.33.0
3232
TERRAFORM_VERSION: 1.0.9
33+
FLUTTER_VERSION: 2.5.2-stable
3334
steps:
3435
- name: Check out the repo
3536
uses: actions/checkout@v2
36-
- name: Set up JDK 1.8
37-
uses: joschi/setup-jdk@v2
37+
- uses: actions/setup-java@v2
3838
with:
39+
distribution: 'zulu'
3940
java-version: '8'
4041
- name: install flutter
41-
uses: subosito/flutter-action@v1
42-
with:
43-
channel: 'stable'
42+
run: |
43+
wget https://storage.googleapis.com/flutter_infra_release/releases/stable/linux/flutter_linux_$FLUTTER_VERSION.tar.xz &&\
44+
tar -xf flutter_linux_$FLUTTER_VERSION.tar.xz &&\
45+
mv flutter /opt/ &&\
46+
ln -s /opt/flutter/bin/flutter /usr/local/bin/flutter &&\
47+
ln -s /opt/flutter/bin/dart /usr/local/bin/dart &&\
48+
dart pub global activate protoc_plugin &&\
49+
ln -s /root/.pub-cache/bin/protoc-gen-dart /usr/local/bin/
4450
- name: Remove default github maven configuration
4551
# This step is a workaround to avoid a decryption issue
4652
run: rm ~/.m2/settings.xml

‎.github/workflows/java_tests.yml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,13 @@ on:
3838

3939

4040
jobs:
41+
setup-go:
42+
runs-on: [ubuntu-latest, macos-latest, windows-latest]
43+
steps:
44+
- uses: actions/checkout@v2
45+
- uses: actions/setup-go@v2
46+
with:
47+
go-version: '1.16.12'
4148

4249
check_gcp_variables:
4350
timeout-minutes: 5
@@ -108,6 +115,7 @@ jobs:
108115
path: runners/core-java/build/reports/tests/test
109116

110117
java_wordcount_direct_runner:
118+
needs: setup-go
111119
name: 'Java Wordcount Direct Runner'
112120
runs-on: ${{ matrix.os }}
113121
strategy:
@@ -137,6 +145,7 @@ jobs:
137145
name: 'Java Wordcount Dataflow'
138146
needs:
139147
- check_gcp_variables
148+
- setup-go
140149
runs-on: ${{ matrix.os }}
141150
if: |
142151
needs.check_gcp_variables.outputs.gcp-variables-set == 'true' && (

‎.github/workflows/local_env_tests.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,9 @@ jobs:
3535
runs-on: ubuntu-latest
3636
steps:
3737
- uses: actions/checkout@v2
38+
- uses: actions/setup-go@v2
39+
with:
40+
go-version: '1.16.12'
3841
- name: "Installing local env dependencies"
3942
run: "sudo ./local-env-setup.sh"
4043
id: local_env_install_ubuntu
@@ -47,6 +50,9 @@ jobs:
4750
runs-on: macos-latest
4851
steps:
4952
- uses: actions/checkout@v2
53+
- uses: actions/setup-go@v2
54+
with:
55+
go-version: '1.16.12'
5056
- name: "Installing local env dependencies"
5157
run: "./local-env-setup.sh"
5258
id: local_env_install_mac

‎.github/workflows/playground_deploy_examples.yml‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
name: Collect And Deploy Playground Examples
17+
18+
on:
19+
push:
20+
tags: 'v*'
21+
workflow_dispatch:
22+
23+
jobs:
24+
deploy_examples:
25+
name: Deploy examples
26+
runs-on: ubuntu-latest
27+
env:
28+
BEAM_ROOT_DIR: ../../
29+
BEAM_EXAMPLE_CATEGORIES: ../categories.yaml
30+
BEAM_VERSION: 2.33.0
31+
SERVER_ADDRESS: backend-python-dot-datatokenization.uc.r.appspot.com
32+
K8S_NAMESPACE: playground-backend
33+
HELM_APP_NAME: playground-backend
34+
steps:
35+
- name: Check out the repo
36+
uses: actions/checkout@v2
37+
- uses: actions/setup-python@v2
38+
with:
39+
python-version: '3.8'
40+
- uses: actions/setup-java@v2
41+
with:
42+
distribution: 'zulu'
43+
java-version: '8'
44+
- name: Install kubectl
45+
run: |
46+
curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" &&\
47+
chmod +x kubectl &&\
48+
mv kubectl /usr/local/bin/
49+
- name: Install helm
50+
run: |
51+
curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 &&\
52+
chmod 700 get_helm.sh &&\
53+
./get_helm.sh
54+
- name: Set up Cloud SDK
55+
uses: google-github-actions/[email protected]
56+
- name: install deps
57+
run: pip install -r requirements.txt
58+
working-directory: playground/infrastructure
59+
- name: Setup GCP account
60+
run: |
61+
echo "${{ secrets.GCP_ACCESS_KEY }}" | base64 -d > /tmp/gcp_access.json
62+
gcloud auth activate-service-account --project=datatokenization --key-file=/tmp/gcp_access.json
63+
- name: Get K8s Config
64+
run: gcloud container clusters get-credentials --region us-central1-a playground-examples
65+
- name: Login to Docker Registry
66+
run: cat /tmp/gcp_access.json | docker login -u _json_key --password-stdin https://${{ secrets.REGISTRY_NAME }}
67+
- name: Build And Push Java Backend
68+
run: ./gradlew playground:backend:containers:java:dockerTagPush -Pdocker-repository-root='${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository' -Pbase-image='apache/beam_java8_sdk:${{ env.BEAM_VERSION }}' -Pdocker-tag="$GITHUB_SHA"
69+
- name: Build And Push Go Backend
70+
run: ./gradlew playground:backend:containers:go:dockerTagPush -Pdocker-repository-root='${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository' -Pdocker-tag="$GITHUB_SHA"
71+
- name: Build And Push Python Backend
72+
run: ./gradlew playground:backend:containers:python:dockerTagPush -Pdocker-repository-root='${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository' -Pdocker-tag="$GITHUB_SHA"
73+
- name: Install helm chart
74+
run: |
75+
kubectl create namespace $K8S_NAMESPACE --dry-run=client -o yaml | kubectl apply -f - &&\
76+
helm install --namespace $K8S_NAMESPACE $HELM_APP_NAME . --set global.registry="${{ secrets.REGISTRY_NAME }}/${{ secrets.PROJECT_ID }}/playground-repository" --set global.tag="$GITHUB_SHA" &&\
77+
sleep 120
78+
working-directory: playground/infrastructure/helm
79+
- name: Run Python Examples CI
80+
run: |
81+
K8S_SERVER_ADDRESS=$(kubectl get svc -n $K8S_NAMESPACE -l "app=service-python" -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}')
82+
BEAM_ROOT_DIR="../../"
83+
BEAM_EXAMPLE_CATEGORIES="../../playground/categories.yaml"
84+
export SERVER_ADDRESS="$K8S_SERVER_ADDRESS:8081" && python3 ci_cd.py --step CI --sdk SDK_PYTHON
85+
working-directory: playground/infrastructure
86+
- name: Run Python Examples CD
87+
run: |
88+
K8S_SERVER_ADDRESS=$(kubectl get svc -n $K8S_NAMESPACE -l "app=service-python" -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}')
89+
BEAM_ROOT_DIR="../../"
90+
BEAM_EXAMPLE_CATEGORIES="../../playground/categories.yaml"
91+
export SERVER_ADDRESS="$K8S_SERVER_ADDRESS:8081" && python3 ci_cd.py --step CD --sdk SDK_PYTHON
92+
working-directory: playground/infrastructure
93+
env:
94+
GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json
95+
- name: Run Go Examples CI
96+
run: |
97+
K8S_SERVER_ADDRESS=$(kubectl get svc -n $K8S_NAMESPACE -l "app=service-go" -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}')
98+
BEAM_ROOT_DIR="../../"
99+
BEAM_EXAMPLE_CATEGORIES="../../playground/categories.yaml"
100+
export SERVER_ADDRESS="$K8S_SERVER_ADDRESS:8082" && python3 ci_cd.py --step CI --sdk SDK_GO
101+
working-directory: playground/infrastructure
102+
- name: Run Go Examples CD
103+
run: |
104+
K8S_SERVER_ADDRESS=$(kubectl get svc -n $K8S_NAMESPACE -l "app=service-go" -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}')
105+
BEAM_ROOT_DIR="../../"
106+
BEAM_EXAMPLE_CATEGORIES="../../playground/categories.yaml"
107+
export SERVER_ADDRESS="$K8S_SERVER_ADDRESS:8082" && python3 ci_cd.py --step CD --sdk SDK_GO
108+
working-directory: playground/infrastructure
109+
env:
110+
GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json
111+
- name: Run Java Examples CI
112+
run: |
113+
K8S_SERVER_ADDRESS=$(kubectl get svc -n $K8S_NAMESPACE -l "app=service-java" -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}')
114+
BEAM_ROOT_DIR="../../"
115+
BEAM_EXAMPLE_CATEGORIES="../../playground/categories.yaml"
116+
export SERVER_ADDRESS="$K8S_SERVER_ADDRESS:8080" && python3 ci_cd.py --step CI --sdk SDK_JAVA
117+
working-directory: playground/infrastructure
118+
- name: Run Java Examples CD
119+
run: |
120+
K8S_SERVER_ADDRESS=$(kubectl get svc -n $K8S_NAMESPACE -l "app=service-java" -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}')
121+
BEAM_ROOT_DIR="../../"
122+
BEAM_EXAMPLE_CATEGORIES="../../playground/categories.yaml"
123+
export SERVER_ADDRESS="$K8S_SERVER_ADDRESS:8080" && python3 ci_cd.py --step CD --sdk SDK_JAVA
124+
working-directory: playground/infrastructure
125+
env:
126+
GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json
127+
- name: Delete Helm Chart
128+
run: |
129+
helm del --namespace $K8S_NAMESPACE $HELM_APP_NAME

‎.github/workflows/python_tests.yml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,14 @@ on:
3838

3939
jobs:
4040

41+
setup-go:
42+
runs-on: [ubuntu-latest, macos-latest, windows-latest]
43+
steps:
44+
- uses: actions/checkout@v2
45+
- uses: actions/setup-go@v2
46+
with:
47+
go-version: '1.16.12'
48+
4149
check_gcp_variables:
4250
timeout-minutes: 5
4351
name: "Check GCP variables"
@@ -168,6 +176,7 @@ jobs:
168176
name: 'Python Wordcount Dataflow'
169177
needs:
170178
- build_python_sdk_source
179+
- setup-go
171180
runs-on: ${{ matrix.os }}
172181
strategy:
173182
fail-fast: false

‎.test-infra/dataproc/flink_cluster.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
# Provide the following environment to run this script:
1818
#
1919
# GCLOUD_ZONE: Google cloud zone. Optional. Default: "us-central1-a"
20-
# DATAPROC_VERSION: Dataproc version. Optional. Default: 1.2
20+
# DATAPROC_VERSION: Dataproc version. Optional. Default: 1.5
2121
# CLUSTER_NAME: Cluster name
2222
# GCS_BUCKET: GCS bucket url for Dataproc resources (init actions)
2323
# HARNESS_IMAGES_TO_PULL: Urls to SDK Harness' images to pull on dataproc workers (optional: 0, 1 or multiple urls for every harness image)
@@ -46,7 +46,7 @@ set -Eeuxo pipefail
4646

4747
# GCloud properties
4848
GCLOUD_ZONE="${GCLOUD_ZONE:=us-central1-a}"
49-
DATAPROC_VERSION="${DATAPROC_VERSION:=1.2}"
49+
DATAPROC_VERSION="${DATAPROC_VERSION:=1.5}"
5050

5151
MASTER_NAME="$CLUSTER_NAME-m"
5252

‎.test-infra/jenkins/dependency_check/generate_report.sh‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -44,10 +44,9 @@ REPORT_DESCRIPTION="
4444
# Virtualenv for the rest of the script to run setup
4545
$PYTHON -m venv dependency/check
4646

47-
. dependency/check/bin/activate
48-
pip install --upgrade google-cloud-bigquery
49-
pip install --upgrade google-cloud-bigtable
50-
pip install --upgrade google-cloud-core
47+
. ./dependency/check/bin/activate
48+
pip install --upgrade pip setuptools wheel
49+
pip install --upgrade google-cloud-bigquery google-cloud-bigtable google-cloud-core
5150
rm -f build/dependencyUpdates/beam-dependency-check-report.txt
5251

5352
# Insall packages and run the unit tests of the report generator and the jira manager

‎.test-infra/jenkins/job_00_seed.groovy‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,8 @@ job('beam_SeedJob') {
108108
command("""
109109
( cd .test-infra/jenkins/committers_list_generator &&
110110
python3.8 -m venv ve3 && source ve3/bin/activate &&
111-
pip install -r requirements.txt &&
111+
pip install --retries 10 --upgrade pip setuptools wheel &&
112+
pip install --retries 10 -r requirements.txt &&
112113
python main.py -o .. &&
113114
deactivate ) ||
114115
{ echo "ERROR: Failed to fetch committers"; exit 3; }

‎.test-infra/jenkins/job_PostCommit_Java_PortableValidatesRunner_Spark_Batch.groovy‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_PVR_Spark_Batch',
2525
description('Runs the Java PortableValidatesRunner suite on the Spark runner in batch mode.')
2626

2727
// Set common parameters.
28-
commonJobProperties.setTopLevelMainJobProperties(delegate)
28+
commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240)
2929

3030
// Publish all test results to Jenkins
3131
publishers {
@@ -38,6 +38,8 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_PVR_Spark_Batch',
3838
rootBuildScriptDir(commonJobProperties.checkoutDir)
3939
tasks(':runners:spark:2:job-server:validatesPortableRunnerBatch')
4040
tasks(':runners:spark:3:job-server:validatesPortableRunnerBatch')
41+
tasks(':runners:spark:2:job-server:validatesPortableRunnerDocker')
42+
tasks(':runners:spark:3:job-server:validatesPortableRunnerDocker')
4143
commonJobProperties.setGradleSwitches(delegate)
4244
}
4345
}

0 commit comments

Comments
 (0)