Skip to content

Commit e862b2e

Browse files
authored
Spark historical job + runner (feast-dev#37)
1 parent 0e92b49 commit e862b2e

52 files changed

Lines changed: 2538 additions & 277 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/docker_build_test_push.yml‎

Lines changed: 42 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -183,9 +183,6 @@ jobs:
183183
- name: Terraform apply app
184184
run: terraform apply -input=false -auto-approve tfplan
185185
working-directory: infra/terraform/app
186-
env:
187-
# capture debugging info, see https://www.terraform.io/docs/internals/debugging.html
188-
TF_LOG: TRACE
189186

190187
- name: Cancel any active Databricks runs
191188
run: |
@@ -202,21 +199,59 @@ jobs:
202199
- name: Prepare e2e test pod and clear Redis data
203200
run: |
204201
set -ex
202+
203+
kubectl delete secret azure-e2e-test || true
204+
kubectl create secret generic azure-e2e-test --from-literal=AZURE_CLIENT_SECRET="$ARM_CLIENT_SECRET"
205+
205206
# Start e2e test pod
206207
kubectl delete pod "$E2E_TEST_POD" || true
207-
kubectl run "$E2E_TEST_POD" --restart=Never --image="$FEAST_JUPYTER_IMAGE:$FEAST_VERSION" --limits 'memory=4Gi'
208+
kubectl run "$E2E_TEST_POD" --restart=Never --image "$FEAST_JUPYTER_IMAGE:$FEAST_VERSION" --overrides='
209+
{
210+
"spec": {
211+
"containers": [
212+
{
213+
"name": "'$E2E_TEST_POD'",
214+
"image": "'$FEAST_JUPYTER_IMAGE':'$FEAST_VERSION'",
215+
"resources": {
216+
"limits": {
217+
"memory": "4Gi"
218+
}
219+
},
220+
"env": [
221+
{
222+
"name": "AZURE_TENANT_ID",
223+
"value": "'$ARM_TENANT_ID'"
224+
},
225+
{
226+
"name": "AZURE_CLIENT_ID",
227+
"value": "'$ARM_CLIENT_ID'"
228+
},
229+
{
230+
"name": "AZURE_CLIENT_SECRET",
231+
"valueFrom": {
232+
"secretKeyRef": {
233+
"name": "azure-e2e-test",
234+
"key": "AZURE_CLIENT_SECRET"
235+
}
236+
}
237+
}
238+
]
239+
}
240+
]
241+
}
242+
}'
208243
209244
# While the test pod is starting, clear all data from Redis
210245
kubectl run -it --rm --restart=Never --image=redis redisflush$GITHUB_RUN_NUMBER --command -- redis-cli -h $TF_VAR_redis_hostname FLUSHALL
211246
212-
kubectl wait --timeout 45m --for=condition=ContainersReady pod "$E2E_TEST_POD"
247+
kubectl wait --timeout 15m --for=condition=ContainersReady pod "$E2E_TEST_POD"
213248
214-
- name: Run e2e tests for Redis
249+
- name: Run e2e tests with Databricks
215250
run: |
216251
set -ex
217252
kubectl exec "$E2E_TEST_POD" -- bash -c "
218253
cd /feast/tests/e2e
219-
pytest -x -rA -s databricks-tests.py --core_url feast-core:6565 --serving_url=feast-serving-feast-online-serving:6566
254+
pytest -x -rA -s databricks-tests.py --core_url feast-core:6565 --serving_url=feast-serving-feast-online-serving:6566 --batch_serving_url=feast-serving-feast-batch-serving:6566
220255
"
221256
222257
- name: Run ingestion tests for FF Data Science scenarios

‎.github/workflows/unit_tests.yml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,29 @@ jobs:
2222
name: java-coverage-report
2323
path: ${{ github.workspace }}/docs/coverage/java/target/site/jacoco-aggregate/
2424

25+
unit-test-java-8:
26+
runs-on: ubuntu-latest
27+
container: gcr.io/kf-feast/feast-ci:latest
28+
name: unit test java 8
29+
steps:
30+
- uses: actions/checkout@v2
31+
- uses: actions/cache@v1
32+
with:
33+
path: ~/.m2/repository
34+
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
35+
restore-keys: |
36+
${{ runner.os }}-maven-
37+
- name: test java
38+
# TODO add JDK 8 to gcr.io/kf-feast/feast-ci image
39+
run: |
40+
apt update
41+
apt install -y software-properties-common
42+
wget -qO - https://adoptopenjdk.jfrog.io/adoptopenjdk/api/gpg/key/public | apt-key add -
43+
add-apt-repository --yes https://adoptopenjdk.jfrog.io/adoptopenjdk/deb/
44+
apt update
45+
apt install -y adoptopenjdk-8-hotspot
46+
mvn -B test -pl spark/spark-ingestion-job,spark/spark-historical-retriever-job -Djvm=/usr/lib/jvm/adoptopenjdk-8-hotspot-amd64/bin/java
47+
2548
unit-test-python:
2649
runs-on: ubuntu-latest
2750
container: gcr.io/kf-feast/feast-ci:latest

‎Makefile‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,9 @@ test-java:
4949
test-java-with-coverage:
5050
mvn -B test jacoco:report-aggregate
5151

52+
test-java-8-with-coverage:
53+
mvn -B test -pl spark/spark-ingestion-job,spark/spark-historical-retriever-job jacoco:report-aggregate
54+
5255
build-java:
5356
mvn clean verify
5457

‎infra/docker-compose/docker-compose.databricks.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,9 @@ services:
55
build:
66
context: ../..
77
dockerfile: infra/docker/databricks-emulator/Dockerfile
8-
image: ${FEAST_DATABRICKS_EMULATOR_IMAGE}:${FEAST_VERSION}
8+
image: ${FEAST_DATABRICKS_EMULATOR_IMAGE:-feast-databricks-emulator-dev}:${FEAST_VERSION}
99
volumes:
10-
- ./temp/databricks-emulator-storage:/mnt/storage
10+
- ${FEAST_BATCH_JOB_STAGING_LOCATION}:${FEAST_BATCH_JOB_STAGING_LOCATION}
1111
restart: on-failure
1212
ports:
1313
- 9080:8080

‎infra/docker-compose/docker-compose.yml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,10 +28,12 @@ services:
2828
build:
2929
context: ../..
3030
dockerfile: infra/docker/jupyter/Dockerfile
31-
image: ${FEAST_JUPYTER_IMAGE}:${FEAST_VERSION}
31+
image: ${FEAST_JUPYTER_IMAGE:-feast-jupyter}:${FEAST_VERSION}
32+
user: root
3233
volumes:
3334
- ./gcp-service-accounts/${FEAST_JUPYTER_GCP_SERVICE_ACCOUNT_KEY}:/etc/gcloud/service-accounts/key.json
3435
- ../../:/home/jovyan/feast
36+
- ${FEAST_BATCH_JOB_STAGING_LOCATION}:${FEAST_BATCH_JOB_STAGING_LOCATION}
3537
depends_on:
3638
- core
3739
environment:
Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
feast:
2+
core-host: core
3+
active-store: delta
4+
stores:
5+
- name: delta
6+
type: DELTA
7+
config:
8+
path: /tmp/databricks-emulator-storage/delta
9+
databricks_host: http://databricks-emulator:8080
10+
databricks_token: unused
11+
staging_location: file:${FEAST_JOB_STAGING_LOCATION}
12+
jar_file: /opt/sparkjars/spark-historical-retriever-job.jar
13+
num_workers: 1
14+
spark_version: unused
15+
spark_conf: |
16+
spark.driver.memory 1G
17+
instance_pool_id: unused
18+
timeout_seconds: 600
19+
subscriptions:
20+
- name: "*"
21+
project: "*"
22+
job_store:
23+
redis_host: redis
24+
redis_port: 6379
25+
26+
grpc:
27+
port: 6567

‎infra/docker/databricks-emulator/Dockerfile‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ FROM maven:3.6-jdk-11 as builder
66

77
ARG SPARK_VERSION=2.4.5
88
ARG HADOOP_VERSION=2.7
9+
ARG SCALA_VERSION=2.11
910

1011
# Install Spark runtime
1112
WORKDIR /
@@ -15,9 +16,10 @@ RUN wget -q https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-$
1516
&& mv spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION} spark
1617

1718
RUN cd spark/jars && for jar in \
18-
org/apache/spark/spark-sql-kafka-0-10_2.11/2.4.5/spark-sql-kafka-0-10_2.11-2.4.5.jar \
19+
org/apache/spark/spark-sql-kafka-0-10_${SCALA_VERSION}/${SPARK_VERSION}/spark-sql-kafka-0-10_${SCALA_VERSION}-${SPARK_VERSION}.jar \
1920
org/apache/kafka/kafka-clients/2.3.0/kafka-clients-2.3.0.jar \
20-
io/delta/delta-core_2.11/0.6.1/delta-core_2.11-0.6.1.jar \
21+
io/delta/delta-core_${SCALA_VERSION}/0.6.1/delta-core_${SCALA_VERSION}-0.6.1.jar \
22+
org/apache/spark/spark-avro_${SCALA_VERSION}/${SPARK_VERSION}/spark-avro_${SCALA_VERSION}-${SPARK_VERSION}.jar \
2123
; do \
2224
wget -q https://repo1.maven.org/maven2/$jar; \
2325
done
@@ -48,7 +50,7 @@ COPY LICENSE .m[2] .m2/
4850
# the existing .m2 directory to $FEAST_REPO_ROOT/.m2
4951
#
5052
ENV MAVEN_OPTS="-Dmaven.repo.local=/build/.m2/repository -DdependencyLocationsEnabled=false"
51-
RUN mvn --also-make --projects spark/databricks-emulator,spark/spark-ingestion-job -Drevision=$REVISION \
53+
RUN mvn --also-make --projects spark/databricks-emulator,spark/spark-ingestion-job,spark/spark-historical-retriever-job -Drevision=$REVISION \
5254
--batch-mode clean package
5355

5456
# ============================================================
@@ -63,6 +65,7 @@ ENV SPARK_HOME /spark
6365
COPY --from=builder /spark /spark
6466
COPY --from=builder /build/spark/databricks-emulator/target/databricks-emulator-$REVISION.jar /opt/databricks-emulator.jar
6567
COPY --from=builder /build/spark/spark-ingestion-job/target/spark-ingestion-job-$REVISION.jar /opt/sparkjars/spark-ingestion-job.jar
68+
COPY --from=builder /build/spark/spark-historical-retriever-job/target/spark-historical-retriever-job-$REVISION.jar /opt/sparkjars/spark-historical-retriever-job.jar
6669
CMD ["java",\
6770
"-Xms2048m",\
6871
"-Xmx2048m",\

‎infra/scripts/test-docker-compose-databricks.sh‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,18 @@
22

33
set -e
44

5-
export COMPOSE_ARGS=${COMPOSE_ARGS:-"-f docker-compose.yml -f docker-compose.online.yml -f docker-compose.databricks.yml"}
5+
export COMPOSE_ARGS=${COMPOSE_ARGS:-"-f docker-compose.yml -f docker-compose.online.yml -f docker-compose.batch.yml -f docker-compose.databricks.yml"}
66

77
export FEAST_CORE_CONFIG=databricks.yml
8-
export FEAST_DATABRICKS_EMULATOR_IMAGE=feast-databricks-emulator-dev
8+
export FEAST_BATCH_SERVING_CONFIG=databricks.yml
9+
export FEAST_BATCH_JOB_STAGING_LOCATION=/tmp/databricks-emulator-staging
10+
11+
mkdir -p "$FEAST_BATCH_JOB_STAGING_LOCATION"
912

1013
# Run e2e tests for Redis
11-
export TEST_CMD="cd feast/tests/e2e/ && pytest -s -rA -x basic-ingest-redis-serving.py --core_url core:6565 --serving_url=online-serving:6566 --databricks"
14+
export TEST_CMD="cd feast/tests/e2e/ && pytest -s -rA -x databricks-tests.py --core_url core:6565 --serving_url=online-serving:6566 --batch_serving_url=batch-serving:6567"
1215
infra/scripts/test-docker-compose.sh
1316

1417
# Run ingestion tests for FF Data Science scenarios
15-
export TEST_CMD="cd feast/tests/ds_scenarios/ && pytest -s -rA -x test-ingest.py --core_url core:6565 --serving_url=online-serving:6566"
18+
export TEST_CMD="cd feast/tests/ds_scenarios/ && pytest -s -rA -x test-ingest.py --core_url core:6565 --serving_url=online-serving:6566 --batch_serving_url=batch-serving:6567"
1619
infra/scripts/test-docker-compose.sh

‎infra/terraform/app/main.tf‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,8 @@ feast-core:
147147
service.beta.kubernetes.io/azure-load-balancer-internal-subnet: "internal-load-balancers"
148148
loadBalancerIP: ${var.feast_core_vnet_ip}
149149
150+
logLevel: INFO
151+
150152
application-secret.yaml:
151153
152154
spring:
@@ -236,6 +238,8 @@ feast-online-serving:
236238
service.beta.kubernetes.io/azure-load-balancer-internal-subnet: "internal-load-balancers"
237239
loadBalancerIP: ${var.feast_online_serving_vnet_ip}
238240
241+
logLevel: INFO
242+
239243
application-override.yaml:
240244
feast:
241245
core-host: "${var.feast_core_vnet_ip}"
@@ -262,15 +266,27 @@ feast-batch-serving:
262266
service.beta.kubernetes.io/azure-load-balancer-internal-subnet: "internal-load-balancers"
263267
loadBalancerIP: ${var.feast_batch_serving_vnet_ip}
264268
265-
application-override.yaml:
269+
logLevel: INFO
270+
271+
application-secret.yaml:
266272
feast:
267273
core-host: "${var.feast_core_vnet_ip}"
268274
active_store: delta
269275
stores:
270276
- name: delta
271277
type: DELTA
272278
config:
273-
path: "abfss://${var.datalake_filesystem}@${var.datalake_name}.dfs.core.windows.net/feast${var.run_number}"
279+
path: "abfss://${var.datalake_filesystem}@${var.datalake_name}.dfs.core.windows.net/"
280+
databricks_host: "${var.databricks_workspace_url}"
281+
databricks_token: "${databricks_token.feast.token_value}"
282+
staging_location: "${var.storage_staging_url}"
283+
jar_file: "${local.databricks_dbfs_jar_folder}/sparkjars/spark-historical-retriever-job.jar"
284+
num_workers: 1
285+
spark_version: "${local.databricks_spark_version}"
286+
spark_conf: |
287+
fs.azure.account.key.${var.datalake_name}.dfs.core.windows.net {{secrets/${local.databricks_secret_scope}/${local.databricks_secret_datalake_key}}}
288+
instance_pool_id: "${databricks_instance_pool.feast.id}"
289+
timeout_seconds: 3600
274290
subscriptions:
275291
- name: "*"
276292
project: "*"

‎infra/terraform/app/variables.tf‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,10 @@ variable "datalake_filesystem" {
5858
description = "The filesystem of the Azure Data Lake Storage Gen2 account for storing Delta data."
5959
type = string
6060
}
61+
variable "storage_staging_url" {
62+
description = "The ABFSS URL of the storage location for exchanging batch data between Feast Serving (Historical Retriever) and Feast client."
63+
type = string
64+
}
6165
variable "redis_hostname" {
6266
description = "The hostname of the Azure Redis Cache instance."
6367
type = string

0 commit comments

Comments
 (0)