Skip to content

Commit 88e43a8

Browse files
algattikalgattik
authored
Created databricks-emulator (feast-dev#13)
Closes KE-610 Closes KE-643 - Added 4 projects in the Maven structure Spark: - spark/databricks-emulator: Implemented a Databricks REST API emulator running in Docker - spark/databricks-types: Databricks REST API types - used by databricks-emulator, and also to be used by DatabricksJobManager (in Feast Core) - spark/spark-historical-retriever-job: Placeholder for implementing Historical Retriever Spark job - spark/spark-ingestion-job: Placeholder for implementing Ingestion Spark job The Databricks emulator Dockerfile exists, but is not yet built by the CI pipeline & deployed in end-to-end test (to be done later).
1 parent 1ae84df commit 88e43a8

32 files changed

Lines changed: 1376 additions & 5 deletions

File tree

‎.github/workflows/docker_build_test_push.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ jobs:
5252
FEAST_SERVING_IMAGE: ${{ secrets.CONTAINERREGISTRY_URL }}/${{ secrets.CONTAINERREGISTRY_IMAGENAMEBASE }}-serving
5353

5454
- name: test docker compose
55-
run: ./infra/scripts/test-docker-compose.sh
55+
run: ./infra/scripts/test-docker-compose-databricks.sh
5656
env:
5757
COMPOSE_PROJECT_NAME: feast
5858
FEAST_VERSION: v${{ github.sha }}

‎infra/docker-compose/core/databricks.yml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
feast:
2+
jobs:
3+
polling_interval_milliseconds: 30000
4+
job_update_timeout_seconds: 240
5+
# TODO replace with DatabricksRunner
6+
active_runner: direct
7+
runners:
8+
# TODO replace with DatabricksRunner
9+
- name: direct
10+
type: DirectRunner
11+
options: {}
12+
stream:
13+
type: kafka
14+
options:
15+
topic: feast-features
16+
bootstrapServers: "kafka:9092,localhost:9094"

‎infra/docker-compose/docker-compose.databricks.yml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
version: "3.7"
2+
3+
services:
4+
databricks-emulator:
5+
build:
6+
context: ../..
7+
dockerfile: infra/docker/databricks-emulator/Dockerfile
8+
image: ${FEAST_DATABRICKS_EMULATOR_IMAGE}:${FEAST_VERSION}
9+
volumes:
10+
- ./temp/databricks-emulator-storage:/mnt/storage
11+
restart: on-failure
12+
ports:
13+
- 9080:8080

‎infra/docker-compose/docker-compose.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ services:
2828
image: jupyter/minimal-notebook:619e9cc2fc07
2929
volumes:
3030
- ./gcp-service-accounts/${FEAST_JUPYTER_GCP_SERVICE_ACCOUNT_KEY}:/etc/gcloud/service-accounts/key.json
31+
- ../../:/home/jovyan/feast
3132
- ./jupyter/startup.sh:/etc/startup.sh
3233
depends_on:
3334
- core

‎infra/docker-compose/jupyter/startup.sh‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,6 @@
22

33
set -ex
44

5-
# Clone Feast repository into Jupyter container
6-
git clone -b ${FEAST_REPOSITORY_VERSION} --single-branch https://github.com/feast-dev/feast.git || true
7-
85
# Install Python dependencies
96
make -C feast/ compile-protos-python
107

‎infra/docker/core/Dockerfile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ COPY core core
3030
COPY ingestion ingestion
3131
COPY protos protos
3232
COPY serving serving
33+
COPY spark spark
3334
COPY pom.xml pom.xml
3435

3536
# Trick to copy .m2 directory only if it exists.

‎infra/docker/databricks-emulator/Dockerfile‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
# ============================================================
2+
# Build stage 1: Builder
3+
# ============================================================
4+
5+
FROM maven:3.6-jdk-11 as builder
6+
7+
ARG SPARK_VERSION=2.4.5
8+
ARG HADOOP_VERSION=2.7
9+
10+
# Install Spark runtime
11+
WORKDIR /
12+
13+
RUN wget -q https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz \
14+
&& tar -zxf spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz \
15+
&& mv spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION} spark
16+
17+
RUN cd spark/jars && for jar in \
18+
org/apache/spark/spark-sql-kafka-0-10_2.11/2.4.5/spark-sql-kafka-0-10_2.11-2.4.5.jar \
19+
org/apache/kafka/kafka-clients/2.3.0/kafka-clients-2.3.0.jar \
20+
; do \
21+
wget https://repo1.maven.org/maven2/$jar; \
22+
done
23+
24+
RUN mv spark/conf/log4j.properties.template spark/conf/log4j.properties
25+
26+
ARG REVISION=dev
27+
28+
WORKDIR /build
29+
30+
COPY datatypes datatypes
31+
COPY storage storage
32+
COPY sdk/java sdk/java
33+
COPY core core
34+
COPY ingestion ingestion
35+
COPY protos protos
36+
COPY serving serving
37+
COPY spark spark
38+
COPY pom.xml pom.xml
39+
40+
# Trick to copy .m2 directory only if it exists.
41+
# The LICENSE file is any file that actually exists, to make sure the command doesn't fail.
42+
COPY LICENSE .m[2] .m2/
43+
44+
#
45+
# Setting Maven repository .m2 directory relative to /build folder gives the
46+
# user to optionally use cached repository when building the image by copying
47+
# the existing .m2 directory to $FEAST_REPO_ROOT/.m2
48+
#
49+
ENV MAVEN_OPTS="-Dmaven.repo.local=/build/.m2/repository -DdependencyLocationsEnabled=false"
50+
RUN mvn --also-make --projects spark/databricks-emulator -Drevision=$REVISION \
51+
--batch-mode clean package
52+
53+
# ============================================================
54+
# Build stage 2: Production
55+
# ============================================================
56+
57+
FROM openjdk:8u252-jre as production
58+
ARG REVISION=dev
59+
60+
ENV SPARK_HOME /spark
61+
62+
COPY --from=builder /spark /spark
63+
COPY --from=builder /build/spark/databricks-emulator/target/databricks-emulator-$REVISION.jar /opt/databricks-emulator.jar
64+
CMD ["java",\
65+
"-Xms2048m",\
66+
"-Xmx2048m",\
67+
"-jar",\
68+
"/opt/databricks-emulator.jar"]

‎infra/docker/serving/Dockerfile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ COPY core core
2424
COPY ingestion ingestion
2525
COPY protos protos
2626
COPY serving serving
27+
COPY spark spark
2728
COPY pom.xml pom.xml
2829

2930
# Trick to copy .m2 directory only if it exists.

‎infra/scripts/test-docker-compose-databricks.sh‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
#!/usr/bin/env bash
2+
3+
set -e
4+
5+
echo "
6+
============================================================
7+
Running Docker Compose tests with pytest at 'tests/e2e'
8+
============================================================
9+
"
10+
11+
COMPOSE_ARGS="-f docker-compose.yml -f docker-compose.online.yml -f docker-compose.databricks.yml"
12+
13+
clean_up () {
14+
ARG=$?
15+
16+
# Shut down docker-compose images
17+
docker-compose $COMPOSE_ARGS down
18+
19+
# Remove configuration file
20+
rm .env
21+
22+
exit $ARG
23+
}
24+
25+
trap clean_up EXIT
26+
27+
export PROJECT_ROOT_DIR=$(git rev-parse --show-toplevel)
28+
export COMPOSE_INTERACTIVE_NO_CLI=1
29+
30+
# Create Docker Compose configuration file
31+
cd ${PROJECT_ROOT_DIR}/infra/docker-compose/
32+
cp .env.sample .env
33+
34+
export FEAST_CORE_CONFIG=${FEAST_CORE_CONFIG:-databricks.yml}
35+
export FEAST_DATABRICKS_EMULATOR_IMAGE=${FEAST_DATABRICKS_EMULATOR_IMAGE:-gcr.io/kf-feast/feast-databricks-emulator}
36+
37+
# Build Databricks emulator image
38+
docker-compose -f docker-compose.databricks.yml build
39+
40+
# Start Docker Compose containers
41+
docker-compose $COMPOSE_ARGS up -d
42+
43+
# Get Jupyter container IP address
44+
export JUPYTER_DOCKER_CONTAINER_IP_ADDRESS=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' feast_jupyter_1)
45+
46+
# Print Jupyter container information
47+
docker logs feast_jupyter_1
48+
49+
# Wait for Jupyter Notebook Container to come online
50+
${PROJECT_ROOT_DIR}/infra/scripts/wait-for-it.sh ${JUPYTER_DOCKER_CONTAINER_IP_ADDRESS}:8888 --timeout=300
51+
52+
# Run e2e tests for Redis
53+
docker exec feast_jupyter_1 bash -c 'cd feast/tests/e2e/ && pytest -s basic-ingest-redis-serving.py --core_url core:6565 --serving_url=online-serving:6566 --databricks'

‎pom.xml‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,10 @@
3535
<module>core</module>
3636
<module>serving</module>
3737
<module>sdk/java</module>
38-
<module>docs/coverage/java</module>
38+
<module>spark/databricks-emulator</module>
39+
<module>spark/databricks-types</module>
40+
<module>spark/spark-historical-retriever-job</module>
41+
<module>spark/spark-ingestion-job</module>
3942
</modules>
4043

4144
<properties>
@@ -61,6 +64,8 @@
6164
<opencensus.version>0.21.0</opencensus.version>
6265
<!-- Force log4j2 to 2.11+ to support objectMessageAsJsonObject -->
6366
<log4jVersion>2.12.1</log4jVersion>
67+
<spark.version>2.4.5</spark.version>
68+
<scala.compat.version>2.11</scala.compat.version>
6469
</properties>
6570

6671
<organization>

0 commit comments

Comments
 (0)