Skip to content

Commit 211828e

Browse files
authored
CI/CD pipeline on SparkRunner (feast-dev#36)
Closes KE-687, KE-688 Deploys CI/CD pipeline on SparkRunner, and run tests for Online serving. Developed a smaller representative subset of tests, to limit resource consumption on Databricks. NB: The Spark job also produces output Delta files in Azure Storage. That functionality is currently not integration tested, as that will be achieved with the Historical Serving feature. NB: an Online Retriever must be connected for Feast to be spun up, but it's implemented as null until we integrate the real retriever.
1 parent dad9ffe commit 211828e

13 files changed

Lines changed: 422 additions & 25 deletions

File tree

‎.github/workflows/docker_build_test_push.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -195,7 +195,7 @@ jobs:
195195
working-directory: infra/terraform/app
196196
env:
197197
# capture debugging info, see https://www.terraform.io/docs/internals/debugging.html
198-
TF_LOG: debug
198+
TF_LOG: TRACE
199199

200200
- name: Cancel any active Databricks runs
201201
run: |
@@ -226,7 +226,7 @@ jobs:
226226
set -ex
227227
kubectl exec "$E2E_TEST_POD" -- bash -c "
228228
cd /feast/tests/e2e
229-
pytest -x -rA -s basic-ingest-redis-serving.py --core_url feast-core:6565 --serving_url=feast-serving-feast-online-serving:6566
229+
pytest -x -rA -s databricks-tests.py --core_url feast-core:6565 --serving_url=feast-serving-feast-online-serving:6566
230230
"
231231
232232
- name: Run ingestion tests for FF Data Science scenarios

‎core/src/main/java/feast/core/job/databricks/DatabricksJobManager.java‎

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,10 @@
3535
import java.net.http.HttpResponse;
3636
import java.nio.charset.StandardCharsets;
3737
import java.util.*;
38+
import java.util.function.Consumer;
3839
import java.util.stream.Collectors;
3940
import lombok.extern.slf4j.Slf4j;
41+
import org.apache.commons.lang3.StringUtils;
4042
import org.apache.http.HttpException;
4143
import org.apache.http.HttpStatus;
4244

@@ -297,13 +299,14 @@ private JobsCreateRequest getJobRequest(String jobName, List<String> params) {
297299
.map(s -> s.strip().split("\\s+", 2))
298300
.collect(Collectors.toMap(s -> s[0], s -> s[1]));
299301

300-
NewCluster newCluster =
302+
NewCluster.Builder newCluster =
301303
NewCluster.builder()
302304
.setNumWorkers(newClusterConfigOptions.getNumWorkers())
303-
.setNodeTypeId(newClusterConfigOptions.getNodeTypeId())
304305
.setSparkVersion(newClusterConfigOptions.getSparkVersion())
305-
.setSparkConf(sparkConf)
306-
.build();
306+
.setSparkConf(sparkConf);
307+
308+
ifPresent(newClusterConfigOptions.getNodeTypeId(), newCluster::setNodeTypeId);
309+
ifPresent(newClusterConfigOptions.getInstancePoolId(), newCluster::setInstancePoolId);
307310

308311
Library library = Library.builder().setJar(jarFile).build();
309312
List<Library> libraries = Collections.singletonList(library);
@@ -320,13 +323,21 @@ private JobsCreateRequest getJobRequest(String jobName, List<String> params) {
320323
.setMaxRetries(maxRetries)
321324
.setTimeoutSeconds(timeoutSeconds)
322325
.setName(jobName)
323-
.setNewCluster(newCluster)
326+
.setNewCluster(newCluster.build())
324327
.setSparkJarTask(sparkJarTask)
325328
.build();
326329

327330
return createRequest;
328331
}
329332

333+
private static void ifPresent(String nodeTypeId, Consumer<? super String> action) {
334+
asOptional(nodeTypeId).ifPresent(action);
335+
}
336+
337+
private static Optional<String> asOptional(String field) {
338+
return Optional.ofNullable(StringUtils.trimToNull(field));
339+
}
340+
330341
private JobStatus waitForJobToStart(Job job) {
331342
JobStatus jobStatus = waitForJobStatus(job, Set.of(JobStatus.RUNNING));
332343

‎core/src/main/java/feast/core/model/Store.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import feast.proto.core.StoreProto.Store.BigQueryConfig;
2222
import feast.proto.core.StoreProto.Store.Builder;
2323
import feast.proto.core.StoreProto.Store.CassandraConfig;
24+
import feast.proto.core.StoreProto.Store.DeltaConfig;
2425
import feast.proto.core.StoreProto.Store.RedisClusterConfig;
2526
import feast.proto.core.StoreProto.Store.RedisConfig;
2627
import feast.proto.core.StoreProto.Store.StoreType;
@@ -86,6 +87,9 @@ public static Store fromProto(StoreProto.Store storeProto) throws IllegalArgumen
8687
case REDIS_CLUSTER:
8788
config = storeProto.getRedisClusterConfig().toByteArray();
8889
break;
90+
case DELTA:
91+
config = storeProto.getDeltaConfig().toByteArray();
92+
break;
8993
default:
9094
throw new IllegalArgumentException("Invalid store provided");
9195
}
@@ -113,6 +117,9 @@ public StoreProto.Store toProto() throws InvalidProtocolBufferException {
113117
case REDIS_CLUSTER:
114118
RedisClusterConfig redisClusterConfig = RedisClusterConfig.parseFrom(config);
115119
return storeProtoBuilder.setRedisClusterConfig(redisClusterConfig).build();
120+
case DELTA:
121+
DeltaConfig deltaConfig = DeltaConfig.parseFrom(config);
122+
return storeProtoBuilder.setDeltaConfig(deltaConfig).build();
116123
default:
117124
throw new InvalidProtocolBufferException("Invalid store set");
118125
}

‎infra/terraform/app/main.tf‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ locals {
1313
databricks_secret_datalake_key = "azure_account_key"
1414
databricks_dbfs_jar_folder = "dbfs:/feast/run${var.run_number}"
1515
databricks_spark_version = "6.6.x-scala2.11"
16+
databricks_vm_type = "Standard_D3_v2"
17+
databricks_instance_pool_name = "Feast"
1618
}
1719

1820
resource "azurerm_postgresql_database" "feast" {
@@ -28,6 +30,14 @@ resource "databricks_token" "feast" {
2830
comment = "Token used by CI/CD pipeline"
2931
}
3032

33+
resource "databricks_instance_pool" "feast" {
34+
instance_pool_name = local.databricks_instance_pool_name
35+
min_idle_instances = 2
36+
max_capacity = 6
37+
node_type_id = local.databricks_vm_type
38+
idle_instance_autotermination_minutes = 60
39+
}
40+
3141
resource "databricks_secret_scope" "feast" {
3242
name = local.databricks_secret_scope
3343
initial_manage_principal = "users"
@@ -150,7 +160,7 @@ feast-core:
150160
polling_interval_milliseconds: 5000
151161
# databricks job can take several minutes to start (on new clusters)
152162
job_update_timeout_seconds: 1200
153-
active_runner: direct
163+
active_runner: databricks
154164
runners:
155165
- name: direct
156166
type: DirectRunner
@@ -164,8 +174,8 @@ feast-core:
164174
maxRetries: 0
165175
timeoutSeconds: 1200
166176
newCluster:
167-
nodeTypeId: "Standard_D3_v2"
168177
sparkVersion: "${local.databricks_spark_version}"
178+
instancePoolId: "${databricks_instance_pool.feast.id}"
169179
numWorkers: 1
170180
sparkConf: |
171181
fs.azure.account.key.${var.datalake_name}.dfs.core.windows.net {{secrets/${local.databricks_secret_scope}/${local.databricks_secret_datalake_key}}}
@@ -240,8 +250,7 @@ feast-online-serving:
240250
project: "*"
241251
242252
feast-batch-serving:
243-
# TODO enable batch serving
244-
enabled: false
253+
enabled: true
245254
image:
246255
repository: ${var.feast_serving_image_repository}
247256
tag: ${var.feast_version}
@@ -256,6 +265,7 @@ feast-batch-serving:
256265
application-override.yaml:
257266
feast:
258267
core-host: "${var.feast_core_vnet_ip}"
268+
active_store: delta
259269
stores:
260270
- name: delta
261271
type: DELTA

‎protos/feast/core/Runner.proto‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,8 @@ message DatabricksRunnerConfigOptions {
8686
int32 numWorkers = 3;
8787

8888
string sparkConf = 4;
89+
90+
string instancePoolId = 5;
8991
}
9092

9193
/* Databricks host */

‎serving/src/main/java/feast/serving/config/FeastProperties.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -262,6 +262,11 @@ public StoreProto.Store toProto()
262262
StoreProto.Store.CassandraConfig.newBuilder();
263263
JsonFormat.parser().merge(jsonWriter.writeValueAsString(config), cassandraConfig);
264264
return storeProtoBuilder.setCassandraConfig(cassandraConfig.build()).build();
265+
case DELTA:
266+
StoreProto.Store.DeltaConfig.Builder deltaConfig =
267+
StoreProto.Store.DeltaConfig.newBuilder();
268+
JsonFormat.parser().merge(jsonWriter.writeValueAsString(config), deltaConfig);
269+
return storeProtoBuilder.setDeltaConfig(deltaConfig.build()).build();
265270
default:
266271
throw new InvalidProtocolBufferException("Invalid store set");
267272
}

‎serving/src/main/java/feast/serving/config/ServingServiceConfig.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,10 @@ public ServingService servingService(
7070
HistoricalRetriever bqRetriever = BigQueryHistoricalRetriever.create(config);
7171
servingService = new HistoricalServingService(bqRetriever, specService, jobService);
7272
break;
73+
case DELTA:
74+
OnlineRetriever deltaRetriever = null;
75+
servingService = new OnlineServingService(deltaRetriever, specService, tracer);
76+
break;
7377
case CASSANDRA:
7478
case UNRECOGNIZED:
7579
case INVALID:

‎spark/databricks-types/src/main/java/feast/databricks/types/NewCluster.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,11 +34,14 @@ public abstract class NewCluster {
3434
public abstract String sparkVersion();
3535

3636
@JsonProperty("node_type_id")
37-
public abstract String nodeTypeId();
37+
public abstract Optional<String> nodeTypeId();
3838

3939
@JsonProperty("spark_conf")
4040
public abstract Optional<Map<String, String>> sparkConf();
4141

42+
@JsonProperty("instance_pool_id")
43+
public abstract Optional<String> instancePoolId();
44+
4245
public static Builder builder() {
4346
return new AutoValue_NewCluster.Builder();
4447
}
@@ -57,6 +60,9 @@ public abstract static class Builder {
5760
@JsonProperty("spark_conf")
5861
public abstract Builder setSparkConf(Map<String, String> value);
5962

63+
@JsonProperty("instance_pool_id")
64+
public abstract Builder setInstancePoolId(String value);
65+
6066
public abstract NewCluster build();
6167
}
6268
}

‎spark/spark-ingestion-job/src/main/java/feast/spark/ingestion/SparkIngestion.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,7 @@ public StreamingQuery createQuery() {
119119
.readStream()
120120
.format("kafka")
121121
.option("kafka.bootstrap.servers", kafkaConfig.getBootstrapServers())
122+
.option("startingOffsets", "earliest")
122123
.option("subscribe", kafkaConfig.getTopic())
123124
.load();
124125

@@ -164,7 +165,7 @@ public StreamingQuery createQuery() {
164165
})
165166
.collect(Collectors.toList());
166167

167-
// Start running the query that writes the data to Delta Lake
168+
// Start running the query that writes the data to sink
168169
return input
169170
.select("value")
170171
.map(

‎spark/spark-ingestion-job/src/main/java/feast/spark/ingestion/delta/SparkDeltaSink.java‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,6 @@
2424
import feast.spark.ingestion.SparkSink;
2525
import io.delta.tables.DeltaTable;
2626
import java.io.Serializable;
27-
import java.nio.file.Path;
28-
import java.nio.file.Paths;
2927
import java.util.ArrayList;
3028
import java.util.Collections;
3129
import java.util.HashMap;
@@ -74,7 +72,7 @@ public VoidFunction2<Dataset<byte[]>, Long> configure() {
7472
log.info("Table: {} schema: {}", spec.getKey(), schema);
7573

7674
// Initialize Delta table if needed
77-
Path deltaTablePath = getDeltaTablePath(deltaPath, spec.getValue());
75+
String deltaTablePath = getDeltaTablePath(deltaPath, spec.getValue());
7876
spark
7977
.createDataFrame(Collections.emptyList(), schema)
8078
.write()
@@ -158,10 +156,10 @@ private void upsertToDelta(
158156
}
159157
}
160158

161-
public static Path getDeltaTablePath(String deltaPath, FeatureSetSpec featureSetSpec) {
162-
return Paths.get(deltaPath)
163-
.resolve(featureSetSpec.getProject())
164-
.resolve(SpecUtil.getFeatureSetReference(featureSetSpec));
159+
public static String getDeltaTablePath(String deltaPath, FeatureSetSpec featureSetSpec) {
160+
return String.format(
161+
"%s/%s/%s",
162+
deltaPath, featureSetSpec.getProject(), SpecUtil.getFeatureSetReference(featureSetSpec));
165163
}
166164

167165
@SuppressWarnings("serial")
@@ -171,7 +169,7 @@ private static class FeatureSetInfo implements Serializable {
171169
private final StructType schema;
172170
private final String tablePath;
173171

174-
private FeatureSetInfo(String key, FeatureSetSpec spec, StructType schema, Path tablePath) {
172+
private FeatureSetInfo(String key, FeatureSetSpec spec, StructType schema, String tablePath) {
175173
this.key = key;
176174
this.spec = spec;
177175
this.schema = schema;

0 commit comments

Comments
 (0)