Skip to content

Commit eb095f3

Browse files
MattDelacMattDelac
authored
chore: Add Trino Offline Store in the contrib folder (#2517)
* Add Trino Offline Store in the contrib folder Signed-off-by: Matt Delacour <[email protected]> * Assess comments Signed-off-by: Matt Delacour <[email protected]> * Assess comments of Tomas Signed-off-by: Matt Delacour <[email protected]> Co-authored-by: Matt Delacour <[email protected]>
1 parent d2fb048 commit eb095f3

21 files changed

Lines changed: 1455 additions & 3 deletions

File tree

‎Makefile‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ OS := linux
2020
ifeq ($(shell uname -s), Darwin)
2121
OS = osx
2222
endif
23+
TRINO_VERSION ?= 376
2324

2425
# General
2526

@@ -121,6 +122,18 @@ build-java:
121122
build-java-no-tests:
122123
${MVN} --no-transfer-progress -Dmaven.javadoc.skip=true -Dgpg.skip -DskipUTs=true -DskipITs=true -Drevision=${REVISION} clean package
123124

125+
# Trino plugin
126+
127+
start-trino-locally:
128+
cd ${ROOT_DIR}; docker run --detach --rm -p 8080:8080 --name trino -v ${ROOT_DIR}/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/test_config/properties/:/etc/catalog/:ro trinodb/trino:${TRINO_VERSION}
129+
sleep 15
130+
131+
test-trino-plugin-locally:
132+
cd ${ROOT_DIR}/sdk/python; FULL_REPO_CONFIGS_MODULE=feast.infra.offline_stores.contrib.trino_offline_store.test_config.manual_tests FEAST_USAGE=False IS_TEST=True python -m pytest --integration --universal tests/
133+
134+
kill-trino-locally:
135+
cd ${ROOT_DIR}; docker stop trino
136+
124137
# Go SDK & embedded
125138

126139
install-go-proto-dependencies:

‎go.mod‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,21 +25,24 @@ require (
2525
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
2626
github.com/goccy/go-json v0.7.10 // indirect
2727
github.com/golang/snappy v0.0.4 // indirect
28+
github.com/gonuts/commander v0.1.0 // indirect
29+
github.com/gonuts/flag v0.1.0 // indirect
2830
github.com/google/flatbuffers v2.0.5+incompatible // indirect
2931
github.com/klauspost/asmfmt v1.3.1 // indirect
3032
github.com/klauspost/compress v1.15.1 // indirect
3133
github.com/klauspost/cpuid/v2 v2.0.9 // indirect
3234
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
3335
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
3436
github.com/pierrec/lz4/v4 v4.1.12 // indirect
37+
github.com/pkg/errors v0.9.1 // indirect
3538
github.com/pmezard/go-difflib v1.0.0 // indirect
3639
github.com/zeebo/xxh3 v1.0.1 // indirect
3740
golang.org/x/exp v0.0.0-20211216164055-b2b84827b756 // indirect
38-
golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 // indirect
41+
golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
3942
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
4043
golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27 // indirect
4144
golang.org/x/text v0.3.7 // indirect
42-
golang.org/x/tools v0.1.9 // indirect
45+
golang.org/x/tools v0.1.10 // indirect
4346
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
4447
google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350 // indirect
4548
gopkg.in/yaml.v2 v2.4.0 // indirect

‎go.sum‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,9 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu
143143
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
144144
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
145145
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
146+
github.com/gonuts/commander v0.1.0 h1:EcDTiVw9oAVORFjQOEOuHQqcl6OXMyTgELocTq6zJ0I=
146147
github.com/gonuts/commander v0.1.0/go.mod h1:qkb5mSlcWodYgo7vs8ulLnXhfinhZsZcm6+H/z1JjgY=
148+
github.com/gonuts/flag v0.1.0 h1:fqMv/MZ+oNGu0i9gp0/IQ/ZaPIDoAZBOBaJoV7viCWM=
147149
github.com/gonuts/flag v0.1.0/go.mod h1:ZTmTGtrSPejTo/SRNhCqwLTmiAgyBdCkLYhHrAoBdz4=
148150
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
149151
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -294,6 +296,7 @@ github.com/pierrec/lz4/v4 v4.1.12 h1:44l88ehTZAUGW4VlO1QC4zkilL99M6Y9MXNwEs0uzP8
294296
github.com/pierrec/lz4/v4 v4.1.12/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
295297
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
296298
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
299+
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
297300
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
298301
github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
299302
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -426,6 +429,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
426429
golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
427430
golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 h1:LQmS1nU0twXLA96Kt7U9qtHJEbBk3z6Q0V4UXjZkpr4=
428431
golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
432+
golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o=
433+
golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
429434
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
430435
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
431436
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -531,6 +536,8 @@ golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
531536
golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
532537
golang.org/x/tools v0.1.9 h1:j9KsMiaP1c3B0OTQGth0/k+miLGTgLsAFUCrF2vLcF8=
533538
golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
539+
golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20=
540+
golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
534541
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
535542
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
536543
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

‎protos/feast/core/DataSource.proto‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ message DataSource {
4545
CUSTOM_SOURCE = 6;
4646
REQUEST_SOURCE = 7;
4747
PUSH_SOURCE = 9;
48+
BATCH_TRINO = 10;
4849
}
4950

5051
// Unique name of data source within the project
@@ -105,6 +106,16 @@ message DataSource {
105106
string query = 2;
106107
}
107108

109+
// Defines options for DataSource that sources features from a Trino Query
110+
message TrinoOptions {
111+
// Full table reference in the form of [project:dataset.table]
112+
string table = 1;
113+
114+
// SQL query that returns a table containing feature data. Must contain an event_timestamp column, and respective
115+
// entity columns
116+
string query = 2;
117+
}
118+
108119
// Defines options for DataSource that sources features from Kafka messages.
109120
// Each message should be a Protobuf that can be decoded with the generated
110121
// Java Protobuf class at the given class path
@@ -204,5 +215,6 @@ message DataSource {
204215
CustomSourceOptions custom_options = 16;
205216
SnowflakeOptions snowflake_options = 19;
206217
PushOptions push_options = 22;
218+
TrinoOptions trino_options = 30;
207219
}
208220
}

‎protos/feast/core/SavedDataset.proto‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ message SavedDatasetStorage {
5656
DataSource.BigQueryOptions bigquery_storage = 5;
5757
DataSource.RedshiftOptions redshift_storage = 6;
5858
DataSource.SnowflakeOptions snowflake_storage = 7;
59+
DataSource.TrinoOptions trino_storage = 8;
5960
}
6061
}
6162

‎sdk/python/feast/data_source.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ class SourceType(enum.Enum):
3636
BATCH_BIGQUERY = 2
3737
STREAM_KAFKA = 3
3838
STREAM_KINESIS = 4
39+
BATCH_TRINO = 5
3940

4041

4142
class KafkaOptions:
@@ -141,6 +142,7 @@ def to_proto(self) -> DataSourceProto.KinesisOptions:
141142
DataSourceProto.SourceType.BATCH_BIGQUERY: "feast.infra.offline_stores.bigquery_source.BigQuerySource",
142143
DataSourceProto.SourceType.BATCH_REDSHIFT: "feast.infra.offline_stores.redshift_source.RedshiftSource",
143144
DataSourceProto.SourceType.BATCH_SNOWFLAKE: "feast.infra.offline_stores.snowflake_source.SnowflakeSource",
145+
DataSourceProto.SourceType.BATCH_TRINO: "feast.infra.offline_stores.contrib.trino_offline_store.trino_source.TrinoSource",
144146
DataSourceProto.SourceType.STREAM_KAFKA: "feast.data_source.KafkaSource",
145147
DataSourceProto.SourceType.STREAM_KINESIS: "feast.data_source.KinesisSource",
146148
DataSourceProto.SourceType.REQUEST_SOURCE: "feast.data_source.RequestSource",

‎sdk/python/feast/infra/offline_stores/contrib/contrib_repo_configuration.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,11 @@
44
from tests.integration.feature_repos.universal.data_sources.spark_data_source_creator import (
55
SparkDataSourceCreator,
66
)
7+
from tests.integration.feature_repos.universal.data_sources.trino import (
8+
TrinoSourceCreator,
9+
)
710

811
FULL_REPO_CONFIGS = [
9-
IntegrationTestRepoConfig(offline_store_creator=SparkDataSourceCreator)
12+
IntegrationTestRepoConfig(offline_store_creator=SparkDataSourceCreator),
13+
IntegrationTestRepoConfig(offline_store_creator=TrinoSourceCreator),
1014
]

‎sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/README.md‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# Trino offline store
2+
This is a walkthrough to talk about how we can test the trino plugin locally
3+
4+
## Start Trino in a docker container
5+
```sh
6+
make start-trino-locally
7+
```
8+
9+
Normally this should start a docker container named `trino` listening on the port 8080.
10+
You can see the docker command executed by looking at the `Makefile` at the root.
11+
12+
You can look at the queries being executed during the tests with the [local cluster UI](http://0.0.0.0:8080/ui/#) running.
13+
This can be helpful to debug the Trino plugin while executing tests.
14+
15+
## Run the universal suites locally
16+
```sh
17+
make test-trino-plugin-locally
18+
```
19+
20+
## Kill the local Trino container
21+
```sh
22+
make kill-trino-locally
23+
```
24+
25+
You can always look at the running containers and kill the ones you don't need anymore
26+
```sh
27+
docker ps
28+
docker stop {NAME/SHA OF THE CONTAINER}
29+
```
30+
feast.sdk.python.feast.infra.offline_stores.contrib.trino_offline_store

‎sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/__init__.py‎

Whitespace-only changes.

‎sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/connectors/__init__.py‎

Whitespace-only changes.

0 commit comments

Comments
 (0)