Skip to content

Commit 318a2b8

Browse files
tokokotokoko
authored
feat: Pull duckdb from contribs, add to CI (#4059)
* pull duckdb/ibis from contrib folder Signed-off-by: tokoko <[email protected]> * fix linter failures Signed-off-by: tokoko <[email protected]> --------- Signed-off-by: tokoko <[email protected]>
1 parent 1ba65b4 commit 318a2b8

13 files changed

Lines changed: 80 additions & 52 deletions

File tree

‎Makefile‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -176,18 +176,6 @@ test-python-universal-athena:
176176
not s3_registry and \
177177
not test_snowflake" \
178178
sdk/python/tests
179-
180-
test-python-universal-duckdb:
181-
PYTHONPATH='.' \
182-
FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.duckdb_repo_configuration \
183-
python -m pytest -n 8 --integration \
184-
-k "not test_nullable_online_store and \
185-
not gcs_registry and \
186-
not s3_registry and \
187-
not test_snowflake and \
188-
not bigquery and \
189-
not test_spark_materialization_consistency" \
190-
sdk/python/tests
191179

192180
test-python-universal-postgres-offline:
193181
PYTHONPATH='.' \

‎docs/project/development-guide.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -187,7 +187,7 @@ make lint-python
187187
### Unit Tests
188188
Unit tests (`pytest`) for the Feast Python SDK / CLI can run as follows:
189189
```sh
190-
make test-python
190+
make test-python-unit
191191
```
192192

193193
> :warning: Local configuration can interfere with Unit tests and cause them to fail:

‎environment-setup.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,5 +19,5 @@ make install-python-ci-dependencies PYTHON=3.9
1919
4. start the docker daemon
2020
5. run unit tests:
2121
```bash
22-
make test-python
22+
make test-python-unit
2323
```

‎sdk/python/feast/infra/offline_stores/contrib/duckdb_offline_store/__init__.py‎

Whitespace-only changes.

‎sdk/python/feast/infra/offline_stores/contrib/duckdb_repo_configuration.py‎

Lines changed: 0 additions & 19 deletions
This file was deleted.

‎sdk/python/feast/infra/offline_stores/contrib/ibis_offline_store/__init__.py‎

Whitespace-only changes.

sdk/python/feast/infra/offline_stores/contrib/duckdb_offline_store/duckdb.py renamed to sdk/python/feast/infra/offline_stores/duckdb.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import ibis
22
from pydantic import StrictStr
33

4-
from feast.infra.offline_stores.contrib.ibis_offline_store.ibis import IbisOfflineStore
4+
from feast.infra.offline_stores.ibis import IbisOfflineStore
55
from feast.repo_config import FeastConfigBaseModel
66

77

sdk/python/feast/infra/offline_stores/contrib/ibis_offline_store/ibis.py renamed to sdk/python/feast/infra/offline_stores/ibis.py

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,9 @@ def get_historical_features(
134134
entity_table, feature_views, event_timestamp_col
135135
)
136136

137-
def read_fv(feature_view, feature_refs, full_feature_names):
137+
def read_fv(
138+
feature_view: FeatureView, feature_refs: List[str], full_feature_names: bool
139+
) -> Tuple:
138140
fv_table: Table = ibis.read_parquet(feature_view.batch_source.name)
139141

140142
for old_name, new_name in feature_view.batch_source.field_mapping.items():
@@ -175,6 +177,7 @@ def read_fv(feature_view, feature_refs, full_feature_names):
175177
return (
176178
fv_table,
177179
feature_view.batch_source.timestamp_field,
180+
feature_view.batch_source.created_timestamp_column,
178181
feature_view.projection.join_key_map
179182
or {e.name: e.name for e in feature_view.entity_columns},
180183
feature_refs,
@@ -351,12 +354,19 @@ def metadata(self) -> Optional[RetrievalMetadata]:
351354

352355
def point_in_time_join(
353356
entity_table: Table,
354-
feature_tables: List[Tuple[Table, str, Dict[str, str], List[str], timedelta]],
357+
feature_tables: List[Tuple[Table, str, str, Dict[str, str], List[str], timedelta]],
355358
event_timestamp_col="event_timestamp",
356359
):
357360
# TODO handle ttl
358361
all_entities = [event_timestamp_col]
359-
for feature_table, timestamp_field, join_key_map, _, _ in feature_tables:
362+
for (
363+
feature_table,
364+
timestamp_field,
365+
created_timestamp_field,
366+
join_key_map,
367+
_,
368+
_,
369+
) in feature_tables:
360370
all_entities.extend(join_key_map.values())
361371

362372
r = ibis.literal("")
@@ -371,6 +381,7 @@ def point_in_time_join(
371381
for (
372382
feature_table,
373383
timestamp_field,
384+
created_timestamp_field,
374385
join_key_map,
375386
feature_refs,
376387
ttl,
@@ -395,9 +406,13 @@ def point_in_time_join(
395406

396407
feature_table = feature_table.drop(s.endswith("_y"))
397408

409+
order_by_fields = [ibis.desc(feature_table[timestamp_field])]
410+
if created_timestamp_field:
411+
order_by_fields.append(ibis.desc(feature_table[created_timestamp_field]))
412+
398413
feature_table = (
399414
feature_table.group_by(by="entity_row_id")
400-
.order_by(ibis.desc(feature_table[timestamp_field]))
415+
.order_by(order_by_fields)
401416
.mutate(rn=ibis.row_number())
402417
)
403418

‎sdk/python/feast/repo_config.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@
7575
"postgres": "feast.infra.offline_stores.contrib.postgres_offline_store.postgres.PostgreSQLOfflineStore",
7676
"athena": "feast.infra.offline_stores.contrib.athena_offline_store.athena.AthenaOfflineStore",
7777
"mssql": "feast.infra.offline_stores.contrib.mssql_offline_store.mssql.MsSqlServerOfflineStore",
78-
"duckdb": "feast.infra.offline_stores.contrib.duckdb_offline_store.duckdb.DuckDBOfflineStore",
78+
"duckdb": "feast.infra.offline_stores.duckdb.DuckDBOfflineStore",
7979
}
8080

8181
FEATURE_SERVER_CONFIG_CLASS_FOR_TYPE = {

‎sdk/python/tests/integration/feature_repos/repo_configuration.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
BigQueryDataSourceCreator,
3232
)
3333
from tests.integration.feature_repos.universal.data_sources.file import (
34+
DuckDBDataSourceCreator,
3435
FileDataSourceCreator,
3536
)
3637
from tests.integration.feature_repos.universal.data_sources.redshift import (
@@ -108,6 +109,7 @@
108109

109110
AVAILABLE_OFFLINE_STORES: List[Tuple[str, Type[DataSourceCreator]]] = [
110111
("local", FileDataSourceCreator),
112+
("local", DuckDBDataSourceCreator),
111113
]
112114

113115
AVAILABLE_ONLINE_STORES: Dict[

0 commit comments

Comments
 (0)