Skip to content

Commit 2f79430

Browse files
jenoOvchijenoOvchi
authored
Merge branch 'feast-dev:master' into master
2 parents 6f1409b + 6589f15 commit 2f79430

259 files changed

Lines changed: 8534 additions & 3815 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/java_pr.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ jobs:
5454
integration-test:
5555
# all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes.
5656
if:
57-
(github.event.action == 'labeled' && (github.event.label.name == 'lgtm' || github.event.label.name == 'approved' || github.event.label.name == 'ok-to-test')) ||
57+
(github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'ok-to-test')) ||
5858
(github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved')))
5959
runs-on: ubuntu-latest
6060
needs: unit-test-java

‎.github/workflows/pr_integration_tests.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,9 @@ on:
1414

1515
jobs:
1616
build-docker-image:
17-
# all jobs MUST have this if check for 'ok-to-test' or 'approved' or 'lgtm' for security purposes.
17+
# all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes.
1818
if:
19-
(github.event.action == 'labeled' && (github.event.label.name == 'lgtm' || github.event.label.name == 'approved' || github.event.label.name == 'ok-to-test')) ||
19+
(github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'ok-to-test')) ||
2020
(github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved')))
2121
runs-on: ubuntu-latest
2222
steps:
@@ -72,9 +72,9 @@ jobs:
7272
outputs:
7373
DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }}
7474
integration-test-python:
75-
# all jobs MUST have this if check for 'ok-to-test' or 'approved' or 'lgtm' for security purposes.
75+
# all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes.
7676
if:
77-
(github.event.action == 'labeled' && (github.event.label.name == 'lgtm' || github.event.label.name == 'approved' || github.event.label.name == 'ok-to-test')) ||
77+
(github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'ok-to-test')) ||
7878
(github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved')))
7979
needs: build-docker-image
8080
runs-on: ${{ matrix.os }}

‎.github/workflows/publish.yml‎

Lines changed: 21 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,7 @@ jobs:
127127

128128
publish-python-sdk:
129129
runs-on: ubuntu-latest
130-
needs: [build-python-sdk, build-python-sdk-no-telemetry, build-python-sdk-macos-py310]
130+
needs: [verify-python-wheel]
131131
steps:
132132
- uses: actions/download-artifact@v2
133133
with:
@@ -139,65 +139,41 @@ jobs:
139139
password: ${{ secrets.PYPI_PASSWORD }}
140140

141141

142-
build-python-sdk:
143-
name: Build wheels on ${{ matrix.os }}
144-
runs-on: ${{ matrix.os }}
145-
strategy:
146-
matrix:
147-
os: [ ubuntu-latest, macos-10.15 ]
148-
142+
verify-python-wheel:
143+
runs-on: ubuntu-latest
144+
needs: [build-python-sdk, build-python-sdk-macos-py310]
149145
steps:
150-
- uses: actions/checkout@v2
151-
152-
- name: Build wheels
153-
uses: pypa/[email protected]
146+
- name: Setup Python
147+
id: setup-python
148+
uses: actions/setup-python@v2
154149
with:
155-
package-dir: sdk/python
156-
env:
157-
CIBW_BUILD: "cp3*_x86_64"
158-
CIBW_SKIP: "cp36-* *-musllinux_x86_64 cp310-macosx_x86_64"
159-
CIBW_ARCHS: "native"
160-
CIBW_ENVIRONMENT: >
161-
COMPILE_GO=True
162-
CIBW_BEFORE_ALL_LINUX: |
163-
yum install -y golang
164-
CIBW_BEFORE_ALL_MACOS: |
165-
curl -o python.pkg https://www.python.org/ftp/python/3.9.12/python-3.9.12-macosx10.9.pkg
166-
sudo installer -pkg python.pkg -target /
167-
CIBW_BEFORE_BUILD: |
168-
make install-protoc-dependencies
169-
make install-go-proto-dependencies
170-
make install-go-ci-dependencies
171-
172-
- uses: actions/upload-artifact@v2
150+
python-version: "3.8"
151+
architecture: x64
152+
- uses: actions/download-artifact@v2
173153
with:
174154
name: wheels
175-
path: ./wheelhouse/*.whl
176-
155+
path: dist
156+
- name: Install wheel
157+
run: pip install dist/*38*linux*.whl
158+
- name: Install sdist
159+
run: pip install dist/*tar.gz
177160

178-
build-python-sdk-no-telemetry:
179-
name: Build no telemetry wheels on ${{ matrix.os }}
161+
build-python-sdk:
162+
name: Build wheels on ${{ matrix.os }}
180163
runs-on: ${{ matrix.os }}
181164
strategy:
182165
matrix:
183166
os: [ ubuntu-latest, macos-10.15 ]
184-
needs: get-version
185167
steps:
186168
- uses: actions/checkout@v2
187-
- run: |
188-
cd sdk/python
189-
sed -i.bak 's/DEFAULT_FEAST_USAGE_VALUE = "True"/DEFAULT_FEAST_USAGE_VALUE = "False"/g' feast/constants.py
190-
sed -i.bak 's/NAME = "feast"/NAME = "feast-no-telemetry"/g' setup.py
191169
- name: Build wheels
192170
uses: pypa/[email protected]
193-
with:
194-
package-dir: sdk/python
195171
env:
196172
CIBW_BUILD: "cp3*_x86_64"
197173
CIBW_SKIP: "cp36-* *-musllinux_x86_64 cp310-macosx_x86_64"
198174
CIBW_ARCHS: "native"
199175
CIBW_ENVIRONMENT: >
200-
COMPILE_GO=True SETUPTOOLS_SCM_PRETEND_VERSION="${{ needs.get-version.outputs.version_without_prefix }}"
176+
COMPILE_GO=True
201177
CIBW_BEFORE_ALL_LINUX: |
202178
yum install -y golang
203179
CIBW_BEFORE_ALL_MACOS: |
@@ -213,6 +189,7 @@ jobs:
213189
name: wheels
214190
path: ./wheelhouse/*.whl
215191

192+
216193
build-python-sdk-macos-py310:
217194
runs-on: macos-10.15
218195
env:
@@ -233,13 +210,12 @@ jobs:
233210
make install-go-ci-dependencies
234211
- name: Build
235212
run: |
236-
cd sdk/python
237213
python3 setup.py sdist bdist_wheel
238214
239215
- uses: actions/upload-artifact@v2
240216
with:
241217
name: wheels
242-
path: sdk/python/dist/*
218+
path: dist/*
243219

244220

245221
publish-java-sdk:
@@ -294,6 +270,7 @@ jobs:
294270
working-directory: ./ui
295271
run: yarn build:lib
296272
- name: Publish UI package
273+
working-directory: ./ui
297274
run: npm publish
298275
env:
299276
NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}}

‎.gitignore‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@ coverage.xml
105105
.hypothesis/
106106
.pytest_cache/
107107
infra/scripts/*.conf
108-
go/cmd/server/logging/feature_repo/data/
108+
go/internal/test/feature_repo/data/
109109

110110
# Translations
111111
*.mo

‎.releaserc.js‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,8 @@ module.exports = {
5757
assets: [
5858
"CHANGELOG.md",
5959
"java/pom.xml",
60-
"infra/charts/**/*.*"
60+
"infra/charts/**/*.*",
61+
"ui/package.json"
6162
],
6263
message: "chore(release): release ${nextRelease.version}\n\n${nextRelease.notes}"
6364
}

‎CONTRIBUTING.md‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ pip install --upgrade pip
9191

9292
4. Install development dependencies for Feast Python SDK / CLI
9393
```sh
94-
pip install -e "sdk/python[dev]"
94+
pip install -e ".[dev]"
9595
```
9696

9797
### Code Style & Linting
@@ -177,6 +177,8 @@ The services with containerized replacements currently implemented are:
177177
- DynamoDB
178178
- Redis
179179
- Trino
180+
- HBase
181+
- Postgres
180182

181183
You can run `make test-python-integration-container` to run tests against the containerized versions of dependencies.
182184

‎MANIFEST.in‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
prune sdk/python/tests
2+
prune ui
3+
prune java/
4+
prune docs
5+
prune infra
6+
prune examples

‎Makefile‎

Lines changed: 42 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -37,24 +37,24 @@ build: protos build-java build-docker
3737
# Python SDK
3838

3939
install-python-ci-dependencies: install-go-proto-dependencies install-go-ci-dependencies
40-
cd sdk/python && python -m piptools sync requirements/py$(PYTHON)-ci-requirements.txt
41-
cd sdk/python && COMPILE_GO=true python setup.py develop
40+
python -m piptools sync sdk/python/requirements/py$(PYTHON)-ci-requirements.txt
41+
COMPILE_GO=true python setup.py develop
4242

4343
lock-python-ci-dependencies:
44-
cd sdk/python && python -m piptools compile -U --extra ci --output-file requirements/py$(PYTHON)-ci-requirements.txt
44+
python -m piptools compile -U --extra ci --output-file sdk/python/requirements/py$(PYTHON)-ci-requirements.txt
4545

4646
package-protos:
4747
cp -r ${ROOT_DIR}/protos ${ROOT_DIR}/sdk/python/feast/protos
4848

4949
compile-protos-python:
50-
cd sdk/python && python setup.py build_python_protos
50+
python setup.py build_python_protos
5151

5252
install-python:
53-
cd sdk/python && python -m piptools sync requirements/py$(PYTHON)-requirements.txt
54-
cd sdk/python && python setup.py develop
53+
python -m piptools sync sdk/python/requirements/py$(PYTHON)-requirements.txt
54+
python setup.py develop
5555

5656
lock-python-dependencies:
57-
cd sdk/python && python -m piptools compile -U --output-file requirements/py$(PYTHON)-requirements.txt
57+
python -m piptools compile -U --output-file sdk/python/requirements/py$(PYTHON)-requirements.txt
5858

5959
benchmark-python:
6060
FEAST_USAGE=False IS_TEST=True python -m pytest --integration --benchmark --benchmark-autosave --benchmark-save-data sdk/python/tests
@@ -72,7 +72,37 @@ test-python-integration-container:
7272
FEAST_USAGE=False IS_TEST=True FEAST_LOCAL_ONLINE_CONTAINER=True python -m pytest -n 8 --integration sdk/python/tests
7373

7474
test-python-universal-contrib:
75-
PYTHONPATH='.' FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.contrib_repo_configuration FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration --universal sdk/python/tests
75+
PYTHONPATH='.' \
76+
FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.contrib_repo_configuration \
77+
PYTEST_PLUGINS=feast.infra.offline_stores.contrib.trino_offline_store.tests \
78+
FEAST_USAGE=False IS_TEST=True \
79+
python -m pytest -n 8 --integration --universal \
80+
-k "not test_historical_retrieval_fails_on_validation and \
81+
not test_historical_retrieval_with_validation and \
82+
not test_historical_features_persisting and \
83+
not test_historical_retrieval_fails_on_validation and \
84+
not test_universal_cli and \
85+
not test_go_feature_server and \
86+
not test_feature_logging and \
87+
not test_universal_types" \
88+
sdk/python/tests
89+
90+
test-python-universal-postgres:
91+
PYTHONPATH='.' \
92+
FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.postgres_repo_configuration \
93+
PYTEST_PLUGINS=sdk.python.feast.infra.offline_stores.contrib.postgres_offline_store.tests \
94+
FEAST_USAGE=False \
95+
IS_TEST=True \
96+
python -m pytest -x --integration --universal \
97+
-k "not test_historical_retrieval_fails_on_validation and \
98+
not test_historical_retrieval_with_validation and \
99+
not test_historical_features_persisting and \
100+
not test_historical_retrieval_fails_on_validation and \
101+
not test_universal_cli and \
102+
not test_go_feature_server and \
103+
not test_feature_logging and \
104+
not test_universal_types" \
105+
sdk/python/tests
76106

77107
test-python-universal-local:
78108
FEAST_USAGE=False IS_TEST=True FEAST_IS_LOCAL_TEST=True python -m pytest -n 8 --integration --universal sdk/python/tests
@@ -151,21 +181,21 @@ install-protoc-dependencies:
151181
pip install grpcio-tools==1.44.0 mypy-protobuf==3.1.0
152182

153183
compile-protos-go: install-go-proto-dependencies install-protoc-dependencies
154-
cd sdk/python && python setup.py build_go_protos
184+
python setup.py build_go_protos
155185

156186
compile-go-lib: install-go-proto-dependencies install-go-ci-dependencies
157-
cd sdk/python && python setup.py build_go_lib
187+
COMPILE_GO=True python setup.py build_ext --inplace
158188

159189
# Needs feast package to setup the feature store
160190
test-go: compile-protos-go
161-
pip install -e "sdk/python[ci]"
191+
pip install -e ".[ci]"
162192
go test ./...
163193

164194
format-go:
165195
gofmt -s -w go/
166196

167197
lint-go: compile-protos-go
168-
go vet ./go/internal/feast ./go/cmd/server
198+
go vet ./go/internal/feast ./go/embedded
169199

170200
# Docker
171201

‎README.md‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -146,8 +146,8 @@ The list below contains the functionality that contributors are planning to deve
146146
* [x] [Parquet file source](https://docs.feast.dev/reference/data-sources/file)
147147
* [x] [Synapse source (community plugin)](https://github.com/Azure/feast-azure)
148148
* [x] [Hive (community plugin)](https://github.com/baineng/feast-hive)
149-
* [x] [Postgres (community plugin)](https://github.com/nossrannug/feast-postgres)
150-
* [x] [Spark (community plugin)](https://docs.feast.dev/reference/data-sources/spark)
149+
* [x] [Postgres (contrib plugin)](https://docs.feast.dev/reference/data-sources/postgres)
150+
* [x] [Spark (contrib plugin)](https://docs.feast.dev/reference/data-sources/spark)
151151
* [x] Kafka / Kinesis sources (via [push support into the online store](https://docs.feast.dev/reference/data-sources/push))
152152
* [ ] HTTP source
153153
* **Offline Stores**
@@ -156,9 +156,9 @@ The list below contains the functionality that contributors are planning to deve
156156
* [x] [BigQuery](https://docs.feast.dev/reference/offline-stores/bigquery)
157157
* [x] [Synapse (community plugin)](https://github.com/Azure/feast-azure)
158158
* [x] [Hive (community plugin)](https://github.com/baineng/feast-hive)
159-
* [x] [Postgres (community plugin)](https://github.com/nossrannug/feast-postgres)
160-
* [x] [Trino (community plugin)](https://github.com/Shopify/feast-trino)
161-
* [x] [Spark (community plugin)](https://docs.feast.dev/reference/offline-stores/spark)
159+
* [x] [Postgres (contrib plugin)](https://docs.feast.dev/reference/offline-stores/postgres)
160+
* [x] [Trino (contrib plugin)](https://github.com/Shopify/feast-trino)
161+
* [x] [Spark (contrib plugin)](https://docs.feast.dev/reference/offline-stores/spark)
162162
* [x] [In-memory / Pandas](https://docs.feast.dev/reference/offline-stores/file)
163163
* [x] [Custom offline store support](https://docs.feast.dev/how-to-guides/adding-a-new-offline-store)
164164
* **Online Stores**
@@ -167,7 +167,7 @@ The list below contains the functionality that contributors are planning to deve
167167
* [x] [Datastore](https://docs.feast.dev/reference/online-stores/datastore)
168168
* [x] [SQLite](https://docs.feast.dev/reference/online-stores/sqlite)
169169
* [x] [Azure Cache for Redis (community plugin)](https://github.com/Azure/feast-azure)
170-
* [x] [Postgres (community plugin)](https://github.com/nossrannug/feast-postgres)
170+
* [x] [Postgres (contrib plugin)](https://docs.feast.dev/reference/online-stores/postgres)
171171
* [x] [Custom online store support](https://docs.feast.dev/how-to-guides/adding-support-for-a-new-online-store)
172172
* [ ] Bigtable (in progress)
173173
* [ ] Cassandra

‎docs/SUMMARY.md‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,19 +60,22 @@
6060
* [Snowflake](reference/data-sources/snowflake.md)
6161
* [BigQuery](reference/data-sources/bigquery.md)
6262
* [Redshift](reference/data-sources/redshift.md)
63-
* [Spark](reference/data-sources/spark.md)
6463
* [Push](reference/data-sources/push.md)
64+
* [Spark (contrib)](reference/data-sources/spark.md)
65+
* [PostgreSQL (contrib)](reference/data-sources/postgres.md)
6566
* [Offline stores](reference/offline-stores/README.md)
6667
* [File](reference/offline-stores/file.md)
6768
* [Snowflake](reference/offline-stores/snowflake.md)
6869
* [BigQuery](reference/offline-stores/bigquery.md)
6970
* [Redshift](reference/offline-stores/redshift.md)
70-
* [Spark](reference/offline-stores/spark.md)
71+
* [Spark (contrib)](reference/offline-stores/spark.md)
72+
* [PostgreSQL (contrib)](reference/offline-stores/postgres.md)
7173
* [Online stores](reference/online-stores/README.md)
7274
* [SQLite](reference/online-stores/sqlite.md)
7375
* [Redis](reference/online-stores/redis.md)
7476
* [Datastore](reference/online-stores/datastore.md)
7577
* [DynamoDB](reference/online-stores/dynamodb.md)
78+
* [PostgreSQL (contrib)](reference/online-stores/postgres.md)
7679
* [Providers](reference/providers/README.md)
7780
* [Local](reference/providers/local.md)
7881
* [Google Cloud Platform](reference/providers/google-cloud-platform.md)

0 commit comments

Comments
 (0)