Skip to content

Commit ce35835

Browse files
achalsachals
authored
fix: Remove date partition column field from datasources that don't s… (#2478)
* fix: Remove date partition column field from datasources that don't support them Signed-off-by: Achal Shah <[email protected]> * remove for file sources as well Signed-off-by: Achal Shah <[email protected]> * remove other references Signed-off-by: Achal Shah <[email protected]> * remove other references Signed-off-by: Achal Shah <[email protected]> * reference to removal Signed-off-by: Achal Shah <[email protected]> * reference to removal Signed-off-by: Achal Shah <[email protected]> * reorder Signed-off-by: Achal Shah <[email protected]> * remove more: Signed-off-by: Achal Shah <[email protected]> * docs Signed-off-by: Achal Shah <[email protected]> * comment Signed-off-by: Achal Shah <[email protected]> * cr Signed-off-by: Achal Shah <[email protected]>
1 parent 495b435 commit ce35835

11 files changed

Lines changed: 68 additions & 48 deletions

File tree

‎sdk/python/feast/infra/offline_stores/bigquery_source.py‎

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ def __init__(
2121
table_ref: Optional[str] = None,
2222
created_timestamp_column: Optional[str] = "",
2323
field_mapping: Optional[Dict[str, str]] = None,
24-
date_partition_column: Optional[str] = "",
24+
date_partition_column: Optional[str] = None,
2525
query: Optional[str] = None,
2626
name: Optional[str] = None,
2727
description: Optional[str] = "",
@@ -37,7 +37,7 @@ def __init__(
3737
created_timestamp_column (optional): Timestamp column when row was created, used for deduplicating rows.
3838
field_mapping: A dictionary mapping of column names in this data source to feature names in a feature table
3939
or view. Only used for feature columns, not entities or timestamp columns.
40-
date_partition_column (optional): Timestamp column used for partitioning.
40+
date_partition_column (deprecated): Timestamp column used for partitioning.
4141
query (optional): SQL query to execute to generate data for this data source.
4242
name (optional): Name for the source. Defaults to the table_ref if not specified.
4343
description (optional): A human-readable description.
@@ -61,6 +61,15 @@ def __init__(
6161
table = table_ref
6262
self.bigquery_options = BigQueryOptions(table_ref=table, query=query)
6363

64+
if date_partition_column:
65+
warnings.warn(
66+
(
67+
"The argument 'date_partition_column' is not supported for BigQuery sources. "
68+
"It will be removed in Feast 0.21+"
69+
),
70+
DeprecationWarning,
71+
)
72+
6473
# If no name, use the table_ref as the default name
6574
_name = name
6675
if not _name:
@@ -78,10 +87,9 @@ def __init__(
7887

7988
super().__init__(
8089
_name if _name else "",
81-
event_timestamp_column,
82-
created_timestamp_column,
83-
field_mapping,
84-
date_partition_column,
90+
event_timestamp_column=event_timestamp_column,
91+
created_timestamp_column=created_timestamp_column,
92+
field_mapping=field_mapping,
8593
description=description,
8694
tags=tags,
8795
owner=owner,
@@ -128,7 +136,6 @@ def from_proto(data_source: DataSourceProto):
128136
table_ref=data_source.bigquery_options.table_ref,
129137
event_timestamp_column=data_source.event_timestamp_column,
130138
created_timestamp_column=data_source.created_timestamp_column,
131-
date_partition_column=data_source.date_partition_column,
132139
query=data_source.bigquery_options.query,
133140
description=data_source.description,
134141
tags=dict(data_source.tags),
@@ -148,7 +155,6 @@ def to_proto(self) -> DataSourceProto:
148155

149156
data_source_proto.event_timestamp_column = self.event_timestamp_column
150157
data_source_proto.created_timestamp_column = self.created_timestamp_column
151-
data_source_proto.date_partition_column = self.date_partition_column
152158

153159
return data_source_proto
154160

‎sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark_source.py‎

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -51,12 +51,21 @@ def __init__(
5151
_name = table
5252
else:
5353
raise DataSourceNoNameException()
54+
55+
if date_partition_column:
56+
warnings.warn(
57+
(
58+
"The argument 'date_partition_column' is not supported for Spark sources."
59+
"It will be removed in Feast 0.21+"
60+
),
61+
DeprecationWarning,
62+
)
63+
5464
super().__init__(
55-
_name,
56-
event_timestamp_column,
57-
created_timestamp_column,
58-
field_mapping,
59-
date_partition_column,
65+
name=_name,
66+
event_timestamp_column=event_timestamp_column,
67+
created_timestamp_column=created_timestamp_column,
68+
field_mapping=field_mapping,
6069
description=description,
6170
tags=tags,
6271
owner=owner,
@@ -130,7 +139,6 @@ def from_proto(data_source: DataSourceProto) -> Any:
130139
file_format=spark_options.file_format,
131140
event_timestamp_column=data_source.event_timestamp_column,
132141
created_timestamp_column=data_source.created_timestamp_column,
133-
date_partition_column=data_source.date_partition_column,
134142
description=data_source.description,
135143
tags=dict(data_source.tags),
136144
owner=data_source.owner,
@@ -149,7 +157,6 @@ def to_proto(self) -> DataSourceProto:
149157

150158
data_source_proto.event_timestamp_column = self.event_timestamp_column
151159
data_source_proto.created_timestamp_column = self.created_timestamp_column
152-
data_source_proto.date_partition_column = self.date_partition_column
153160

154161
return data_source_proto
155162

‎sdk/python/feast/infra/offline_stores/file_source.py‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -66,12 +66,20 @@ def __init__(
6666
s3_endpoint_override=s3_endpoint_override,
6767
)
6868

69+
if date_partition_column:
70+
warnings.warn(
71+
(
72+
"The argument 'date_partition_column' is not supported for File sources."
73+
"It will be removed in Feast 0.21+"
74+
),
75+
DeprecationWarning,
76+
)
77+
6978
super().__init__(
70-
name if name else path,
71-
event_timestamp_column,
72-
created_timestamp_column,
73-
field_mapping,
74-
date_partition_column,
79+
name=name if name else path,
80+
event_timestamp_column=event_timestamp_column,
81+
created_timestamp_column=created_timestamp_column,
82+
field_mapping=field_mapping,
7583
description=description,
7684
tags=tags,
7785
owner=owner,
@@ -114,7 +122,6 @@ def from_proto(data_source: DataSourceProto):
114122
path=data_source.file_options.uri,
115123
event_timestamp_column=data_source.event_timestamp_column,
116124
created_timestamp_column=data_source.created_timestamp_column,
117-
date_partition_column=data_source.date_partition_column,
118125
s3_endpoint_override=data_source.file_options.s3_endpoint_override,
119126
description=data_source.description,
120127
tags=dict(data_source.tags),
@@ -134,7 +141,6 @@ def to_proto(self) -> DataSourceProto:
134141

135142
data_source_proto.event_timestamp_column = self.event_timestamp_column
136143
data_source_proto.created_timestamp_column = self.created_timestamp_column
137-
data_source_proto.date_partition_column = self.date_partition_column
138144

139145
return data_source_proto
140146

‎sdk/python/feast/infra/offline_stores/redshift_source.py‎

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ def __init__(
2121
schema: Optional[str] = None,
2222
created_timestamp_column: Optional[str] = "",
2323
field_mapping: Optional[Dict[str, str]] = None,
24-
date_partition_column: Optional[str] = "",
24+
date_partition_column: Optional[str] = None,
2525
query: Optional[str] = None,
2626
name: Optional[str] = None,
2727
description: Optional[str] = "",
@@ -41,7 +41,7 @@ def __init__(
4141
row was created, used for deduplicating rows.
4242
field_mapping (optional): A dictionary mapping of column names in this data
4343
source to column names in a feature table or view.
44-
date_partition_column (optional): Timestamp column used for partitioning.
44+
date_partition_column (deprecated): Timestamp column used for partitioning.
4545
query (optional): The query to be executed to obtain the features.
4646
name (optional): Name for the source. Defaults to the table_ref if not specified.
4747
description (optional): A human-readable description.
@@ -70,13 +70,20 @@ def __init__(
7070
),
7171
DeprecationWarning,
7272
)
73+
if date_partition_column:
74+
warnings.warn(
75+
(
76+
"The argument 'date_partition_column' is not supported for Redshift sources."
77+
"It will be removed in Feast 0.21+"
78+
),
79+
DeprecationWarning,
80+
)
7381

7482
super().__init__(
7583
_name if _name else "",
76-
event_timestamp_column,
77-
created_timestamp_column,
78-
field_mapping,
79-
date_partition_column,
84+
event_timestamp_column=event_timestamp_column,
85+
created_timestamp_column=created_timestamp_column,
86+
field_mapping=field_mapping,
8087
description=description,
8188
tags=tags,
8289
owner=owner,
@@ -99,7 +106,6 @@ def from_proto(data_source: DataSourceProto):
99106
schema=data_source.redshift_options.schema,
100107
event_timestamp_column=data_source.event_timestamp_column,
101108
created_timestamp_column=data_source.created_timestamp_column,
102-
date_partition_column=data_source.date_partition_column,
103109
query=data_source.redshift_options.query,
104110
description=data_source.description,
105111
tags=dict(data_source.tags),
@@ -169,7 +175,6 @@ def to_proto(self) -> DataSourceProto:
169175

170176
data_source_proto.event_timestamp_column = self.event_timestamp_column
171177
data_source_proto.created_timestamp_column = self.created_timestamp_column
172-
data_source_proto.date_partition_column = self.date_partition_column
173178

174179
return data_source_proto
175180

‎sdk/python/feast/infra/offline_stores/snowflake_source.py‎

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,9 @@ def __init__(
2020
table: Optional[str] = None,
2121
query: Optional[str] = None,
2222
event_timestamp_column: Optional[str] = "",
23+
date_partition_column: Optional[str] = None,
2324
created_timestamp_column: Optional[str] = "",
2425
field_mapping: Optional[Dict[str, str]] = None,
25-
date_partition_column: Optional[str] = "",
2626
name: Optional[str] = None,
2727
description: Optional[str] = "",
2828
tags: Optional[Dict[str, str]] = None,
@@ -42,7 +42,7 @@ def __init__(
4242
row was created, used for deduplicating rows.
4343
field_mapping (optional): A dictionary mapping of column names in this data
4444
source to column names in a feature table or view.
45-
date_partition_column (optional): Timestamp column used for partitioning.
45+
date_partition_column (deprecated): Timestamp column used for partitioning.
4646
name (optional): Name for the source. Defaults to the table if not specified.
4747
description (optional): A human-readable description.
4848
tags (optional): A dictionary of key-value pairs to store arbitrary metadata.
@@ -72,12 +72,20 @@ def __init__(
7272
DeprecationWarning,
7373
)
7474

75+
if date_partition_column:
76+
warnings.warn(
77+
(
78+
"The argument 'date_partition_column' is not supported for Snowflake sources."
79+
"It will be removed in Feast 0.21+"
80+
),
81+
DeprecationWarning,
82+
)
83+
7584
super().__init__(
7685
_name if _name else "",
77-
event_timestamp_column,
78-
created_timestamp_column,
79-
field_mapping,
80-
date_partition_column,
86+
event_timestamp_column=event_timestamp_column,
87+
created_timestamp_column=created_timestamp_column,
88+
field_mapping=field_mapping,
8189
description=description,
8290
tags=tags,
8391
owner=owner,
@@ -101,7 +109,6 @@ def from_proto(data_source: DataSourceProto):
101109
table=data_source.snowflake_options.table,
102110
event_timestamp_column=data_source.event_timestamp_column,
103111
created_timestamp_column=data_source.created_timestamp_column,
104-
date_partition_column=data_source.date_partition_column,
105112
query=data_source.snowflake_options.query,
106113
description=data_source.description,
107114
tags=dict(data_source.tags),
@@ -170,7 +177,6 @@ def to_proto(self) -> DataSourceProto:
170177

171178
data_source_proto.event_timestamp_column = self.event_timestamp_column
172179
data_source_proto.created_timestamp_column = self.created_timestamp_column
173-
data_source_proto.date_partition_column = self.date_partition_column
174180

175181
return data_source_proto
176182

‎sdk/python/tests/integration/feature_repos/universal/data_sources/bigquery.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,6 @@ def create_data_source(
7777
table_ref=destination_name,
7878
event_timestamp_column=event_timestamp_column,
7979
created_timestamp_column=created_timestamp_column,
80-
date_partition_column="",
8180
field_mapping=field_mapping or {"ts_1": "ts"},
8281
)
8382

‎sdk/python/tests/integration/feature_repos/universal/data_sources/file.py‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,6 @@ def create_data_source(
4848
path=f"{f.name}",
4949
event_timestamp_column=event_timestamp_column,
5050
created_timestamp_column=created_timestamp_column,
51-
date_partition_column="",
5251
field_mapping=field_mapping or {"ts_1": "ts"},
5352
)
5453

@@ -130,7 +129,6 @@ def create_data_source(
130129
path=f"s3://{self.bucket}/{filename}",
131130
event_timestamp_column=event_timestamp_column,
132131
created_timestamp_column=created_timestamp_column,
133-
date_partition_column="",
134132
field_mapping=field_mapping or {"ts_1": "ts"},
135133
s3_endpoint_override=f"http://{host}:{port}",
136134
)

‎sdk/python/tests/integration/feature_repos/universal/data_sources/redshift.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,6 @@ def create_data_source(
6363
table=destination_name,
6464
event_timestamp_column=event_timestamp_column,
6565
created_timestamp_column=created_timestamp_column,
66-
date_partition_column="",
6766
field_mapping=field_mapping or {"ts_1": "ts"},
6867
database=self.offline_store_config.database,
6968
)

‎sdk/python/tests/integration/feature_repos/universal/data_sources/snowflake.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,6 @@ def create_data_source(
5555
table=destination_name,
5656
event_timestamp_column=event_timestamp_column,
5757
created_timestamp_column=created_timestamp_column,
58-
date_partition_column="",
5958
field_mapping=field_mapping or {"ts_1": "ts"},
6059
)
6160

‎sdk/python/tests/integration/feature_repos/universal/data_sources/spark_data_source_creator.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,6 @@ def create_data_source(
9595
table=destination_name,
9696
event_timestamp_column=event_timestamp_column,
9797
created_timestamp_column=created_timestamp_column,
98-
date_partition_column="",
9998
# maps certain column names to other names
10099
field_mapping=field_mapping or {"ts_1": "ts"},
101100
)

0 commit comments

Comments
 (0)