Skip to content

Commit 682e314

Browse files
committed
feat: Add dbt integration for importing models as FeatureViews (feast-dev#3335)
This PR implements the dbt-Feast integration feature requested in feast-dev#3335, enabling users to import dbt models as Feast FeatureViews. ## New CLI Commands - `feast dbt list` - List dbt models available for import - `feast dbt import` - Import dbt models as Feast objects ## Features - Parse dbt manifest.json files to extract model metadata - Map dbt types to Feast types (38 types supported) - Generate Entity, DataSource, and FeatureView objects - Support for BigQuery, Snowflake, and File data sources - Tag-based filtering (--tag) to select specific models - Code generation (--output) to create Python files - Dry-run mode to preview changes before applying ## Usage Examples ```bash # List models with 'feast' tag feast dbt list -m target/manifest.json --tag feast # Import models to registry feast dbt import -m target/manifest.json -e driver_id --tag feast # Generate Python file instead feast dbt import -m target/manifest.json -e driver_id --output features.py ``` Closes feast-dev#3335 Signed-off-by: yassinnouh21 <[email protected]>
1 parent 11122ce commit 682e314

10 files changed

Lines changed: 2199 additions & 0 deletions

File tree

‎sdk/python/feast/cli/cli.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626

2727
from feast import utils
2828
from feast.cli.data_sources import data_sources_cmd
29+
from feast.cli.dbt_import import dbt_cmd
2930
from feast.cli.entities import entities_cmd
3031
from feast.cli.feature_services import feature_services_cmd
3132
from feast.cli.feature_views import feature_views_cmd
@@ -553,6 +554,7 @@ def validate(
553554
cli.add_command(serve_offline_command)
554555
cli.add_command(serve_registry_command)
555556
cli.add_command(serve_transformations_command)
557+
cli.add_command(dbt_cmd)
556558

557559
if __name__ == "__main__":
558560
cli()

‎sdk/python/feast/cli/dbt_import.py‎

Lines changed: 379 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,379 @@
1+
"""
2+
CLI commands for importing dbt models as Feast features.
3+
4+
This module provides the `feast dbt` command group for integrating
5+
dbt models with Feast feature stores.
6+
"""
7+
8+
from typing import List, Optional
9+
10+
import click
11+
from colorama import Fore, Style
12+
13+
from feast.repo_operations import cli_check_repo, create_feature_store
14+
15+
16+
@click.group(name="dbt")
17+
def dbt_cmd():
18+
"""Import dbt models as Feast features."""
19+
pass
20+
21+
22+
@dbt_cmd.command("import")
23+
@click.option(
24+
"--manifest-path",
25+
"-m",
26+
required=True,
27+
type=click.Path(exists=True),
28+
help="Path to dbt manifest.json file (typically target/manifest.json)",
29+
)
30+
@click.option(
31+
"--entity-column",
32+
"-e",
33+
required=True,
34+
help="Primary key / entity column name (e.g., driver_id, customer_id)",
35+
)
36+
@click.option(
37+
"--data-source-type",
38+
"-d",
39+
type=click.Choice(["bigquery", "snowflake", "file"]),
40+
default="bigquery",
41+
show_default=True,
42+
help="Type of data source to create",
43+
)
44+
@click.option(
45+
"--timestamp-field",
46+
"-t",
47+
default="event_timestamp",
48+
show_default=True,
49+
help="Timestamp field name for point-in-time joins",
50+
)
51+
@click.option(
52+
"--tag",
53+
"tag_filter",
54+
default=None,
55+
help="Only import models with this dbt tag (e.g., --tag feast)",
56+
)
57+
@click.option(
58+
"--model",
59+
"model_names",
60+
multiple=True,
61+
help="Specific model names to import (can be specified multiple times)",
62+
)
63+
@click.option(
64+
"--ttl-days",
65+
type=int,
66+
default=1,
67+
show_default=True,
68+
help="TTL (time-to-live) in days for feature views",
69+
)
70+
@click.option(
71+
"--dry-run",
72+
is_flag=True,
73+
default=False,
74+
help="Preview what would be created without applying changes",
75+
)
76+
@click.option(
77+
"--exclude-columns",
78+
default=None,
79+
help="Comma-separated list of columns to exclude from features",
80+
)
81+
@click.option(
82+
"--output",
83+
"-o",
84+
type=click.Path(),
85+
default=None,
86+
help="Output Python file path (e.g., features.py). Generates code instead of applying to registry.",
87+
)
88+
@click.pass_context
89+
def import_command(
90+
ctx: click.Context,
91+
manifest_path: str,
92+
entity_column: str,
93+
data_source_type: str,
94+
timestamp_field: str,
95+
tag_filter: Optional[str],
96+
model_names: tuple,
97+
ttl_days: int,
98+
dry_run: bool,
99+
exclude_columns: Optional[str],
100+
output: Optional[str],
101+
):
102+
"""
103+
Import dbt models as Feast FeatureViews.
104+
105+
This command parses a dbt manifest.json file and creates corresponding
106+
Feast DataSource and FeatureView objects.
107+
108+
Examples:
109+
110+
# Import all models with 'feast' tag
111+
feast dbt import -m target/manifest.json -e driver_id --tag feast
112+
113+
# Import specific models
114+
feast dbt import -m target/manifest.json -e customer_id --model orders --model customers
115+
116+
# Dry run to preview changes
117+
feast dbt import -m target/manifest.json -e driver_id --tag feast --dry-run
118+
119+
# Generate Python file instead of applying to registry
120+
feast dbt import -m target/manifest.json -e driver_id --tag feast --output features.py
121+
"""
122+
from feast.dbt.parser import DbtManifestParser
123+
from feast.dbt.mapper import DbtToFeastMapper
124+
125+
# Parse manifest
126+
click.echo(f"{Fore.CYAN}Parsing dbt manifest: {manifest_path}{Style.RESET_ALL}")
127+
128+
try:
129+
parser = DbtManifestParser(manifest_path)
130+
parser.parse()
131+
except FileNotFoundError as e:
132+
click.echo(f"{Fore.RED}Error: {e}{Style.RESET_ALL}", err=True)
133+
raise SystemExit(1)
134+
except ValueError as e:
135+
click.echo(f"{Fore.RED}Error: {e}{Style.RESET_ALL}", err=True)
136+
raise SystemExit(1)
137+
138+
# Display manifest info
139+
if parser.dbt_version:
140+
click.echo(f" dbt version: {parser.dbt_version}")
141+
if parser.project_name:
142+
click.echo(f" Project: {parser.project_name}")
143+
144+
# Get models with filters
145+
model_list: Optional[List[str]] = list(model_names) if model_names else None
146+
models = parser.get_models(model_names=model_list, tag_filter=tag_filter)
147+
148+
if not models:
149+
click.echo(
150+
f"{Fore.YELLOW}No models found matching the criteria.{Style.RESET_ALL}"
151+
)
152+
if tag_filter:
153+
click.echo(f" Tag filter: {tag_filter}")
154+
if model_names:
155+
click.echo(f" Model names: {', '.join(model_names)}")
156+
raise SystemExit(0)
157+
158+
click.echo(
159+
f"{Fore.GREEN}Found {len(models)} model(s) to import:{Style.RESET_ALL}"
160+
)
161+
for model in models:
162+
tags_str = f" [tags: {', '.join(model.tags)}]" if model.tags else ""
163+
click.echo(f" - {model.name} ({len(model.columns)} columns){tags_str}")
164+
165+
# Parse exclude columns
166+
excluded: Optional[List[str]] = None
167+
if exclude_columns:
168+
excluded = [c.strip() for c in exclude_columns.split(",")]
169+
170+
# Create mapper
171+
mapper = DbtToFeastMapper(
172+
data_source_type=data_source_type,
173+
timestamp_field=timestamp_field,
174+
ttl_days=ttl_days,
175+
)
176+
177+
# Generate Feast objects
178+
click.echo(f"\n{Fore.CYAN}Generating Feast objects...{Style.RESET_ALL}")
179+
180+
all_objects = []
181+
entities_created = {}
182+
183+
for model in models:
184+
# Validate timestamp field exists
185+
column_names = [c.name for c in model.columns]
186+
if timestamp_field not in column_names:
187+
click.echo(
188+
f"{Fore.YELLOW}Warning: Model '{model.name}' missing timestamp "
189+
f"field '{timestamp_field}'. Skipping.{Style.RESET_ALL}"
190+
)
191+
continue
192+
193+
# Validate entity column exists
194+
if entity_column not in column_names:
195+
click.echo(
196+
f"{Fore.YELLOW}Warning: Model '{model.name}' missing entity "
197+
f"column '{entity_column}'. Skipping.{Style.RESET_ALL}"
198+
)
199+
continue
200+
201+
# Create or reuse entity
202+
if entity_column not in entities_created:
203+
entity = mapper.create_entity(
204+
name=entity_column,
205+
description=f"Entity key for dbt models",
206+
)
207+
entities_created[entity_column] = entity
208+
all_objects.append(entity)
209+
else:
210+
entity = entities_created[entity_column]
211+
212+
# Create data source
213+
data_source = mapper.create_data_source(
214+
model=model,
215+
timestamp_field=timestamp_field,
216+
)
217+
all_objects.append(data_source)
218+
219+
# Create feature view
220+
feature_view = mapper.create_feature_view(
221+
model=model,
222+
source=data_source,
223+
entity_column=entity_column,
224+
entity=entity,
225+
timestamp_field=timestamp_field,
226+
ttl_days=ttl_days,
227+
exclude_columns=excluded,
228+
)
229+
all_objects.append(feature_view)
230+
231+
click.echo(
232+
f" {Fore.GREEN}✓{Style.RESET_ALL} {model.name}: "
233+
f"DataSource + FeatureView ({len(feature_view.features)} features)"
234+
)
235+
236+
if not all_objects:
237+
click.echo(
238+
f"{Fore.YELLOW}No valid models to import (check warnings above).{Style.RESET_ALL}"
239+
)
240+
raise SystemExit(0)
241+
242+
# Filter models that were actually processed (have valid columns)
243+
valid_models = [
244+
m for m in models
245+
if timestamp_field in [c.name for c in m.columns]
246+
and entity_column in [c.name for c in m.columns]
247+
]
248+
249+
# Summary
250+
click.echo(f"\n{Fore.CYAN}Summary:{Style.RESET_ALL}")
251+
click.echo(f" Entities: {len(entities_created)}")
252+
click.echo(f" DataSources: {len(valid_models)}")
253+
click.echo(f" FeatureViews: {len(valid_models)}")
254+
255+
# Generate Python file if --output specified
256+
if output:
257+
from feast.dbt.codegen import generate_feast_code
258+
259+
code = generate_feast_code(
260+
models=valid_models,
261+
entity_column=entity_column,
262+
data_source_type=data_source_type,
263+
timestamp_field=timestamp_field,
264+
ttl_days=ttl_days,
265+
manifest_path=manifest_path,
266+
project_name=parser.project_name or "",
267+
exclude_columns=excluded,
268+
online=True,
269+
)
270+
271+
with open(output, "w") as f:
272+
f.write(code)
273+
274+
click.echo(
275+
f"\n{Fore.GREEN}✓ Generated Feast definitions: {output}{Style.RESET_ALL}"
276+
)
277+
click.echo(f" You can now import this file in your feature_store.yaml repo.")
278+
return
279+
280+
if dry_run:
281+
click.echo(
282+
f"\n{Fore.YELLOW}Dry run - no changes applied.{Style.RESET_ALL}"
283+
)
284+
click.echo("Remove --dry-run flag to apply changes.")
285+
return
286+
287+
# Apply to Feast
288+
click.echo(f"\n{Fore.CYAN}Applying to Feast registry...{Style.RESET_ALL}")
289+
290+
repo = ctx.obj["CHDIR"]
291+
fs_yaml_file = ctx.obj["FS_YAML_FILE"]
292+
cli_check_repo(repo, fs_yaml_file)
293+
store = create_feature_store(ctx)
294+
295+
store.apply(all_objects)
296+
297+
click.echo(
298+
f"{Fore.GREEN}✓ Successfully imported {len(valid_models)} dbt model(s) "
299+
f"to Feast project '{store.project}'{Style.RESET_ALL}"
300+
)
301+
302+
303+
@dbt_cmd.command("list")
304+
@click.option(
305+
"--manifest-path",
306+
"-m",
307+
required=True,
308+
type=click.Path(exists=True),
309+
help="Path to dbt manifest.json file",
310+
)
311+
@click.option(
312+
"--tag",
313+
"tag_filter",
314+
default=None,
315+
help="Filter models by dbt tag",
316+
)
317+
@click.option(
318+
"--show-columns",
319+
is_flag=True,
320+
default=False,
321+
help="Show column details for each model",
322+
)
323+
def list_command(
324+
manifest_path: str,
325+
tag_filter: Optional[str],
326+
show_columns: bool,
327+
):
328+
"""
329+
List dbt models available for import.
330+
331+
Examples:
332+
333+
# List all models
334+
feast dbt list -m target/manifest.json
335+
336+
# List models with specific tag
337+
feast dbt list -m target/manifest.json --tag feast
338+
339+
# Show column details
340+
feast dbt list -m target/manifest.json --show-columns
341+
"""
342+
from feast.dbt.parser import DbtManifestParser
343+
344+
click.echo(f"{Fore.CYAN}Parsing dbt manifest: {manifest_path}{Style.RESET_ALL}")
345+
346+
try:
347+
parser = DbtManifestParser(manifest_path)
348+
parser.parse()
349+
except (FileNotFoundError, ValueError) as e:
350+
click.echo(f"{Fore.RED}Error: {e}{Style.RESET_ALL}", err=True)
351+
raise SystemExit(1)
352+
353+
if parser.dbt_version:
354+
click.echo(f" dbt version: {parser.dbt_version}")
355+
if parser.project_name:
356+
click.echo(f" Project: {parser.project_name}")
357+
358+
models = parser.get_models(tag_filter=tag_filter)
359+
360+
if not models:
361+
click.echo(f"{Fore.YELLOW}No models found.{Style.RESET_ALL}")
362+
return
363+
364+
click.echo(f"\n{Fore.GREEN}Found {len(models)} model(s):{Style.RESET_ALL}\n")
365+
366+
for model in models:
367+
tags_str = f" [tags: {', '.join(model.tags)}]" if model.tags else ""
368+
click.echo(f"{Fore.CYAN}{model.name}{Style.RESET_ALL}{tags_str}")
369+
click.echo(f" Table: {model.full_table_name}")
370+
if model.description:
371+
click.echo(f" Description: {model.description[:80]}...")
372+
373+
if show_columns and model.columns:
374+
click.echo(f" Columns ({len(model.columns)}):")
375+
for col in model.columns:
376+
type_str = col.data_type or "unknown"
377+
click.echo(f" - {col.name}: {type_str}")
378+
379+
click.echo()

0 commit comments

Comments
 (0)