|
| 1 | +""" |
| 2 | +CLI commands for importing dbt models as Feast features. |
| 3 | +
|
| 4 | +This module provides the `feast dbt` command group for integrating |
| 5 | +dbt models with Feast feature stores. |
| 6 | +""" |
| 7 | + |
| 8 | +from typing import List, Optional |
| 9 | + |
| 10 | +import click |
| 11 | +from colorama import Fore, Style |
| 12 | + |
| 13 | +from feast.repo_operations import cli_check_repo, create_feature_store |
| 14 | + |
| 15 | + |
| 16 | +@click.group(name="dbt") |
| 17 | +def dbt_cmd(): |
| 18 | + """Import dbt models as Feast features.""" |
| 19 | + pass |
| 20 | + |
| 21 | + |
| 22 | +@dbt_cmd.command("import") |
| 23 | +@click.option( |
| 24 | + "--manifest-path", |
| 25 | + "-m", |
| 26 | + required=True, |
| 27 | + type=click.Path(exists=True), |
| 28 | + help="Path to dbt manifest.json file (typically target/manifest.json)", |
| 29 | +) |
| 30 | +@click.option( |
| 31 | + "--entity-column", |
| 32 | + "-e", |
| 33 | + required=True, |
| 34 | + help="Primary key / entity column name (e.g., driver_id, customer_id)", |
| 35 | +) |
| 36 | +@click.option( |
| 37 | + "--data-source-type", |
| 38 | + "-d", |
| 39 | + type=click.Choice(["bigquery", "snowflake", "file"]), |
| 40 | + default="bigquery", |
| 41 | + show_default=True, |
| 42 | + help="Type of data source to create", |
| 43 | +) |
| 44 | +@click.option( |
| 45 | + "--timestamp-field", |
| 46 | + "-t", |
| 47 | + default="event_timestamp", |
| 48 | + show_default=True, |
| 49 | + help="Timestamp field name for point-in-time joins", |
| 50 | +) |
| 51 | +@click.option( |
| 52 | + "--tag", |
| 53 | + "tag_filter", |
| 54 | + default=None, |
| 55 | + help="Only import models with this dbt tag (e.g., --tag feast)", |
| 56 | +) |
| 57 | +@click.option( |
| 58 | + "--model", |
| 59 | + "model_names", |
| 60 | + multiple=True, |
| 61 | + help="Specific model names to import (can be specified multiple times)", |
| 62 | +) |
| 63 | +@click.option( |
| 64 | + "--ttl-days", |
| 65 | + type=int, |
| 66 | + default=1, |
| 67 | + show_default=True, |
| 68 | + help="TTL (time-to-live) in days for feature views", |
| 69 | +) |
| 70 | +@click.option( |
| 71 | + "--dry-run", |
| 72 | + is_flag=True, |
| 73 | + default=False, |
| 74 | + help="Preview what would be created without applying changes", |
| 75 | +) |
| 76 | +@click.option( |
| 77 | + "--exclude-columns", |
| 78 | + default=None, |
| 79 | + help="Comma-separated list of columns to exclude from features", |
| 80 | +) |
| 81 | +@click.option( |
| 82 | + "--output", |
| 83 | + "-o", |
| 84 | + type=click.Path(), |
| 85 | + default=None, |
| 86 | + help="Output Python file path (e.g., features.py). Generates code instead of applying to registry.", |
| 87 | +) |
| 88 | +@click.pass_context |
| 89 | +def import_command( |
| 90 | + ctx: click.Context, |
| 91 | + manifest_path: str, |
| 92 | + entity_column: str, |
| 93 | + data_source_type: str, |
| 94 | + timestamp_field: str, |
| 95 | + tag_filter: Optional[str], |
| 96 | + model_names: tuple, |
| 97 | + ttl_days: int, |
| 98 | + dry_run: bool, |
| 99 | + exclude_columns: Optional[str], |
| 100 | + output: Optional[str], |
| 101 | +): |
| 102 | + """ |
| 103 | + Import dbt models as Feast FeatureViews. |
| 104 | +
|
| 105 | + This command parses a dbt manifest.json file and creates corresponding |
| 106 | + Feast DataSource and FeatureView objects. |
| 107 | +
|
| 108 | + Examples: |
| 109 | +
|
| 110 | + # Import all models with 'feast' tag |
| 111 | + feast dbt import -m target/manifest.json -e driver_id --tag feast |
| 112 | +
|
| 113 | + # Import specific models |
| 114 | + feast dbt import -m target/manifest.json -e customer_id --model orders --model customers |
| 115 | +
|
| 116 | + # Dry run to preview changes |
| 117 | + feast dbt import -m target/manifest.json -e driver_id --tag feast --dry-run |
| 118 | +
|
| 119 | + # Generate Python file instead of applying to registry |
| 120 | + feast dbt import -m target/manifest.json -e driver_id --tag feast --output features.py |
| 121 | + """ |
| 122 | + from feast.dbt.parser import DbtManifestParser |
| 123 | + from feast.dbt.mapper import DbtToFeastMapper |
| 124 | + |
| 125 | + # Parse manifest |
| 126 | + click.echo(f"{Fore.CYAN}Parsing dbt manifest: {manifest_path}{Style.RESET_ALL}") |
| 127 | + |
| 128 | + try: |
| 129 | + parser = DbtManifestParser(manifest_path) |
| 130 | + parser.parse() |
| 131 | + except FileNotFoundError as e: |
| 132 | + click.echo(f"{Fore.RED}Error: {e}{Style.RESET_ALL}", err=True) |
| 133 | + raise SystemExit(1) |
| 134 | + except ValueError as e: |
| 135 | + click.echo(f"{Fore.RED}Error: {e}{Style.RESET_ALL}", err=True) |
| 136 | + raise SystemExit(1) |
| 137 | + |
| 138 | + # Display manifest info |
| 139 | + if parser.dbt_version: |
| 140 | + click.echo(f" dbt version: {parser.dbt_version}") |
| 141 | + if parser.project_name: |
| 142 | + click.echo(f" Project: {parser.project_name}") |
| 143 | + |
| 144 | + # Get models with filters |
| 145 | + model_list: Optional[List[str]] = list(model_names) if model_names else None |
| 146 | + models = parser.get_models(model_names=model_list, tag_filter=tag_filter) |
| 147 | + |
| 148 | + if not models: |
| 149 | + click.echo( |
| 150 | + f"{Fore.YELLOW}No models found matching the criteria.{Style.RESET_ALL}" |
| 151 | + ) |
| 152 | + if tag_filter: |
| 153 | + click.echo(f" Tag filter: {tag_filter}") |
| 154 | + if model_names: |
| 155 | + click.echo(f" Model names: {', '.join(model_names)}") |
| 156 | + raise SystemExit(0) |
| 157 | + |
| 158 | + click.echo( |
| 159 | + f"{Fore.GREEN}Found {len(models)} model(s) to import:{Style.RESET_ALL}" |
| 160 | + ) |
| 161 | + for model in models: |
| 162 | + tags_str = f" [tags: {', '.join(model.tags)}]" if model.tags else "" |
| 163 | + click.echo(f" - {model.name} ({len(model.columns)} columns){tags_str}") |
| 164 | + |
| 165 | + # Parse exclude columns |
| 166 | + excluded: Optional[List[str]] = None |
| 167 | + if exclude_columns: |
| 168 | + excluded = [c.strip() for c in exclude_columns.split(",")] |
| 169 | + |
| 170 | + # Create mapper |
| 171 | + mapper = DbtToFeastMapper( |
| 172 | + data_source_type=data_source_type, |
| 173 | + timestamp_field=timestamp_field, |
| 174 | + ttl_days=ttl_days, |
| 175 | + ) |
| 176 | + |
| 177 | + # Generate Feast objects |
| 178 | + click.echo(f"\n{Fore.CYAN}Generating Feast objects...{Style.RESET_ALL}") |
| 179 | + |
| 180 | + all_objects = [] |
| 181 | + entities_created = {} |
| 182 | + |
| 183 | + for model in models: |
| 184 | + # Validate timestamp field exists |
| 185 | + column_names = [c.name for c in model.columns] |
| 186 | + if timestamp_field not in column_names: |
| 187 | + click.echo( |
| 188 | + f"{Fore.YELLOW}Warning: Model '{model.name}' missing timestamp " |
| 189 | + f"field '{timestamp_field}'. Skipping.{Style.RESET_ALL}" |
| 190 | + ) |
| 191 | + continue |
| 192 | + |
| 193 | + # Validate entity column exists |
| 194 | + if entity_column not in column_names: |
| 195 | + click.echo( |
| 196 | + f"{Fore.YELLOW}Warning: Model '{model.name}' missing entity " |
| 197 | + f"column '{entity_column}'. Skipping.{Style.RESET_ALL}" |
| 198 | + ) |
| 199 | + continue |
| 200 | + |
| 201 | + # Create or reuse entity |
| 202 | + if entity_column not in entities_created: |
| 203 | + entity = mapper.create_entity( |
| 204 | + name=entity_column, |
| 205 | + description=f"Entity key for dbt models", |
| 206 | + ) |
| 207 | + entities_created[entity_column] = entity |
| 208 | + all_objects.append(entity) |
| 209 | + else: |
| 210 | + entity = entities_created[entity_column] |
| 211 | + |
| 212 | + # Create data source |
| 213 | + data_source = mapper.create_data_source( |
| 214 | + model=model, |
| 215 | + timestamp_field=timestamp_field, |
| 216 | + ) |
| 217 | + all_objects.append(data_source) |
| 218 | + |
| 219 | + # Create feature view |
| 220 | + feature_view = mapper.create_feature_view( |
| 221 | + model=model, |
| 222 | + source=data_source, |
| 223 | + entity_column=entity_column, |
| 224 | + entity=entity, |
| 225 | + timestamp_field=timestamp_field, |
| 226 | + ttl_days=ttl_days, |
| 227 | + exclude_columns=excluded, |
| 228 | + ) |
| 229 | + all_objects.append(feature_view) |
| 230 | + |
| 231 | + click.echo( |
| 232 | + f" {Fore.GREEN}✓{Style.RESET_ALL} {model.name}: " |
| 233 | + f"DataSource + FeatureView ({len(feature_view.features)} features)" |
| 234 | + ) |
| 235 | + |
| 236 | + if not all_objects: |
| 237 | + click.echo( |
| 238 | + f"{Fore.YELLOW}No valid models to import (check warnings above).{Style.RESET_ALL}" |
| 239 | + ) |
| 240 | + raise SystemExit(0) |
| 241 | + |
| 242 | + # Filter models that were actually processed (have valid columns) |
| 243 | + valid_models = [ |
| 244 | + m for m in models |
| 245 | + if timestamp_field in [c.name for c in m.columns] |
| 246 | + and entity_column in [c.name for c in m.columns] |
| 247 | + ] |
| 248 | + |
| 249 | + # Summary |
| 250 | + click.echo(f"\n{Fore.CYAN}Summary:{Style.RESET_ALL}") |
| 251 | + click.echo(f" Entities: {len(entities_created)}") |
| 252 | + click.echo(f" DataSources: {len(valid_models)}") |
| 253 | + click.echo(f" FeatureViews: {len(valid_models)}") |
| 254 | + |
| 255 | + # Generate Python file if --output specified |
| 256 | + if output: |
| 257 | + from feast.dbt.codegen import generate_feast_code |
| 258 | + |
| 259 | + code = generate_feast_code( |
| 260 | + models=valid_models, |
| 261 | + entity_column=entity_column, |
| 262 | + data_source_type=data_source_type, |
| 263 | + timestamp_field=timestamp_field, |
| 264 | + ttl_days=ttl_days, |
| 265 | + manifest_path=manifest_path, |
| 266 | + project_name=parser.project_name or "", |
| 267 | + exclude_columns=excluded, |
| 268 | + online=True, |
| 269 | + ) |
| 270 | + |
| 271 | + with open(output, "w") as f: |
| 272 | + f.write(code) |
| 273 | + |
| 274 | + click.echo( |
| 275 | + f"\n{Fore.GREEN}✓ Generated Feast definitions: {output}{Style.RESET_ALL}" |
| 276 | + ) |
| 277 | + click.echo(f" You can now import this file in your feature_store.yaml repo.") |
| 278 | + return |
| 279 | + |
| 280 | + if dry_run: |
| 281 | + click.echo( |
| 282 | + f"\n{Fore.YELLOW}Dry run - no changes applied.{Style.RESET_ALL}" |
| 283 | + ) |
| 284 | + click.echo("Remove --dry-run flag to apply changes.") |
| 285 | + return |
| 286 | + |
| 287 | + # Apply to Feast |
| 288 | + click.echo(f"\n{Fore.CYAN}Applying to Feast registry...{Style.RESET_ALL}") |
| 289 | + |
| 290 | + repo = ctx.obj["CHDIR"] |
| 291 | + fs_yaml_file = ctx.obj["FS_YAML_FILE"] |
| 292 | + cli_check_repo(repo, fs_yaml_file) |
| 293 | + store = create_feature_store(ctx) |
| 294 | + |
| 295 | + store.apply(all_objects) |
| 296 | + |
| 297 | + click.echo( |
| 298 | + f"{Fore.GREEN}✓ Successfully imported {len(valid_models)} dbt model(s) " |
| 299 | + f"to Feast project '{store.project}'{Style.RESET_ALL}" |
| 300 | + ) |
| 301 | + |
| 302 | + |
| 303 | +@dbt_cmd.command("list") |
| 304 | +@click.option( |
| 305 | + "--manifest-path", |
| 306 | + "-m", |
| 307 | + required=True, |
| 308 | + type=click.Path(exists=True), |
| 309 | + help="Path to dbt manifest.json file", |
| 310 | +) |
| 311 | +@click.option( |
| 312 | + "--tag", |
| 313 | + "tag_filter", |
| 314 | + default=None, |
| 315 | + help="Filter models by dbt tag", |
| 316 | +) |
| 317 | +@click.option( |
| 318 | + "--show-columns", |
| 319 | + is_flag=True, |
| 320 | + default=False, |
| 321 | + help="Show column details for each model", |
| 322 | +) |
| 323 | +def list_command( |
| 324 | + manifest_path: str, |
| 325 | + tag_filter: Optional[str], |
| 326 | + show_columns: bool, |
| 327 | +): |
| 328 | + """ |
| 329 | + List dbt models available for import. |
| 330 | +
|
| 331 | + Examples: |
| 332 | +
|
| 333 | + # List all models |
| 334 | + feast dbt list -m target/manifest.json |
| 335 | +
|
| 336 | + # List models with specific tag |
| 337 | + feast dbt list -m target/manifest.json --tag feast |
| 338 | +
|
| 339 | + # Show column details |
| 340 | + feast dbt list -m target/manifest.json --show-columns |
| 341 | + """ |
| 342 | + from feast.dbt.parser import DbtManifestParser |
| 343 | + |
| 344 | + click.echo(f"{Fore.CYAN}Parsing dbt manifest: {manifest_path}{Style.RESET_ALL}") |
| 345 | + |
| 346 | + try: |
| 347 | + parser = DbtManifestParser(manifest_path) |
| 348 | + parser.parse() |
| 349 | + except (FileNotFoundError, ValueError) as e: |
| 350 | + click.echo(f"{Fore.RED}Error: {e}{Style.RESET_ALL}", err=True) |
| 351 | + raise SystemExit(1) |
| 352 | + |
| 353 | + if parser.dbt_version: |
| 354 | + click.echo(f" dbt version: {parser.dbt_version}") |
| 355 | + if parser.project_name: |
| 356 | + click.echo(f" Project: {parser.project_name}") |
| 357 | + |
| 358 | + models = parser.get_models(tag_filter=tag_filter) |
| 359 | + |
| 360 | + if not models: |
| 361 | + click.echo(f"{Fore.YELLOW}No models found.{Style.RESET_ALL}") |
| 362 | + return |
| 363 | + |
| 364 | + click.echo(f"\n{Fore.GREEN}Found {len(models)} model(s):{Style.RESET_ALL}\n") |
| 365 | + |
| 366 | + for model in models: |
| 367 | + tags_str = f" [tags: {', '.join(model.tags)}]" if model.tags else "" |
| 368 | + click.echo(f"{Fore.CYAN}{model.name}{Style.RESET_ALL}{tags_str}") |
| 369 | + click.echo(f" Table: {model.full_table_name}") |
| 370 | + if model.description: |
| 371 | + click.echo(f" Description: {model.description[:80]}...") |
| 372 | + |
| 373 | + if show_columns and model.columns: |
| 374 | + click.echo(f" Columns ({len(model.columns)}):") |
| 375 | + for col in model.columns: |
| 376 | + type_str = col.data_type or "unknown" |
| 377 | + click.echo(f" - {col.name}: {type_str}") |
| 378 | + |
| 379 | + click.echo() |
0 commit comments