# SPDX-License-Identifier: Apache-2.0

# SPDX-FileCopyrightText: Copyright The LanceDB Authors

from __future__ import annotations

import asyncio

import inspect

import deprecation

import warnings

from abc import ABC, abstractmethod

from dataclasses import dataclass

from datetime import datetime, timedelta

from functools import cached_property

from typing import (

TYPE_CHECKING,

Any,

Callable,

Dict,

Iterable,

List,

Literal,

Optional,

Tuple,

Union,

overload,

)

from urllib.parse import urlparse

from lancedb.scannable import _register_optional_converters, to_scannable

from . import __version__

from ._blob import (

BlobFile,

_normalize_blob_row_ids,

_wrap_blob_files,

strip_auto_row_ids,

validate_blob_mode,

)

from .types import BlobMode

from lancedb.arrow import peek_reader

from lancedb.background_loop import LOOP, embedding_executor

from .dependencies import (

_check_for_hugging_face,

_check_for_lance,

_check_for_pandas,

lance,

pandas as pd,

polars as pl,

)

import pyarrow as pa

import pyarrow.dataset

import pyarrow.compute as pc

import pyarrow.fs as pa_fs

import numpy as np

from .common import DATA, VEC, VECTOR_COLUMN_NAME

from .embeddings import EmbeddingFunctionConfig, EmbeddingFunctionRegistry

from .index import (

BTree,

IvfFlat,

IvfPq,

IvfSq,

Bitmap,

IvfRq,

LabelList,

Fm,

HnswPq,

HnswSq,

HnswFlat,

FTS,

)

from .expr import Expr

from .merge import LanceMergeInsertBuilder

from .pydantic import LanceModel, model_to_dict

from .query import (

AnalyzePlanDistributedMetrics,

AsyncFTSQuery,

AsyncHybridQuery,

AsyncQuery,

AsyncTakeQuery,

AsyncVectorQuery,

FullTextQuery,

LanceEmptyQueryBuilder,

LanceFtsQueryBuilder,

LanceHybridQueryBuilder,

LanceQueryBuilder,

LanceVectorQueryBuilder,

LanceTakeQueryBuilder,

Query,

)

from .util import (

add_note,

fs_from_uri,

get_uri_scheme,

infer_vector_column_name,

join_uri,

value_to_sql,

)

from .index import lang_mapping

from .schema import blob_v2_column_paths, schema_has_blob_field

def _should_push_down_query_table(

namespace_client: Optional[Any], pushdown_operations: set

) -> bool:

return namespace_client is not None and "QueryTable" in pushdown_operations

_MODEL_BACKED_TOKENIZER_PREFIXES = ("jieba", "lindera")

_MODEL_BACKED_TOKENIZER_ERRORS = (

"unknown base tokenizer",

"Invalid directory path:",

"Failed to load Jieba",

"Failed to load tokenizer config",

"Failed to initialize default tokenizer",

)

def _add_unique_note(exception: BaseException, note: str) -> None:

existing_notes = getattr(exception, "__notes__", ()) or ()

message = (

exception.args[0]

if exception.args and isinstance(exception.args[0], str)

else ""

)

if note not in existing_notes and note not in message:

add_note(exception, note)

def _is_model_backed_tokenizer(base_tokenizer: str) -> bool:

return any(

base_tokenizer == prefix or base_tokenizer.startswith(f"{prefix}/")

for prefix in _MODEL_BACKED_TOKENIZER_PREFIXES

)

def _maybe_add_fts_error_note(

exception: BaseException, *, base_tokenizer: str, language: Optional[str] = None

) -> None:

message = str(exception)

if language is not None and "not support the requested language" in message:

supported_langs = ", ".join(lang_mapping.values())

_add_unique_note(exception, f"Supported languages: {supported_langs}")

return

if not _is_model_backed_tokenizer(base_tokenizer):

return

if not any(marker in message for marker in _MODEL_BACKED_TOKENIZER_ERRORS):

return

_add_unique_note(

exception,

"Model-backed tokenizers such as 'jieba/default' and 'lindera/ipadic' "

"require tokenizer models in Lance's language model home. Set "

"LANCE_LANGUAGE_MODEL_HOME to override the default platform data "

"directory under 'lance/language_models'. Expected layouts include "

"'<model-home>/jieba/default/...' and "

"'<model-home>/lindera/ipadic/...'.",

)

if TYPE_CHECKING:

from .db import LanceDBConnection

from ._lancedb import (

Table as LanceDBTable,

OptimizeStats,

CleanupStats,

CompactionStats,

Tag,

AddColumnsResult,

AddResult,

AlterColumnsResult,

UpdateFieldMetadataResult,

DeleteResult,

DropColumnsResult,

FtsToken,

LsmWriteSpec,

MergeResult,

UpdateResult,

)

from .index import IndexConfig

import pandas

import PIL

from .types import (

QueryType,

OnBadVectorsType,

AddMode,

CreateMode,

VectorIndexType,

ScalarIndexType,

BaseTokenizerType,

DistanceType,

)

# Type alias for index configuration objects

IndexConfigType = Union[

IvfFlat,

IvfPq,

IvfSq,

IvfRq,

HnswFlat,

HnswPq,

HnswSq,

BTree,

Bitmap,

LabelList,

Fm,

FTS,

]

# Known distance metrics for legacy API detection

KNOWN_METRICS = {"l2", "cosine", "dot", "hamming"}

def _into_pyarrow_reader(

data, schema: Optional[pa.Schema] = None

) -> pa.RecordBatchReader:

from lancedb.dependencies import datasets

if _check_for_hugging_face(data):

if isinstance(data, datasets.Dataset):

schema = data.features.arrow_schema

return pa.RecordBatchReader.from_batches(schema, data.data.to_batches())

elif isinstance(data, datasets.dataset_dict.DatasetDict):

schema = _schema_from_hf(data, None)

if "split" not in schema.names:

schema = schema.append(pa.field("split", pa.string()))

return pa.RecordBatchReader.from_batches(

schema, _to_batches_with_split(data)

)

if isinstance(data, LanceModel):

raise ValueError("Cannot add a single LanceModel to a table. Use a list.")

if isinstance(data, dict):

raise ValueError(

"Cannot create or add rows from a single dictionary. "

"Use a list of dictionaries instead."

)

if isinstance(data, list):

# Handle empty list case

if not data:

if schema is None:

raise ValueError("Cannot create table from empty list without a schema")

return pa.Table.from_pylist(data, schema=schema).to_reader()

# convert to list of dict if data is a bunch of LanceModels

if isinstance(data[0], LanceModel):

schema = data[0].__class__.to_arrow_schema()

data = [model_to_dict(d) for d in data]

return pa.Table.from_pylist(data, schema=schema).to_reader()

elif isinstance(data[0], pa.RecordBatch):

return pa.Table.from_batches(data).to_reader()

else:

return pa.Table.from_pylist(data).to_reader()

elif _check_for_pandas(data) and isinstance(data, pd.DataFrame):

table = pa.Table.from_pandas(data, preserve_index=False)

# Do not serialize Pandas metadata

meta = table.schema.metadata if table.schema.metadata is not None else {}

meta = {k: v for k, v in meta.items() if k != b"pandas"}

return table.replace_schema_metadata(meta).to_reader()

elif isinstance(data, pa.Table):

return data.to_reader()

elif isinstance(data, pa.RecordBatch):

return pa.RecordBatchReader.from_batches(data.schema, [data])

elif _check_for_lance(data) and isinstance(data, lance.LanceDataset):

return data.scanner().to_reader()

elif isinstance(data, pa.dataset.Dataset):

return data.scanner().to_reader()

elif isinstance(data, pa.dataset.Scanner):

return data.to_reader()

elif isinstance(data, pa.RecordBatchReader):

return data

elif (

type(data).__module__.startswith("polars")

and data.__class__.__name__ == "DataFrame"

):

return data.to_arrow().to_reader()

elif (

type(data).__module__.startswith("polars")

and data.__class__.__name__ == "LazyFrame"

):

return data.collect().to_arrow().to_reader()

elif isinstance(data, Iterable):

return _iterator_to_reader(data)

else:

raise TypeError(

f"Unknown data type {type(data)}. "

"Supported types: list of dicts, pandas DataFrame, polars DataFrame, "

"pyarrow Table/RecordBatch, or Pydantic models. "

"See https://docs.lancedb.com/tables/ for examples."

)

def _iterator_to_reader(data: Iterable) -> pa.RecordBatchReader:

# Each batch is treated as it's own reader, mainly so we can

# re-use the _into_pyarrow_reader logic.

first = _into_pyarrow_reader(next(data))

schema = first.schema

def gen():

yield from first

for batch in data:

table: pa.Table = _into_pyarrow_reader(batch).read_all()

if table.schema != schema:

try:

table = table.cast(schema)

except pa.lib.ArrowInvalid:

raise ValueError(

f"Input iterator yielded a batch with schema that "

f"does not match the schema of other batches.\n"

f"Expected:\n{schema}\nGot:\n{batch.schema}"

)

yield from table.to_batches()

return pa.RecordBatchReader.from_batches(schema, gen())

def _sanitize_data(

data: "DATA",

target_schema: Optional[pa.Schema] = None,

metadata: Optional[dict] = None, # embedding metadata

on_bad_vectors: OnBadVectorsType = "error",

fill_value: float = 0.0,

*,

allow_subschema: bool = False,

) -> pa.RecordBatchReader:

"""

Handle input data, applying all standard transformations.

This includes:

* Converting the data to a PyArrow Table

* Adding vector columns defined in the metadata

* Adding embedding metadata into the schema

* Casting the table to the target schema

* Handling bad vectors

Parameters

----------

target_schema : Optional[pa.Schema], default None

The schema to cast the table to. This is typically the schema of the table

if it already exists. Otherwise it might be a user-requested schema.

allow_subschema : bool, default False

If True, the input table is allowed to omit columns from the target schema.

The target schema will be filtered to only include columns that are present

in the input table before casting.

metadata : Optional[dict], default None

The embedding metadata to add to the schema.

on_bad_vectors : Literal["error", "drop", "fill", "null"], default "error"

What to do if any of the vectors are not the same size or contains NaNs.

fill_value : float, default 0.0

The value to use when filling vectors. Only used if on_bad_vectors="fill".

All entries in the vector will be set to this value.

"""

# At this point, the table might not match the schema we are targeting:

# 1. There might be embedding columns missing that will be added

# in the add_embeddings step.

# 2. If `allow_subschemas` is True, there might be columns missing.

reader = _into_pyarrow_reader(data, target_schema)

reader = _append_vector_columns(reader, target_schema, metadata=metadata)

# This happens before the cast so we can fix vector columns with

# incorrect lengths before they are cast to FSL.

reader = _handle_bad_vectors(

reader,

on_bad_vectors=on_bad_vectors,

fill_value=fill_value,

target_schema=target_schema,

metadata=metadata,

)

if target_schema is None:

target_schema, reader = _infer_target_schema(reader)

if metadata:

target_schema = target_schema.with_metadata(

_merge_metadata(target_schema.metadata, metadata)

)

_validate_schema(target_schema)

reader = _cast_to_target_schema(reader, target_schema, allow_subschema)

return reader

def _cast_to_target_schema(

reader: pa.RecordBatchReader,

target_schema: pa.Schema,

allow_subschema: bool = False,

) -> pa.RecordBatchReader:

# pa.Table.cast expects field order not to be changed.

# Lance doesn't care about field order, so we don't need to rearrange fields

# to match the target schema. We just need to correctly cast the fields.

if reader.schema.equals(target_schema, check_metadata=True):

# Fast path when the schemas are already the same

return reader

fields = _align_field_types(list(iter(reader.schema)), list(iter(target_schema)))

reordered_schema = pa.schema(fields, metadata=target_schema.metadata)

if not allow_subschema and len(reordered_schema) != len(target_schema):

raise ValueError(

"Input table has different number of columns than target schema"

)

if allow_subschema and len(reordered_schema) != len(target_schema):

fields = _infer_subschema(

list(iter(reader.schema)), list(iter(reordered_schema))

)

reordered_schema = pa.schema(fields, metadata=target_schema.metadata)

def gen():

for batch in reader:

# Table but not RecordBatch has cast.

cast_batches = (

pa.Table.from_batches([batch]).cast(reordered_schema).to_batches()

)

if cast_batches:

yield pa.RecordBatch.from_arrays(

cast_batches[0].columns, schema=reordered_schema

)

return pa.RecordBatchReader.from_batches(reordered_schema, gen())

def _align_field_types(

fields: List[pa.Field],

target_fields: List[pa.Field],

) -> List[pa.Field]:

"""

Apply the data types from the target_fields to the fields.

"""

new_fields = []

for field in fields:

target_field = next((f for f in target_fields if f.name == field.name), None)

if target_field is None:

raise ValueError(f"Field '{field.name}' not found in target schema")

if pa.types.is_struct(target_field.type):

if pa.types.is_struct(field.type):

new_type = pa.struct(

_align_field_types(

field.type.fields,

target_field.type.fields,

)

)

else:

new_type = target_field.type

elif pa.types.is_list(target_field.type):

if _is_list_like(field.type):

new_type = pa.list_(

_align_field_types(

[field.type.value_field],

[target_field.type.value_field],

)[0]

)

else:

new_type = target_field.type

elif pa.types.is_large_list(target_field.type):

if _is_list_like(field.type):

new_type = pa.large_list(

_align_field_types(

[field.type.value_field],

[target_field.type.value_field],

)[0]

)

else:

new_type = target_field.type

elif pa.types.is_fixed_size_list(target_field.type):

if _is_list_like(field.type):

new_type = pa.list_(

_align_field_types(

[field.type.value_field],

[target_field.type.value_field],

)[0],

target_field.type.list_size,

)

else:

new_type = target_field.type

else:

new_type = target_field.type

new_fields.append(

pa.field(field.name, new_type, field.nullable, target_field.metadata)

)

return new_fields

def _infer_subschema(

schema: List[pa.Field],

reference_fields: List[pa.Field],

) -> List[pa.Field]:

"""

Transform the list of fields so the types match the reference_fields.

The order of the fields is preserved.

``schema`` may have fewer fields than `reference_fields`, but it may not have

more fields.

"""

fields = []

lookup = {f.name: f for f in reference_fields}

for field in schema:

reference = lookup.get(field.name)

if reference is None:

raise ValueError("Unexpected field in schema: {}".format(field))

if pa.types.is_struct(reference.type):

new_type = pa.struct(

_infer_subschema(

field.type.fields,

reference.type.fields,

)

)

new_field = pa.field(

field.name,

new_type,

reference.nullable,

)

else:

new_field = reference

fields.append(new_field)

return fields

def sanitize_create_table(

data,

schema: Union[pa.Schema, LanceModel],

metadata=None,

on_bad_vectors: OnBadVectorsType = "error",

fill_value: float = 0.0,

):

if inspect.isclass(schema) and issubclass(schema, LanceModel):

# convert LanceModel to pyarrow schema

# note that it's possible this contains

# embedding function metadata already

schema: pa.Schema = schema.to_arrow_schema()

if data is not None:

if metadata is None and schema is not None:

metadata = schema.metadata

data = _sanitize_data(

data,

schema,

metadata=metadata,

on_bad_vectors=on_bad_vectors,

fill_value=fill_value,

)

schema = data.schema

else:

if schema is not None:

data = pa.Table.from_pylist([], schema)

if schema is None:

if data is None:

raise ValueError("Either data or schema must be provided")

elif hasattr(data, "schema"):

schema = data.schema

if metadata:

metadata = _merge_metadata(schema.metadata, metadata)

schema = schema.with_metadata(metadata)

# Need to apply metadata to the data as well

if isinstance(data, pa.Table):

data = data.replace_schema_metadata(metadata)

elif isinstance(data, pa.RecordBatchReader):

data = pa.RecordBatchReader.from_batches(schema, data)

return data, schema

def _schema_from_hf(data, schema) -> pa.Schema:

"""

Extract pyarrow schema from HuggingFace DatasetDict

and validate that they're all the same schema between

splits

"""

for dataset in data.values():

if schema is None:

schema = dataset.features.arrow_schema

elif schema != dataset.features.arrow_schema:

msg = "All datasets in a HuggingFace DatasetDict must have the same schema"

raise TypeError(msg)

return schema

def _to_batches_with_split(data):

"""

Return a generator of RecordBatches from a HuggingFace DatasetDict

with an extra `split` column

"""

for key, dataset in data.items():

for batch in dataset.data.to_batches():

table = pa.Table.from_batches([batch])

if "split" not in table.column_names:

table = table.append_column(

"split", pa.array([key] * batch.num_rows, pa.string())

)

for b in table.to_batches():

yield b

def _append_vector_columns(

reader: pa.RecordBatchReader,

schema: Optional[pa.Schema] = None,

*,

metadata: Optional[dict] = None,

) -> pa.RecordBatchReader:

"""

Use the embedding function to automatically embed the source columns and add the

vector columns to the table.

"""

if schema is None:

metadata = _merge_metadata(metadata)

else:

metadata = _merge_metadata(schema.metadata, metadata)

functions = EmbeddingFunctionRegistry.get_instance().parse_functions(metadata)

if not functions:

return reader

fields = list(reader.schema)

for vector_column, conf in functions.items():

if vector_column not in reader.schema.names:

if schema is not None:

field = schema.field(vector_column)

else:

dtype = pa.list_(pa.float32(), conf.function.ndims())

field = pa.field(vector_column, type=dtype, nullable=True)

fields.append(field)

schema = pa.schema(fields, metadata=reader.schema.metadata)

def gen():

for batch in reader:

for vector_column, conf in functions.items():

func = conf.function

no_vector_column = vector_column not in batch.column_names

if no_vector_column or pc.all(pc.is_null(batch[vector_column])).as_py():

col_data = func.compute_source_embeddings_with_retry(

batch[conf.source_column]

)

# Replace vectors with wrong length (including empty lists

# returned for inputs like empty strings) with None so that

# _handle_bad_vectors can process them according to the

# on_bad_vectors policy instead of crashing when PyArrow

# tries to cast them into a fixed-size list array.

expected_ndims = conf.function.ndims()

col_data = [

v if v is not None and len(v) == expected_ndims else None

for v in col_data

]

if no_vector_column:

batch = batch.append_column(

schema.field(vector_column),

pa.array(col_data, type=schema.field(vector_column).type),

)

else:

batch = batch.set_column(

batch.column_names.index(vector_column),

schema.field(vector_column),

pa.array(col_data, type=schema.field(vector_column).type),

)

yield batch

return pa.RecordBatchReader.from_batches(schema, gen())

def _table_path(base: str, table_name: str) -> str:

"""

Get a table path that can be used in PyArrow FS.

Removes any weird schemes (such as "s3+ddb") and drops any query params.

"""

uri = _table_uri(base, table_name)

# Parse as URL

parsed = urlparse(uri)

# If scheme is s3+ddb, convert to s3

if parsed.scheme == "s3+ddb":

parsed = parsed._replace(scheme="s3")

# Remove query parameters

return parsed._replace(query=None).geturl()

def _table_uri(base: str, table_name: str) -> str:

return join_uri(base, f"{table_name}.lance")

def _normalize_progress(progress):

"""Normalize a ``progress`` parameter for :meth:`Table.add`.

Returns ``(progress_obj, owns)`` where *owns* is True when we created a

tqdm bar that the caller must close.

"""

if progress is True:

from tqdm.auto import tqdm

return tqdm(unit=" rows"), True

if progress is False or progress is None:

return None, False

return progress, False

class Table(ABC):

"""

A Table is a collection of Records in a LanceDB Database.

Examples

--------

Create using [DBConnection.create_table][lancedb.DBConnection.create_table]

(more examples in that method's documentation).

>>> import lancedb

>>> db = lancedb.connect("./.lancedb")

>>> table = db.create_table("my_table", data=[{"vector": [1.1, 1.2], "b": 2}])

>>> table.head()

pyarrow.Table

vector: fixed_size_list<item: float>[2]

child 0, item: float

b: int64

----

vector: [[[1.1,1.2]]]

b: [[2]]

Can append new data with [Table.add()][lancedb.table.Table.add].

>>> table.add([{"vector": [0.5, 1.3], "b": 4}])

AddResult(version=2)

Can query the table with [Table.search][lancedb.table.Table.search].

>>> table.search([0.4, 0.4]).select(["b", "vector"]).to_pandas()

b vector _distance

0 4 [0.5, 1.3] 0.82

1 2 [1.1, 1.2] 1.13

Search queries are much faster when an index is created. See

[Table.create_index][lancedb.table.Table.create_index].

"""

@property

@abstractmethod

def name(self) -> str:

"""The name of this Table"""

raise NotImplementedError

@property

@abstractmethod

def version(self) -> int:

"""The version of this Table"""

raise NotImplementedError

@property

@abstractmethod

def schema(self) -> pa.Schema:

"""The [Arrow Schema](https://arrow.apache.org/docs/python/api/datatypes.html#)

of this Table

"""

raise NotImplementedError

@property

@abstractmethod

def tags(self) -> Tags:

"""Tag management for the table.

Similar to Git, tags are a way to add metadata to a specific version of the

table.

.. warning::

Tagged versions are exempted from the :py:meth:`cleanup_old_versions()`

process.

To remove a version that has been tagged, you must first

:py:meth:`~Tags.delete` the associated tag.

Examples

--------

.. code-block:: python

table = db.open_table("my_table")

table.tags.create("v2-prod-20250203", 10)

tags = table.tags.list()

"""

raise NotImplementedError

@property

def branches(self) -> "Branches":

"""Branch management for the table.

Branches are isolated, writable lines of history forked from another

branch (or version). Writes on a branch do not affect ``main``.

"""

raise NotImplementedError

def current_branch(self) -> Optional[str]:

"""The branch this table handle is scoped to, or ``None`` for ``main``."""

raise NotImplementedError

def __len__(self) -> int:

"""The number of rows in this Table"""

return self.count_rows(None)

@property

@abstractmethod

def embedding_functions(self) -> Dict[str, EmbeddingFunctionConfig]:

"""

Get a mapping from vector column name to it's configured embedding function.

"""

@abstractmethod

def count_rows(self, filter: Optional[str] = None) -> int:

"""

Count the number of rows in the table.

Parameters

----------

filter: str, optional

A SQL where clause to filter the rows to count.

"""

raise NotImplementedError

def to_pandas(self, blob_mode: BlobMode = "lazy", **kwargs) -> "pandas.DataFrame":

"""Return the table as a pandas DataFrame.

Parameters

----------

blob_mode: str, default "lazy"

Controls how blob columns are returned for backends that support

Lance blob-aware pandas conversion.

**kwargs

Forwarded to PyArrow / Lance pandas conversion.

Returns

-------

pd.DataFrame

"""

return self.to_arrow().to_pandas(**kwargs)

@abstractmethod

def to_arrow(self) -> pa.Table:

"""Return the table as a pyarrow Table.

Returns

-------

pa.Table

"""

raise NotImplementedError

def to_lance(self, **kwargs) -> lance.LanceDataset:

"""Return the table as a lance.LanceDataset.

Returns

-------

lance.LanceDataset

"""

raise NotImplementedError

def to_polars(self, **kwargs) -> "pl.DataFrame":

"""Return the table as a polars.DataFrame.

Returns

-------

polars.DataFrame

"""

raise NotImplementedError

# New unified API overload

@overload

def create_index(

self,

column: str,

/,

*,

config: IndexConfigType,

replace: bool = ...,

wait_timeout: Optional[timedelta] = ...,

name: Optional[str] = ...,

train: bool = ...,

) -> None: ...

# Legacy API overload (deprecated)

@overload

def create_index(

self,

metric: Literal["l2", "cosine", "dot", "hamming"] = ...,

num_partitions: Optional[int] = ...,

num_sub_vectors: Optional[int] = ...,

vector_column_name: str = ...,

replace: bool = ...,

accelerator: Optional[str] = ...,

index_cache_size: Optional[int] = ...,

*,

index_type: VectorIndexType = ...,

wait_timeout: Optional[timedelta] = ...,

num_bits: int = ...,

max_iterations: int = ...,

sample_rate: int = ...,

m: int = ...,

ef_construction: int = ...,

name: Optional[str] = ...,

train: bool = ...,

target_partition_size: Optional[int] = ...,

) -> None: ...

def create_index(

self,

metric: DistanceType = "l2",

num_partitions: Optional[int] = None,

num_sub_vectors: Optional[int] = None,

vector_column_name: str = VECTOR_COLUMN_NAME,

replace: bool = True,

accelerator: Optional[str] = None,

index_cache_size: Optional[int] = None,

*,

index_type: VectorIndexType = "IVF_PQ",

wait_timeout: Optional[timedelta] = None,

num_bits: int = 8,

max_iterations: int = 50,

sample_rate: int = 256,

m: int = 20,

ef_construction: int = 300,

config: Optional[IndexConfigType] = None,

name: Optional[str] = None,

train: bool = True,

target_partition_size: Optional[int] = None,

):

"""Create an index on a column.

This method supports both the new unified API and the legacy API

for backwards compatibility. The new API takes the column name as the

first positional argument and an index configuration object via

``config``; the legacy API takes the distance metric as the first

argument plus separate ``vector_column_name`` / ``num_partitions`` /

etc. parameters, and emits a ``DeprecationWarning``.

Parameters

----------

metric : str

For new API: the column name to index.

For legacy API: the distance metric ("l2", "cosine", "dot", "hamming").

config : IndexConfigType, optional

The index configuration object. If provided, uses the new unified API.

Can be one of: IvfFlat, IvfPq, IvfSq, IvfRq, HnswPq, HnswSq,

BTree, Bitmap, LabelList, Fm, FTS.

replace : bool, default True

Whether to replace an existing index on this column.

wait_timeout : timedelta, optional

Timeout to wait for async indexing to complete.

name : str, optional

Custom name for the index.

train : bool, default True

Whether to train the index with existing data.

Examples

--------

New API (recommended):

>>> table.create_index( # doctest: +SKIP

... "vector", config=IvfPq(distance_type="l2")

... )

>>> table.create_index("category", config=BTree()) # doctest: +SKIP

>>> table.create_index("content", config=FTS()) # doctest: +SKIP

Legacy API (deprecated):

>>> table.create_index( # doctest: +SKIP

... "l2", vector_column_name="vector"

... )

"""

raise NotImplementedError

def drop_index(self, name: str) -> None:

"""

Drop an index from the table.

Parameters

----------

name: str

The name of the index to drop.

Notes

-----

This does not delete the index from disk, it just removes it from the table.

To delete the index, run [optimize][lancedb.table.Table.optimize]

after dropping the index.

Use [list_indices][lancedb.table.Table.list_indices] to find the names of

the indices.

"""

raise NotImplementedError

def wait_for_index(

self, index_names: Iterable[str], timeout: timedelta = timedelta(seconds=300)