volcengine · qin-ctx · Mar 16, 2026 · Mar 15, 2026 · qin-ctx · Mar 16, 2026
diff --git a/.github/workflows/_build.yml b/.github/workflows/_build.yml
@@ -455,21 +455,22 @@ jobs:
       shell: bash
       run: |
         python - <<'PY'
+        import importlib
         import importlib.util
-        import pathlib
-        import site
-
-        candidates = []
-        for base in site.getsitepackages():
-            candidates.extend(pathlib.Path(base).glob("openviking/storage/vectordb/engine*.so"))
-
-        if not candidates:
-            raise SystemExit("openviking storage engine extension was not installed")
-
-        engine_path = candidates[0]
-        spec = importlib.util.spec_from_file_location("engine", engine_path)
-        module = importlib.util.module_from_spec(spec)
-        assert spec.loader is not None
-        spec.loader.exec_module(module)
-        print(f"Loaded native extension from {engine_path}")
+
+        import openviking.storage.vectordb.engine as engine
+
+        native_spec = importlib.util.find_spec("openviking.storage.vectordb.engine._native")
+        if native_spec is None or native_spec.origin is None:
+            raise SystemExit("openviking storage native backend extension was not installed")
+
+        native_module = importlib.import_module("openviking.storage.vectordb.engine._native")
+        if engine.ENGINE_VARIANT != "native":
+            raise SystemExit(
+                f"expected native engine variant on macOS arm64 wheel, got {engine.ENGINE_VARIANT}"
+            )
+
+        print(f"Loaded runtime engine variant {engine.ENGINE_VARIANT}")
+        print(f"Loaded native extension from {native_spec.origin}")
+        print(f"Imported backend module {native_module.__name__}")
         PY
diff --git a/build_support/__init__.py b/build_support/__init__.py
@@ -0,0 +1 @@
+"""Build helpers for OpenViking native artifacts."""
diff --git a/build_support/x86_profiles.py b/build_support/x86_profiles.py
@@ -0,0 +1,64 @@
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from typing import Iterable
+
+DEFAULT_X86_VARIANTS = ("sse3", "avx2", "avx512")
+KNOWN_X86_VARIANTS = frozenset(DEFAULT_X86_VARIANTS)
+X86_ARCHITECTURES = ("x86_64", "amd64", "x64", "i386", "i686")
+
+
+@dataclass(frozen=True)
+class EngineBuildConfig:
+    is_x86: bool
+    primary_extension: str
+    cmake_variants: tuple[str, ...]
+
+
+def _normalize_machine(machine: str | None) -> str:
+    return (machine or "").strip().lower()
+
+
+def is_x86_machine(machine: str | None) -> bool:
+    normalized = _normalize_machine(machine)
+    return any(token in normalized for token in X86_ARCHITECTURES)
+
+
+def _normalize_x86_variants(raw_variants: Iterable[str]) -> tuple[str, ...]:
+    requested = []
+    for variant in raw_variants:
+        normalized = variant.strip().lower()
+        if not normalized or normalized not in KNOWN_X86_VARIANTS or normalized in requested:
+            continue
+        requested.append(normalized)
+
+    if "sse3" not in requested:
+        requested.insert(0, "sse3")
+
+    return tuple(requested or DEFAULT_X86_VARIANTS)
+
+
+def get_requested_x86_build_variants(raw_value: str | None = None) -> tuple[str, ...]:
+    if raw_value is None:
+        raw_value = os.environ.get("OV_X86_BUILD_VARIANTS", "")
+
+    if not raw_value.strip():
+        return DEFAULT_X86_VARIANTS
+
+    return _normalize_x86_variants(raw_value.replace(";", ",").split(","))
+
+
+def get_host_engine_build_config(machine: str | None) -> EngineBuildConfig:
+    if is_x86_machine(machine):
+        return EngineBuildConfig(
+            is_x86=True,
+            primary_extension="openviking.storage.vectordb.engine._x86_sse3",
+            cmake_variants=get_requested_x86_build_variants(),
+        )
+
+    return EngineBuildConfig(
+        is_x86=False,
+        primary_extension="openviking.storage.vectordb.engine._native",
+        cmake_variants=(),
+    )
diff --git a/openviking/storage/vectordb/engine/__init__.py b/openviking/storage/vectordb/engine/__init__.py
@@ -0,0 +1,176 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: Apache-2.0
+"""Stable runtime loader for vectordb native engine variants."""
+
+from __future__ import annotations
+
+import importlib
+import importlib.util
+import os
+import platform
+from types import ModuleType
+
+_BACKEND_MODULES = {
+    "x86_sse3": "_x86_sse3",
+    "x86_avx2": "_x86_avx2",
+    "x86_avx512": "_x86_avx512",
+    "native": "_native",
+}
+_X86_DISPLAY_ORDER = ("x86_sse3", "x86_avx2", "x86_avx512")
+_X86_PRIORITY = ("x86_avx512", "x86_avx2", "x86_sse3")
+_REQUEST_ALIASES = {
+    "sse3": "x86_sse3",
+    "avx2": "x86_avx2",
+    "avx512": "x86_avx512",
+}
+
+
+def _is_x86_machine(machine: str | None = None) -> bool:
+    normalized = (machine or platform.machine() or "").strip().lower()
+    return any(token in normalized for token in ("x86_64", "amd64", "x64", "i386", "i686"))
+
+
+def _module_exists(module_name: str) -> bool:
+    return importlib.util.find_spec(f".{module_name}", __name__) is not None
+
+
+def _available_variants(is_x86: bool) -> tuple[str, ...]:
+    ordered = _X86_DISPLAY_ORDER if is_x86 else ("native",)
+    return tuple(variant for variant in ordered if _module_exists(_BACKEND_MODULES[variant]))
+
+
+def _supported_x86_variants() -> set[str]:
+    supported = {"x86_sse3"}
+    if not _module_exists("_x86_caps"):
+        return supported
+
+    try:
+        caps = importlib.import_module("._x86_caps", __name__)
+    except ImportError:
+        return supported
+
+    reported = getattr(caps, "get_supported_variants", lambda: [])()
+    for variant in reported:
+        normalized = str(variant).strip().lower()
+        if normalized in _BACKEND_MODULES:
+            supported.add(normalized)
+    return supported
+
+
+def _normalize_requested_variant(value: str | None) -> str:
+    normalized = (value or "auto").strip().lower()
+    return _REQUEST_ALIASES.get(normalized, normalized)
+
+
+def _validate_forced_variant(
+    requested: str, *, is_x86: bool, available: tuple[str, ...], supported_x86: set[str]
+) -> None:
+    if is_x86 and requested == "native":
+        raise ImportError("OV_ENGINE_VARIANT=native is only valid on non-x86 platforms")
+
+    if not is_x86 and requested != "native":
+        raise ImportError(
+            f"OV_ENGINE_VARIANT={requested} is not valid on non-x86 platforms; use native"
+        )
+
+    if requested not in _BACKEND_MODULES:
+        raise ImportError(f"Unknown OV_ENGINE_VARIANT={requested}")
+
+    if requested not in available:
+        raise ImportError(
+            f"Requested engine variant {requested} is not packaged in this wheel. "
+            f"Available variants: {', '.join(available) or 'none'}"
+        )
+
+    if is_x86 and requested not in supported_x86:
+        raise ImportError(f"Requested engine variant {requested} is not supported by this CPU")
+
+
+def _select_variant() -> tuple[str | None, tuple[str, ...], str | None]:
+    is_x86 = _is_x86_machine()
+    available = _available_variants(is_x86)
+    requested = _normalize_requested_variant(os.environ.get("OV_ENGINE_VARIANT"))
+
+    if requested != "auto":
+        supported_x86 = _supported_x86_variants() if is_x86 else set()
+        _validate_forced_variant(
+            requested, is_x86=is_x86, available=available, supported_x86=supported_x86
+        )
+        return requested, available, None
+
+    if not is_x86:
+        if "native" not in available:
+            return None, available, "Native engine backend is missing from this wheel"
+        return "native", available, None
+
+    supported_x86 = _supported_x86_variants()
+    for variant in _X86_PRIORITY:
+        if variant in available and variant in supported_x86:
+            return variant, available, None
+
+    if "x86_sse3" in available:
+        return "x86_sse3", available, None
+
+    return None, available, "No compatible x86 engine backend was packaged in this wheel"
+
+
+def _load_backend(variant: str) -> ModuleType:
+    return importlib.import_module(f".{_BACKEND_MODULES[variant]}", __name__)
+
+
+def _export_backend(module: ModuleType) -> tuple[str, ...]:
+    names = getattr(module, "__all__", None)
+    if names is None:
+        names = tuple(name for name in dir(module) if not name.startswith("_"))
+
+    for name in names:
+        globals()[name] = getattr(module, name)
+
+    return tuple(names)
+
+
+class _MissingBackendSymbol:
+    def __init__(self, symbol_name: str, message: str):
+        self._symbol_name = symbol_name
+        self._message = message
+
+    def __call__(self, *args, **kwargs):
+        raise ImportError(f"{self._message}. Missing symbol: {self._symbol_name}")
+
+    def __getattr__(self, name: str):
+        return _MissingBackendSymbol(f"{self._symbol_name}.{name}", self._message)
+
+    def __bool__(self) -> bool:
+        return False
+
+    def __repr__(self) -> str:
+        return f"<missing vectordb engine symbol {self._symbol_name}>"
+
+
+_SELECTED_VARIANT, AVAILABLE_ENGINE_VARIANTS, _ENGINE_IMPORT_ERROR = _select_variant()
+if _SELECTED_VARIANT is None:
+    ENGINE_VARIANT = "unavailable"
+    _BACKEND = None
+    _EXPORTED_NAMES = ()
+else:
+    ENGINE_VARIANT = _SELECTED_VARIANT
+    _BACKEND = _load_backend(ENGINE_VARIANT)
+    _EXPORTED_NAMES = _export_backend(_BACKEND)
+
+
+def __getattr__(name: str):
+    if _BACKEND is None and _ENGINE_IMPORT_ERROR is not None:
+        return _MissingBackendSymbol(name, _ENGINE_IMPORT_ERROR)
+    raise AttributeError(name)
+
+
+__all__ = tuple(
+    sorted(
+        set(_EXPORTED_NAMES).union(
+            {
+                "AVAILABLE_ENGINE_VARIANTS",
+                "ENGINE_VARIANT",
+            }
+        )
+    )
+)
diff --git a/openviking/storage/vectordb/store/bytes_row.py b/openviking/storage/vectordb/store/bytes_row.py
@@ -274,6 +274,9 @@ def deserialize(self, serialized_data):
 try:
     import openviking.storage.vectordb.engine as engine
 
+    if getattr(engine, "ENGINE_VARIANT", "unavailable") == "unavailable":
+        raise ImportError("vectordb engine backend is unavailable")
+
     # Use C++ implementation if available
     BytesRow = engine.BytesRow
     Schema = engine.Schema

diff --git a/pyproject.toml b/pyproject.toml
@@ -175,7 +175,9 @@ openviking = [
     "lib/libagfsbinding.dylib",
     "lib/libagfsbinding.dll",
     "bin/ov",
-    "bin/ov.exe"
+    "bin/ov.exe",
+    "storage/vectordb/engine/*.so",
+    "storage/vectordb/engine/*.pyd",
 ]
 vikingbot = [
     "**/*.mjs",

diff --git a/setup.py b/setup.py
@@ -1,5 +1,7 @@
+import importlib
 import json
 import os
+import platform
 import shutil
 import subprocess
 import sys
@@ -10,10 +12,19 @@
 from setuptools import Extension, setup
 from setuptools.command.build_ext import build_ext
 
+SETUP_DIR = Path(__file__).resolve().parent
+if str(SETUP_DIR) not in sys.path:
+    sys.path.insert(0, str(SETUP_DIR))
+
+get_host_engine_build_config = importlib.import_module(
+    "build_support.x86_profiles"
+).get_host_engine_build_config
+
 CMAKE_PATH = shutil.which("cmake") or "cmake"
 C_COMPILER_PATH = shutil.which("gcc") or "gcc"
 CXX_COMPILER_PATH = shutil.which("g++") or "g++"
 ENGINE_SOURCE_DIR = "src/"
+ENGINE_BUILD_CONFIG = get_host_engine_build_config(platform.machine())
 
 
 class OpenVikingBuildExt(build_ext):
@@ -320,6 +331,9 @@ def _build_ov_cli_artifact_impl(self, ov_cli_dir, binary_name, ov_target_binary)
 
     def build_extension(self, ext):
         """Build a single Python native extension artifact using CMake."""
+        if getattr(self, "_engine_extensions_built", False):
+            return
+
         ext_fullpath = Path(self.get_ext_fullpath(ext.name))
         ext_dir = ext_fullpath.parent.resolve()
         build_dir = Path(self.build_temp) / "cmake_build"
@@ -330,19 +344,19 @@ def build_extension(self, ext):
             lambda: self._build_extension_impl(ext_fullpath, ext_dir, build_dir),
             [(ext_fullpath, f"native extension '{ext.name}'")],
         )
+        self._engine_extensions_built = True
 
     def _build_extension_impl(self, ext_fullpath, ext_dir, build_dir):
         """Invoke CMake to build the Python native extension."""
-        py_output_name = ext_fullpath.stem
-        py_output_suffix = ext_fullpath.suffix
+        py_ext_suffix = sysconfig.get_config_var("EXT_SUFFIX") or ext_fullpath.suffix
 
         cmake_args = [
             f"-S{Path(ENGINE_SOURCE_DIR).resolve()}",
             f"-B{build_dir}",
             "-DCMAKE_BUILD_TYPE=Release",
-            f"-DPY_OUTPUT_DIR={ext_dir}",
-            f"-DPY_OUTPUT_NAME={py_output_name}",
-            f"-DPY_OUTPUT_SUFFIX={py_output_suffix}",
+            f"-DOV_PY_OUTPUT_DIR={ext_dir}",
+            f"-DOV_PY_EXT_SUFFIX={py_ext_suffix}",
+            f"-DOV_X86_BUILD_VARIANTS={';'.join(ENGINE_BUILD_CONFIG.cmake_variants)}",
             "-DCMAKE_VERBOSE_MAKEFILE=ON",
             "-DCMAKE_INSTALL_RPATH=$ORIGIN",
             f"-DPython3_EXECUTABLE={sys.executable}",
@@ -351,7 +365,6 @@ def _build_extension_impl(self, ext_fullpath, ext_dir, build_dir):
             f"-Dpybind11_DIR={pybind11.get_cmake_dir()}",
             f"-DCMAKE_C_COMPILER={C_COMPILER_PATH}",
             f"-DCMAKE_CXX_COMPILER={CXX_COMPILER_PATH}",
-            f"-DOV_X86_SIMD_LEVEL={os.environ.get('OV_X86_SIMD_LEVEL', 'AVX2')}",
         ]
 
         if sys.platform == "darwin":
@@ -374,7 +387,7 @@ def _build_extension_impl(self, ext_fullpath, ext_dir, build_dir):
     # ],
     ext_modules=[
         Extension(
-            name="openviking.storage.vectordb.engine",
+            name=ENGINE_BUILD_CONFIG.primary_extension,
             sources=[],
         )
     ],
@@ -390,6 +403,8 @@ def _build_extension_impl(self, ext_fullpath, ext_dir, build_dir):
             "lib/libagfsbinding.dll",
             "bin/ov",
             "bin/ov.exe",
+            "storage/vectordb/engine/*.so",
+            "storage/vectordb/engine/*.pyd",
         ],
     },
     include_package_data=True,
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		"""Build helpers for OpenViking native artifacts."""