Skip to content

Commit ea34b03

Browse files
evertlammertsevertlammerts
authored
Fix alias of DuckDBPyRelation.query (#521)
The arguments for `QueryRelation` are reversed causing `DuckDBPyRelation.alias` created via `query` to be the full SQL query string: https://github.com/duckdb/duckdb-python/blob/664d7e4667f92c88138641a9bf65043b586cc49e/src/duckdb_py/pyrelation.cpp#L1564-L1565 This PR simply reverses the 3rd and 4th argument and sets the alias to `"query_relation_" + StringUtil::GenerateRandomName(16)` following conventions in other parts of the relational API. Closes #468.
2 parents 94733ba + 44c45e2 commit ea34b03

87 files changed

Lines changed: 4032 additions & 2938 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/actions/ccache-action/action.yml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
name: 'Setup Ccache'
2+
inputs:
3+
key:
4+
description: 'Cache key (defaults to github.job)'
5+
required: false
6+
default: ''
7+
runs:
8+
using: "composite"
9+
steps:
10+
- name: Setup Ccache
11+
uses: hendrikmuhs/ccache-action@main
12+
with:
13+
key: ${{ inputs.key || github.job }}
14+
save: ${{ github.repository != 'duckdb/duckdb-python' || contains('["refs/heads/main", "refs/heads/v1.4-andium", "refs/heads/v1.5-variegata"]', github.ref) }}
15+
# Dump verbose ccache statistics report at end of CI job.
16+
verbose: 1
17+
# Increase per-directory limit: 5*1024 MB / 16 = 320 MB.
18+
# Note: `layout=subdirs` computes the size limit divided by 16 dirs.
19+
# See also: https://ccache.dev/manual/4.9.html#_cache_size_management
20+
max-size: 1500MB
21+
# Evicts all cache files that were not touched during the job run.
22+
# Removing cache files from previous runs avoids creating huge caches.
23+
evict-old-files: 'job'

‎.github/workflows/packaging_wheels.yml‎

Lines changed: 119 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -25,15 +25,112 @@ on:
2525
type: string
2626

2727
jobs:
28+
seed_wheels:
29+
name: 'Seed: cp314-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}'
30+
strategy:
31+
fail-fast: false
32+
matrix:
33+
python: [ cp314 ]
34+
platform:
35+
- { os: windows-2022, arch: amd64, cibw_system: win }
36+
- { os: windows-11-arm, arch: ARM64, cibw_system: win }
37+
- { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux }
38+
- { os: ubuntu-24.04-arm, arch: aarch64, cibw_system: manylinux }
39+
- { os: macos-15, arch: arm64, cibw_system: macosx }
40+
- { os: macos-15, arch: universal2, cibw_system: macosx }
41+
- { os: macos-15-intel, arch: x86_64, cibw_system: macosx }
42+
minimal:
43+
- ${{ inputs.minimal }}
44+
exclude:
45+
- { minimal: true, platform: { arch: universal2 } }
46+
runs-on: ${{ matrix.platform.os }}
47+
env:
48+
CCACHE_DIR: ${{ github.workspace }}/.ccache
49+
### cibuildwheel configuration
50+
#
51+
# This is somewhat brittle, so be careful with changes. Some notes for our future selves (and others):
52+
# - cibw will change its cwd to a temp dir and create a separate venv for testing. It then installs the wheel it
53+
# built into that venv, and run the CIBW_TEST_COMMAND. We have to install all dependencies ourselves, and make
54+
# sure that the pytest config in pyproject.toml is available.
55+
# - CIBW_BEFORE_TEST installs the test dependencies by exporting them into a pylock.toml. At the time of writing,
56+
# `uv sync --no-install-project` had problems correctly resolving dependencies using resolution environments
57+
# across all platforms we build for. This might be solved in newer uv versions.
58+
# - CIBW_TEST_COMMAND specifies pytest conf from pyproject.toml. --confcutdir is needed to prevent pytest from
59+
# traversing the full filesystem, which produces an error on Windows.
60+
# - CIBW_TEST_SKIP we always skip tests for *-macosx_universal2 builds, because we run tests for arm64 and x86_64.
61+
CIBW_TEST_SKIP: ${{ inputs.testsuite == 'none' && '*' || '*-macosx_universal2' }}
62+
CIBW_TEST_SOURCES: tests
63+
CIBW_BEFORE_TEST: >
64+
uv export --only-group test --no-emit-project --quiet --output-file pylock.toml --directory {project} &&
65+
uv pip install -r pylock.toml
66+
CIBW_TEST_COMMAND: >
67+
uv run -v pytest --confcutdir=. --rootdir . -c {project}/pyproject.toml ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }}
68+
69+
steps:
70+
- name: Checkout DuckDB Python
71+
uses: actions/checkout@v4
72+
with:
73+
ref: ${{ inputs.duckdb-python-sha }}
74+
fetch-depth: 0
75+
submodules: true
76+
77+
- name: Checkout DuckDB
78+
shell: bash
79+
if: ${{ inputs.duckdb-sha }}
80+
run: |
81+
cd external/duckdb
82+
git fetch origin
83+
git checkout ${{ inputs.duckdb-sha }}
84+
85+
- name: Set CIBW_ENVIRONMENT
86+
shell: bash
87+
run: |
88+
cibw_env=""
89+
if [[ "${{ matrix.platform.cibw_system }}" == "manylinux" ]]; then
90+
cibw_env="CCACHE_DIR=/host${{ github.workspace }}/.ccache"
91+
fi
92+
if [[ -n "${{ inputs.set-version }}" ]]; then
93+
cibw_env="${cibw_env:+$cibw_env }OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }}"
94+
fi
95+
if [[ -n "$cibw_env" ]]; then
96+
echo "CIBW_ENVIRONMENT=${cibw_env}" >> $GITHUB_ENV
97+
fi
98+
99+
- name: Setup Ccache
100+
uses: ./.github/actions/ccache-action
101+
with:
102+
key: ${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}
103+
104+
# Install Astral UV, which will be used as build-frontend for cibuildwheel
105+
- uses: astral-sh/setup-uv@v7
106+
with:
107+
version: "0.9.0"
108+
enable-cache: false
109+
cache-suffix: -${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}
110+
111+
- name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels
112+
uses: pypa/[email protected]
113+
env:
114+
CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }}
115+
CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}
116+
117+
- name: Upload wheel
118+
uses: actions/upload-artifact@v4
119+
with:
120+
name: wheel-${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}
121+
path: wheelhouse/*.whl
122+
compression-level: 0
123+
28124
build_wheels:
29125
name: 'Wheel: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}'
126+
needs: seed_wheels
30127
strategy:
31128
fail-fast: false
32129
matrix:
33-
python: [ cp310, cp311, cp312, cp313, cp314 ]
130+
python: [ cp310, cp311, cp312, cp313 ]
34131
platform:
35-
- { os: windows-2022, arch: amd64, cibw_system: win }
36-
- { os: windows-11-arm, arch: ARM64, cibw_system: win } # cibw requires ARM64 to be uppercase
132+
- { os: windows-2025, arch: amd64, cibw_system: win }
133+
- { os: windows-11-arm, arch: ARM64, cibw_system: win }
37134
- { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux }
38135
- { os: ubuntu-24.04-arm, arch: aarch64, cibw_system: manylinux }
39136
- { os: macos-15, arch: arm64, cibw_system: macosx }
@@ -46,9 +143,10 @@ jobs:
46143
- { minimal: true, python: cp312 }
47144
- { minimal: true, python: cp313 }
48145
- { minimal: true, platform: { arch: universal2 } }
49-
- { python: cp310, platform: { os: windows-11-arm, arch: ARM64 } } # too many dependency problems for win arm64
146+
- { python: cp310, platform: { os: windows-11-arm, arch: ARM64 } }
50147
runs-on: ${{ matrix.platform.os }}
51148
env:
149+
CCACHE_DIR: ${{ github.workspace }}/.ccache
52150
### cibuildwheel configuration
53151
#
54152
# This is somewhat brittle, so be careful with changes. Some notes for our future selves (and others):
@@ -85,11 +183,24 @@ jobs:
85183
git fetch origin
86184
git checkout ${{ inputs.duckdb-sha }}
87185
88-
# Make sure that OVERRIDE_GIT_DESCRIBE is propagated to cibuildwhel's env, also when it's running linux builds
89-
- name: Set OVERRIDE_GIT_DESCRIBE
186+
- name: Set CIBW_ENVIRONMENT
90187
shell: bash
91-
if: ${{ inputs.set-version != '' }}
92-
run: echo "CIBW_ENVIRONMENT=OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }}" >> $GITHUB_ENV
188+
run: |
189+
cibw_env=""
190+
if [[ "${{ matrix.platform.cibw_system }}" == "manylinux" ]]; then
191+
cibw_env="CCACHE_DIR=/host${{ github.workspace }}/.ccache"
192+
fi
193+
if [[ -n "${{ inputs.set-version }}" ]]; then
194+
cibw_env="${cibw_env:+$cibw_env }OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }}"
195+
fi
196+
if [[ -n "$cibw_env" ]]; then
197+
echo "CIBW_ENVIRONMENT=${cibw_env}" >> $GITHUB_ENV
198+
fi
199+
200+
- name: Setup Ccache
201+
uses: ./.github/actions/ccache-action
202+
with:
203+
key: ${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}
93204

94205
# Install Astral UV, which will be used as build-frontend for cibuildwheel
95206
- uses: astral-sh/setup-uv@v7

‎.github/workflows/release.yml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,6 @@ defaults:
3939
jobs:
4040
build_sdist:
4141
name: Build an sdist and determine versions
42-
if: ${{ github.ref != 'refs/heads/main' }}
4342
uses: ./.github/workflows/packaging_sdist.yml
4443
with:
4544
testsuite: all

‎CMakeLists.txt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@ cmake_minimum_required(VERSION 3.29)
22

33
project(duckdb_py LANGUAGES CXX)
44

5-
# Always use C++11
6-
set(CMAKE_CXX_STANDARD 11)
5+
# Always use C++17
6+
set(CMAKE_CXX_STANDARD 17)
77
set(CMAKE_CXX_STANDARD_REQUIRED ON)
88
set(CMAKE_CXX_EXTENSIONS OFF)
99
set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")

‎_duckdb-stubs/__init__.pyi‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -537,7 +537,9 @@ class DuckDBPyRelation:
537537
def distinct(self) -> DuckDBPyRelation: ...
538538
def except_(self, other_rel: Self) -> DuckDBPyRelation: ...
539539
def execute(self) -> DuckDBPyRelation: ...
540-
def explain(self, type: ExplainType | ExplainTypeLiteral = ExplainType.STANDARD) -> str: ...
540+
def explain(
541+
self, type: ExplainType | ExplainTypeLiteral = ExplainType.STANDARD, format: str | None = None
542+
) -> str: ...
541543
def favg(
542544
self, expression: str, groups: str = "", window_spec: str = "", projected_columns: str = ""
543545
) -> DuckDBPyRelation: ...

‎duckdb/experimental/spark/sql/type_utils.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
IntegerType,
2020
LongType,
2121
MapType,
22+
NullType,
2223
ShortType,
2324
StringType,
2425
StructField,
@@ -27,6 +28,7 @@
2728
TimeNTZType,
2829
TimestampMillisecondNTZType,
2930
TimestampNanosecondNTZType,
31+
TimestampNanosecondType,
3032
TimestampNTZType,
3133
TimestampSecondNTZType,
3234
TimestampType,
@@ -41,6 +43,7 @@
4143
)
4244

4345
_sqltype_to_spark_class = {
46+
"null": NullType,
4447
"boolean": BooleanType,
4548
"utinyint": UnsignedByteType,
4649
"tinyint": ByteType,
@@ -62,9 +65,10 @@
6265
"time with time zone": TimeType,
6366
"timestamp": TimestampNTZType,
6467
"timestamp with time zone": TimestampType,
65-
"timestamp_ms": TimestampNanosecondNTZType,
66-
"timestamp_ns": TimestampMillisecondNTZType,
68+
"timestamp_ms": TimestampMillisecondNTZType,
69+
"timestamp_ns": TimestampNanosecondNTZType,
6770
"timestamp_s": TimestampSecondNTZType,
71+
"timestamptz_ns": TimestampNanosecondType,
6872
"interval": DayTimeIntervalType,
6973
"list": ArrayType,
7074
"struct": StructType,

‎duckdb/experimental/spark/sql/types.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
"TimestampMillisecondNTZType",
5050
"TimestampNTZType",
5151
"TimestampNanosecondNTZType",
52+
"TimestampNanosecondType",
5253
"TimestampSecondNTZType",
5354
"TimestampType",
5455
"UUIDType",
@@ -239,6 +240,26 @@ def fromInternal(self, ts: int) -> datetime.datetime: # noqa: D102
239240
return datetime.datetime.fromtimestamp(ts // 1000000).replace(microsecond=ts % 1000000)
240241

241242

243+
class TimestampNanosecondType(AtomicType, metaclass=DataTypeSingleton):
244+
"""Timestamp (datetime.datetime) data type with timezone information with nanosecond precision."""
245+
246+
def __init__(self) -> None: # noqa: D107
247+
super().__init__(DuckDBPyType("TIMESTAMPTZ_NS"))
248+
249+
def needConversion(self) -> bool: # noqa: D102
250+
return True
251+
252+
@classmethod
253+
def typeName(cls) -> str: # noqa: D102
254+
return "timestamptz_ns"
255+
256+
def toInternal(self, dt: datetime.datetime) -> int: # noqa: D102
257+
raise ContributionsAcceptedError
258+
259+
def fromInternal(self, ts: int) -> datetime.datetime: # noqa: D102
260+
raise ContributionsAcceptedError
261+
262+
242263
class TimestampNTZType(AtomicType, metaclass=DataTypeSingleton):
243264
"""Timestamp (datetime.datetime) data type without timezone information with microsecond precision."""
244265

‎duckdb_packaging/setuptools_scm_version.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
from ._versioning import format_version, parse_version
1313

1414
# MAIN_BRANCH_VERSIONING should be 'True' on main branch only
15-
MAIN_BRANCH_VERSIONING = False
15+
MAIN_BRANCH_VERSIONING = True
1616

1717
SCM_PRETEND_ENV_VAR = "SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DUCKDB"
1818
SCM_GLOBAL_PRETEND_ENV_VAR = "SETUPTOOLS_SCM_PRETEND_VERSION"

‎pyproject.toml‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,12 @@ version_scheme = "duckdb_packaging.setuptools_scm_version:version_scheme"
8888
local_scheme = "no-local-version"
8989
fallback_version = "0.0.1.dev1"
9090

91+
# main only: count dev distance from the last *minor* tag (v*.*.0), so a patch
92+
# tag (e.g. v1.5.4) merged in from a release branch can't reset .devN.
93+
# Release branches must NOT have this, they correctly count from v*.*.* (the default).
94+
[tool.setuptools_scm.scm.git]
95+
describe_command = "git describe --dirty --tags --long --abbrev=40 --match v*.*.0"
96+
9197
# Override: if COVERAGE is set then:
9298
# - we create a RelWithDebInfo build
9399
# - we make sure we use a persistent build dir so we get access to the .gcda files
@@ -122,7 +128,7 @@ cmake.build-type = "Debug"
122128
[[tool.scikit-build.overrides]]
123129
if.state = "editable"
124130
if.env.COVERAGE = false
125-
if.platform-system = "Darwin"
131+
if.platform-system = "(?i)darwin"
126132
inherit.cmake.define = "append"
127133
cmake.define.DISABLE_UNITY = "1"
128134

‎scripts/cache_data.json‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -532,7 +532,9 @@
532532
"polars.DataFrame",
533533
"polars.LazyFrame",
534534
"polars.col",
535-
"polars.lit"
535+
"polars.lit",
536+
"polars.Series",
537+
"polars.Decimal"
536538
],
537539
"required": false
538540
},
@@ -822,5 +824,17 @@
822824
"full_path": "polars.lit",
823825
"name": "lit",
824826
"children": []
827+
},
828+
"polars.Series": {
829+
"type": "attribute",
830+
"full_path": "polars.Series",
831+
"name": "Series",
832+
"children": []
833+
},
834+
"polars.Decimal": {
835+
"type": "attribute",
836+
"full_path": "polars.Decimal",
837+
"name": "Decimal",
838+
"children": []
825839
}
826840
}

0 commit comments

Comments
 (0)