Skip to content

Commit 8bbac25

Browse files
Fix dict to STRUCT issue (#660)
Converting a dict to a `STRUCT` type filled a field whose key was missing from another key's value. A `TUPLE` target took the first value in every member. `TransformDictionaryToStruct` in src/native/python_conversion.cpp now matches every field to its key, ignoring case, before converting any value. It raises the existing "We could not convert the object ... to the desired target type" error in two cases: - a field has no key; - two keys are equal ignoring case. Each key fills at most one field. A TUPLE is filled by position, and a dict with no target type works exactly as before. Notes: - A pandas object column whose renamed dict row falls outside the type-inference sample now raises during the scan. Before, it returned corrupted rows. - duckdb.Value({b'a': 42}, con.struct_type({b'a': 'INTEGER'})) now raises. It returned the right value only by luck, because struct_type decodes bytes names; that's a follow-up.
2 parents e38a20d + e54f315 commit 8bbac25

2 files changed

Lines changed: 232 additions & 16 deletions

File tree

‎src/native/python_conversion.cpp‎

Lines changed: 48 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -61,12 +61,12 @@ static Value EmptyMapValue() {
6161
return Value::MAP(ListType::GetChildType(map_type), vector<Value>());
6262
}
6363

64-
vector<Identifier> TransformStructKeys(nb::handle keys, idx_t size, const LogicalType &type = LogicalType::UNKNOWN) {
64+
static vector<Identifier> TransformStructKeys(nb::handle keys, idx_t size) {
6565
vector<Identifier> res;
6666
res.reserve(size);
6767
for (idx_t i = 0; i < size; i++) {
68-
// Stringify via str() so non-string keys (e.g. the integer keys of a hashable-key MAP, which DuckDB
69-
// produces as a plain {1: 10} dict) are accepted -- nanobind's nb::cast<std::string> rejects non-str.
68+
// str() accepts non-string keys, such as the integer keys of a MAP that DuckDB returns as a plain {1: 10}
69+
// dict, which nanobind's nb::cast<std::string> rejects.
7070
res.emplace_back(Identifier(nb::cast<std::string>(nb::str(keys.attr("__getitem__")(i)))));
7171
}
7272
return res;
@@ -122,26 +122,58 @@ bool DictionaryHasMapFormat(const PyDictionary &dict) {
122122

123123
Value TransformDictionaryToStruct(optional_ptr<ClientContext> context, const PyDictionary &dict,
124124
const LogicalType &target_type = LogicalType::UNKNOWN) {
125-
auto struct_keys = TransformStructKeys(dict.keys, dict.len, target_type);
126-
127-
bool struct_target = target_type.id() == LogicalTypeId::STRUCT || target_type.id() == LogicalTypeId::TUPLE;
125+
auto conversion_error = [&]() {
126+
return InvalidInputException("We could not convert the object %s to the desired target type (%s)",
127+
dict.ToString(), target_type.ToString());
128+
};
129+
bool struct_target = StructType::IsStruct(target_type);
128130
if (struct_target && dict.len != StructType::GetChildCount(target_type)) {
129-
throw InvalidInputException("We could not convert the object %s to the desired target type (%s)",
130-
dict.ToString(), target_type.ToString());
131+
throw conversion_error();
132+
}
133+
134+
child_list_t<Value> struct_values;
135+
if (StructType::IsUnnamed(target_type)) {
136+
// A TUPLE has no member names, so its members take the dict's values in dict order.
137+
for (idx_t i = 0; i < dict.len; i++) {
138+
auto val = TransformPythonValue(context, dict.values.attr("__getitem__")(i),
139+
StructType::GetChildType(target_type, i));
140+
struct_values.emplace_back(make_pair(StructType::GetChildName(target_type, i), std::move(val)));
141+
}
142+
return Value::STRUCT(std::move(struct_values));
143+
}
144+
145+
auto struct_keys = TransformStructKeys(dict.keys, dict.len);
146+
if (!struct_target) {
147+
for (idx_t i = 0; i < dict.len; i++) {
148+
auto val = TransformPythonValue(context, dict.values.attr("__getitem__")(i), LogicalType::UNKNOWN);
149+
struct_values.emplace_back(make_pair(std::move(struct_keys[i]), std::move(val)));
150+
}
151+
return Value::STRUCT(std::move(struct_values));
131152
}
132153

154+
// Keys that are distinct in Python but equal as identifiers would both claim one STRUCT field.
133155
identifier_map_t<idx_t> key_mapping;
134156
for (idx_t i = 0; i < struct_keys.size(); i++) {
135-
key_mapping[struct_keys[i]] = i;
157+
if (!key_mapping.emplace(struct_keys[i], i).second) {
158+
throw conversion_error();
159+
}
160+
}
161+
// Every field finds its key before any value converts, so a key mismatch never surfaces as a value cast error.
162+
vector<idx_t> value_indexes;
163+
value_indexes.reserve(dict.len);
164+
for (idx_t i = 0; i < dict.len; i++) {
165+
auto entry = key_mapping.find(StructType::GetChildName(target_type, i));
166+
if (entry == key_mapping.end()) {
167+
throw conversion_error();
168+
}
169+
value_indexes.push_back(entry->second);
170+
// Fields equal ignoring case would otherwise all read one key.
171+
key_mapping.erase(entry);
136172
}
137-
138-
child_list_t<Value> struct_values;
139173
for (idx_t i = 0; i < dict.len; i++) {
140-
auto &key = struct_target ? StructType::GetChildName(target_type, i) : struct_keys[i];
141-
auto value_index = struct_target ? key_mapping[key] : i;
142-
auto &child_type = struct_target ? StructType::GetChildType(target_type, i) : LogicalType::UNKNOWN;
143-
auto val = TransformPythonValue(context, dict.values.attr("__getitem__")(value_index), child_type);
144-
struct_values.emplace_back(make_pair(std::move(key), std::move(val)));
174+
auto val = TransformPythonValue(context, dict.values.attr("__getitem__")(value_indexes[i]),
175+
StructType::GetChildType(target_type, i));
176+
struct_values.emplace_back(make_pair(StructType::GetChildName(target_type, i), std::move(val)));
145177
}
146178
return Value::STRUCT(std::move(struct_values));
147179
}
Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,184 @@
1+
import re
2+
3+
import pytest
4+
5+
import duckdb
6+
7+
AB = "STRUCT(a INTEGER, b INTEGER)"
8+
NESTED = f"STRUCT(s {AB})"
9+
PAIR = "TUPLE(INTEGER, INTEGER)"
10+
# SQL refuses field names equal ignoring case, but struct_type accepts them.
11+
FIELDS_EQUAL_IGNORING_CASE = {"a": "INTEGER", "A": "INTEGER"}
12+
13+
14+
def resolve(con, target):
15+
return con.type(target) if isinstance(target, str) else con.struct_type(target)
16+
17+
18+
def via_value_parameter(con, target, obj):
19+
return con.execute("SELECT ?", [duckdb.Value(obj, resolve(con, target))]).fetchone()[0]
20+
21+
22+
def via_constant_expression(con, target, obj):
23+
expression = duckdb.ConstantExpression(duckdb.Value(obj, resolve(con, target)))
24+
return con.sql("SELECT 1").select(expression).fetchall()[0][0]
25+
26+
27+
def via_udf_result(con, target, obj):
28+
con.create_function("produce", lambda: obj, [], resolve(con, target))
29+
return con.sql("SELECT produce()").fetchall()[0][0]
30+
31+
32+
def via_udf_list_result(con, target, obj):
33+
con.create_function("produce", lambda: [obj], [], con.list_type(resolve(con, target)))
34+
(result,) = con.sql("SELECT produce()").fetchall()[0][0]
35+
return result
36+
37+
38+
PATHS = [
39+
pytest.param(via_value_parameter, id="value_parameter"),
40+
pytest.param(via_constant_expression, id="constant_expression"),
41+
pytest.param(via_udf_result, id="udf_result"),
42+
pytest.param(via_udf_list_result, id="udf_list_result"),
43+
]
44+
45+
CONVERTED = [
46+
pytest.param(AB, {"a": 1, "b": 2}, {"a": 1, "b": 2}, id="same_keys"),
47+
pytest.param(AB, {"b": 2, "a": 1}, {"a": 1, "b": 2}, id="keys_in_another_order"),
48+
pytest.param(AB, {"A": 1, "B": 2}, {"a": 1, "b": 2}, id="keys_in_another_case"),
49+
pytest.param(AB, {"a": None, "b": 2}, {"a": None, "b": 2}, id="null_field"),
50+
pytest.param(NESTED, {"s": {"b": 2, "a": 1}}, {"s": {"a": 1, "b": 2}}, id="nested_keys_in_another_order"),
51+
pytest.param(NESTED, {"s": None}, {"s": None}, id="nested_null_struct"),
52+
pytest.param(PAIR, {"k1": 1, "k2": 2}, (1, 2), id="unnamed_target_by_position"),
53+
pytest.param(PAIR, {"b": 2, "a": 1}, (2, 1), id="unnamed_target_follows_dict_order"),
54+
pytest.param("STRUCT(\"b'a'\" INTEGER)", {b"a": 42}, {"b'a'": 42}, id="bytes_key_named_by_str"),
55+
pytest.param(
56+
"STRUCT(\"b'a'\" INTEGER, a INTEGER)",
57+
{b"a": 1, "a": 2},
58+
{"b'a'": 1, "a": 2},
59+
id="bytes_key_and_its_text_are_distinct_keys",
60+
),
61+
pytest.param('STRUCT("1" INTEGER)', {1: 10}, {"1": 10}, id="integer_key"),
62+
]
63+
64+
# Each case names the dict and the STRUCT type the error reports, which for a nested case is the inner pair.
65+
REFUSED = [
66+
pytest.param(AB, {"a": 1, "c": 3}, {"a": 1, "c": 3}, AB, id="one_key_missing"),
67+
pytest.param(AB, {"a": "bad", "c": 2}, {"a": "bad", "c": 2}, AB, id="key_missing_after_a_bad_value"),
68+
pytest.param(AB, {"k1": 1, "k2": 2}, {"k1": 1, "k2": 2}, AB, id="no_key_matches"),
69+
pytest.param(AB, {"a": 1, "A": 2}, {"a": 1, "A": 2}, AB, id="keys_equal_ignoring_case"),
70+
pytest.param(AB, {"a": 1, "b": 2, "c": 3}, {"a": 1, "b": 2, "c": 3}, AB, id="extra_key"),
71+
pytest.param(AB, {"a": 1}, {"a": 1}, AB, id="fewer_keys"),
72+
pytest.param(NESTED, {"s": {"k1": 1, "k2": 2}}, {"k1": 1, "k2": 2}, AB, id="nested_no_key_matches"),
73+
pytest.param(PAIR, {"a": 1, "b": 2, "c": 3}, {"a": 1, "b": 2, "c": 3}, PAIR, id="unnamed_target_extra_key"),
74+
pytest.param(AB, {"key": [1], "value": [2]}, {"key": [1], "value": [2]}, AB, id="map_shaped_dict"),
75+
pytest.param(AB, {"b": 20, b"a": 10}, {"b": 20, b"a": 10}, AB, id="bytes_key_does_not_name_its_text"),
76+
pytest.param(
77+
FIELDS_EQUAL_IGNORING_CASE,
78+
{"a": 1, "A": 2},
79+
{"a": 1, "A": 2},
80+
"STRUCT(a INTEGER, A INTEGER)",
81+
id="keys_and_fields_equal_ignoring_case",
82+
),
83+
pytest.param(
84+
FIELDS_EQUAL_IGNORING_CASE,
85+
{"a": 1, "b": 2},
86+
{"a": 1, "b": 2},
87+
"STRUCT(a INTEGER, A INTEGER)",
88+
id="fields_equal_ignoring_case",
89+
),
90+
]
91+
92+
93+
def conversion_error(obj, target):
94+
return re.escape(f"We could not convert the object {obj!r} to the desired target type ({target})")
95+
96+
97+
class TestDictToStruct:
98+
@pytest.mark.parametrize("path", PATHS)
99+
@pytest.mark.parametrize(("target", "obj", "expected"), CONVERTED)
100+
def test_converted(self, path, target, obj, expected):
101+
con = duckdb.connect()
102+
assert path(con, target, obj) == expected
103+
104+
@pytest.mark.parametrize("path", PATHS)
105+
@pytest.mark.parametrize(("target", "obj", "reported_obj", "reported_target"), REFUSED)
106+
def test_refused(self, path, target, obj, reported_obj, reported_target):
107+
con = duckdb.connect()
108+
with pytest.raises(duckdb.InvalidInputException, match=conversion_error(reported_obj, reported_target)):
109+
path(con, target, obj)
110+
111+
def test_untyped_bytes_key_named_by_str(self):
112+
con = duckdb.connect()
113+
assert con.execute("SELECT ?", [{b"a": 42}]).fetchall() == [({"b'a'": 42},)]
114+
115+
@pytest.mark.parametrize(
116+
("rows", "expected_type", "expected"),
117+
[
118+
pytest.param([{b"a": 1}], "STRUCT(\"b'a'\" INTEGER)", [({"b'a'": 1},)], id="bytes_key"),
119+
pytest.param(
120+
[{b"a": 1, "b": 2}],
121+
"STRUCT(\"b'a'\" INTEGER, b INTEGER)",
122+
[({"b'a'": 1, "b": 2},)],
123+
id="bytes_and_str_keys",
124+
),
125+
],
126+
)
127+
def test_pandas_bytes_keys_named_by_str(self, rows, expected_type, expected):
128+
pd = pytest.importorskip("pandas")
129+
con = duckdb.connect()
130+
df = pd.DataFrame({"col": rows}) # noqa: F841
131+
rel = con.sql("SELECT col FROM df")
132+
assert str(rel.types[0]) == expected_type
133+
assert rel.fetchall() == expected
134+
135+
def test_pandas_keys_equal_ignoring_case_refused(self):
136+
pd = pytest.importorskip("pandas")
137+
con = duckdb.connect()
138+
obj = {"a": 1, "A": 2}
139+
df = pd.DataFrame({"col": [obj]}) # noqa: F841
140+
with pytest.raises(duckdb.InvalidInputException, match=conversion_error(obj, "STRUCT(a INTEGER, A INTEGER)")):
141+
con.sql("SELECT col FROM df").fetchall()
142+
143+
def test_numpy_object_row_outside_the_sample(self):
144+
np = pytest.importorskip("numpy")
145+
con = duckdb.connect()
146+
con.execute("SET pandas_analyze_sample=1")
147+
col = np.empty(2, dtype=object)
148+
col[0] = {"a": 10, "b": 20}
149+
col[1] = {"b": 2, "a": 1}
150+
data = {"col": col} # noqa: F841
151+
assert con.sql("SELECT col FROM data").fetchall() == [({"a": 10, "b": 20},), ({"a": 1, "b": 2},)]
152+
col[1] = {"k1": 1, "k2": 2}
153+
with pytest.raises(duckdb.InvalidInputException, match=conversion_error(col[1], AB)):
154+
con.sql("SELECT col FROM data").fetchall()
155+
156+
@pytest.mark.parametrize(
157+
("row", "expected"),
158+
[
159+
pytest.param({"b": 2, "a": 1}, {"a": 1, "b": 2}, id="keys_in_another_order"),
160+
pytest.param({"A": 1, "B": 2}, {"a": 1, "b": 2}, id="keys_in_another_case"),
161+
],
162+
)
163+
def test_pandas_row_outside_the_sample_converted(self, row, expected):
164+
pd = pytest.importorskip("pandas")
165+
con = duckdb.connect()
166+
con.execute("SET pandas_analyze_sample=1")
167+
df = pd.DataFrame({"col": [{"a": 10, "b": 20}, row]}) # noqa: F841
168+
assert con.sql("SELECT col FROM df").fetchall() == [({"a": 10, "b": 20},), (expected,)]
169+
170+
@pytest.mark.parametrize(
171+
"row",
172+
[
173+
pytest.param({"a": 1, "c": 3}, id="one_key_missing"),
174+
pytest.param({"k1": 1, "k2": 2}, id="no_key_matches"),
175+
pytest.param({"a": 1, "b": 2, "c": 3}, id="extra_key"),
176+
],
177+
)
178+
def test_pandas_row_outside_the_sample_refused(self, row):
179+
pd = pytest.importorskip("pandas")
180+
con = duckdb.connect()
181+
con.execute("SET pandas_analyze_sample=1")
182+
df = pd.DataFrame({"col": [{"a": 10, "b": 20}, row]}) # noqa: F841
183+
with pytest.raises(duckdb.InvalidInputException, match=conversion_error(row, AB)):
184+
con.sql("SELECT col FROM df").fetchall()

0 commit comments

Comments
 (0)