Skip to content

Commit 3040f77

Browse files
robot-clickhouserobot-clickhouse
committed
Backport #84020 to 25.7: Fix geoparquet types breaking client-server protocol
1 parent 571a5d6 commit 3040f77

File tree

4 files changed

+40
-22
lines changed

4 files changed

+40
-22
lines changed

‎src/Formats/SchemaInferenceUtils.cpp‎

Lines changed: 5 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1584,6 +1584,11 @@ DataTypePtr makeNullableRecursively(DataTypePtr type, const FormatSettings & set
15841584
if (which.isNullable())
15851585
return type;
15861586

1587+
/// Leave named compound types unchanged.
1588+
/// E.g. don't turn `Point` into `Tuple(Nullable(Float64), Nullable(Float64))`.
1589+
if (type->hasCustomName())
1590+
return makeNullableSafe(type);
1591+
15871592
if (which.isArray())
15881593
{
15891594
const auto * array_type = assert_cast<const DataTypeArray *>(type.get());
@@ -1655,25 +1660,8 @@ DataTypePtr makeNullableRecursively(DataTypePtr type, const FormatSettings & set
16551660
NamesAndTypesList getNamesAndRecursivelyNullableTypes(const Block & header, const FormatSettings & settings)
16561661
{
16571662
NamesAndTypesList result;
1658-
1659-
std::unordered_map<String, DataTypeCustomDescPtr> custom_descs;
1660-
const auto & prev_schema = header.getNamesAndTypesList();
1661-
for (const auto & [name, type] : prev_schema)
1662-
if (type->hasCustomName() && (type->getTypeId() == TypeIndex::Tuple || type->getTypeId() == TypeIndex::Array))
1663-
custom_descs[name] = std::make_unique<DataTypeCustomDesc>(std::make_unique<DataTypeCustomFixedName>(type->getCustomName()->getName()));
1664-
16651663
for (auto & [name, type] : header.getNamesAndTypesList())
16661664
result.emplace_back(name, makeNullableRecursively(type, settings));
1667-
1668-
for (auto & [name, type] : result)
1669-
{
1670-
auto it = custom_descs.find(name);
1671-
if (it != custom_descs.end())
1672-
{
1673-
type->setCustomization(std::move(it->second));
1674-
}
1675-
}
1676-
16771665
return result;
16781666
}
16791667

‎src/Formats/SchemaInferenceUtils.h‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -101,11 +101,15 @@ void transformFinalInferredJSONTypeIfNeeded(DataTypePtr & data_type, const Forma
101101
void transformInferredJSONTypesFromDifferentFilesIfNeeded(DataTypePtr & first, DataTypePtr & second, const FormatSettings & settings);
102102

103103
/// Make type Nullable recursively:
104-
/// - Type -> Nullable(type)
105-
/// - Array(Type) -> Array(Nullable(Type))
106-
/// - Tuple(Type1, ..., TypeN) -> Tuple(Nullable(Type1), ..., Nullable(TypeN))
107-
/// - Map(KeyType, ValueType) -> Map(KeyType, Nullable(ValueType))
108-
/// - LowCardinality(Type) -> LowCardinality(Nullable(Type))
104+
/// - Type -> Nullable(type)
105+
/// - Array(Type) -> Array(Nullable(Type))
106+
/// - Tuple(Type1, ..., TypeN) -> Tuple(Nullable(Type1), ..., Nullable(TypeN))
107+
/// - Map(KeyType, ValueType) -> Map(KeyType, Nullable(ValueType))
108+
/// - LowCardinality(Type) -> LowCardinality(Nullable(Type))
109+
/// Does not recurse into types with custom name.
110+
/// E.g. type `Point` (aka `Tuple(Float64, Float64)`) stays unchanged as `Point`, it does not become
111+
/// `Tuple(Nullable(Float64), Nullable(Float64))`.
112+
/// But `Bool` becomes `Nullable(Bool)`.
109113
DataTypePtr makeNullableRecursively(DataTypePtr type, const FormatSettings & settings);
110114

111115
/// Call makeNullableRecursively for all types

‎tests/queries/0_stateless/03571_geoparquet_nullable_bug.reference‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
(10,20) [(0,0),(10,0),(10,10),(0,10)] [[(20,20),(50,20),(50,50),(20,50)],[(30,30),(50,50),(50,30)]] [[(0,0),(10,0),(10,10),(0,10)],[(1,1),(2,2),(3,3)]] [[[(0,0),(10,0),(10,10),(0,10)]],[[(20,20),(50,20),(50,50),(20,50)],[(30,30),(50,50),(50,30)]]]
2+
(30,40) [(1,1),(11,1),(11,11),(1,11)] [[(21,21),(51,21),(51,51),(21,51)],[(31,31),(51,51),(51,31)]] [[(1,1),(11,1),(11,11),(1,11)],[(2,2),(3,3),(4,4)]] [[[(1,1),(11,1),(11,11),(1,11)]],[[(21,21),(51,21),(51,51),(21,51)],[(31,31),(51,51),(51,31)]]]

‎tests/queries/0_stateless/03571_geoparquet_nullable_bug.sh‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#!/usr/bin/env bash
2+
# Tags: no-fasttest
3+
4+
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
5+
# shellcheck source=../shell_config.sh
6+
. "$CURDIR"/../shell_config.sh
7+
8+
$CLICKHOUSE_CLIENT --query="
9+
insert into function file('${CLICKHOUSE_TEST_UNIQUE_NAME}.parquet', Parquet, 'point Point, linestring LineString, polygon Polygon, multilinestring MultiLineString, multipolygon MultiPolygon') values (
10+
(10, 20),
11+
[(0, 0), (10, 0), (10, 10), (0, 10)],
12+
[[(20, 20), (50, 20), (50, 50), (20, 50)], [(30, 30), (50, 50), (50, 30)]],
13+
[[(0, 0), (10, 0), (10, 10), (0, 10)], [(1, 1), (2, 2), (3, 3)]],
14+
[[[(0, 0), (10, 0), (10, 10), (0, 10)]], [[(20, 20), (50, 20), (50, 50), (20, 50)],[(30, 30), (50, 50), (50, 30)]]]
15+
), (
16+
(30, 40),
17+
[(1, 1), (11, 1), (11, 11), (1, 11)],
18+
[[(21, 21), (51, 21), (51, 51), (21, 51)], [(31, 31), (51, 51), (51, 31)]],
19+
[[(1, 1), (11, 1), (11, 11), (1, 11)], [(2, 2), (3, 3), (4, 4)]],
20+
[[[(1, 1), (11, 1), (11, 11), (1, 11)]], [[(21, 21), (51, 21), (51, 51), (21, 51)],[(31, 31), (51, 51), (51, 31)]]]
21+
);
22+
select * from file('${CLICKHOUSE_TEST_UNIQUE_NAME}.parquet');"
23+
24+
rm "${CLICKHOUSE_USER_FILES}/${CLICKHOUSE_TEST_UNIQUE_NAME}.parquet"

0 commit comments

Comments
 (0)