Skip to content

Commit c1d0f74

Browse files
arthurpassosarthurpassos
committed
poll system.part_log instead of system.exports
1 parent ebe2ad1 commit c1d0f74

File tree

1 file changed

+30
-10
lines changed

1 file changed

+30
-10
lines changed

‎tests/queries/0_stateless/03572_export_merge_tree_part_limits_and_table_functions.sh‎

Lines changed: 30 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,14 @@ tf_schema_explicit="tf_schema_explicit_${RANDOM}"
1414
mt_table_tf="mt_table_tf_${RANDOM}"
1515

1616
query() {
17-
$CLICKHOUSE_CLIENT --query "$1"
17+
local query_text="$1"
18+
local query_id="$2"
19+
20+
if [ -n "$query_id" ]; then
21+
$CLICKHOUSE_CLIENT --query_id="$query_id" --query "$query_text"
22+
else
23+
$CLICKHOUSE_CLIENT --query "$query_text"
24+
fi
1825
}
1926

2027
query "DROP TABLE IF EXISTS $big_table, $big_destination_max_bytes, $big_destination_max_rows, $mt_table_tf"
@@ -44,16 +51,22 @@ big_part_max_rows=$(query "SELECT name FROM system.parts WHERE database = curren
4451
# ALL EXPORTS HAPPEN HERE
4552
# ============================================================================
4653

54+
# Generate unique query_ids for each export to track them in part_log
55+
export_query_id_1="export_${RANDOM}_1"
56+
export_query_id_2="export_${RANDOM}_2"
57+
export_query_id_3="export_${RANDOM}_3"
58+
export_query_id_4="export_${RANDOM}_4"
59+
4760
# this should generate ~4 files
48-
query "ALTER TABLE $big_table EXPORT PART '$big_part_max_bytes' TO TABLE $big_destination_max_bytes SETTINGS allow_experimental_export_merge_tree_part = 1, export_merge_tree_part_max_bytes_per_file=3500000, output_format_parquet_row_group_size_bytes=1000000"
61+
query "ALTER TABLE $big_table EXPORT PART '$big_part_max_bytes' TO TABLE $big_destination_max_bytes SETTINGS allow_experimental_export_merge_tree_part = 1, export_merge_tree_part_max_bytes_per_file=3500000, output_format_parquet_row_group_size_bytes=1000000" "$export_query_id_1"
4962
# export_merge_tree_part_max_rows_per_file = 1048576 (which is 4194304/4) to generate 4 files
50-
query "ALTER TABLE $big_table EXPORT PART '$big_part_max_rows' TO TABLE $big_destination_max_rows SETTINGS allow_experimental_export_merge_tree_part = 1, export_merge_tree_part_max_rows_per_file=1048576"
63+
query "ALTER TABLE $big_table EXPORT PART '$big_part_max_rows' TO TABLE $big_destination_max_rows SETTINGS allow_experimental_export_merge_tree_part = 1, export_merge_tree_part_max_rows_per_file=1048576" "$export_query_id_2"
5164

5265
echo "---- Table function with schema inheritance (no schema specified)"
53-
query "ALTER TABLE $mt_table_tf EXPORT PART '2022_1_1_0' TO TABLE FUNCTION s3(s3_conn, filename='$tf_schema_inherit', format='Parquet', partition_strategy='hive') PARTITION BY year SETTINGS allow_experimental_export_merge_tree_part = 1"
66+
query "ALTER TABLE $mt_table_tf EXPORT PART '2022_1_1_0' TO TABLE FUNCTION s3(s3_conn, filename='$tf_schema_inherit', format='Parquet', partition_strategy='hive') PARTITION BY year SETTINGS allow_experimental_export_merge_tree_part = 1" "$export_query_id_3"
5467

5568
echo "---- Table function with explicit compatible schema"
56-
query "ALTER TABLE $mt_table_tf EXPORT PART '2023_2_2_0' TO TABLE FUNCTION s3(s3_conn, filename='$tf_schema_explicit', format='Parquet', structure='id UInt64, value String, year UInt16', partition_strategy='hive') PARTITION BY year SETTINGS allow_experimental_export_merge_tree_part = 1"
69+
query "ALTER TABLE $mt_table_tf EXPORT PART '2023_2_2_0' TO TABLE FUNCTION s3(s3_conn, filename='$tf_schema_explicit', format='Parquet', structure='id UInt64, value String, year UInt16', partition_strategy='hive') PARTITION BY year SETTINGS allow_experimental_export_merge_tree_part = 1" "$export_query_id_4"
5770

5871
# Wait for all exports to complete
5972
wait_for_exports() {
@@ -65,10 +78,14 @@ wait_for_exports() {
6578
echo "Waiting for exports to complete (timeout: ${timeout}s)..."
6679

6780
while [ $elapsed -lt $timeout ]; do
68-
# Check if any exports are still in progress for our tables/parts
69-
local active_exports=$(query "SELECT count() FROM system.exports WHERE (source_table = '$big_table' AND part_name IN ('$big_part_max_bytes', '$big_part_max_rows')) OR (source_table = '$mt_table_tf' AND part_name IN ('2022_1_1_0', '2023_2_2_0'))" | tr -d '\n')
81+
# Flush logs to ensure part_log entries are visible
82+
query "SYSTEM FLUSH LOGS" > /dev/null 2>&1 || true
83+
84+
# Wait for part_log entries - these are written synchronously when export completes
85+
# Check if all expected exports have corresponding part_log entries by query_id
86+
local completed_count=$(query "SELECT count() FROM system.part_log WHERE event_type = 'ExportPart' AND query_id IN ('$export_query_id_1', '$export_query_id_2', '$export_query_id_3', '$export_query_id_4')" | tr -d '\n')
7087

71-
if [ "$active_exports" = "0" ]; then
88+
if [ "$completed_count" = "4" ]; then
7289
echo "All exports completed."
7390
return 0
7491
fi
@@ -78,8 +95,11 @@ wait_for_exports() {
7895
done
7996

8097
echo "Timeout waiting for exports to complete after ${timeout}s"
81-
echo "Remaining exports:"
82-
query "SELECT source_table, part_name, elapsed, rows_read, total_rows_to_read FROM system.exports WHERE (source_table = '$big_table' AND part_name IN ('$big_part_max_bytes', '$big_part_max_rows')) OR (source_table = '$mt_table_tf' AND part_name IN ('2022_1_1_0', '2023_2_2_0'))"
98+
query "SYSTEM FLUSH LOGS" > /dev/null 2>&1 || true
99+
echo "Completed exports in part_log:"
100+
query "SELECT query_id, table, part_name, event_time FROM system.part_log WHERE event_type = 'ExportPart' AND query_id IN ('$export_query_id_1', '$export_query_id_2', '$export_query_id_3', '$export_query_id_4')"
101+
echo "Remaining exports in system.exports:"
102+
query "SELECT source_table, part_name, elapsed, rows_read, total_rows_to_read FROM system.exports WHERE ((source_table = '$big_table' AND part_name IN ('$big_part_max_bytes', '$big_part_max_rows')) OR (source_table = '$mt_table_tf' AND part_name IN ('2022_1_1_0', '2023_2_2_0')))"
83103
return 1
84104
}
85105

0 commit comments

Comments
 (0)