Skip to content

Commit 3afa946

Browse files
azatazat
committed
Add a test to ensure that server will wait the server thread pools
v2: add into skip_list v3: print server log on error v4: increase sleep time to trigger some issues under TSAN v5: avoid ports overlaps v6: avoid endless loops to print server log on failure
1 parent c849686 commit 3afa946

File tree

5 files changed

+126
-0
lines changed

5 files changed

+126
-0
lines changed

‎tests/queries/0_stateless/01737_clickhouse_server_wait_server_pool_long.config.xml‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
<?xml version="1.0"?>
2+
<yandex>
3+
<logger>
4+
<level>trace</level>
5+
<console>true</console>
6+
</logger>
7+
8+
<tcp_port>9000</tcp_port>
9+
10+
<path>./</path>
11+
12+
<mark_cache_size>0</mark_cache_size>
13+
14+
<users>
15+
<default>
16+
<password></password>
17+
18+
<networks>
19+
<ip>::/0</ip>
20+
</networks>
21+
22+
<profile>default</profile>
23+
<quota>default</quota>
24+
<access_management>1</access_management>
25+
</default>
26+
</users>
27+
28+
<profiles>
29+
<default/>
30+
</profiles>
31+
32+
<quotas>
33+
<default />
34+
</quotas>
35+
</yandex>

‎tests/queries/0_stateless/01737_clickhouse_server_wait_server_pool_long.reference‎

Whitespace-only changes.

‎tests/queries/0_stateless/01737_clickhouse_server_wait_server_pool_long.sh‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
#!/usr/bin/env bash
2+
3+
CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
4+
# shellcheck source=../shell_config.sh
5+
. "$CUR_DIR"/../shell_config.sh
6+
7+
server_opts=(
8+
"--config-file=$CUR_DIR/$(basename "${BASH_SOURCE[0]}" .sh).config.xml"
9+
"--"
10+
# to avoid multiple listen sockets (complexity for port discovering)
11+
"--listen_host=127.1"
12+
# we will discover the real port later.
13+
"--tcp_port=0"
14+
"--shutdown_wait_unfinished=0"
15+
)
16+
CLICKHOUSE_WATCHDOG_ENABLE=0 $CLICKHOUSE_SERVER_BINARY "${server_opts[@]}" >& clickhouse-server.log &
17+
server_pid=$!
18+
19+
trap cleanup EXIT
20+
function cleanup()
21+
{
22+
kill -9 $server_pid
23+
kill -9 $client_pid
24+
25+
echo "Test failed. Server log:"
26+
cat clickhouse-server.log
27+
rm -f clickhouse-server.log
28+
29+
exit 1
30+
}
31+
32+
server_port=
33+
i=0 retries=300
34+
# wait until server will start to listen (max 30 seconds)
35+
while [[ -z $server_port ]] && [[ $i -lt $retries ]]; do
36+
server_port=$(lsof -n -a -P -i tcp -s tcp:LISTEN -p $server_pid 2>/dev/null | awk -F'[ :]' '/LISTEN/ { print $(NF-1) }')
37+
((++i))
38+
sleep 0.1
39+
done
40+
if [[ -z $server_port ]]; then
41+
echo "Cannot wait for LISTEN socket" >&2
42+
exit 1
43+
fi
44+
45+
# wait for the server to start accepting tcp connections (max 30 seconds)
46+
i=0 retries=300
47+
while ! $CLICKHOUSE_CLIENT_BINARY --host 127.1 --port "$server_port" --format Null -q 'select 1' 2>/dev/null && [[ $i -lt $retries ]]; do
48+
sleep 0.1
49+
done
50+
if ! $CLICKHOUSE_CLIENT_BINARY --host 127.1 --port "$server_port" --format Null -q 'select 1'; then
51+
echo "Cannot wait until server will start accepting connections on <tcp_port>" >&2
52+
exit 1
53+
fi
54+
55+
query_id="$CLICKHOUSE_DATABASE-$SECONDS"
56+
$CLICKHOUSE_CLIENT_BINARY --query_id "$query_id" --host 127.1 --port "$server_port" --format Null -q 'select sleepEachRow(1) from numbers(10)' 2>/dev/null &
57+
client_pid=$!
58+
59+
# wait until the query will appear in processlist (max 10 second)
60+
# (it is enough to trigger the problem)
61+
i=0 retries=1000
62+
while [[ $($CLICKHOUSE_CLIENT_BINARY --host 127.1 --port "$server_port" -q "select count() from system.processes where query_id = '$query_id'") != "1" ]] && [[ $i -lt $retries ]]; do
63+
sleep 0.01
64+
done
65+
if [[ $($CLICKHOUSE_CLIENT_BINARY --host 127.1 --port "$server_port" -q "select count() from system.processes where query_id = '$query_id'") != "1" ]]; then
66+
echo "Cannot wait until the query will start" >&2
67+
exit 1
68+
fi
69+
70+
# send TERM and save the error code to ensure that it is 0 (EXIT_SUCCESS)
71+
kill $server_pid
72+
wait $server_pid
73+
return_code=$?
74+
75+
wait $client_pid
76+
77+
trap '' EXIT
78+
if [ $return_code != 0 ]; then
79+
cat clickhouse-server.log
80+
fi
81+
rm -f clickhouse-server.log
82+
83+
exit $return_code

‎tests/queries/shell_config.sh‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,14 +23,21 @@ export CLICKHOUSE_TEST_ZOOKEEPER_PREFIX="${CLICKHOUSE_TEST_NAME}_${CLICKHOUSE_DA
2323
[ -v CLICKHOUSE_LOG_COMMENT ] && CLICKHOUSE_BENCHMARK_OPT0+=" --log_comment='${CLICKHOUSE_LOG_COMMENT}' "
2424

2525
export CLICKHOUSE_BINARY=${CLICKHOUSE_BINARY:="clickhouse"}
26+
# client
2627
[ -x "$CLICKHOUSE_BINARY-client" ] && CLICKHOUSE_CLIENT_BINARY=${CLICKHOUSE_CLIENT_BINARY:=$CLICKHOUSE_BINARY-client}
2728
[ -x "$CLICKHOUSE_BINARY" ] && CLICKHOUSE_CLIENT_BINARY=${CLICKHOUSE_CLIENT_BINARY:=$CLICKHOUSE_BINARY client}
2829
export CLICKHOUSE_CLIENT_BINARY=${CLICKHOUSE_CLIENT_BINARY:=$CLICKHOUSE_BINARY-client}
2930
export CLICKHOUSE_CLIENT_OPT="${CLICKHOUSE_CLIENT_OPT0:-} ${CLICKHOUSE_CLIENT_OPT:-}"
3031
export CLICKHOUSE_CLIENT=${CLICKHOUSE_CLIENT:="$CLICKHOUSE_CLIENT_BINARY ${CLICKHOUSE_CLIENT_OPT:-}"}
32+
# local
3133
[ -x "${CLICKHOUSE_BINARY}-local" ] && CLICKHOUSE_LOCAL=${CLICKHOUSE_LOCAL:="${CLICKHOUSE_BINARY}-local"}
3234
[ -x "${CLICKHOUSE_BINARY}" ] && CLICKHOUSE_LOCAL=${CLICKHOUSE_LOCAL:="${CLICKHOUSE_BINARY} local"}
3335
export CLICKHOUSE_LOCAL=${CLICKHOUSE_LOCAL:="${CLICKHOUSE_BINARY}-local"}
36+
# server
37+
[ -x "${CLICKHOUSE_BINARY}-server" ] && CLICKHOUSE_SERVER_BINARY=${CLICKHOUSE_SERVER_BINARY:="${CLICKHOUSE_BINARY}-server"}
38+
[ -x "${CLICKHOUSE_BINARY}" ] && CLICKHOUSE_SERVER_BINARY=${CLICKHOUSE_SERVER_BINARY:="${CLICKHOUSE_BINARY} server"}
39+
export CLICKHOUSE_SERVER_BINARY=${CLICKHOUSE_SERVER_BINARY:="${CLICKHOUSE_BINARY}-server"}
40+
# others
3441
export CLICKHOUSE_OBFUSCATOR=${CLICKHOUSE_OBFUSCATOR:="${CLICKHOUSE_BINARY}-obfuscator"}
3542
export CLICKHOUSE_COMPRESSOR=${CLICKHOUSE_COMPRESSOR:="${CLICKHOUSE_BINARY}-compressor"}
3643
export CLICKHOUSE_BENCHMARK=${CLICKHOUSE_BENCHMARK:="${CLICKHOUSE_BINARY}-benchmark ${CLICKHOUSE_BENCHMARK_OPT0:-}"}

‎tests/queries/skip_list.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -696,6 +696,7 @@
696696
"01682_cache_dictionary_complex_key",
697697
"01684_ssd_cache_dictionary_simple_key",
698698
"01685_ssd_cache_dictionary_complex_key",
699+
"01737_clickhouse_server_wait_server_pool_long", // This test is fully compatible to run in parallel, however under ASAN processes are pretty heavy and may fail under flaky adress check.
699700
"01760_system_dictionaries",
700701
"01760_polygon_dictionaries",
701702
"01778_hierarchical_dictionaries",

0 commit comments

Comments
 (0)