tests: limit memory usage for tests

azat · azat · commit 05a901514709 · 2025-02-19T11:00:55.000+01:00
To avoid situation when the tests eats too much RAM and we spend dozen of time to figure out why MEMORY_LIMIT_EXCEEDED happens on the server. Refs: #75966
diff --git a/ci/jobs/functional_stateful_tests.py b/ci/jobs/functional_stateful_tests.py
@@ -36,7 +36,7 @@ def run_test(
 ):
     test_output_file = f"{temp_dir}/test_result.txt"
 
-    test_command = f"clickhouse-test --jobs 2 --testname --shard --zookeeper --check-zookeeper-session --no-stateless \
+    test_command = f"clickhouse-test --memory-limit {20<<30} --jobs 2 --testname --shard --zookeeper --check-zookeeper-session --no-stateless \
         --hung-check --print-time \
         --capture-client-stacktrace --queries ./tests/queries -- '{test}' \
         | ts '%Y-%m-%d %H:%M:%S' | tee -a \"{test_output_file}\""
diff --git a/ci/jobs/functional_stateless_tests.py b/ci/jobs/functional_stateless_tests.py
@@ -40,7 +40,7 @@ def run_stateless_test(
     nproc = int(Utils.cpu_count() / 2)
     if batch_num and batch_total:
         aux = f"--run-by-hash-total {batch_total} --run-by-hash-num {batch_num-1}"
-    statless_test_command = f"clickhouse-test --testname --shard --zookeeper --check-zookeeper-session --hung-check --print-time \
+    statless_test_command = f"clickhouse-test --memory-limit {20<<30} --testname --shard --zookeeper --check-zookeeper-session --hung-check --print-time \
                 --capture-client-stacktrace --queries /repo/tests/queries --test-runs 1 --hung-check \
                 {'--no-parallel' if no_parallel else ''}  {'--no-sequential' if no_sequiential else ''} \
                 --print-time --jobs {nproc} --report-coverage --report-logs-stats {aux} \
diff --git a/tests/clickhouse-test b/tests/clickhouse-test
@@ -1169,6 +1169,11 @@ def replace_in_file(filename, what, with_what):
     os.system(f"LC_ALL=C sed -i -e 's|{what}|{with_what}|g' {filename}")
 
 
+def prlimit(command, limit):
+    """Limit memory usage for each test to avoid MEMORY_LIMIT_EXCEEDED errors on server"""
+    return f"prlimit --as={limit} {command}"
+
+
 class TestResult:
     def __init__(
         self,
@@ -1890,6 +1895,8 @@ class TestCase:
             )
 
         command = pattern.format(**params)
+        if args.memory_limit:
+            command = prlimit(command, args.memory_limit)
 
         # pylint:disable-next=consider-using-with; TODO: fix
         proc = Popen(command, shell=True, env=os.environ, start_new_session=True)
@@ -3769,6 +3776,12 @@ def parse_args():
         action="store_true",
         help="Capture stacktraces from clickhouse-client/local on errors",
     )
+    parser.add_argument(
+        "--memory-limit",
+        type=int,
+        default=5 << 30,
+        help="Limit memory for the test (for now it is address space, not a real memory, so the limit could be higher the real memory usage)",
+    )
 
     return parser.parse_args()
 
diff --git a/tests/docker_scripts/fasttest_runner.sh b/tests/docker_scripts/fasttest_runner.sh
@@ -275,6 +275,7 @@ function run_tests
     export CLICKHOUSE_SCHEMA_FILES="$FASTTEST_DATA/format_schemas"
 
     local test_opts=(
+        --memory-limit $((5<<30))
         --hung-check
         --fast-tests-only
         --no-random-settings
diff --git a/tests/docker_scripts/stress_runner.sh b/tests/docker_scripts/stress_runner.sh
@@ -266,7 +266,7 @@ fi
 start_server
 
 cd /repo/tests/ || exit 1  # clickhouse-test can find queries dir from there
-python3 /repo/tests/ci/stress.py --hung-check --drop-databases --output-folder /test_output --skip-func-tests "$SKIP_TESTS_OPTION" --global-time-limit 1200 \
+python3 /repo/tests/ci/stress.py --memory-limit $((20<<30)) --hung-check --drop-databases --output-folder /test_output --skip-func-tests "$SKIP_TESTS_OPTION" --global-time-limit 1200 \
     && echo -e "Test script exit code$OK" >> /test_output/test_results.tsv \
     || echo -e "Test script failed$FAIL script exit code: $?" >> /test_output/test_results.tsv
 
diff --git a/tests/docker_scripts/upgrade_runner.sh b/tests/docker_scripts/upgrade_runner.sh
@@ -111,7 +111,7 @@ clickhouse-client --query="SELECT 'Server version: ', version()"
 
 mkdir tmp_stress_output
 
-stress --test-cmd="/usr/bin/clickhouse-test --queries=\"previous_release_repository/tests/queries\""  --upgrade-check --output-folder tmp_stress_output --global-time-limit=1200 \
+stress --test-cmd="/usr/bin/clickhouse-test --memory-limit $((20<<30)) --queries=\"previous_release_repository/tests/queries\""  --upgrade-check --output-folder tmp_stress_output --global-time-limit=1200 \
     && echo -e "Test script exit code$OK" >> /test_output/test_results.tsv \
     || echo -e "Test script failed$FAIL script exit code: $?" >> /test_output/test_results.tsv
 
