Skip to content

Commit ff9b291

Browse files
steipetesteipete
authored
fix(docker): explain unsupported E2E resource limits (#109587)
* fix(docker): diagnose unsupported E2E resource limits * test(docker): track diagnostic temp dirs * fix(docker): bound resource diagnostic capture * fix(docker): restrict diagnostic FIFO access
1 parent 6c07572 commit ff9b291

4 files changed

Lines changed: 385 additions & 38 deletions

File tree

‎scripts/lib/docker-e2e-container.sh‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,15 @@
33
# Shared helpers for Docker E2E scripts that keep a named container running
44
# while polling readiness from the host.
55

6+
DOCKER_E2E_CONTAINER_LIB_DIR="${BASH_SOURCE[0]}"
7+
if [[ "$DOCKER_E2E_CONTAINER_LIB_DIR" == */* ]]; then
8+
DOCKER_E2E_CONTAINER_LIB_DIR="${DOCKER_E2E_CONTAINER_LIB_DIR%/*}"
9+
else
10+
DOCKER_E2E_CONTAINER_LIB_DIR=.
11+
fi
12+
DOCKER_E2E_CONTAINER_LIB_DIR="$(cd "$DOCKER_E2E_CONTAINER_LIB_DIR" && pwd)"
13+
source "$DOCKER_E2E_CONTAINER_LIB_DIR/docker-e2e-resource-diagnostics.sh"
14+
615
docker_e2e_timeout_bin() {
716
if command -v timeout >/dev/null 2>&1; then
817
printf '%s\n' timeout
@@ -142,11 +151,7 @@ docker_e2e_docker_cmd() {
142151
if [ "${1:-}" = "run" ]; then
143152
shift
144153
docker_e2e_docker_run_resource_args "$@" || return $?
145-
if [ "${#DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" -gt 0 ]; then
146-
docker_e2e_timeout_cmd "$timeout_value" docker run "${DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" "$@"
147-
else
148-
docker_e2e_timeout_cmd "$timeout_value" docker run "$@"
149-
fi
154+
docker_e2e_docker_run_with_resource_diagnostics "$timeout_value" "$@"
150155
return
151156
fi
152157
docker_e2e_timeout_cmd "$timeout_value" docker "$@"
@@ -157,11 +162,7 @@ docker_e2e_docker_run_cmd() {
157162
if [ "${1:-}" = "run" ]; then
158163
shift
159164
docker_e2e_docker_run_resource_args "$@" || return $?
160-
if [ "${#DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" -gt 0 ]; then
161-
docker_e2e_timeout_cmd "$timeout_value" docker run "${DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" "$@"
162-
else
163-
docker_e2e_timeout_cmd "$timeout_value" docker run "$@"
164-
fi
165+
docker_e2e_docker_run_with_resource_diagnostics "$timeout_value" "$@"
165166
return
166167
fi
167168
docker_e2e_timeout_cmd "$timeout_value" docker "$@"

‎scripts/lib/docker-e2e-package.sh‎

Lines changed: 23 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,10 @@
77
DOCKER_E2E_PACKAGE_LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
88
ROOT_DIR="${ROOT_DIR:-$(cd "$DOCKER_E2E_PACKAGE_LIB_DIR/../.." && pwd)}"
99

10+
if ! declare -F docker_e2e_docker_run_with_resource_diagnostics >/dev/null 2>&1; then
11+
source "$DOCKER_E2E_PACKAGE_LIB_DIR/docker-e2e-resource-diagnostics.sh"
12+
fi
13+
1014
if ! declare -F run_logged >/dev/null 2>&1; then
1115
source "$DOCKER_E2E_PACKAGE_LIB_DIR/docker-e2e-logs.sh"
1216
fi
@@ -108,30 +112,10 @@ if ! declare -F docker_e2e_docker_run_resource_args >/dev/null 2>&1; then
108112
fi
109113
}
110114
fi
111-
if ! declare -F docker_e2e_docker_run_cmd >/dev/null 2>&1; then
112-
docker_e2e_docker_run_cmd() {
113-
if [ "${1:-}" = "run" ]; then
114-
shift
115-
docker_e2e_docker_run_resource_args "$@" || return $?
116-
if declare -F docker_e2e_timeout_cmd >/dev/null 2>&1; then
117-
if [ "${#DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" -gt 0 ]; then
118-
docker_e2e_timeout_cmd "${DOCKER_COMMAND_TIMEOUT:-${OPENCLAW_DOCKER_E2E_RUN_TIMEOUT:-3600s}}" docker run "${DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" "$@"
119-
else
120-
docker_e2e_timeout_cmd "${DOCKER_COMMAND_TIMEOUT:-${OPENCLAW_DOCKER_E2E_RUN_TIMEOUT:-3600s}}" docker run "$@"
121-
fi
122-
return
123-
fi
124-
if [ "${#DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" -gt 0 ]; then
125-
set -- run "${DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" "$@"
126-
else
127-
set -- run "$@"
128-
fi
129-
fi
130-
if declare -F docker_e2e_timeout_cmd >/dev/null 2>&1; then
131-
docker_e2e_timeout_cmd "${DOCKER_COMMAND_TIMEOUT:-${OPENCLAW_DOCKER_E2E_RUN_TIMEOUT:-3600s}}" docker "$@"
132-
return
133-
fi
134-
local timeout_value="${DOCKER_COMMAND_TIMEOUT:-${OPENCLAW_DOCKER_E2E_RUN_TIMEOUT:-3600s}}"
115+
if ! declare -F docker_e2e_timeout_cmd >/dev/null 2>&1; then
116+
docker_e2e_timeout_cmd() {
117+
local timeout_value="$1"
118+
shift
135119
local timeout_bin=""
136120
if command -v timeout >/dev/null 2>&1; then
137121
timeout_bin="timeout"
@@ -140,16 +124,29 @@ if ! declare -F docker_e2e_docker_run_cmd >/dev/null 2>&1; then
140124
fi
141125
if [ -n "$timeout_bin" ]; then
142126
if "$timeout_bin" --kill-after=1s 1s true >/dev/null 2>&1; then
143-
"$timeout_bin" --kill-after=30s "$timeout_value" docker "$@"
127+
"$timeout_bin" --kill-after=30s "$timeout_value" "$@"
144128
else
145-
"$timeout_bin" "$timeout_value" docker "$@"
129+
"$timeout_bin" "$timeout_value" "$@"
146130
fi
147131
return
148132
fi
149133
echo "timeout command not found; cannot bound Docker run after ${timeout_value}" >&2
150134
return 127
151135
}
152136
fi
137+
if ! declare -F docker_e2e_docker_run_cmd >/dev/null 2>&1; then
138+
docker_e2e_docker_run_cmd() {
139+
if [ "${1:-}" = "run" ]; then
140+
shift
141+
docker_e2e_docker_run_resource_args "$@" || return $?
142+
docker_e2e_docker_run_with_resource_diagnostics \
143+
"${DOCKER_COMMAND_TIMEOUT:-${OPENCLAW_DOCKER_E2E_RUN_TIMEOUT:-3600s}}" \
144+
"$@"
145+
return
146+
fi
147+
docker_e2e_timeout_cmd "${DOCKER_COMMAND_TIMEOUT:-${OPENCLAW_DOCKER_E2E_RUN_TIMEOUT:-3600s}}" docker "$@"
148+
}
149+
fi
153150

154151
docker_e2e_abs_path() {
155152
local file="$1"

‎scripts/lib/docker-e2e-resource-diagnostics.sh‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
#!/usr/bin/env bash
2+
3+
docker_e2e_resource_limit_error_file() {
4+
local status="$1"
5+
local stderr_file="$2"
6+
local line
7+
[ "$status" = "125" ] || return 1
8+
9+
local text=""
10+
while IFS= read -r line || [ -n "$line" ]; do
11+
text="${text}${line}
12+
"
13+
done <"$stderr_file"
14+
15+
case "$text" in
16+
*"controller pids is not available"* | *"cgroup controller pids is not available"* | \
17+
*"NanoCPUs can not be set"* | *"CPU CFS scheduler"* | \
18+
*"cgroup is not mounted"* | *"cgroup not mounted"* | \
19+
*"resource limit not supported"* | *"resource limits not supported"*)
20+
return 0
21+
;;
22+
esac
23+
return 1
24+
}
25+
26+
docker_e2e_resource_limit_stderr_file() {
27+
local template="${TMPDIR:-/tmp}/openclaw-docker-resource-limits.XXXXXX"
28+
if command -v mktemp >/dev/null 2>&1; then
29+
mktemp "$template"
30+
return
31+
fi
32+
if [ -x /usr/bin/mktemp ]; then
33+
/usr/bin/mktemp "$template"
34+
return
35+
fi
36+
echo "mktemp command not found; cannot capture Docker resource-limit diagnostics" >&2
37+
return 127
38+
}
39+
40+
docker_e2e_tee_bin() {
41+
if command -v tee >/dev/null 2>&1; then
42+
command -v tee
43+
return
44+
fi
45+
if [ -x /usr/bin/tee ]; then
46+
printf '%s\n' /usr/bin/tee
47+
return
48+
fi
49+
return 1
50+
}
51+
52+
docker_e2e_tail_bin() {
53+
if command -v tail >/dev/null 2>&1; then
54+
command -v tail
55+
return
56+
fi
57+
if [ -x /usr/bin/tail ]; then
58+
printf '%s\n' /usr/bin/tail
59+
return
60+
fi
61+
return 1
62+
}
63+
64+
docker_e2e_remove_diagnostic_file() {
65+
if command -v rm >/dev/null 2>&1; then
66+
rm -f "$@"
67+
return
68+
fi
69+
/bin/rm -f "$@"
70+
}
71+
72+
docker_e2e_print_resource_limit_error() {
73+
echo "Docker E2E resource limits are incompatible with this Docker runtime. Fix its cgroup support or explicitly opt out with OPENCLAW_DOCKER_E2E_DISABLE_RESOURCE_LIMITS=1." >&2
74+
}
75+
76+
docker_e2e_docker_run_with_resource_diagnostics() {
77+
local timeout_value="$1"
78+
shift
79+
if [ "${#DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" -eq 0 ]; then
80+
docker_e2e_timeout_cmd "$timeout_value" docker run "$@"
81+
return
82+
fi
83+
84+
local stderr_file=""
85+
if ! stderr_file="$(docker_e2e_resource_limit_stderr_file)"; then
86+
docker_e2e_timeout_cmd \
87+
"$timeout_value" \
88+
docker run "${DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" "$@"
89+
return
90+
fi
91+
local tee_bin=""
92+
if ! tee_bin="$(docker_e2e_tee_bin)"; then
93+
docker_e2e_remove_diagnostic_file "$stderr_file"
94+
docker_e2e_timeout_cmd \
95+
"$timeout_value" \
96+
docker run "${DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" "$@"
97+
return
98+
fi
99+
local tail_bin=""
100+
if ! tail_bin="$(docker_e2e_tail_bin)"; then
101+
docker_e2e_remove_diagnostic_file "$stderr_file"
102+
docker_e2e_timeout_cmd \
103+
"$timeout_value" \
104+
docker run "${DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" "$@"
105+
return
106+
fi
107+
local stderr_fifo="${stderr_file}.stderr.pipe"
108+
local capture_fifo="${stderr_file}.capture.pipe"
109+
local mkfifo_bin=""
110+
if command -v mkfifo >/dev/null 2>&1; then
111+
mkfifo_bin="$(command -v mkfifo)"
112+
elif [ -x /usr/bin/mkfifo ]; then
113+
mkfifo_bin=/usr/bin/mkfifo
114+
fi
115+
local fifo_status=1
116+
if [ -n "$mkfifo_bin" ]; then
117+
local previous_umask=""
118+
previous_umask="$(umask)"
119+
umask 077
120+
"$mkfifo_bin" "$stderr_fifo" "$capture_fifo" && fifo_status=0 || fifo_status="$?"
121+
umask "$previous_umask"
122+
fi
123+
if [ "$fifo_status" -ne 0 ]; then
124+
docker_e2e_remove_diagnostic_file "$stderr_file" "$stderr_fifo" "$capture_fifo"
125+
docker_e2e_timeout_cmd \
126+
"$timeout_value" \
127+
docker run "${DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" "$@"
128+
return
129+
fi
130+
131+
"$tail_bin" -c 65536 <"$capture_fifo" >"$stderr_file" &
132+
local tail_pid="$!"
133+
"$tee_bin" "$capture_fifo" <"$stderr_fifo" >&2 &
134+
local tee_pid="$!"
135+
local run_status=0
136+
if docker_e2e_timeout_cmd \
137+
"$timeout_value" \
138+
docker run "${DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" "$@" \
139+
2>"$stderr_fifo"; then
140+
run_status=0
141+
else
142+
run_status="$?"
143+
fi
144+
wait "$tee_pid" || true
145+
wait "$tail_pid" || true
146+
147+
if docker_e2e_resource_limit_error_file "$run_status" "$stderr_file"; then
148+
docker_e2e_print_resource_limit_error
149+
fi
150+
docker_e2e_remove_diagnostic_file "$stderr_file" "$stderr_fifo" "$capture_fifo"
151+
return "$run_status"
152+
}

0 commit comments

Comments
 (0)