Skip to content

Commit ef080d8

Browse files
committed
perf(e2e): streamline guest gateway startup
Signed-off-by: Drew Newberry <[email protected]>
1 parent 78d3d33 commit ef080d8

3 files changed

Lines changed: 78 additions & 42 deletions

File tree

‎e2e/configs/gateway/docker.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,4 +24,4 @@ ttl_secs = 0
2424
default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest"
2525
image_pull_policy = "IfNotPresent"
2626
sandbox_namespace = "openshell-e2e"
27-
supervisor_bin = ".cache/openshell-e2e/bin/openshell-sandbox"
27+
supervisor_image = "localhost/openshell/supervisor:e2e-vm"

‎e2e/run.sh‎

Lines changed: 63 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,10 @@ gateway_config_source=${gateway_config}
139139
if ! gateway_config="$(resolve_file "${gateway_config_source}")"; then
140140
die "gateway config does not exist: ${gateway_config_source}"
141141
fi
142+
gateway_driver="$(python3 -c '
143+
import sys, tomllib
144+
print(tomllib.load(open(sys.argv[1], "rb"))["openshell"]["gateway"]["compute_drivers"][0])
145+
' "${gateway_config}")"
142146
if [[ ! ${suite_name} =~ ^[a-z0-9][a-z0-9-]*$ ]]; then
143147
die "suite name must contain only lowercase letters, digits, and hyphens: ${suite_name}"
144148
fi
@@ -153,7 +157,6 @@ if [ -n "${vm}" ] || [ "${#with_configurations[@]}" -gt 0 ]; then
153157
vm=ubuntu
154158
fi
155159
fi
156-
vm_uses_podman=0
157160
if [ "${mode}" = vm ]; then
158161
if [[ ! ${vm} =~ ^[a-z0-9][a-z0-9-]*$ ]]; then
159162
die "invalid VM distro name: ${vm}"
@@ -179,9 +182,6 @@ if [ "${mode}" = vm ]; then
179182
if ! catalog_has_entry "${vm_catalog}" Configurations "${configuration}"; then
180183
die "unknown VM configuration in the Nix test-guest catalog: ${configuration}"
181184
fi
182-
if [ "${configuration}" = podman ]; then
183-
vm_uses_podman=1
184-
fi
185185
done
186186
fi
187187
@@ -283,6 +283,15 @@ done
283283
run_parent="${ROOT}/.cache/openshell-e2e/runs"
284284
mkdir -p "${run_parent}"
285285
run_dir="$(mktemp -d "${run_parent%/}/run.XXXXXX")"
286+
if ! command -v tar >/dev/null 2>&1; then
287+
die "tar is required to package the supervisor image"
288+
fi
289+
supervisor_image=localhost/openshell/supervisor:e2e-vm
290+
supervisor_rootfs="${run_dir}/supervisor-rootfs"
291+
supervisor_archive="${run_dir}/supervisor.tar"
292+
mkdir -p "${supervisor_rootfs}"
293+
install -m 0555 "${linux_sandbox_bin}" "${supervisor_rootfs}/openshell-sandbox"
294+
tar -C "${supervisor_rootfs}" -cf "${supervisor_archive}" openshell-sandbox
286295
child_pid=
287296
runtime_log=
288297
keep=0
@@ -376,10 +385,21 @@ export OPENSHELL_GATEWAY="${gateway_name}"
376385
export OPENSHELL_BIN="${host_cli_bin}"
377386
378387
if [ "${mode}" = host ]; then
379-
e2e_align_docker_host_with_cli_context
380-
staged_supervisor="${host_runtime_dir}/.cache/openshell-e2e/bin/openshell-sandbox"
381-
mkdir -p "$(dirname "${staged_supervisor}")"
382-
ln -sfn "${linux_sandbox_bin}" "${staged_supervisor}"
388+
case "${gateway_driver}" in
389+
docker)
390+
e2e_align_docker_host_with_cli_context
391+
docker import \
392+
--change 'ENTRYPOINT ["/openshell-sandbox"]' \
393+
"${supervisor_archive}" \
394+
"${supervisor_image}" >/dev/null
395+
;;
396+
podman)
397+
podman import \
398+
--change 'ENTRYPOINT ["/openshell-sandbox"]' \
399+
"${supervisor_archive}" \
400+
"${supervisor_image}" >/dev/null
401+
;;
402+
esac
383403
384404
runtime_log="${run_dir}/gateway.log"
385405
echo "==> Starting host gateway at ${gateway_endpoint}"
@@ -396,17 +416,6 @@ else
396416
guest_launcher="${run_dir}/launch-gateway.sh"
397417
guest_launcher_path=/home/openshell/.cache/openshell-e2e/bin/launch-gateway
398418
guest_supervisor_archive_path=/home/openshell/.cache/openshell-e2e/supervisor.tar
399-
supervisor_archive=
400-
if [ "${vm_uses_podman}" -eq 1 ]; then
401-
if ! command -v tar >/dev/null 2>&1; then
402-
die "tar is required for Podman VM mode"
403-
fi
404-
supervisor_rootfs="${run_dir}/supervisor-rootfs"
405-
supervisor_archive="${run_dir}/supervisor.tar"
406-
mkdir -p "${supervisor_rootfs}"
407-
install -m 0555 "${linux_sandbox_bin}" "${supervisor_rootfs}/openshell-sandbox"
408-
tar -C "${supervisor_rootfs}" -cf "${supervisor_archive}" openshell-sandbox
409-
fi
410419
config_payload="$(base64 <"${gateway_config}" | tr -d '\r\n')"
411420
jwt_signing_payload="$(base64 <"${jwt_source_dir}/signing.pem" | tr -d '\r\n')"
412421
jwt_public_payload="$(base64 <"${jwt_source_dir}/public.pem" | tr -d '\r\n')"
@@ -441,14 +450,26 @@ export XDG_CACHE_HOME=\${state_root}/xdg/cache
441450
export XDG_DATA_HOME=\${state_root}/xdg/data
442451
export XDG_STATE_HOME=\${state_root}/xdg/state
443452
report_timing "guest gateway setup" "\${phase_started_at}"
444-
if [ -f "${guest_supervisor_archive_path}" ]; then
445-
phase_started_at=\${SECONDS}
453+
phase_started_at=\${SECONDS}
454+
case '${gateway_driver}' in
455+
docker)
456+
docker import \
457+
--change 'ENTRYPOINT ["/openshell-sandbox"]' \
458+
"${guest_supervisor_archive_path}" \
459+
"${supervisor_image}" >/dev/null
460+
;;
461+
podman)
446462
podman --url "unix:///run/user/\$(id -u)/podman/podman.sock" import \
447463
--change 'ENTRYPOINT ["/openshell-sandbox"]' \
448464
"${guest_supervisor_archive_path}" \
449-
localhost/openshell/supervisor:e2e-vm >/dev/null
450-
report_timing "Podman supervisor import" "\${phase_started_at}"
465+
"${supervisor_image}" >/dev/null
466+
;;
467+
esac
468+
report_timing "${gateway_driver} supervisor import" "\${phase_started_at}"
469+
if [ '${gateway_driver}' = podman ]; then
451470
phase_started_at=\${SECONDS}
471+
# Keep the primary gateway listener on loopback and bridge the rootless
472+
# container callback path temporarily. Remove this relay when #2492 lands.
452473
relay_socket=\${state_root}/podman-gateway.sock
453474
rm -f "\${relay_socket}"
454475
socat "UNIX-LISTEN:\${relay_socket},fork" TCP:127.0.0.1:8080 &
@@ -475,7 +496,7 @@ fi
475496
cd /home/openshell
476497
exec /usr/local/bin/openshell-gateway \
477498
--config "\${config_path}" \
478-
--bind-address 0.0.0.0 \
499+
--bind-address 127.0.0.1 \
479500
--port ${guest_port} \
480501
--disable-tls
481502
EOF
@@ -491,14 +512,9 @@ EOF
491512
vm_args+=(
492513
--copy "${guest_gateway_bin}:/usr/local/bin/openshell-gateway"
493514
--copy "${guest_launcher}:${guest_launcher_path}"
515+
--copy "${supervisor_archive}:${guest_supervisor_archive_path}"
494516
--forward-port "${host_port}:${guest_port}"
495517
)
496-
if [ "${vm_uses_podman}" -eq 0 ]; then
497-
vm_args+=(--copy "${linux_sandbox_bin}:/home/openshell/.cache/openshell-e2e/bin/openshell-sandbox")
498-
fi
499-
if [ -n "${supervisor_archive}" ]; then
500-
vm_args+=(--copy "${supervisor_archive}:${guest_supervisor_archive_path}")
501-
fi
502518
if [ "${keep}" -eq 1 ]; then
503519
vm_args+=(--keep)
504520
fi
@@ -535,6 +551,21 @@ wait_for_gateway() {
535551
local elapsed=0
536552
local process_status
537553
local probe_log="${run_dir}/gateway-probe.log"
554+
local reported_timings=0
555+
local timing_count
556+
557+
report_vm_progress() {
558+
if [ "${mode}" != vm ]; then
559+
return
560+
fi
561+
timing_count="$(grep -c '^==> Timing:' "${runtime_log}" || true)"
562+
if [ "${timing_count}" -le "${reported_timings}" ]; then
563+
return
564+
fi
565+
sed -n 's/^==> Timing: / /p' "${runtime_log}" |
566+
sed -n "$((reported_timings + 1)),${timing_count}p"
567+
reported_timings=${timing_count}
568+
}
538569
539570
echo "==> Waiting up to ${gateway_ready_timeout}s for gateway readiness"
540571
while :; do
@@ -555,13 +586,11 @@ wait_for_gateway() {
555586
fi
556587
return "${process_status}"
557588
fi
589+
report_vm_progress
558590
if probe_gateway "${probe_log}" &&
559591
grep -q "Connected" "${probe_log}"; then
592+
report_vm_progress
560593
echo "==> Gateway ready after ${elapsed}s"
561-
if [ "${mode}" = vm ] && grep -q '^==> Timing:' "${runtime_log}"; then
562-
echo "==> VM gateway readiness breakdown"
563-
sed -n 's/^==> Timing: / /p' "${runtime_log}"
564-
fi
565594
return 0
566595
fi
567596
sleep 1

‎nix/test-guest/run.sh‎

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -332,6 +332,7 @@ qemu_log=${run_dir}/qemu.log
332332
qemu_pid=
333333
ssh_port=
334334
ssh_args=()
335+
ssh_forward_args=()
335336
scp_args=()
336337
ansible_config=${run_dir}/ansible.cfg
337338
ansible_inventory=${run_dir}/inventory.ini
@@ -439,11 +440,6 @@ for attempt in $(seq 1 5); do
439440
done
440441
fi
441442
netdev_arg="user,id=net0,hostfwd=tcp:127.0.0.1:${ssh_port}-:22"
442-
for forward_spec in "${forward_ports[@]}"; do
443-
host_port=${forward_spec%%:*}
444-
guest_port=${forward_spec#*:}
445-
netdev_arg+=",hostfwd=tcp:127.0.0.1:${host_port}-:${guest_port}"
446-
done
447443
: >"${qemu_log}"
448444
echo "==> Booting ${distro} (${TEST_GUEST_ARCHITECTURE}) with QEMU/${TEST_GUEST_ACCELERATOR}"
449445
"${TEST_GUEST_QEMU}" \
@@ -519,6 +515,16 @@ scp_args=(
519515
-o StrictHostKeyChecking=no
520516
-o UserKnownHostsFile=/dev/null
521517
)
518+
if [ "${#forward_ports[@]}" -gt 0 ]; then
519+
ssh_forward_args+=(-o ExitOnForwardFailure=yes)
520+
for forward_spec in "${forward_ports[@]}"; do
521+
host_port=${forward_spec%%:*}
522+
guest_port=${forward_spec#*:}
523+
ssh_forward_args+=(
524+
-L "127.0.0.1:${host_port}:127.0.0.1:${guest_port}"
525+
)
526+
done
527+
fi
522528

523529
echo "==> Waiting up to ${ssh_wait_seconds} seconds for SSH on 127.0.0.1:${ssh_port}"
524530
ssh_ready=0
@@ -635,12 +641,13 @@ ssh "${ssh_args[@]}" -O exit [email protected] >/dev/null 2>&1 || true
635641

636642
echo "==> Test guest ready: ${distro} (SSH port ${ssh_port})"
637643
if [ "${#guest_command[@]}" -eq 0 ]; then
638-
ssh -t "${ssh_args[@]}" [email protected]
644+
ssh -t "${ssh_args[@]}" "${ssh_forward_args[@]}" [email protected]
639645
else
640646
printf -v quoted_command '%q ' "${guest_command[@]}"
641647
# quoted_command is shell-escaped locally before it reaches the guest.
642648
# shellcheck disable=SC2029
643-
ssh "${ssh_args[@]}" [email protected] "bash -lc $(printf '%q' "${quoted_command}")"
649+
ssh "${ssh_args[@]}" "${ssh_forward_args[@]}" \
650+
[email protected] "bash -lc $(printf '%q' "${quoted_command}")"
644651
fi
645652

646653
echo "==> Shutting down ${distro}"

0 commit comments

Comments
 (0)