Skip to content

Commit b14566a

Browse files
test(conformance): wait for a connected gateway on every installer
Replace the K3s-specific boot readiness hook with one shared gateway wait in the conformance playbook. Skip conformance and collect the existing diagnostics when the gateway never connects. Discover the gateway Service port with its ClusterIP, and capture both iptables backends in K3s diagnostics. Signed-off-by: Matthew Grossman <[email protected]>
1 parent 6538559 commit b14566a

10 files changed

Lines changed: 62 additions & 116 deletions

File tree

‎CI.md‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -106,14 +106,17 @@ K3s development default, currently v1.0.3. OpenShell also supports v0.4.6 throug
106106
its v1alpha1 fallback, so v0.5.0 is not the overall minimum supported version.
107107

108108
The conformance CLI runs on the K3s node inside the tmachine guest and connects
109-
directly to the gateway's default ClusterIP Service on port 8080. Installation
110-
discovers the Service IP and registers it once; the installed disk preserves that
111-
registration across test boots. Automated boots wait for the API, nodes, gateway
112-
StatefulSet and a connected CLI before conformance. Interactive shell boots skip
113-
this preparation so failed gateways remain debuggable. Recreating the Service
114-
requires reprovisioning the fixture rather than repairing registrations at boot.
115-
Installation, preparation and conformance failures preserve bounded diagnostics
116-
under `artifacts/tmachine-diagnostics`, also uploaded by `Integration Tests`.
109+
directly to the gateway's default ClusterIP Service on its gRPC port. Installation
110+
discovers the Service address and registers it once; the installed disk preserves that
111+
registration across test boots. Recreating the Service requires reprovisioning
112+
the fixture rather than repairing registrations at boot. K3s installation and
113+
conformance failures preserve bounded diagnostics under
114+
`artifacts/tmachine-diagnostics`, also uploaded by `Integration Tests`.
115+
116+
On every installer, the conformance playbook waits up to five minutes for
117+
`openshell status` to report a connected gateway before running tests, because
118+
each test boot restarts the gateway. Interactive shell boots skip this wait so
119+
failed gateways remain debuggable.
117120

118121
### Run only the policy advisor conformance tests
119122

‎tests/ansible/playbooks/conformance/cli.yaml‎

Lines changed: 25 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,20 @@
5555
register: openshell_cli
5656
changed_when: false
5757

58+
# Every test boot restarts the gateway, and some installers take minutes to
59+
# recover. Wait once here; the runner's per-scenario preflight stays short.
60+
- name: Wait for the registered gateway
61+
ansible.builtin.command:
62+
argv: [timeout, 10s, "{{ openshell_cli.stdout }}", status, --output, json]
63+
register: gateway_status
64+
changed_when: false
65+
until: >-
66+
gateway_status.rc == 0 and
67+
(gateway_status.stdout | from_json).status == 'connected'
68+
retries: 60
69+
delay: 5
70+
ignore_errors: true
71+
5872
- name: Run OpenShell conformance archive
5973
ansible.builtin.command:
6074
argv:
@@ -77,13 +91,14 @@
7791
register: conformance_result
7892
changed_when: false
7993
failed_when: false
94+
when: gateway_status is succeeded
8095

8196
# Preserve both streams for failures without adding passing-test output to
8297
# every tmachine run.
8398
- name: Show OpenShell conformance diagnostics
8499
ansible.builtin.debug:
85100
var: conformance_result
86-
when: conformance_result.rc != 0
101+
when: conformance_result.rc | default(0) != 0
87102

88103
- name: Read OpenShell gateway logs
89104
become: true
@@ -99,17 +114,23 @@
99114
register: openshell_gateway_logs
100115
changed_when: false
101116
failed_when: false
102-
when: conformance_result.rc != 0
117+
when: conformance_result.rc | default(1) != 0
103118

104119
- name: Show OpenShell gateway logs
105120
ansible.builtin.debug:
106121
var: openshell_gateway_logs.stdout_lines
107-
when: conformance_result.rc != 0
122+
when: conformance_result.rc | default(1) != 0
108123

109124
- name: Collect K3s conformance diagnostics
110125
ansible.builtin.include_role:
111126
name: openshell_k3s_diagnostics
112-
when: conformance_result.rc != 0
127+
when: conformance_result.rc | default(1) != 0
128+
129+
- name: Require a connected gateway
130+
ansible.builtin.assert:
131+
that:
132+
- gateway_status is succeeded
133+
fail_msg: OpenShell gateway did not connect before conformance; see gateway diagnostics
113134

114135
- name: Require OpenShell conformance success
115136
ansible.builtin.assert:

‎tests/ansible/playbooks/openshell-k3s-ready.yaml‎

Lines changed: 0 additions & 9 deletions
This file was deleted.

‎tests/ansible/playbooks/openshell-k3s.yaml‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -139,16 +139,27 @@
139139
environment:
140140
KUBECONFIG: /etc/rancher/k3s/k3s.yaml
141141

142-
- name: Discover the gateway ClusterIP
143-
ansible.builtin.command: /usr/local/bin/k3s kubectl --request-timeout=10s --namespace openshell get service openshell --output=jsonpath={.spec.clusterIP}
144-
register: k3s_gateway_ip
142+
- name: Discover the gateway ClusterIP and gRPC port
143+
ansible.builtin.command:
144+
argv:
145+
- /usr/local/bin/k3s
146+
- kubectl
147+
- --request-timeout=10s
148+
- --namespace
149+
- openshell
150+
- get
151+
- service
152+
- openshell
153+
- --output=jsonpath={.spec.clusterIP}:{.spec.ports[?(@.name=="grpc")].port}
154+
register: k3s_gateway_address
145155
changed_when: false
146156

157+
# Rejects headless Services (ClusterIP "None") and a missing grpc port.
147158
- name: Require a gateway Service address
148159
ansible.builtin.assert:
149160
that:
150-
- k3s_gateway_ip.stdout | length > 0
151-
- k3s_gateway_ip.stdout != 'None'
161+
- k3s_gateway_address.stdout is match('^[0-9.]+:[0-9]+$')
162+
fail_msg: "Unexpected gateway Service address: {{ k3s_gateway_address.stdout }}"
152163
rescue:
153164
- name: Collect K3s installation diagnostics
154165
ansible.builtin.include_role:
@@ -162,7 +173,6 @@
162173
hosts: all
163174
gather_facts: false
164175
vars:
165-
openshell_client_gateway_endpoint: "http://{{ k3s_gateway_ip.stdout }}:8080"
176+
openshell_client_gateway_endpoint: "http://{{ k3s_gateway_address.stdout }}"
166177
roles:
167178
- openshell_client
168-
- openshell_k3s_ready

‎tests/ansible/roles/openshell_k3s_diagnostics/files/collect.sh‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,14 @@ collect() {
2323
-p ExecMainPID -p ActiveEnterTimestamp -p ExecMainStartTimestamp \
2424
-p StartLimitIntervalUSec -p StartLimitBurst -p RestartUSec
2525
collect 'K3s boot journal' journalctl -b --no-pager --lines=500 -u k3s.service
26-
collect 'Gateway Service routing rules' bash -c \
27-
'iptables-save | awk "/openshell\/openshell:grpc/ { print }"'
26+
# kube-proxy may program either iptables backend; dump whichever exist.
27+
# shellcheck disable=SC2016 # Expanded by the inner shell.
28+
collect 'Gateway Service routing rules' bash -c '
29+
for save in iptables-nft-save iptables-legacy-save; do
30+
command -v "$save" >/dev/null || continue
31+
echo "# $save"
32+
"$save" 2>/dev/null | awk "/openshell\/openshell:grpc/ { print }"
33+
done'
2834
collect 'K3s version' /usr/local/bin/k3s --version
2935
collect 'API readiness' /usr/local/bin/k3s kubectl --request-timeout=5s get --raw=/readyz
3036
collect 'Nodes' /usr/local/bin/k3s kubectl --request-timeout=5s get nodes -o wide

‎tests/ansible/roles/openshell_k3s_diagnostics/tasks/main.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,6 @@
2626
- name: Fetch K3s diagnostics
2727
ansible.builtin.fetch:
2828
src: /var/lib/openshell/diagnostics/k3s.txt
29-
dest: "{{ lookup('env', 'TMACHINE_DIAGNOSTICS_DIR') | default(role_path + '/../../../../artifacts/tmachine-diagnostics', true) }}/{{ inventory_hostname }}/k3s.txt"
29+
dest: "{{ role_path }}/../../../../artifacts/tmachine-diagnostics/{{ inventory_hostname }}/k3s.txt"
3030
flat: true
3131
failed_when: false

‎tests/ansible/roles/openshell_k3s_ready/tasks/main.yaml‎

Lines changed: 0 additions & 50 deletions
This file was deleted.

‎tests/config.nix‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,6 @@ let
8585
name = "k3s";
8686
use_galaxy = false;
8787
playbooks = [ "ansible/playbooks/openshell-k3s.yaml" ];
88-
prepare_playbooks = [ "ansible/playbooks/openshell-k3s-ready.yaml" ];
8988
inputs = {
9089
agent_sandbox_version = "0.5.0";
9190
openshell_cli_binary = "../artifacts/binaries/${muslTarget}/openshell";
@@ -214,7 +213,6 @@ let
214213
root=$(git rev-parse --show-toplevel)
215214
cd "$root/tests"
216215
export ANSIBLE_CONFIG="$PWD/ansible/ansible.cfg"
217-
export TMACHINE_DIAGNOSTICS_DIR="$root/artifacts/tmachine-diagnostics"
218216
exec ${tmachine}/bin/tmachine --config ${config} "$@"
219217
'';
220218
};

‎tests/tmachine/src/config.rs‎

Lines changed: 0 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -40,34 +40,6 @@ pub struct Installer {
4040
pub use_galaxy: bool,
4141
pub playbooks: Vec<PathBuf>,
4242
pub inputs: BTreeMap<String, PathBuf>,
43-
/// Readiness checks run on non-interactive test boots, including cached installations.
44-
#[serde(default)]
45-
pub prepare_playbooks: Vec<PathBuf>,
46-
}
47-
48-
#[cfg(test)]
49-
mod tests {
50-
use super::Installer;
51-
52-
#[test]
53-
fn existing_installers_need_no_boot_preparation() {
54-
let installer: Installer =
55-
serde_saphyr::from_str("name: none\nuse_galaxy: false\nplaybooks: []\ninputs: {}\n")
56-
.unwrap();
57-
assert!(installer.prepare_playbooks.is_empty());
58-
}
59-
60-
#[test]
61-
fn installer_can_request_boot_preparation() {
62-
let installer: Installer = serde_saphyr::from_str(
63-
"name: k3s\nuse_galaxy: false\nplaybooks: []\ninputs: {}\nprepare_playbooks: [ready.yaml]\n",
64-
)
65-
.unwrap();
66-
assert_eq!(
67-
installer.prepare_playbooks,
68-
[std::path::PathBuf::from("ready.yaml")]
69-
);
70-
}
7143
}
7244

7345
#[derive(Clone, Deserialize)]

‎tests/tmachine/src/qemu/test.rs‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -26,11 +26,6 @@ pub async fn test(
2626
let image = QemuImage::create(&install_disk, test_disk).await;
2727
let vm = QemuVm::start(&image).await;
2828

29-
// Installation readiness does not survive the shutdown of the cached VM.
30-
// Keep the interactive shell accessible when diagnosing boot failures.
31-
if !testsuite.interactive {
32-
run_playbooks(&installer.prepare_playbooks, &installer.inputs).await?;
33-
}
3429
run_playbooks(&testsuite.playbooks, &testsuite.inputs).await?;
3530

3631
if testsuite.interactive {

0 commit comments

Comments
 (0)