Skip to content

Commit 764f0f3

Browse files
JimCodeymichaelneale
andcommitted
test(mesh): exercise remote-only inference with isolated nodes
Co-authored-by: Michael Neale <[email protected]> Signed-off-by: JimCodey <fab731feb32f528977dec0f8e40b557d552d1c795ae5c8d75e7dc071c2b05aa0@meshllm.communities.buzz.xyz>
1 parent 3ea63af commit 764f0f3

6 files changed

Lines changed: 416 additions & 0 deletions

File tree

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
name: Mesh cross-node CPU
2+
on:
3+
workflow_dispatch:
4+
pull_request:
5+
paths:
6+
- 'crates/mesh-compute/**'
7+
- 'scripts/*mesh-cross-node*'
8+
- '.github/workflows/mesh-cross-node.yml'
9+
- 'Cargo.lock'
10+
- 'Cargo.toml'
11+
push:
12+
branches: [main]
13+
paths:
14+
- 'crates/mesh-compute/**'
15+
- 'scripts/*mesh-cross-node*'
16+
- '.github/workflows/mesh-cross-node.yml'
17+
- 'Cargo.lock'
18+
- 'Cargo.toml'
19+
permissions:
20+
contents: read
21+
concurrency:
22+
group: mesh-cross-node-${{ github.ref }}
23+
cancel-in-progress: true
24+
jobs:
25+
remote-only:
26+
runs-on: ubuntu-24.04
27+
timeout-minutes: 20
28+
steps:
29+
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
30+
with:
31+
persist-credentials: false
32+
- uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1
33+
- uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2
34+
with:
35+
key: mesh-cross-node
36+
save-if: ${{ github.event_name == 'push' }}
37+
- uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
38+
with:
39+
path: .cache/mesh-cross-node
40+
key: mesh-smoke-assets-linux-v0.78.1-8030f045
41+
- name: Full mesh crate tests
42+
run: ./bin/cargo test --locked -p buzz-mesh-compute --features mesh
43+
- name: Remote-only completion and provider outage
44+
run: scripts/ci-mesh-cross-node.sh
45+
- name: Evidence (includes ephemeral test identities only)
46+
if: always()
47+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
48+
with:
49+
name: mesh-cross-node
50+
path: test-results/mesh-cross-node
51+
retention-days: 7

‎crates/mesh-compute/Cargo.toml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,3 +31,7 @@ ed25519-dalek = { version = "2", optional = true }
3131

3232
[dev-dependencies]
3333
tokio = { version = "1", features = ["test-util"] }
34+
35+
[[example]]
36+
name = "cross_node"
37+
required-features = ["mesh"]
Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
//! Child process for scripts/mesh-cross-node.py; never opens the desktop or Keychain.
2+
use anyhow::{bail, Context, Result};
3+
use buzz_mesh_compute::{
4+
config::{ClientConfig, ServeConfig},
5+
identity::ensure_owner_at,
6+
lifecycle::{Lifecycle, Phase},
7+
};
8+
use serde_json::{json, Value};
9+
use std::{
10+
io::{BufRead, Write},
11+
path::PathBuf,
12+
time::Duration,
13+
};
14+
15+
fn main() -> Result<()> {
16+
let root = PathBuf::from(std::env::var("BUZZ_MESH_TEST_HOME")?);
17+
anyhow::ensure!(
18+
root.is_absolute() && root.join("SMOKE_ONLY").is_file(),
19+
"isolated test home required"
20+
);
21+
anyhow::ensure!(
22+
std::env::var("HOME")? == root.to_string_lossy(),
23+
"HOME must be isolated"
24+
);
25+
anyhow::ensure!(
26+
std::env::var("BUZZ_MESH_IROH_RELAYS")? == "0",
27+
"public relays must be disabled"
28+
);
29+
let runtime = tokio::runtime::Builder::new_multi_thread()
30+
.enable_all()
31+
.build()?;
32+
let lifecycle = Lifecycle::default();
33+
// stdin is the bounded parent/child control channel. EOF always requests shutdown.
34+
let result = (|| -> Result<()> {
35+
for line in std::io::stdin().lock().lines() {
36+
let command: Value = serde_json::from_str(&line?)?;
37+
let response = runtime.block_on(command_reply(&lifecycle, &root, command));
38+
let response = match response {
39+
Ok(value) => json!({"ok": value}),
40+
Err(error) => json!({"error": format!("{error:#}")}),
41+
};
42+
println!("BUZZ_SMOKE {}", response);
43+
std::io::stdout().flush()?;
44+
}
45+
Ok(())
46+
})();
47+
let stopped = runtime.block_on(lifecycle.stop_and_wait());
48+
result.and(stopped)
49+
}
50+
51+
async fn command_reply(
52+
lifecycle: &Lifecycle,
53+
root: &std::path::Path,
54+
command: Value,
55+
) -> Result<Value> {
56+
match command["op"].as_str().context("op required")? {
57+
"identity" => Ok(json!(ensure_owner_at(&root.join("owner.json"))?)),
58+
"start" => {
59+
let port = |name: &str| -> Result<u16> {
60+
Ok(u16::try_from(
61+
command[name].as_u64().context("port required")?,
62+
)?)
63+
};
64+
let node = ClientConfig {
65+
api_port: port("api_port")?,
66+
console_port: port("console_port")?,
67+
owner_key: root.join("owner.json"),
68+
owner_id: ensure_owner_at(&root.join("owner.json"))?,
69+
trusted_owners: serde_json::from_value(command["owners"].clone())?,
70+
mesh_name: Some(command["mesh"].as_str().context("mesh required")?.into()),
71+
join_token: None,
72+
};
73+
if let Some(model) = command["model"].as_str() {
74+
lifecycle.serve(ServeConfig {
75+
node,
76+
model: model.into(),
77+
max_vram_gb: None,
78+
})?;
79+
} else {
80+
lifecycle.start(node)?;
81+
}
82+
tokio::time::timeout(Duration::from_secs(240), async {
83+
while lifecycle.phase() == Phase::Starting {
84+
tokio::time::sleep(Duration::from_millis(100)).await;
85+
}
86+
})
87+
.await?;
88+
anyhow::ensure!(
89+
lifecycle.phase() == Phase::Ready,
90+
"startup failed: {:?}",
91+
lifecycle.phase()
92+
);
93+
Ok(json!({"serving": lifecycle.is_serving()}))
94+
}
95+
"status" => {
96+
let status = lifecycle.status().await?;
97+
Ok(
98+
json!({"payload": status.payload, "invite": status.invite_token, "api": status.api_base_url}),
99+
)
100+
}
101+
"join" => {
102+
lifecycle.dial(command["token"].as_str().context("token required")?)?;
103+
Ok(json!(true))
104+
}
105+
"stop" => {
106+
lifecycle.stop_and_wait().await?;
107+
Ok(json!(true))
108+
}
109+
other => bail!("unknown operation {other}"),
110+
}
111+
}

‎scripts/MESH_CROSS_NODE.md‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# Cross-node inference smoke
2+
3+
`ci-mesh-cross-node.sh` builds the real Buzz lifecycle example with the `mesh`
4+
feature and runs two separate processes on Linux x86-64 using a checksum-pinned
5+
v0.78.1 CPU runtime and revision/checksum-pinned SmolLM2-135M GGUF. The workflow
6+
runs on relevant PRs/main pushes and supports manual dispatch. Budget: 20 minutes
7+
including compilation; the Python smoke has a 10-minute outer deadline.
8+
9+
The provider alone receives the model. Both processes receive isolated temporary
10+
homes, new owner identities, a private mesh name, explicit owner allowlists and
11+
no public discovery/iroh relay configuration. No desktop credentials are inherited.
12+
The smoke checks exact remote model visibility, a nonempty `model=mesh` response,
13+
increased provider-local request accounting and zero consumer-local requests.
14+
After provider shutdown a fresh request must fail. Temporary homes and owned
15+
processes are cleaned on success, exceptions and SIGTERM/SIGINT; SIGKILL cannot
16+
run cleanup. Evidence goes to `test-results/mesh-cross-node`.
17+
18+
For macOS or an already provisioned runtime:
19+
20+
```sh
21+
bin/cargo build --locked -p buzz-mesh-compute --features mesh --example cross_node
22+
python3 scripts/mesh-cross-node.py --binary target/debug/examples/cross_node \
23+
--runtime-bundle /path/to/native-runtime --cache /path/to/model-cache \
24+
--output /path/to/evidence
25+
```
26+
27+
Negative control: append `--skip-join --readiness-timeout 3`; this must exit
28+
nonzero with `deadline: remote model visible`. It is not a passing smoke.
29+
30+
This covers the old serve/client smoke boundary, **not** signed relay discovery,
31+
stranger admission, the leased gateway/GDK agent loop, desktop packaging or
32+
cross-host firewall/NAT behavior. Two processes on one runner are deliberate:
33+
this detects accidental local-model dependence without requiring two machines.
34+
Native macOS runs do not establish that the Linux CPU workflow passes.

‎scripts/ci-mesh-cross-node.sh‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
#!/usr/bin/env bash
2+
# Linux CPU lane. No desktop, public mesh, credentials or GPU required.
3+
set -euo pipefail
4+
cd "$(dirname "$0")/.."
5+
cache="${MESH_SMOKE_CACHE:-$PWD/.cache/mesh-cross-node}"
6+
output="${MESH_SMOKE_OUTPUT:-$PWD/test-results/mesh-cross-node}"
7+
mkdir -p "$cache" "$output"
8+
archive="$cache/runtime-linux.tar.gz"
9+
if [[ ! -f "$archive" ]]; then
10+
curl --fail --location --max-time 180 \
11+
https://github.com/Mesh-LLM/mesh-llm/releases/download/v0.78.1/meshllm-native-runtime-linux-x86_64-cpu.tar.gz \
12+
--output "$archive"
13+
fi
14+
echo "19fe012ae952e50a536f21c8c9f7864281656c8c20bc53902832850d6203106d $archive" | sha256sum --check
15+
tar -xzf "$archive" -C "$cache"
16+
./bin/cargo build --locked -p buzz-mesh-compute --features mesh --example cross_node
17+
python3 scripts/mesh-cross-node.py \
18+
--binary "${CARGO_TARGET_DIR:-target}/debug/examples/cross_node" \
19+
--runtime-bundle "$cache/meshllm-native-runtime-linux-x86_64-cpu" \
20+
--cache "$cache" --output "$output"

0 commit comments

Comments
 (0)