Skip to content

Commit 0535679

Browse files
vincentkocvincentkoc
authored
fix(release): validate exact prepared npm package sets (#102759)
* fix(release): validate prepared npm package sets Forward-port the beta3 package-set preflight, Telegram, and Parallels validation to main while preserving main's existing Bun install smoke. Recompute decoded proxy response lengths and require artifact tarballs to exactly match the verified manifest. * fix(release): lock fixture registry proxy origin
1 parent aaf5ddd commit 0535679

19 files changed

Lines changed: 957 additions & 43 deletions

‎.github/workflows/npm-telegram-beta-e2e.yml‎

Lines changed: 109 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -247,14 +247,117 @@ jobs:
247247
trap append_telegram_summary EXIT
248248
249249
if [[ -n "${PACKAGE_ARTIFACT_NAME// }" ]]; then
250-
mapfile -t package_tgzs < <(find .artifacts/telegram-package-under-test -type f -name "*.tgz" | sort)
251-
if [[ "${#package_tgzs[@]}" -ne 1 ]]; then
252-
echo "package artifact ${PACKAGE_ARTIFACT_NAME} must contain exactly one .tgz; found ${#package_tgzs[@]}" >&2
253-
exit 1
250+
package_dir=".artifacts/telegram-package-under-test"
251+
manifest="${package_dir}/preflight-manifest.json"
252+
if [[ -f "${manifest}" ]]; then
253+
package_tgz="$(
254+
node - "${manifest}" "${package_dir}" <<'NODE'
255+
const crypto = require("node:crypto");
256+
const childProcess = require("node:child_process");
257+
const fs = require("node:fs");
258+
const path = require("node:path");
259+
const [manifestPath, packageDir] = process.argv.slice(2);
260+
const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8"));
261+
const entries = [
262+
{
263+
packageName: "openclaw",
264+
packageVersion: manifest.packageVersion,
265+
tarballName: manifest.tarballName,
266+
tarballSha256: manifest.tarballSha256,
267+
},
268+
...(Array.isArray(manifest.dependencyTarballs) ? manifest.dependencyTarballs : []),
269+
];
270+
const packageNames = new Set();
271+
const tarballNames = new Set();
272+
for (const entry of entries) {
273+
if (
274+
!entry.packageName ||
275+
!entry.packageVersion ||
276+
!entry.tarballName ||
277+
!entry.tarballSha256 ||
278+
entry.tarballName !== path.basename(entry.tarballName)
279+
) {
280+
throw new Error("package artifact manifest contains invalid tarball metadata");
281+
}
282+
if (packageNames.has(entry.packageName) || tarballNames.has(entry.tarballName)) {
283+
throw new Error("package artifact manifest contains duplicate package metadata");
284+
}
285+
packageNames.add(entry.packageName);
286+
tarballNames.add(entry.tarballName);
287+
const tarballPath = path.join(packageDir, entry.tarballName);
288+
if (!fs.existsSync(tarballPath)) {
289+
throw new Error(`package artifact is missing ${entry.tarballName}`);
290+
}
291+
const digest = crypto.createHash("sha256").update(fs.readFileSync(tarballPath)).digest("hex");
292+
if (digest !== entry.tarballSha256) {
293+
throw new Error(`package artifact digest mismatch for ${entry.packageName}`);
294+
}
295+
const packageJson = JSON.parse(
296+
childProcess.execFileSync("tar", ["-xOf", tarballPath, "package/package.json"], {
297+
encoding: "utf8",
298+
}),
299+
);
300+
if (packageJson.name !== entry.packageName || packageJson.version !== entry.packageVersion) {
301+
throw new Error(`package artifact metadata mismatch for ${entry.packageName}`);
302+
}
303+
}
304+
const artifactTarballs = fs
305+
.readdirSync(packageDir, { withFileTypes: true })
306+
.filter((entry) => entry.isFile() && entry.name.endsWith(".tgz"))
307+
.map((entry) => entry.name);
308+
if (
309+
artifactTarballs.length !== tarballNames.size ||
310+
artifactTarballs.some((name) => !tarballNames.has(name))
311+
) {
312+
throw new Error("package artifact tarball set does not match preflight manifest");
313+
}
314+
process.stdout.write(path.join(packageDir, manifest.tarballName));
315+
NODE
316+
)"
317+
export OPENCLAW_NPM_TELEGRAM_PACKAGE_DIR="${package_dir}"
318+
else
319+
mapfile -t package_tgzs < <(find "${package_dir}" -type f -name "*.tgz" | sort)
320+
if [[ "${#package_tgzs[@]}" -ne 1 ]]; then
321+
echo "package artifact ${PACKAGE_ARTIFACT_NAME} without a preflight manifest must contain exactly one tgz; found ${#package_tgzs[@]}" >&2
322+
exit 1
323+
fi
324+
package_tgz="${package_tgzs[0]}"
325+
candidate_manifest="${package_dir}/package-candidate.json"
326+
node - "${package_tgz}" "${candidate_manifest}" <<'NODE'
327+
const crypto = require("node:crypto");
328+
const childProcess = require("node:child_process");
329+
const fs = require("node:fs");
330+
const path = require("node:path");
331+
const [tarballPath, manifestPath] = process.argv.slice(2);
332+
const packageJson = JSON.parse(
333+
childProcess.execFileSync("tar", ["-xOf", tarballPath, "package/package.json"], {
334+
encoding: "utf8",
335+
}),
336+
);
337+
if (packageJson.name !== "openclaw" || typeof packageJson.version !== "string") {
338+
throw new Error("package artifact tgz is not a versioned openclaw package");
339+
}
340+
if (fs.existsSync(manifestPath)) {
341+
const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8"));
342+
const expectedTarballName = path.basename(manifest.tarball || "");
343+
if (
344+
manifest.name !== "openclaw" ||
345+
manifest.version !== packageJson.version ||
346+
expectedTarballName !== path.basename(tarballPath) ||
347+
typeof manifest.sha256 !== "string"
348+
) {
349+
throw new Error("package candidate manifest does not match the OpenClaw tarball");
350+
}
351+
const digest = crypto.createHash("sha256").update(fs.readFileSync(tarballPath)).digest("hex");
352+
if (digest !== manifest.sha256) {
353+
throw new Error("package candidate digest mismatch");
354+
}
355+
}
356+
NODE
254357
fi
255-
export OPENCLAW_NPM_TELEGRAM_PACKAGE_TGZ="${package_tgzs[0]}"
358+
export OPENCLAW_NPM_TELEGRAM_PACKAGE_TGZ="${package_tgz}"
256359
if [[ -z "${OPENCLAW_NPM_TELEGRAM_PACKAGE_LABEL// }" ]]; then
257-
export OPENCLAW_NPM_TELEGRAM_PACKAGE_LABEL="$(basename "${package_tgzs[0]}")"
360+
export OPENCLAW_NPM_TELEGRAM_PACKAGE_LABEL="$(basename "${package_tgz}")"
258361
fi
259362
elif [[ -z "${OPENCLAW_NPM_TELEGRAM_PACKAGE_LABEL// }" ]]; then
260363
export OPENCLAW_NPM_TELEGRAM_PACKAGE_LABEL="${OPENCLAW_NPM_TELEGRAM_PACKAGE_SPEC}"

‎.github/workflows/openclaw-npm-release.yml‎

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -396,17 +396,33 @@ jobs:
396396
fi
397397
TARBALL_NAME="$PACK_NAME"
398398
TARBALL_SHA256="$(sha256sum "$PACK_PATH" | awk '{print $1}')"
399+
mapfile -t AI_TARBALLS < <(find "$AI_RUNTIME_TARBALL_DIR" -maxdepth 1 -type f -name 'openclaw-ai-*.tgz' -print | sort)
400+
if [[ "${#AI_TARBALLS[@]}" -ne 1 ]]; then
401+
echo "Expected exactly one packed @openclaw/ai tarball, found ${#AI_TARBALLS[@]}." >&2
402+
exit 1
403+
fi
404+
AI_TARBALL_PATH="${AI_TARBALLS[0]}"
405+
AI_TARBALL_NAME="$(basename "$AI_TARBALL_PATH")"
406+
AI_TARBALL_SHA256="$(sha256sum "$AI_TARBALL_PATH" | awk '{print $1}')"
407+
AI_PACKAGE_VERSION="$(
408+
tar -xOf "$AI_TARBALL_PATH" package/package.json |
409+
node -e 'let raw = ""; process.stdin.on("data", (chunk) => (raw += chunk)); process.stdin.on("end", () => process.stdout.write(JSON.parse(raw).version));'
410+
)"
411+
if [[ "$AI_PACKAGE_VERSION" != "$PACKAGE_VERSION" ]]; then
412+
echo "Prepared @openclaw/ai version ${AI_PACKAGE_VERSION} does not match openclaw ${PACKAGE_VERSION}." >&2
413+
exit 1
414+
fi
399415
ARTIFACT_DIR="$RUNNER_TEMP/openclaw-npm-preflight"
400416
rm -rf "$ARTIFACT_DIR"
401417
mkdir -p "$ARTIFACT_DIR"
402418
cp "$PACK_PATH" "$ARTIFACT_DIR/"
403-
cp "$AI_RUNTIME_TARBALL_DIR"/*.tgz "$ARTIFACT_DIR/"
419+
cp "$AI_TARBALL_PATH" "$ARTIFACT_DIR/"
404420
cp "$AI_RUNTIME_TARBALL_DIR/SHA256SUMS" "$ARTIFACT_DIR/ai-runtime-SHA256SUMS"
405421
cp -R "$DEPENDENCY_EVIDENCE_DIR" "$ARTIFACT_DIR/dependency-evidence"
406422
printf '%s\n' "$RELEASE_TAG" > "$ARTIFACT_DIR/release-tag.txt"
407423
printf '%s\n' "$RELEASE_SHA" > "$ARTIFACT_DIR/release-sha.txt"
408424
printf '%s\n' "$RELEASE_NPM_DIST_TAG" > "$ARTIFACT_DIR/release-npm-dist-tag.txt"
409-
ARTIFACT_DIR="$ARTIFACT_DIR" RELEASE_TAG="$RELEASE_TAG" RELEASE_SHA="$RELEASE_SHA" RELEASE_NPM_DIST_TAG="$RELEASE_NPM_DIST_TAG" PACKAGE_VERSION="$PACKAGE_VERSION" TARBALL_NAME="$TARBALL_NAME" TARBALL_SHA256="$TARBALL_SHA256" node <<'NODE'
425+
ARTIFACT_DIR="$ARTIFACT_DIR" RELEASE_TAG="$RELEASE_TAG" RELEASE_SHA="$RELEASE_SHA" RELEASE_NPM_DIST_TAG="$RELEASE_NPM_DIST_TAG" PACKAGE_VERSION="$PACKAGE_VERSION" TARBALL_NAME="$TARBALL_NAME" TARBALL_SHA256="$TARBALL_SHA256" AI_PACKAGE_VERSION="$AI_PACKAGE_VERSION" AI_TARBALL_NAME="$AI_TARBALL_NAME" AI_TARBALL_SHA256="$AI_TARBALL_SHA256" node <<'NODE'
410426
const fs = require("node:fs");
411427
const path = require("node:path");
412428
const manifest = {
@@ -418,6 +434,14 @@ jobs:
418434
packageVersion: process.env.PACKAGE_VERSION,
419435
tarballName: process.env.TARBALL_NAME,
420436
tarballSha256: process.env.TARBALL_SHA256,
437+
dependencyTarballs: [
438+
{
439+
packageName: "@openclaw/ai",
440+
packageVersion: process.env.AI_PACKAGE_VERSION,
441+
tarballName: process.env.AI_TARBALL_NAME,
442+
tarballSha256: process.env.AI_TARBALL_SHA256,
443+
},
444+
],
421445
dependencyEvidenceDir: "dependency-evidence",
422446
dependencyEvidenceManifest: "dependency-evidence/dependency-evidence-manifest.json",
423447
};

‎scripts/e2e/lib/plugins/npm-registry-server.mjs‎

Lines changed: 99 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,30 @@
1+
import { execFileSync } from "node:child_process";
12
// Fixture npm registry server for plugin E2E scenarios.
23
import crypto from "node:crypto";
34
import fs from "node:fs";
45
import http from "node:http";
56
import path from "node:path";
67

78
const [portFile, ...packageArgs] = process.argv.slice(2);
9+
function normalizeUpstreamRegistry(raw) {
10+
if (!raw) {
11+
return undefined;
12+
}
13+
const url = new URL(raw);
14+
if (
15+
(url.protocol !== "http:" && url.protocol !== "https:") ||
16+
url.username ||
17+
url.password ||
18+
url.pathname !== "/" ||
19+
url.search ||
20+
url.hash
21+
) {
22+
throw new Error("OPENCLAW_NPM_REGISTRY_UPSTREAM must be an HTTP(S) origin");
23+
}
24+
return url.origin;
25+
}
26+
27+
const upstreamRegistry = normalizeUpstreamRegistry(process.env.OPENCLAW_NPM_REGISTRY_UPSTREAM);
828

929
if (!portFile || packageArgs.length === 0 || packageArgs.length % 3 !== 0) {
1030
console.error(
@@ -14,6 +34,25 @@ if (!portFile || packageArgs.length === 0 || packageArgs.length % 3 !== 0) {
1434
}
1535

1636
const packages = new Map();
37+
38+
function readPackageManifest(tarballPath, packageName) {
39+
try {
40+
const packageJson = JSON.parse(
41+
execFileSync("tar", ["-xOf", tarballPath, "package/package.json"], {
42+
encoding: "utf8",
43+
stdio: ["ignore", "pipe", "ignore"],
44+
}),
45+
);
46+
return packageJson && typeof packageJson === "object" && !Array.isArray(packageJson)
47+
? packageJson
48+
: {};
49+
} catch {
50+
return packageName === "@openclaw/demo-plugin-npm"
51+
? { dependencies: { "is-number": "7.0.0" } }
52+
: {};
53+
}
54+
}
55+
1756
for (let index = 0; index < packageArgs.length; index += 3) {
1857
const packageName = packageArgs[index];
1958
const version = packageArgs[index + 1];
@@ -28,8 +67,8 @@ for (let index = 0; index < packageArgs.length; index += 3) {
2867
existing.latestVersion = version;
2968
existing.versions.set(version, {
3069
archive,
31-
dependencies: packageName === "@openclaw/demo-plugin-npm" ? { "is-number": "7.0.0" } : {},
3270
integrity: `sha512-${crypto.createHash("sha512").update(archive).digest("base64")}`,
71+
manifest: readPackageManifest(tarballPath, packageName),
3372
shasum: crypto.createHash("sha1").update(archive).digest("hex"),
3473
tarballName: path.basename(tarballPath),
3574
version,
@@ -44,7 +83,7 @@ const metadataFor = (entry, baseUrl) => ({
4483
[...entry.versions.entries()].map(([version, versionEntry]) => [
4584
version,
4685
{
47-
dependencies: versionEntry.dependencies,
86+
...versionEntry.manifest,
4887
name: entry.packageName,
4988
version,
5089
dist: {
@@ -85,9 +124,46 @@ function findTarballForPath(pathname) {
85124
return undefined;
86125
}
87126

88-
const server = http.createServer((request, response) => {
89-
const url = new URL(request.url ?? "/", "http://127.0.0.1");
90-
const baseUrl = `http://127.0.0.1:${server.address().port}`;
127+
function resolveUpstreamRequestUrl(rawRequestUrl) {
128+
const raw = rawRequestUrl || "/";
129+
if (!raw.startsWith("/") || raw.startsWith("//") || raw.includes("\\")) {
130+
throw new Error(`refusing non-origin registry request URL: ${JSON.stringify(raw)}`);
131+
}
132+
const requestUrl = new URL(raw, "http://127.0.0.1");
133+
return `${upstreamRegistry}${requestUrl.pathname}${requestUrl.search}`;
134+
}
135+
136+
async function proxyUpstream(rawRequestUrl, response) {
137+
if (!upstreamRegistry) {
138+
return false;
139+
}
140+
try {
141+
const upstreamUrl = resolveUpstreamRequestUrl(rawRequestUrl);
142+
const upstreamResponse = await fetch(upstreamUrl, { redirect: "manual" });
143+
const body = Buffer.from(await upstreamResponse.arrayBuffer());
144+
// Fetch decodes compressed bodies but preserves upstream length metadata.
145+
// Emit the decoded size so npm clients do not truncate proxied responses.
146+
const headers = { "content-length": String(body.length) };
147+
for (const name of ["content-type", "location"]) {
148+
const value = upstreamResponse.headers.get(name);
149+
if (value) {
150+
headers[name] = value;
151+
}
152+
}
153+
response.writeHead(upstreamResponse.status, headers);
154+
response.end(body);
155+
} catch (error) {
156+
response.writeHead(502, { "content-type": "text/plain" });
157+
response.end(`upstream registry request failed: ${String(error)}`);
158+
}
159+
return true;
160+
}
161+
162+
async function handleRequest(request, response) {
163+
const fallbackHost = `127.0.0.1:${server.address().port}`;
164+
const requestHost = request.headers.host || fallbackHost;
165+
const url = new URL(request.url ?? "/", `http://${requestHost}`);
166+
const baseUrl = url.origin;
91167
if (request.method !== "GET") {
92168
response.writeHead(405, { "content-type": "text/plain" });
93169
response.end("method not allowed");
@@ -111,10 +187,27 @@ const server = http.createServer((request, response) => {
111187
return;
112188
}
113189

190+
if (await proxyUpstream(request.url, response)) {
191+
return;
192+
}
193+
114194
response.writeHead(404, { "content-type": "text/plain" });
115195
response.end(`not found: ${url.pathname}`);
196+
}
197+
198+
const server = http.createServer((request, response) => {
199+
void handleRequest(request, response).catch((/** @type {unknown} */ error) => {
200+
if (!response.headersSent) {
201+
response.writeHead(500, { "content-type": "text/plain" });
202+
response.end(`registry request failed: ${String(error)}`);
203+
return;
204+
}
205+
response.destroy(error instanceof Error ? error : new Error(String(error)));
206+
});
116207
});
117208

118-
server.listen(0, "127.0.0.1", () => {
209+
const bindHost = process.env.OPENCLAW_NPM_REGISTRY_BIND_HOST || "127.0.0.1";
210+
const requestedPort = Number(process.env.OPENCLAW_NPM_REGISTRY_PORT || 0);
211+
server.listen(requestedPort, bindHost, () => {
119212
fs.writeFileSync(portFile, String(server.address().port));
120213
});

0 commit comments

Comments
 (0)