Skip to content

Commit 127452c

Browse files
vincentkocvincentkoc
committed
fix(release): validate prepared npm package sets
Forward-port the beta3 package-set preflight, Telegram, and Parallels validation to main while preserving main's existing Bun install smoke. Recompute decoded proxy response lengths and require artifact tarballs to exactly match the verified manifest.
1 parent 18fd6ee commit 127452c

19 files changed

Lines changed: 833 additions & 43 deletions

‎.github/workflows/npm-telegram-beta-e2e.yml‎

Lines changed: 109 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -247,14 +247,117 @@ jobs:
247247
trap append_telegram_summary EXIT
248248
249249
if [[ -n "${PACKAGE_ARTIFACT_NAME// }" ]]; then
250-
mapfile -t package_tgzs < <(find .artifacts/telegram-package-under-test -type f -name "*.tgz" | sort)
251-
if [[ "${#package_tgzs[@]}" -ne 1 ]]; then
252-
echo "package artifact ${PACKAGE_ARTIFACT_NAME} must contain exactly one .tgz; found ${#package_tgzs[@]}" >&2
253-
exit 1
250+
package_dir=".artifacts/telegram-package-under-test"
251+
manifest="${package_dir}/preflight-manifest.json"
252+
if [[ -f "${manifest}" ]]; then
253+
package_tgz="$(
254+
node - "${manifest}" "${package_dir}" <<'NODE'
255+
const crypto = require("node:crypto");
256+
const childProcess = require("node:child_process");
257+
const fs = require("node:fs");
258+
const path = require("node:path");
259+
const [manifestPath, packageDir] = process.argv.slice(2);
260+
const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8"));
261+
const entries = [
262+
{
263+
packageName: "openclaw",
264+
packageVersion: manifest.packageVersion,
265+
tarballName: manifest.tarballName,
266+
tarballSha256: manifest.tarballSha256,
267+
},
268+
...(Array.isArray(manifest.dependencyTarballs) ? manifest.dependencyTarballs : []),
269+
];
270+
const packageNames = new Set();
271+
const tarballNames = new Set();
272+
for (const entry of entries) {
273+
if (
274+
!entry.packageName ||
275+
!entry.packageVersion ||
276+
!entry.tarballName ||
277+
!entry.tarballSha256 ||
278+
entry.tarballName !== path.basename(entry.tarballName)
279+
) {
280+
throw new Error("package artifact manifest contains invalid tarball metadata");
281+
}
282+
if (packageNames.has(entry.packageName) || tarballNames.has(entry.tarballName)) {
283+
throw new Error("package artifact manifest contains duplicate package metadata");
284+
}
285+
packageNames.add(entry.packageName);
286+
tarballNames.add(entry.tarballName);
287+
const tarballPath = path.join(packageDir, entry.tarballName);
288+
if (!fs.existsSync(tarballPath)) {
289+
throw new Error(`package artifact is missing ${entry.tarballName}`);
290+
}
291+
const digest = crypto.createHash("sha256").update(fs.readFileSync(tarballPath)).digest("hex");
292+
if (digest !== entry.tarballSha256) {
293+
throw new Error(`package artifact digest mismatch for ${entry.packageName}`);
294+
}
295+
const packageJson = JSON.parse(
296+
childProcess.execFileSync("tar", ["-xOf", tarballPath, "package/package.json"], {
297+
encoding: "utf8",
298+
}),
299+
);
300+
if (packageJson.name !== entry.packageName || packageJson.version !== entry.packageVersion) {
301+
throw new Error(`package artifact metadata mismatch for ${entry.packageName}`);
302+
}
303+
}
304+
const artifactTarballs = fs
305+
.readdirSync(packageDir, { withFileTypes: true })
306+
.filter((entry) => entry.isFile() && entry.name.endsWith(".tgz"))
307+
.map((entry) => entry.name);
308+
if (
309+
artifactTarballs.length !== tarballNames.size ||
310+
artifactTarballs.some((name) => !tarballNames.has(name))
311+
) {
312+
throw new Error("package artifact tarball set does not match preflight manifest");
313+
}
314+
process.stdout.write(path.join(packageDir, manifest.tarballName));
315+
NODE
316+
)"
317+
export OPENCLAW_NPM_TELEGRAM_PACKAGE_DIR="${package_dir}"
318+
else
319+
mapfile -t package_tgzs < <(find "${package_dir}" -type f -name "*.tgz" | sort)
320+
if [[ "${#package_tgzs[@]}" -ne 1 ]]; then
321+
echo "package artifact ${PACKAGE_ARTIFACT_NAME} without a preflight manifest must contain exactly one tgz; found ${#package_tgzs[@]}" >&2
322+
exit 1
323+
fi
324+
package_tgz="${package_tgzs[0]}"
325+
candidate_manifest="${package_dir}/package-candidate.json"
326+
node - "${package_tgz}" "${candidate_manifest}" <<'NODE'
327+
const crypto = require("node:crypto");
328+
const childProcess = require("node:child_process");
329+
const fs = require("node:fs");
330+
const path = require("node:path");
331+
const [tarballPath, manifestPath] = process.argv.slice(2);
332+
const packageJson = JSON.parse(
333+
childProcess.execFileSync("tar", ["-xOf", tarballPath, "package/package.json"], {
334+
encoding: "utf8",
335+
}),
336+
);
337+
if (packageJson.name !== "openclaw" || typeof packageJson.version !== "string") {
338+
throw new Error("package artifact tgz is not a versioned openclaw package");
339+
}
340+
if (fs.existsSync(manifestPath)) {
341+
const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8"));
342+
const expectedTarballName = path.basename(manifest.tarball || "");
343+
if (
344+
manifest.name !== "openclaw" ||
345+
manifest.version !== packageJson.version ||
346+
expectedTarballName !== path.basename(tarballPath) ||
347+
typeof manifest.sha256 !== "string"
348+
) {
349+
throw new Error("package candidate manifest does not match the OpenClaw tarball");
350+
}
351+
const digest = crypto.createHash("sha256").update(fs.readFileSync(tarballPath)).digest("hex");
352+
if (digest !== manifest.sha256) {
353+
throw new Error("package candidate digest mismatch");
354+
}
355+
}
356+
NODE
254357
fi
255-
export OPENCLAW_NPM_TELEGRAM_PACKAGE_TGZ="${package_tgzs[0]}"
358+
export OPENCLAW_NPM_TELEGRAM_PACKAGE_TGZ="${package_tgz}"
256359
if [[ -z "${OPENCLAW_NPM_TELEGRAM_PACKAGE_LABEL// }" ]]; then
257-
export OPENCLAW_NPM_TELEGRAM_PACKAGE_LABEL="$(basename "${package_tgzs[0]}")"
360+
export OPENCLAW_NPM_TELEGRAM_PACKAGE_LABEL="$(basename "${package_tgz}")"
258361
fi
259362
elif [[ -z "${OPENCLAW_NPM_TELEGRAM_PACKAGE_LABEL// }" ]]; then
260363
export OPENCLAW_NPM_TELEGRAM_PACKAGE_LABEL="${OPENCLAW_NPM_TELEGRAM_PACKAGE_SPEC}"

‎.github/workflows/openclaw-npm-release.yml‎

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -396,17 +396,33 @@ jobs:
396396
fi
397397
TARBALL_NAME="$PACK_NAME"
398398
TARBALL_SHA256="$(sha256sum "$PACK_PATH" | awk '{print $1}')"
399+
mapfile -t AI_TARBALLS < <(find "$AI_RUNTIME_TARBALL_DIR" -maxdepth 1 -type f -name 'openclaw-ai-*.tgz' -print | sort)
400+
if [[ "${#AI_TARBALLS[@]}" -ne 1 ]]; then
401+
echo "Expected exactly one packed @openclaw/ai tarball, found ${#AI_TARBALLS[@]}." >&2
402+
exit 1
403+
fi
404+
AI_TARBALL_PATH="${AI_TARBALLS[0]}"
405+
AI_TARBALL_NAME="$(basename "$AI_TARBALL_PATH")"
406+
AI_TARBALL_SHA256="$(sha256sum "$AI_TARBALL_PATH" | awk '{print $1}')"
407+
AI_PACKAGE_VERSION="$(
408+
tar -xOf "$AI_TARBALL_PATH" package/package.json |
409+
node -e 'let raw = ""; process.stdin.on("data", (chunk) => (raw += chunk)); process.stdin.on("end", () => process.stdout.write(JSON.parse(raw).version));'
410+
)"
411+
if [[ "$AI_PACKAGE_VERSION" != "$PACKAGE_VERSION" ]]; then
412+
echo "Prepared @openclaw/ai version ${AI_PACKAGE_VERSION} does not match openclaw ${PACKAGE_VERSION}." >&2
413+
exit 1
414+
fi
399415
ARTIFACT_DIR="$RUNNER_TEMP/openclaw-npm-preflight"
400416
rm -rf "$ARTIFACT_DIR"
401417
mkdir -p "$ARTIFACT_DIR"
402418
cp "$PACK_PATH" "$ARTIFACT_DIR/"
403-
cp "$AI_RUNTIME_TARBALL_DIR"/*.tgz "$ARTIFACT_DIR/"
419+
cp "$AI_TARBALL_PATH" "$ARTIFACT_DIR/"
404420
cp "$AI_RUNTIME_TARBALL_DIR/SHA256SUMS" "$ARTIFACT_DIR/ai-runtime-SHA256SUMS"
405421
cp -R "$DEPENDENCY_EVIDENCE_DIR" "$ARTIFACT_DIR/dependency-evidence"
406422
printf '%s\n' "$RELEASE_TAG" > "$ARTIFACT_DIR/release-tag.txt"
407423
printf '%s\n' "$RELEASE_SHA" > "$ARTIFACT_DIR/release-sha.txt"
408424
printf '%s\n' "$RELEASE_NPM_DIST_TAG" > "$ARTIFACT_DIR/release-npm-dist-tag.txt"
409-
ARTIFACT_DIR="$ARTIFACT_DIR" RELEASE_TAG="$RELEASE_TAG" RELEASE_SHA="$RELEASE_SHA" RELEASE_NPM_DIST_TAG="$RELEASE_NPM_DIST_TAG" PACKAGE_VERSION="$PACKAGE_VERSION" TARBALL_NAME="$TARBALL_NAME" TARBALL_SHA256="$TARBALL_SHA256" node <<'NODE'
425+
ARTIFACT_DIR="$ARTIFACT_DIR" RELEASE_TAG="$RELEASE_TAG" RELEASE_SHA="$RELEASE_SHA" RELEASE_NPM_DIST_TAG="$RELEASE_NPM_DIST_TAG" PACKAGE_VERSION="$PACKAGE_VERSION" TARBALL_NAME="$TARBALL_NAME" TARBALL_SHA256="$TARBALL_SHA256" AI_PACKAGE_VERSION="$AI_PACKAGE_VERSION" AI_TARBALL_NAME="$AI_TARBALL_NAME" AI_TARBALL_SHA256="$AI_TARBALL_SHA256" node <<'NODE'
410426
const fs = require("node:fs");
411427
const path = require("node:path");
412428
const manifest = {
@@ -418,6 +434,14 @@ jobs:
418434
packageVersion: process.env.PACKAGE_VERSION,
419435
tarballName: process.env.TARBALL_NAME,
420436
tarballSha256: process.env.TARBALL_SHA256,
437+
dependencyTarballs: [
438+
{
439+
packageName: "@openclaw/ai",
440+
packageVersion: process.env.AI_PACKAGE_VERSION,
441+
tarballName: process.env.AI_TARBALL_NAME,
442+
tarballSha256: process.env.AI_TARBALL_SHA256,
443+
},
444+
],
421445
dependencyEvidenceDir: "dependency-evidence",
422446
dependencyEvidenceManifest: "dependency-evidence/dependency-evidence-manifest.json",
423447
};

‎scripts/e2e/lib/plugins/npm-registry-server.mjs‎

Lines changed: 72 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
1+
import { execFileSync } from "node:child_process";
12
// Fixture npm registry server for plugin E2E scenarios.
23
import crypto from "node:crypto";
34
import fs from "node:fs";
45
import http from "node:http";
56
import path from "node:path";
67

78
const [portFile, ...packageArgs] = process.argv.slice(2);
9+
const upstreamRegistry = process.env.OPENCLAW_NPM_REGISTRY_UPSTREAM?.replace(/\/+$/u, "");
810

911
if (!portFile || packageArgs.length === 0 || packageArgs.length % 3 !== 0) {
1012
console.error(
@@ -14,6 +16,25 @@ if (!portFile || packageArgs.length === 0 || packageArgs.length % 3 !== 0) {
1416
}
1517

1618
const packages = new Map();
19+
20+
function readPackageManifest(tarballPath, packageName) {
21+
try {
22+
const packageJson = JSON.parse(
23+
execFileSync("tar", ["-xOf", tarballPath, "package/package.json"], {
24+
encoding: "utf8",
25+
stdio: ["ignore", "pipe", "ignore"],
26+
}),
27+
);
28+
return packageJson && typeof packageJson === "object" && !Array.isArray(packageJson)
29+
? packageJson
30+
: {};
31+
} catch {
32+
return packageName === "@openclaw/demo-plugin-npm"
33+
? { dependencies: { "is-number": "7.0.0" } }
34+
: {};
35+
}
36+
}
37+
1738
for (let index = 0; index < packageArgs.length; index += 3) {
1839
const packageName = packageArgs[index];
1940
const version = packageArgs[index + 1];
@@ -28,8 +49,8 @@ for (let index = 0; index < packageArgs.length; index += 3) {
2849
existing.latestVersion = version;
2950
existing.versions.set(version, {
3051
archive,
31-
dependencies: packageName === "@openclaw/demo-plugin-npm" ? { "is-number": "7.0.0" } : {},
3252
integrity: `sha512-${crypto.createHash("sha512").update(archive).digest("base64")}`,
53+
manifest: readPackageManifest(tarballPath, packageName),
3354
shasum: crypto.createHash("sha1").update(archive).digest("hex"),
3455
tarballName: path.basename(tarballPath),
3556
version,
@@ -44,7 +65,7 @@ const metadataFor = (entry, baseUrl) => ({
4465
[...entry.versions.entries()].map(([version, versionEntry]) => [
4566
version,
4667
{
47-
dependencies: versionEntry.dependencies,
68+
...versionEntry.manifest,
4869
name: entry.packageName,
4970
version,
5071
dist: {
@@ -85,9 +106,37 @@ function findTarballForPath(pathname) {
85106
return undefined;
86107
}
87108

88-
const server = http.createServer((request, response) => {
89-
const url = new URL(request.url ?? "/", "http://127.0.0.1");
90-
const baseUrl = `http://127.0.0.1:${server.address().port}`;
109+
async function proxyUpstream(url, response) {
110+
if (!upstreamRegistry) {
111+
return false;
112+
}
113+
try {
114+
const upstreamUrl = new URL(`${url.pathname}${url.search}`, `${upstreamRegistry}/`);
115+
const upstreamResponse = await fetch(upstreamUrl, { redirect: "manual" });
116+
const body = Buffer.from(await upstreamResponse.arrayBuffer());
117+
// Fetch decodes compressed bodies but preserves upstream length metadata.
118+
// Emit the decoded size so npm clients do not truncate proxied responses.
119+
const headers = { "content-length": String(body.length) };
120+
for (const name of ["content-type", "location"]) {
121+
const value = upstreamResponse.headers.get(name);
122+
if (value) {
123+
headers[name] = value;
124+
}
125+
}
126+
response.writeHead(upstreamResponse.status, headers);
127+
response.end(body);
128+
} catch (error) {
129+
response.writeHead(502, { "content-type": "text/plain" });
130+
response.end(`upstream registry request failed: ${String(error)}`);
131+
}
132+
return true;
133+
}
134+
135+
async function handleRequest(request, response) {
136+
const fallbackHost = `127.0.0.1:${server.address().port}`;
137+
const requestHost = request.headers.host || fallbackHost;
138+
const url = new URL(request.url ?? "/", `http://${requestHost}`);
139+
const baseUrl = url.origin;
91140
if (request.method !== "GET") {
92141
response.writeHead(405, { "content-type": "text/plain" });
93142
response.end("method not allowed");
@@ -111,10 +160,27 @@ const server = http.createServer((request, response) => {
111160
return;
112161
}
113162

163+
if (await proxyUpstream(url, response)) {
164+
return;
165+
}
166+
114167
response.writeHead(404, { "content-type": "text/plain" });
115168
response.end(`not found: ${url.pathname}`);
169+
}
170+
171+
const server = http.createServer((request, response) => {
172+
void handleRequest(request, response).catch((/** @type {unknown} */ error) => {
173+
if (!response.headersSent) {
174+
response.writeHead(500, { "content-type": "text/plain" });
175+
response.end(`registry request failed: ${String(error)}`);
176+
return;
177+
}
178+
response.destroy(error instanceof Error ? error : new Error(String(error)));
179+
});
116180
});
117181

118-
server.listen(0, "127.0.0.1", () => {
182+
const bindHost = process.env.OPENCLAW_NPM_REGISTRY_BIND_HOST || "127.0.0.1";
183+
const requestedPort = Number(process.env.OPENCLAW_NPM_REGISTRY_PORT || 0);
184+
server.listen(requestedPort, bindHost, () => {
119185
fs.writeFileSync(portFile, String(server.address().port));
120186
});

0 commit comments

Comments
 (0)