Skip to content

Commit d2d1e4f

Browse files
t3dotggsvanvlietclaude
authored
fix(updates): version checks respect your configured npm registry (pingdotgg#27847)
Co-authored-by: Scott Van Vliet <[email protected]> Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 96c21aa commit d2d1e4f

22 files changed

Lines changed: 821 additions & 183 deletions

File tree

‎apps/server/src/auth/RpcAuthorization.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ export const RPC_REQUIRED_SCOPES = {
6161
[WS_METHODS.projectsMutate]: AuthOrchestrationOperateScope,
6262
[WS_METHODS.serverProbe]: AuthOrchestrationReadScope,
6363
[WS_METHODS.serverGetConfig]: AuthOrchestrationReadScope,
64+
[WS_METHODS.serverGetNpmRelease]: AuthOrchestrationReadScope,
6465
[WS_METHODS.serverRefreshProviders]: AuthOrchestrationReadScope,
6566
[WS_METHODS.serverUpdateProvider]: AuthProvidersManageScope,
6667
[WS_METHODS.providerAuthStart]: AuthProvidersManageScope,
Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
import * as NodeServices from "@effect/platform-node/NodeServices";
2+
import { expect, it } from "@effect/vitest";
3+
import { ServerNpmReleaseError } from "@t3tools/contracts";
4+
import * as HostProcess from "@t3tools/shared/HostProcess";
5+
import * as Deferred from "effect/Deferred";
6+
import * as Effect from "effect/Effect";
7+
import * as FileSystem from "effect/FileSystem";
8+
import * as Path from "effect/Path";
9+
import * as Queue from "effect/Queue";
10+
import * as Fiber from "effect/Fiber";
11+
import { HttpClient, HttpClientResponse } from "effect/http";
12+
import * as Result from "effect/Result";
13+
import * as TestClock from "effect/testing/TestClock";
14+
import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner";
15+
16+
import * as ProcessRunner from "../processRunner.ts";
17+
import * as NpmRelease from "./npmRelease.ts";
18+
19+
const version = "0.1.5-nightly.20261030.1";
20+
const output = (stdout: string, code = 0): ProcessRunner.ProcessRunOutput => ({
21+
stdout,
22+
stderr: "",
23+
code: ChildProcessSpawner.ExitCode(code),
24+
timedOut: false,
25+
stdoutTruncated: false,
26+
stderrTruncated: false,
27+
stdoutInvalidUtf8: false,
28+
stderrInvalidUtf8: false,
29+
});
30+
31+
const noPublicRegistry = HttpClient.make(() =>
32+
Effect.die("a configured npm lookup must not fall back to public npm"),
33+
);
34+
// A fake npm on PATH decides whether npm counts as installed.
35+
const makeService = Effect.fnUntraced(function* (
36+
run: ProcessRunner.ProcessRunner["Service"]["run"],
37+
publicRegistry: HttpClient.HttpClient = noPublicRegistry,
38+
npmInstalled = true,
39+
) {
40+
const fs = yield* FileSystem.FileSystem;
41+
const path = yield* Path.Path;
42+
const npmBin = yield* fs.makeTempDirectoryScoped({ prefix: "t3-npm-release-" });
43+
for (const name of ["npm", "npm.cmd"]) {
44+
yield* fs.writeFileString(path.join(npmBin, name), "", { mode: 0o755 });
45+
}
46+
return yield* NpmRelease.make().pipe(
47+
Effect.provideService(ProcessRunner.ProcessRunner, { run }),
48+
Effect.provideService(HttpClient.HttpClient, publicRegistry),
49+
Effect.provideService(HostProcess.Environment, {
50+
PATH: npmInstalled ? npmBin : "",
51+
PATHEXT: ".CMD",
52+
}),
53+
);
54+
}, Effect.provide(NodeServices.layer));
55+
56+
it.effect.each([
57+
{
58+
spec: version,
59+
stdout: JSON.stringify({ version, t3code: { databaseMigration: 42 } }),
60+
migration: 42,
61+
},
62+
{
63+
spec: "nightly",
64+
stdout: JSON.stringify([{ version, t3code: { databaseMigration: 42 } }]),
65+
migration: 42,
66+
},
67+
{ spec: "latest", stdout: JSON.stringify(version), migration: undefined },
68+
])("reads $spec through npm and retains migration compatibility metadata", (fixture) =>
69+
Effect.gen(function* () {
70+
const calls: ProcessRunner.ProcessRunInput[] = [];
71+
const service = yield* makeService((input) => {
72+
calls.push(input);
73+
return Effect.succeed(output(fixture.stdout));
74+
});
75+
const expected = {
76+
version,
77+
...(fixture.migration === undefined ? {} : { databaseMigration: fixture.migration }),
78+
};
79+
expect(yield* service.get({ spec: fixture.spec })).toEqual(expected);
80+
expect(yield* service.get({ spec: fixture.spec })).toEqual(expected);
81+
expect(calls).toHaveLength(1);
82+
expect(calls[0]).toMatchObject({
83+
command: expect.stringMatching(/[/\\]npm(?:\.cmd)?$/i),
84+
args: ["view", "--global", "--json", `t3@${fixture.spec}`, "version", "t3code"],
85+
maxOutputBytes: 256 * 1024,
86+
});
87+
yield* TestClock.adjust("1 minute");
88+
expect(yield* service.get({ spec: fixture.spec })).toEqual(expected);
89+
expect(calls).toHaveLength(2);
90+
}),
91+
);
92+
93+
it.effect("treats a registry 404 as unpublished without trying another registry", () =>
94+
Effect.gen(function* () {
95+
const service = yield* makeService(() =>
96+
Effect.succeed(output('{"error":{"code":"E404"}}', 1)),
97+
);
98+
expect(yield* service.get({ spec: version })).toBeNull();
99+
}),
100+
);
101+
102+
const spawnFailure = () =>
103+
Effect.fail(
104+
new ProcessRunner.ProcessSpawnError({
105+
command: "npm",
106+
argumentCount: 6,
107+
cause: new Error("EACCES"),
108+
}),
109+
);
110+
111+
it.effect.each([
112+
output("not json"),
113+
output('{"error":{"code":"E401","detail":"sensitive"}}', 1),
114+
output('{"version":"9.9.9"}'),
115+
output("[]"),
116+
output("", 9009),
117+
{ ...output(JSON.stringify({ version })), stdoutTruncated: true },
118+
{ ...output(JSON.stringify({ version })), timedOut: true },
119+
"spawn failure",
120+
] as const)("reports an unavailable lookup rather than an unpublished release: %j", (result) =>
121+
Effect.gen(function* () {
122+
const service = yield* makeService(() =>
123+
result === "spawn failure" ? spawnFailure() : Effect.succeed(result),
124+
);
125+
const found = yield* service.get({ spec: version }).pipe(Effect.result);
126+
expect(Result.isFailure(found)).toBe(true);
127+
if (Result.isFailure(found)) {
128+
expect(found.failure).toBeInstanceOf(ServerNpmReleaseError);
129+
expect(found.failure.message).not.toContain("sensitive");
130+
}
131+
}),
132+
);
133+
134+
it.effect("falls back to public npm only when npm is not installed", () =>
135+
Effect.gen(function* () {
136+
const urls: string[] = [];
137+
const service = yield* makeService(
138+
() => Effect.die("npm is not installed"),
139+
HttpClient.make((request) => {
140+
urls.push(request.url);
141+
const response = request.url.endsWith("/nightly")
142+
? new Response(JSON.stringify({ version, t3code: { databaseMigration: 42 } }))
143+
: request.url.endsWith("/latest")
144+
? new Response(new ReadableStream())
145+
: new Response("{}", { status: 404 });
146+
return Effect.succeed(HttpClientResponse.fromWeb(request, response));
147+
}),
148+
false,
149+
);
150+
expect(yield* service.get({ spec: "nightly" })).toEqual({ version, databaseMigration: 42 });
151+
expect(yield* service.get({ spec: "9.9.9" })).toBeNull();
152+
const stalled = yield* service.get({ spec: "latest" }).pipe(Effect.result, Effect.forkChild);
153+
yield* TestClock.adjust("15 seconds");
154+
expect(Result.isFailure(yield* Fiber.join(stalled))).toBe(true);
155+
expect(urls).toEqual([
156+
"https://registry.npmjs.org/t3/nightly",
157+
"https://registry.npmjs.org/t3/9.9.9",
158+
"https://registry.npmjs.org/t3/latest",
159+
]);
160+
}),
161+
);
162+
163+
it.effect("runs at most two npm lookups at once", () =>
164+
Effect.gen(function* () {
165+
const started = yield* Queue.unbounded<void>();
166+
const release = yield* Deferred.make<void>();
167+
let running = 0;
168+
let peak = 0;
169+
const service = yield* makeService(() =>
170+
Effect.gen(function* () {
171+
peak = Math.max(peak, ++running);
172+
yield* Queue.offer(started, undefined);
173+
yield* Deferred.await(release);
174+
running--;
175+
return output('{"error":{"code":"E404"}}', 1);
176+
}),
177+
);
178+
const lookups = yield* Effect.forEach(
179+
["1.0.0", "1.0.1", "1.0.2"],
180+
(spec) => service.get({ spec }),
181+
{ concurrency: "unbounded" },
182+
).pipe(Effect.forkChild);
183+
yield* Queue.take(started);
184+
yield* Queue.take(started);
185+
yield* Deferred.succeed(release, undefined);
186+
expect(yield* Fiber.join(lookups)).toEqual([null, null, null]);
187+
expect(peak).toBe(2);
188+
}),
189+
);
190+
191+
it.effect.each(["--registry=https://example.com", "../other", "other-package@latest", "*", ""])(
192+
"rejects invalid release specs before spawning npm: %s",
193+
(spec) =>
194+
Effect.gen(function* () {
195+
const service = yield* makeService(() => Effect.die("invalid specs must not spawn npm"));
196+
expect(Result.isFailure(yield* service.get({ spec }).pipe(Effect.result))).toBe(true);
197+
}),
198+
);
Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
import {
2+
ServerNpmReleaseError,
3+
ServerNpmReleaseInput,
4+
type ServerNpmRelease,
5+
} from "@t3tools/contracts";
6+
import { npmReleaseDatabaseMigration } from "@t3tools/shared/cliRelease";
7+
import { resolveCommandPath } from "@t3tools/shared/shell";
8+
import * as Cache from "effect/Cache";
9+
import * as Context from "effect/Context";
10+
import * as Duration from "effect/Duration";
11+
import * as Effect from "effect/Effect";
12+
import * as Layer from "effect/Layer";
13+
import * as Option from "effect/Option";
14+
import * as Schema from "effect/Schema";
15+
import * as Semaphore from "effect/Semaphore";
16+
import { FetchHttpClient, HttpClient } from "effect/http";
17+
18+
import * as ProcessRunner from "../processRunner.ts";
19+
20+
const Version = Schema.String.check(Schema.isPattern(/^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/));
21+
const Manifest = Schema.Struct({ version: Version, t3code: Schema.optionalKey(Schema.Unknown) });
22+
// npm prints a bare version string when the release has no `t3code` field.
23+
const decodeManifest = Schema.decodeUnknownOption(
24+
Schema.fromJsonString(Schema.Union([Manifest, Schema.Tuple([Manifest]), Version])),
25+
);
26+
const decodeError = Schema.decodeUnknownOption(
27+
Schema.fromJsonString(Schema.Struct({ error: Schema.Struct({ code: Schema.String }) })),
28+
);
29+
const decodeInput = Schema.decodeUnknownEffect(ServerNpmReleaseInput);
30+
31+
export class NpmRelease extends Context.Service<
32+
NpmRelease,
33+
{
34+
readonly get: (
35+
input: ServerNpmReleaseInput,
36+
) => Effect.Effect<ServerNpmRelease | null, ServerNpmReleaseError>;
37+
}
38+
>()("t3/cloud/npmRelease") {}
39+
40+
function toRelease(spec: string, json: string) {
41+
const decoded = Option.getOrUndefined(decodeManifest(json));
42+
const manifest =
43+
typeof decoded === "string"
44+
? { version: decoded }
45+
: decoded && ("version" in decoded ? decoded : decoded[0]);
46+
if (!manifest || (spec !== "latest" && spec !== "nightly" && manifest.version !== spec)) {
47+
return Effect.fail(
48+
new ServerNpmReleaseError({ reason: "npm returned invalid release metadata." }),
49+
);
50+
}
51+
const databaseMigration = npmReleaseDatabaseMigration(manifest);
52+
return Effect.succeed({
53+
version: manifest.version,
54+
...(databaseMigration === undefined ? {} : { databaseMigration }),
55+
});
56+
}
57+
58+
export const make = Effect.fn("NpmRelease.make")(function* () {
59+
const runner = yield* ProcessRunner.ProcessRunner;
60+
const http = yield* HttpClient.HttpClient;
61+
const lookupPublic = Effect.fn("NpmRelease.lookupPublic")(function* (spec: string) {
62+
const unreachable = new ServerNpmReleaseError({
63+
reason: "npm is not installed and the public npm registry could not be reached.",
64+
});
65+
const json = yield* Effect.gen(function* () {
66+
const response = yield* http.get(`https://registry.npmjs.org/t3/${spec}`);
67+
if (response.status === 404) return Option.none<string>();
68+
if (response.status < 200 || response.status >= 300) return yield* unreachable;
69+
return Option.some(yield* response.text);
70+
}).pipe(
71+
Effect.timeout(Duration.seconds(15)),
72+
Effect.mapError(() => unreachable),
73+
);
74+
return Option.isNone(json) ? null : yield* toRelease(spec, json.value);
75+
});
76+
// npm loads the whole package history per lookup, so a burst of specs must not spawn one each.
77+
const npmPermits = yield* Semaphore.make(2);
78+
const cache = yield* Cache.make({
79+
capacity: 32,
80+
timeToLive: Duration.minutes(1),
81+
lookup: Effect.fn("NpmRelease.lookup")(function* (spec: string) {
82+
// Without npm there is no configured registry to honor, so use the public one.
83+
const npmPath = yield* resolveCommandPath("npm");
84+
if (npmPath === null) return yield* lookupPublic(spec);
85+
const output = yield* npmPermits
86+
.withPermits(1)(
87+
runner.run({
88+
command: npmPath,
89+
// Named fields keep the output small; the full package document grows with every release.
90+
args: ["view", "--global", "--json", `t3@${spec}`, "version", "t3code"],
91+
timeout: Duration.seconds(15),
92+
maxOutputBytes: 256 * 1024,
93+
}),
94+
)
95+
.pipe(
96+
Effect.tapError((error) =>
97+
Effect.logInfo("npm release lookup failed", { tag: error._tag }),
98+
),
99+
Effect.mapError(
100+
() => new ServerNpmReleaseError({ reason: "npm could not complete the lookup." }),
101+
),
102+
);
103+
if (output.timedOut || output.stdoutTruncated || output.stdoutInvalidUtf8) {
104+
return yield* new ServerNpmReleaseError({ reason: "npm returned an incomplete response." });
105+
}
106+
if (output.code !== 0) {
107+
const error = Option.getOrUndefined(decodeError(output.stdout));
108+
if (error?.error.code === "E404") return null;
109+
yield* Effect.logInfo("npm release lookup failed", {
110+
exitCode: output.code,
111+
npmErrorCode: error?.error.code,
112+
});
113+
return yield* new ServerNpmReleaseError({ reason: "npm could not read release metadata." });
114+
}
115+
return yield* toRelease(spec, output.stdout);
116+
}),
117+
});
118+
return NpmRelease.of({
119+
get: Effect.fn("NpmRelease.get")(function* (input) {
120+
const { spec } = yield* decodeInput(input).pipe(
121+
Effect.mapError(
122+
() => new ServerNpmReleaseError({ reason: "Invalid release version or track." }),
123+
),
124+
);
125+
return yield* Cache.get(cache, spec);
126+
}),
127+
});
128+
});
129+
130+
export const layer = Layer.effect(NpmRelease, make()).pipe(
131+
Layer.provide([ProcessRunner.layer, FetchHttpClient.layer]),
132+
);

‎apps/server/src/environment/ServerEnvironment.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -265,6 +265,7 @@ export const make = Effect.gen(function* () {
265265
projectCloneTracking: true,
266266
ignoredFileListing: true,
267267
triage: true,
268+
npmReleaseLookup: true,
268269
...(serverSelfUpdate === null ? {} : { serverSelfUpdate }),
269270
...(serverInstallation === null ? {} : { serverInstallation }),
270271
...(serverSelfUpdate === "boot-service" || desktopAppUpdate

‎apps/server/src/observability/RpcInstrumentation.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ const RPC_AGGREGATES = {
3333
[WS_METHODS.projectsMutate]: "orchestration",
3434
[WS_METHODS.serverProbe]: "server",
3535
[WS_METHODS.serverGetConfig]: "server",
36+
[WS_METHODS.serverGetNpmRelease]: "server",
3637
[WS_METHODS.serverRefreshProviders]: "server",
3738
[WS_METHODS.serverUpdateProvider]: "server",
3839
[WS_METHODS.providerAuthStart]: "provider",

‎apps/server/src/provider/Drivers/CodexDriver.test.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -667,7 +667,9 @@ it.layer(layerTest)("CodexDriver", (it) => {
667667
homePath: NodePath.join(tempDir, "codex-home"),
668668
},
669669
}).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, metadataSpawner));
670-
const capabilities = yield* instance.snapshot.resolveMaintenance();
670+
const capabilities = yield* instance.snapshot
671+
.resolveMaintenance()
672+
.pipe(Effect.provideService(HostProcess.Environment, { PATH: "" }));
671673
const latestVersion = yield* resolveLatestProviderVersion(capabilities).pipe(
672674
Effect.provideServiceEffect(
673675
ProviderLatestVersions.ProviderLatestVersions,

‎apps/server/src/provider/ProviderRegistry.ts‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ import type {
4444
import {
4545
makeManualOnlyProviderMaintenanceCapabilities,
4646
type ProviderMaintenanceCapabilities,
47+
latestProviderVersionCacheKeys,
4748
} from "@t3tools/provider-core/server/maintenanceResolver";
4849
import type { ProviderSnapshotSource } from "./builtInProviderCatalog.ts";
4950

@@ -718,9 +719,11 @@ export const layer = Layer.effect(
718719
Effect.gen(function* () {
719720
yield* instance.invalidateCaches ?? Effect.void;
720721
const maintenance = yield* instance.snapshot.resolveMaintenance({ fresh: true });
721-
if (maintenance.packageName) {
722-
yield* latestVersions.invalidate(maintenance.packageName);
723-
}
722+
yield* Effect.forEach(
723+
latestProviderVersionCacheKeys(maintenance),
724+
latestVersions.invalidate,
725+
{ discard: true },
726+
);
724727
}).pipe(Effect.catchCause((cause) => recoverInstanceFailure(instance.instanceId, cause))),
725728
{ concurrency: "unbounded", discard: true },
726729
);

0 commit comments

Comments
 (0)