Skip to content

Commit 7186a80

Browse files
chore(repos): sync reference repos from shallow clones
`git subtree add/pull <url> <tag>` fetched the upstream repos' commits and tags into this repo. The foreign tags (about 14k from Effect and alchemy-effect) left shallow boundaries behind, and git.cafe then rejected every fetch with "upload-pack: invalid request". `sync:repos` now shallow-clones the resolved ref with --depth=1 --no-tags into a temp dir next to `.repos/<id>`, drops its .git, renames it into place (so relative symlinks stay relative), and commits only that prefix with the upstream SHA in the message. A sync with no changes skips the commit. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 2b37449 commit 7186a80

2 files changed

Lines changed: 114 additions & 63 deletions

File tree

‎scripts/sync-reference-repos.test.ts‎

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ it.layer(NodeServices.layer)("sync-reference-repos", (it) => {
3131
}),
3232
);
3333

34-
it.effect("plans an add for a missing subtree and a pull for an existing subtree", () =>
34+
it.effect("plans a shallow, tagless clone of the resolved ref", () =>
3535
Effect.gen(function* () {
3636
const fs = yield* FileSystem.FileSystem;
3737
const path = yield* Path.Path;
@@ -43,19 +43,17 @@ it.layer(NodeServices.layer)("sync-reference-repos", (it) => {
4343
"catalog:\n effect: 4.0.0-beta.73\n",
4444
);
4545

46-
const addPlan = yield* planReferenceRepoSync(effectSmol, rootDir, false);
47-
assert.equal(addPlan.action, "add");
48-
assert.deepStrictEqual(addPlan.args, [
49-
"subtree",
50-
"add",
51-
"--prefix=.repos/effect-smol",
46+
const plan = yield* planReferenceRepoSync(effectSmol, rootDir, false);
47+
assert.equal(plan.ref, "[email protected]");
48+
assert.deepStrictEqual(plan.cloneArgs, [
49+
"clone",
50+
"--quiet",
51+
"--depth=1",
52+
"--no-tags",
53+
"--single-branch",
54+
5255
"https://github.com/Effect-TS/effect.git",
53-
54-
"--squash",
5556
]);
56-
57-
yield* fs.makeDirectory(path.join(rootDir, effectSmol.prefix), { recursive: true });
58-
assert.equal((yield* planReferenceRepoSync(effectSmol, rootDir, false)).action, "pull");
5957
}),
6058
);
6159
});

‎scripts/sync-reference-repos.ts‎

Lines changed: 104 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,6 @@ import { fromYaml } from "@t3tools/shared/schemaYaml";
1515

1616
import { referenceRepos, type ReferenceRepo } from "./lib/reference-repos.ts";
1717

18-
export type ReferenceRepoSyncAction = "add" | "pull";
19-
2018
export interface ReferenceRepoSyncOptions {
2119
readonly rootDir?: string | undefined;
2220
readonly repoId?: string | undefined;
@@ -26,9 +24,9 @@ export interface ReferenceRepoSyncOptions {
2624

2725
export interface ReferenceRepoSyncPlan {
2826
readonly repo: ReferenceRepo;
29-
readonly action: ReferenceRepoSyncAction;
3027
readonly ref: string;
31-
readonly args: ReadonlyArray<string>;
28+
/** `git clone` arguments, without the destination directory. */
29+
readonly cloneArgs: ReadonlyArray<string>;
3230
}
3331

3432
export class ReferenceRepoSelectionError extends Schema.TaggedError<ReferenceRepoSelectionError>()(
@@ -70,32 +68,44 @@ export class ReferenceRepoVersionResolutionError extends Schema.TaggedError<Refe
7068
}
7169
}
7270

73-
export class ReferenceRepoGitSubtreeError extends Schema.TaggedError<ReferenceRepoGitSubtreeError>()(
74-
"ReferenceRepoGitSubtreeError",
71+
export class ReferenceRepoGitError extends Schema.TaggedError<ReferenceRepoGitError>()(
72+
"ReferenceRepoGitError",
7573
{
7674
operation: Schema.Literals(["spawn", "communicate", "exit"]),
7775
repoId: Schema.String,
78-
action: Schema.Literals(["add", "pull"]),
79-
repository: Schema.String,
76+
command: Schema.String,
8077
ref: Schema.String,
81-
rootDir: Schema.String,
82-
argumentCount: Schema.Number,
78+
cwd: Schema.String,
8379
exitCode: Schema.optional(Schema.Number),
84-
stdoutLength: Schema.optional(Schema.Number),
85-
stderrLength: Schema.optional(Schema.Number),
80+
stderr: Schema.optional(Schema.String),
8681
cause: Schema.optional(Schema.Defect()),
8782
},
8883
) {
8984
override get message(): string {
90-
return `Git subtree ${this.action} for reference repo "${this.repoId}" failed during "${this.operation}".`;
85+
const detail = this.stderr ? `: ${this.stderr.trim()}` : "";
86+
return `git ${this.command} for reference repo "${this.repoId}" at ${this.ref} failed during "${this.operation}"${detail}`;
87+
}
88+
}
89+
90+
export class ReferenceRepoFileSystemError extends Schema.TaggedError<ReferenceRepoFileSystemError>()(
91+
"ReferenceRepoFileSystemError",
92+
{
93+
repoId: Schema.String,
94+
prefix: Schema.String,
95+
cause: Schema.Defect(),
96+
},
97+
) {
98+
override get message(): string {
99+
return `Replacing ${this.prefix} for reference repo "${this.repoId}" failed.`;
91100
}
92101
}
93102

94103
export type ReferenceRepoSyncError =
95104
| ReferenceRepoSelectionError
96105
| ReferenceRepoVersionSourceError
97106
| ReferenceRepoVersionResolutionError
98-
| ReferenceRepoGitSubtreeError;
107+
| ReferenceRepoGitError
108+
| ReferenceRepoFileSystemError;
99109

100110
const decodeJsonSource = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Unknown));
101111
const decodeYamlSource = Schema.decodeEffect(fromYaml(Schema.Unknown));
@@ -197,46 +207,51 @@ export const resolveReferenceRepoRef = Effect.fn("resolveReferenceRepoRef")(func
197207
return `${repo.versionTagPrefix}${version}`;
198208
});
199209

210+
/**
211+
* Shallow-clone the upstream ref into a temp dir and copy its files into `.repos/<id>`.
212+
* Never fetch upstream into this repo: its tags and shallow boundaries break `git fetch`.
213+
*/
200214
export const planReferenceRepoSync = Effect.fn("planReferenceRepoSync")(function* (
201215
repo: ReferenceRepo,
202216
rootDir: string,
203217
latest: boolean,
204218
) {
205-
const fs = yield* FileSystem.FileSystem;
206-
const path = yield* Path.Path;
207-
const action: ReferenceRepoSyncAction = (yield* fs.exists(path.join(rootDir, repo.prefix)))
208-
? "pull"
209-
: "add";
210219
const ref = yield* resolveReferenceRepoRef(repo, rootDir, latest);
211220

212221
return {
213222
repo,
214-
action,
215223
ref,
216-
args: ["subtree", action, `--prefix=${repo.prefix}`, repo.repository, ref, "--squash"],
224+
cloneArgs: [
225+
"clone",
226+
"--quiet",
227+
"--depth=1",
228+
"--no-tags",
229+
"--single-branch",
230+
`--branch=${ref}`,
231+
repo.repository,
232+
],
217233
} satisfies ReferenceRepoSyncPlan;
218234
});
219235

220-
const runGit = Effect.fn("runGit")(function* (rootDir: string, plan: ReferenceRepoSyncPlan) {
236+
const runGit = Effect.fn("runGit")(function* (
237+
plan: ReferenceRepoSyncPlan,
238+
cwd: string,
239+
args: ReadonlyArray<string>,
240+
) {
221241
const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
222242
const errorContext = {
223243
repoId: plan.repo.id,
224-
action: plan.action,
225-
repository: plan.repo.repository,
244+
command: args[0] ?? "",
226245
ref: plan.ref,
227-
rootDir,
228-
argumentCount: plan.args.length,
246+
cwd,
229247
} as const;
230-
const child = yield* spawner.spawn(ChildProcess.make("git", plan.args, { cwd: rootDir })).pipe(
231-
Effect.mapError(
232-
(cause) =>
233-
new ReferenceRepoGitSubtreeError({
234-
...errorContext,
235-
operation: "spawn",
236-
cause,
237-
}),
238-
),
239-
);
248+
const child = yield* spawner
249+
.spawn(ChildProcess.make("git", [...args], { cwd }))
250+
.pipe(
251+
Effect.mapError(
252+
(cause) => new ReferenceRepoGitError({ ...errorContext, operation: "spawn", cause }),
253+
),
254+
);
240255
const [stdout, stderr, exitCode] = yield* Effect.all(
241256
[
242257
collectStreamAsString(child.stdout),
@@ -246,28 +261,66 @@ const runGit = Effect.fn("runGit")(function* (rootDir: string, plan: ReferenceRe
246261
{ concurrency: "unbounded" },
247262
).pipe(
248263
Effect.mapError(
249-
(cause) =>
250-
new ReferenceRepoGitSubtreeError({
251-
...errorContext,
252-
operation: "communicate",
253-
cause,
254-
}),
264+
(cause) => new ReferenceRepoGitError({ ...errorContext, operation: "communicate", cause }),
255265
),
256266
);
257267

258268
if (exitCode !== 0) {
259-
return yield* new ReferenceRepoGitSubtreeError({
269+
return yield* new ReferenceRepoGitError({
260270
...errorContext,
261271
operation: "exit",
262272
exitCode,
263-
stdoutLength: stdout.length,
264-
stderrLength: stderr.length,
273+
stderr,
265274
});
266275
}
267276

268-
if (stdout.trim().length > 0) {
269-
yield* Console.log(stdout.trim());
277+
return stdout;
278+
});
279+
280+
const applyReferenceRepoSync = Effect.fn("applyReferenceRepoSync")(function* (
281+
rootDir: string,
282+
plan: ReferenceRepoSyncPlan,
283+
) {
284+
const fs = yield* FileSystem.FileSystem;
285+
const path = yield* Path.Path;
286+
const { repo } = plan;
287+
const target = path.join(rootDir, repo.prefix);
288+
const mapFsError = Effect.mapError(
289+
(cause: unknown) =>
290+
new ReferenceRepoFileSystemError({ repoId: repo.id, prefix: repo.prefix, cause }),
291+
);
292+
293+
// Clone next to the target so a rename moves it into place: same filesystem, and
294+
// relative symlinks stay relative (`fs.copy` rewrites them to absolute temp paths).
295+
yield* fs.makeDirectory(path.dirname(target), { recursive: true }).pipe(mapFsError);
296+
const tempDir = yield* fs
297+
.makeTempDirectoryScoped({ directory: path.dirname(target), prefix: `.sync-${repo.id}-` })
298+
.pipe(mapFsError);
299+
const checkout = path.join(tempDir, "checkout");
300+
yield* runGit(plan, rootDir, [...plan.cloneArgs, checkout]);
301+
const commit = (yield* runGit(plan, checkout, ["rev-parse", "HEAD"])).trim();
302+
303+
yield* Effect.gen(function* () {
304+
yield* fs.remove(path.join(checkout, ".git"), { recursive: true });
305+
yield* fs.remove(target, { recursive: true, force: true });
306+
yield* fs.rename(checkout, target);
307+
}).pipe(mapFsError);
308+
309+
yield* runGit(plan, rootDir, ["add", "--all", "--", repo.prefix]);
310+
const status = yield* runGit(plan, rootDir, ["status", "--porcelain", "--", repo.prefix]);
311+
if (status.trim().length === 0) {
312+
yield* Console.log(`${repo.id} is already at ${plan.ref}.`);
313+
return;
270314
}
315+
yield* runGit(plan, rootDir, [
316+
"commit",
317+
"--quiet",
318+
"--message",
319+
`chore(repos): sync ${repo.id} to ${plan.ref}\n\nUpstream: ${repo.repository} ${commit}`,
320+
"--",
321+
repo.prefix,
322+
]);
323+
yield* Console.log(`Committed ${repo.prefix} at ${plan.ref} (${commit.slice(0, 12)}).`);
271324
});
272325

273326
export const syncReferenceRepos = Effect.fn("syncReferenceRepos")(function* (
@@ -281,9 +334,9 @@ export const syncReferenceRepos = Effect.fn("syncReferenceRepos")(function* (
281334
for (const repo of repos) {
282335
const plan = yield* planReferenceRepoSync(repo, rootDir, options.latest ?? false);
283336
plans.push(plan);
284-
yield* Console.log(`Syncing ${repo.id} from ${plan.ref} with git subtree ${plan.action}.`);
337+
yield* Console.log(`Syncing ${repo.id} from ${plan.ref} (shallow clone, no history).`);
285338
if (!(options.dryRun ?? false)) {
286-
yield* runGit(rootDir, plan).pipe(Effect.scoped);
339+
yield* applyReferenceRepoSync(rootDir, plan).pipe(Effect.scoped);
287340
}
288341
}
289342

@@ -304,11 +357,11 @@ export const syncReferenceReposCommand = Command.make(
304357
Flag.withDefault(false),
305358
),
306359
root: Flag.String("root").pipe(
307-
Flag.withDescription("Workspace root used to resolve versions and subtree prefixes."),
360+
Flag.withDescription("Workspace root used to resolve versions and .repos/ prefixes."),
308361
Flag.optional,
309362
),
310363
dryRun: Flag.Boolean("dry-run").pipe(
311-
Flag.withDescription("Print planned subtree operations without running git."),
364+
Flag.withDescription("Print the planned syncs without cloning or committing."),
312365
Flag.withDefault(false),
313366
),
314367
},

0 commit comments

Comments
 (0)