Skip to content

Commit 1463f5e

Browse files
committed
fix(ci): large-binary check reads blobs by id and skips submodules
1 parent 358fa6e commit 1463f5e

2 files changed

Lines changed: 82 additions & 30 deletions

File tree

‎scripts/check-large-files.sh‎

Lines changed: 27 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -20,44 +20,39 @@
2020
# a file matching an LFS pattern but added without git-lfs installed is caught.
2121
#
2222
# We read the staged set ourselves rather than taking lefthook's {staged_files}
23-
# expansion: that expands to a bare space-separated string, which splits paths
24-
# containing spaces into separate args. `git diff --cached` + a line-based read
25-
# keeps whole paths intact (only a literal newline in a filename would break it,
26-
# which git quotes/escapes anyway).
23+
# expansion, which splits paths containing spaces into separate args.
2724

2825
set -u
2926

3027
MAX_KB="${HF_MAX_NONLFS_KB:-500}"
3128

32-
# Blob prefix for `git cat-file`: ":" is the index, "<head>:" a commit, empty
33-
# means read the file on disk.
34-
BLOB_REV=""
29+
# staged (default): the index; range: what <head> adds over <base>; files: paths on disk.
30+
MODE=staged
3531
RANGE_BASE=""
3632
RANGE_HEAD=""
3733
if [ "${1:-}" = "--range" ]; then
3834
[ "$#" -eq 3 ] || { echo "usage: $0 --range <base> <head>" >&2; exit 2; }
35+
MODE=range
3936
RANGE_BASE="$2"
4037
RANGE_HEAD="$3"
41-
BLOB_REV="$3:"
4238
shift 3
43-
elif [ "$#" -eq 0 ]; then
44-
BLOB_REV=":"
39+
elif [ "$#" -gt 0 ]; then
40+
MODE=files
4541
fi
4642

47-
# Emit the paths to check, NUL-separated and unquoted (git quotes non-ASCII,
48-
# quotes and tabs otherwise). Added/Copied/Modified/Renamed/Type-changed only.
43+
# Emit a "<raw diff header>" record then a path record per entry, NUL-separated and
44+
# unquoted. The header carries the new mode and blob id, so blobs are read by id,
45+
# never by a path git could parse as revision syntax. Renames count as additions.
4946
list_files() {
50-
if [ -n "$RANGE_BASE" ]; then
51-
git diff -z --name-only --diff-filter=ACMRT "$RANGE_BASE" "$RANGE_HEAD"
52-
elif [ -n "$BLOB_REV" ]; then
53-
git diff -z --cached --name-only --diff-filter=ACMRT
54-
else
55-
printf '%s\0' "$@"
56-
fi
47+
case "$MODE" in
48+
range) git diff -z --raw --no-abbrev --no-renames --diff-filter=ACMT "$RANGE_BASE" "$RANGE_HEAD" ;;
49+
staged) git diff -z --raw --no-abbrev --no-renames --diff-filter=ACMT --cached ;;
50+
files) for f in "$@"; do printf 'file\0%s\0' "$f"; done ;;
51+
esac
5752
}
5853

5954
read_bytes() {
60-
if [ -n "$BLOB_REV" ]; then git cat-file blob "$BLOB_REV$1"; else cat -- "$1"; fi
55+
if [ "$MODE" = files ]; then cat -- "$1"; else git cat-file blob "$2"; fi
6156
}
6257

6358
violations="$(mktemp)"
@@ -68,26 +63,29 @@ trap 'rm -f "$violations" "$errors" "$paths" "$blob"' EXIT INT TERM
6863

6964
# A git error (bad ref, not a repo) must fail the check, not pass an empty list.
7065
list_files "$@" > "$paths" || { echo "ERROR: could not list the files to check." >&2; exit 2; }
71-
# Paths are NUL-separated, so a newline byte can only come from a filename; sh cannot
66+
# Records are NUL-separated, so a newline byte can only come from a filename; sh cannot
7267
# keep such a name whole, so refuse it rather than check the wrong files.
7368
if [ "$(tr -cd '\n' < "$paths" | wc -c)" -gt 0 ]; then
7469
echo "ERROR: a file name contains a newline; rename it." >&2
7570
exit 2
7671
fi
7772

78-
tr '\0' '\n' < "$paths" | while IFS= read -r f; do
73+
tr '\0' '\n' < "$paths" | while IFS= read -r header && IFS= read -r f; do
7974
[ -n "$f" ] || continue
8075

81-
if [ -n "$BLOB_REV" ]; then
82-
# A submodule is not a blob; a symlink's blob is its short target path.
83-
kind="$(git cat-file -t "$BLOB_REV$f")" || { printf '%s\n' "$f" >> "$errors"; continue; }
84-
[ "$kind" = "blob" ] || continue
85-
else
76+
id=""
77+
if [ "$MODE" = files ]; then
8678
# Skip symlinks: `wc -c` would measure the link *target's* bytes, so a symlink
8779
# to a large LFS-tracked asset could be flagged even though the real blob is a
8880
# tiny pointer.
8981
[ -L "$f" ] && continue
9082
[ -f "$f" ] || continue
83+
else
84+
# ":<old mode> <new mode> <old id> <new id> <status>"
85+
set -- $header
86+
# A submodule (160000) records another repository's commit: nothing stored here.
87+
[ "$2" = 160000 ] && continue
88+
id="$4"
9189
fi
9290

9391
# registry/ intentionally ships raw binary assets (block backgrounds, avatar
@@ -108,7 +106,7 @@ tr '\0' '\n' < "$paths" | while IFS= read -r f; do
108106
# git uses to print "Binary files differ". A generated blob of text is still
109107
# caught by review, not here.
110108
# Read once into a file so a failed read is an error, never an empty pass.
111-
read_bytes "$f" > "$blob" || { printf '%s\n' "$f" >> "$errors"; continue; }
109+
read_bytes "$f" "$id" > "$blob" || { printf '%s\n' "$f" >> "$errors"; continue; }
112110
grep -qI . "$blob" && continue
113111

114112
bytes="$(wc -c < "$blob" | tr -d ' ')"
@@ -124,7 +122,7 @@ tr '\0' '\n' < "$paths" | while IFS= read -r f; do
124122
filter="$(git check-attr filter -- "$f" | sed 's/.*: //')"
125123
note=""
126124
if [ "$filter" = "lfs" ]; then
127-
[ -z "$BLOB_REV" ] && continue
125+
[ "$MODE" = files ] && continue
128126
note=" — matches an LFS pattern but was stored raw; run \`git lfs install\` and re-add it"
129127
fi
130128

‎scripts/check-large-files.test.mjs‎

Lines changed: 55 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,13 +24,23 @@ function withRepo(run) {
2424
git("commit", "-q", "-m", "c");
2525
return git("rev-parse", "HEAD").stdout.trim();
2626
};
27+
// `files` null checks the index as it stands (git add -A would drop a gitlink with no checkout).
28+
const staged = (files) => {
29+
if (files) {
30+
for (const [name, contents] of Object.entries(files))
31+
writeFileSync(join(dir, name), contents);
32+
git("add", "-A");
33+
}
34+
const result = spawnSync(SCRIPT, [], { cwd: dir, env, encoding: "utf-8" });
35+
return { ok: result.status === 0, stderr: result.stderr ?? "" };
36+
};
2737
const range = (base, head) => {
2838
const result = spawnSync(SCRIPT, ["--range", base, head], { cwd: dir, env, encoding: "utf-8" });
2939
return { ok: result.status === 0, stderr: result.stderr ?? "" };
3040
};
3141
try {
3242
git("init", "-q");
33-
run({ commit, range, dir, git });
43+
run({ commit, staged, range, dir, git });
3444
} finally {
3545
rmSync(dir, { recursive: true, force: true });
3646
}
@@ -172,4 +182,48 @@ describe("check-large-files", () => {
172182
assert.equal(range(base, head).ok, false);
173183
});
174184
});
185+
186+
it("staged mode rejects a large binary and passes text", () => {
187+
withRepo(({ commit, staged }) => {
188+
commit({ "seed.txt": "x" });
189+
const { ok, stderr } = staged({ "big.bin": bigBinary, "big.txt": bigText });
190+
assert.equal(ok, false);
191+
assert.match(stderr, /big\.bin/);
192+
assert.doesNotMatch(stderr, /big\.txt/);
193+
});
194+
});
195+
196+
it("staged mode sizes a file named like revision syntax, not the file it aliases", () => {
197+
withRepo(({ commit, staged }) => {
198+
commit({ "clip.bin": "small" });
199+
const { ok, stderr } = staged({ "0:clip.bin": bigBinary });
200+
assert.equal(ok, false);
201+
assert.match(stderr, /0:clip\.bin/);
202+
});
203+
});
204+
205+
it("staged mode checks modified and type-changed files", () => {
206+
withRepo(({ commit, staged, dir }) => {
207+
writeFileSync(join(dir, "target"), "t");
208+
symlinkSync("target", join(dir, "link.bin"));
209+
commit({ "clip.bin": "small" });
210+
rmSync(join(dir, "link.bin"));
211+
const { ok, stderr } = staged({ "clip.bin": bigBinary, "link.bin": bigBinary });
212+
assert.equal(ok, false);
213+
assert.match(stderr, /clip\.bin/);
214+
assert.match(stderr, /link\.bin/);
215+
});
216+
});
217+
218+
it("skips a submodule whose commit is not in this repository", () => {
219+
withRepo(({ commit, staged, range, git }) => {
220+
const base = commit({ "seed.txt": "x" });
221+
const child = "1234567890abcdef1234567890abcdef12345678";
222+
git("update-index", "--add", "--cacheinfo", `160000,${child},sub`);
223+
assert.equal(staged(null).ok, true);
224+
git("commit", "-q", "-m", "sub");
225+
const head = git("rev-parse", "HEAD").stdout.trim();
226+
assert.equal(range(base, head).ok, true);
227+
});
228+
});
175229
});

0 commit comments

Comments
 (0)