Skip to content

Commit 7c1d559

Browse files
committed
Add "--filter" flag to enable git partial clone
1 parent 7ffab78 commit 7c1d559

2 files changed

Lines changed: 108 additions & 0 deletions

File tree

‎main.go‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,7 @@ type repoSync struct {
120120
repo string // remote repo to sync
121121
ref string // the ref to sync
122122
depth int // for shallow sync
123+
filter string // for partial clone
123124
submodules submodulesMode // how to handle submodules
124125
gc gcMode // garbage collection
125126
link absPath // absolute path to the symlink to publish
@@ -167,6 +168,9 @@ func main() {
167168
flDepth := pflag.Int("depth",
168169
envInt(1, "GITSYNC_DEPTH", "GIT_SYNC_DEPTH"),
169170
"create a shallow clone with history truncated to the specified number of commits")
171+
flFilter := pflag.String("filter",
172+
envString("", "GITSYNC_FILTER"),
173+
"use partial clone with the specified filter (e.g. 'blob:none', 'tree:0')")
170174
flSubmodules := pflag.String("submodules",
171175
envString("recursive", "GITSYNC_SUBMODULES", "GIT_SYNC_SUBMODULES"),
172176
"git submodule behavior: one of 'recursive', 'shallow', or 'off'")
@@ -709,6 +713,7 @@ func main() {
709713
repo: *flRepo,
710714
ref: *flRef,
711715
depth: *flDepth,
716+
filter: *flFilter,
712717
submodules: submodulesMode(*flSubmodules),
713718
gc: gcMode(*flGitGC),
714719
link: absLink,
@@ -1860,6 +1865,9 @@ func (git *repoSync) fetch(ctx context.Context, ref string) error {
18601865
args = append(args, "--unshallow")
18611866
}
18621867
}
1868+
if git.filter != "" {
1869+
args = append(args, "--filter", git.filter)
1870+
}
18631871
if _, _, err := git.Run(ctx, git.root, args...); err != nil {
18641872
return err
18651873
}
@@ -2445,6 +2453,13 @@ OPTIONS
24452453
single commit. Setting this to 0 will sync the full history of the
24462454
repo.
24472455
2456+
--filter <string>, $GITSYNC_FILTER
2457+
Use partial clone with the specified filter. This can reduce
2458+
the amount of data transferred when cloning large repositories.
2459+
Common values are 'blob:none' (omit all blobs, fetch on demand)
2460+
and 'tree:0' (omit all trees and blobs). This is most effective
2461+
when combined with --depth and --sparse-checkout-file.
2462+
24482463
--error-file <string>, $GITSYNC_ERROR_FILE
24492464
The path to an optional file into which errors will be written.
24502465
This may be an absolute path or a relative path, in which case it

‎test_e2e.sh‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3339,6 +3339,99 @@ function e2e::sparse_checkout() {
33393339
assert_file_eq "$ROOT/link/file2" "${FUNCNAME[0]}"
33403340
}
33413341

3342+
##############################################
3343+
# Test filter (partial clone) with blob:none
3344+
##############################################
3345+
function e2e::filter_partial_clone_blob_none() {
3346+
echo "${FUNCNAME[0]}" > "$REPO/file"
3347+
git -C "$REPO" commit -qam "${FUNCNAME[0]}"
3348+
3349+
GIT_SYNC \
3350+
--one-time \
3351+
--repo="file://$REPO" \
3352+
--root="$ROOT" \
3353+
--link="link" \
3354+
--filter="blob:none" \
3355+
--depth=0
3356+
assert_link_exists "$ROOT/link"
3357+
assert_file_exists "$ROOT/link/file"
3358+
assert_file_eq "$ROOT/link/file" "${FUNCNAME[0]}"
3359+
3360+
# Verify the repo is a partial clone
3361+
if ! git -C "$ROOT/link" config --get remote.origin.promisor >/dev/null 2>&1; then
3362+
# Check if the repo has a partial clone filter configured
3363+
local filter
3364+
filter=$(git -C "$ROOT/link" config --get remote.origin.partialclonefilter 2>/dev/null || true)
3365+
if [[ -z "$filter" ]]; then
3366+
fail "expected partial clone filter to be configured"
3367+
fi
3368+
fi
3369+
}
3370+
3371+
##############################################
3372+
# Test filter with sparse-checkout
3373+
##############################################
3374+
function e2e::filter_with_sparse_checkout() {
3375+
echo "!/*" > "$WORK/sparseconfig"
3376+
echo "!/*/" >> "$WORK/sparseconfig"
3377+
echo "file2" >> "$WORK/sparseconfig"
3378+
echo "${FUNCNAME[0]}" > "$REPO/file"
3379+
echo "${FUNCNAME[0]}" > "$REPO/file2"
3380+
mkdir -p "$REPO/dir"
3381+
echo "${FUNCNAME[0]}" > "$REPO/dir/file3"
3382+
git -C "$REPO" add file2
3383+
git -C "$REPO" add dir
3384+
git -C "$REPO" commit -qam "${FUNCNAME[0]}"
3385+
3386+
GIT_SYNC \
3387+
--one-time \
3388+
--repo="file://$REPO" \
3389+
--root="$ROOT" \
3390+
--link="link" \
3391+
--filter="blob:none" \
3392+
--depth=1 \
3393+
--sparse-checkout-file="$WORK/sparseconfig"
3394+
assert_link_exists "$ROOT/link"
3395+
assert_file_exists "$ROOT/link/file2"
3396+
assert_file_absent "$ROOT/link/dir/file3"
3397+
assert_file_absent "$ROOT/link/dir"
3398+
assert_file_eq "$ROOT/link/file2" "${FUNCNAME[0]}"
3399+
}
3400+
3401+
##############################################
3402+
# Test filter syncing across updates
3403+
##############################################
3404+
function e2e::filter_across_updates() {
3405+
# First sync
3406+
echo "${FUNCNAME[0]} 1" > "$REPO/file"
3407+
git -C "$REPO" commit -qam "${FUNCNAME[0]} 1"
3408+
3409+
GIT_SYNC \
3410+
--period=100ms \
3411+
--repo="file://$REPO" \
3412+
--filter="blob:none" \
3413+
--depth=0 \
3414+
--root="$ROOT" \
3415+
--link="link" \
3416+
&
3417+
wait_for_sync "${MAXWAIT}"
3418+
assert_link_exists "$ROOT/link"
3419+
assert_file_exists "$ROOT/link/file"
3420+
assert_file_eq "$ROOT/link/file" "${FUNCNAME[0]} 1"
3421+
assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 1
3422+
assert_metric_eq "${METRIC_FETCH_COUNT}" 1
3423+
3424+
# Move forward
3425+
echo "${FUNCNAME[0]} 2" > "$REPO/file"
3426+
git -C "$REPO" commit -qam "${FUNCNAME[0]} 2"
3427+
wait_for_sync "${MAXWAIT}"
3428+
assert_link_exists "$ROOT/link"
3429+
assert_file_exists "$ROOT/link/file"
3430+
assert_file_eq "$ROOT/link/file" "${FUNCNAME[0]} 2"
3431+
assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 2
3432+
assert_metric_eq "${METRIC_FETCH_COUNT}" 2
3433+
}
3434+
33423435
##############################################
33433436
# Test additional git configs
33443437
##############################################

0 commit comments

Comments
 (0)