Skip to content

Commit a4d82ec

Browse files
authored
perf(relay): cache Git pack hydration (#2169)
1 parent 984645e commit a4d82ec

13 files changed

Lines changed: 944 additions & 104 deletions

File tree

‎.env.example‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -66,12 +66,15 @@ RELAY_URL=ws://localhost:3000
6666
# -----------------------------------------------------------------------------
6767
# Git (NIP-34 bare repositories)
6868
# -----------------------------------------------------------------------------
69-
# Root directory for bare git repos. Repos are stored at
70-
# {path}/{owner_hex}/{repo_id}.git/. Default: ./repos (relative to CWD).
71-
# Set an absolute path to keep repos stable across worktrees.
69+
# Root directory for ephemeral Git workspaces and the disposable pack cache.
70+
# Default: ./repos (relative to CWD).
7271
# BUZZ_GIT_REPO_PATH=./repos
7372
# BUZZ_GIT_MAX_PACK_BYTES=524288000
7473
# BUZZ_GIT_MAX_REPO_BYTES=1048576000
74+
# Process-local immutable pack/index cache. Zero disables retention.
75+
# BUZZ_GIT_PACK_CACHE_PATH=./repos/.pack-cache
76+
# BUZZ_GIT_PACK_CACHE_MAX_BYTES=5368709120
77+
# BUZZ_GIT_PACK_CACHE_MAX_CONCURRENT_POPULATIONS=2
7578

7679
# -----------------------------------------------------------------------------
7780
# Media Upload Admission

‎crates/buzz-relay/src/api/git/hydrate.rs‎

Lines changed: 122 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,11 @@
66
//!
77
//! 1. GET pointer → manifest digest.
88
//! 2. GET manifest (digest-verified) → parsed [`Manifest`].
9-
//! 3. GET every pack the manifest names (digest-verified, in parallel).
10-
//! 4. **Phase 1** — for each pack: write `pack-<hex>.pack`; install and
11-
//! validate a cached `idx/<hex>` sidecar when present, otherwise run
12-
//! `git index-pack` to materialize `.idx`. Failure here tears down the
13-
//! tempdir with no refs/HEAD ever written.
9+
//! 3. Resolve every pack through the bounded local content-addressed cache.
10+
//! 4. **Phase 1** — hard-link or copy each verified pack/index pair into the
11+
//! ephemeral repo. A cache miss reads object storage and validates or
12+
//! generates the index once per digest. Failure here tears down the tempdir
13+
//! with no refs/HEAD ever written.
1414
//! 5. **Phase 2** — only after all packs are indexed: write loose refs and
1515
//! HEAD from the manifest.
1616
//!
@@ -19,8 +19,8 @@
1919
//! mid-stream. Sami/Max named this explicitly.
2020
//!
2121
//! The returned [`HydratedRepo`] owns a [`tempfile::TempDir`]; dropping it
22-
//! cleans up. Every read currently re-hydrates from scratch — naive but
23-
//! correct; caching is named follow-up work.
22+
//! cleans up. Cached pack/index pairs are immutable performance state; object
23+
//! storage remains authoritative.
2424
2525
// Public surface is consumed by `transport.rs` after Eva's `AppState::git_store`
2626
// wires in; the items below are intentionally `pub` to keep the consumer-side
@@ -35,6 +35,7 @@ use tokio::process::Command;
3535

3636
use super::cas_publish::ParentState;
3737
use super::manifest::{is_hex_oid, is_safe_refname, pointer_key, Manifest, ManifestError};
38+
use super::pack_cache::GitPackCache;
3839
use super::store::{ETag, GitStore, StoreError};
3940
use buzz_core::TenantContext;
4041

@@ -75,6 +76,19 @@ impl HydratedRepo {
7576
}
7677
}
7778

79+
/// Process-local resources and limits used to materialize one repository.
80+
#[derive(Clone, Copy)]
81+
pub struct HydrationOptions<'a> {
82+
/// Immutable pack/index cache shared by Git operations in this process.
83+
pub pack_cache: &'a GitPackCache,
84+
/// Parent directory for ephemeral bare repositories.
85+
pub scratch_dir: &'a Path,
86+
/// Maximum bytes accepted for one pack object.
87+
pub max_pack_bytes: u64,
88+
/// Maximum aggregate pack bytes accepted for one repository.
89+
pub max_repo_bytes: u64,
90+
}
91+
7892
/// Hydration errors.
7993
///
8094
/// "Repo doesn't exist" is signalled by `Ok(None)` from `hydrate_for_read`,
@@ -112,21 +126,10 @@ pub async fn hydrate_for_read(
112126
ctx: &TenantContext,
113127
owner: &str,
114128
repo: &str,
115-
scratch_dir: &Path,
116-
max_pack_bytes: u64,
117-
max_repo_bytes: u64,
129+
options: HydrationOptions<'_>,
118130
) -> Result<Option<HydratedRepo>, HydrateError> {
119131
let started_at = std::time::Instant::now();
120-
let result = hydrate_for_read_inner(
121-
store,
122-
ctx,
123-
owner,
124-
repo,
125-
scratch_dir,
126-
max_pack_bytes,
127-
max_repo_bytes,
128-
)
129-
.await;
132+
let result = hydrate_for_read_inner(store, ctx, owner, repo, options).await;
130133
let outcome = match &result {
131134
Ok(Some(repo)) => {
132135
metrics::histogram!("buzz_git_hydrate_bytes").record(repo.hydrated_bytes() as f64);
@@ -151,23 +154,12 @@ async fn hydrate_for_read_inner(
151154
ctx: &TenantContext,
152155
owner: &str,
153156
repo: &str,
154-
scratch_dir: &Path,
155-
max_pack_bytes: u64,
156-
max_repo_bytes: u64,
157+
options: HydrationOptions<'_>,
157158
) -> Result<Option<HydratedRepo>, HydrateError> {
158159
let Some((_etag, _digest, manifest)) = load_pointer(store, ctx, owner, repo).await? else {
159160
return Ok(None);
160161
};
161-
Ok(Some(
162-
materialize_manifest(
163-
store,
164-
&manifest,
165-
scratch_dir,
166-
max_pack_bytes,
167-
max_repo_bytes,
168-
)
169-
.await?,
170-
))
162+
Ok(Some(materialize_manifest(store, &manifest, options).await?))
171163
}
172164

173165
/// Load the current manifest for a read path without materializing pack objects.
@@ -212,29 +204,21 @@ pub async fn hydrate_for_write(
212204
ctx: &TenantContext,
213205
owner: &str,
214206
repo: &str,
215-
scratch_dir: &Path,
216-
max_pack_bytes: u64,
217-
max_repo_bytes: u64,
207+
options: HydrationOptions<'_>,
218208
) -> Result<(HydratedRepo, ParentState), HydrateError> {
219209
match load_pointer(store, ctx, owner, repo).await? {
220210
Some((etag, digest, manifest)) => {
221-
let repo = materialize_manifest(
222-
store,
223-
&manifest,
224-
scratch_dir,
225-
max_pack_bytes,
226-
max_repo_bytes,
227-
)
228-
.await?;
211+
let repo = materialize_manifest(store, &manifest, options).await?;
229212
let parent = ParentState::from_loaded(etag, digest, manifest);
230213
Ok((repo, parent))
231214
}
232215
None => {
233216
// First push: empty bare repo. No packs to fetch, no refs to
234217
// install. `receive-pack` will accept whatever the client
235218
// sends; `cas_publish` will use `If-None-Match: *`.
236-
let tempdir = TempDir::new_in(scratch_dir)
237-
.map_err(|e| HydrateError::Hydrate(format!("tempdir in {scratch_dir:?}: {e}")))?;
219+
let tempdir = TempDir::new_in(options.scratch_dir).map_err(|e| {
220+
HydrateError::Hydrate(format!("tempdir in {:?}: {e}", options.scratch_dir))
221+
})?;
238222
let path = tempdir.path().to_path_buf();
239223
run_git(&path, &["init", "--bare", "--quiet"]).await?;
240224
Ok((
@@ -280,7 +264,7 @@ async fn load_pointer(
280264
Ok(Some((etag, digest, manifest)))
281265
}
282266

283-
async fn get_verified_limited(
267+
pub(super) async fn get_verified_limited(
284268
store: &GitStore,
285269
key: &str,
286270
digest: &str,
@@ -304,13 +288,11 @@ async fn get_verified_limited(
304288
async fn materialize_manifest(
305289
store: &GitStore,
306290
manifest: &Manifest,
307-
scratch_dir: &Path,
308-
max_pack_bytes: u64,
309-
max_repo_bytes: u64,
291+
options: HydrationOptions<'_>,
310292
) -> Result<HydratedRepo, HydrateError> {
311293
// Init bare repo.
312-
let tempdir = TempDir::new_in(scratch_dir)
313-
.map_err(|e| HydrateError::Hydrate(format!("tempdir in {scratch_dir:?}: {e}")))?;
294+
let tempdir = TempDir::new_in(options.scratch_dir)
295+
.map_err(|e| HydrateError::Hydrate(format!("tempdir in {:?}: {e}", options.scratch_dir)))?;
314296
let path = tempdir.path().to_path_buf();
315297
run_git(&path, &["init", "--bare", "--quiet"]).await?;
316298

@@ -324,27 +306,25 @@ async fn materialize_manifest(
324306
let digest = key
325307
.strip_prefix("packs/")
326308
.ok_or_else(|| HydrateError::Hydrate(format!("malformed pack key {key:?}")))?;
327-
let bytes = get_verified_limited(store, key, digest, max_pack_bytes).await?;
328-
let pack_bytes = u64::try_from(bytes.len()).unwrap_or(u64::MAX);
329-
if pack_bytes > max_pack_bytes {
309+
let pack_bytes = options
310+
.pack_cache
311+
.materialize_pack(store, key, digest, &pack_dir, options.max_pack_bytes)
312+
.await?;
313+
if pack_bytes > options.max_pack_bytes {
330314
return Err(HydrateError::ResourceLimit(format!(
331-
"pack {digest} is {pack_bytes} bytes (max {max_pack_bytes})"
315+
"pack {digest} is {pack_bytes} bytes (max {})",
316+
options.max_pack_bytes
332317
)));
333318
}
334319
hydrated_bytes = hydrated_bytes.checked_add(pack_bytes).ok_or_else(|| {
335320
HydrateError::ResourceLimit("repo byte count overflowed u64".to_string())
336321
})?;
337-
if hydrated_bytes > max_repo_bytes {
322+
if hydrated_bytes > options.max_repo_bytes {
338323
return Err(HydrateError::ResourceLimit(format!(
339-
"repo needs {hydrated_bytes} hydrated bytes (max {max_repo_bytes})"
324+
"repo needs {hydrated_bytes} hydrated bytes (max {})",
325+
options.max_repo_bytes
340326
)));
341327
}
342-
343-
let pack_path = pack_dir.join(format!("pack-{digest}.pack"));
344-
tokio::fs::write(&pack_path, &bytes)
345-
.await
346-
.map_err(|e| HydrateError::Hydrate(format!("write pack {digest}: {e}")))?;
347-
install_or_generate_idx(store, &path, digest, &pack_path).await?;
348328
}
349329

350330
// Phase 2: install refs and HEAD. After this point, the repo advertises.
@@ -393,14 +373,15 @@ async fn materialize_manifest(
393373
})
394374
}
395375

396-
async fn install_or_generate_idx(
376+
pub(super) async fn install_or_generate_idx(
397377
store: &GitStore,
398378
repo_path: &Path,
399379
pack_digest: &str,
400380
pack_path: &Path,
381+
max_idx_bytes: u64,
401382
) -> Result<(), HydrateError> {
402383
let idx_path = pack_path.with_extension("idx");
403-
match store.get_idx(pack_digest).await {
384+
match store.get_idx(pack_digest, max_idx_bytes).await {
404385
Ok(Some(idx_bytes)) => {
405386
tokio::fs::write(&idx_path, &idx_bytes)
406387
.await
@@ -438,7 +419,27 @@ async fn install_or_generate_idx(
438419
let pack_path_str = pack_path
439420
.to_str()
440421
.ok_or_else(|| HydrateError::Hydrate("pack path is not valid utf-8".to_string()))?;
441-
run_git(repo_path, &["index-pack", pack_path_str]).await
422+
run_git(repo_path, &["index-pack", pack_path_str]).await?;
423+
enforce_idx_size(&idx_path, pack_digest, max_idx_bytes).await
424+
}
425+
426+
async fn enforce_idx_size(
427+
idx_path: &Path,
428+
pack_digest: &str,
429+
max_idx_bytes: u64,
430+
) -> Result<(), HydrateError> {
431+
let idx_bytes = tokio::fs::metadata(&idx_path)
432+
.await
433+
.map_err(|error| {
434+
HydrateError::Hydrate(format!("stat generated idx {pack_digest}: {error}"))
435+
})?
436+
.len();
437+
if idx_bytes > max_idx_bytes {
438+
return Err(HydrateError::ResourceLimit(format!(
439+
"generated idx {pack_digest} is {idx_bytes} bytes (max {max_idx_bytes})"
440+
)));
441+
}
442+
Ok(())
442443
}
443444

444445
/// Run `git <args>` in `cwd`, fail on non-zero exit.
@@ -499,6 +500,17 @@ mod tests {
499500
assert!(!is_hex_oid(""));
500501
}
501502

503+
#[tokio::test]
504+
async fn generated_idx_size_is_bounded() {
505+
let scratch = TempDir::new().expect("scratch");
506+
let idx_path = scratch.path().join("pack-test.idx");
507+
tokio::fs::write(&idx_path, [0u8; 2]).await.expect("idx");
508+
509+
let result = enforce_idx_size(&idx_path, "test", 1).await;
510+
511+
assert!(matches!(result, Err(HydrateError::ResourceLimit(_))));
512+
}
513+
502514
// `pointer_key` is tested in `super::manifest::tests` — single source.
503515

504516
fn tenant() -> TenantContext {
@@ -520,10 +532,20 @@ mod tests {
520532
packs: Vec::new(),
521533
parent: None,
522534
};
535+
let cache = GitPackCache::new(scratch.path(), u64::MAX, 2).expect("cache");
523536

524-
let hydrated = materialize_manifest(&store, &manifest, scratch.path(), u64::MAX, u64::MAX)
525-
.await
526-
.expect("materialize empty repo");
537+
let hydrated = materialize_manifest(
538+
&store,
539+
&manifest,
540+
HydrationOptions {
541+
pack_cache: &cache,
542+
scratch_dir: scratch.path(),
543+
max_pack_bytes: u64::MAX,
544+
max_repo_bytes: u64::MAX,
545+
},
546+
)
547+
.await
548+
.expect("materialize empty repo");
527549

528550
assert!(hydrated.path().starts_with(scratch.path()));
529551
}
@@ -643,11 +665,22 @@ mod tests {
643665

644666
// Hydrate.
645667
let scratch = TempDir::new().unwrap();
646-
let hydrated =
647-
hydrate_for_read(&st, &ctx, &owner, repo, scratch.path(), u64::MAX, u64::MAX)
648-
.await
649-
.expect("hydrate")
650-
.expect("hydrate Some");
668+
let cache = GitPackCache::new(scratch.path(), u64::MAX, 2).expect("cache");
669+
let hydrated = hydrate_for_read(
670+
&st,
671+
&ctx,
672+
&owner,
673+
repo,
674+
HydrationOptions {
675+
pack_cache: &cache,
676+
scratch_dir: scratch.path(),
677+
max_pack_bytes: u64::MAX,
678+
max_repo_bytes: u64::MAX,
679+
},
680+
)
681+
.await
682+
.expect("hydrate")
683+
.expect("hydrate Some");
651684
eprintln!("hydrated to {}", hydrated.path().display());
652685

653686
// The hydrated repo must list the same ref with the same oid.
@@ -718,14 +751,18 @@ mod tests {
718751
let owner = format!("nope-{}", uuid::Uuid::new_v4());
719752
let ctx = tenant();
720753
let scratch = TempDir::new().unwrap();
754+
let cache = GitPackCache::new(scratch.path(), u64::MAX, 2).expect("cache");
721755
let result = hydrate_for_read(
722756
&st,
723757
&ctx,
724758
&owner,
725759
"ghost",
726-
scratch.path(),
727-
u64::MAX,
728-
u64::MAX,
760+
HydrationOptions {
761+
pack_cache: &cache,
762+
scratch_dir: scratch.path(),
763+
max_pack_bytes: u64::MAX,
764+
max_repo_bytes: u64::MAX,
765+
},
729766
)
730767
.await
731768
.expect("ok");
@@ -774,14 +811,18 @@ mod tests {
774811
}
775812

776813
let scratch = TempDir::new().unwrap();
814+
let cache = GitPackCache::new(scratch.path(), u64::MAX, 2).expect("cache");
777815
let hydrated = hydrate_for_read(
778816
&st,
779817
&ctx,
780818
&owner,
781819
"void",
782-
scratch.path(),
783-
u64::MAX,
784-
u64::MAX,
820+
HydrationOptions {
821+
pack_cache: &cache,
822+
scratch_dir: scratch.path(),
823+
max_pack_bytes: u64::MAX,
824+
max_repo_bytes: u64::MAX,
825+
},
785826
)
786827
.await
787828
.expect("hydrate")

‎crates/buzz-relay/src/api/git/mod.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ pub mod hook;
2727
pub mod hydrate;
2828
pub mod manifest;
2929
pub mod manifest_event;
30+
pub mod pack_cache;
3031
pub mod policy;
3132
pub mod store;
3233
pub mod transport;

0 commit comments

Comments
 (0)