Skip to content

Commit 2318b30

Browse files
tlongwell-blocknpub12gtutshhh76rx0jx697f32f9tffd4hhp3hx58fp4x6u4uemkm7sqf8f757
andauthored
Route Git scratch through configured volume (#1884)
Signed-off-by: npub12gtutshhh76rx0jx697f32f9tffd4hhp3hx58fp4x6u4uemkm7sqf8f757 <5217c5c2f7bfb4333e46d17c98a9255a52dadee18dcd43a43536b95e6776dfa0@sprout-oss.stage.blox.sqprod.co> Co-authored-by: npub12gtutshhh76rx0jx697f32f9tffd4hhp3hx58fp4x6u4uemkm7sqf8f757 <5217c5c2f7bfb4333e46d17c98a9255a52dadee18dcd43a43536b95e6776dfa0@sprout-oss.stage.blox.sqprod.co>
1 parent c772ac0 commit 2318b30

7 files changed

Lines changed: 116 additions & 41 deletions

File tree

‎crates/buzz-relay/src/api/git/cas_publish.rs‎

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -225,15 +225,16 @@ impl ParentState {
225225
/// HEAD or no HEAD yields an empty string.
226226
async fn snapshot_workspace_state(
227227
repo_path: &Path,
228+
scratch_dir: &Path,
228229
) -> Result<(BTreeMap<String, String>, String), CasError> {
229230
const MAX_REF_SNAPSHOT_BYTES: u64 = 4 * 1024 * 1024;
230231

231-
let refs_stdout_tmp = tempfile::NamedTempFile::new()
232+
let refs_stdout_tmp = tempfile::NamedTempFile::new_in(scratch_dir)
232233
.map_err(|e| CasError::PackCapture(format!("for-each-ref stdout tempfile: {e}")))?;
233234
let refs_stdout_file = refs_stdout_tmp
234235
.reopen()
235236
.map_err(|e| CasError::PackCapture(format!("for-each-ref stdout reopen: {e}")))?;
236-
let refs_stderr_tmp = tempfile::NamedTempFile::new()
237+
let refs_stderr_tmp = tempfile::NamedTempFile::new_in(scratch_dir)
237238
.map_err(|e| CasError::PackCapture(format!("for-each-ref stderr tempfile: {e}")))?;
238239
let refs_stderr_file = refs_stderr_tmp
239240
.reopen()
@@ -325,9 +326,11 @@ async fn write_idx_sidecar(
325326
store: &GitStore,
326327
pack_key: &str,
327328
pack_bytes: &[u8],
329+
scratch_dir: &Path,
328330
) -> Result<(), CasError> {
329331
let pack_digest = digest_from_pack_key(pack_key)?;
330-
let tempdir = TempDir::new().map_err(|e| CasError::PackCapture(format!("idx tempdir: {e}")))?;
332+
let tempdir = TempDir::new_in(scratch_dir)
333+
.map_err(|e| CasError::PackCapture(format!("idx tempdir in {scratch_dir:?}: {e}")))?;
331334
let pack_path = tempdir.path().join(format!("pack-{pack_digest}.pack"));
332335
tokio::fs::write(&pack_path, pack_bytes)
333336
.await
@@ -380,6 +383,7 @@ async fn capture_pack(
380383
refs_before: &BTreeMap<String, String>,
381384
refs_after: &BTreeMap<String, String>,
382385
max_pack_bytes: u64,
386+
scratch_dir: &Path,
383387
) -> Result<Option<Vec<u8>>, CasError> {
384388
// Build rev-spec stdin: positive new tips, negative old tips.
385389
// Deduplicate against the same-oid case — no point feeding `X ^X`.
@@ -401,12 +405,12 @@ async fn capture_pack(
401405
stdin_lines.push('\n');
402406
}
403407

404-
let stdout_tmp = tempfile::NamedTempFile::new()
408+
let stdout_tmp = tempfile::NamedTempFile::new_in(scratch_dir)
405409
.map_err(|e| CasError::PackCapture(format!("pack-objects stdout tempfile: {e}")))?;
406410
let stdout_file = stdout_tmp
407411
.reopen()
408412
.map_err(|e| CasError::PackCapture(format!("pack-objects stdout reopen: {e}")))?;
409-
let stderr_tmp = tempfile::NamedTempFile::new()
413+
let stderr_tmp = tempfile::NamedTempFile::new_in(scratch_dir)
410414
.map_err(|e| CasError::PackCapture(format!("pack-objects stderr tempfile: {e}")))?;
411415
let stderr_file = stderr_tmp
412416
.reopen()
@@ -558,10 +562,17 @@ pub async fn cas_publish(
558562
) -> Result<CasSuccess, CasError> {
559563
let pkey = pointer_key(ctx.community(), owner, repo);
560564

565+
// Hydrated repositories are direct children of the configured Git scratch
566+
// root. Reuse that parent for publication tempfiles so they remain on the
567+
// mounted scratch volume without adding another independent path argument.
568+
let scratch_dir = repo_path
569+
.parent()
570+
.ok_or_else(|| CasError::PackCapture("repository path has no scratch parent".into()))?;
571+
561572
// Snapshot post-receive-pack state from disk. `parent_state.parent.refs`
562573
// are the refs the workspace was hydrated from — `pack-objects --revs`
563574
// below uses them as the "negative" set to produce the delta pack.
564-
let (refs_after, head_observed) = snapshot_workspace_state(repo_path).await?;
575+
let (refs_after, head_observed) = snapshot_workspace_state(repo_path, scratch_dir).await?;
565576

566577
// HEAD fallback: a bare repo serving pushes shouldn't have detached
567578
// HEAD, but if `git symbolic-ref` failed (or returned empty), inherit
@@ -583,6 +594,7 @@ pub async fn cas_publish(
583594
&parent_state.parent.refs,
584595
&refs_after,
585596
limits.max_pack_bytes,
597+
scratch_dir,
586598
)
587599
.await?;
588600
let new_pack_key = if let Some(bytes) = pack_bytes {
@@ -599,7 +611,7 @@ pub async fn cas_publish(
599611
}
600612
debug!(bytes = bytes.len(), "captured push pack");
601613
let pack_key = store.put_pack(&bytes).await?;
602-
if let Err(e) = write_idx_sidecar(store, &pack_key, &bytes).await {
614+
if let Err(e) = write_idx_sidecar(store, &pack_key, &bytes, scratch_dir).await {
603615
warn!(
604616
pack_key = %pack_key,
605617
error = %e,

‎crates/buzz-relay/src/api/git/hydrate.rs‎

Lines changed: 74 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -105,14 +105,22 @@ pub async fn hydrate_for_read(
105105
ctx: &TenantContext,
106106
owner: &str,
107107
repo: &str,
108+
scratch_dir: &Path,
108109
max_pack_bytes: u64,
109110
max_repo_bytes: u64,
110111
) -> Result<Option<HydratedRepo>, HydrateError> {
111112
let Some((_etag, _digest, manifest)) = load_pointer(store, ctx, owner, repo).await? else {
112113
return Ok(None);
113114
};
114115
Ok(Some(
115-
materialize_manifest(store, &manifest, max_pack_bytes, max_repo_bytes).await?,
116+
materialize_manifest(
117+
store,
118+
&manifest,
119+
scratch_dir,
120+
max_pack_bytes,
121+
max_repo_bytes,
122+
)
123+
.await?,
116124
))
117125
}
118126

@@ -158,22 +166,29 @@ pub async fn hydrate_for_write(
158166
ctx: &TenantContext,
159167
owner: &str,
160168
repo: &str,
169+
scratch_dir: &Path,
161170
max_pack_bytes: u64,
162171
max_repo_bytes: u64,
163172
) -> Result<(HydratedRepo, ParentState), HydrateError> {
164173
match load_pointer(store, ctx, owner, repo).await? {
165174
Some((etag, digest, manifest)) => {
166-
let repo =
167-
materialize_manifest(store, &manifest, max_pack_bytes, max_repo_bytes).await?;
175+
let repo = materialize_manifest(
176+
store,
177+
&manifest,
178+
scratch_dir,
179+
max_pack_bytes,
180+
max_repo_bytes,
181+
)
182+
.await?;
168183
let parent = ParentState::from_loaded(etag, digest, manifest);
169184
Ok((repo, parent))
170185
}
171186
None => {
172187
// First push: empty bare repo. No packs to fetch, no refs to
173188
// install. `receive-pack` will accept whatever the client
174189
// sends; `cas_publish` will use `If-None-Match: *`.
175-
let tempdir =
176-
TempDir::new().map_err(|e| HydrateError::Hydrate(format!("tempdir: {e}")))?;
190+
let tempdir = TempDir::new_in(scratch_dir)
191+
.map_err(|e| HydrateError::Hydrate(format!("tempdir in {scratch_dir:?}: {e}")))?;
177192
let path = tempdir.path().to_path_buf();
178193
run_git(&path, &["init", "--bare", "--quiet"]).await?;
179194
Ok((
@@ -242,11 +257,13 @@ async fn get_verified_limited(
242257
async fn materialize_manifest(
243258
store: &GitStore,
244259
manifest: &Manifest,
260+
scratch_dir: &Path,
245261
max_pack_bytes: u64,
246262
max_repo_bytes: u64,
247263
) -> Result<HydratedRepo, HydrateError> {
248264
// Init bare repo.
249-
let tempdir = TempDir::new().map_err(|e| HydrateError::Hydrate(format!("tempdir: {e}")))?;
265+
let tempdir = TempDir::new_in(scratch_dir)
266+
.map_err(|e| HydrateError::Hydrate(format!("tempdir in {scratch_dir:?}: {e}")))?;
250267
let path = tempdir.path().to_path_buf();
251268
run_git(&path, &["init", "--bare", "--quiet"]).await?;
252269

@@ -443,6 +460,26 @@ mod tests {
443460
)
444461
}
445462

463+
#[tokio::test]
464+
async fn materialized_repo_is_created_under_configured_scratch_dir() {
465+
let scratch = TempDir::new().unwrap();
466+
let store = GitStore::new("http://localhost:9000", "x", "x", "x", "us-east-1")
467+
.expect("construct store");
468+
let manifest = Manifest {
469+
version: 1,
470+
head: "refs/heads/main".into(),
471+
refs: BTreeMap::new(),
472+
packs: Vec::new(),
473+
parent: None,
474+
};
475+
476+
let hydrated = materialize_manifest(&store, &manifest, scratch.path(), u64::MAX, u64::MAX)
477+
.await
478+
.expect("materialize empty repo");
479+
480+
assert!(hydrated.path().starts_with(scratch.path()));
481+
}
482+
446483
// -------- Live MinIO + real git roundtrip ----------------------------------
447484
//
448485
// Run manually:
@@ -557,10 +594,12 @@ mod tests {
557594
}
558595

559596
// Hydrate.
560-
let hydrated = hydrate_for_read(&st, &ctx, &owner, repo, u64::MAX, u64::MAX)
561-
.await
562-
.expect("hydrate")
563-
.expect("hydrate Some");
597+
let scratch = TempDir::new().unwrap();
598+
let hydrated =
599+
hydrate_for_read(&st, &ctx, &owner, repo, scratch.path(), u64::MAX, u64::MAX)
600+
.await
601+
.expect("hydrate")
602+
.expect("hydrate Some");
564603
eprintln!("hydrated to {}", hydrated.path().display());
565604

566605
// The hydrated repo must list the same ref with the same oid.
@@ -630,9 +669,18 @@ mod tests {
630669
let st = store();
631670
let owner = format!("nope-{}", uuid::Uuid::new_v4());
632671
let ctx = tenant();
633-
let result = hydrate_for_read(&st, &ctx, &owner, "ghost", u64::MAX, u64::MAX)
634-
.await
635-
.expect("ok");
672+
let scratch = TempDir::new().unwrap();
673+
let result = hydrate_for_read(
674+
&st,
675+
&ctx,
676+
&owner,
677+
"ghost",
678+
scratch.path(),
679+
u64::MAX,
680+
u64::MAX,
681+
)
682+
.await
683+
.expect("ok");
636684
assert!(result.is_none(), "missing pointer must surface as None");
637685
}
638686

@@ -677,10 +725,19 @@ mod tests {
677725
super::super::store::CasOutcome::LostRace => panic!("first INM* must win"),
678726
}
679727

680-
let hydrated = hydrate_for_read(&st, &ctx, &owner, "void", u64::MAX, u64::MAX)
681-
.await
682-
.expect("hydrate")
683-
.expect("hydrate Some");
728+
let scratch = TempDir::new().unwrap();
729+
let hydrated = hydrate_for_read(
730+
&st,
731+
&ctx,
732+
&owner,
733+
"void",
734+
scratch.path(),
735+
u64::MAX,
736+
u64::MAX,
737+
)
738+
.await
739+
.expect("hydrate")
740+
.expect("hydrate Some");
684741

685742
// HEAD points where the manifest said.
686743
let head_file = tokio::fs::read_to_string(hydrated.path().join("HEAD"))

‎crates/buzz-relay/src/api/git/transport.rs‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -581,6 +581,7 @@ async fn info_refs_subprocess(
581581
tenant,
582582
&params.owner,
583583
&params.repo,
584+
&state.config.git_repo_path,
584585
state.config.git_max_pack_bytes,
585586
state.config.git_max_repo_bytes,
586587
)
@@ -596,11 +597,11 @@ async fn info_refs_subprocess(
596597
// "receive-pack" (without the "git-" prefix).
597598
let git_subcmd = service.strip_prefix("git-").unwrap_or(service);
598599

599-
let stdout_tmp = tempfile::NamedTempFile::new().map_err(|e| {
600+
let stdout_tmp = tempfile::NamedTempFile::new_in(&state.config.git_repo_path).map_err(|e| {
600601
error!(error = %e, "git info_refs stdout tempfile failed");
601602
(StatusCode::INTERNAL_SERVER_ERROR, "git error").into_response()
602603
})?;
603-
let stderr_tmp = tempfile::NamedTempFile::new().map_err(|e| {
604+
let stderr_tmp = tempfile::NamedTempFile::new_in(&state.config.git_repo_path).map_err(|e| {
604605
error!(error = %e, "git info_refs stderr tempfile failed");
605606
(StatusCode::INTERNAL_SERVER_ERROR, "git error").into_response()
606607
})?;
@@ -712,6 +713,7 @@ pub async fn upload_pack(
712713
&auth.tenant,
713714
&params.owner,
714715
&params.repo,
716+
&state.config.git_repo_path,
715717
state.config.git_max_pack_bytes,
716718
state.config.git_max_repo_bytes,
717719
)
@@ -799,6 +801,7 @@ pub async fn receive_pack(
799801
&auth.tenant,
800802
&params.owner,
801803
&params.repo,
804+
&state.config.git_repo_path,
802805
state.config.git_max_pack_bytes,
803806
state.config.git_max_repo_bytes,
804807
)
@@ -848,6 +851,7 @@ pub async fn receive_pack(
848851
"receive-pack",
849852
body,
850853
&hook_env,
854+
&state.config.git_repo_path,
851855
RECEIVE_PACK_MAX_OUTPUT_BYTES,
852856
)
853857
.await?;
@@ -900,13 +904,14 @@ async fn run_git_at(
900904
service: &str,
901905
body: Body,
902906
extra_env: &[(&str, String)],
907+
scratch_dir: &Path,
903908
max_output_bytes: u64,
904909
) -> Result<PackOutput, Response> {
905-
let stdout_tmp = tempfile::NamedTempFile::new().map_err(|e| {
910+
let stdout_tmp = tempfile::NamedTempFile::new_in(scratch_dir).map_err(|e| {
906911
error!(error = %e, service = %service, "git stdout tempfile failed");
907912
(StatusCode::INTERNAL_SERVER_ERROR, "git error").into_response()
908913
})?;
909-
let stderr_tmp = tempfile::NamedTempFile::new().map_err(|e| {
914+
let stderr_tmp = tempfile::NamedTempFile::new_in(scratch_dir).map_err(|e| {
910915
error!(error = %e, service = %service, "git stderr tempfile failed");
911916
(StatusCode::INTERNAL_SERVER_ERROR, "git error").into_response()
912917
})?;

‎crates/buzz-relay/src/config.rs‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -169,12 +169,10 @@ pub struct Config {
169169

170170
/// Root directory for the relay's local git scratch. No per-repo bare repos
171171
/// or persistent git state live here — runtime reads/writes hydrate
172-
/// ephemeral repos from object storage per request, and repo-name
173-
/// uniqueness now lives in Postgres (`git_repo_names`), not on disk. Retained
174-
/// for ephemeral working space and env compatibility; the relay no longer
175-
/// depends on this path being persistent or shared across replicas, so it
176-
/// needs no ReadWriteMany volume. (Removing the field entirely is a
177-
/// follow-up cleanup once the deploy chart drops the git PVC mount.)
172+
/// ephemeral repos from object storage per request, and all temporary Git
173+
/// workspaces and buffered subprocess output are created beneath this path.
174+
/// Repo-name uniqueness lives in Postgres (`git_repo_names`), not on disk,
175+
/// so this directory need not be persistent or shared across replicas.
178176
pub git_repo_path: std::path::PathBuf,
179177
/// Maximum pack file size for git push (bytes). Default: 500 MB.
180178
pub git_max_pack_bytes: u64,

‎deploy/charts/buzz/templates/deployment.yaml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -219,5 +219,6 @@ spec:
219219
persistentVolumeClaim:
220220
claimName: {{ default (printf "%s-git" (include "buzz.fullname" .)) .Values.persistence.git.existingClaim }}
221221
{{- else }}
222-
emptyDir: {}
222+
emptyDir:
223+
sizeLimit: {{ .Values.persistence.git.size | quote }}
223224
{{- end }}

‎deploy/charts/buzz/tests/render_test.yaml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,8 @@ tests:
130130
path: spec.template.spec.volumes
131131
content:
132132
name: git-repos
133-
emptyDir: {}
133+
emptyDir:
134+
sizeLimit: 10Gi
134135
template: templates/deployment.yaml
135136
# No PVC is rendered at all when persistence is disabled.
136137
- hasDocuments:

‎deploy/charts/buzz/values.yaml‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -214,17 +214,18 @@ httproute:
214214
# enabled: true → mount a PVC at mountPath (durable across pod restarts, but a
215215
# single ReadWriteOnce PVC binds to one node, so it does NOT support multi-pod
216216
# scheduling across nodes on a Deployment).
217-
# enabled: false → mount a per-pod emptyDir at mountPath (pure scratch). This is
218-
# the correct choice for multi-replica HA: each pod gets its own local working
219-
# space, nothing is shared, and there is no volume to multi-attach. Safe
220-
# because the object store + Postgres are the sources of truth, not this disk.
217+
# enabled: false → mount a per-pod emptyDir at mountPath (pure scratch), bounded
218+
# by size. This is the correct choice for multi-replica HA: each pod gets its
219+
# own local working space, nothing is shared, and there is no volume to
220+
# multi-attach. Safe because the object store + Postgres are the sources of
221+
# truth, not this disk.
221222
persistence:
222223
git:
223224
enabled: true
224225
mountPath: /var/lib/buzz/git
225226
storageClass: ""
226227
accessMode: ReadWriteOnce
227-
size: 10Gi
228+
size: 10Gi # PVC capacity or emptyDir sizeLimit
228229
annotations: {}
229230
existingClaim: ""
230231

0 commit comments

Comments
 (0)