Skip to content

Commit 5f7c93d

Browse files
authored
fix(relay): bound and observe Git read operations (#2167)
1 parent 21dbd32 commit 5f7c93d

3 files changed

Lines changed: 237 additions & 9 deletions

File tree

‎crates/buzz-relay/src/api/git/hydrate.rs‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,8 @@ pub struct HydratedRepo {
5454
path: PathBuf,
5555
/// Total bytes of parent packs materialized into this workspace.
5656
hydrated_bytes: u64,
57+
/// Number of parent packs materialized into this workspace.
58+
hydrated_packs: usize,
5759
}
5860

5961
impl HydratedRepo {
@@ -66,6 +68,11 @@ impl HydratedRepo {
6668
pub fn hydrated_bytes(&self) -> u64 {
6769
self.hydrated_bytes
6870
}
71+
72+
/// Number of parent packs materialized into this workspace.
73+
pub fn hydrated_packs(&self) -> usize {
74+
self.hydrated_packs
75+
}
6976
}
7077

7178
/// Hydration errors.
@@ -108,6 +115,45 @@ pub async fn hydrate_for_read(
108115
scratch_dir: &Path,
109116
max_pack_bytes: u64,
110117
max_repo_bytes: u64,
118+
) -> Result<Option<HydratedRepo>, HydrateError> {
119+
let started_at = std::time::Instant::now();
120+
let result = hydrate_for_read_inner(
121+
store,
122+
ctx,
123+
owner,
124+
repo,
125+
scratch_dir,
126+
max_pack_bytes,
127+
max_repo_bytes,
128+
)
129+
.await;
130+
let outcome = match &result {
131+
Ok(Some(repo)) => {
132+
metrics::histogram!("buzz_git_hydrate_bytes").record(repo.hydrated_bytes() as f64);
133+
metrics::histogram!("buzz_git_hydrate_packs").record(repo.hydrated_packs() as f64);
134+
"success"
135+
}
136+
Ok(None) => "missing",
137+
Err(HydrateError::InvalidPointer) => "invalid_pointer",
138+
Err(HydrateError::Manifest(_)) => "manifest_error",
139+
Err(HydrateError::Store(_)) => "store_error",
140+
Err(HydrateError::Hydrate(_)) => "hydrate_error",
141+
Err(HydrateError::ResourceLimit(_)) => "resource_limit",
142+
};
143+
metrics::counter!("buzz_git_hydrations_total", "outcome" => outcome).increment(1);
144+
metrics::histogram!("buzz_git_hydrate_seconds", "outcome" => outcome)
145+
.record(started_at.elapsed().as_secs_f64());
146+
result
147+
}
148+
149+
async fn hydrate_for_read_inner(
150+
store: &GitStore,
151+
ctx: &TenantContext,
152+
owner: &str,
153+
repo: &str,
154+
scratch_dir: &Path,
155+
max_pack_bytes: u64,
156+
max_repo_bytes: u64,
111157
) -> Result<Option<HydratedRepo>, HydrateError> {
112158
let Some((_etag, _digest, manifest)) = load_pointer(store, ctx, owner, repo).await? else {
113159
return Ok(None);
@@ -196,6 +242,7 @@ pub async fn hydrate_for_write(
196242
_tempdir: tempdir,
197243
path,
198244
hydrated_bytes: 0,
245+
hydrated_packs: 0,
199246
},
200247
ParentState::fresh(),
201248
))
@@ -342,6 +389,7 @@ async fn materialize_manifest(
342389
_tempdir: tempdir,
343390
path,
344391
hydrated_bytes,
392+
hydrated_packs: manifest.packs.len(),
345393
})
346394
}
347395

‎crates/buzz-relay/src/api/git/transport.rs‎

Lines changed: 143 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,10 @@
88
//! Auth: NIP-98 on all routes (clone + push). No public repos for v1.
99
//! Transport: shells out to `git --stateless-rpc` with `env_clear()`.
1010
11+
use std::future::Future;
1112
use std::path::Path;
1213
use std::sync::Arc;
14+
use std::time::{Duration, Instant};
1315

1416
use axum::{
1517
body::Body,
@@ -307,10 +309,16 @@ pub(crate) fn harden_git_env(cmd: &mut Command) {
307309
#[allow(clippy::result_large_err)]
308310
fn acquire_git_permit(
309311
state: &Arc<AppState>,
312+
operation: &'static str,
310313
) -> Result<tokio::sync::OwnedSemaphorePermit, Response> {
311314
Arc::clone(&state.git_semaphore)
312315
.try_acquire_owned()
313316
.map_err(|_| {
317+
metrics::counter!(
318+
"buzz_git_semaphore_rejections_total",
319+
"operation" => operation
320+
)
321+
.increment(1);
314322
Response::builder()
315323
.status(StatusCode::SERVICE_UNAVAILABLE)
316324
.header("Retry-After", "5")
@@ -583,7 +591,7 @@ async fn info_refs_subprocess(
583591
service: &str,
584592
params: &GitRepoParams,
585593
) -> Result<Response, Response> {
586-
let _permit = acquire_git_permit(state)?;
594+
let _permit = acquire_git_permit(state, "info_refs")?;
587595

588596
let repo = match hydrate_for_read(
589597
&state.git_store,
@@ -715,7 +723,7 @@ pub async fn upload_pack(
715723
body: Body,
716724
) -> Result<Response, Response> {
717725
let _ = validate_repo_id(&params.owner, &params.repo)?;
718-
let permit = acquire_git_permit(&state)?;
726+
let permit = acquire_git_permit(&state, "upload_pack")?;
719727

720728
let repo = match hydrate_for_read(
721729
&state.git_store,
@@ -783,7 +791,7 @@ pub async fn receive_pack(
783791
) -> Result<Response, Response> {
784792
let repo_name = validate_repo_id(&params.owner, &params.repo)?;
785793
let pusher_hex = hex::encode(auth.pubkey.to_bytes());
786-
let _permit = acquire_git_permit(&state)?;
794+
let _permit = acquire_git_permit(&state, "receive_pack")?;
787795

788796
// **No per-repo advisory lock — by design.** Writer serialization is
789797
// the pointer CAS at `cas_publish` step 7 (`Inv_NoFork` proves this
@@ -1167,15 +1175,31 @@ impl<'a> Iterator for PktLineIter<'a> {
11671175
/// `receive_pack`, which must buffer into [`PackOutput`] so [`finalize_push`]
11681176
/// can sequence the pointer CAS *before* any 2xx byte exists.
11691177
struct StreamingGit {
1170-
inner: tokio_util::io::ReaderStream<tokio::process::ChildStdout>,
1178+
inner: TimedByteStream<tokio_util::io::ReaderStream<tokio::process::ChildStdout>>,
11711179
/// Held purely to extend lifetime. `kill_on_drop(true)` means dropping
11721180
/// this after the stream completes reaps any lingering process; on the
11731181
/// happy path the child has already exited by then.
1174-
_child: tokio::process::Child,
1182+
child: tokio::process::Child,
11751183
/// The ephemeral bare repo the subprocess reads objects from. Must not be
11761184
/// removed from disk until the subprocess is done — i.e. until the stream
11771185
/// ends.
11781186
_repo: HydratedRepo,
1187+
/// Pumping the request body is detached from response polling. Abort it
1188+
/// when the response is dropped or the subprocess times out.
1189+
stdin_task: tokio::task::JoinHandle<()>,
1190+
}
1191+
1192+
/// Adds a hard deadline and lifecycle metrics to upload-pack stdout.
1193+
///
1194+
/// The response status is already committed when this stream is polled, so a
1195+
/// timeout is surfaced as an in-band body error. [`StreamingGit`] observes
1196+
/// that error and kills the subprocess.
1197+
struct TimedByteStream<S> {
1198+
inner: std::pin::Pin<Box<S>>,
1199+
deadline: std::pin::Pin<Box<tokio::time::Sleep>>,
1200+
started_at: Instant,
1201+
streamed_bytes: u64,
1202+
finished: bool,
11791203
}
11801204

11811205
/// Keeps a git concurrency permit alive for the lifetime of a response-body
@@ -1207,7 +1231,82 @@ impl futures_util::Stream for StreamingGit {
12071231
mut self: std::pin::Pin<&mut Self>,
12081232
cx: &mut std::task::Context<'_>,
12091233
) -> std::task::Poll<Option<Self::Item>> {
1210-
std::pin::Pin::new(&mut self.inner).poll_next(cx)
1234+
let poll = std::pin::Pin::new(&mut self.inner).poll_next(cx);
1235+
if matches!(
1236+
&poll,
1237+
std::task::Poll::Ready(Some(Err(error)))
1238+
if error.kind() == std::io::ErrorKind::TimedOut
1239+
) {
1240+
self.stdin_task.abort();
1241+
if let Err(error) = self.child.start_kill() {
1242+
warn!(error = %error, "timed-out git upload-pack could not be killed");
1243+
}
1244+
}
1245+
poll
1246+
}
1247+
}
1248+
1249+
impl<S> TimedByteStream<S> {
1250+
fn new(inner: S, timeout: Duration) -> Self {
1251+
Self {
1252+
inner: Box::pin(inner),
1253+
deadline: Box::pin(tokio::time::sleep(timeout)),
1254+
started_at: Instant::now(),
1255+
streamed_bytes: 0,
1256+
finished: false,
1257+
}
1258+
}
1259+
}
1260+
1261+
impl<S> futures_util::Stream for TimedByteStream<S>
1262+
where
1263+
S: futures_util::Stream<Item = Result<bytes::Bytes, std::io::Error>>,
1264+
{
1265+
type Item = Result<bytes::Bytes, std::io::Error>;
1266+
1267+
fn poll_next(
1268+
mut self: std::pin::Pin<&mut Self>,
1269+
cx: &mut std::task::Context<'_>,
1270+
) -> std::task::Poll<Option<Self::Item>> {
1271+
if self.finished {
1272+
return std::task::Poll::Ready(None);
1273+
}
1274+
if self.deadline.as_mut().poll(cx).is_ready() {
1275+
self.finished = true;
1276+
metrics::counter!("buzz_git_upload_pack_timeouts_total").increment(1);
1277+
warn!("git upload-pack stream timed out");
1278+
return std::task::Poll::Ready(Some(Err(std::io::Error::new(
1279+
std::io::ErrorKind::TimedOut,
1280+
"git upload-pack timed out",
1281+
))));
1282+
}
1283+
match self.inner.as_mut().poll_next(cx) {
1284+
std::task::Poll::Ready(Some(Ok(bytes))) => {
1285+
self.streamed_bytes = self
1286+
.streamed_bytes
1287+
.saturating_add(u64::try_from(bytes.len()).unwrap_or(u64::MAX));
1288+
std::task::Poll::Ready(Some(Ok(bytes)))
1289+
}
1290+
std::task::Poll::Ready(None) => {
1291+
self.finished = true;
1292+
std::task::Poll::Ready(None)
1293+
}
1294+
other => other,
1295+
}
1296+
}
1297+
}
1298+
1299+
impl<S> Drop for TimedByteStream<S> {
1300+
fn drop(&mut self) {
1301+
metrics::histogram!("buzz_git_upload_pack_stream_seconds")
1302+
.record(self.started_at.elapsed().as_secs_f64());
1303+
metrics::histogram!("buzz_git_upload_pack_stream_bytes").record(self.streamed_bytes as f64);
1304+
}
1305+
}
1306+
1307+
impl Drop for StreamingGit {
1308+
fn drop(&mut self) {
1309+
self.stdin_task.abort();
12111310
}
12121311
}
12131312

@@ -1256,7 +1355,7 @@ fn stream_git_read(
12561355
// Pump the request body into git's stdin, then close it (EOF). Detached:
12571356
// the task ends on its own when the body ends or the write fails.
12581357
let mut stdin = child.stdin.take().expect("stdin piped");
1259-
tokio::spawn(async move {
1358+
let stdin_task = tokio::spawn(async move {
12601359
use futures_util::StreamExt;
12611360
let mut stream = body.into_data_stream();
12621361
while let Some(chunk) = stream.next().await {
@@ -1277,9 +1376,10 @@ fn stream_git_read(
12771376

12781377
let stdout = child.stdout.take().expect("stdout piped");
12791378
let git_stream = StreamingGit {
1280-
inner: tokio_util::io::ReaderStream::new(stdout),
1281-
_child: child,
1379+
inner: TimedByteStream::new(tokio_util::io::ReaderStream::new(stdout), PACK_OPS_TIMEOUT),
1380+
child,
12821381
_repo: repo,
1382+
stdin_task,
12831383
};
12841384

12851385
// Prepend any protocol header (info/refs) ahead of git's stdout. The
@@ -1638,6 +1738,40 @@ mod track_c_tests {
16381738
assert_eq!(semaphore.available_permits(), 1);
16391739
}
16401740

1741+
#[tokio::test]
1742+
async fn upload_pack_stream_returns_timeout_error_at_deadline() {
1743+
use futures_util::StreamExt;
1744+
1745+
let mut stream = TimedByteStream::new(
1746+
futures_util::stream::pending::<Result<bytes::Bytes, std::io::Error>>(),
1747+
Duration::from_millis(10),
1748+
);
1749+
1750+
let error = tokio::time::timeout(Duration::from_secs(1), stream.next())
1751+
.await
1752+
.expect("stream deadline")
1753+
.expect("timeout item")
1754+
.expect_err("timeout error");
1755+
assert_eq!(error.kind(), std::io::ErrorKind::TimedOut);
1756+
assert!(stream.next().await.is_none());
1757+
}
1758+
1759+
#[tokio::test]
1760+
async fn upload_pack_stream_counts_response_bytes() {
1761+
use futures_util::StreamExt;
1762+
1763+
let chunks = vec![
1764+
Ok(bytes::Bytes::from_static(b"abc")),
1765+
Ok(bytes::Bytes::from_static(b"de")),
1766+
];
1767+
let mut stream =
1768+
TimedByteStream::new(futures_util::stream::iter(chunks), Duration::from_secs(1));
1769+
1770+
while stream.next().await.is_some() {}
1771+
1772+
assert_eq!(stream.streamed_bytes, 5);
1773+
}
1774+
16411775
/// Wrap inner status pkt-lines in one side-band-64k band-1 (data) outer
16421776
/// pkt-line — the real shape git emits when side-band is negotiated:
16431777
/// `<outer-len>\x01<inner-pkt-lines>`.

‎crates/buzz-relay/src/metrics.rs‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,27 @@ const LATENCY_BUCKETS_MS: [f64; 11] = [
3232
/// Seconds-scale buckets for internal processing histograms (event, search, audit).
3333
const DURATION_BUCKETS_S: [f64; 10] = [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 5.0];
3434

35+
/// Seconds-scale buckets for Git hydration and pack streams.
36+
const GIT_DURATION_BUCKETS_S: [f64; 13] = [
37+
0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0,
38+
];
39+
40+
/// Byte buckets for hydrated repositories and streamed clone/fetch responses.
41+
const GIT_BYTES_BUCKETS: [f64; 9] = [
42+
0.0,
43+
64.0 * 1024.0,
44+
1024.0 * 1024.0,
45+
10.0 * 1024.0 * 1024.0,
46+
50.0 * 1024.0 * 1024.0,
47+
100.0 * 1024.0 * 1024.0,
48+
250.0 * 1024.0 * 1024.0,
49+
500.0 * 1024.0 * 1024.0,
50+
1024.0 * 1024.0 * 1024.0,
51+
];
52+
53+
/// Pack-count buckets bounded by the manifest's maximum pack count.
54+
const GIT_PACK_BUCKETS: [f64; 9] = [0.0, 1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0];
55+
3556
/// Integer-count buckets for fan-out recipient histograms.
3657
const FANOUT_BUCKETS: [f64; 9] = [0.0, 1.0, 5.0, 10.0, 25.0, 50.0, 100.0, 500.0, 1000.0];
3758

@@ -56,6 +77,31 @@ pub fn install(port: u16, gauge_idle_timeout_secs: u64) {
5677
&LATENCY_BUCKETS_MS,
5778
)
5879
.expect("valid ms bucket boundaries")
80+
.set_buckets_for_metric(
81+
Matcher::Full("buzz_git_hydrate_seconds".to_owned()),
82+
&GIT_DURATION_BUCKETS_S,
83+
)
84+
.expect("valid git hydration duration bucket boundaries")
85+
.set_buckets_for_metric(
86+
Matcher::Full("buzz_git_upload_pack_stream_seconds".to_owned()),
87+
&GIT_DURATION_BUCKETS_S,
88+
)
89+
.expect("valid git stream duration bucket boundaries")
90+
.set_buckets_for_metric(
91+
Matcher::Full("buzz_git_hydrate_bytes".to_owned()),
92+
&GIT_BYTES_BUCKETS,
93+
)
94+
.expect("valid git hydration byte bucket boundaries")
95+
.set_buckets_for_metric(
96+
Matcher::Full("buzz_git_upload_pack_stream_bytes".to_owned()),
97+
&GIT_BYTES_BUCKETS,
98+
)
99+
.expect("valid git stream byte bucket boundaries")
100+
.set_buckets_for_metric(
101+
Matcher::Full("buzz_git_hydrate_packs".to_owned()),
102+
&GIT_PACK_BUCKETS,
103+
)
104+
.expect("valid git pack-count bucket boundaries")
59105
.set_buckets_for_metric(Matcher::Suffix("_seconds".to_owned()), &DURATION_BUCKETS_S)
60106
.expect("valid seconds bucket boundaries")
61107
.set_buckets_for_metric(

0 commit comments

Comments
 (0)