Skip to content

Commit 5ca36e7

Browse files
iamkaaldhairyaclaudenpub1qyvc0c5kl4gqv2fd97fsk46tu378sqgy35vc83rvgfwne90sel7s0ed67dTyler Longwell
authored
fix(relay): decompress gzip-encoded git smart-HTTP request bodies (#2670)
Signed-off-by: Kaal <[email protected]> Signed-off-by: Tyler Longwell <[email protected]> Co-authored-by: Claude Opus 4.8 <[email protected]> Co-authored-by: npub1qyvc0c5kl4gqv2fd97fsk46tu378sqgy35vc83rvgfwne90sel7s0ed67d <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz> Co-authored-by: Tyler Longwell <[email protected]>
1 parent 76aeae7 commit 5ca36e7

3 files changed

Lines changed: 198 additions & 2 deletions

File tree

‎Cargo.lock‎

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎crates/buzz-relay/Cargo.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ metrics = { workspace = true }
7878
metrics-exporter-prometheus = { workspace = true }
7979
metrics-util = { workspace = true }
8080
pulldown-cmark = { version = "0.13.4", default-features = false, features = ["html"] }
81+
async-compression = { version = "0.4.42", features = ["tokio", "gzip"] }
8182

8283
[features]
8384
dev = ["buzz-auth/dev"]
@@ -90,3 +91,4 @@ buzz-auth = { workspace = true, features = ["dev"] }
9091
reqwest = { workspace = true }
9192
tokio-tungstenite = { workspace = true }
9293
futures = "0.3"
94+
flate2 = "1.1.9"

‎crates/buzz-relay/src/api/git/transport.rs‎

Lines changed: 194 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,13 @@ const PACK_OPS_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(300
5050
const RECEIVE_PACK_MAX_OUTPUT_BYTES: u64 = 1024 * 1024;
5151
/// Maximum ref advertisement output after manifest ref-count validation.
5252
const INFO_REFS_MAX_OUTPUT_BYTES: u64 = 4 * 1024 * 1024;
53+
/// Maximum *decoded* upload-pack request body (want/have negotiation).
54+
///
55+
/// Bounds the output of [`decode_git_request_body`] so a small gzip bomb
56+
/// cannot bypass the compressed-body `RequestBodyLimitLayer`. Negotiation
57+
/// bodies are pkt-lines of wants/haves (~50 bytes per ref); even a repo
58+
/// with a million refs stays far under this.
59+
const UPLOAD_PACK_MAX_DECODED_BYTES: u64 = 64 * 1024 * 1024;
5360

5461
/// NIP-98 auth extractor for git routes.
5562
///
@@ -714,6 +721,62 @@ async fn info_refs_subprocess(
714721
.unwrap())
715722
}
716723

724+
/// Decode a smart-HTTP git request body according to its `Content-Encoding`.
725+
///
726+
/// Git's smart-HTTP client gzip-compresses the `git-upload-pack` /
727+
/// `git-receive-pack` request body once it exceeds an internal size
728+
/// threshold (`http.postBuffer`-independent; triggered by the number of
729+
/// want/have lines, so it fires reliably on many-ref clones). The relay
730+
/// pipes the request body straight into the git subprocess's stdin, so a
731+
/// still-compressed body reaches `git upload-pack` as raw gzip and fails
732+
/// with `fatal: protocol error: bad line length character`. Transparently
733+
/// inflate here so the subprocess always sees plain pkt-lines.
734+
///
735+
/// Only `gzip` is decoded (the sole encoding git emits). An unknown
736+
/// non-identity encoding is passed through unchanged rather than rejected;
737+
/// the subprocess surfaces any real mismatch as an in-band protocol error.
738+
///
739+
/// `max_decoded_bytes` bounds the *inflated* size: the router's
740+
/// `RequestBodyLimitLayer` only caps compressed bytes, so without this a
741+
/// small gzip bomb (ratios up to ~1000:1) could feed an effectively
742+
/// unbounded stream to the subprocess — for receive-pack that means
743+
/// unbounded scratch-disk writes. Exceeding the cap errors the stream,
744+
/// which the stdin pumps surface as a logged early-EOF to git.
745+
fn decode_git_request_body(
746+
headers: &axum::http::HeaderMap,
747+
body: Body,
748+
max_decoded_bytes: u64,
749+
) -> Body {
750+
let is_gzip = headers
751+
.get(header::CONTENT_ENCODING)
752+
.and_then(|v| v.to_str().ok())
753+
.map(|v| v.eq_ignore_ascii_case("gzip") || v.eq_ignore_ascii_case("x-gzip"))
754+
.unwrap_or(false);
755+
if !is_gzip {
756+
return body;
757+
}
758+
use futures_util::StreamExt;
759+
use tokio_util::io::{ReaderStream, StreamReader};
760+
let byte_stream = body
761+
.into_data_stream()
762+
.map(|res| res.map_err(std::io::Error::other))
763+
.boxed();
764+
let decoder =
765+
async_compression::tokio::bufread::GzipDecoder::new(StreamReader::new(byte_stream));
766+
let mut decoded: u64 = 0;
767+
let capped = ReaderStream::new(decoder).map(move |chunk| {
768+
let chunk = chunk?;
769+
decoded = decoded.saturating_add(chunk.len() as u64);
770+
if decoded > max_decoded_bytes {
771+
return Err(std::io::Error::other(format!(
772+
"gzip git request body exceeded {max_decoded_bytes} decoded bytes"
773+
)));
774+
}
775+
Ok(chunk)
776+
});
777+
Body::from_stream(capped)
778+
}
779+
717780
/// `POST /git/{owner}/{repo}/git-upload-pack`
718781
///
719782
/// Handles clone/fetch — client sends wants/haves, server sends pack data.
@@ -723,10 +786,12 @@ async fn info_refs_subprocess(
723786
pub async fn upload_pack(
724787
State(state): State<Arc<AppState>>,
725788
auth: GitAuth,
789+
headers: axum::http::HeaderMap,
726790
AxumPath(params): AxumPath<GitRepoParams>,
727791
body: Body,
728792
) -> Result<Response, Response> {
729793
let _ = validate_repo_id(&params.owner, &params.repo)?;
794+
let body = decode_git_request_body(&headers, body, UPLOAD_PACK_MAX_DECODED_BYTES);
730795
let permit = acquire_git_permit(&state, "upload_pack")?;
731796

732797
let repo = match hydrate_for_read(
@@ -793,10 +858,12 @@ pub async fn upload_pack(
793858
pub async fn receive_pack(
794859
State(state): State<Arc<AppState>>,
795860
auth: GitAuth,
861+
headers: axum::http::HeaderMap,
796862
AxumPath(params): AxumPath<GitRepoParams>,
797863
body: Body,
798864
) -> Result<Response, Response> {
799865
let repo_name = validate_repo_id(&params.owner, &params.repo)?;
866+
let body = decode_git_request_body(&headers, body, state.config.git_max_pack_bytes);
800867
let pusher_hex = hex::encode(auth.pubkey.to_bytes());
801868
let _permit = acquire_git_permit(&state, "receive_pack")?;
802869

@@ -968,6 +1035,7 @@ async fn run_git_at(
9681035

9691036
// Stream request body to git stdin.
9701037
let mut stdin = child.stdin.take().unwrap();
1038+
let pump_service = service.to_string();
9711039
let body_task = tokio::spawn(async move {
9721040
use futures_util::StreamExt;
9731041
let mut stream = body.into_data_stream();
@@ -981,7 +1049,14 @@ async fn run_git_at(
9811049
break;
9821050
}
9831051
}
984-
Err(_) => break,
1052+
Err(e) => {
1053+
// Body/decode errors (client abort, malformed gzip,
1054+
// decoded-size cap) surface to git as early EOF; log so
1055+
// there is a server-side signal, not just an opaque
1056+
// client-side hangup.
1057+
warn!(error = %e, service = %pump_service, "git request body stream failed");
1058+
break;
1059+
}
9851060
}
9861061
}
9871062
drop(stdin); // close stdin → EOF for git
@@ -1378,7 +1453,14 @@ fn stream_git_read(
13781453
break;
13791454
}
13801455
}
1381-
Err(_) => break,
1456+
Err(e) => {
1457+
// Body/decode errors (client abort, malformed gzip,
1458+
// decoded-size cap) surface to git as early EOF; log so
1459+
// there is a server-side signal, not just an opaque
1460+
// client-side hangup.
1461+
warn!(error = %e, service = %service, "git request body stream failed");
1462+
break;
1463+
}
13821464
}
13831465
}
13841466
drop(stdin); // close stdin → EOF for git
@@ -1694,6 +1776,116 @@ mod track_c_tests {
16941776
"cb09a769da1c01f458fa6959d4e8eded38fac8d3".to_string()
16951777
}
16961778

1779+
/// A gzip-encoded request body is transparently inflated before it
1780+
/// reaches the git subprocess. Git's smart-HTTP client gzips the
1781+
/// upload-pack/receive-pack request body past a size threshold (fires
1782+
/// on many-ref clones); without this the subprocess sees raw gzip and
1783+
/// dies with `fatal: protocol error: bad line length character`.
1784+
#[tokio::test]
1785+
async fn gzip_request_body_is_inflated() {
1786+
use axum::http::HeaderMap;
1787+
use std::io::Write;
1788+
1789+
let plaintext = b"0032want cb09a769da1c01f458fa6959d4e8eded38fac8d3\n0000";
1790+
let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
1791+
encoder.write_all(plaintext).unwrap();
1792+
let gzipped = encoder.finish().unwrap();
1793+
assert_ne!(
1794+
gzipped, plaintext,
1795+
"precondition: body is actually compressed"
1796+
);
1797+
1798+
let mut headers = HeaderMap::new();
1799+
headers.insert(header::CONTENT_ENCODING, "gzip".parse().unwrap());
1800+
let decoded = decode_git_request_body(&headers, Body::from(gzipped), u64::MAX);
1801+
let bytes = axum::body::to_bytes(decoded, usize::MAX).await.unwrap();
1802+
assert_eq!(bytes.as_ref(), plaintext);
1803+
}
1804+
1805+
/// Without a gzip `Content-Encoding`, the body is passed through byte
1806+
/// for byte (the common small-clone / already-inflated case).
1807+
#[tokio::test]
1808+
async fn identity_request_body_is_passthrough() {
1809+
use axum::http::HeaderMap;
1810+
let plaintext = b"0032want cb09a769da1c01f458fa6959d4e8eded38fac8d3\n0000";
1811+
let decoded =
1812+
decode_git_request_body(&HeaderMap::new(), Body::from(plaintext.to_vec()), u64::MAX);
1813+
let bytes = axum::body::to_bytes(decoded, usize::MAX).await.unwrap();
1814+
assert_eq!(bytes.as_ref(), plaintext);
1815+
}
1816+
1817+
/// A gzip bomb is cut off at the decoded-byte cap: the router's
1818+
/// `RequestBodyLimitLayer` only bounds *compressed* bytes, so the
1819+
/// decode seam must enforce the inflated bound itself. Highly
1820+
/// compressible input (1 MiB of zeros → ~1 KiB gzip) must error once
1821+
/// the decoded stream crosses the cap.
1822+
#[tokio::test]
1823+
async fn gzip_request_body_over_decoded_cap_errors() {
1824+
use axum::http::HeaderMap;
1825+
use std::io::Write;
1826+
1827+
let plaintext = vec![0u8; 1024 * 1024]; // inflates 1 MiB from ~1 KiB wire
1828+
let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
1829+
encoder.write_all(&plaintext).unwrap();
1830+
let gzipped = encoder.finish().unwrap();
1831+
assert!(
1832+
gzipped.len() < 64 * 1024,
1833+
"precondition: bomb is small on the wire (got {} bytes)",
1834+
gzipped.len()
1835+
);
1836+
1837+
let mut headers = HeaderMap::new();
1838+
headers.insert(header::CONTENT_ENCODING, "gzip".parse().unwrap());
1839+
let cap: u64 = 64 * 1024;
1840+
let decoded = decode_git_request_body(&headers, Body::from(gzipped), cap);
1841+
let err = axum::body::to_bytes(decoded, usize::MAX)
1842+
.await
1843+
.expect_err("decoded stream must error past the cap");
1844+
assert!(
1845+
err.to_string().contains("decoded bytes"),
1846+
"error should name the decoded-size cap, got: {err}"
1847+
);
1848+
}
1849+
1850+
/// The decoded cap does not truncate bodies at or under the limit —
1851+
/// exactly-at-cap input passes through complete.
1852+
#[tokio::test]
1853+
async fn gzip_request_body_at_decoded_cap_passes() {
1854+
use axum::http::HeaderMap;
1855+
use std::io::Write;
1856+
1857+
let plaintext = vec![7u8; 8 * 1024];
1858+
let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
1859+
encoder.write_all(&plaintext).unwrap();
1860+
let gzipped = encoder.finish().unwrap();
1861+
1862+
let mut headers = HeaderMap::new();
1863+
headers.insert(header::CONTENT_ENCODING, "gzip".parse().unwrap());
1864+
let decoded =
1865+
decode_git_request_body(&headers, Body::from(gzipped), plaintext.len() as u64);
1866+
let bytes = axum::body::to_bytes(decoded, usize::MAX).await.unwrap();
1867+
assert_eq!(bytes.as_ref(), plaintext);
1868+
}
1869+
1870+
/// Malformed gzip (valid header, corrupt deflate stream) surfaces as a
1871+
/// stream error rather than silently yielding garbage — the stdin pump
1872+
/// logs it and closes the subprocess's stdin early.
1873+
#[tokio::test]
1874+
async fn malformed_gzip_request_body_errors() {
1875+
use axum::http::HeaderMap;
1876+
1877+
// Valid 10-byte gzip header followed by garbage.
1878+
let mut bogus = vec![0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff];
1879+
bogus.extend_from_slice(b"this is not deflate data at all");
1880+
1881+
let mut headers = HeaderMap::new();
1882+
headers.insert(header::CONTENT_ENCODING, "gzip".parse().unwrap());
1883+
let decoded = decode_git_request_body(&headers, Body::from(bogus), u64::MAX);
1884+
axum::body::to_bytes(decoded, usize::MAX)
1885+
.await
1886+
.expect_err("corrupt gzip must error the decoded stream");
1887+
}
1888+
16971889
fn branches_only_manifest() -> Manifest {
16981890
let mut refs = BTreeMap::new();
16991891
refs.insert("refs/heads/feature".to_string(), oid_sha1());

0 commit comments

Comments
 (0)