Skip to content

Commit 8059e6b

Browse files
committed
build(seaweedfs): build SeaweedFS in-tree with a bounded read-ahead
The S3 gateway is OOM-killed while serving a large object. SeaweedFS starts read-ahead downloads after every read of a sequential stream and counts only the downloads it newly starts toward its prefetch window, so a GET reading 256 KiB at a time out of 4 MiB chunks pulls up to about a gigabyte of the object into memory, past the 512Mi limit of the S3 deployment. Upstream master carries the same defect, so there is no release to move to instead. Build weed from the 4.45 sources with a patch that keeps the window at the next prefetchCount chunks, and lay it over the upstream image of the same tag so the rest of the image stays upstream's own. The image build runs the tests of the packages the patch touches, including the regression test the patch adds, because go build never compiles them. Assisted-by: LLM Signed-off-by: Alexey Artamonov <[email protected]>
1 parent 7c46fff commit 8059e6b

5 files changed

Lines changed: 179 additions & 2 deletions

File tree

‎Makefile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ build: build-deps
3737
make -C packages/system/capi-providers-infraprovider image
3838
make -C packages/system/multus image
3939
make -C packages/system/bucket image
40+
make -C packages/system/seaweedfs image
4041
make -C packages/system/objectstorage-controller image
4142
make -C packages/system/securitygroup-controller image
4243
make -C packages/system/grafana-operator image

‎hack/build-matrix_test.bats‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -110,15 +110,15 @@ load test_helper
110110
tmp=$(mktemp)
111111
echo "packages/system/seaweedfs/values.yaml" > "$tmp"
112112
out=$(hack/build-matrix.sh "$tmp")
113-
[ "$out" = '["packages/system/objectstorage-controller"]' ]
113+
[ "$out" = '["packages/system/seaweedfs","packages/system/objectstorage-controller"]' ]
114114
rm -f "$tmp"
115115
}
116116

117117
@test "seaweedfs change does not duplicate an already-selected objectstorage-controller" {
118118
tmp=$(mktemp)
119119
printf 'packages/system/seaweedfs/values.yaml\npackages/system/objectstorage-controller/values.yaml\n' > "$tmp"
120120
out=$(hack/build-matrix.sh "$tmp")
121-
[ "$out" = '["packages/system/objectstorage-controller"]' ]
121+
[ "$out" = '["packages/system/seaweedfs","packages/system/objectstorage-controller"]' ]
122122
rm -f "$tmp"
123123
}
124124

‎packages/system/seaweedfs/Makefile‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,35 @@
11
export NAME=seaweedfs-system
2+
SEAWEEDFS_TAG=$(shell grep -F 'ARG VERSION=' images/seaweedfs/Dockerfile | cut -f2 -d=)
23

34
include ../../../hack/common-envs.mk
45
include ../../../hack/package.mk
56

7+
image:
8+
docker buildx build images/seaweedfs \
9+
$(call image-tags,seaweedfs,$(SEAWEEDFS_TAG)) \
10+
$(call cache-args,seaweedfs) \
11+
--metadata-file images/seaweedfs.json \
12+
$(BUILDX_ARGS)
13+
REPOSITORY="$(REGISTRY)/seaweedfs" \
14+
yq -i '.seaweedfs.image.repository = strenv(REPOSITORY)' values.yaml
15+
TAG=$(IMAGE_TAG)@$$(yq e '."containerimage.digest"' images/seaweedfs.json -o json -r) \
16+
yq -i '.seaweedfs.image.tag = strenv(TAG)' values.yaml
17+
rm -f images/seaweedfs.json
18+
619
test:
720
helm unittest .
821
$(MAKE) test-guard-fail-closed
22+
$(MAKE) test-image-version
23+
24+
# The image replaces weed in the upstream image of ARG VERSION, so a chart moved
25+
# by `make update` without the Dockerfile would run the previous release's
26+
# binary under the new chart.
27+
.PHONY: test-image-version
28+
test-image-version:
29+
@chart=$$(yq e '.appVersion' charts/seaweedfs/Chart.yaml); \
30+
[ "$(SEAWEEDFS_TAG)" = "$$chart" ] \
31+
|| { echo "FAIL: images/seaweedfs/Dockerfile builds $(SEAWEEDFS_TAG), the vendored chart is $$chart"; exit 1; }
32+
@echo "ok: images/seaweedfs builds the chart's appVersion $(SEAWEEDFS_TAG)"
933

1034
# The naming guard refuses to render an UPGRADE that cannot see the cluster
1135
# (templates/naming-guard.yaml): a client-side lookup silently returns nothing,
Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
# Rebuild the weed binary from the upstream sources with the Cozystack patches
2+
# applied, and lay it over the upstream image of the same tag. Only the Go
3+
# binary changes: the Rust volume server and worker, the entrypoint and the
4+
# runtime packages are the upstream image's own, byte for byte.
5+
#
6+
# patches/reader-cache-prefetch-window.diff bounds the read-ahead of a
7+
# sequential chunk reader to the prefetch count it was given. Upstream counts
8+
# only the downloads it newly starts toward that window and runs the prefetch
9+
# after every ReadAt, so the S3 GET path, reading 256 KiB at a time out of
10+
# 4 MiB chunks, pushes the window four chunks further on each read until the
11+
# ReaderCache limit of 256 downloaders is reached. The gateway then holds up to
12+
# about a gigabyte of a single object in memory, is OOM-killed under the 512Mi
13+
# limit the S3 deployment gets from its resources preset, and the client sees
14+
# the response cut off tens of megabytes in:
15+
# https://github.com/cozystack/cozystack/issues/3782. Upstream carries the same
16+
# defect on master, tracked as seaweedfs/seaweedfs#11672; drop the patch on the
17+
# first release that carries a fix.
18+
19+
# Keep in step with the chart appVersion pulled by `make update`, and with the
20+
# base image below: the patched binary replaces this version's, so building it
21+
# from another tag would run one version's weed against the rest of another's
22+
# image.
23+
ARG VERSION=4.45
24+
25+
FROM --platform=$BUILDPLATFORM golang:1.26@sha256:eb36c1664dd974cde625f736e02c204383deebe03977365caaec5bf49f794348 AS builder
26+
ARG GOPROXY
27+
ARG VERSION
28+
ARG TARGETOS
29+
ARG TARGETARCH
30+
31+
WORKDIR /workspace
32+
33+
RUN curl -sSL https://github.com/seaweedfs/seaweedfs/archive/refs/tags/${VERSION}.tar.gz | tar -xzf- --strip=1
34+
35+
COPY patches /patches
36+
RUN git apply /patches/*.diff
37+
38+
# go build never compiles a _test.go, so nothing else runs the tests the
39+
# patches carry. The list is read into a variable first: a for loop drops the
40+
# exit status of a command substitution, so a patch git cannot read would leave
41+
# the list empty and the step green.
42+
RUN files=$(git apply --numstat /patches/*.diff) || exit 1; \
43+
for dir in $(printf '%s\n' "$files" | cut -f3 | grep '\.go$' | xargs -r -n1 dirname | sort -u); do \
44+
go test -race "./$dir" || exit 1; \
45+
done
46+
47+
RUN cd weed && GOOS=$TARGETOS GOARCH=$TARGETARCH CGO_ENABLED=0 go build -o /workspace/weed-bin .
48+
49+
FROM chrislusf/seaweedfs:${VERSION}@sha256:fc9f76fa993ad69966ffeb2f65d0318fcae39c6f8e20cf68ef7b3a5cb97769e5
50+
COPY --from=builder /workspace/weed-bin /usr/bin/weed
Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
diff --git a/weed/filer/reader_cache.go b/weed/filer/reader_cache.go
2+
--- a/weed/filer/reader_cache.go
3+
+++ b/weed/filer/reader_cache.go
4+
@@ -75,8 +75,10 @@
5+
return
6+
}
7+
8+
- cached := 0
9+
- for x := chunkViews; x != nil && cached < count; x = x.Next {
10+
+ // The window is the next count chunks, whether or not a download for them
11+
+ // is already running: counting only newly started downloads would push the
12+
+ // window count chunks further on every call.
13+
+ for x, i := chunkViews, 0; x != nil && i < count; x, i = x.Next, i+1 {
14+
chunkView := x.Value
15+
if _, found := rc.downloaders[chunkView.FileId]; found {
16+
continue
17+
@@ -98,7 +100,6 @@
18+
go cacher.startCaching()
19+
<-cacher.cacheStartedCh
20+
rc.downloaders[chunkView.FileId] = cacher
21+
- cached++
22+
}
23+
24+
return
25+
diff --git a/weed/filer/reader_at_prefetch_window_test.go b/weed/filer/reader_at_prefetch_window_test.go
26+
new file mode 100644
27+
--- /dev/null
28+
+++ b/weed/filer/reader_at_prefetch_window_test.go
29+
@@ -0,0 +1,73 @@
30+
+package filer
31+
+
32+
+import (
33+
+ "context"
34+
+ "fmt"
35+
+ "strconv"
36+
+ "strings"
37+
+ "sync/atomic"
38+
+ "testing"
39+
+
40+
+ "github.com/seaweedfs/seaweedfs/weed/util/chunk_cache"
41+
+ util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
42+
+)
43+
+
44+
+// A sequential reader that consumes a chunk in several small ReadAt calls, as
45+
+// the S3 GET path does with its 256 KiB copy buffer over 4 MiB chunks, must
46+
+// keep its read-ahead at prefetchCount chunks. Each ReadAt triggers a
47+
+// prefetch; if chunks already being fetched are not counted toward the
48+
+// window, every call starts prefetchCount more downloads and the read-ahead
49+
+// runs to the ReaderCache downloader limit, buffering most of the object.
50+
+func TestChunkReadAtSequentialPrefetchStaysWithinWindow(t *testing.T) {
51+
+ const chunkSize = 64 << 10
52+
+ const chunkCount = 64
53+
+ const sliceSize = 16 << 10
54+
+
55+
+ var readerChunk atomic.Int64
56+
+ var maxLead atomic.Int64
57+
+ rc := NewReaderCache(256, (*chunk_cache.TieredChunkCache)(nil), func(context.Context, string) ([]string, error) {
58+
+ return []string{"unused"}, nil
59+
+ }, nil)
60+
+ defer rc.destroy()
61+
+ rc.fetchChunkDataFn = func(_ context.Context, buffer []byte, _ []string, _ []byte, _ bool, _ bool, _ int64, fileId string, _ util_http.RefreshUrlsFunc) (int, error) {
62+
+ idx, err := strconv.Atoi(strings.TrimPrefix(fileId, "chunk"))
63+
+ if err != nil {
64+
+ return 0, err
65+
+ }
66+
+ lead := int64(idx) - readerChunk.Load()
67+
+ for {
68+
+ cur := maxLead.Load()
69+
+ if lead <= cur || maxLead.CompareAndSwap(cur, lead) {
70+
+ break
71+
+ }
72+
+ }
73+
+ return len(buffer), nil
74+
+ }
75+
+
76+
+ views := NewIntervalList[*ChunkView]()
77+
+ for i := 0; i < chunkCount; i++ {
78+
+ views.AppendInterval(&Interval[*ChunkView]{
79+
+ StartOffset: int64(i * chunkSize),
80+
+ StopOffset: int64((i + 1) * chunkSize),
81+
+ Value: &ChunkView{
82+
+ FileId: fmt.Sprintf("chunk%d", i),
83+
+ ViewSize: chunkSize,
84+
+ ViewOffset: int64(i * chunkSize),
85+
+ ChunkSize: chunkSize,
86+
+ },
87+
+ })
88+
+ }
89+
+ reader := NewChunkReaderAtFromClient(context.Background(), rc, views, chunkSize*chunkCount, DefaultPrefetchCount)
90+
+
91+
+ buf := make([]byte, sliceSize)
92+
+ for off := int64(0); off < chunkSize*chunkCount; off += sliceSize {
93+
+ readerChunk.Store(off / chunkSize)
94+
+ if _, err := reader.ReadAt(buf, off); err != nil && off+sliceSize < chunkSize*chunkCount {
95+
+ t.Fatalf("ReadAt(%d): %v", off, err)
96+
+ }
97+
+ }
98+
+
99+
+ if got := maxLead.Load(); got > DefaultPrefetchCount {
100+
+ t.Fatalf("read-ahead reached %d chunks past the reader, want at most %d", got, DefaultPrefetchCount)
101+
+ }
102+
+}

0 commit comments

Comments
 (0)