Skip to content

Commit b77ea36

Browse files
authored
[Backport][v1.83.x] Memory optimization (#43297)
Back port of #43272 to v1.83.x --------- Co-authored-by: Tanvi Jagtap
1 parent 9dcd5aa commit b77ea36

8 files changed

Lines changed: 185 additions & 6 deletions

File tree

‎bazel/experiments.bzl‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/core/ext/transport/chttp2/transport/frame_data.cc‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ absl::Status grpc_chttp2_data_parser_begin_frame(uint8_t flags,
4848
} else {
4949
s->received_last_frame = false;
5050
}
51+
++s->num_frames;
5152

5253
return absl::OkStatus();
5354
}
@@ -156,6 +157,7 @@ grpc_core::Poll<grpc_error_handle> grpc_deframe_unprocessed_incoming_frames(
156157
grpc_slice_buffer_move_first_into_buffer(slices, GRPC_HEADER_SIZE_IN_BYTES,
157158
header);
158159
grpc_slice_buffer_move_first(slices, length, stream_out->c_slice_buffer());
160+
s->num_frames = 0;
159161
}
160162

161163
return absl::OkStatus();
@@ -166,8 +168,20 @@ grpc_error_handle grpc_chttp2_data_parser_parse(void* /*parser*/,
166168
grpc_chttp2_stream* s,
167169
const grpc_slice& slice,
168170
int is_last) {
169-
grpc_core::CSliceRef(slice);
170-
grpc_slice_buffer_add(&s->frame_storage, slice);
171+
const size_t slice_len = GRPC_SLICE_LENGTH(slice);
172+
bool is_small_frame = 0 < slice_len && slice_len < GRPC_SLICE_INLINED_SIZE;
173+
bool multiple_small_frames =
174+
(s->num_frames >= 64) &&
175+
((s->frame_storage.length / s->num_frames) < GRPC_SLICE_INLINED_SIZE);
176+
if (GPR_UNLIKELY(multiple_small_frames && is_small_frame &&
177+
grpc_core::IsHeaderDataFrameEnabled())) {
178+
uint8_t* append_ptr =
179+
grpc_slice_buffer_tiny_add(&s->frame_storage, slice_len);
180+
memcpy(append_ptr, GRPC_SLICE_START_PTR(slice), slice_len);
181+
} else {
182+
grpc_core::CSliceRef(slice);
183+
grpc_slice_buffer_add(&s->frame_storage, slice);
184+
}
171185
grpc_chttp2_maybe_complete_recv_message(t, s);
172186

173187
if (is_last) {

‎src/core/ext/transport/chttp2/transport/internal.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -720,6 +720,7 @@ struct grpc_chttp2_stream {
720720
grpc_metadata_batch trailing_metadata_buffer;
721721

722722
grpc_slice_buffer frame_storage; // protected by t combiner
723+
size_t num_frames = 0; // protected by t combiner
723724

724725
grpc_core::Timestamp deadline = grpc_core::Timestamp::InfFuture();
725726

‎src/core/lib/experiments/experiments.cc‎

Lines changed: 15 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/core/lib/experiments/experiments.h‎

Lines changed: 11 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/core/lib/experiments/experiments.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,11 @@
172172
expiry: 2026/09/01
173173
174174
test_tags: []
175+
- name: header_data_frame
176+
description: Managing header and data memory better
177+
expiry: 2026/10/31
178+
179+
test_tags: []
175180
- name: inproc_cancel_stream
176181
description: If set, cancel inproc stream inside the transport mutex.
177182
expiry: 2026/08/01

‎src/core/lib/experiments/rollouts.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,8 @@
7272
default: false
7373
- name: fuse_filters
7474
default: false
75+
- name : header_data_frame
76+
default : true
7577
- name: inproc_cancel_stream
7678
default: true
7779
- name: keep_alive_ping_timer_batch

‎test/core/slice/c_slice_buffer_test.cc‎

Lines changed: 134 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727

2828
static constexpr size_t kTotalDataLength = 4096;
2929

30-
void test_slice_buffer_add() {
30+
TEST(SliceBufferTest, Add) {
3131
grpc_slice_buffer buf;
3232
grpc_slice aaa = grpc_slice_from_copied_string("aaa");
3333
grpc_slice bb = grpc_slice_from_copied_string("bb");
@@ -68,7 +68,7 @@ static void free_data(void* data, size_t len) {
6868
gpr_free(data);
6969
}
7070

71-
void test_slice_buffer_add_contiguous_slices() {
71+
TEST(SliceBufferTest, AddContiguousSlices) {
7272
grpc_slice_buffer buf;
7373
grpc_slice_buffer_init(&buf);
7474
char* data = reinterpret_cast<char*>(gpr_malloc(kTotalDataLength));
@@ -92,7 +92,7 @@ void test_slice_buffer_add_contiguous_slices() {
9292
grpc_slice_buffer_destroy(&buf);
9393
}
9494

95-
void test_slice_buffer_move_first() {
95+
TEST(SliceBufferTest, MoveFirst) {
9696
grpc_slice slices[3];
9797
grpc_slice_buffer src;
9898
grpc_slice_buffer dst;
@@ -140,7 +140,7 @@ void test_slice_buffer_move_first() {
140140
ASSERT_EQ(dst.length, dst_len);
141141
}
142142

143-
void test_slice_buffer_first() {
143+
TEST(SliceBufferTest, First) {
144144
grpc_slice slices[3];
145145
slices[0] = grpc_slice_from_copied_string("aaa");
146146
slices[1] = grpc_slice_from_copied_string("bbbb");
@@ -181,6 +181,136 @@ void test_slice_buffer_first() {
181181
ASSERT_EQ(buf.length, 0);
182182
}
183183

184+
TEST(SliceBufferTest, TinyAddEmptyBuffer) {
185+
grpc_slice_buffer buf;
186+
grpc_slice_buffer_init(&buf);
187+
188+
// Tiny add to an empty buffer.
189+
uint8_t* p = grpc_slice_buffer_tiny_add(&buf, 5);
190+
ASSERT_NE(p, nullptr);
191+
memcpy(p, "hello", 5);
192+
193+
ASSERT_EQ(buf.count, 1);
194+
ASSERT_EQ(buf.length, 5);
195+
196+
grpc_slice slice = grpc_slice_buffer_peek_first(&buf)[0];
197+
ASSERT_EQ(GRPC_SLICE_LENGTH(slice), 5);
198+
ASSERT_EQ(memcmp(GRPC_SLICE_START_PTR(slice), "hello", 5), 0);
199+
200+
grpc_slice_buffer_destroy(&buf);
201+
}
202+
203+
TEST(SliceBufferTest, TinyAddAppendToInlined) {
204+
grpc_slice_buffer buf;
205+
grpc_slice_buffer_init(&buf);
206+
207+
// First tiny add
208+
uint8_t* p1 = grpc_slice_buffer_tiny_add(&buf, 5);
209+
memcpy(p1, "hello", 5);
210+
ASSERT_EQ(buf.count, 1);
211+
ASSERT_EQ(buf.length, 5);
212+
213+
// Second tiny add should append to the same slice.
214+
uint8_t* p2 = grpc_slice_buffer_tiny_add(&buf, 6);
215+
ASSERT_EQ(p2, p1 + 5);
216+
memcpy(p2, " world", 6);
217+
218+
ASSERT_EQ(buf.count, 1);
219+
ASSERT_EQ(buf.length, 11);
220+
221+
grpc_slice slice = grpc_slice_buffer_peek_first(&buf)[0];
222+
ASSERT_EQ(GRPC_SLICE_LENGTH(slice), 11);
223+
ASSERT_EQ(memcmp(GRPC_SLICE_START_PTR(slice), "hello world", 11), 0);
224+
225+
grpc_slice_buffer_destroy(&buf);
226+
}
227+
228+
TEST(SliceBufferTest, TinyAddNotEnoughSpace) {
229+
grpc_slice_buffer buf;
230+
grpc_slice_buffer_init(&buf);
231+
232+
const size_t first_add_size = GRPC_SLICE_INLINED_SIZE - 3;
233+
const size_t second_add_size = 5;
234+
235+
uint8_t* p1 = grpc_slice_buffer_tiny_add(&buf, first_add_size);
236+
memset(p1, 'a', first_add_size);
237+
ASSERT_EQ(buf.count, 1);
238+
ASSERT_EQ(buf.length, first_add_size);
239+
240+
// This one should fall back to a new slice.
241+
uint8_t* p2 = grpc_slice_buffer_tiny_add(&buf, second_add_size);
242+
ASSERT_NE(p2, p1 + first_add_size);
243+
memset(p2, 'b', second_add_size);
244+
245+
ASSERT_EQ(buf.count, 2);
246+
ASSERT_EQ(buf.length, first_add_size + second_add_size);
247+
248+
// verify first slice
249+
grpc_slice slice1 = grpc_slice_buffer_peek_first(&buf)[0];
250+
ASSERT_EQ(GRPC_SLICE_LENGTH(slice1), first_add_size);
251+
252+
// verify second slice
253+
grpc_slice slice2 = buf.slices[1];
254+
ASSERT_EQ(GRPC_SLICE_LENGTH(slice2), second_add_size);
255+
256+
grpc_slice_buffer_destroy(&buf);
257+
}
258+
259+
TEST(SliceBufferTest, TinyAddAfterRefcounted) {
260+
grpc_slice_buffer buf;
261+
grpc_slice_buffer_init(&buf);
262+
263+
// Add a refcounted slice (one that is longer than INLINED_SIZE)
264+
grpc_slice s = grpc_slice_from_copied_string(
265+
"this is a refcounted string that is large enough");
266+
size_t initial_len = GRPC_SLICE_LENGTH(s);
267+
grpc_slice_buffer_add(&buf, s);
268+
269+
ASSERT_EQ(buf.count, 1);
270+
ASSERT_EQ(buf.length, initial_len);
271+
272+
// Tiny add should create a new slice, since previous is refcounted
273+
uint8_t* p = grpc_slice_buffer_tiny_add(&buf, 5);
274+
memcpy(p, "hello", 5);
275+
276+
ASSERT_EQ(buf.count, 2);
277+
ASSERT_EQ(buf.length, initial_len + 5);
278+
279+
// verify second slice
280+
grpc_slice slice2 = buf.slices[1];
281+
ASSERT_EQ(GRPC_SLICE_LENGTH(slice2), 5);
282+
ASSERT_EQ(memcmp(GRPC_SLICE_START_PTR(slice2), "hello", 5), 0);
283+
284+
grpc_slice_buffer_destroy(&buf);
285+
}
286+
287+
TEST(SliceBufferTest, TinyAddEmbiggen) {
288+
grpc_slice_buffer buf;
289+
grpc_slice_buffer_init(&buf);
290+
291+
// Fill up the buffer capacity to trigger embiggen logic.
292+
// capacity starts at GRPC_SLICE_BUFFER_INLINE_ELEMENTS.
293+
for (size_t i = 0; i < GRPC_SLICE_BUFFER_INLINE_ELEMENTS + 1; ++i) {
294+
grpc_slice s = grpc_slice_from_copied_string(
295+
"large enough string to be refcounted so tiny add creates new slice");
296+
grpc_slice_buffer_add(&buf, s);
297+
}
298+
299+
size_t initial_count = buf.count;
300+
size_t initial_len = buf.length;
301+
302+
uint8_t* p = grpc_slice_buffer_tiny_add(&buf, 5);
303+
memcpy(p, "hello", 5);
304+
305+
ASSERT_EQ(buf.count, initial_count + 1);
306+
ASSERT_EQ(buf.length, initial_len + 5);
307+
308+
grpc_slice slice_last = buf.slices[buf.count - 1];
309+
ASSERT_EQ(GRPC_SLICE_LENGTH(slice_last), 5);
310+
ASSERT_EQ(memcmp(GRPC_SLICE_START_PTR(slice_last), "hello", 5), 0);
311+
312+
grpc_slice_buffer_destroy(&buf);
313+
}
184314
int main(int argc, char** argv) {
185315
grpc::testing::TestEnvironment env(&argc, argv);
186316
::testing::InitGoogleTest(&argc, argv);

0 commit comments

Comments
 (0)