Skip to content

Commit 91dd64f

Browse files
authored
transport: surface subsequent data when receiving non-gRPC header (#8929)
Fixes #7406 If content-type is not grpc, we read the next data frames till 1kb or endStream, and append the subsequent data to error message. Example: When receiving ``` [header] ":status", "200", "content-type", "text/html", [payload] <html><body>Hello World</body></html> ``` Return ``` rpc error: code = Unknown desc = unexpected HTTP status code received from server: 200 (OK); transport: received unexpected content-type "text/html" data: "<html><body>Hello World</body></html>"`, ``` RELEASE NOTES: * client: Append response data to the error message when receiving a non-gRPC content-type header.
1 parent adc97de commit 91dd64f

4 files changed

Lines changed: 324 additions & 16 deletions

File tree

‎internal/transport/client_stream.go‎

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
package transport
2020

2121
import (
22+
"fmt"
2223
"sync/atomic"
2324

2425
"golang.org/x/net/http2"
@@ -28,6 +29,12 @@ import (
2829
"google.golang.org/grpc/status"
2930
)
3031

32+
// nonGRPCDataMaxLen is the maximum length of nonGRPCDataBuf.
33+
//
34+
// NOTE: If changed this value, you MUST update the corresponding test in:
35+
// - /test/end2end_test.go:TestHTTPServerSendsNonGRPCHeaderSurfaceFurtherData
36+
const nonGRPCDataMaxLen = 1024
37+
3138
// ClientStream implements streaming functionality for a gRPC client.
3239
type ClientStream struct {
3340
Stream // Embed for common stream functionality.
@@ -46,14 +53,41 @@ type ClientStream struct {
4653
// headerValid indicates whether a valid header was received. Only
4754
// meaningful after headerChan is closed (always call waitOnHeader() before
4855
// reading its value).
49-
headerValid bool
56+
headerValid bool
57+
58+
nonGRPCStatus *status.Status // the initial status from the non-gRPC response header, finalized with collected data before closing.
59+
nonGRPCDataBuf []byte // stores the data of a non-gRPC response.
60+
5061
noHeaders bool // set if the client never received headers (set only after the stream is done).
5162
headerChanClosed uint32 // set when headerChan is closed. Used to avoid closing headerChan multiple times.
5263
bytesReceived atomic.Bool // indicates whether any bytes have been received on this stream
5364
unprocessed atomic.Bool // set if the server sends a refused stream or GOAWAY including this stream
5465
statsHandler stats.Handler // nil for internal streams (e.g., health check, ORCA) where telemetry is not supported.
5566
}
5667

68+
func (s *ClientStream) startNonGRPCDataCollection(st *status.Status) {
69+
s.nonGRPCStatus = st
70+
s.nonGRPCDataBuf = make([]byte, 0, nonGRPCDataMaxLen)
71+
}
72+
73+
// finalizeNonGRPCStatus builds the terminal status by appending the collected
74+
// response body to the original non-gRPC status message.
75+
func (s *ClientStream) finalizeNonGRPCStatus() *status.Status {
76+
msg := fmt.Sprintf("%s\ndata: %q", s.nonGRPCStatus.Message(), s.nonGRPCDataBuf)
77+
return status.New(s.nonGRPCStatus.Code(), msg)
78+
}
79+
80+
// handleNonGRPCData collects non-gRPC body from the given data frame.
81+
// It returns non-nil value when the stream should be closed with it.
82+
func (s *ClientStream) handleNonGRPCData(f *parsedDataFrame) *status.Status {
83+
n := min(f.data.Len(), nonGRPCDataMaxLen-len(s.nonGRPCDataBuf))
84+
s.nonGRPCDataBuf = append(s.nonGRPCDataBuf, f.data.ReadOnlyData()[0:n]...)
85+
if len(s.nonGRPCDataBuf) >= nonGRPCDataMaxLen || f.StreamEnded() {
86+
return s.finalizeNonGRPCStatus()
87+
}
88+
return nil
89+
}
90+
5791
// Read reads an n byte message from the input stream.
5892
func (s *ClientStream) Read(n int) (mem.BufferSlice, error) {
5993
b, err := s.Stream.read(n)

‎internal/transport/http2_client.go‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1231,6 +1231,23 @@ func (t *http2Client) handleData(f *parsedDataFrame) {
12311231
t.closeStream(s, io.EOF, true, http2.ErrCodeFlowControl, status.New(codes.Internal, err.Error()), nil, false)
12321232
return
12331233
}
1234+
1235+
if s.nonGRPCStatus != nil {
1236+
// The frame should be handled as a non-gRPC response body
1237+
st := s.handleNonGRPCData(f)
1238+
if st != nil {
1239+
t.closeStream(s, st.Err(), true, http2.ErrCodeProtocol, st, nil, true)
1240+
return
1241+
}
1242+
if w := s.fc.onRead(size); w > 0 {
1243+
t.controlBuf.put(&outgoingWindowUpdate{
1244+
streamID: s.id,
1245+
increment: w,
1246+
})
1247+
}
1248+
return
1249+
}
1250+
12341251
dataLen := f.data.Len()
12351252
if f.Header().Flags.Has(http2.FlagDataPadded) {
12361253
if w := s.fc.onRead(size - uint32(dataLen)); w > 0 {
@@ -1475,6 +1492,17 @@ func (t *http2Client) operateHeaders(frame *http2.MetaHeadersFrame) {
14751492
return
14761493
}
14771494

1495+
// If we are collecting non-gRPC response data and receive a trailing
1496+
// HEADERS frame with END_STREAM, finalize the buffered data and close
1497+
// the stream.
1498+
if s.nonGRPCStatus != nil {
1499+
if endStream {
1500+
st := s.finalizeNonGRPCStatus()
1501+
t.closeStream(s, st.Err(), true, http2.ErrCodeProtocol, st, nil, true)
1502+
}
1503+
return
1504+
}
1505+
14781506
var (
14791507
// If a gRPC Response-Headers has already been received, then it means
14801508
// that the peer is speaking gRPC and we are in gRPC mode.
@@ -1575,7 +1603,12 @@ func (t *http2Client) operateHeaders(frame *http2.MetaHeadersFrame) {
15751603
}
15761604

15771605
se := status.New(grpcErrorCode, strings.Join(errs, "; "))
1578-
t.closeStream(s, se.Err(), true, http2.ErrCodeProtocol, se, nil, endStream)
1606+
if endStream {
1607+
t.closeStream(s, se.Err(), true, http2.ErrCodeProtocol, se, nil, true)
1608+
return
1609+
}
1610+
1611+
s.startNonGRPCDataCollection(se)
15791612
return
15801613
}
15811614

‎internal/transport/transport_test.go‎

Lines changed: 134 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,9 @@ const (
111111
notifyCall
112112
misbehaved
113113
encodingRequiredStatus
114-
invalidHeaderField
114+
invalidContentType
115+
invalidContentTypeWithMultipleFrame
116+
malformedHeader
115117
delayRead
116118
pingpong
117119
)
@@ -220,14 +222,60 @@ func (h *testStreamHandler) handleStreamEncodingRequiredStatus(s *ServerStream)
220222
s.Read(math.MaxInt)
221223
}
222224

223-
func (h *testStreamHandler) handleStreamInvalidHeaderField(s *ServerStream) {
225+
func (h *testStreamHandler) handleStreamInvalidContentType(s *ServerStream) {
226+
headerFields := []hpack.HeaderField{}
227+
headerFields = append(headerFields, hpack.HeaderField{Name: "content-type", Value: expectedInvalidHeaderField})
228+
h.t.controlBuf.put(&headerFrame{
229+
streamID: s.id,
230+
hf: headerFields,
231+
endStream: true,
232+
cleanup: &cleanupStream{
233+
streamID: s.id,
234+
onWrite: func() {},
235+
},
236+
})
237+
}
238+
239+
func (h *testStreamHandler) handleStreamInvalidContentTypeWithMultipleFrame(s *ServerStream) {
224240
headerFields := []hpack.HeaderField{}
225241
headerFields = append(headerFields, hpack.HeaderField{Name: "content-type", Value: expectedInvalidHeaderField})
226242
h.t.controlBuf.put(&headerFrame{
227243
streamID: s.id,
228244
hf: headerFields,
229245
endStream: false,
230246
})
247+
d1 := newBufferSlice([]byte("first"))
248+
d1.Ref()
249+
h.t.controlBuf.put(&dataFrame{
250+
streamID: s.id,
251+
h: nil,
252+
data: d1,
253+
onEachWrite: func() {},
254+
})
255+
// Wait for the test to verify the first frame before sending the next frame.
256+
<-h.notify
257+
d2 := newBufferSlice([]byte(" second"))
258+
d2.Ref()
259+
h.t.controlBuf.put(&dataFrame{
260+
streamID: s.id,
261+
h: nil,
262+
data: d2,
263+
onEachWrite: func() {},
264+
endStream: true,
265+
})
266+
}
267+
268+
func (h *testStreamHandler) handleStreamMalformedHeader(s *ServerStream) {
269+
headerFields := []hpack.HeaderField{
270+
{Name: ":status", Value: "200"},
271+
{Name: "content-type", Value: "application/grpc"},
272+
{Name: "x-bad-bin", Value: "!!!invalid-base64!!!"},
273+
}
274+
h.t.controlBuf.put(&headerFrame{
275+
streamID: s.id,
276+
hf: headerFields,
277+
endStream: false,
278+
})
231279
}
232280

233281
// handleStreamDelayRead delays reads so that the other side has to halt on
@@ -425,12 +473,36 @@ func (s *server) start(t *testing.T, port int, serverConfig *ServerConfig, ht hT
425473
})
426474
wg.Done()
427475
}()
428-
case invalidHeaderField:
476+
case invalidContentType:
477+
go func() {
478+
transport.HandleStreams(ctx, func(s *ServerStream) {
479+
wg.Add(1)
480+
go func() {
481+
h.handleStreamInvalidContentType(s)
482+
wg.Done()
483+
}()
484+
})
485+
wg.Done()
486+
}()
487+
case invalidContentTypeWithMultipleFrame:
488+
h.notify = make(chan struct{})
489+
close(s.ready)
490+
go func() {
491+
transport.HandleStreams(ctx, func(s *ServerStream) {
492+
wg.Add(1)
493+
go func() {
494+
h.handleStreamInvalidContentTypeWithMultipleFrame(s)
495+
wg.Done()
496+
}()
497+
})
498+
wg.Done()
499+
}()
500+
case malformedHeader:
429501
go func() {
430502
transport.HandleStreams(ctx, func(s *ServerStream) {
431503
wg.Add(1)
432504
go func() {
433-
h.handleStreamInvalidHeaderField(s)
505+
h.handleStreamMalformedHeader(s)
434506
wg.Done()
435507
}()
436508
})
@@ -1638,8 +1710,8 @@ func (s) TestEncodingRequiredStatus(t *testing.T) {
16381710
s.Read(math.MaxInt)
16391711
}
16401712

1641-
func (s) TestInvalidHeaderField(t *testing.T) {
1642-
server, ct, cancel := setUp(t, 0, invalidHeaderField)
1713+
func (s) TestInvalidContentType(t *testing.T) {
1714+
server, ct, cancel := setUp(t, 0, invalidContentType)
16431715
defer cancel()
16441716
callHdr := &CallHdr{
16451717
Host: "localhost",
@@ -1660,8 +1732,53 @@ func (s) TestInvalidHeaderField(t *testing.T) {
16601732
server.stop()
16611733
}
16621734

1735+
func (s) TestNonGRPCDataCollectionAcrossMultipleFrames(t *testing.T) {
1736+
server, ct, cancel := setUp(t, 0, invalidContentTypeWithMultipleFrame)
1737+
defer cancel()
1738+
defer server.stop()
1739+
defer ct.Close(fmt.Errorf("closed manually by test"))
1740+
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
1741+
defer cancel()
1742+
1743+
select {
1744+
case <-server.ready:
1745+
case <-ctx.Done():
1746+
t.Fatal("timed out waiting for server handler to be initialized")
1747+
}
1748+
1749+
s, err := ct.NewStream(ctx, &CallHdr{Host: "localhost", Method: "foo"}, nil)
1750+
if err != nil {
1751+
t.Fatalf("failed to create the stream")
1752+
}
1753+
1754+
// After the first Data frame (without EOS), handleNonGRPCData returns
1755+
// nil so the stream stays open and continues collecting.
1756+
select {
1757+
case <-s.Done():
1758+
t.Fatal("stream closed after first DATA frame, want it to stay open")
1759+
case <-time.After(defaultTestShortTimeout):
1760+
}
1761+
1762+
// Signal the server to send the second Data frame with EOS.
1763+
close(server.h.notify)
1764+
1765+
_, err = s.readTo(make([]byte, 1))
1766+
if err == nil {
1767+
t.Fatal("Read succeeded, want error")
1768+
}
1769+
// Both frames should be collected: "first" + " second".
1770+
wantCode := codes.Internal
1771+
wantMsg := "first second"
1772+
if got := status.Code(err); got != wantCode {
1773+
t.Fatalf("Read error code = %v, want %v\nfull error: %v", got, wantCode, err)
1774+
}
1775+
if got := status.Convert(err).Message(); !strings.Contains(got, wantMsg) {
1776+
t.Fatalf("Read error message = %q, want it to contain %q", got, wantMsg)
1777+
}
1778+
}
1779+
16631780
func (s) TestHeaderChanClosedAfterReceivingAnInvalidHeader(t *testing.T) {
1664-
server, ct, cancel := setUp(t, 0, invalidHeaderField)
1781+
server, ct, cancel := setUp(t, 0, malformedHeader)
16651782
defer cancel()
16661783
defer server.stop()
16671784
defer ct.Close(fmt.Errorf("closed manually by test"))
@@ -2685,6 +2802,7 @@ func (s) TestClientDecodeHeader(t *testing.T) {
26852802
name string
26862803
metaHeaderFrame *http2.MetaHeadersFrame
26872804
wantStatus *status.Status
2805+
isNonGRPCStatus bool
26882806
}{
26892807
{
26902808
name: "valid_header",
@@ -2708,6 +2826,7 @@ func (s) TestClientDecodeHeader(t *testing.T) {
27082826
codes.Unknown,
27092827
"unexpected HTTP status code received from server: 200 (OK); malformed header: missing HTTP content-type",
27102828
),
2829+
isNonGRPCStatus: true,
27112830
},
27122831
{
27132832
name: "invalid_grpc_status",
@@ -2734,6 +2853,7 @@ func (s) TestClientDecodeHeader(t *testing.T) {
27342853
codes.Internal,
27352854
"malformed header: missing HTTP status; transport: received unexpected content-type \"application/json\"",
27362855
),
2856+
isNonGRPCStatus: true,
27372857
},
27382858
{
27392859
name: "invalid_content_type_with_http_status_504",
@@ -2747,6 +2867,7 @@ func (s) TestClientDecodeHeader(t *testing.T) {
27472867
codes.Unavailable,
27482868
"unexpected HTTP status code received from server: 504 (Gateway Timeout); transport: received unexpected content-type \"application/json\"",
27492869
),
2870+
isNonGRPCStatus: true,
27502871
},
27512872
{
27522873
name: "http_fallback_and_invalid_http_status",
@@ -2803,7 +2924,12 @@ func (s) TestClientDecodeHeader(t *testing.T) {
28032924
}
28042925

28052926
s.operateHeaders(tc.metaHeaderFrame)
2806-
got := cs.status
2927+
var got *status.Status
2928+
if tc.isNonGRPCStatus {
2929+
got = cs.nonGRPCStatus
2930+
} else {
2931+
got = cs.status
2932+
}
28072933
want := tc.wantStatus
28082934
if got.Code() != want.Code() || got.Message() != want.Message() {
28092935
t.Errorf("operateHeaders(%v) got status %q, want %q", tc.metaHeaderFrame, got, want)

0 commit comments

Comments
 (0)