Skip to content

Commit 017b08c

Browse files
authored
Merge commit from fork
* Fix attribute length limits for replacement runes * Link security advisory in changelog * Apply suggestion from @pellared * Explain UTF-8 truncation invariants * Clarify rebuilt output bound * Fix replacement rune limits in auto trace attributes * Fold auto trace replacement rune coverage into limit table * Fold SDK replacement rune coverage into span limits test * Keep attribute truncation comments focused
1 parent d9d3ac2 commit 017b08c

11 files changed

Lines changed: 187 additions & 23 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
3737

3838
### Fixed
3939

40+
- Count valid U+FFFD replacement characters toward attribute value length limits in `go.opentelemetry.io/otel/trace`, `go.opentelemetry.io/otel/sdk/trace`, and `go.opentelemetry.io/otel/sdk/log`.
4041
- Truncate string attribute values to empty strings when the configured length limit is zero, including malformed UTF-8, in `go.opentelemetry.io/otel/trace`, `go.opentelemetry.io/otel/sdk/trace`, and `go.opentelemetry.io/otel/sdk/log`. (#9054)
4142
- Fix a data race in `NewPeriodicReader` where `r.inst` was assigned after the background goroutine was launched in `go.opentelemetry.io/otel/sdk/metric`. (#9028)
4243
- Prevent precision loss for large `int64` values in `Sum`, `Histogram`, and `ExponentialHistogram` aggregations in `go.opentelemetry.io/otel/sdk/metric`. (#8981)

‎internal/shared/attrnorm/truncate.go.tmpl‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -339,7 +339,7 @@ func truncate(limit int, s string) string {
339339
// This prioritizes performance in the following order based on the most
340340
// common expected use cases.
341341
//
342-
// - Values shorter than the default limit (128).
342+
// - Values whose byte length does not exceed the limit.
343343
// - Strings with valid encodings that exceed the limit.
344344
// - No limit.
345345
// - Strings with invalid encodings that exceed the limit.
@@ -363,22 +363,33 @@ func truncate(limit int, s string) string {
363363
if limit == 0 {
364364
return ""
365365
}
366+
// Range reports RuneError for both an invalid byte and a valid U+FFFD.
367+
// Only a size-1 decoding indicates invalid UTF-8.
366368
_, size := utf8.DecodeRuneInString(s[i:])
367369
if size == 1 {
368-
// Invalid encoding.
370+
// Removing this byte makes len(s)-1 an upper bound for a rebuilt
371+
// result. Keep the valid prefix before rescanning s[i:].
369372
b.Grow(len(s) - 1)
370373
_, _ = b.WriteString(s[:i])
371374
s = s[i:]
372375
break
373376
}
377+
378+
// A valid U+FFFD still consumes one character of the limit.
379+
count++
380+
if count > limit {
381+
return s[:i]
382+
}
374383
}
375384

376385
// Fast-path, no invalid input.
377386
if b.Cap() == 0 {
378387
return s
379388
}
380389

381-
// Truncate while validating UTF-8.
390+
// The builder holds the valid prefix before the first invalid byte, and
391+
// count includes its runes. Skip malformed bytes without counting them
392+
// while copying the remaining valid runes up to the limit.
382393
for i := 0; i < len(s) && count < limit; {
383394
c := s[i]
384395
if c < utf8.RuneSelf {

‎internal/shared/attrnorm/truncate_test.go.tmpl‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ package attrnorm
99
import (
1010
"bytes"
1111
"fmt"
12+
"strings"
1213
"testing"
1314

1415
"github.com/stretchr/testify/assert"
@@ -178,6 +179,16 @@ func TestTruncateAttr(t *testing.T) {
178179
attr: attribute.StringSlice(key, []string{"日", "本"}),
179180
want: attribute.StringSlice(key, []string{"日", "本"}),
180181
},
182+
{
183+
limit: 2,
184+
attr: attribute.String(key, "a\uFFFDbcd"),
185+
want: attribute.String(key, "a\uFFFD"),
186+
},
187+
{
188+
limit: 1,
189+
attr: attribute.StringSlice(key, []string{"\uFFFD\uFFFD", "ok"}),
190+
want: attribute.StringSlice(key, []string{"\uFFFD", "o"}),
191+
},
181192
// SLICE cases
182193
{
183194
limit: -1,
@@ -200,6 +211,11 @@ func TestTruncateAttr(t *testing.T) {
200211
attr: attribute.Slice(key, attribute.SliceValue(attribute.StringValue("value"))),
201212
want: attribute.Slice(key, attribute.SliceValue(attribute.StringValue("v"))),
202213
},
214+
{
215+
limit: 1,
216+
attr: attribute.Slice(key, attribute.StringValue("\uFFFD\uFFFD")),
217+
want: attribute.Slice(key, attribute.StringValue("\uFFFD")),
218+
},
203219
{
204220
// STRINGSLICE within SLICE: each string element is truncated.
205221
limit: 2,
@@ -327,6 +343,11 @@ func TestTruncateAttr(t *testing.T) {
327343
attr: attribute.Map(key, attribute.Map("map", attribute.String("nested", "value"))),
328344
want: attribute.Map(key, attribute.Map("map", attribute.String("nested", "v"))),
329345
},
346+
{
347+
limit: 1,
348+
attr: attribute.Map(key, attribute.String("nested", "\uFFFD\uFFFD")),
349+
want: attribute.Map(key, attribute.String("nested", "\uFFFD")),
350+
},
330351
{
331352
// SLICE within MAP: recursive truncation.
332353
limit: 2,
@@ -744,6 +765,19 @@ func TestTruncateString(t *testing.T) {
744765
{12, "こんにちは", "こんにちは"},
745766
},
746767
},
768+
{
769+
name: "ReplacementRune",
770+
groups: []group{
771+
{5, strings.Repeat("\uFFFD", 1000), strings.Repeat("\uFFFD", 5)},
772+
{5, strings.Repeat("\uFFFD", 100) + "abcdef", strings.Repeat("\uFFFD", 5)},
773+
{2, "a\uFFFDbcd", "a\uFFFD"},
774+
{3, "ab\uFFFDcd", "ab\uFFFD"},
775+
{2, "a\uFFFD", "a\uFFFD"},
776+
{0, "\uFFFD", ""},
777+
{2, "a\x80\uFFFDcd", "a\uFFFD"},
778+
{3, "a\uFFFD\x80cd", "a\uFFFDc"},
779+
},
780+
},
747781

748782
// Truncation with invalid UTF-8 characters
749783
{
@@ -1007,6 +1041,7 @@ func BenchmarkTruncate(b *testing.B) {
10071041
b.Run("Short", run(10, "Short Text"))
10081042
b.Run("ASCII", run(5, "Hello, World!"))
10091043
b.Run("ValidUTF-8", run(10, "hello 😊 world 🌍🚀"))
1044+
b.Run("ReplacementRune", run(5, strings.Repeat("\uFFFD", 100)))
10101045
b.Run("InvalidUTF-8", run(6, "€"[0:2]+"hello€€"))
10111046
b.Run("MixedUTF-8", run(14, "\x80😊\x80 Hello\x80World🌍\x80🚀\x80"))
10121047
}

‎sdk/internal/attrnorm/truncate.go‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -339,7 +339,7 @@ func truncate(limit int, s string) string {
339339
// This prioritizes performance in the following order based on the most
340340
// common expected use cases.
341341
//
342-
// - Values shorter than the default limit (128).
342+
// - Values whose byte length does not exceed the limit.
343343
// - Strings with valid encodings that exceed the limit.
344344
// - No limit.
345345
// - Strings with invalid encodings that exceed the limit.
@@ -363,22 +363,33 @@ func truncate(limit int, s string) string {
363363
if limit == 0 {
364364
return ""
365365
}
366+
// Range reports RuneError for both an invalid byte and a valid U+FFFD.
367+
// Only a size-1 decoding indicates invalid UTF-8.
366368
_, size := utf8.DecodeRuneInString(s[i:])
367369
if size == 1 {
368-
// Invalid encoding.
370+
// Removing this byte makes len(s)-1 an upper bound for a rebuilt
371+
// result. Keep the valid prefix before rescanning s[i:].
369372
b.Grow(len(s) - 1)
370373
_, _ = b.WriteString(s[:i])
371374
s = s[i:]
372375
break
373376
}
377+
378+
// A valid U+FFFD still consumes one character of the limit.
379+
count++
380+
if count > limit {
381+
return s[:i]
382+
}
374383
}
375384

376385
// Fast-path, no invalid input.
377386
if b.Cap() == 0 {
378387
return s
379388
}
380389

381-
// Truncate while validating UTF-8.
390+
// The builder holds the valid prefix before the first invalid byte, and
391+
// count includes its runes. Skip malformed bytes without counting them
392+
// while copying the remaining valid runes up to the limit.
382393
for i := 0; i < len(s) && count < limit; {
383394
c := s[i]
384395
if c < utf8.RuneSelf {

‎sdk/internal/attrnorm/truncate_test.go‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ package attrnorm
99
import (
1010
"bytes"
1111
"fmt"
12+
"strings"
1213
"testing"
1314

1415
"github.com/stretchr/testify/assert"
@@ -178,6 +179,16 @@ func TestTruncateAttr(t *testing.T) {
178179
attr: attribute.StringSlice(key, []string{"日", "本"}),
179180
want: attribute.StringSlice(key, []string{"日", "本"}),
180181
},
182+
{
183+
limit: 2,
184+
attr: attribute.String(key, "a\uFFFDbcd"),
185+
want: attribute.String(key, "a\uFFFD"),
186+
},
187+
{
188+
limit: 1,
189+
attr: attribute.StringSlice(key, []string{"\uFFFD\uFFFD", "ok"}),
190+
want: attribute.StringSlice(key, []string{"\uFFFD", "o"}),
191+
},
181192
// SLICE cases
182193
{
183194
limit: -1,
@@ -200,6 +211,11 @@ func TestTruncateAttr(t *testing.T) {
200211
attr: attribute.Slice(key, attribute.SliceValue(attribute.StringValue("value"))),
201212
want: attribute.Slice(key, attribute.SliceValue(attribute.StringValue("v"))),
202213
},
214+
{
215+
limit: 1,
216+
attr: attribute.Slice(key, attribute.StringValue("\uFFFD\uFFFD")),
217+
want: attribute.Slice(key, attribute.StringValue("\uFFFD")),
218+
},
203219
{
204220
// STRINGSLICE within SLICE: each string element is truncated.
205221
limit: 2,
@@ -327,6 +343,11 @@ func TestTruncateAttr(t *testing.T) {
327343
attr: attribute.Map(key, attribute.Map("map", attribute.String("nested", "value"))),
328344
want: attribute.Map(key, attribute.Map("map", attribute.String("nested", "v"))),
329345
},
346+
{
347+
limit: 1,
348+
attr: attribute.Map(key, attribute.String("nested", "\uFFFD\uFFFD")),
349+
want: attribute.Map(key, attribute.String("nested", "\uFFFD")),
350+
},
330351
{
331352
// SLICE within MAP: recursive truncation.
332353
limit: 2,
@@ -744,6 +765,19 @@ func TestTruncateString(t *testing.T) {
744765
{12, "こんにちは", "こんにちは"},
745766
},
746767
},
768+
{
769+
name: "ReplacementRune",
770+
groups: []group{
771+
{5, strings.Repeat("\uFFFD", 1000), strings.Repeat("\uFFFD", 5)},
772+
{5, strings.Repeat("\uFFFD", 100) + "abcdef", strings.Repeat("\uFFFD", 5)},
773+
{2, "a\uFFFDbcd", "a\uFFFD"},
774+
{3, "ab\uFFFDcd", "ab\uFFFD"},
775+
{2, "a\uFFFD", "a\uFFFD"},
776+
{0, "\uFFFD", ""},
777+
{2, "a\x80\uFFFDcd", "a\uFFFD"},
778+
{3, "a\uFFFD\x80cd", "a\uFFFDc"},
779+
},
780+
},
747781

748782
// Truncation with invalid UTF-8 characters
749783
{
@@ -1007,6 +1041,7 @@ func BenchmarkTruncate(b *testing.B) {
10071041
b.Run("Short", run(10, "Short Text"))
10081042
b.Run("ASCII", run(5, "Hello, World!"))
10091043
b.Run("ValidUTF-8", run(10, "hello 😊 world 🌍🚀"))
1044+
b.Run("ReplacementRune", run(5, strings.Repeat("\uFFFD", 100)))
10101045
b.Run("InvalidUTF-8", run(6, "€"[0:2]+"hello€€"))
10111046
b.Run("MixedUTF-8", run(14, "\x80😊\x80 Hello\x80World🌍\x80🚀\x80"))
10121047
}

‎sdk/log/internal/attrnorm/truncate.go‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -339,7 +339,7 @@ func truncate(limit int, s string) string {
339339
// This prioritizes performance in the following order based on the most
340340
// common expected use cases.
341341
//
342-
// - Values shorter than the default limit (128).
342+
// - Values whose byte length does not exceed the limit.
343343
// - Strings with valid encodings that exceed the limit.
344344
// - No limit.
345345
// - Strings with invalid encodings that exceed the limit.
@@ -363,22 +363,33 @@ func truncate(limit int, s string) string {
363363
if limit == 0 {
364364
return ""
365365
}
366+
// Range reports RuneError for both an invalid byte and a valid U+FFFD.
367+
// Only a size-1 decoding indicates invalid UTF-8.
366368
_, size := utf8.DecodeRuneInString(s[i:])
367369
if size == 1 {
368-
// Invalid encoding.
370+
// Removing this byte makes len(s)-1 an upper bound for a rebuilt
371+
// result. Keep the valid prefix before rescanning s[i:].
369372
b.Grow(len(s) - 1)
370373
_, _ = b.WriteString(s[:i])
371374
s = s[i:]
372375
break
373376
}
377+
378+
// A valid U+FFFD still consumes one character of the limit.
379+
count++
380+
if count > limit {
381+
return s[:i]
382+
}
374383
}
375384

376385
// Fast-path, no invalid input.
377386
if b.Cap() == 0 {
378387
return s
379388
}
380389

381-
// Truncate while validating UTF-8.
390+
// The builder holds the valid prefix before the first invalid byte, and
391+
// count includes its runes. Skip malformed bytes without counting them
392+
// while copying the remaining valid runes up to the limit.
382393
for i := 0; i < len(s) && count < limit; {
383394
c := s[i]
384395
if c < utf8.RuneSelf {

‎sdk/log/internal/attrnorm/truncate_test.go‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ package attrnorm
99
import (
1010
"bytes"
1111
"fmt"
12+
"strings"
1213
"testing"
1314

1415
"github.com/stretchr/testify/assert"
@@ -178,6 +179,16 @@ func TestTruncateAttr(t *testing.T) {
178179
attr: attribute.StringSlice(key, []string{"日", "本"}),
179180
want: attribute.StringSlice(key, []string{"日", "本"}),
180181
},
182+
{
183+
limit: 2,
184+
attr: attribute.String(key, "a\uFFFDbcd"),
185+
want: attribute.String(key, "a\uFFFD"),
186+
},
187+
{
188+
limit: 1,
189+
attr: attribute.StringSlice(key, []string{"\uFFFD\uFFFD", "ok"}),
190+
want: attribute.StringSlice(key, []string{"\uFFFD", "o"}),
191+
},
181192
// SLICE cases
182193
{
183194
limit: -1,
@@ -200,6 +211,11 @@ func TestTruncateAttr(t *testing.T) {
200211
attr: attribute.Slice(key, attribute.SliceValue(attribute.StringValue("value"))),
201212
want: attribute.Slice(key, attribute.SliceValue(attribute.StringValue("v"))),
202213
},
214+
{
215+
limit: 1,
216+
attr: attribute.Slice(key, attribute.StringValue("\uFFFD\uFFFD")),
217+
want: attribute.Slice(key, attribute.StringValue("\uFFFD")),
218+
},
203219
{
204220
// STRINGSLICE within SLICE: each string element is truncated.
205221
limit: 2,
@@ -327,6 +343,11 @@ func TestTruncateAttr(t *testing.T) {
327343
attr: attribute.Map(key, attribute.Map("map", attribute.String("nested", "value"))),
328344
want: attribute.Map(key, attribute.Map("map", attribute.String("nested", "v"))),
329345
},
346+
{
347+
limit: 1,
348+
attr: attribute.Map(key, attribute.String("nested", "\uFFFD\uFFFD")),
349+
want: attribute.Map(key, attribute.String("nested", "\uFFFD")),
350+
},
330351
{
331352
// SLICE within MAP: recursive truncation.
332353
limit: 2,
@@ -744,6 +765,19 @@ func TestTruncateString(t *testing.T) {
744765
{12, "こんにちは", "こんにちは"},
745766
},
746767
},
768+
{
769+
name: "ReplacementRune",
770+
groups: []group{
771+
{5, strings.Repeat("\uFFFD", 1000), strings.Repeat("\uFFFD", 5)},
772+
{5, strings.Repeat("\uFFFD", 100) + "abcdef", strings.Repeat("\uFFFD", 5)},
773+
{2, "a\uFFFDbcd", "a\uFFFD"},
774+
{3, "ab\uFFFDcd", "ab\uFFFD"},
775+
{2, "a\uFFFD", "a\uFFFD"},
776+
{0, "\uFFFD", ""},
777+
{2, "a\x80\uFFFDcd", "a\uFFFD"},
778+
{3, "a\uFFFD\x80cd", "a\uFFFDc"},
779+
},
780+
},
747781

748782
// Truncation with invalid UTF-8 characters
749783
{
@@ -1007,6 +1041,7 @@ func BenchmarkTruncate(b *testing.B) {
10071041
b.Run("Short", run(10, "Short Text"))
10081042
b.Run("ASCII", run(5, "Hello, World!"))
10091043
b.Run("ValidUTF-8", run(10, "hello 😊 world 🌍🚀"))
1044+
b.Run("ReplacementRune", run(5, strings.Repeat("\uFFFD", 100)))
10101045
b.Run("InvalidUTF-8", run(6, "€"[0:2]+"hello€€"))
10111046
b.Run("MixedUTF-8", run(14, "\x80😊\x80 Hello\x80World🌍\x80🚀\x80"))
10121047
}

‎sdk/log/record_test.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1615,6 +1615,12 @@ func TestApplyAttrLimitsTruncation(t *testing.T) {
16151615
input: attribute.StringValue("This is a very long string that should be truncated"),
16161616
want: attribute.StringValue("This "),
16171617
},
1618+
{
1619+
name: "ReplacementRuneTruncated",
1620+
limit: 5,
1621+
input: attribute.StringValue(strings.Repeat("\uFFFD", 1000)),
1622+
want: attribute.StringValue(strings.Repeat("\uFFFD", 5)),
1623+
},
16181624
{
16191625
name: "LongBytesTruncated",
16201626
limit: 5,

0 commit comments

Comments
 (0)