Skip to content
This repository was archived by the owner on Mar 11, 2026. It is now read-only.

Commit ec66e4d

Browse files
losalexlosalex
authored
fix: Setting maxEntrySize does not truncate big json payloads correctly (#1177)
* fix: Setting maxEntrySize does not truncate big json payloads correctly * Change variable names * Add more validations and adjust test * Address PR comments * Fix msg check to validate null and undefined
1 parent eada910 commit ec66e4d

4 files changed

Lines changed: 89 additions & 14 deletions

File tree

‎src/log.ts‎

Lines changed: 38 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ export interface TailEntriesRequest {
5959
export interface LogOptions {
6060
removeCircular?: boolean;
6161
maxEntrySize?: number; // see: https://cloud.google.com/logging/quotas
62+
jsonFieldsToTruncate?: string[];
6263
}
6364

6465
// eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -86,6 +87,8 @@ export type DeleteCallback = ApiResponseCallback;
8687
* @param {boolean} [options.removeCircular] Replace circular references in
8788
* logged objects with a string value, `[Circular]`. (Default: false)
8889
* @param {number} [options.maxEntrySize] A max entry size
90+
* @param {string[]} [options.jsonFieldsToTruncate] A list of JSON properties at the given full path to be truncated.
91+
* Received values will be prepended to predefined list in the order received and duplicates discarded.
8992
*
9093
* @example
9194
* ```
@@ -100,6 +103,7 @@ class Log implements LogSeverityFunctions {
100103
maxEntrySize?: number;
101104
logging: Logging;
102105
name: string;
106+
jsonFieldsToTruncate: string[];
103107

104108
constructor(logging: Logging, name: string, options?: LogOptions) {
105109
options = options || {};
@@ -112,6 +116,35 @@ class Log implements LogSeverityFunctions {
112116
* @type {string}
113117
*/
114118
this.name = this.formattedName_.split('/').pop()!;
119+
this.jsonFieldsToTruncate = [
120+
// Winston:
121+
'jsonPayload.fields.metadata.structValue.fields.stack.stringValue',
122+
// Bunyan:
123+
'jsonPayload.fields.msg.stringValue',
124+
'jsonPayload.fields.err.structValue.fields.stack.stringValue',
125+
'jsonPayload.fields.err.structValue.fields.message.stringValue',
126+
// All:
127+
'jsonPayload.fields.message.stringValue',
128+
];
129+
130+
// Prepend all custom fields to be truncated to a list with defaults, thus
131+
// custom fields will be truncated first. Make sure to filter out fields
132+
// which are not in EntryJson.jsonPayload
133+
if (
134+
options.jsonFieldsToTruncate !== null &&
135+
options.jsonFieldsToTruncate !== undefined
136+
) {
137+
const filteredList = options.jsonFieldsToTruncate.filter(
138+
str =>
139+
str !== null &&
140+
!this.jsonFieldsToTruncate.includes(str) &&
141+
str.startsWith('jsonPayload')
142+
);
143+
const uniqueSet = new Set(filteredList);
144+
this.jsonFieldsToTruncate = Array.from(uniqueSet).concat(
145+
this.jsonFieldsToTruncate
146+
);
147+
}
115148
}
116149

117150
/**
@@ -988,25 +1021,18 @@ class Log implements LogSeverityFunctions {
9881021
Math.max(entry.textPayload.length - delta, 0)
9891022
);
9901023
} else {
991-
const fieldsToTruncate = [
992-
// Winston:
993-
'jsonPayload.fields.metadata.structValue.fields.stack.stringValue',
994-
// Bunyan:
995-
'jsonPayload.fields.msg.stringValue',
996-
'jsonPayload.fields.err.structValue.fields.stack.stringValue',
997-
'jsonPayload.fields.err.structValue.fields.message.stringValue',
998-
// All:
999-
'jsonPayload.fields.message.stringValue',
1000-
];
1001-
for (const field of fieldsToTruncate) {
1024+
for (const field of this.jsonFieldsToTruncate) {
10021025
const msg: string = dotProp.get(entry, field, '');
1003-
if (msg !== '') {
1026+
if (msg !== null && msg !== undefined && msg !== '') {
10041027
dotProp.set(
10051028
entry,
10061029
field,
10071030
msg.slice(0, Math.max(msg.length - delta, 0))
10081031
);
10091032
delta -= Math.min(msg.length, delta);
1033+
if (delta <= 0) {
1034+
break;
1035+
}
10101036
}
10111037
}
10121038
}

‎test/entry.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -362,7 +362,7 @@ describe('Entry', () => {
362362
headers: {
363363
['x-cloud-trace-context']: '1/1',
364364
},
365-
} as any as http.IncomingMessage;
365+
} as unknown as http.IncomingMessage;
366366
const json = entry.toStructuredJSON();
367367
assert.strictEqual(json[entryTypes.TRACE_KEY], 'projects//traces/1');
368368
assert.strictEqual(json[entryTypes.SPAN_ID_KEY], '1');

‎test/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1772,6 +1772,7 @@ describe('Logging', () => {
17721772
logging = new Logging();
17731773
sinon.stub(metadata, 'getDefaultResource').resolves({type: 'bar'});
17741774
await logging.setDetectedResource();
1775+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
17751776
assert.strictEqual((logging.detectedResource as any).type, 'bar');
17761777
sinon.restore();
17771778
});

‎test/log.ts‎

Lines changed: 49 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,9 @@ describe('Log', () => {
3131
const PROJECT_ID = 'project-id';
3232
const FAKE_RESOURCE = 'fake-resource';
3333
const LOG_NAME = 'escaping/required/for/this/log-name';
34+
const TRUNCATE_FIELD =
35+
'jsonPayload.fields.metadata.structValue.fields.custom.stringValue';
36+
const INVALID_TRUNCATE_FIELD = 'insertId';
3437
const LOG_NAME_ENCODED = encodeURIComponent(LOG_NAME);
3538
const LOG_NAME_FORMATTED = [
3639
'projects',
@@ -95,7 +98,17 @@ describe('Log', () => {
9598
},
9699
} as {} as Logging;
97100

98-
const options: LogOptions = {};
101+
// Add some custom defined field to truncate which can be tested later - the idea is to
102+
// see that constructor works properly and provides correct order of fields to be truncated.
103+
// Also append same value twice to make sure that duplicates should be discarded.
104+
// Adding illegal field to be truncated should be discared as well
105+
const options: LogOptions = {
106+
jsonFieldsToTruncate: [
107+
INVALID_TRUNCATE_FIELD,
108+
TRUNCATE_FIELD,
109+
TRUNCATE_FIELD,
110+
],
111+
};
99112
if (maxEntrySize) {
100113
options.maxEntrySize = maxEntrySize;
101114
}
@@ -633,6 +646,41 @@ describe('Log', () => {
633646
assert.ok(message.startsWith('hello world'));
634647
assert.ok(message.length < maxSize + entryMetaMaxLength);
635648
});
649+
650+
it('should not contin duplicate or illegal fields to be truncated and defaults should present', async () => {
651+
assert.ok(log.jsonFieldsToTruncate.length > 1);
652+
assert.ok(log.jsonFieldsToTruncate[0] === TRUNCATE_FIELD);
653+
const notExists = log.jsonFieldsToTruncate.filter(
654+
(str: string) => str === INVALID_TRUNCATE_FIELD
655+
);
656+
assert.strictEqual(notExists.length, 0);
657+
const existOnce = log.jsonFieldsToTruncate.filter(
658+
(str: string) => str === TRUNCATE_FIELD
659+
);
660+
assert.strictEqual(existOnce.length, 1);
661+
});
662+
663+
it('should truncate custom defined field', async () => {
664+
const maxSize = 300;
665+
const entries = entriesFactory({
666+
message: 'hello world'.padEnd(2000, '.'),
667+
metadata: {
668+
custom: 'custom world'.padEnd(2000, '.'),
669+
},
670+
});
671+
672+
log.maxEntrySize = maxSize;
673+
log.truncateEntries(entries);
674+
675+
const message: string =
676+
entries[0].jsonPayload!.fields!.message.stringValue!;
677+
const custom: string =
678+
entries[0].jsonPayload!.fields!.metadata.structValue!.fields!.custom
679+
.stringValue!;
680+
assert.ok(message.startsWith('hello world'));
681+
assert.strictEqual(custom, '');
682+
assert.ok(message.length < maxSize + entryMetaMaxLength);
683+
});
636684
});
637685

638686
describe('severity shortcuts', () => {

0 commit comments

Comments
 (0)