Skip to content

Commit d523e1f

Browse files
authored
[unified_analytics] Append to the log file instead of rewriting it (#2613)
1 parent 6993cba commit d523e1f

5 files changed

Lines changed: 164 additions & 25 deletions

File tree

‎pkgs/unified_analytics/CHANGELOG.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,10 @@
33
- Added optional `isExternal` parameter to `Analytics.fake` to allow overriding
44
the build configuration during testing.
55
- Dropped dependency on `package:platform` in favor of `dart:io`.
6+
- Made sending an event append to the log file instead of reading it and,
7+
once full, rewriting it; the log is now trimmed only when it passes 4 MiB.
8+
This cuts the synchronous cost of `send` with a full log from about 16 ms
9+
to under 1 ms.
610

711
## 8.0.18
812

‎pkgs/unified_analytics/lib/src/constants.dart‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,8 +76,20 @@ const String kGoogleAnalyticsApiSecret = 'Ka1jc8tZSzWc_GXMWHfPHA';
7676
const String kGoogleAnalyticsMeasurementId = 'G-04BXPVBCWJ';
7777

7878
/// How many data records to store in the log file.
79+
///
80+
/// The file holds more between trims, see [kLogFileTrimSize]; readers use
81+
/// the newest [kLogFileLength] records.
7982
const int kLogFileLength = 2500;
8083

84+
/// The log file size that triggers trimming it to the newest
85+
/// [kLogFileLength] records.
86+
///
87+
/// Between trims, saving a record only appends to the file, so the
88+
/// cost of trimming is amortized over many records.
89+
///
90+
/// 4 MiB.
91+
const int kLogFileTrimSize = 4 * (1 << 20);
92+
8193
/// The maximum allowed size of the telemetry log file.
8294
///
8395
/// 25 MiB.

‎pkgs/unified_analytics/lib/src/log_handler.dart‎

Lines changed: 64 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,12 @@
33
// BSD-style license that can be found in the LICENSE file.
44

55
import 'dart:convert';
6+
import 'dart:io' as io;
7+
import 'dart:math';
68

79
import 'package:clock/clock.dart';
810
import 'package:file/file.dart';
11+
import 'package:meta/meta.dart';
912

1013
import 'constants.dart';
1114
import 'event.dart';
@@ -154,18 +157,23 @@ class LogFileStats {
154157
/// This class is responsible for writing to a log
155158
/// file that has been initialized by the [createLogFile].
156159
///
157-
/// It will be treated as an append only log and will be limited
158-
/// to have has many data records as specified by [kLogFileLength].
160+
/// It will be treated as an append only log, trimmed to the newest
161+
/// [kLogFileLength] records when it grows past [kLogFileTrimSize].
159162
class LogHandler {
160163
final File logFile;
161164

165+
final int _trimSize;
166+
162167
/// Contains instances of [Event.analyticsException] that were encountered
163168
/// during a workflow and will be sent to GA4 for collection.
164169
final Set<Event> errorSet = {};
165170

166171
/// A log handler constructor that will delegate saving
167172
/// logs and retrieving stats from the persisted log.
168-
LogHandler({required this.logFile});
173+
LogHandler({
174+
required this.logFile,
175+
@visibleForTesting int trimSize = kLogFileTrimSize,
176+
}) : _trimSize = trimSize;
169177

170178
/// Get stats from the persisted log file.
171179
///
@@ -176,8 +184,9 @@ class LogHandler {
176184
// Parse each line of the log file through [LogItem],
177185
// some returned records may be null if malformed, they will be
178186
// removed later through `whereType<LogItem>`
179-
final records = logFile
180-
.readAsLinesSync()
187+
final lines = logFile.readAsLinesSync();
188+
final records = lines
189+
.skip(max(0, lines.length - kLogFileLength))
181190
.map((String e) {
182191
try {
183192
return LogItem.fromRecord(jsonDecode(e) as Map<String, Object?>);
@@ -273,31 +282,22 @@ class LogHandler {
273282

274283
/// Saves the data passed in as a single line in the log file.
275284
///
276-
/// This will keep the max number of records limited to equal to
277-
/// or less than [kLogFileLength] records.
285+
/// This only appends to the file, unless the file has grown past the trim
286+
/// size, see [kLogFileTrimSize]; then it is first trimmed to the newest
287+
/// [kLogFileLength] records.
278288
void save({required Map<String, Object?> data}) {
279289
try {
280-
List<String> records;
281290
final stat = logFile.statSync();
282291
if (stat.size > kMaxLogFileSize) {
283292
logFile.deleteSync();
284293
logFile.createSync();
285-
records = [];
286-
} else {
287-
records = logFile.readAsLinesSync();
288-
}
289-
final content = '${jsonEncode(data)}\n';
290-
291-
// When the record count is less than the max, add as normal;
292-
// else drop the oldest records until equal to max
293-
if (records.length < kLogFileLength) {
294-
logFile.writeAsStringSync(content, mode: FileMode.writeOnlyAppend);
295-
} else {
296-
records.add(content);
297-
records = records.skip(records.length - kLogFileLength).toList();
298-
299-
logFile.writeAsStringSync(records.join('\n'));
294+
} else if (stat.size > _trimSize) {
295+
_trim();
300296
}
297+
logFile.writeAsStringSync(
298+
'${jsonEncode(data)}\n',
299+
mode: FileMode.writeOnlyAppend,
300+
);
301301
} on Object {
302302
// Logging isn't important enough to warrant raising an
303303
// exception or error that will surprise consumers of this package.
@@ -308,6 +308,47 @@ class LogHandler {
308308
logFile.createSync();
309309
}
310310
}
311+
312+
/// Rewrites the log file keeping only its newest records.
313+
///
314+
/// Keeps at most [kLogFileLength] records and at most about half the trim
315+
/// size, so that trims stay rare however large the records are.
316+
///
317+
/// The new content is written to a temporary file which is then renamed
318+
/// over the log file, so other processes see the old or the new file and
319+
/// never a partial one. A record that another process appends during the
320+
/// trim can be lost; the log is only a local record, so that is accepted.
321+
void _trim() {
322+
File? tempFile;
323+
try {
324+
final lines = logFile.readAsLinesSync();
325+
final maxBytes = _trimSize ~/ 2;
326+
var start = lines.length;
327+
var bytes = 0;
328+
while (start > 0 && lines.length - start < kLogFileLength) {
329+
// UTF-16 length approximates UTF-8 bytes; log records are almost
330+
// entirely ASCII.
331+
bytes += lines[start - 1].length + 1;
332+
if (bytes > maxBytes) break;
333+
start--;
334+
}
335+
final kept = lines.sublist(start);
336+
// The random part keeps trims in different isolates of one process
337+
// apart.
338+
final suffix = '${io.pid}.${Random().nextInt(1 << 32)}';
339+
tempFile = logFile.fileSystem.file('${logFile.path}.$suffix.tmp');
340+
tempFile.writeAsStringSync(kept.isEmpty ? '' : '${kept.join('\n')}\n');
341+
tempFile.renameSync(logFile.path);
342+
} on Object {
343+
// Trimming is best effort; the next save tries again. If it keeps
344+
// failing, the file grows until [kMaxLogFileSize] and is deleted.
345+
try {
346+
if (tempFile != null && tempFile.existsSync()) tempFile.deleteSync();
347+
} on Object {
348+
// Nothing more to do.
349+
}
350+
}
351+
}
311352
}
312353

313354
/// Data class for each record persisted on the client's machine.

‎pkgs/unified_analytics/test/log_handler_test.dart‎

Lines changed: 83 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -241,7 +241,11 @@ void main() {
241241
expect(analytics.sentEvents.last, testEvent);
242242
expect(secondLogFileStats, isNotNull);
243243
expect(secondLogFileStats!.recordCount, kLogFileLength);
244-
expect(logFile.readAsLinesSync()[0].trim(), isNot('{{'));
244+
expect(
245+
logFile.readAsLinesSync().length,
246+
kLogFileLength + 1,
247+
reason: 'The file is append only until it is trimmed',
248+
);
245249
});
246250

247251
test('Catches and discards any FileSystemException raised from attempting '
@@ -324,6 +328,84 @@ void main() {
324328
expect(wroteDataToLogFile, isTrue);
325329
});
326330

331+
test('does not read the log file if smaller than kLogFileTrimSize', () {
332+
var readLogFile = false;
333+
var wroteDataToLogFile = false;
334+
final logFile = _FakeFile('log.txt')
335+
.._readAsLinesSyncImpl = () {
336+
readLogFile = true;
337+
return [];
338+
}
339+
.._statSyncImpl = (() => _FakeFileStat(kLogFileTrimSize))
340+
.._writeAsStringSync = (contents, {mode = FileMode.append}) {
341+
expect(mode, FileMode.writeOnlyAppend);
342+
wroteDataToLogFile = true;
343+
};
344+
final logHandler = LogHandler(logFile: logFile);
345+
346+
logHandler.save(data: const {});
347+
expect(readLogFile, isFalse);
348+
expect(wroteDataToLogFile, isTrue);
349+
});
350+
351+
test('trims the log file to its newest records', () {
352+
final logFile = fs.file('trim.log')..createSync();
353+
const trimSize = 1000;
354+
final logHandler = LogHandler(logFile: logFile, trimSize: trimSize);
355+
356+
const count = 200;
357+
for (var i = 0; i < count; i++) {
358+
logHandler.save(data: {'i': i});
359+
expect(
360+
logFile.lengthSync(),
361+
lessThan(trimSize + 20),
362+
reason: 'The file is trimmed once past the trim size',
363+
);
364+
}
365+
366+
final lines = logFile.readAsLinesSync();
367+
final indexes = [
368+
for (final line in lines)
369+
(jsonDecode(line) as Map<String, Object?>)['i'] as int,
370+
];
371+
expect(indexes.last, count - 1);
372+
expect(indexes, [
373+
for (var i = count - indexes.length; i < count; i++) i,
374+
], reason: 'The newest records are kept, whole and in order');
375+
expect(
376+
fs.directory('.').listSync().map((e) => e.basename),
377+
isNot(contains(endsWith('.tmp'))),
378+
reason: 'The temporary file is renamed over the log file',
379+
);
380+
});
381+
382+
test('a failed trim keeps the log file and still saves', () {
383+
final fs = MemoryFileSystem.test(
384+
opHandle: (context, operation) {
385+
if (context.endsWith('.tmp') && operation == FileSystemOp.write) {
386+
throw FileSystemException('write failed', context);
387+
}
388+
},
389+
);
390+
final logFile = fs.file('trim.log')..createSync();
391+
const trimSize = 1000;
392+
final logHandler = LogHandler(logFile: logFile, trimSize: trimSize);
393+
394+
const count = 200;
395+
for (var i = 0; i < count; i++) {
396+
logHandler.save(data: {'i': i});
397+
}
398+
399+
final lines = logFile.readAsLinesSync();
400+
expect(lines, hasLength(count), reason: 'Every record is still saved');
401+
expect(jsonDecode(lines.last), {'i': count - 1});
402+
expect(
403+
fs.directory('.').listSync().map((e) => e.basename),
404+
isNot(contains(endsWith('.tmp'))),
405+
reason: 'The temporary file is deleted after a failed trim',
406+
);
407+
});
408+
327409
test('Catching cast errors for each log record silently', () async {
328410
// Write a json array to the log file which will cause
329411
// a cast error when parsing each line

‎pkgs/unified_analytics/test/unified_analytics_test.dart‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1341,7 +1341,7 @@ ${initialTool.label}=$dateStamp,$toolsMessageVersion
13411341
}
13421342

13431343
expect(
1344-
logFile.readAsLinesSync().length,
1344+
analytics.logFileStats()!.recordCount,
13451345
kLogFileLength,
13461346
reason: 'The number of events should be capped at $kLogFileLength',
13471347
);

0 commit comments

Comments
 (0)