Skip to content

Commit 3751e61

Browse files
aspire-repo-bot[bot]JamesNKCopilot
authored
[release/13.6] Reduce dashboard CPU under sustained metric ingestion (#20747)
Backport of #20736 to release/13.6 /cc @JamesNK ## Customer Impact In Aspire 13.6, long-running AppHosts with sustained metric ingestion can consume steadily increasing dashboard CPU even under a fixed or idle workload. A reported 15-project AppHost reached approximately 70% of one core after 6.8 hours, with significant SQLite churn and database growth. ## Testing Passed 59 targeted SQLite metrics and persistence tests, excluding quarantined and outerloop tests. The source PR also validated the before/after ingestion benchmarks and smoke-tested all 12 existing metrics-query benchmark cases. Backport PR CI is currently in progress. ## Risk Low. The change is localized to SQLite metric-retention cleanup, preserves the existing retention limit and cascading deletes, and adds focused coverage for retention, database reopening, and transaction-failure recovery. It makes no public API or database schema changes. ## Regression? Yes — introduced in 13.6 with SQLite-backed telemetry in #18924. Co-authored-by: James Newton-King <[email protected]> Co-authored-by: Copilot <[email protected]>
1 parent 660f92c commit 3751e61

5 files changed

Lines changed: 270 additions & 140 deletions

File tree

‎benchmarks/Aspire.Dashboard.Benchmarks/TelemetryRepositoryMetricsBenchmarks.cs‎

Lines changed: 83 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
using BenchmarkDotNet.Configs;
1313
using BenchmarkDotNet.Diagnosers;
1414
using BenchmarkDotNet.Jobs;
15+
using BenchmarkDotNet.Toolchains.InProcess.NoEmit;
1516
using Google.Protobuf;
1617
using Google.Protobuf.Collections;
1718
using Microsoft.Extensions.Logging.Abstractions;
@@ -23,14 +24,14 @@
2324
namespace Aspire.Dashboard.Benchmarks;
2425

2526
[MemoryDiagnoser]
26-
[ThreadingDiagnoser]
2727
[Config(typeof(Config))]
2828
public class TelemetryRepositoryMetricsBenchmarks
2929
{
3030
private const int MetricSamplesPerBatch = 100;
3131
private const string MetricMeterName = "benchmark-meter";
3232
private const string MetricInstrumentName = "benchmark.metric";
3333
private const string HistogramMetricInstrumentName = "benchmark.histogram";
34+
private static readonly double[] s_histogramObservations = [5, 25, 75, 150];
3435
private static readonly TimeSpan s_metricDataDuration = TimeSpan.FromHours(6);
3536
private static readonly TimeSpan s_metricDisplayDuration = TimeSpan.FromHours(6);
3637
private static readonly TimeSpan s_metricInterval = TimeSpan.FromSeconds(2);
@@ -43,6 +44,8 @@ public class TelemetryRepositoryMetricsBenchmarks
4344
private DashboardSqliteDatabase _database = null!;
4445
private SqliteTelemetryRepository _queryRepository = null!;
4546
private IReadOnlyList<MetricDimensionCursor> _incrementalCursors = null!;
47+
private RepeatedField<ResourceMetrics> _ingestionMetrics = null!;
48+
private HistogramDataPoint[] _ingestionPoints = null!;
4649

4750
[Params(1, 5)]
4851
public int DimensionCount { get; set; }
@@ -65,13 +68,50 @@ public class TelemetryRepositoryMetricsBenchmarks
6568
[GlobalSetup(Target = nameof(GetHistogramMetricsIncrementalRollup))]
6669
public Task SetupHistogramMetricsIncrementalRollup() => SetupIncrementalAsync(isHistogram: true, HistogramMetricInstrumentName);
6770

68-
private async Task SetupAsync(bool isHistogram)
71+
[GlobalSetup(Target = nameof(AddHistogramMetricsAtCapacity))]
72+
public async Task SetupHistogramMetricsIngestion()
73+
{
74+
await InitializeRepositoryAsync();
75+
var retainedPointCount = new TelemetryLimitOptions().MaxMetricsCount;
76+
var startTime = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
77+
var exemplars = s_histogramObservations.Select(value => CreateMetricExemplar(startTime, value)).ToArray();
78+
var context = new AddContext();
79+
foreach (var batch in CreateLongDurationMetricBatches(DimensionCount, isHistogram: true, retainedPointCount))
80+
{
81+
foreach (var point in batch[0].ScopeMetrics[0].Metrics[0].Histogram.DataPoints)
82+
{
83+
// Cumulative reservoirs replay unrefreshed exemplars. Include them in every export to
84+
// exercise point eviction and its cascading exemplar deletes in a long-running app.
85+
point.Exemplars.Clear();
86+
point.Exemplars.Add(exemplars);
87+
}
88+
await _queryRepository.AddMetricsAsync(context, batch);
89+
_ingestionMetrics = batch;
90+
}
91+
if (context.FailureCount > 0)
92+
{
93+
throw new InvalidOperationException($"Failed to add {context.FailureCount} benchmark metric points.");
94+
}
95+
96+
var points = _ingestionMetrics[0].ScopeMetrics[0].Metrics[0].Histogram.DataPoints;
97+
_ingestionPoints = points.TakeLast(DimensionCount).ToArray();
98+
points.Clear();
99+
points.Add(_ingestionPoints);
100+
}
101+
102+
private async Task InitializeRepositoryAsync()
69103
{
70104
_temporaryDirectory = Directory.CreateTempSubdirectory("aspire-dashboard-metrics-benchmark-").FullName;
71105
_database = new DashboardSqliteDatabase(Path.Combine(_temporaryDirectory, "query.db"));
106+
await _database.InitializeSchemaAsync(CancellationToken.None);
72107
_queryRepository = CreateRepository(_database);
108+
}
109+
110+
private async Task SetupAsync(bool isHistogram)
111+
{
112+
await InitializeRepositoryAsync();
73113
var addContext = new AddContext();
74-
foreach (var batch in CreateLongDurationMetricBatches(DimensionCount, isHistogram))
114+
foreach (var batch in CreateLongDurationMetricBatches(DimensionCount, isHistogram, (int)(s_metricDataDuration / s_metricInterval)))
75115
{
76116
await _queryRepository.AddMetricsAsync(addContext, batch);
77117
}
@@ -84,7 +124,7 @@ private async Task SetupAsync(bool isHistogram)
84124
private async Task SetupIncrementalAsync(bool isHistogram, string instrumentName)
85125
{
86126
await SetupAsync(isHistogram);
87-
var instrument = GetLongDurationInstrument(instrumentName, s_metricDataPointInterval);
127+
var instrument = await GetLongDurationInstrumentAsync(instrumentName, s_metricDataPointInterval);
88128
_incrementalCursors = instrument.Dimensions.Select(dimension =>
89129
{
90130
var latestValue = dimension.Values[^1];
@@ -106,57 +146,78 @@ public void Cleanup()
106146
}
107147

108148
[Benchmark(Description = "TelemetryRepository: query 6h metrics display")]
109-
public int GetMetricsLongDuration()
149+
public async Task<int> GetMetricsLongDuration()
110150
{
111-
var instrument = GetLongDurationInstrument(MetricInstrumentName);
151+
var instrument = await GetLongDurationInstrumentAsync(MetricInstrumentName);
112152

113153
return instrument.Dimensions.Sum(dimension => dimension.Values.Count);
114154
}
115155

116156
[Benchmark(Description = "TelemetryRepository: query 6h histogram metrics display")]
117-
public int GetHistogramMetricsLongDuration()
157+
public async Task<int> GetHistogramMetricsLongDuration()
118158
{
119-
var instrument = GetLongDurationInstrument(HistogramMetricInstrumentName);
159+
var instrument = await GetLongDurationInstrumentAsync(HistogramMetricInstrumentName);
120160

121161
return instrument.Dimensions.Sum(dimension =>
122162
dimension.Values.Count + dimension.Values.Sum(value => value.Exemplars.Count));
123163
}
124164

125165
[Benchmark(Description = "TelemetryRepository: query 6h metrics with dashboard rollup")]
126-
public int GetMetricsLongDurationRollup()
166+
public async Task<int> GetMetricsLongDurationRollup()
127167
{
128-
var instrument = GetLongDurationInstrument(MetricInstrumentName, s_metricDataPointInterval);
168+
var instrument = await GetLongDurationInstrumentAsync(MetricInstrumentName, s_metricDataPointInterval);
129169

130170
return instrument.Dimensions.Sum(dimension => dimension.Values.Count);
131171
}
132172

133173
[Benchmark(Description = "TelemetryRepository: query 6h histogram metrics with dashboard rollup")]
134-
public int GetHistogramMetricsLongDurationRollup()
174+
public async Task<int> GetHistogramMetricsLongDurationRollup()
135175
{
136-
var instrument = GetLongDurationInstrument(HistogramMetricInstrumentName, s_metricDataPointInterval);
176+
var instrument = await GetLongDurationInstrumentAsync(HistogramMetricInstrumentName, s_metricDataPointInterval);
137177

138178
return instrument.Dimensions.Sum(dimension =>
139179
dimension.Values.Count + dimension.Values.Sum(value => value.Exemplars.Count));
140180
}
141181

142182
[Benchmark(Description = "TelemetryRepository: query incremental metrics with dashboard rollup")]
143-
public int GetMetricsIncrementalRollup()
183+
public async Task<int> GetMetricsIncrementalRollup()
144184
{
145-
var instrument = GetLongDurationInstrument(MetricInstrumentName, s_metricDataPointInterval, _incrementalCursors);
185+
var instrument = await GetLongDurationInstrumentAsync(MetricInstrumentName, s_metricDataPointInterval, _incrementalCursors);
146186

147187
return instrument.Dimensions.Sum(dimension => dimension.Values.Count);
148188
}
149189

150190
[Benchmark(Description = "TelemetryRepository: query incremental histogram metrics with dashboard rollup")]
151-
public int GetHistogramMetricsIncrementalRollup()
191+
public async Task<int> GetHistogramMetricsIncrementalRollup()
152192
{
153-
var instrument = GetLongDurationInstrument(HistogramMetricInstrumentName, s_metricDataPointInterval, _incrementalCursors);
193+
var instrument = await GetLongDurationInstrumentAsync(HistogramMetricInstrumentName, s_metricDataPointInterval, _incrementalCursors);
154194

155195
return instrument.Dimensions.Sum(dimension =>
156196
dimension.Values.Count + dimension.Values.Sum(value => value.Exemplars.Count));
157197
}
158198

159-
private OtlpInstrumentData GetLongDurationInstrument(
199+
[Benchmark(Description = "TelemetryRepository: ingest histogram metrics at retention capacity")]
200+
public async Task<int> AddHistogramMetricsAtCapacity()
201+
{
202+
foreach (var point in _ingestionPoints)
203+
{
204+
point.Count++;
205+
point.Sum += s_histogramObservations[0];
206+
point.BucketCounts[0]++;
207+
point.TimeUnixNano += (ulong)s_metricInterval.Ticks * 100;
208+
}
209+
210+
var context = new AddContext();
211+
await _queryRepository.AddMetricsAsync(context, _ingestionMetrics);
212+
if (context.SuccessCount != DimensionCount || context.FailureCount > 0)
213+
{
214+
throw new InvalidOperationException($"Expected {DimensionCount} benchmark metric points, added {context.SuccessCount} and rejected {context.FailureCount}.");
215+
}
216+
217+
return context.SuccessCount;
218+
}
219+
220+
private async Task<OtlpInstrumentData> GetLongDurationInstrumentAsync(
160221
string instrumentName,
161222
TimeSpan? dataPointInterval = null,
162223
IReadOnlyList<MetricDimensionCursor>? dimensionCursors = null)
@@ -165,7 +226,7 @@ private OtlpInstrumentData GetLongDurationInstrument(
165226
?? throw new InvalidOperationException($"Unable to find the benchmark metric '{instrumentName}' end time.");
166227

167228
// Match the dashboard metrics display query, which includes one preceding rollup for histogram calculations.
168-
return _queryRepository.GetInstrument(new GetInstrumentRequest
229+
return await _queryRepository.GetInstrumentAsync(new GetInstrumentRequest
169230
{
170231
ResourceKey = s_metricResourceKey,
171232
MeterName = MetricMeterName,
@@ -175,7 +236,7 @@ private OtlpInstrumentData GetLongDurationInstrument(
175236
DataPointInterval = dataPointInterval,
176237
PopulateExemplarAttributes = false,
177238
DimensionCursors = dimensionCursors ?? []
178-
}) ?? throw new InvalidOperationException($"Unable to find the benchmark metric '{instrumentName}'.");
239+
}, cancellationToken: CancellationToken.None) ?? throw new InvalidOperationException($"Unable to find the benchmark metric '{instrumentName}'.");
179240
}
180241

181242
private static SqliteTelemetryRepository CreateRepository(DashboardSqliteDatabase database)
@@ -189,13 +250,12 @@ private static SqliteTelemetryRepository CreateRepository(DashboardSqliteDatabas
189250
[]);
190251
}
191252

192-
private static IEnumerable<RepeatedField<ResourceMetrics>> CreateLongDurationMetricBatches(int dimensionCount, bool isHistogram)
253+
private static IEnumerable<RepeatedField<ResourceMetrics>> CreateLongDurationMetricBatches(int dimensionCount, bool isHistogram, int totalSampleCount)
193254
{
194255
var startTime = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
195-
double[] observations = [5, 25, 75, 150];
256+
var observations = s_histogramObservations;
196257
var bucketCounts = Enumerable.Range(0, dimensionCount).Select(_ => new ulong[observations.Length]).ToArray();
197258
var sums = new double[dimensionCount];
198-
var totalSampleCount = (int)(s_metricDataDuration / s_metricInterval);
199259

200260
for (var firstSampleIndex = 0; firstSampleIndex < totalSampleCount; firstSampleIndex += MetricSamplesPerBatch)
201261
{
@@ -386,7 +446,7 @@ private sealed class Config : ManualConfig
386446
{
387447
public Config()
388448
{
389-
AddJob(Job.Dry);
449+
AddJob(Job.Dry.WithToolchain(InProcessNoEmitToolchain.Instance).DontEnforcePowerPlan());
390450

391451
AddDiagnoser(MemoryDiagnoser.Default);
392452
}

‎docs/specs/dashboard-persistence.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -295,6 +295,8 @@ The Dashboard applies these default ingestion limits:
295295

296296
The oldest logs, traces, and metric points are removed when their limits are exceeded. Fixed limits of 10,000 also apply to resource views per resource, scopes per database, instruments per resource, and dimensions per instrument; additional identities are rejected.
297297

298+
The `AddHistogramMetricsAtCapacity` benchmark in [TelemetryRepositoryMetricsBenchmarks](../../benchmarks/Aspire.Dashboard.Benchmarks/TelemetryRepositoryMetricsBenchmarks.cs) measures steady-state ingestion with one or five dimensions already at the default retention limit. Each invocation adds one cumulative histogram point per dimension with four replayed exemplars, exercising eviction and cascading exemplar deletes. Payload construction and history population are outside the measured operation.
299+
298300
Console logs are persisted only after their stream is viewed or exported. The frontend keeps up to `Dashboard:Frontend:MaxConsoleLogCount` entries in memory, which defaults to 100,000. The same limit applies across all console logs in the database, with the oldest entries removed when the limit is exceeded. Historical runs can therefore omit uncaptured logs.
299301

300302
### Database file size

‎src/Aspire.Dashboard/Otlp/Storage/SqliteTelemetryRepository.Metrics.Writes.cs‎

Lines changed: 20 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -389,6 +389,7 @@ FROM updates
389389
foreach (var point in pointBatch.Inserts)
390390
{
391391
point.Context.SuccessCount += point.SourcePointCount;
392+
point.Dimension.PointCount++;
392393

393394
if (ReferenceEquals(point.Dimension.PendingPoint, point))
394395
{
@@ -423,6 +424,7 @@ private MetricDimensionState GetOrAddMetricDimension(
423424
var dimensions = connection.Query<MetricDimensionStateRecord>("""
424425
SELECT
425426
d.dimension_id AS DimensionId,
427+
(SELECT COUNT(*) FROM telemetry_metric_points WHERE dimension_id = d.dimension_id) AS PointCount,
426428
a.attribute_key AS AttributeKey,
427429
a.attribute_value AS AttributeValue,
428430
p.point_id AS PointId,
@@ -451,6 +453,7 @@ LIMIT 1
451453
return new MetricDimensionState
452454
{
453455
DimensionId = group.Key,
456+
PointCount = first.PointCount,
454457
Attributes = group
455458
.Where(record => record.AttributeKey is not null)
456459
.Select(record => KeyValuePair.Create(record.AttributeKey!, record.AttributeValue!))
@@ -733,22 +736,27 @@ INSERT OR IGNORE INTO telemetry_metric_exemplars (
733736

734737
private void TrimMetricDimensions(SqliteConnection connection, IDbTransaction transaction, IEnumerable<MetricDimensionState> dimensions)
735738
{
736-
foreach (var batch in dimensions.Chunk(MaxMetricPointBatchSize))
739+
foreach (var dimension in dimensions)
737740
{
741+
var removeCount = dimension.PointCount - _otlpContext.Options.MaxMetricsCount;
742+
if (removeCount <= 0)
743+
{
744+
continue;
745+
}
746+
747+
// Count cached points instead of ranking the entire history on every insert. The dimension-order
748+
// index lets SQLite visit only the oldest surplus points, even after the retention limit is reached.
738749
connection.Execute("""
739750
DELETE FROM telemetry_metric_points
740751
WHERE point_id IN (
741752
SELECT point_id
742-
FROM (
743-
SELECT
744-
point_id,
745-
ROW_NUMBER() OVER (PARTITION BY dimension_id ORDER BY point_id DESC) AS point_rank
746-
FROM telemetry_metric_points
747-
WHERE dimension_id IN @DimensionIds
748-
)
749-
WHERE point_rank > @MaxMetricsCount
753+
FROM telemetry_metric_points
754+
WHERE dimension_id = @DimensionId
755+
ORDER BY point_id
756+
LIMIT @RemoveCount
750757
);
751-
""", new { DimensionIds = batch.Select(dimension => dimension.DimensionId).ToArray(), _otlpContext.Options.MaxMetricsCount }, transaction);
758+
""", new { dimension.DimensionId, RemoveCount = removeCount }, transaction);
759+
dimension.PointCount -= removeCount;
752760
}
753761
}
754762

@@ -861,6 +869,7 @@ private sealed record PendingMetricDimensionAttribute(MetricDimensionState Dimen
861869
private sealed class MetricDimensionState
862870
{
863871
public long DimensionId { get; set; }
872+
public long PointCount { get; set; }
864873
public required KeyValuePair<string, string>[] Attributes { get; init; }
865874
public MetricPointRecord? LatestPoint { get; set; }
866875
public PendingMetricPoint? PendingPoint { get; set; }
@@ -940,6 +949,7 @@ private sealed class PendingMetricPoint
940949
internal sealed class MetricDimensionStateRecord
941950
{
942951
public required long DimensionId { get; init; }
952+
public required long PointCount { get; init; }
943953
public string? AttributeKey { get; init; }
944954
public string? AttributeValue { get; init; }
945955
public long? PointId { get; init; }

0 commit comments

Comments
 (0)