Skip to content

Prepare client side aggregation #133

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ogaca-dd merged 14 commits into from
Oct 8, 2020
Merged

Prepare client side aggregation #133

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
a8539c9
Introduce Stats class.
ogaca-dd Aug 25, 2020
49e1e71
Use StatsMetric in MetricSerializer
ogaca-dd Aug 25, 2020
a824f82
Use StatsEvent in EventSerializer
ogaca-dd Aug 25, 2020
725898b
Use StatsServiceCheck in ServiceCheckSerializer
ogaca-dd Aug 25, 2020
ab18f95
Remove generics in DogStatsdService except for Set
ogaca-dd Aug 26, 2020
6f38849
Replace BufferBuilder by StatsRouter in StatsBufferize
ogaca-dd Aug 26, 2020
d0036fe
Use Stats inside MetricsSender.
ogaca-dd Aug 26, 2020
9b2f008
Do not used a pool for SerializedMetric.
ogaca-dd Aug 26, 2020
22d7fda
Use a memory pool for Stats.
ogaca-dd Aug 26, 2020
b18dcc9
Remove missing namespace StatsdClient.Aggregator
ogaca-dd Aug 26, 2020
d5b7edc
Remove `Tags` from `Stats`.
ogaca-dd Aug 28, 2020
175353f
Fix StatsBufferize
ogaca-dd Aug 28, 2020
502128b
Merge branch 'olivierg/object-pool' of github.com:DataDog/dogstatsd-c…
ogaca-dd Aug 31, 2020
f72bada
Fix compilation
ogaca-dd Aug 31, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion src/StatsdClient/Bufferize/IStatsBufferizeFactory.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,15 @@ namespace StatsdClient.Bufferize
internal interface IStatsBufferizeFactory
{
StatsBufferize CreateStatsBufferize(
BufferBuilder bufferBuilder,
StatsRouter statsRouter,
int workerMaxItemCount,
TimeSpan? blockingQueueTimeout,
TimeSpan maxIdleWaitBeforeSending);

StatsRouter CreateStatsRouter(
Serializers serializers,
BufferBuilder bufferBuilder);

ITransport CreateUDPTransport(IPEndPoint endPoint);

ITransport CreateUnixDomainSocketTransport(
Expand Down
37 changes: 17 additions & 20 deletions src/StatsdClient/Bufferize/StatsBufferize.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
using System;
using System.Threading.Tasks;
using StatsdClient.Statistic;
using StatsdClient.Worker;

namespace StatsdClient.Bufferize
Expand All @@ -9,26 +10,26 @@ namespace StatsdClient.Bufferize
/// </summary>
internal class StatsBufferize : IDisposable
{
private readonly AsynchronousWorker<SerializedMetric> _worker;
private readonly AsynchronousWorker<Stats> _worker;

public StatsBufferize(
BufferBuilder bufferBuilder,
StatsRouter statsRouter,
int workerMaxItemCount,
TimeSpan? blockingQueueTimeout,
TimeSpan maxIdleWaitBeforeSending)
{
var handler = new WorkerHandler(bufferBuilder, maxIdleWaitBeforeSending);
var handler = new WorkerHandler(statsRouter, maxIdleWaitBeforeSending);

// `handler` (and also `bufferBuilder`) do not need to be thread safe as long as workerMaxItemCount is 1.
this._worker = new AsynchronousWorker<SerializedMetric>(
// `handler` (and also `statsRouter`) do not need to be thread safe as long as `workerThreadCount` is 1.
this._worker = new AsynchronousWorker<Stats>(
handler,
new Waiter(),
1,
workerThreadCount: 1,
workerMaxItemCount,
blockingQueueTimeout);
}

public bool Send(SerializedMetric serializedMetric)
public bool Send(Stats serializedMetric)
{
if (!this._worker.TryEnqueue(serializedMetric))
{
Expand All @@ -44,27 +45,23 @@ public void Dispose()
this._worker.Dispose();
}

private class WorkerHandler : IAsynchronousWorkerHandler<SerializedMetric>
private class WorkerHandler : IAsynchronousWorkerHandler<Stats>
{
private readonly BufferBuilder _bufferBuilder;
private readonly StatsRouter _statsRouter;
private readonly TimeSpan _maxIdleWaitBeforeSending;
private System.Diagnostics.Stopwatch _stopwatch;

public WorkerHandler(BufferBuilder bufferBuilder, TimeSpan maxIdleWaitBeforeSending)
public WorkerHandler(StatsRouter statsRouter, TimeSpan maxIdleWaitBeforeSending)
{
_bufferBuilder = bufferBuilder;
_statsRouter = statsRouter;
_maxIdleWaitBeforeSending = maxIdleWaitBeforeSending;
}

public void OnNewValue(SerializedMetric serializedMetric)
public void OnNewValue(Stats stats)
{
using (serializedMetric)
using (stats)
{
if (!_bufferBuilder.Add(serializedMetric))
{
throw new InvalidOperationException($"The metric size exceeds the buffer capacity: {serializedMetric.ToString()}");
}

_statsRouter.Route(stats);
_stopwatch = null;
}
}
Expand All @@ -78,7 +75,7 @@ public bool OnIdle()

if (_stopwatch.ElapsedMilliseconds > _maxIdleWaitBeforeSending.TotalMilliseconds)
{
this._bufferBuilder.HandleBufferAndReset();
this._statsRouter.TryFlush();

return true;
}
Expand All @@ -88,7 +85,7 @@ public bool OnIdle()

public void OnShutdown()
{
this._bufferBuilder.HandleBufferAndReset();
this._statsRouter.TryFlush();
}
}
}
Expand Down
11 changes: 9 additions & 2 deletions src/StatsdClient/Bufferize/StatsBufferizeFactory.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,18 +8,25 @@ namespace StatsdClient.Bufferize
internal class StatsBufferizeFactory : IStatsBufferizeFactory
{
public StatsBufferize CreateStatsBufferize(
BufferBuilder bufferBuilder,
StatsRouter statsRouter,
int workerMaxItemCount,
TimeSpan? blockingQueueTimeout,
TimeSpan maxIdleWaitBeforeSending)
{
return new StatsBufferize(
bufferBuilder,
statsRouter,
workerMaxItemCount,
blockingQueueTimeout,
maxIdleWaitBeforeSending);
}

public StatsRouter CreateStatsRouter(
Serializers serializers,
BufferBuilder bufferBuilder)
{
return new StatsRouter(serializers, bufferBuilder);
}

public ITransport CreateUDPTransport(IPEndPoint endPoint)
{
return new UDPTransport(endPoint);
Expand Down
31 changes: 20 additions & 11 deletions src/StatsdClient/DogStatsdService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using System;
using System.Globalization;
using StatsdClient.Bufferize;

namespace StatsdClient
Expand Down Expand Up @@ -71,8 +72,7 @@ public void Event(string title, string text, string alertType = null, string agg
/// <param name="value">A given delta.</param>
/// <param name="sampleRate">Percentage of metric to be sent.</param>
/// <param name="tags">Array of tags to be added to the data.</param>
/// <typeparam name="T">The type of the value.</typeparam>
public void Counter<T>(string statName, T value, double sampleRate = 1.0, string[] tags = null)
public void Counter(string statName, double value, double sampleRate = 1.0, string[] tags = null)
Copy link
Copy Markdown
Member

@truthbk truthbk Oct 6, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The generic to double should not break the API as an implicit cast would happen here. Correct?

Copy link
Copy Markdown
Contributor Author

@ogaca-dd ogaca-dd Oct 6, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There is an implicit cast to double when no information is lost.

The problem is that T can be BitInteger https://docs.microsoft.com/en-us/dotnet/api/system.numerics.biginteger?view=netcore-3.1 and there is no implicit conversion to double. It can also be the string "42" as the previous code only assumes that there is a ToString() method which is available for all objects in C#.

This is a breaking change even if it is unlikely to break anything.

Copy link
Copy Markdown
Member

@truthbk truthbk Oct 7, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you plan to release this in a new major?

Copy link
Copy Markdown
Contributor Author

@ogaca-dd ogaca-dd Oct 8, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am going to release this in a new major as it may break compilation in some cases.

{
_metricsSender?.SendMetric(MetricType.Count, statName, value, sampleRate, tags);
}
Expand Down Expand Up @@ -108,8 +108,7 @@ public void Decrement(string statName, int value = 1, double sampleRate = 1.0, s
/// <param name="value">The value of the gauge.</param>
/// <param name="sampleRate">Percentage of metric to be sent.</param>
/// <param name="tags">Array of tags to be added to the data.</param>
/// <typeparam name="T">The type of the value.</typeparam>
public void Gauge<T>(string statName, T value, double sampleRate = 1.0, string[] tags = null)
public void Gauge(string statName, double value, double sampleRate = 1.0, string[] tags = null)
{
_metricsSender?.SendMetric(MetricType.Gauge, statName, value, sampleRate, tags);
}
Expand All @@ -121,8 +120,7 @@ public void Gauge<T>(string statName, T value, double sampleRate = 1.0, string[]
/// <param name="value">The value of the histogram.</param>
/// <param name="sampleRate">Percentage of metric to be sent.</param>
/// <param name="tags">Array of tags to be added to the data.</param>
/// <typeparam name="T">The type of the value.</typeparam>
public void Histogram<T>(string statName, T value, double sampleRate = 1.0, string[] tags = null)
public void Histogram(string statName, double value, double sampleRate = 1.0, string[] tags = null)
{
_metricsSender?.SendMetric(MetricType.Histogram, statName, value, sampleRate, tags);
}
Expand All @@ -134,8 +132,7 @@ public void Histogram<T>(string statName, T value, double sampleRate = 1.0, stri
/// <param name="value">The value of the distribution.</param>
/// <param name="sampleRate">Percentage of metric to be sent.</param>
/// <param name="tags">Array of tags to be added to the data.</param>
/// <typeparam name="T">The type of the value.</typeparam>
public void Distribution<T>(string statName, T value, double sampleRate = 1.0, string[] tags = null)
public void Distribution(string statName, double value, double sampleRate = 1.0, string[] tags = null)
{
_metricsSender?.SendMetric(MetricType.Distribution, statName, value, sampleRate, tags);
}
Expand All @@ -150,7 +147,20 @@ public void Distribution<T>(string statName, T value, double sampleRate = 1.0, s
/// <typeparam name="T">The type of the value.</typeparam>
public void Set<T>(string statName, T value, double sampleRate = 1.0, string[] tags = null)
{
_metricsSender?.SendMetric(MetricType.Set, statName, value, sampleRate, tags);
var strValue = string.Format(CultureInfo.InvariantCulture, "{0}", value);
_metricsSender?.SendSetMetric(statName, strValue, sampleRate, tags);
}

/// <summary>
/// Records a value for the specified set.
/// </summary>
/// <param name="statName">The name of the metric.</param>
/// <param name="value">The value to set.</param>
/// <param name="sampleRate">Percentage of metric to be sent.</param>
/// <param name="tags">Array of tags to be added to the data.</param>
public void Set(string statName, string value, double sampleRate = 1.0, string[] tags = null)
{
_metricsSender?.SendSetMetric(statName, value, sampleRate, tags);
}

/// <summary>
Expand All @@ -160,8 +170,7 @@ public void Set<T>(string statName, T value, double sampleRate = 1.0, string[] t
/// <param name="value">The time in millisecond.</param>
/// <param name="sampleRate">Percentage of metric to be sent.</param>
/// <param name="tags">Array of tags to be added to the data.</param>
/// <typeparam name="T">The type of value parameter.</typeparam>
public void Timer<T>(string statName, T value, double sampleRate = 1.0, string[] tags = null)
public void Timer(string statName, double value, double sampleRate = 1.0, string[] tags = null)
{
_metricsSender?.SendMetric(MetricType.Timing, statName, value, sampleRate, tags);
}
Expand Down
35 changes: 20 additions & 15 deletions src/StatsdClient/Dogstatsd.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,9 +93,8 @@ public static void Event(
/// <param name="value">A given delta.</param>
/// <param name="sampleRate">Percentage of metric to be sent.</param>
/// <param name="tags">Array of tags to be added to the data.</param>
/// <typeparam name="T">The type of the value.</typeparam>
public static void Counter<T>(string statName, T value, double sampleRate = 1.0, string[] tags = null) =>
_dogStatsdService.Counter<T>(statName: statName, value: value, sampleRate: sampleRate, tags: tags);
public static void Counter(string statName, double value, double sampleRate = 1.0, string[] tags = null) =>
_dogStatsdService.Counter(statName: statName, value: value, sampleRate: sampleRate, tags: tags);

/// <summary>
/// Increments the specified counter.
Expand Down Expand Up @@ -124,9 +123,8 @@ public static void Decrement(string statName, int value = 1, double sampleRate =
/// <param name="value">The value of the gauge.</param>
/// <param name="sampleRate">Percentage of metric to be sent.</param>
/// <param name="tags">Array of tags to be added to the data.</param>
/// <typeparam name="T">The type of the value.</typeparam>
public static void Gauge<T>(string statName, T value, double sampleRate = 1.0, string[] tags = null) =>
_dogStatsdService.Gauge<T>(statName: statName, value: value, sampleRate: sampleRate, tags: tags);
public static void Gauge(string statName, double value, double sampleRate = 1.0, string[] tags = null) =>
_dogStatsdService.Gauge(statName: statName, value: value, sampleRate: sampleRate, tags: tags);

/// <summary>
/// Records a value for the specified named histogram.
Expand All @@ -135,9 +133,8 @@ public static void Gauge<T>(string statName, T value, double sampleRate = 1.0, s
/// <param name="value">The value of the histogram.</param>
/// <param name="sampleRate">Percentage of metric to be sent.</param>
/// <param name="tags">Array of tags to be added to the data.</param>
/// <typeparam name="T">The type of the value.</typeparam>
public static void Histogram<T>(string statName, T value, double sampleRate = 1.0, string[] tags = null) =>
_dogStatsdService.Histogram<T>(statName: statName, value: value, sampleRate: sampleRate, tags: tags);
public static void Histogram(string statName, double value, double sampleRate = 1.0, string[] tags = null) =>
_dogStatsdService.Histogram(statName: statName, value: value, sampleRate: sampleRate, tags: tags);

/// <summary>
/// Records a value for the specified named distribution.
Expand All @@ -146,9 +143,8 @@ public static void Histogram<T>(string statName, T value, double sampleRate = 1.
/// <param name="value">The value of the distribution.</param>
/// <param name="sampleRate">Percentage of metric to be sent.</param>
/// <param name="tags">Array of tags to be added to the data.</param>
/// <typeparam name="T">The type of the value.</typeparam>
public static void Distribution<T>(string statName, T value, double sampleRate = 1.0, string[] tags = null) =>
_dogStatsdService.Distribution<T>(statName: statName, value: value, sampleRate: sampleRate, tags: tags);
public static void Distribution(string statName, double value, double sampleRate = 1.0, string[] tags = null) =>
_dogStatsdService.Distribution(statName: statName, value: value, sampleRate: sampleRate, tags: tags);

/// <summary>
/// Records a value for the specified set.
Expand All @@ -161,16 +157,25 @@ public static void Distribution<T>(string statName, T value, double sampleRate =
public static void Set<T>(string statName, T value, double sampleRate = 1.0, string[] tags = null) =>
_dogStatsdService.Set<T>(statName: statName, value: value, sampleRate: sampleRate, tags: tags);

/// <summary>
/// Records a value for the specified set.
/// </summary>
/// <param name="statName">The name of the metric.</param>
/// <param name="value">The value to set.</param>
/// <param name="sampleRate">Percentage of metric to be sent.</param>
/// <param name="tags">Array of tags to be added to the data.</param>
public static void Set(string statName, string value, double sampleRate = 1.0, string[] tags = null) =>
_dogStatsdService.Set(statName: statName, value: value, sampleRate: sampleRate, tags: tags);

/// <summary>
/// Records an execution time in milliseconds.
/// </summary>
/// <param name="statName">The name of the metric.</param>
/// <param name="value">The time in millisecond.</param>
/// <param name="sampleRate">Percentage of metric to be sent.</param>
/// <param name="tags">Array of tags to be added to the data.</param>
/// <typeparam name="T">The type of value parameter.</typeparam>
public static void Timer<T>(string statName, T value, double sampleRate = 1.0, string[] tags = null) =>
_dogStatsdService.Timer<T>(statName: statName, value: value, sampleRate: sampleRate, tags: tags);
public static void Timer(string statName, double value, double sampleRate = 1.0, string[] tags = null) =>
_dogStatsdService.Timer(statName: statName, value: value, sampleRate: sampleRate, tags: tags);

/// <summary>
/// Creates a timer that records the execution time until Dispose is called on the returned value.
Expand Down
24 changes: 14 additions & 10 deletions src/StatsdClient/IDogStatsd.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,7 @@ public interface IDogStatsd : IDisposable
/// <param name="value">A given delta.</param>
/// <param name="sampleRate">Percentage of metric to be sent.</param>
/// <param name="tags">Array of tags to be added to the data.</param>
/// <typeparam name="T">The type of the value.</typeparam>
void Counter<T>(string statName, T value, double sampleRate = 1, string[] tags = null);
void Counter(string statName, double value, double sampleRate = 1, string[] tags = null);

/// <summary>
/// Decrements the specified counter.
Expand Down Expand Up @@ -69,8 +68,7 @@ void Event(
/// <param name="value">The value of the gauge.</param>
/// <param name="sampleRate">Percentage of metric to be sent.</param>
/// <param name="tags">Array of tags to be added to the data.</param>
/// <typeparam name="T">The type of the value.</typeparam>
void Gauge<T>(string statName, T value, double sampleRate = 1, string[] tags = null);
void Gauge(string statName, double value, double sampleRate = 1, string[] tags = null);

/// <summary>
/// Records a value for the specified named histogram.
Expand All @@ -79,8 +77,7 @@ void Event(
/// <param name="value">The value of the histogram.</param>
/// <param name="sampleRate">Percentage of metric to be sent.</param>
/// <param name="tags">Array of tags to be added to the data.</param>
/// <typeparam name="T">The type of the value.</typeparam>
void Histogram<T>(string statName, T value, double sampleRate = 1, string[] tags = null);
void Histogram(string statName, double value, double sampleRate = 1, string[] tags = null);

/// <summary>
/// Records a value for the specified named distribution.
Expand All @@ -89,8 +86,7 @@ void Event(
/// <param name="value">The value of the distribution.</param>
/// <param name="sampleRate">Percentage of metric to be sent.</param>
/// <param name="tags">Array of tags to be added to the data.</param>
/// <typeparam name="T">The type of the value.</typeparam>
void Distribution<T>(string statName, T value, double sampleRate = 1, string[] tags = null);
void Distribution(string statName, double value, double sampleRate = 1, string[] tags = null);

/// <summary>
/// Increments the specified counter.
Expand All @@ -111,6 +107,15 @@ void Event(
/// <typeparam name="T">The type of the value.</typeparam>
void Set<T>(string statName, T value, double sampleRate = 1, string[] tags = null);

/// <summary>
/// Records a value for the specified set.
/// </summary>
/// <param name="statName">The name of the metric.</param>
/// <param name="value">The value to set.</param>
/// <param name="sampleRate">Percentage of metric to be sent.</param>
/// <param name="tags">Array of tags to be added to the data.</param>
void Set(string statName, string value, double sampleRate = 1, string[] tags = null);

/// <summary>
/// Creates a timer that records the execution time until Dispose is called on the returned value.
/// </summary>
Expand Down Expand Up @@ -147,8 +152,7 @@ void Event(
/// <param name="value">The time in millisecond.</param>
/// <param name="sampleRate">Percentage of metric to be sent.</param>
/// <param name="tags">Array of tags to be added to the data.</param>
/// <typeparam name="T">The type of value parameter.</typeparam>
void Timer<T>(string statName, T value, double sampleRate = 1, string[] tags = null);
void Timer(string statName, double value, double sampleRate = 1, string[] tags = null);

/// <summary>
/// Records a run status for the specified named service check.
Expand Down
Loading