Skip to content

Store benchmark results as an artifact #1799

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
andrewlock merged 6 commits into from
Sep 23, 2021
Merged

Store benchmark results as an artifact #1799

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
b6add0e
Add json exporter to benchmarks
andrewlock Sep 20, 2021
987e042
Upload benchmark results as an artifact for later processing
andrewlock Sep 20, 2021
0a4679e
Ignore the build_data folder (used in CI)
andrewlock Sep 21, 2021
f4f370b
Add default exporters
andrewlock Sep 22, 2021
9441416
Remove manual exporters
andrewlock Sep 22, 2021
c59520e
Fix exporter
andrewlock Sep 22, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .azure-pipelines/ultimate-pipeline.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -689,6 +689,10 @@ stages:
continueOnError: true
displayName: RunBenchmarks

- publish: tracer/build_data/benchmarks
artifact: benchmarks_results
continueOnError: true

## Is this really necessary?
- task: PowerShell@2
inputs:
Expand Down
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -324,6 +324,7 @@ tracer/deploy/AzureAppServices/
!tracer/build/artifacts

# profiler build files
build_data
tracer/src/Datadog.Trace.ClrProfiler.Native/build/
tracer/src/Datadog.Trace.ClrProfiler.Native/deps/
tracer/src/Datadog.Trace.ClrProfiler.Native/CMakeFiles/
Expand Down
52 changes: 34 additions & 18 deletions tracer/build/_build/Build.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -287,25 +287,41 @@ void DeleteReparsePoints(string path)
.Description("Runs the Benchmarks project")
.Executes(() =>
{
DotNetBuild(s => s
.SetProjectFile(Solution.GetProject(Projects.BenchmarksTrace))
.SetConfiguration(BuildConfiguration)
.SetFramework(TargetFramework.NETCOREAPP3_1)
.EnableNoDependencies()
.When(!string.IsNullOrEmpty(NugetPackageDirectory), o => o.SetPackageDirectory(NugetPackageDirectory))
);
var benchmarksProject = Solution.GetProject(Projects.BenchmarksTrace);
var resultsDirectory = benchmarksProject.Directory / "BenchmarkDotNet.Artifacts" / "results";
EnsureCleanDirectory(resultsDirectory);

DotNetRun(s => s
.SetProjectFile(Solution.GetProject(Projects.BenchmarksTrace))
.SetConfiguration(BuildConfiguration)
.SetFramework(TargetFramework.NETCOREAPP3_1)
.EnableNoRestore()
.EnableNoBuild()
.SetApplicationArguments("-r net472 netcoreapp3.1 -m -f * --iterationTime 2000")
.SetProcessEnvironmentVariable("DD_SERVICE", "dd-trace-dotnet")
.SetProcessEnvironmentVariable("DD_ENV", "CI")
.When(!string.IsNullOrEmpty(NugetPackageDirectory), o => o.SetPackageDirectory(NugetPackageDirectory))
);
try
{
DotNetBuild(s => s
.SetProjectFile(benchmarksProject)
.SetConfiguration(BuildConfiguration)
.SetFramework(TargetFramework.NETCOREAPP3_1)
.EnableNoDependencies()
.When(!string.IsNullOrEmpty(NugetPackageDirectory), o => o.SetPackageDirectory(NugetPackageDirectory))
);

DotNetRun(s => s
.SetProjectFile(benchmarksProject)
.SetConfiguration(BuildConfiguration)
.SetFramework(TargetFramework.NETCOREAPP3_1)
.EnableNoRestore()
.EnableNoBuild()
.SetApplicationArguments("-r net472 netcoreapp3.1 -m -f * --iterationTime 2000")
.SetProcessEnvironmentVariable("DD_SERVICE", "dd-trace-dotnet")
.SetProcessEnvironmentVariable("DD_ENV", "CI")
.When(!string.IsNullOrEmpty(NugetPackageDirectory), o => o.SetPackageDirectory(NugetPackageDirectory))
);
}
finally
{
MoveLogsToBuildData();
if (Directory.Exists(resultsDirectory))
{
CopyDirectoryRecursively(resultsDirectory, BuildDataDirectory / "benchmarks",
DirectoryExistsPolicy.Merge, FileExistsPolicy.Overwrite);
}
}
});

/// <summary>
Expand Down
2 changes: 1 addition & 1 deletion tracer/test/benchmarks/Benchmarks.Trace/AgentWriterBenchmark.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
using System.IO;
using System.Threading.Tasks;
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Exporters.Json;
using Datadog.Trace;
using Datadog.Trace.Agent;
using Datadog.Trace.Agent.Transports;
Expand All @@ -12,7 +13,6 @@

namespace Benchmarks.Trace
{
[DatadogExporter]
[MemoryDiagnoser]
public class AgentWriterBenchmark
{
Expand Down
6 changes: 5 additions & 1 deletion tracer/test/benchmarks/Benchmarks.Trace/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Running;
using Datadog.Trace.BenchmarkDotNet;
using BenchmarkDotNet.Exporters.Json;

namespace Benchmarks.Trace
{
Expand All @@ -22,7 +23,10 @@ private static void Main(string[] args)
}
else
{
var config = DefaultConfig.Instance.AddExporter(DatadogExporter.Default);
var config = DefaultConfig.Instance
.AddExporter(DatadogExporter.Default)
.AddExporter(JsonExporter.FullCompressed);

BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args, config);
}
}
Expand Down
1 change: 0 additions & 1 deletion tracer/test/benchmarks/Benchmarks.Trace/SpanBenchmark.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ namespace Benchmarks.Trace
/// <summary>
/// Span benchmarks
/// </summary>
[DatadogExporter]
[MemoryDiagnoser]
public class SpanBenchmark
{
Expand Down