Skip to content

Commit 37e75fe

Browse files
author
Ilias Tsakiridis
committed
feat: add OpenMetrics exporter support and update related tests
1 parent e97c686 commit 37e75fe

14 files changed

+951
-2
lines changed

‎BenchmarkDotNet.sln.DotSettings‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,7 @@
161161
<s:Boolean x:Key="/Default/UserDictionary/Words/=NONINFRINGEMENT/@EntryIndexedValue">True</s:Boolean>
162162
<s:Boolean x:Key="/Default/UserDictionary/Words/=notcs/@EntryIndexedValue">True</s:Boolean>
163163
<s:Boolean x:Key="/Default/UserDictionary/Words/=nuget/@EntryIndexedValue">True</s:Boolean>
164+
<s:Boolean x:Key="/Default/UserDictionary/Words/=openmetrics/@EntryIndexedValue">True</s:Boolean>
164165
<s:Boolean x:Key="/Default/UserDictionary/Words/=outofproc/@EntryIndexedValue">True</s:Boolean>
165166
<s:Boolean x:Key="/Default/UserDictionary/Words/=parameterless/@EntryIndexedValue">True</s:Boolean>
166167
<s:Boolean x:Key="/Default/UserDictionary/Words/=Partitioner/@EntryIndexedValue">True</s:Boolean>
@@ -185,6 +186,7 @@
185186
<s:Boolean x:Key="/Default/UserDictionary/Words/=sitnik/@EntryIndexedValue">True</s:Boolean>
186187
<s:Boolean x:Key="/Default/UserDictionary/Words/=sproj/@EntryIndexedValue">True</s:Boolean>
187188
<s:Boolean x:Key="/Default/UserDictionary/Words/=stackoverflow/@EntryIndexedValue">True</s:Boolean>
189+
<s:Boolean x:Key="/Default/UserDictionary/Words/=stddev/@EntryIndexedValue">True</s:Boolean>
188190
<s:Boolean x:Key="/Default/UserDictionary/Words/=stloc/@EntryIndexedValue">True</s:Boolean>
189191
<s:Boolean x:Key="/Default/UserDictionary/Words/=Sturges/@EntryIndexedValue">True</s:Boolean>
190192
<s:Boolean x:Key="/Default/UserDictionary/Words/=subfolder/@EntryIndexedValue">True</s:Boolean>

‎src/BenchmarkDotNet/Attributes/Exporters/OpenMetricsExporterAttribute.cs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
using BenchmarkDotNet.Exporters;
2+
using BenchmarkDotNet.Exporters.OpenMetrics;
3+
using JetBrains.Annotations;
4+
5+
namespace BenchmarkDotNet.Attributes
6+
{
7+
[PublicAPI]
8+
public class OpenMetricsExporterAttribute : ExporterConfigBaseAttribute
9+
{
10+
public OpenMetricsExporterAttribute() : base(OpenMetricsExporter.Default)
11+
{
12+
}
13+
}
14+
}

‎src/BenchmarkDotNet/Exporters/DefaultExporters.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
using BenchmarkDotNet.Exporters.Csv;
22
using BenchmarkDotNet.Exporters.Json;
3+
using BenchmarkDotNet.Exporters.OpenMetrics;
34
using BenchmarkDotNet.Exporters.Xml;
45
using JetBrains.Annotations;
56

@@ -12,6 +13,7 @@ public static class DefaultExporters
1213
[PublicAPI] public static readonly IExporter CsvMeasurements = CsvMeasurementsExporter.Default;
1314
[PublicAPI] public static readonly IExporter Html = HtmlExporter.Default;
1415
[PublicAPI] public static readonly IExporter Markdown = MarkdownExporter.Default;
16+
[PublicAPI] public static readonly IExporter OpenMetrics = OpenMetricsExporter.Default;
1517
[PublicAPI] public static readonly IExporter Plain = PlainExporter.Default;
1618
[PublicAPI] public static readonly IExporter RPlot = RPlotExporter.Default;
1719

‎src/BenchmarkDotNet/Exporters/OpenMetrics/OpenMetricsExporter.cs‎

Lines changed: 323 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,323 @@
1+
using System.Collections.Generic;
2+
using System.Collections.Immutable;
3+
using System.Linq;
4+
using BenchmarkDotNet.Loggers;
5+
using BenchmarkDotNet.Parameters;
6+
using BenchmarkDotNet.Reports;
7+
using BenchmarkDotNet.Running;
8+
using System;
9+
using System.Text;
10+
using BenchmarkDotNet.Engines;
11+
using BenchmarkDotNet.Extensions;
12+
using BenchmarkDotNet.Mathematics;
13+
using System.Globalization;
14+
15+
namespace BenchmarkDotNet.Exporters.OpenMetrics;
16+
17+
public class OpenMetricsExporter : ExporterBase
18+
{
19+
private const string MetricPrefix = "benchmark_";
20+
protected override string FileExtension => "metrics";
21+
protected override string FileCaption => "openmetrics";
22+
23+
public static readonly IExporter Default = new OpenMetricsExporter();
24+
25+
public override void ExportToLog(Summary summary, ILogger logger)
26+
{
27+
var metricsSet = new HashSet<OpenMetric>();
28+
29+
foreach (var report in summary.Reports)
30+
{
31+
var benchmark = report.BenchmarkCase;
32+
var gcStats = report.GcStats;
33+
var descriptor = benchmark.Descriptor;
34+
var parameters = benchmark.Parameters;
35+
36+
var stats = report.ResultStatistics;
37+
var metrics = report.Metrics;
38+
if (stats == null)
39+
continue;
40+
41+
AddCommonMetrics(metricsSet, descriptor, parameters, stats, gcStats);
42+
AddAdditionalMetrics(metricsSet, metrics, descriptor, parameters);
43+
}
44+
45+
WriteMetricsToLogger(logger, metricsSet);
46+
}
47+
48+
private static void AddCommonMetrics(HashSet<OpenMetric> metricsSet, Descriptor descriptor, ParameterInstances parameters, Statistics stats, GcStats gcStats)
49+
{
50+
metricsSet.AddRange([
51+
// Mean
52+
OpenMetric.FromStatistics(
53+
$"{MetricPrefix}execution_time_nanoseconds",
54+
"Mean execution time in nanoseconds.",
55+
"gauge",
56+
"nanoseconds",
57+
descriptor,
58+
parameters,
59+
stats.Mean),
60+
// Error
61+
OpenMetric.FromStatistics(
62+
$"{MetricPrefix}error_nanoseconds",
63+
"Standard error of the mean execution time in nanoseconds.",
64+
"gauge",
65+
"nanoseconds",
66+
descriptor,
67+
parameters,
68+
stats.StandardError),
69+
// Standard Deviation
70+
OpenMetric.FromStatistics(
71+
$"{MetricPrefix}stddev_nanoseconds",
72+
"Standard deviation of execution time in nanoseconds.",
73+
"gauge",
74+
"nanoseconds",
75+
descriptor,
76+
parameters,
77+
stats.StandardDeviation),
78+
// GC Stats Gen0 - these are counters, not gauges
79+
OpenMetric.FromStatistics(
80+
$"{MetricPrefix}gc_gen0_collections_total",
81+
"Total number of Gen 0 garbage collections during the benchmark execution.",
82+
"counter",
83+
"",
84+
descriptor,
85+
parameters,
86+
gcStats.Gen0Collections),
87+
// GC Stats Gen1
88+
OpenMetric.FromStatistics(
89+
$"{MetricPrefix}gc_gen1_collections_total",
90+
"Total number of Gen 1 garbage collections during the benchmark execution.",
91+
"counter",
92+
"",
93+
descriptor,
94+
parameters,
95+
gcStats.Gen1Collections),
96+
// GC Stats Gen2
97+
OpenMetric.FromStatistics(
98+
$"{MetricPrefix}gc_gen2_collections_total",
99+
"Total number of Gen 2 garbage collections during the benchmark execution.",
100+
"counter",
101+
"",
102+
descriptor,
103+
parameters,
104+
gcStats.Gen2Collections),
105+
// Total GC Operations
106+
OpenMetric.FromStatistics(
107+
$"{MetricPrefix}gc_total_operations_total",
108+
"Total number of garbage collection operations during the benchmark execution.",
109+
"counter",
110+
"",
111+
descriptor,
112+
parameters,
113+
gcStats.TotalOperations),
114+
// P90 - in nanoseconds
115+
OpenMetric.FromStatistics(
116+
$"{MetricPrefix}p90_nanoseconds",
117+
"90th percentile execution time in nanoseconds.",
118+
"gauge",
119+
"nanoseconds",
120+
descriptor,
121+
parameters,
122+
stats.Percentiles.P90),
123+
// P95 - in nanoseconds
124+
OpenMetric.FromStatistics(
125+
$"{MetricPrefix}p95_nanoseconds",
126+
"95th percentile execution time in nanoseconds.",
127+
"gauge",
128+
"nanoseconds",
129+
descriptor,
130+
parameters,
131+
stats.Percentiles.P95)
132+
]);
133+
}
134+
135+
private static void AddAdditionalMetrics(HashSet<OpenMetric> metricsSet, IReadOnlyDictionary<string, Metric> metrics, Descriptor descriptor, ParameterInstances parameters)
136+
{
137+
var reservedMetricNames = new HashSet<string>
138+
{
139+
$"{MetricPrefix}execution_time_nanoseconds",
140+
$"{MetricPrefix}error_nanoseconds",
141+
$"{MetricPrefix}stddev_nanoseconds",
142+
$"{MetricPrefix}gc_gen0_collections_total",
143+
$"{MetricPrefix}gc_gen1_collections_total",
144+
$"{MetricPrefix}gc_gen2_collections_total",
145+
$"{MetricPrefix}gc_total_operations_total",
146+
$"{MetricPrefix}p90_nanoseconds",
147+
$"{MetricPrefix}p95_nanoseconds"
148+
};
149+
150+
foreach (var metric in metrics)
151+
{
152+
string metricName = SanitizeMetricName(metric.Key);
153+
string fullMetricName = $"{MetricPrefix}{metricName}";
154+
155+
if (reservedMetricNames.Contains(fullMetricName))
156+
continue;
157+
158+
metricsSet.Add(OpenMetric.FromMetric(
159+
fullMetricName,
160+
metric,
161+
"gauge", // Assuming all additional metrics are of type "gauge"
162+
descriptor,
163+
parameters));
164+
}
165+
}
166+
167+
private static void WriteMetricsToLogger(ILogger logger, HashSet<OpenMetric> metricsSet)
168+
{
169+
var emittedHelpType = new HashSet<string>();
170+
171+
foreach (var metric in metricsSet.OrderBy(m => m.Name))
172+
{
173+
if (!emittedHelpType.Contains(metric.Name))
174+
{
175+
logger.WriteLine($"# HELP {metric.Name} {metric.Help}");
176+
logger.WriteLine($"# TYPE {metric.Name} {metric.Type}");
177+
if (!string.IsNullOrEmpty(metric.Unit))
178+
{
179+
logger.WriteLine($"# UNIT {metric.Name} {metric.Unit}");
180+
}
181+
emittedHelpType.Add(metric.Name);
182+
}
183+
184+
logger.WriteLine(metric.ToString());
185+
}
186+
187+
logger.WriteLine("# EOF");
188+
}
189+
190+
private static string SanitizeMetricName(string name)
191+
{
192+
var builder = new StringBuilder();
193+
bool lastWasUnderscore = false;
194+
195+
foreach (char c in name.ToLowerInvariant())
196+
{
197+
if (char.IsLetterOrDigit(c) || c == '_')
198+
{
199+
builder.Append(c);
200+
lastWasUnderscore = false;
201+
}
202+
else if (!lastWasUnderscore)
203+
{
204+
builder.Append('_');
205+
lastWasUnderscore = true;
206+
}
207+
}
208+
209+
string? result = builder.ToString().Trim('_'); // <-- Trim here
210+
211+
if (result.Length > 0 && char.IsDigit(result[0]))
212+
result = "_" + result;
213+
214+
return result;
215+
}
216+
217+
private class OpenMetric : IEquatable<OpenMetric>
218+
{
219+
internal string Name { get; }
220+
internal string Help { get; }
221+
internal string Type { get; }
222+
internal string Unit { get; }
223+
private readonly ImmutableSortedDictionary<string, string> labels;
224+
private readonly double value;
225+
226+
private OpenMetric(string name, string help, string type, string unit, ImmutableSortedDictionary<string, string> labels, double value)
227+
{
228+
if (string.IsNullOrWhiteSpace(name)) throw new ArgumentException("Metric name cannot be null or empty.");
229+
if (string.IsNullOrWhiteSpace(type)) throw new ArgumentException("Metric type cannot be null or empty.");
230+
231+
Name = name;
232+
Help = help;
233+
Type = type;
234+
Unit = unit ?? "";
235+
this.labels = labels ?? throw new ArgumentNullException(nameof(labels));
236+
this.value = value;
237+
}
238+
239+
public static OpenMetric FromStatistics(string name, string help, string type, string unit, Descriptor descriptor, ParameterInstances parameters, double value)
240+
{
241+
var labels = BuildLabelDict(descriptor, parameters);
242+
return new OpenMetric(name, help, type, unit, labels, value);
243+
}
244+
245+
public static OpenMetric FromMetric(string fullMetricName, KeyValuePair<string, Metric> metric, string type, Descriptor descriptor, ParameterInstances parameters)
246+
{
247+
string help = $"Additional metric {metric.Key}";
248+
var labels = BuildLabelDict(descriptor, parameters);
249+
return new OpenMetric(fullMetricName, help, type, "", labels, metric.Value.Value);
250+
}
251+
252+
private static readonly Dictionary<string, string> NormalizedLabelKeyCache = new();
253+
private static string NormalizeLabelKey(string key)
254+
{
255+
string normalized = new(key
256+
.ToLowerInvariant()
257+
.Select(c => char.IsLetterOrDigit(c) ? c : '_')
258+
.ToArray());
259+
return normalized;
260+
}
261+
262+
private static ImmutableSortedDictionary<string, string> BuildLabelDict(Descriptor descriptor, ParameterInstances parameters)
263+
{
264+
var dict = new SortedDictionary<string, string>
265+
{
266+
["method"] = descriptor.WorkloadMethod.Name,
267+
["type"] = descriptor.TypeInfo
268+
};
269+
foreach (var param in parameters.Items)
270+
{
271+
string key = NormalizeLabelKey(param.Name);
272+
string value = EscapeLabelValue(param.Value?.ToString() ?? "");
273+
dict[key] = value;
274+
}
275+
return dict.ToImmutableSortedDictionary();
276+
}
277+
278+
private static string EscapeLabelValue(string value)
279+
{
280+
return value.Replace("\\", @"\\")
281+
.Replace("\"", "\\\"")
282+
.Replace("\n", "\\n")
283+
.Replace("\r", "\\r")
284+
.Replace("\t", "\\t");
285+
}
286+
287+
public override bool Equals(object? obj) => Equals(obj as OpenMetric);
288+
289+
public bool Equals(OpenMetric? other)
290+
{
291+
if (other is null)
292+
return false;
293+
294+
return Name == other.Name
295+
&& value.Equals(other.value)
296+
&& labels.Count == other.labels.Count
297+
&& labels.All(kv => other.labels.TryGetValue(kv.Key, out string? otherValue) && kv.Value == otherValue);
298+
}
299+
300+
public override int GetHashCode()
301+
{
302+
var hash = new HashCode();
303+
hash.Add(Name);
304+
hash.Add(value);
305+
306+
foreach (var kv in labels)
307+
{
308+
hash.Add(kv.Key);
309+
hash.Add(kv.Value);
310+
}
311+
312+
return hash.ToHashCode();
313+
}
314+
315+
public override string ToString()
316+
{
317+
string labelStr = labels.Count > 0
318+
? $"{{{string.Join(", ", labels.Select(kvp => $"{kvp.Key}=\"{kvp.Value}\""))}}}"
319+
: string.Empty;
320+
return $"{Name}{labelStr} {value.ToString(CultureInfo.InvariantCulture)}";
321+
}
322+
}
323+
}

‎tests/BenchmarkDotNet.IntegrationTests/ValidatorsTest.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
using Xunit;
99
using Xunit.Abstractions;
1010
using BenchmarkDotNet.Exporters.Json;
11+
using BenchmarkDotNet.Exporters.OpenMetrics;
1112
using BenchmarkDotNet.Exporters.Xml;
1213

1314
namespace BenchmarkDotNet.IntegrationTests
@@ -23,6 +24,7 @@ public ValidatorsTest(ITestOutputHelper output) : base(output) { }
2324
MarkdownExporter.Default,
2425
MarkdownExporter.GitHub,
2526
MarkdownExporter.StackOverflow,
27+
OpenMetricsExporter.Default,
2628
CsvExporter.Default,
2729
CsvMeasurementsExporter.Default,
2830
HtmlExporter.Default,

0 commit comments

Comments
 (0)