Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
using BenchmarkDotNet.Diagnosers;
using BenchmarkDotNet.Engines;
using BenchmarkDotNet.Extensions;
using BenchmarkDotNet.Helpers;
using BenchmarkDotNet.Loggers;
using BenchmarkDotNet.Reports;
using BenchmarkDotNet.Running;
Expand Down Expand Up @@ -245,7 +246,7 @@ bool IsCallStackIn(StackSourceCallStackIndex index)

logger.WriteLine(
$"Native memory allocated per single operation: " +
$"{PerfolizerMeasurementFormatter.Instance.Format(
$"{UnitHelper.Format(
SizeValue.FromBytes(memoryAllocatedPerOperation).ToMeasurement(SizeUnit.B),
formatProvider: benchmarkCase.Config.CultureInfo)}");
logger.WriteLine($"Count of allocated object: {countOfAllocatedObject / totalOperation}");
Expand All @@ -254,7 +255,7 @@ bool IsCallStackIn(StackSourceCallStackIndex index)
{
logger.WriteLine(
$"Native memory leak per single operation: " +
$"{PerfolizerMeasurementFormatter.Instance.Format(
$"{UnitHelper.Format(
SizeValue.FromBytes(memoryLeakPerOperation).ToMeasurement(SizeUnit.B),
formatProvider: benchmarkCase.Config.CultureInfo)}");
}
Expand Down
9 changes: 9 additions & 0 deletions src/BenchmarkDotNet/Analysers/ZeroMeasurementHelper.cs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
using Perfolizer.Metrology;
using Pragmastat;
using Pragmastat.Estimators;
using Threshold = Perfolizer.Metrology.Threshold;

namespace BenchmarkDotNet.Analysers
{
Expand All @@ -24,7 +25,15 @@ public static bool AreIndistinguishable(double[] workload, double[] overhead, Th
public static bool AreIndistinguishable(Sample workload, Sample overhead, Threshold? threshold = null)
{
threshold ??= MathHelper.DefaultThreshold;
// Perfolizer deprecated its significance testing API in favor of Pragmastat.Toolkit.Compare2.
// Compare2 reports per-metric verdicts against typed thresholds instead of a single
// equivalence result, so adopting it would change every verdict this codebase produces:
// zero-measurement detection, ranks, and the statistical test column. Staying on the
// deprecated path keeps the current statistics intact; the switch needs its own change,
// with its own validation against real runs.
#pragma warning disable CS0618 // Type or member is obsolete
var tost = new SimpleEquivalenceTest(MannWhitneyTest.Instance);
#pragma warning restore CS0618
if (workload.Size == 1 || overhead.Size == 1)
return false;
return tost.Perform(workload, overhead, threshold, SignificanceLevel.P1E5) == ComparisonResult.Indistinguishable;
Expand Down
2 changes: 1 addition & 1 deletion src/BenchmarkDotNet/BenchmarkDotNet.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
<PackageReference Include="Gee.External.Capstone" Version="2.3.0" />
<PackageReference Include="Iced" Version="1.21.0" />
<PackageReference Include="Microsoft.Diagnostics.Runtime" Version="4.0.726401" />
<PackageReference Include="Perfolizer" Version="[0.6.6]" />
<PackageReference Include="Perfolizer" Version="[0.7.5]" />
<PackageReference Include="Microsoft.Diagnostics.Tracing.TraceEvent" Version="3.2.4" PrivateAssets="contentfiles;analyzers" />
<PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="5.3.0" />
<PackageReference Include="System.Management" Version="10.0.9" />
Expand Down
8 changes: 4 additions & 4 deletions src/BenchmarkDotNet/Columns/MetricColumn.cs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using BenchmarkDotNet.Extensions;
using BenchmarkDotNet.Helpers;
using BenchmarkDotNet.Reports;
using BenchmarkDotNet.Running;
using Perfolizer.Horology;
Expand Down Expand Up @@ -44,25 +45,24 @@ public string GetValue(Summary summary, BenchmarkCase benchmarkCase, SummaryStyl
var cultureInfo = summary.GetCultureInfo();

bool printUnits = style.PrintUnitsInContent || style.PrintUnitsInHeader;
var unitPresentation = new UnitPresentation(style.PrintUnitsInContent, minUnitWidth: 0, gap: true);
string numberFormat = descriptor.NumberFormat;

if (printUnits && descriptor.UnitType == UnitType.CodeSize)
{
var measurement = SizeValue.FromBytes((long)metric.Value).ToMeasurement(style.CodeSizeUnit);
return PerfolizerMeasurementFormatter.Instance.Format(measurement, numberFormat, cultureInfo, unitPresentation);
return UnitHelper.Format(measurement, numberFormat, cultureInfo, style.PrintUnitsInContent);
}
if (printUnits && descriptor.UnitType == UnitType.Size)
{
var measurement = SizeValue.FromBytes((long)metric.Value).ToMeasurement(style.SizeUnit);
return PerfolizerMeasurementFormatter.Instance.Format(measurement, numberFormat, cultureInfo, unitPresentation);
return UnitHelper.Format(measurement, numberFormat, cultureInfo, style.PrintUnitsInContent);
}
if (printUnits && descriptor.UnitType == UnitType.Time)
{
if (numberFormat.IsBlank())
numberFormat = "N4";
var measurement = TimeInterval.FromNanoseconds(metric.Value).ToMeasurement(style.TimeUnit);
return PerfolizerMeasurementFormatter.Instance.Format(measurement, numberFormat, cultureInfo, unitPresentation);
return UnitHelper.Format(measurement, numberFormat, cultureInfo, style.PrintUnitsInContent);
}

return metric.Value.ToString(numberFormat, cultureInfo);
Expand Down
5 changes: 3 additions & 2 deletions src/BenchmarkDotNet/Columns/StatisticColumn.cs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Helpers;
using BenchmarkDotNet.Mathematics;
using BenchmarkDotNet.Reports;
using BenchmarkDotNet.Running;
Expand Down Expand Up @@ -151,11 +152,11 @@ private string Format(Summary summary, ImmutableConfig config, Statistics? stati
if (double.IsNaN(value))
return "NA";
return UnitType == UnitType.Time
? PerfolizerMeasurementFormatter.Instance.Format(
? UnitHelper.Format(
TimeInterval.FromNanoseconds(value).ToMeasurement(style.TimeUnit),
format,
style.CultureInfo,
new UnitPresentation(style.PrintUnitsInContent, minUnitWidth: 0, gap: true))
style.PrintUnitsInContent)
: value.ToString(format, style.CultureInfo);
}

Expand Down
7 changes: 6 additions & 1 deletion src/BenchmarkDotNet/Columns/StatisticalTestColumn.cs
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
using BenchmarkDotNet.Helpers;
using BenchmarkDotNet.Mathematics;
using BenchmarkDotNet.Reports;
using BenchmarkDotNet.Running;
Expand All @@ -18,7 +19,7 @@ public class StatisticalTestColumn(Threshold threshold, SignificanceLevel? signi

public static StatisticalTestColumn Create(string threshold, SignificanceLevel? significanceLevel = null)
{
if (!Threshold.TryParse(threshold, out var parsedThreshold))
if (!Threshold.TryParse(UnitHelper.NormalizeUnits(threshold), out var parsedThreshold))
throw new ArgumentException($"Can't parse threshold '{threshold}'");
return new StatisticalTestColumn(parsedThreshold, significanceLevel);
}
Expand All @@ -37,7 +38,11 @@ public override string GetValue(Summary summary, BenchmarkCase benchmarkCase, St
if (current.Sample.Size == 1 && baseline.Sample.Size == 1)
return "?";

// See ZeroMeasurementHelper: moving to Pragmastat.Toolkit.Compare2 would change the
// reported verdicts, so it needs its own change rather than riding along with a bump.
#pragma warning disable CS0618 // Type or member is obsolete
var test = new SimpleEquivalenceTest(MannWhitneyTest.Instance);
#pragma warning restore CS0618
var comparisonResult = test.Perform(current.Sample, baseline.Sample, Threshold, SignificanceLevel);
return comparisonResult switch
{
Expand Down
2 changes: 1 addition & 1 deletion src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs
Original file line number Diff line number Diff line change
Expand Up @@ -318,7 +318,7 @@ private static bool Validate(CommandLineOptions options, ILogger logger)

if (options.StatisticalTestThreshold.IsNotBlank())
{
options.StatisticalTestThreshold = options.StatisticalTestThreshold.Trim();
options.StatisticalTestThreshold = UnitHelper.NormalizeUnits(options.StatisticalTestThreshold.Trim());
if (IsUnitlessNumber(options.StatisticalTestThreshold))
{
string original = options.StatisticalTestThreshold;
Expand Down
9 changes: 2 additions & 7 deletions src/BenchmarkDotNet/Environments/HostEnvironmentInfo.cs
Original file line number Diff line number Diff line change
Expand Up @@ -99,13 +99,8 @@ public override IEnumerable<string> ToFormattedString()

if (HardwareTimerKind != HardwareTimerKind.Unknown)
{
string frequency = PerfolizerMeasurementFormatter.Instance.Format(
ChronometerFrequency.ToMeasurement(FrequencyUnit.Hz),
unitPresentation: UnitHelper.DefaultPresentation);
string resolution = PerfolizerMeasurementFormatter.Instance.Format(
ChronometerResolution.ToMeasurement(),
format: "0.000",
unitPresentation: UnitHelper.DefaultPresentation);
string frequency = UnitHelper.Format(ChronometerFrequency.ToMeasurement(FrequencyUnit.Hz));
string resolution = UnitHelper.Format(ChronometerResolution.ToMeasurement(), format: "0.000");
string timer = HardwareTimerKind.ToString().ToUpper();
yield return $"Frequency: {frequency}, Resolution: {resolution}, Timer: {timer}";
}
Expand Down
7 changes: 4 additions & 3 deletions src/BenchmarkDotNet/Extensions/CommonExtensions.cs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using BenchmarkDotNet.Columns;
using BenchmarkDotNet.Helpers;
using BenchmarkDotNet.Reports;
using System.Diagnostics.CodeAnalysis;

Expand All @@ -17,14 +18,14 @@ public static string GetColumnTitle(this IColumn column, SummaryStyle style)
switch (column.UnitType)
{
case UnitType.CodeSize:
return $"{column.ColumnName} [{style.CodeSizeUnit.Abbreviation}]";
return $"{column.ColumnName} [{style.CodeSizeUnit.GetAbbreviation()}]";
case UnitType.Size:
return style.SizeUnit != null
? $"{column.ColumnName} [{style.SizeUnit.Abbreviation}]"
? $"{column.ColumnName} [{style.SizeUnit.GetAbbreviation()}]"
: $"{column.ColumnName}";
case UnitType.Time:
return style.TimeUnit != null
? $"{column.ColumnName} [{style.TimeUnit.Abbreviation}]"
? $"{column.ColumnName} [{style.TimeUnit.GetAbbreviation()}]"
: $"{column.ColumnName}";
case UnitType.Dimensionless:
return column.ColumnName;
Expand Down
4 changes: 2 additions & 2 deletions src/BenchmarkDotNet/Extensions/StatisticsExtensions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,9 @@ public static class StatisticsExtensions
public static Func<double, string> CreateNanosecondFormatter(this Statistics s, CultureInfo cultureInfo, string format = "N3")
{
var timeUnit = TimeUnit.GetBestTimeUnit(s.Mean);
return x => PerfolizerMeasurementFormatter.Instance.Format(
return x => UnitHelper.Format(
TimeInterval.FromNanoseconds(x).ToMeasurement(timeUnit),
format, cultureInfo, UnitHelper.DefaultPresentation
format, cultureInfo
);
}

Expand Down
2 changes: 1 addition & 1 deletion src/BenchmarkDotNet/Helpers/AsciiHelper.cs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ internal static class AsciiHelper
/// <summary>
/// The 'μ' symbol
/// </summary>
private const string Mu = "\u03BC";
internal const string Mu = "\u03BC";

public static string ToAscii(this string s)
{
Expand Down
33 changes: 30 additions & 3 deletions src/BenchmarkDotNet/Helpers/UnitHelper.cs
Original file line number Diff line number Diff line change
@@ -1,11 +1,38 @@
using System;
using System.Globalization;
using Perfolizer.Horology;
using Pragmastat.Metrology;

namespace BenchmarkDotNet.Helpers;

public static class UnitHelper
{
public static readonly UnitPresentation DefaultPresentation = new(true, 0, gap: true);
/// <summary>
/// The abbreviation to print for the given unit.
/// Perfolizer reports ASCII-only abbreviations ("us"), while BenchmarkDotNet prints the Unicode ones ("μs")
/// and downgrades them via ToAscii for terminals without Unicode support.
/// </summary>
public static string GetAbbreviation(this MeasurementUnit unit) =>
unit == TimeUnit.Microsecond ? AsciiHelper.Mu + "s" : unit.Abbreviation;

public static string ToDefaultString(this TimeInterval timeInterval, string? format = null) => timeInterval.ToString(format, null, DefaultPresentation);
}
/// <summary>
/// Formats a measurement the way BenchmarkDotNet presents it:
/// the nominal value, a gap, and the unit abbreviation.
/// </summary>
public static string Format(Measurement measurement, string? format = null,
IFormatProvider? formatProvider = null, bool printUnit = true)
{
string nominalPart = measurement.NominalValue.ToString(format, formatProvider ?? CultureInfo.InvariantCulture);
string abbreviation = measurement.Unit.GetAbbreviation();
return printUnit && abbreviation.Length > 0 ? $"{nominalPart} {abbreviation}" : nominalPart;
}

/// <summary>
/// Normalizes user input so that units can be spelled with Unicode abbreviations ("5μs")
/// even though Perfolizer parsers only know the ASCII ones.
/// </summary>
public static string NormalizeUnits(string s) => s.ToAscii();

public static string ToDefaultString(this TimeInterval timeInterval, string? format = null) =>
Format(timeInterval.ToMeasurement(), format ?? "0.###");
}
4 changes: 4 additions & 0 deletions src/BenchmarkDotNet/Mathematics/RankHelper.cs
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,11 @@ public static int[] GetRanks(params Statistics[] stats)

private static bool AreSame(Statistics x, Statistics y)
{
// See ZeroMeasurementHelper: moving to Pragmastat.Toolkit.Compare2 would change the
// computed ranks, so it needs its own change rather than riding along with a bump.
#pragma warning disable CS0618 // Type or member is obsolete
var test = new SimpleEquivalenceTest(MannWhitneyTest.Instance);
#pragma warning restore CS0618
var comparisonResult = test.Perform(x.Sample, y.Sample, MathHelper.DefaultThreshold, MathHelper.DefaultSignificanceLevel);
return comparisonResult == ComparisonResult.Indistinguishable;
}
Expand Down
4 changes: 2 additions & 2 deletions src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs
Original file line number Diff line number Diff line change
Expand Up @@ -376,13 +376,13 @@ private static async ValueTask PrintSummary(ILogger logger, ImmutableConfig conf
if (columnWithLegends.Any())
maxNameWidth = Math.Max(maxNameWidth, columnWithLegends.Select(c => c.ColumnName.Length).Max());
if (effectiveTimeUnit != null)
maxNameWidth = Math.Max(maxNameWidth, effectiveTimeUnit.Abbreviation.ToString(cultureInfo).Length + 2);
maxNameWidth = Math.Max(maxNameWidth, effectiveTimeUnit.GetAbbreviation().ToString(cultureInfo).Length + 2);

foreach (var column in columnWithLegends)
logger.WriteLineHint($" {column.ColumnName.PadRight(maxNameWidth, ' ')} : {column.Legend}");

if (effectiveTimeUnit != null)
logger.WriteLineHint($" {("1 " + effectiveTimeUnit.Abbreviation).PadRight(maxNameWidth, ' ')} :" +
logger.WriteLineHint($" {("1 " + effectiveTimeUnit.GetAbbreviation()).PadRight(maxNameWidth, ' ')} :" +
$" 1 {effectiveTimeUnit.FullName} ({TimeUnit.Convert(1, effectiveTimeUnit, TimeUnit.Second).ToString("0.#########", summary.GetCultureInfo())} sec)");
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ public static EntryInfo AddMetrics(this EntryInfo entry, params string[] metrics
{
for (int i = 0; i < metrics.Length; i++)
{
var measurement = PerfolizerMeasurementFormatter.Instance.Parse(metrics[i]);
var measurement = MeasurementFormatter.Default.Parse(metrics[i]);
entry.Add(new EntryInfo
{
IterationIndex = i,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@ MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 phys

Type = Bench

| Method | Center | Spread |
|:-------|---------:|--------:|
| Foo | 11.0 ns | 0.81 ns |
| Bar | 201.0 ns | 0.81 ns |
| Method | Center | Spread |
|:-------|--------:|-------:|
| Foo | 11.0ns | 0.81ns |
| Bar | 201.0ns | 0.81ns |

{
"engine": {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,12 @@ MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 phys

Type = Bench

| Method | Runtime | Center | Spread |
|:-------|:--------|--------:|--------:|
| Foo | Net481 | 11.0 ns | 0.81 ns |
| Bar | Net481 | 21.0 ns | 0.81 ns |
| Foo | Net70 | 31.0 ns | 0.81 ns |
| Bar | Net70 | 41.0 ns | 0.81 ns |
| Method | Runtime | Center | Spread |
|:-------|:--------|-------:|-------:|
| Foo | Net481 | 11.0ns | 0.81ns |
| Bar | Net481 | 21.0ns | 0.81ns |
| Foo | Net70 | 31.0ns | 0.81ns |
| Bar | Net70 | 41.0ns | 0.81ns |

{
"engine": {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@ MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 phys

Type = Bench, Runtime = Net70, Jit = RyuJit

| Method | Center | Spread |
|:-------|--------:|--------:|
| Foo | 31.0 ns | 0.81 ns |
| Bar | 41.0 ns | 0.81 ns |
| Method | Center | Spread |
|:-------|-------:|-------:|
| Foo | 31.0ns | 0.81ns |
| Bar | 41.0ns | 0.81ns |

{
"engine": {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,12 @@ Type = Bench
@Net481LegacyJit: Runtime = Net481, Jit = LegacyJit
@Net70RyuJit: Runtime = Net70, Jit = RyuJit

| Method | Job | Center | Spread |
|:-------|:----------------|--------:|--------:|
| Foo | Net481LegacyJit | 11.0 ns | 0.81 ns |
| Bar | Net481LegacyJit | 21.0 ns | 0.81 ns |
| Foo | Net70RyuJit | 31.0 ns | 0.81 ns |
| Bar | Net70RyuJit | 41.0 ns | 0.81 ns |
| Method | Job | Center | Spread |
|:-------|:----------------|-------:|-------:|
| Foo | Net481LegacyJit | 11.0ns | 0.81ns |
| Bar | Net481LegacyJit | 21.0ns | 0.81ns |
| Foo | Net70RyuJit | 31.0ns | 0.81ns |
| Bar | Net70RyuJit | 41.0ns | 0.81ns |

{
"engine": {
Expand Down
Loading