From 73b6e30c1baf9b8d2300026149463ddc118844fc Mon Sep 17 00:00:00 2001 From: Mikel Blanchard Date: Fri, 23 Feb 2024 15:50:19 -0800 Subject: [PATCH] [tools] Stress test improvements (#5381) --- src/OpenTelemetry/AssemblyInfo.cs | 1 + .../OpenTelemetry.Tests.Stress.Logs.csproj | 10 +- .../Program.cs | 58 +++-- .../OpenTelemetry.Tests.Stress.Metrics.csproj | 12 +- .../Program.cs | 161 ++++++++++---- .../OpenTelemetry.Tests.Stress.Traces.csproj | 12 +- .../Program.cs | 42 ++-- test/OpenTelemetry.Tests.Stress/Meat.cs | 19 -- .../OpenTelemetry.Tests.Stress.csproj | 2 + test/OpenTelemetry.Tests.Stress/Program.cs | 24 ++ test/OpenTelemetry.Tests.Stress/README.md | 6 +- test/OpenTelemetry.Tests.Stress/Skeleton.cs | 172 -------------- test/OpenTelemetry.Tests.Stress/StressTest.cs | 209 ++++++++++++++++++ .../StressTestFactory.cs | 34 +++ .../StressTestNativeMethods.cs | 28 +++ .../StressTestOptions.cs | 18 ++ 16 files changed, 508 insertions(+), 300 deletions(-) delete mode 100644 test/OpenTelemetry.Tests.Stress/Meat.cs create mode 100644 test/OpenTelemetry.Tests.Stress/Program.cs delete mode 100644 test/OpenTelemetry.Tests.Stress/Skeleton.cs create mode 100644 test/OpenTelemetry.Tests.Stress/StressTest.cs create mode 100644 test/OpenTelemetry.Tests.Stress/StressTestFactory.cs create mode 100644 test/OpenTelemetry.Tests.Stress/StressTestNativeMethods.cs create mode 100644 test/OpenTelemetry.Tests.Stress/StressTestOptions.cs diff --git a/src/OpenTelemetry/AssemblyInfo.cs b/src/OpenTelemetry/AssemblyInfo.cs index 62254638d8a..90823131448 100644 --- a/src/OpenTelemetry/AssemblyInfo.cs +++ b/src/OpenTelemetry/AssemblyInfo.cs @@ -16,6 +16,7 @@ [assembly: InternalsVisibleTo("OpenTelemetry.Exporter.Console" + AssemblyInfo.PublicKey)] [assembly: InternalsVisibleTo("OpenTelemetry.Exporter.OpenTelemetryProtocol" + AssemblyInfo.PublicKey)] [assembly: InternalsVisibleTo("OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests" + AssemblyInfo.PublicKey)] +[assembly: InternalsVisibleTo("OpenTelemetry.Tests.Stress.Metrics" + AssemblyInfo.PublicKey)] #endif #if SIGNED diff --git a/test/OpenTelemetry.Tests.Stress.Logs/OpenTelemetry.Tests.Stress.Logs.csproj b/test/OpenTelemetry.Tests.Stress.Logs/OpenTelemetry.Tests.Stress.Logs.csproj index 1f1225d551a..e75a64bbcbf 100644 --- a/test/OpenTelemetry.Tests.Stress.Logs/OpenTelemetry.Tests.Stress.Logs.csproj +++ b/test/OpenTelemetry.Tests.Stress.Logs/OpenTelemetry.Tests.Stress.Logs.csproj @@ -3,21 +3,13 @@ Exe $(TargetFrameworksForTests) - - disable - + - - - - - - diff --git a/test/OpenTelemetry.Tests.Stress.Logs/Program.cs b/test/OpenTelemetry.Tests.Stress.Logs/Program.cs index 6d2cb88fad0..dececdacb51 100644 --- a/test/OpenTelemetry.Tests.Stress.Logs/Program.cs +++ b/test/OpenTelemetry.Tests.Stress.Logs/Program.cs @@ -1,39 +1,55 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 -using System.Runtime.CompilerServices; using Microsoft.Extensions.Logging; namespace OpenTelemetry.Tests.Stress; -public partial class Program +public static class Program { - private static ILogger logger; - private static Payload payload = new Payload(); + public static int Main(string[] args) + { + return StressTestFactory.RunSynchronously(args); + } - public static void Main() + private sealed class LogsStressTest : StressTest { - using var loggerFactory = LoggerFactory.Create(builder => + private static readonly Payload Payload = new(); + private readonly ILoggerFactory loggerFactory; + private readonly ILogger logger; + + public LogsStressTest(StressTestOptions options) + : base(options) { - builder.AddOpenTelemetry(options => + this.loggerFactory = LoggerFactory.Create(builder => { - options.AddProcessor(new DummyProcessor()); + builder.AddOpenTelemetry(options => + { + options.AddProcessor(new DummyProcessor()); + }); }); - }); - logger = loggerFactory.CreateLogger(); + this.logger = this.loggerFactory.CreateLogger(); + } - Stress(prometheusPort: 9464); - } + protected override void RunWorkItemInParallel() + { + this.logger.Log( + logLevel: LogLevel.Information, + eventId: 2, + state: Payload, + exception: null, + formatter: (state, ex) => string.Empty); + } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - protected static void Run() - { - logger.Log( - logLevel: LogLevel.Information, - eventId: 2, - state: payload, - exception: null, - formatter: (state, ex) => string.Empty); + protected override void Dispose(bool isDisposing) + { + if (isDisposing) + { + this.loggerFactory.Dispose(); + } + + base.Dispose(isDisposing); + } } } diff --git a/test/OpenTelemetry.Tests.Stress.Metrics/OpenTelemetry.Tests.Stress.Metrics.csproj b/test/OpenTelemetry.Tests.Stress.Metrics/OpenTelemetry.Tests.Stress.Metrics.csproj index a783ed18d71..d162e31f732 100644 --- a/test/OpenTelemetry.Tests.Stress.Metrics/OpenTelemetry.Tests.Stress.Metrics.csproj +++ b/test/OpenTelemetry.Tests.Stress.Metrics/OpenTelemetry.Tests.Stress.Metrics.csproj @@ -3,23 +3,15 @@ Exe $(TargetFrameworksForTests) - - disable - - - - - - - + + - diff --git a/test/OpenTelemetry.Tests.Stress.Metrics/Program.cs b/test/OpenTelemetry.Tests.Stress.Metrics/Program.cs index 102ad6d1df8..f43e4d12fe8 100644 --- a/test/OpenTelemetry.Tests.Stress.Metrics/Program.cs +++ b/test/OpenTelemetry.Tests.Stress.Metrics/Program.cs @@ -2,65 +2,140 @@ // SPDX-License-Identifier: Apache-2.0 using System.Diagnostics.Metrics; -using System.Runtime.CompilerServices; +using System.Text.Json.Serialization; +using CommandLine; using OpenTelemetry.Metrics; namespace OpenTelemetry.Tests.Stress; -public partial class Program +public static class Program { - private const int ArraySize = 10; - - // Note: Uncomment the below line if you want to run Histogram stress test - private const int MaxHistogramMeasurement = 1000; + private enum MetricsStressTestType + { + /// Histogram. + Histogram, - private static readonly Meter TestMeter = new(Utils.GetCurrentMethodName()); - private static readonly Counter TestCounter = TestMeter.CreateCounter("TestCounter"); - private static readonly string[] DimensionValues = new string[ArraySize]; - private static readonly ThreadLocal ThreadLocalRandom = new(() => new Random()); + /// Counter. + Counter, + } - // Note: Uncomment the below line if you want to run Histogram stress test - private static readonly Histogram TestHistogram = TestMeter.CreateHistogram("TestHistogram"); + public static int Main(string[] args) + { + return StressTestFactory.RunSynchronously(args); + } - public static void Main() + private sealed class MetricsStressTest : StressTest { - for (int i = 0; i < ArraySize; i++) + private const int ArraySize = 10; + private const int MaxHistogramMeasurement = 1000; + + private static readonly Meter TestMeter = new(Utils.GetCurrentMethodName()); + private static readonly Histogram TestHistogram = TestMeter.CreateHistogram("TestHistogram"); + private static readonly Counter TestCounter = TestMeter.CreateCounter("TestCounter"); + private static readonly string[] DimensionValues = new string[ArraySize]; + private static readonly ThreadLocal ThreadLocalRandom = new(() => new Random()); + private readonly MeterProvider meterProvider; + + static MetricsStressTest() { - DimensionValues[i] = $"DimValue{i}"; + for (int i = 0; i < ArraySize; i++) + { + DimensionValues[i] = $"DimValue{i}"; + } } - using var meterProvider = Sdk.CreateMeterProviderBuilder() - .AddMeter(TestMeter.Name) + public MetricsStressTest(MetricsStressTestOptions options) + : base(options) + { + var builder = Sdk.CreateMeterProviderBuilder().AddMeter(TestMeter.Name); + + if (options.PrometheusTestMetricsPort != 0) + { + builder.AddPrometheusHttpListener(o => o.UriPrefixes = new string[] { $"http://localhost:{options.PrometheusTestMetricsPort}/" }); + } + + if (options.EnableExemplars) + { + builder.SetExemplarFilter(new AlwaysOnExemplarFilter()); + } + + if (options.AddViewToFilterTags) + { + builder + .AddView("TestCounter", new MetricStreamConfiguration { TagKeys = new string[] { "DimName1" } }) + .AddView("TestHistogram", new MetricStreamConfiguration { TagKeys = new string[] { "DimName1" } }); + } - // .SetExemplarFilter(new AlwaysOnExemplarFilter()) - .AddPrometheusHttpListener( - options => options.UriPrefixes = new string[] { $"http://localhost:9185/" }) - .Build(); + if (options.AddOtlpExporter) + { + builder.AddOtlpExporter((exporterOptions, readerOptions) => + { + readerOptions.PeriodicExportingMetricReaderOptions.ExportIntervalMilliseconds = options.OtlpExporterExportIntervalMilliseconds; + }); + } - Stress(prometheusPort: 9464); + this.meterProvider = builder.Build(); + } + + protected override void WriteRunInformationToConsole() + { + if (this.Options.PrometheusTestMetricsPort != 0) + { + Console.Write($", testPrometheusEndpoint = http://localhost:{this.Options.PrometheusTestMetricsPort}/metrics/"); + } + } + + protected override void RunWorkItemInParallel() + { + var random = ThreadLocalRandom.Value!; + if (this.Options.TestType == MetricsStressTestType.Histogram) + { + TestHistogram.Record( + random.Next(MaxHistogramMeasurement), + new("DimName1", DimensionValues[random.Next(0, ArraySize)]), + new("DimName2", DimensionValues[random.Next(0, ArraySize)]), + new("DimName3", DimensionValues[random.Next(0, ArraySize)])); + } + else if (this.Options.TestType == MetricsStressTestType.Counter) + { + TestCounter.Add( + 100, + new("DimName1", DimensionValues[random.Next(0, ArraySize)]), + new("DimName2", DimensionValues[random.Next(0, ArraySize)]), + new("DimName3", DimensionValues[random.Next(0, ArraySize)])); + } + } + + protected override void Dispose(bool isDisposing) + { + if (isDisposing) + { + this.meterProvider.Dispose(); + } + + base.Dispose(isDisposing); + } } - // Note: Uncomment the below lines if you want to run Counter stress test - // [MethodImpl(MethodImplOptions.AggressiveInlining)] - // protected static void Run() - // { - // var random = ThreadLocalRandom.Value; - // TestCounter.Add( - // 100, - // new("DimName1", DimensionValues[random.Next(0, ArraySize)]), - // new("DimName2", DimensionValues[random.Next(0, ArraySize)]), - // new("DimName3", DimensionValues[random.Next(0, ArraySize)])); - // } - - // Note: Uncomment the below lines if you want to run Histogram stress test - [MethodImpl(MethodImplOptions.AggressiveInlining)] - protected static void Run() + private sealed class MetricsStressTestOptions : StressTestOptions { - var random = ThreadLocalRandom.Value; - TestHistogram.Record( - random.Next(MaxHistogramMeasurement), - new("DimName1", DimensionValues[random.Next(0, ArraySize)]), - new("DimName2", DimensionValues[random.Next(0, ArraySize)]), - new("DimName3", DimensionValues[random.Next(0, ArraySize)])); + [JsonConverter(typeof(JsonStringEnumConverter))] + [Option('t', "type", HelpText = "The metrics stress test type to run. Valid values: [Histogram, Counter]. Default value: Histogram.", Required = false)] + public MetricsStressTestType TestType { get; set; } = MetricsStressTestType.Histogram; + + [Option('m', "metrics_port", HelpText = "The Prometheus http listener port where Prometheus will be exposed for retrieving test metrics while the stress test is running. Set to '0' to disable. Default value: 9185.", Required = false)] + public int PrometheusTestMetricsPort { get; set; } = 9185; + + [Option('v', "view", HelpText = "Whether or not a view should be configured to filter tags for the stress test. Default value: False.", Required = false)] + public bool AddViewToFilterTags { get; set; } + + [Option('o', "otlp", HelpText = "Whether or not an OTLP exporter should be added for the stress test. Default value: False.", Required = false)] + public bool AddOtlpExporter { get; set; } + + [Option('i', "interval", HelpText = "The OTLP exporter export interval in milliseconds. Default value: 5000.", Required = false)] + public int OtlpExporterExportIntervalMilliseconds { get; set; } = 5000; + + [Option('e', "exemplars", HelpText = "Whether or not to enable exemplars for the stress test. Default value: False.", Required = false)] + public bool EnableExemplars { get; set; } } } diff --git a/test/OpenTelemetry.Tests.Stress.Traces/OpenTelemetry.Tests.Stress.Traces.csproj b/test/OpenTelemetry.Tests.Stress.Traces/OpenTelemetry.Tests.Stress.Traces.csproj index 41f6d28bc55..7a32563d8b5 100644 --- a/test/OpenTelemetry.Tests.Stress.Traces/OpenTelemetry.Tests.Stress.Traces.csproj +++ b/test/OpenTelemetry.Tests.Stress.Traces/OpenTelemetry.Tests.Stress.Traces.csproj @@ -6,17 +6,7 @@ - - - - - - - - - - - + diff --git a/test/OpenTelemetry.Tests.Stress.Traces/Program.cs b/test/OpenTelemetry.Tests.Stress.Traces/Program.cs index 743da46b638..422a44a99ef 100644 --- a/test/OpenTelemetry.Tests.Stress.Traces/Program.cs +++ b/test/OpenTelemetry.Tests.Stress.Traces/Program.cs @@ -2,31 +2,45 @@ // SPDX-License-Identifier: Apache-2.0 using System.Diagnostics; -using System.Runtime.CompilerServices; -using OpenTelemetry.Metrics; using OpenTelemetry.Trace; namespace OpenTelemetry.Tests.Stress; -public partial class Program +public static class Program { - private static readonly ActivitySource ActivitySource = new ActivitySource("OpenTelemetry.Tests.Stress"); - - public static void Main() + public static int Main(string[] args) { - using var tracerProvider = Sdk.CreateTracerProviderBuilder() - .AddSource(ActivitySource.Name) - .Build(); - - Stress(prometheusPort: 9464); + return StressTestFactory.RunSynchronously(args); } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - protected static void Run() + private sealed class TracesStressTest : StressTest { - using (var activity = ActivitySource.StartActivity("test")) + private static readonly ActivitySource ActivitySource = new("OpenTelemetry.Tests.Stress"); + private readonly TracerProvider tracerProvider; + + public TracesStressTest(StressTestOptions options) + : base(options) + { + this.tracerProvider = Sdk.CreateTracerProviderBuilder() + .AddSource(ActivitySource.Name) + .Build(); + } + + protected override void RunWorkItemInParallel() { + using var activity = ActivitySource.StartActivity("test"); + activity?.SetTag("foo", "value"); } + + protected override void Dispose(bool isDisposing) + { + if (isDisposing) + { + this.tracerProvider.Dispose(); + } + + base.Dispose(isDisposing); + } } } diff --git a/test/OpenTelemetry.Tests.Stress/Meat.cs b/test/OpenTelemetry.Tests.Stress/Meat.cs deleted file mode 100644 index 65e66535349..00000000000 --- a/test/OpenTelemetry.Tests.Stress/Meat.cs +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright The OpenTelemetry Authors -// SPDX-License-Identifier: Apache-2.0 - -using System.Runtime.CompilerServices; - -namespace OpenTelemetry.Tests.Stress; - -public partial class Program -{ - public static void Main() - { - Stress(concurrency: 1, prometheusPort: 9464); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - protected static void Run() - { - } -} diff --git a/test/OpenTelemetry.Tests.Stress/OpenTelemetry.Tests.Stress.csproj b/test/OpenTelemetry.Tests.Stress/OpenTelemetry.Tests.Stress.csproj index 60e3c917910..01af1c993ae 100644 --- a/test/OpenTelemetry.Tests.Stress/OpenTelemetry.Tests.Stress.csproj +++ b/test/OpenTelemetry.Tests.Stress/OpenTelemetry.Tests.Stress.csproj @@ -5,8 +5,10 @@ + + diff --git a/test/OpenTelemetry.Tests.Stress/Program.cs b/test/OpenTelemetry.Tests.Stress/Program.cs new file mode 100644 index 00000000000..a5f6fb8975e --- /dev/null +++ b/test/OpenTelemetry.Tests.Stress/Program.cs @@ -0,0 +1,24 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +namespace OpenTelemetry.Tests.Stress; + +public static class Program +{ + public static int Main(string[] args) + { + return StressTestFactory.RunSynchronously(args); + } + + private sealed class DemoStressTest : StressTest + { + public DemoStressTest(StressTestOptions options) + : base(options) + { + } + + protected override void RunWorkItemInParallel() + { + } + } +} diff --git a/test/OpenTelemetry.Tests.Stress/README.md b/test/OpenTelemetry.Tests.Stress/README.md index 890b1d0cc9b..1f953b1def1 100644 --- a/test/OpenTelemetry.Tests.Stress/README.md +++ b/test/OpenTelemetry.Tests.Stress/README.md @@ -73,6 +73,10 @@ process_runtime_dotnet_gc_allocations_size_bytes 5485192 1658950184752 ## Writing your own stress test +> [!WARNING] +> These instructions are out of date and should NOT be followed. They will be + updated soon. + Create a simple console application with the following code: ```csharp @@ -93,7 +97,7 @@ public partial class Program } ``` -Add the [`Skeleton.cs`](./Skeleton.cs) file to your `*.csproj` file: +Add the Skeleton.cs file to your `*.csproj` file: ```xml diff --git a/test/OpenTelemetry.Tests.Stress/Skeleton.cs b/test/OpenTelemetry.Tests.Stress/Skeleton.cs deleted file mode 100644 index cd3e5af7a8a..00000000000 --- a/test/OpenTelemetry.Tests.Stress/Skeleton.cs +++ /dev/null @@ -1,172 +0,0 @@ -// Copyright The OpenTelemetry Authors -// SPDX-License-Identifier: Apache-2.0 - -using System.Diagnostics; -using System.Diagnostics.Metrics; -using System.Runtime.InteropServices; -using OpenTelemetry.Metrics; - -namespace OpenTelemetry.Tests.Stress; - -public partial class Program -{ - private static volatile bool bContinue = true; - private static volatile string output = "Test results not available yet."; - - static Program() - { - } - - public static void Stress(int concurrency = 0, int prometheusPort = 0) - { -#if DEBUG - Console.WriteLine("***WARNING*** The current build is DEBUG which may affect timing!"); - Console.WriteLine(); -#endif - - if (concurrency < 0) - { - throw new ArgumentOutOfRangeException(nameof(concurrency), "concurrency level should be a non-negative number."); - } - - if (concurrency == 0) - { - concurrency = Environment.ProcessorCount; - } - - using var meter = new Meter("OpenTelemetry.Tests.Stress." + Guid.NewGuid().ToString("D")); - var cntLoopsTotal = 0UL; - meter.CreateObservableCounter( - "OpenTelemetry.Tests.Stress.Loops", - () => unchecked((long)cntLoopsTotal), - description: "The total number of `Run()` invocations that are completed."); - var dLoopsPerSecond = 0D; - meter.CreateObservableGauge( - "OpenTelemetry.Tests.Stress.LoopsPerSecond", - () => dLoopsPerSecond, - description: "The rate of `Run()` invocations based on a small sliding window of few hundreds of milliseconds."); - var dCpuCyclesPerLoop = 0D; - - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - meter.CreateObservableGauge( - "OpenTelemetry.Tests.Stress.CpuCyclesPerLoop", - () => dCpuCyclesPerLoop, - description: "The average CPU cycles for each `Run()` invocation, based on a small sliding window of few hundreds of milliseconds."); - } - - using var meterProvider = prometheusPort != 0 ? Sdk.CreateMeterProviderBuilder() - .AddMeter(meter.Name) - .AddRuntimeInstrumentation() - .AddPrometheusHttpListener( - options => options.UriPrefixes = new string[] { $"http://localhost:{prometheusPort}/" }) - .Build() : null; - - var statistics = new long[concurrency]; - var watchForTotal = Stopwatch.StartNew(); - - Parallel.Invoke( - () => - { - Console.Write($"Running (concurrency = {concurrency}"); - - if (prometheusPort != 0) - { - Console.Write($", prometheusEndpoint = http://localhost:{prometheusPort}/metrics/"); - } - - Console.WriteLine("), press to stop..."); - - var bOutput = false; - var watch = new Stopwatch(); - while (true) - { - if (Console.KeyAvailable) - { - var key = Console.ReadKey(true).Key; - - switch (key) - { - case ConsoleKey.Enter: - Console.WriteLine(string.Format("{0} {1}", DateTime.UtcNow.ToString("O"), output)); - break; - case ConsoleKey.Escape: - bContinue = false; - return; - case ConsoleKey.Spacebar: - bOutput = !bOutput; - break; - } - - continue; - } - - if (bOutput) - { - Console.WriteLine(string.Format("{0} {1}", DateTime.UtcNow.ToString("O"), output)); - } - - var cntLoopsOld = (ulong)statistics.Sum(); - var cntCpuCyclesOld = GetCpuCycles(); - - watch.Restart(); - Thread.Sleep(200); - watch.Stop(); - - cntLoopsTotal = (ulong)statistics.Sum(); - var cntCpuCyclesNew = GetCpuCycles(); - - var nLoops = cntLoopsTotal - cntLoopsOld; - var nCpuCycles = cntCpuCyclesNew - cntCpuCyclesOld; - - dLoopsPerSecond = (double)nLoops / ((double)watch.ElapsedMilliseconds / 1000.0); - dCpuCyclesPerLoop = nLoops == 0 ? 0 : nCpuCycles / nLoops; - - output = $"Loops: {cntLoopsTotal:n0}, Loops/Second: {dLoopsPerSecond:n0}, CPU Cycles/Loop: {dCpuCyclesPerLoop:n0}, RunwayTime (Seconds): {watchForTotal.Elapsed.TotalSeconds:n0} "; - Console.Title = output; - } - }, - () => - { - Parallel.For(0, concurrency, (i) => - { - statistics[i] = 0; - while (bContinue) - { - Run(); - statistics[i]++; - } - }); - }); - - watchForTotal.Stop(); - cntLoopsTotal = (ulong)statistics.Sum(); - var totalLoopsPerSecond = (double)cntLoopsTotal / ((double)watchForTotal.ElapsedMilliseconds / 1000.0); - var cntCpuCyclesTotal = GetCpuCycles(); - var cpuCyclesPerLoopTotal = cntLoopsTotal == 0 ? 0 : cntCpuCyclesTotal / cntLoopsTotal; - Console.WriteLine("Stopping the stress test..."); - Console.WriteLine($"* Total Runaway Time (seconds) {watchForTotal.Elapsed.TotalSeconds:n0}"); - Console.WriteLine($"* Total Loops: {cntLoopsTotal:n0}"); - Console.WriteLine($"* Average Loops/Second: {totalLoopsPerSecond:n0}"); - Console.WriteLine($"* Average CPU Cycles/Loop: {cpuCyclesPerLoopTotal:n0}"); - } - - [DllImport("kernel32.dll")] - [return: MarshalAs(UnmanagedType.Bool)] - private static extern bool QueryProcessCycleTime(IntPtr hProcess, out ulong cycles); - - private static ulong GetCpuCycles() - { - if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - return 0; - } - - if (!QueryProcessCycleTime((IntPtr)(-1), out var cycles)) - { - return 0; - } - - return cycles; - } -} diff --git a/test/OpenTelemetry.Tests.Stress/StressTest.cs b/test/OpenTelemetry.Tests.Stress/StressTest.cs new file mode 100644 index 00000000000..ae19c7f8ece --- /dev/null +++ b/test/OpenTelemetry.Tests.Stress/StressTest.cs @@ -0,0 +1,209 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +using System.Diagnostics; +using System.Diagnostics.Metrics; +using System.Runtime.InteropServices; +using System.Text.Json; +using OpenTelemetry.Metrics; + +namespace OpenTelemetry.Tests.Stress; + +public abstract class StressTest : IDisposable + where T : StressTestOptions +{ + private volatile bool bContinue = true; + private volatile string output = "Test results not available yet."; + + protected StressTest(T options) + { + this.Options = options ?? throw new ArgumentNullException(nameof(options)); + } + + public T Options { get; } + + public void Dispose() + { + this.Dispose(true); + GC.SuppressFinalize(this); + } + + public void RunSynchronously() + { +#if DEBUG + Console.WriteLine("***WARNING*** The current build is DEBUG which may affect timing!"); + Console.WriteLine(); +#endif + + var options = this.Options; + + if (options.Concurrency < 0) + { + throw new ArgumentOutOfRangeException(nameof(options.Concurrency), "Concurrency level should be a non-negative number."); + } + + if (options.Concurrency == 0) + { + options.Concurrency = Environment.ProcessorCount; + } + + using var meter = new Meter("OpenTelemetry.Tests.Stress." + Guid.NewGuid().ToString("D")); + var cntLoopsTotal = 0UL; + meter.CreateObservableCounter( + "OpenTelemetry.Tests.Stress.Loops", + () => unchecked((long)cntLoopsTotal), + description: "The total number of `Run()` invocations that are completed."); + var dLoopsPerSecond = 0D; + meter.CreateObservableGauge( + "OpenTelemetry.Tests.Stress.LoopsPerSecond", + () => dLoopsPerSecond, + description: "The rate of `Run()` invocations based on a small sliding window of few hundreds of milliseconds."); + var dCpuCyclesPerLoop = 0D; + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + meter.CreateObservableGauge( + "OpenTelemetry.Tests.Stress.CpuCyclesPerLoop", + () => dCpuCyclesPerLoop, + description: "The average CPU cycles for each `Run()` invocation, based on a small sliding window of few hundreds of milliseconds."); + } + + using var meterProvider = options.PrometheusInternalMetricsPort != 0 ? Sdk.CreateMeterProviderBuilder() + .AddMeter(meter.Name) + .AddRuntimeInstrumentation() + .AddPrometheusHttpListener(o => o.UriPrefixes = new string[] { $"http://localhost:{options.PrometheusInternalMetricsPort}/" }) + .Build() : null; + + var statistics = new long[options.Concurrency]; + var watchForTotal = Stopwatch.StartNew(); + + TimeSpan? duration = options.DurationSeconds > 0 + ? TimeSpan.FromSeconds(options.DurationSeconds) + : null; + + Parallel.Invoke( + () => + { + Console.WriteLine($"Options: {JsonSerializer.Serialize(options)}"); + Console.WriteLine($"Run {Process.GetCurrentProcess().ProcessName}.exe --help to see available options."); + Console.Write($"Running (concurrency = {options.Concurrency}"); + + if (options.PrometheusInternalMetricsPort != 0) + { + Console.Write($", internalPrometheusEndpoint = http://localhost:{options.PrometheusInternalMetricsPort}/metrics/"); + } + + this.WriteRunInformationToConsole(); + + Console.WriteLine("), press to stop, press to toggle statistics in the console..."); + Console.WriteLine(this.output); + + var outputCursorTop = Console.CursorTop - 1; + + var bOutput = true; + var watch = new Stopwatch(); + while (true) + { + if (Console.KeyAvailable) + { + var key = Console.ReadKey(true).Key; + + switch (key) + { + case ConsoleKey.Enter: + Console.WriteLine(string.Format("{0} {1}", DateTime.UtcNow.ToString("O"), this.output)); + break; + case ConsoleKey.Escape: + this.bContinue = false; + return; + case ConsoleKey.Spacebar: + bOutput = !bOutput; + break; + } + + continue; + } + + if (bOutput) + { + var tempCursorLeft = Console.CursorLeft; + var tempCursorTop = Console.CursorTop; + Console.SetCursorPosition(0, outputCursorTop); + Console.WriteLine(this.output.PadRight(Console.BufferWidth)); + Console.SetCursorPosition(tempCursorLeft, tempCursorTop); + } + + var cntLoopsOld = (ulong)statistics.Sum(); + var cntCpuCyclesOld = StressTestNativeMethods.GetCpuCycles(); + + watch.Restart(); + Thread.Sleep(200); + watch.Stop(); + + cntLoopsTotal = (ulong)statistics.Sum(); + var cntCpuCyclesNew = StressTestNativeMethods.GetCpuCycles(); + + var nLoops = cntLoopsTotal - cntLoopsOld; + var nCpuCycles = cntCpuCyclesNew - cntCpuCyclesOld; + + dLoopsPerSecond = (double)nLoops / ((double)watch.ElapsedMilliseconds / 1000.0); + dCpuCyclesPerLoop = nLoops == 0 ? 0 : nCpuCycles / nLoops; + + var totalElapsedTime = watchForTotal.Elapsed; + + if (duration.HasValue) + { + this.output = $"Loops: {cntLoopsTotal:n0}, Loops/Second: {dLoopsPerSecond:n0}, CPU Cycles/Loop: {dCpuCyclesPerLoop:n0}, RemainingTime (Seconds): {(duration.Value - totalElapsedTime).TotalSeconds:n0}"; + if (totalElapsedTime > duration) + { + this.bContinue = false; + return; + } + } + else + { + this.output = $"Loops: {cntLoopsTotal:n0}, Loops/Second: {dLoopsPerSecond:n0}, CPU Cycles/Loop: {dCpuCyclesPerLoop:n0}, RunningTime (Seconds): {totalElapsedTime.TotalSeconds:n0}"; + } + + Console.Title = this.output; + } + }, + () => + { + Parallel.For(0, options.Concurrency, (i) => + { + ref var count = ref statistics[i]; + + while (this.bContinue) + { + this.RunWorkItemInParallel(); + count++; + } + }); + }); + + watchForTotal.Stop(); + cntLoopsTotal = (ulong)statistics.Sum(); + var totalLoopsPerSecond = (double)cntLoopsTotal / ((double)watchForTotal.ElapsedMilliseconds / 1000.0); + var cntCpuCyclesTotal = StressTestNativeMethods.GetCpuCycles(); + var cpuCyclesPerLoopTotal = cntLoopsTotal == 0 ? 0 : cntCpuCyclesTotal / cntLoopsTotal; + Console.WriteLine("Stopping the stress test..."); + Console.WriteLine($"* Total Running Time (Seconds) {watchForTotal.Elapsed.TotalSeconds:n0}"); + Console.WriteLine($"* Total Loops: {cntLoopsTotal:n0}"); + Console.WriteLine($"* Average Loops/Second: {totalLoopsPerSecond:n0}"); + Console.WriteLine($"* Average CPU Cycles/Loop: {cpuCyclesPerLoopTotal:n0}"); +#if !NETFRAMEWORK + Console.WriteLine($"* GC Total Allocated Bytes: {GC.GetTotalAllocatedBytes()}"); +#endif + } + + protected virtual void WriteRunInformationToConsole() + { + } + + protected abstract void RunWorkItemInParallel(); + + protected virtual void Dispose(bool isDisposing) + { + } +} diff --git a/test/OpenTelemetry.Tests.Stress/StressTestFactory.cs b/test/OpenTelemetry.Tests.Stress/StressTestFactory.cs new file mode 100644 index 00000000000..6f3e7ff9ea7 --- /dev/null +++ b/test/OpenTelemetry.Tests.Stress/StressTestFactory.cs @@ -0,0 +1,34 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +using CommandLine; + +namespace OpenTelemetry.Tests.Stress; + +public static class StressTestFactory +{ + public static int RunSynchronously(string[] commandLineArguments) + where TStressTest : StressTest + { + return RunSynchronously(commandLineArguments); + } + + public static int RunSynchronously(string[] commandLineArguments) + where TStressTest : StressTest + where TStressTestOptions : StressTestOptions + { + return Parser.Default.ParseArguments(commandLineArguments) + .MapResult( + CreateStressTestAndRunSynchronously, + _ => 1); + + static int CreateStressTestAndRunSynchronously(TStressTestOptions options) + { + using var stressTest = (TStressTest)Activator.CreateInstance(typeof(TStressTest), options)!; + + stressTest.RunSynchronously(); + + return 0; + } + } +} diff --git a/test/OpenTelemetry.Tests.Stress/StressTestNativeMethods.cs b/test/OpenTelemetry.Tests.Stress/StressTestNativeMethods.cs new file mode 100644 index 00000000000..da3df1c2864 --- /dev/null +++ b/test/OpenTelemetry.Tests.Stress/StressTestNativeMethods.cs @@ -0,0 +1,28 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +using System.Runtime.InteropServices; + +namespace OpenTelemetry.Tests.Stress; + +internal static class StressTestNativeMethods +{ + public static ulong GetCpuCycles() + { + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + return 0; + } + + if (!QueryProcessCycleTime((IntPtr)(-1), out var cycles)) + { + return 0; + } + + return cycles; + } + + [DllImport("kernel32.dll")] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool QueryProcessCycleTime(IntPtr hProcess, out ulong cycles); +} diff --git a/test/OpenTelemetry.Tests.Stress/StressTestOptions.cs b/test/OpenTelemetry.Tests.Stress/StressTestOptions.cs new file mode 100644 index 00000000000..2dcb2b2e47c --- /dev/null +++ b/test/OpenTelemetry.Tests.Stress/StressTestOptions.cs @@ -0,0 +1,18 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +using CommandLine; + +namespace OpenTelemetry.Tests.Stress; + +public class StressTestOptions +{ + [Option('c', "concurrency", HelpText = "The concurrency (maximum degree of parallelism) for the stress test. Default value: Environment.ProcessorCount.", Required = false)] + public int Concurrency { get; set; } + + [Option('p', "internal_port", HelpText = "The Prometheus http listener port where Prometheus will be exposed for retrieving internal metrics while the stress test is running. Set to '0' to disable. Default value: 9464.", Required = false)] + public int PrometheusInternalMetricsPort { get; set; } = 9464; + + [Option('d', "duration", HelpText = "The duration for the stress test to run in seconds. If set to '0' or a negative value the stress test will run until canceled. Default value: 0.", Required = false)] + public int DurationSeconds { get; set; } +}