diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/ARMTemplate/benchmarkTemplate.json b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/ARMTemplate/benchmarkTemplate.json index 13bb3070da..ac163dbad9 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/ARMTemplate/benchmarkTemplate.json +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/ARMTemplate/benchmarkTemplate.json @@ -45,7 +45,7 @@ "metadata": { "description": "Name for the container group" }, - "defaultValue": "CosmosDBBenchmark" + "defaultValue": "CosmosDBBenchmark" }, "containerName": { "type": "string", @@ -88,9 +88,22 @@ "metadata": { "description": "Location for all resources." } + }, + "applicationInsightsName": { + "type": "string", + "defaultValue": "CosmosDBBenchmarkTestAppInsights", + "metadata": { + "description": "Specifies Application Insights Account Nmae" + } + }, + "dashboardName": { + "type": "string", + "defaultValue": "Cosmos Benchmark Statistics" } }, - "variables": {}, + "variables": { + "appInsightsResourceIds": "[concat('/subscriptions/', subscription().subscriptionId, '/resourceGroups/', resourceGroup().name, '/providers/microsoft.insights/components/', parameters('applicationInsightsName'))]" + }, "resources": [ { "name": "[parameters('containerGroupName')]", @@ -149,6 +162,190 @@ } ] } + }, + { + "name": "[parameters('applicationInsightsName')]", + "type": "Microsoft.Insights/components", + "apiVersion": "2015-05-01", + "location": "[resourceGroup().location]", + "properties": { + "ApplicationId": "[parameters('applicationInsightsName')]", + "Application_Type": "web" + } + }, + { + "name": "CosmosDBBenchmarkDashboard", + "type": "Microsoft.Portal/dashboards", + "location": "[resourceGroup().location]", + "apiVersion": "2015-08-01-preview", + "tags": { + "hidden-title": "[parameters('dashboardName')]" + }, + + "properties": { + "lenses": { + "0": { + "order": 0, + "parts": { + "0": { + "position": { + "x": 0, + "y": 0, + "colSpan": 11, + "rowSpan": 5 + }, + "metadata": { + "inputs": [ + { + "name": "resourceTypeMode", + "isOptional": true + }, + { + "name": "ComponentId", + "isOptional": true + }, + { + "name": "Scope", + "value": { + "resourceIds": [ + "[variables('appInsightsResourceIds')]" + ] + }, + "isOptional": true + }, + { + "name": "PartId", + "value": "104528fc-0216-49c6-bf70-fffe9d37f93d", + "isOptional": true + }, + { + "name": "Version", + "value": "2.0", + "isOptional": true + }, + { + "name": "TimeRange", + "isOptional": true + }, + { + "name": "DashboardId", + "isOptional": true + }, + { + "name": "DraftRequestParameters", + "isOptional": true + }, + { + "name": "Query", + "value": "customMetrics\n| where name == \"ReadOperationLatencyInMs\" and timestamp > ago(1d)\n| summarize\n percentile(value, 50),\n percentile(value, 75),\n percentile(value, 90),\n percentile(value, 95),\n percentile(value, 99),\n percentile(value, 99.9),\n percentile(value, 99.99)\n by ts = bin(timestamp, 5s)\n| render timechart \n\n", + "isOptional": true + }, + { + "name": "ControlType", + "value": "FrameControlChart", + "isOptional": true + }, + { + "name": "SpecificChart", + "value": "Line", + "isOptional": true + }, + { + "name": "PartTitle", + "value": "Analytics", + "isOptional": true + }, + { + "name": "PartSubTitle", + "value": "CosmosDBBenchmarkTestAppInsights", + "isOptional": true + }, + { + "name": "Dimensions", + "value": { + "xAxis": { + "name": "ts", + "type": "datetime" + }, + "yAxis": [ + { + "name": "percentile_value_50", + "type": "real" + }, + { + "name": "percentile_value_75", + "type": "real" + }, + { + "name": "percentile_value_90", + "type": "real" + }, + { + "name": "percentile_value_95", + "type": "real" + } + ], + "splitBy": [], + "aggregation": "Sum" + }, + "isOptional": true + }, + { + "name": "LegendOptions", + "value": { + "isEnabled": true, + "position": "Bottom" + }, + "isOptional": true + }, + { + "name": "IsQueryContainTimeRange", + "value": true, + "isOptional": true + } + ], + "type": "Extension/Microsoft_OperationsManagementSuite_Workspace/PartType/LogsDashboardPart", + "settings": {} + } + } + } + } + }, + "metadata": { + "model": { + "timeRange": { + "value": { + "relative": { + "duration": 24, + "timeUnit": 1 + } + }, + "type": "MsPortalFx.Composition.Configuration.ValueTypes.TimeRange" + }, + "filterLocale": { + "value": "en-us" + }, + "filters": { + "value": { + "MsPortalFx_TimeRange": { + "model": { + "format": "utc", + "granularity": "auto", + "relative": "24h" + }, + "displayCache": { + "name": "UTC Time", + "value": "Past 24 hours" + }, + "filteredPartIds": [ + "StartboardPart-LogsDashboardPart-4f9d44ab-efbe-4310-8809-63ae942a3091" + ] + } + } + } + } + } + } } ] } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfig.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfig.cs index 6f89776370..71e2c5eef5 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfig.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfig.cs @@ -123,6 +123,18 @@ public class BenchmarkConfig [Option(Required = false, HelpText = "Container to publish results to")] public string ResultsContainer { get; set; } = "runsummary"; + [Option(Required = false, HelpText = "Metrics reporting interval in seconds")] + public int MetricsReportingIntervalInSec { get; set; } = 5; + + [Option(Required = false, HelpText = "Application Insights instrumentation key")] + public string AppInsightsInstrumentationKey { get; set; } + + [Option(Required = false, HelpText = "The reservoir sample size.")] + public int ReservoirSampleSize { get; set; } = 1028; + + [Option(Required = false, HelpText = "Logging context name. The default value is \"CosmosDBBenchmarkLoggingContext\"")] + public string LoggingContextIdentifier { get; set; } = "CosmosDBBenchmarkLoggingContext"; + internal int GetTaskCount(int containerThroughput) { int taskCount = this.DegreeOfParallelism; @@ -270,4 +282,4 @@ private static void HandleParseError(IEnumerable errors) Environment.Exit(errors.Count()); } } -} +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/CosmosBenchmark.csproj b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/CosmosBenchmark.csproj index c8ac26f5aa..94b99d2501 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/CosmosBenchmark.csproj +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/CosmosBenchmark.csproj @@ -17,9 +17,15 @@ + + + + + + diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/BenchmarkOperation.cs new file mode 100644 index 0000000000..8cf2da3278 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/BenchmarkOperation.cs @@ -0,0 +1,15 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace CosmosBenchmark +{ + using System.Threading.Tasks; + + internal abstract class BenchmarkOperation : IBenchmarkOperation + { + public abstract Task ExecuteOnceAsync(); + + public abstract Task PrepareAsync(); + } +} diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutionStrategy.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutionStrategy.cs index b18fc34dc7..fddbd0808f 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutionStrategy.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutionStrategy.cs @@ -6,6 +6,9 @@ namespace CosmosBenchmark { using System; using System.Threading.Tasks; + using Microsoft.ApplicationInsights; + using Microsoft.Extensions.Logging; + using OpenTelemetry.Metrics; internal interface IExecutionStrategy { @@ -19,7 +22,8 @@ public Task ExecuteAsync( BenchmarkConfig benchmarkConfig, int serialExecutorConcurrency, int serialExecutorIterationCount, - double warmupFraction); - + double warmupFraction, + ILogger logger, + MeterProvider meterProvider); } } diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutor.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutor.cs index be2ee2e761..a820306e3d 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutor.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutor.cs @@ -6,6 +6,8 @@ namespace CosmosBenchmark { using System; using System.Threading.Tasks; + using Microsoft.Extensions.Logging; + using OpenTelemetry.Metrics; internal interface IExecutor { @@ -17,6 +19,9 @@ public Task ExecuteAsync( int iterationCount, bool isWarmup, bool traceFailures, - Action completionCallback); + Action completionCallback, + BenchmarkConfig benchmarkConfig, + ILogger logger, + MeterProvider meterProvider); } } diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IMetricsCollector.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IMetricsCollector.cs new file mode 100644 index 0000000000..faba3d7870 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IMetricsCollector.cs @@ -0,0 +1,15 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace CosmosBenchmark +{ + public interface IMetricsCollector + { + void RecordLatencyAndRps(double milliseconds); + + void CollectMetricsOnSuccess(); + + void CollectMetricsOnFailure(); + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertBenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertBenchmarkOperation.cs new file mode 100644 index 0000000000..117fe5ae09 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertBenchmarkOperation.cs @@ -0,0 +1,10 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace CosmosBenchmark +{ + internal abstract class InsertBenchmarkOperation : BenchmarkOperation + { + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertOperationMetricsCollector.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertOperationMetricsCollector.cs new file mode 100644 index 0000000000..060a85780d --- /dev/null +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertOperationMetricsCollector.cs @@ -0,0 +1,28 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace CosmosBenchmark +{ + using Microsoft.ApplicationInsights; + using System.Diagnostics.Metrics; + + internal class InsertOperationMetricsCollector : MetricsCollector + { + public InsertOperationMetricsCollector(Meter meter) : base(meter) + { + } + + protected override string RpsHistogramName => "InsertOperationRpsHistogram"; + + protected override string LatencyInMsHistogramName => "InsertOperationLatencyInMsHistogram"; + + protected override string RpsMetricName => "InsertOperationRps"; + + protected override string LatencyInMsMetricName => "InsertOperationLatencyInMs"; + + protected override string FailureOperationMetricName => "InsertOperationFailure"; + + protected override string SuccessOperationMetricName => "InsertOperationSuccess"; + } +} diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollector.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollector.cs new file mode 100644 index 0000000000..8925bff3a2 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollector.cs @@ -0,0 +1,74 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace CosmosBenchmark +{ + using System.Diagnostics.Metrics; + + internal abstract class MetricsCollector : IMetricsCollector + { + private readonly Meter meter; + + private readonly Histogram operationLatencyHistogram; + + private readonly Histogram rpsMetricNameHistogram; + + private readonly Counter successOperationCounter; + + private readonly Counter failureOperationCounter; + + private readonly ObservableGauge latencyInMsMetricNameGauge; + + private readonly ObservableGauge rpsNameGauge; + + private double latencyInMs; + + private double rps; + + public MetricsCollector(Meter meter) + { + this.meter = meter; + this.rpsMetricNameHistogram = meter.CreateHistogram(this.RpsHistogramName); + this.operationLatencyHistogram = meter.CreateHistogram(this.LatencyInMsHistogramName); + this.successOperationCounter = meter.CreateCounter(this.SuccessOperationMetricName); + this.failureOperationCounter = meter.CreateCounter(this.FailureOperationMetricName); + + this.latencyInMsMetricNameGauge = this.meter.CreateObservableGauge(this.LatencyInMsMetricName, + () => new Measurement(this.latencyInMs)); + + this.rpsNameGauge = this.meter.CreateObservableGauge(this.LatencyInMsMetricName, + () => new Measurement(this.rps)); + } + + public void CollectMetricsOnSuccess() + { + this.successOperationCounter.Add(1); + } + + public void CollectMetricsOnFailure() + { + this.failureOperationCounter.Add(1); + } + + public void RecordLatencyAndRps(double milliseconds) + { + this.rps = 1000 / milliseconds; + this.latencyInMs = milliseconds; + this.rpsMetricNameHistogram.Record(this.rps); + this.operationLatencyHistogram.Record(this.latencyInMs); + } + + protected abstract string RpsHistogramName { get; } + + protected abstract string LatencyInMsHistogramName { get; } + + protected abstract string RpsMetricName { get; } + + protected abstract string LatencyInMsMetricName { get; } + + protected abstract string FailureOperationMetricName { get; } + + protected abstract string SuccessOperationMetricName { get; } + } +} diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollectorProvider.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollectorProvider.cs new file mode 100644 index 0000000000..104551f240 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollectorProvider.cs @@ -0,0 +1,67 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace CosmosBenchmark +{ + using System; + using System.Diagnostics.Metrics; + using OpenTelemetry.Metrics; + + internal static class MetricsCollectorProvider + { + private static MetricCollectionWindow metricCollectionWindow; + + private static readonly object metricCollectionWindowLock = new object(); + + private static readonly Meter insertOperationMeter = new("CosmosBenchmarkInsertOperationMeter"); + + private static readonly Meter queryOperationMeter = new("CosmosBenchmarkQueryOperationMeter"); + + private static readonly Meter readOperationMeter = new ("CosmosBenchmarkReadOperationMeter"); + + private static MetricCollectionWindow GetCurrentMetricCollectionWindow(MeterProvider meterProvider, BenchmarkConfig config) + { + if (CheckMetricCollectionInvalid(metricCollectionWindow, config)) + { + lock (metricCollectionWindowLock) + { + if (CheckMetricCollectionInvalid(metricCollectionWindow, config)) + { + // Reset Meter provider and meters. + metricCollectionWindow = new MetricCollectionWindow(); + meterProvider.ForceFlush(); + } + } + } + + return metricCollectionWindow; + } + + private static bool CheckMetricCollectionInvalid(MetricCollectionWindow metricCollectionWindow, BenchmarkConfig config) + { + return metricCollectionWindow is null || DateTime.Now > metricCollectionWindow.DateTimeCreated.AddSeconds(config.MetricsReportingIntervalInSec); + } + + public static IMetricsCollector GetMetricsCollector(IBenchmarkOperation benchmarkOperation, MeterProvider meterProvider, BenchmarkConfig config) + { + Type benchmarkOperationType = benchmarkOperation.GetType(); + if (typeof(InsertBenchmarkOperation).IsAssignableFrom(benchmarkOperationType)) + { + return GetCurrentMetricCollectionWindow(meterProvider, config).GetInsertOperationMetricsCollector(insertOperationMeter); + } + else if (typeof(QueryBenchmarkOperation).IsAssignableFrom(benchmarkOperationType)) + { + return GetCurrentMetricCollectionWindow(meterProvider, config).GetQueryOperationMetricsCollector(queryOperationMeter); + } + else if (typeof(ReadBenchmarkOperation).IsAssignableFrom(benchmarkOperationType)) + { + return GetCurrentMetricCollectionWindow(meterProvider, config).GetReadOperationMetricsCollector(readOperationMeter); + } + else + { + throw new NotSupportedException($"The type {nameof(benchmarkOperationType)} is not supported for collecting metrics."); + } + } + } +} diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ParallelExecutionStrategy.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ParallelExecutionStrategy.cs index e5dc7bb58d..a88e365db9 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ParallelExecutionStrategy.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ParallelExecutionStrategy.cs @@ -10,7 +10,10 @@ namespace CosmosBenchmark using System.Linq; using System.Threading; using System.Threading.Tasks; + using Microsoft.ApplicationInsights; + using Microsoft.Extensions.Logging; using Newtonsoft.Json; + using OpenTelemetry.Metrics; internal class ParallelExecutionStrategy : IExecutionStrategy { @@ -28,7 +31,9 @@ public async Task ExecuteAsync( BenchmarkConfig benchmarkConfig, int serialExecutorConcurrency, int serialExecutorIterationCount, - double warmupFraction) + double warmupFraction, + ILogger logger, + MeterProvider meterProvider) { IExecutor warmupExecutor = new SerialOperationExecutor( executorId: "Warmup", @@ -37,7 +42,10 @@ await warmupExecutor.ExecuteAsync( (int)(serialExecutorIterationCount * warmupFraction), isWarmup: true, traceFailures: benchmarkConfig.TraceFailures, - completionCallback: () => { }); + completionCallback: () => { }, + benchmarkConfig, + logger, + meterProvider); IExecutor[] executors = new IExecutor[serialExecutorConcurrency]; for (int i = 0; i < serialExecutorConcurrency; i++) @@ -54,17 +62,24 @@ await warmupExecutor.ExecuteAsync( iterationCount: serialExecutorIterationCount, isWarmup: false, traceFailures: benchmarkConfig.TraceFailures, - completionCallback: () => Interlocked.Decrement(ref this.pendingExecutorCount)); + completionCallback: () => Interlocked.Decrement(ref this.pendingExecutorCount), + benchmarkConfig, + logger, + meterProvider); } return await this.LogOutputStats( benchmarkConfig, - executors); + executors, + logger, + meterProvider); } private async Task LogOutputStats( BenchmarkConfig benchmarkConfig, - IExecutor[] executors) + IExecutor[] executors, + ILogger logger, + MeterProvider meterProvider) { const int outputLoopDelayInSeconds = 1; IList perLoopCounters = new List(); @@ -135,6 +150,8 @@ private async Task LogOutputStats( runSummary.Top80PercentAverageRps = Math.Round(summaryCounters.Take((int)(0.8 * summaryCounters.Length)).Average(), 0); runSummary.Top90PercentAverageRps = Math.Round(summaryCounters.Take((int)(0.9 * summaryCounters.Length)).Average(), 0); runSummary.Top95PercentAverageRps = Math.Round(summaryCounters.Take((int)(0.95 * summaryCounters.Length)).Average(), 0); + runSummary.Top95PercentAverageRps = Math.Round(summaryCounters.Take((int)(0.999 * summaryCounters.Length)).Average(), 0); + runSummary.Top95PercentAverageRps = Math.Round(summaryCounters.Take((int)(0.9999 * summaryCounters.Length)).Average(), 0); runSummary.Top99PercentAverageRps = Math.Round(summaryCounters.Take((int)(0.99 * summaryCounters.Length)).Average(), 0); runSummary.AverageRps = Math.Round(summaryCounters.Average(), 0); @@ -144,6 +161,8 @@ private async Task LogOutputStats( runSummary.Top95PercentLatencyInMs = TelemetrySpan.GetLatencyPercentile(95); runSummary.Top98PercentLatencyInMs = TelemetrySpan.GetLatencyPercentile(98); runSummary.Top99PercentLatencyInMs = TelemetrySpan.GetLatencyPercentile(99); + runSummary.Top999PercentLatencyInMs = TelemetrySpan.GetLatencyQuantile(0.999); + runSummary.Top9999PercentLatencyInMs = TelemetrySpan.GetLatencyQuantile(0.9999); runSummary.MaxLatencyInMs = TelemetrySpan.GetLatencyPercentile(100); string summary = JsonConvert.SerializeObject(runSummary); @@ -160,4 +179,4 @@ private async Task LogOutputStats( } } } -} +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/QueryBenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/QueryBenchmarkOperation.cs new file mode 100644 index 0000000000..1308a9b83e --- /dev/null +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/QueryBenchmarkOperation.cs @@ -0,0 +1,10 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace CosmosBenchmark +{ + internal abstract class QueryBenchmarkOperation : BenchmarkOperation + { + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/QueryOperationMetricsCollector.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/QueryOperationMetricsCollector.cs new file mode 100644 index 0000000000..c9a247d530 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/QueryOperationMetricsCollector.cs @@ -0,0 +1,27 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace CosmosBenchmark +{ + using System.Diagnostics.Metrics; + + internal class QueryOperationMetricsCollector : MetricsCollector + { + public QueryOperationMetricsCollector(Meter meter) : base(meter) + { + } + + protected override string RpsHistogramName => "QueryOperationRpsHistogram"; + + protected override string LatencyInMsHistogramName => "QueryOperationLatencyInMsHistogram"; + + protected override string RpsMetricName => "QueryOperationRps"; + + protected override string LatencyInMsMetricName => "QueryOperationLatencyInMs"; + + protected override string FailureOperationMetricName => "QueryOperationFailure"; + + protected override string SuccessOperationMetricName => "QueryOperationSuccess"; + } +} diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ReadBenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ReadBenchmarkOperation.cs new file mode 100644 index 0000000000..55e42af491 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ReadBenchmarkOperation.cs @@ -0,0 +1,10 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace CosmosBenchmark +{ + internal abstract class ReadBenchmarkOperation : BenchmarkOperation + { + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ReadOperationMetricsCollector.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ReadOperationMetricsCollector.cs new file mode 100644 index 0000000000..2ba49cc438 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ReadOperationMetricsCollector.cs @@ -0,0 +1,27 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace CosmosBenchmark +{ + using System.Diagnostics.Metrics; + + internal class ReadOperationMetricsCollector : MetricsCollector + { + public ReadOperationMetricsCollector(Meter meter) : base(meter) + { + } + + protected override string RpsHistogramName => "ReadOperationRpsHistogram"; + + protected override string LatencyInMsHistogramName => "ReadOperationLatencyInMsHistogram"; + + protected override string RpsMetricName => "ReadOperationRps"; + + protected override string LatencyInMsMetricName => "ReadOperationLatencyInMs"; + + protected override string FailureOperationMetricName => "ReadOperationFailure"; + + protected override string SuccessOperationMetricName => "ReadOperationSuccess"; + } +} diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/SerialOperationExecutor.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/SerialOperationExecutor.cs index 1086c31bd5..99428d154d 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/SerialOperationExecutor.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/SerialOperationExecutor.cs @@ -8,10 +8,13 @@ namespace CosmosBenchmark using System.Diagnostics; using System.Threading.Tasks; using Microsoft.Azure.Cosmos; + using Microsoft.Extensions.Logging; + using OpenTelemetry.Metrics; internal class SerialOperationExecutor : IExecutor { private readonly IBenchmarkOperation operation; + private readonly string executorId; public SerialOperationExecutor( @@ -26,6 +29,7 @@ public SerialOperationExecutor( } public int SuccessOperationCount { get; private set; } + public int FailedOperationCount { get; private set; } public double TotalRuCharges { get; private set; } @@ -34,27 +38,37 @@ public async Task ExecuteAsync( int iterationCount, bool isWarmup, bool traceFailures, - Action completionCallback) + Action completionCallback, + BenchmarkConfig benchmarkConfig, + ILogger logger, + MeterProvider meterProvider) { - Trace.TraceInformation($"Executor {this.executorId} started"); + logger.LogInformation($"Executor {this.executorId} started"); + + logger.LogInformation("Initializing counters and metrics."); try { int currentIterationCount = 0; do { + IMetricsCollector metricsCollector = MetricsCollectorProvider.GetMetricsCollector(this.operation, meterProvider, benchmarkConfig); + OperationResult? operationResult = null; await this.operation.PrepareAsync(); using (IDisposable telemetrySpan = TelemetrySpan.StartNew( () => operationResult.Value, - disableTelemetry: isWarmup)) + disableTelemetry: isWarmup, + metricsCollector.RecordLatencyAndRps)) { try { operationResult = await this.operation.ExecuteOnceAsync(); + metricsCollector.CollectMetricsOnSuccess(); + // Success case this.SuccessOperationCount++; this.TotalRuCharges += operationResult.Value.RuCharges; @@ -71,6 +85,8 @@ public async Task ExecuteAsync( Console.WriteLine(ex.ToString()); } + metricsCollector.CollectMetricsOnFailure(); + // failure case this.FailedOperationCount++; @@ -102,4 +118,4 @@ public async Task ExecuteAsync( } } } -} +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/TelemetrySpan.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/TelemetrySpan.cs index 13d971aa2a..564f3253e1 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/TelemetrySpan.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/TelemetrySpan.cs @@ -18,11 +18,13 @@ internal struct TelemetrySpan : IDisposable private Stopwatch stopwatch; private Func lazyOperationResult; + private Action recordLatencyAction; private bool disableTelemetry; public static IDisposable StartNew( Func lazyOperationResult, - bool disableTelemetry) + bool disableTelemetry, + Action recordLatencyAction) { if (disableTelemetry || !TelemetrySpan.IncludePercentile) { @@ -33,6 +35,7 @@ public static IDisposable StartNew( { stopwatch = Stopwatch.StartNew(), lazyOperationResult = lazyOperationResult, + recordLatencyAction = recordLatencyAction, disableTelemetry = disableTelemetry }; } @@ -47,6 +50,8 @@ public void Dispose() if (TelemetrySpan.IncludePercentile) { RecordLatency(this.stopwatch.Elapsed.TotalMilliseconds); + + this.recordLatencyAction?.Invoke(this.stopwatch.Elapsed.TotalMilliseconds); } BenchmarkLatencyEventSource.Instance.LatencyDiagnostics( @@ -79,6 +84,16 @@ internal static void ResetLatencyHistogram(int totalNumberOfIterations) return MathNet.Numerics.Statistics.Statistics.Percentile(latencyHistogram.Take(latencyIndex + 1), percentile); } + internal static double? GetLatencyQuantile(double quantile) + { + if (latencyHistogram == null) + { + return null; + } + + return MathNet.Numerics.Statistics.Statistics.Quantile(latencyHistogram.Take(latencyIndex + 1), quantile); + } + private class NoOpDisposable : IDisposable { public static readonly NoOpDisposable Instance = new NoOpDisposable(); diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/IBenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/IBenchmarkOperation.cs index d6933fcce5..9a3ea70894 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/IBenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/IBenchmarkOperation.cs @@ -8,8 +8,8 @@ namespace CosmosBenchmark internal interface IBenchmarkOperation { - Task PrepareAsync(); - Task ExecuteOnceAsync(); + + Task PrepareAsync(); } } diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/MetricCollectionWindow.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/MetricCollectionWindow.cs new file mode 100644 index 0000000000..9b2a3ac13a --- /dev/null +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/MetricCollectionWindow.cs @@ -0,0 +1,63 @@ +namespace CosmosBenchmark +{ + using System; + using System.Diagnostics.Metrics; + + internal class MetricCollectionWindow + { + private InsertOperationMetricsCollector insertOperationMetricsCollector; + private readonly object insertOperationMetricsCollectorLock = new object(); + + private QueryOperationMetricsCollector queryOperationMetricsCollector; + private readonly object queryOperationMetricsCollectorLock = new object(); + + private ReadOperationMetricsCollector readOperationMetricsCollector; + private readonly object readOperationMetricsCollectorLock = new object(); + + public DateTime DateTimeCreated { get; init; } + + public MetricCollectionWindow() + { + this.DateTimeCreated = DateTime.Now; + } + + public InsertOperationMetricsCollector GetInsertOperationMetricsCollector(Meter meter) + { + if (this.insertOperationMetricsCollector is null) + { + lock (this.insertOperationMetricsCollectorLock) + { + this.insertOperationMetricsCollector ??= new InsertOperationMetricsCollector(meter); + } + } + + return this.insertOperationMetricsCollector; + } + + public QueryOperationMetricsCollector GetQueryOperationMetricsCollector(Meter meter) + { + if (this.queryOperationMetricsCollector is null) + { + lock (this.queryOperationMetricsCollectorLock) + { + this.queryOperationMetricsCollector ??= new QueryOperationMetricsCollector(meter); + } + } + + return this.queryOperationMetricsCollector; + } + + public ReadOperationMetricsCollector GetReadOperationMetricsCollector(Meter meter) + { + if (this.readOperationMetricsCollector is null) + { + lock (this.readOperationMetricsCollectorLock) + { + this.readOperationMetricsCollector ??= new ReadOperationMetricsCollector(meter); + } + } + + return this.readOperationMetricsCollector; + } + } +} diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Program.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Program.cs index 99131262d7..6a28432d40 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Program.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Program.cs @@ -14,8 +14,12 @@ namespace CosmosBenchmark using System.Reflection; using System.Threading; using System.Threading.Tasks; + using Azure.Monitor.OpenTelemetry.Exporter; using Microsoft.Azure.Cosmos; + using Microsoft.Extensions.Logging; using Newtonsoft.Json.Linq; + using OpenTelemetry; + using OpenTelemetry.Metrics; /// /// This sample demonstrates how to achieve high performance writes using Azure Comsos DB. @@ -32,7 +36,19 @@ public static async Task Main(string[] args) { BenchmarkConfig config = BenchmarkConfig.From(args); await Program.AddAzureInfoToRunSummary(); - + + OpenTelemetry.Trace.TracerProviderBuilder tracerProviderBuilder = Sdk.CreateTracerProviderBuilder() + .AddAzureMonitorTraceExporter(); + + MeterProvider meterProvider = Sdk.CreateMeterProviderBuilder() + .AddAzureMonitorMetricExporter(configure: new Action((options) => Configure(options, config))) + .AddMeter("CosmosBenchmarkInsertOperationMeter") + .AddMeter("CosmosBenchmarkQueryOperationMeter") + .AddMeter("CosmosBenchmarkReadOperationMeter") + .AddConsoleExporter() + .AddAzureMonitorMetricExporter() + .Build(); + ThreadPool.SetMinThreads(config.MinThreadPoolSize, config.MinThreadPoolSize); if (config.EnableLatencyPercentiles) @@ -43,9 +59,12 @@ public static async Task Main(string[] args) config.Print(); + using ILoggerFactory loggerFactory = LoggerFactory.Create(builder => builder.AddConsole()); + ILogger logger = loggerFactory.CreateLogger(); + Program program = new Program(); - RunSummary runSummary = await program.ExecuteAsync(config); + RunSummary runSummary = await program.ExecuteAsync(config, logger, meterProvider); } finally { @@ -75,17 +94,22 @@ private static async Task AddAzureInfoToRunSummary() RunSummary.Location = jObject["compute"]["location"].ToString(); Console.WriteLine($"Azure VM Location:{RunSummary.Location}"); } - catch(Exception e) + catch (Exception e) { Console.WriteLine("Failed to get Azure VM info:" + e.ToString()); } } + private static void Configure(AzureMonitorExporterOptions options, BenchmarkConfig config) + { + options.ConnectionString = $"InstrumentationKey={config.AppInsightsInstrumentationKey}"; + } + /// /// Executing benchmarks for V2/V3 cosmosdb SDK. /// /// a Task object. - private async Task ExecuteAsync(BenchmarkConfig config) + private async Task ExecuteAsync(BenchmarkConfig config, ILogger logger, MeterProvider meterProvider) { // V3 SDK client initialization using (CosmosClient cosmosClient = config.CreateCosmosClient(config.Key)) @@ -137,7 +161,7 @@ private async Task ExecuteAsync(BenchmarkConfig config) } IExecutionStrategy execution = IExecutionStrategy.StartNew(benchmarkOperationFactory); - runSummary = await execution.ExecuteAsync(config, taskCount, opsPerTask, 0.01); + runSummary = await execution.ExecuteAsync(config, taskCount, opsPerTask, 0.01, logger, meterProvider); } if (config.CleanupOnFinish) @@ -159,8 +183,8 @@ private async Task ExecuteAsync(BenchmarkConfig config) { runSummary.Diagnostics = CosmosDiagnosticsLogger.GetDiagnostics(); await this.PublishResults( - config, - runSummary, + config, + runSummary, cosmosClient); } @@ -169,8 +193,8 @@ await this.PublishResults( } private async Task PublishResults( - BenchmarkConfig config, - RunSummary runSummary, + BenchmarkConfig config, + RunSummary runSummary, CosmosClient benchmarkClient) { if (string.IsNullOrEmpty(config.ResultsEndpoint)) @@ -266,8 +290,8 @@ private static async Task CreatePartitionedContainerAsync(Ben { return await container.ReadContainerAsync(); } - catch(CosmosException ex) when (ex.StatusCode == HttpStatusCode.NotFound) - { + catch (CosmosException ex) when (ex.StatusCode == HttpStatusCode.NotFound) + { // Show user cost of running this test double estimatedCostPerMonth = 0.06 * options.Throughput; double estimatedCostPerHour = estimatedCostPerMonth / (24 * 30); @@ -288,4 +312,4 @@ private static void ClearCoreSdkListeners() traceSource.Listeners.Clear(); } } -} +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/RunSummary.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/RunSummary.cs index 4f84322554..7b74b8702a 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/RunSummary.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/RunSummary.cs @@ -79,6 +79,8 @@ public RunSummary( public double? Top95PercentLatencyInMs { get; set; } public double? Top98PercentLatencyInMs { get; set; } public double? Top99PercentLatencyInMs { get; set; } + public double? Top999PercentLatencyInMs { get; set; } + public double? Top9999PercentLatencyInMs { get; set; } public double? MaxLatencyInMs { get; set; } public double AverageRps { get; set; } diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/InsertV2BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/InsertV2BenchmarkOperation.cs index 744bb9f577..e01ab060e8 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/InsertV2BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/InsertV2BenchmarkOperation.cs @@ -10,7 +10,7 @@ namespace CosmosBenchmark using Microsoft.Azure.Documents; using Microsoft.Azure.Documents.Client; - internal class InsertV2BenchmarkOperation : IBenchmarkOperation + internal class InsertV2BenchmarkOperation : InsertBenchmarkOperation { private readonly DocumentClient documentClient; private readonly Uri containerUri; @@ -38,7 +38,7 @@ public InsertV2BenchmarkOperation( this.sampleJObject = JsonHelper.Deserialize>(sampleJson); } - public async Task ExecuteOnceAsync() + public override async Task ExecuteOnceAsync() { ResourceResponse itemResponse = await this.documentClient.CreateDocumentAsync( this.containerUri, @@ -58,7 +58,7 @@ public async Task ExecuteOnceAsync() }; } - public Task PrepareAsync() + public override Task PrepareAsync() { string newPartitionKey = Guid.NewGuid().ToString(); diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/QueryStreamSinglePkV2BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/QueryStreamSinglePkV2BenchmarkOperation.cs index 1bf3bf245e..908850dd39 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/QueryStreamSinglePkV2BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/QueryStreamSinglePkV2BenchmarkOperation.cs @@ -12,7 +12,7 @@ namespace CosmosBenchmark using Microsoft.Azure.Documents.Client; using Microsoft.Azure.Documents.Linq; - internal class QueryStreamSinglePkV2BenchmarkOperation : IBenchmarkOperation + internal class QueryStreamSinglePkV2BenchmarkOperation : QueryBenchmarkOperation { private readonly DocumentClient documentClient; private readonly string partitionKeyPath; @@ -47,7 +47,7 @@ public QueryStreamSinglePkV2BenchmarkOperation( this.containerUri = UriFactory.CreateDocumentCollectionUri(this.databsaeName, this.containerName); } - public async Task ExecuteOnceAsync() + public override async Task ExecuteOnceAsync() { IDocumentQuery query = this.documentClient.CreateDocumentQuery( this.containerUri, @@ -85,7 +85,7 @@ public async Task ExecuteOnceAsync() }; } - public async Task PrepareAsync() + public override async Task PrepareAsync() { if (this.initialized) { diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/QueryTSinglePkV2BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/QueryTSinglePkV2BenchmarkOperation.cs index 7d58447859..3cc32d295e 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/QueryTSinglePkV2BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/QueryTSinglePkV2BenchmarkOperation.cs @@ -12,7 +12,7 @@ namespace CosmosBenchmark using Microsoft.Azure.Documents.Client; using Microsoft.Azure.Documents.Linq; - internal class QueryTSinglePkV2BenchmarkOperation : IBenchmarkOperation + internal class QueryTSinglePkV2BenchmarkOperation : QueryBenchmarkOperation { private readonly DocumentClient documentClient; private readonly string partitionKeyPath; @@ -46,7 +46,7 @@ public QueryTSinglePkV2BenchmarkOperation( this.sampleJObject[this.partitionKeyPath] = this.executionItemPartitionKey; } - public async Task ExecuteOnceAsync() + public override async Task ExecuteOnceAsync() { IDocumentQuery> query = this.documentClient.CreateDocumentQuery>( this.containerUri, @@ -93,7 +93,7 @@ public async Task ExecuteOnceAsync() }; } - public async Task PrepareAsync() + public override async Task PrepareAsync() { if (this.initialized) { diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadFeedStreamV2BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadFeedStreamV2BenchmarkOperation.cs index 379cd363e6..dd687e0f8b 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadFeedStreamV2BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadFeedStreamV2BenchmarkOperation.cs @@ -12,7 +12,7 @@ namespace CosmosBenchmark using Microsoft.Azure.Documents; using Microsoft.Azure.Documents.Client; - internal class ReadFeedStreamV2BenchmarkOperation : IBenchmarkOperation + internal class ReadFeedStreamV2BenchmarkOperation : ReadBenchmarkOperation { private readonly DocumentClient documentClient; private readonly string partitionKeyPath; @@ -40,7 +40,7 @@ public ReadFeedStreamV2BenchmarkOperation( this.sampleJObject = JsonHelper.Deserialize>(sampleJson); } - public async Task ExecuteOnceAsync() + public override async Task ExecuteOnceAsync() { Uri containerUri = UriFactory.CreateDocumentCollectionUri(this.databsaeName, this.containerName); FeedResponse feedResponse = await this.documentClient.ReadDocumentFeedAsync( @@ -57,7 +57,7 @@ public async Task ExecuteOnceAsync() }; } - public async Task PrepareAsync() + public override async Task PrepareAsync() { if (string.IsNullOrEmpty(this.nextExecutionItemId) || string.IsNullOrEmpty(this.nextExecutionItemPartitionKey)) diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadNotExistsV2BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadNotExistsV2BenchmarkOperation.cs index f5e8f94c68..4c66893495 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadNotExistsV2BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadNotExistsV2BenchmarkOperation.cs @@ -12,7 +12,7 @@ namespace CosmosBenchmark using Microsoft.Azure.Documents; using Microsoft.Azure.Documents.Client; - internal class ReadNotExistsV2BenchmarkOperation : IBenchmarkOperation + internal class ReadNotExistsV2BenchmarkOperation : ReadBenchmarkOperation { private readonly string databsaeName; private readonly string containerName; @@ -35,7 +35,7 @@ public ReadNotExistsV2BenchmarkOperation( this.containerName = containerName; } - public async Task ExecuteOnceAsync() + public override async Task ExecuteOnceAsync() { Uri itemUri = UriFactory.CreateDocumentUri(this.databsaeName, this.containerName, this.nextExecutionItemId); @@ -65,7 +65,7 @@ public async Task ExecuteOnceAsync() } } - public Task PrepareAsync() + public override Task PrepareAsync() { if (string.IsNullOrEmpty(this.nextExecutionItemId) || string.IsNullOrEmpty(this.nextExecutionItemPartitionKey)) diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadStreamExistsV2BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadStreamExistsV2BenchmarkOperation.cs index 7afd4cdf71..fda0c58c43 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadStreamExistsV2BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadStreamExistsV2BenchmarkOperation.cs @@ -12,7 +12,7 @@ namespace CosmosBenchmark using Microsoft.Azure.Documents; using Microsoft.Azure.Documents.Client; - internal class ReadStreamExistsV2BenchmarkOperation : IBenchmarkOperation + internal class ReadStreamExistsV2BenchmarkOperation : ReadBenchmarkOperation { private readonly string partitionKeyPath; private readonly Dictionary sampleJObject; @@ -41,7 +41,7 @@ public ReadStreamExistsV2BenchmarkOperation( this.sampleJObject = JsonHelper.Deserialize>(sampleJson); } - public async Task ExecuteOnceAsync() + public override async Task ExecuteOnceAsync() { Uri itemUri = UriFactory.CreateDocumentUri(this.databsaeName, this.containerName, this.nextExecutionItemId); ResourceResponse itemResponse = await this.documentClient.ReadDocumentAsync( @@ -61,7 +61,7 @@ public async Task ExecuteOnceAsync() }; } - public async Task PrepareAsync() + public override async Task PrepareAsync() { if (string.IsNullOrEmpty(this.nextExecutionItemId) || string.IsNullOrEmpty(this.nextExecutionItemPartitionKey)) diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadTExistsV2BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadTExistsV2BenchmarkOperation.cs index 061183c575..fa9cd81c98 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadTExistsV2BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadTExistsV2BenchmarkOperation.cs @@ -13,7 +13,7 @@ namespace CosmosBenchmark using Microsoft.Azure.Documents.Client; using Newtonsoft.Json.Linq; - internal class ReadTExistsV2BenchmarkOperation : IBenchmarkOperation + internal class ReadTExistsV2BenchmarkOperation : ReadBenchmarkOperation { private readonly string partitionKeyPath; private readonly Dictionary sampleJObject; @@ -42,7 +42,7 @@ public ReadTExistsV2BenchmarkOperation( this.sampleJObject = JsonHelper.Deserialize>(sampleJson); } - public async Task ExecuteOnceAsync() + public override async Task ExecuteOnceAsync() { Uri itemUri = UriFactory.CreateDocumentUri(this.databsaeName, this.containerName, this.nextExecutionItemId); DocumentResponse> itemResponse = await this.documentClient.ReadDocumentAsync>( @@ -62,7 +62,7 @@ public async Task ExecuteOnceAsync() }; } - public async Task PrepareAsync() + public override async Task PrepareAsync() { if (string.IsNullOrEmpty(this.nextExecutionItemId) || string.IsNullOrEmpty(this.nextExecutionItemPartitionKey)) diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/InsertV3BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/InsertV3BenchmarkOperation.cs index 1ab5861242..81f9da9c87 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/InsertV3BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/InsertV3BenchmarkOperation.cs @@ -12,7 +12,7 @@ namespace CosmosBenchmark using Newtonsoft.Json; using Newtonsoft.Json.Serialization; - internal class InsertV3BenchmarkOperation : IBenchmarkOperation + internal class InsertV3BenchmarkOperation : InsertBenchmarkOperation { private readonly Container container; private readonly string partitionKeyPath; @@ -37,7 +37,7 @@ public InsertV3BenchmarkOperation( this.sampleJObject = JsonHelper.Deserialize>(sampleJson); } - public async Task ExecuteOnceAsync() + public override async Task ExecuteOnceAsync() { using (MemoryStream input = JsonHelper.ToStream(this.sampleJObject)) { @@ -60,7 +60,7 @@ public async Task ExecuteOnceAsync() } } - public Task PrepareAsync() + public override Task PrepareAsync() { string newPartitionKey = Guid.NewGuid().ToString(); this.sampleJObject["id"] = Guid.NewGuid().ToString(); diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/QueryTV3BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/QueryTV3BenchmarkOperation.cs index bdb0b38cf8..5eb8fba252 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/QueryTV3BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/QueryTV3BenchmarkOperation.cs @@ -11,7 +11,7 @@ namespace CosmosBenchmark using System.Threading.Tasks; using Microsoft.Azure.Cosmos; - internal abstract class QueryTV3BenchmarkOperation : IBenchmarkOperation + internal abstract class QueryTV3BenchmarkOperation : QueryBenchmarkOperation { protected readonly Container container; protected readonly Dictionary sampleJObject; @@ -55,7 +55,7 @@ public QueryTV3BenchmarkOperation( /// /// /// - public async Task ExecuteOnceAsync() + public override async Task ExecuteOnceAsync() { if (this.IsQueryStream) { @@ -258,7 +258,7 @@ private async Task ExecuteOnceAsyncWithStreamsAndPagination() /// /// /// - public virtual async Task PrepareAsync() + public override async Task PrepareAsync() { if (this.initialized) { diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadFeedStreamV3BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadFeedStreamV3BenchmarkOperation.cs index 99789040b0..3917470691 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadFeedStreamV3BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadFeedStreamV3BenchmarkOperation.cs @@ -11,7 +11,7 @@ namespace CosmosBenchmark using System.Threading.Tasks; using Microsoft.Azure.Cosmos; - internal class ReadFeedStreamV3BenchmarkOperation : IBenchmarkOperation + internal class ReadFeedStreamV3BenchmarkOperation : ReadBenchmarkOperation { private readonly Container container; private readonly string partitionKeyPath; @@ -39,7 +39,7 @@ public ReadFeedStreamV3BenchmarkOperation( this.sampleJObject = JsonHelper.Deserialize>(sampleJson); } - public async Task ExecuteOnceAsync() + public override async Task ExecuteOnceAsync() { FeedIterator feedIterator = this.container .GetItemQueryStreamIterator( @@ -63,7 +63,7 @@ public async Task ExecuteOnceAsync() }; } - public async Task PrepareAsync() + public override async Task PrepareAsync() { if (string.IsNullOrEmpty(this.nextExecutionItemId) || string.IsNullOrEmpty(this.nextExecutionItemPartitionKey)) diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadNotExistsV3BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadNotExistsV3BenchmarkOperation.cs index c97d1af4b7..fcec5a46fd 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadNotExistsV3BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadNotExistsV3BenchmarkOperation.cs @@ -9,7 +9,7 @@ namespace CosmosBenchmark using System.Threading.Tasks; using Microsoft.Azure.Cosmos; - internal class ReadNotExistsV3BenchmarkOperation : IBenchmarkOperation + internal class ReadNotExistsV3BenchmarkOperation : ReadBenchmarkOperation { private readonly Container container; @@ -32,7 +32,7 @@ public ReadNotExistsV3BenchmarkOperation( this.container = cosmosClient.GetContainer(this.databsaeName, this.containerName); } - public async Task ExecuteOnceAsync() + public override async Task ExecuteOnceAsync() { using (ResponseMessage itemResponse = await this.container.ReadItemStreamAsync( this.nextExecutionItemId, @@ -54,7 +54,7 @@ public async Task ExecuteOnceAsync() } } - public Task PrepareAsync() + public override Task PrepareAsync() { if (string.IsNullOrEmpty(this.nextExecutionItemId) || string.IsNullOrEmpty(this.nextExecutionItemPartitionKey)) diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadStreamExistsV3BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadStreamExistsV3BenchmarkOperation.cs index aba0a4ea7a..6695f721b6 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadStreamExistsV3BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadStreamExistsV3BenchmarkOperation.cs @@ -11,7 +11,7 @@ namespace CosmosBenchmark using System.Threading.Tasks; using Microsoft.Azure.Cosmos; - internal class ReadStreamExistsV3BenchmarkOperation : IBenchmarkOperation + internal class ReadStreamExistsV3BenchmarkOperation : ReadBenchmarkOperation { private readonly Container container; private readonly string partitionKeyPath; @@ -39,7 +39,7 @@ public ReadStreamExistsV3BenchmarkOperation( this.sampleJObject = JsonHelper.Deserialize>(sampleJson); } - public async Task ExecuteOnceAsync() + public override async Task ExecuteOnceAsync() { using (ResponseMessage itemResponse = await this.container.ReadItemStreamAsync( this.nextExecutionItemId, @@ -61,7 +61,7 @@ public async Task ExecuteOnceAsync() } } - public async Task PrepareAsync() + public override async Task PrepareAsync() { if (string.IsNullOrEmpty(this.nextExecutionItemId) || string.IsNullOrEmpty(this.nextExecutionItemPartitionKey)) diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadStreamExistsWithDiagnosticsV3BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadStreamExistsWithDiagnosticsV3BenchmarkOperation.cs index ebdc14924b..580b9cd375 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadStreamExistsWithDiagnosticsV3BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadStreamExistsWithDiagnosticsV3BenchmarkOperation.cs @@ -11,7 +11,7 @@ namespace CosmosBenchmark using System.Threading.Tasks; using Microsoft.Azure.Cosmos; - internal class ReadStreamExistsWithDiagnosticsV3BenchmarkOperation : IBenchmarkOperation + internal class ReadStreamExistsWithDiagnosticsV3BenchmarkOperation : ReadBenchmarkOperation { private readonly Container container; private readonly string partitionKeyPath; @@ -39,7 +39,7 @@ public ReadStreamExistsWithDiagnosticsV3BenchmarkOperation( this.sampleJObject = JsonHelper.Deserialize>(sampleJson); } - public async Task ExecuteOnceAsync() + public override async Task ExecuteOnceAsync() { using (ResponseMessage itemResponse = await this.container.ReadItemStreamAsync( this.nextExecutionItemId, @@ -67,7 +67,7 @@ public async Task ExecuteOnceAsync() } } - public async Task PrepareAsync() + public override async Task PrepareAsync() { if (string.IsNullOrEmpty(this.nextExecutionItemId) || string.IsNullOrEmpty(this.nextExecutionItemPartitionKey)) diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadTExistsV3BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadTExistsV3BenchmarkOperation.cs index 7fd40397be..392a38691e 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadTExistsV3BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadTExistsV3BenchmarkOperation.cs @@ -12,7 +12,7 @@ namespace CosmosBenchmark using Microsoft.Azure.Cosmos; using Newtonsoft.Json.Linq; - internal class ReadTExistsV3BenchmarkOperation : IBenchmarkOperation + internal class ReadTExistsV3BenchmarkOperation : ReadBenchmarkOperation { private readonly Container container; private readonly string partitionKeyPath; @@ -40,7 +40,7 @@ public ReadTExistsV3BenchmarkOperation( this.sampleJObject = JsonHelper.Deserialize>(sampleJson); } - public async Task ExecuteOnceAsync() + public override async Task ExecuteOnceAsync() { ItemResponse> itemResponse = await this.container.ReadItemAsync>( this.nextExecutionItemId, @@ -60,7 +60,7 @@ public async Task ExecuteOnceAsync() }; } - public async Task PrepareAsync() + public override async Task PrepareAsync() { if (string.IsNullOrEmpty(this.nextExecutionItemId) || string.IsNullOrEmpty(this.nextExecutionItemPartitionKey))