Draft: 3892: Adding metrics to Azure Benchmark tool

Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/ARMTemplate/benchmarkTemplate.json

@@ -45,7 +45,7 @@
  "metadata": {
  "description": "Name for the container group"
  },
- "defaultValue":  "CosmosDBBenchmark"
+ "defaultValue": "CosmosDBBenchmark"
  },
  "containerName": {
  "type": "string",
@@ -88,9 +88,22 @@
  "metadata": {
  "description": "Location for all resources."
  }
+ },
+ "applicationInsightsName": {
+ "type": "string",
+ "defaultValue": "CosmosDBBenchmarkTestAppInsights",
+ "metadata": {
+ "description": "Specifies Application Insights Account Nmae"
+ }
+ },
+ "dashboardName": {
+ "type": "string",
+ "defaultValue": "Cosmos Benchmark Statistics"
  }
  },
- "variables": {},
+ "variables": {
+ "appInsightsResourceIds": "[concat('/subscriptions/', subscription().subscriptionId, '/resourceGroups/', resourceGroup().name, '/providers/microsoft.insights/components/', parameters('applicationInsightsName'))]"
+ },
  "resources": [
  {
  "name": "[parameters('containerGroupName')]",
@@ -149,6 +162,190 @@
  }
  ]
  }
+ },
+ {
+ "name": "[parameters('applicationInsightsName')]",
+ "type": "Microsoft.Insights/components",
+ "apiVersion": "2015-05-01",
+ "location": "[resourceGroup().location]",
+ "properties": {
+ "ApplicationId": "[parameters('applicationInsightsName')]",
+ "Application_Type": "web"
+ }
+ },
+ {
+ "name": "CosmosDBBenchmarkDashboard",
+ "type": "Microsoft.Portal/dashboards",
+ "location": "[resourceGroup().location]",
+ "apiVersion": "2015-08-01-preview",
+ "tags": {
+ "hidden-title": "[parameters('dashboardName')]"
+ },
+
+ "properties": {
+ "lenses": {
+ "0": {
+ "order": 0,
+ "parts": {
+ "0": {
+ "position": {
+ "x": 0,
+ "y": 0,
+ "colSpan": 11,
+ "rowSpan": 5
+ },
+ "metadata": {
+ "inputs": [
+ {
+ "name": "resourceTypeMode",
+ "isOptional": true
+ },
+ {
+ "name": "ComponentId",
+ "isOptional": true
+ },
+ {
+ "name": "Scope",
+ "value": {
+ "resourceIds": [
+ "[variables('appInsightsResourceIds')]"
+ ]
+ },
+ "isOptional": true
+ },
+ {
+ "name": "PartId",
+ "value": "104528fc-0216-49c6-bf70-fffe9d37f93d",
+ "isOptional": true
+ },
+ {
+ "name": "Version",
+ "value": "2.0",
+ "isOptional": true
+ },
+ {
+ "name": "TimeRange",
+ "isOptional": true
+ },
+ {
+ "name": "DashboardId",
+ "isOptional": true
+ },
+ {
+ "name": "DraftRequestParameters",
+ "isOptional": true
+ },
+ {
+ "name": "Query",
+ "value": "customMetrics\n| where name == \"ReadOperationLatencyInMs\" and timestamp > ago(1d)\n| summarize\n percentile(value, 50),\n percentile(value, 75),\n percentile(value, 90),\n percentile(value, 95),\n percentile(value, 99),\n percentile(value, 99.9),\n percentile(value, 99.99)\n by ts = bin(timestamp, 5s)\n| render timechart \n\n",
+ "isOptional": true
+ },
+ {
+ "name": "ControlType",
+ "value": "FrameControlChart",
+ "isOptional": true
+ },
+ {
+ "name": "SpecificChart",
+ "value": "Line",
+ "isOptional": true
+ },
+ {
+ "name": "PartTitle",
+ "value": "Analytics",
+ "isOptional": true
+ },
+ {
+ "name": "PartSubTitle",
+ "value": "CosmosDBBenchmarkTestAppInsights",
+ "isOptional": true
+ },
+ {
+ "name": "Dimensions",
+ "value": {
+ "xAxis": {
+ "name": "ts",
+ "type": "datetime"
+ },
+ "yAxis": [
+ {
+ "name": "percentile_value_50",
+ "type": "real"
+ },
+ {
+ "name": "percentile_value_75",
+ "type": "real"
+ },
+ {
+ "name": "percentile_value_90",
+ "type": "real"
+ },
+ {
+ "name": "percentile_value_95",
+ "type": "real"
+ }
+ ],
+ "splitBy": [],
+ "aggregation": "Sum"
+ },
+ "isOptional": true
+ },
+ {
+ "name": "LegendOptions",
+ "value": {
+ "isEnabled": true,
+ "position": "Bottom"
+ },
+ "isOptional": true
+ },
+ {
+ "name": "IsQueryContainTimeRange",
+ "value": true,
+ "isOptional": true
+ }
+ ],
+ "type": "Extension/Microsoft_OperationsManagementSuite_Workspace/PartType/LogsDashboardPart",
+ "settings": {}
+ }
+ }
+ }
+ }
+ },
+ "metadata": {
+ "model": {
+ "timeRange": {
+ "value": {
+ "relative": {
+ "duration": 24,
+ "timeUnit": 1
+ }
+ },
+ "type": "MsPortalFx.Composition.Configuration.ValueTypes.TimeRange"
+ },
+ "filterLocale": {
+ "value": "en-us"
+ },
+ "filters": {
+ "value": {
+ "MsPortalFx_TimeRange": {
+ "model": {
+ "format": "utc",
+ "granularity": "auto",
+ "relative": "24h"
+ },
+ "displayCache": {
+ "name": "UTC Time",
+ "value": "Past 24 hours"
+ },
+ "filteredPartIds": [
+ "StartboardPart-LogsDashboardPart-4f9d44ab-efbe-4310-8809-63ae942a3091"
+ ]
+ }
+ }
+ }
+ }
+ }
+ }
  }
  ]
 }

Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfig.cs

@@ -123,6 +123,18 @@ public class BenchmarkConfig
  [Option(Required = false, HelpText = "Container to publish results to")]
  public string ResultsContainer { get; set; } = "runsummary";
 
+ [Option(Required = false, HelpText = "Metrics reporting interval in seconds")]
+ public int MetricsReportingIntervalInSec { get; set; } = 5;
+
+ [Option(Required = false, HelpText = "Application Insights instrumentation key")]
+ public string AppInsightsInstrumentationKey { get; set; }
+
+ [Option(Required = false, HelpText = "The reservoir sample size.")]
+ public int ReservoirSampleSize { get; set; } = 1028;
+
+ [Option(Required = false, HelpText = "Logging context name. The default value is \"CosmosDBBenchmarkLoggingContext\"")]
+ public string LoggingContextIdentifier { get; set; } = "CosmosDBBenchmarkLoggingContext";
+
  internal int GetTaskCount(int containerThroughput)
  {
  int taskCount = this.DegreeOfParallelism;
@@ -270,4 +282,4 @@ private static void HandleParseError(IEnumerable<Error> errors)
  Environment.Exit(errors.Count());
  }
  }
-}
+}

Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/CosmosBenchmark.csproj

@@ -17,9 +17,15 @@
  </None>
  </ItemGroup>
  <ItemGroup>
+ <PackageReference Include="Azure.Monitor.OpenTelemetry.Exporter" Version="1.0.0-beta.12" />
  <PackageReference Include="CommandLineParser" Version="2.8.0" />
  <PackageReference Include="MathNet.Numerics" Version="4.15.0" />
+ <PackageReference Include="Microsoft.ApplicationInsights" Version="2.21.0" />
  <PackageReference Include="Microsoft.Azure.DocumentDB.Core" Version="*" />
+ <PackageReference Include="Microsoft.Extensions.Logging" Version="7.0.0" />
+ <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="7.0.0" />
+ <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="7.0.0" />
+ <PackageReference Include="OpenTelemetry.Exporter.Console" Version="1.5.0" />
  </ItemGroup>
 
  <ItemGroup Condition=" '$(ProjectRef)' != 'True' ">

Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/BenchmarkOperation.cs

@@ -0,0 +1,15 @@
+//------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+//------------------------------------------------------------
+
+namespace CosmosBenchmark
+{
+ using System.Threading.Tasks;
+
+ internal abstract class BenchmarkOperation : IBenchmarkOperation
+ {
+ public abstract Task<OperationResult> ExecuteOnceAsync();
+
+ public abstract Task PrepareAsync();
+ }
+}

Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutionStrategy.cs

@@ -6,6 +6,9 @@ namespace CosmosBenchmark
 {
  using System;
  using System.Threading.Tasks;
+ using Microsoft.ApplicationInsights;
+ using Microsoft.Extensions.Logging;
+ using OpenTelemetry.Metrics;
 
  internal interface IExecutionStrategy
  {
@@ -19,7 +22,8 @@ public Task<RunSummary> ExecuteAsync(
  BenchmarkConfig benchmarkConfig,
  int serialExecutorConcurrency,
  int serialExecutorIterationCount,
- double warmupFraction);
-
+ double warmupFraction,
+ ILogger logger,
+ MeterProvider meterProvider);
  }
 }

Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutor.cs

@@ -6,6 +6,8 @@ namespace CosmosBenchmark
 {
  using System;
  using System.Threading.Tasks;
+ using Microsoft.Extensions.Logging;
+ using OpenTelemetry.Metrics;
 
  internal interface IExecutor
  {
@@ -17,6 +19,9 @@ public Task ExecuteAsync(
  int iterationCount,
  bool isWarmup,
  bool traceFailures,
- Action completionCallback);
+ Action completionCallback,
+ BenchmarkConfig benchmarkConfig,
+ ILogger logger,
+ MeterProvider meterProvider);
  }
 }

Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IMetricsCollector.cs

@@ -0,0 +1,15 @@
+//------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+//------------------------------------------------------------
+
+namespace CosmosBenchmark
+{
+ public interface IMetricsCollector
+ {
+ void RecordLatencyAndRps(double milliseconds);
+
+ void CollectMetricsOnSuccess();
+
+ void CollectMetricsOnFailure();
+ }
+}

Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertBenchmarkOperation.cs

@@ -0,0 +1,10 @@
+//------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+//------------------------------------------------------------
+
+namespace CosmosBenchmark
+{
+ internal abstract class InsertBenchmarkOperation : BenchmarkOperation
+ {
+ }
+}

Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertOperationMetricsCollector.cs

@@ -0,0 +1,28 @@
+//------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+//------------------------------------------------------------
+
+namespace CosmosBenchmark
+{
+ using Microsoft.ApplicationInsights;
+ using System.Diagnostics.Metrics;
+
+ internal class InsertOperationMetricsCollector : MetricsCollector
+ {
+ public InsertOperationMetricsCollector(Meter meter) : base(meter)
+ {
+ }
+
+ protected override string RpsHistogramName => "InsertOperationRpsHistogram";
+
+ protected override string LatencyInMsHistogramName => "InsertOperationLatencyInMsHistogram";
+
+ protected override string RpsMetricName => "InsertOperationRps";
+
+ protected override string LatencyInMsMetricName => "InsertOperationLatencyInMs";
+
+ protected override string FailureOperationMetricName => "InsertOperationFailure";
+
+ protected override string SuccessOperationMetricName => "InsertOperationSuccess";
+ }
+}