Skip to content

Commit

Permalink
fix: Provide all metric labels in OpenTelemetry & StatsD (#2183)
Browse files Browse the repository at this point in the history
Co-authored-by: Sagi Vaknin <[email protected]>
  • Loading branch information
tomkerkhove and SagiVaknin authored Jul 17, 2023
1 parent d89f60c commit 2ae09eb
Show file tree
Hide file tree
Showing 27 changed files with 231 additions and 113 deletions.
6 changes: 5 additions & 1 deletion build/azure-devops/agents-ci-scraper.yml
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,10 @@ stages:
parameters:
agentName: 'Resource Discovery'
url: '$(Agent.ResourceDiscovery.BaseUrl)/$(Agent.ResourceDiscovery.Prometheus.ScrapeUri)'
- template: templates/prometheus/show-prometheus-metrics.yml
parameters:
agentName: 'OpenTelemetry'
url: '$(OpenTelemetry.Collector.Uri)/$(Agent.OpenTelemetry.Prometheus.ScrapeUri)'
- ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
- template: templates/docker/push-image.yml
parameters:
Expand Down Expand Up @@ -271,4 +275,4 @@ stages:
- ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
- template: templates/docker/push-image.yml
parameters:
imageName: '$(Image.TaggedName)'
imageName: '$(Image.TaggedName)'
2 changes: 2 additions & 0 deletions build/azure-devops/variables/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,5 @@ variables:
value: metrics
- name: Agent.ResourceDiscovery.Prometheus.ScrapeUri
value: metrics
- name: Agent.OpenTelemetry.Prometheus.ScrapeUri
value: metrics
2 changes: 2 additions & 0 deletions changelog/content/experimental/unreleased.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ version:

#### Scraper

- {{% tag added %}} Provide support for all label scenarios in StatsD & OpenTelemetry metric sink. This includes
dimensions, customer & default labels.
- {{% tag changed %}} Switch to Mariner distroless base images
- {{% tag security %}} Patch for [CVE-2023-29331](https:/advisories/GHSA-555c-2p6r-68mm) (High)

Expand Down
4 changes: 2 additions & 2 deletions config/opentelemetry-collector/collector-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ exporters:
endpoint: "0.0.0.0:8889"
namespace: otel
const_labels:
app: promitor
source_app: promitor
send_timestamps: true

processors:
Expand All @@ -22,4 +22,4 @@ service:
metrics:
receivers: [otlp]
processors: [batch]
exporters: [prometheus]
exporters: [prometheus]
5 changes: 4 additions & 1 deletion src/Promitor.Agents.Core/AgentStartup.cs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
using Promitor.Agents.Core.Observability;
using Serilog;
using Serilog.Configuration;
using Serilog.Events;

namespace Promitor.Agents.Core
{
Expand Down Expand Up @@ -53,7 +54,9 @@ protected virtual LoggerConfiguration CreateSerilogConfiguration(string componen
{
var defaultLogLevel = SerilogFactory.DetermineSinkLogLevel(telemetryConfiguration.DefaultVerbosity);
var loggerConfiguration = new LoggerConfiguration()
.MinimumLevel.Is(defaultLogLevel);
.MinimumLevel.Is(defaultLogLevel)
.MinimumLevel.Override("Microsoft", LogEventLevel.Warning)
.MinimumLevel.Override("Microsoft.AspNetCore", LogEventLevel.Warning);

loggerConfiguration = EnrichTelemetry(componentName, serviceProvider, loggerConfiguration);
loggerConfiguration = FilterTelemetry(loggerConfiguration);
Expand Down
1 change: 1 addition & 0 deletions src/Promitor.Agents.ResourceDiscovery/Dockerfile.linux
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ COPY Promitor.Integrations.Azure/* Promitor.Integrations.Azure/
COPY Promitor.Integrations.AzureMonitor/* Promitor.Integrations.AzureMonitor/
COPY Promitor.Integrations.AzureStorage/* Promitor.Integrations.AzureStorage/
COPY Promitor.Integrations.LogAnalytics/* Promitor.Integrations.LogAnalytics/
COPY Promitor.Integrations.Sinks.Core/* Promitor.Integrations.Sinks.Core/
COPY Promitor.Integrations.Sinks.Prometheus/* Promitor.Integrations.Sinks.Prometheus/
RUN dotnet publish Promitor.Agents.ResourceDiscovery/Promitor.Agents.ResourceDiscovery.csproj --configuration release --output /app /p:Version=$VERSION

Expand Down
1 change: 1 addition & 0 deletions src/Promitor.Agents.ResourceDiscovery/Dockerfile.windows
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ COPY Promitor.Integrations.Azure/* Promitor.Integrations.Azure/
COPY Promitor.Integrations.AzureMonitor/* Promitor.Integrations.AzureMonitor/
COPY Promitor.Integrations.AzureStorage/* Promitor.Integrations.AzureStorage/
COPY Promitor.Integrations.LogAnalytics/* Promitor.Integrations.LogAnalytics/
COPY Promitor.Integrations.Sinks.Core/* Promitor.Integrations.Sinks.Core/
COPY Promitor.Integrations.Sinks.Prometheus/* Promitor.Integrations.Sinks.Prometheus/
RUN dotnet publish Promitor.Agents.ResourceDiscovery/Promitor.Agents.ResourceDiscovery.csproj --configuration release --output /app /p:Version=%VERSION%

Expand Down
1 change: 1 addition & 0 deletions src/Promitor.Agents.Scraper/Dockerfile.linux
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ COPY Promitor.Integrations.Azure/* Promitor.Integrations.Azure/
COPY Promitor.Integrations.AzureMonitor/* Promitor.Integrations.AzureMonitor/
COPY Promitor.Integrations.AzureStorage/* Promitor.Integrations.AzureStorage/
COPY Promitor.Integrations.LogAnalytics/* Promitor.Integrations.LogAnalytics/
COPY Promitor.Integrations.Sinks.Core/* Promitor.Integrations.Sinks.Core/
COPY Promitor.Integrations.Sinks.Atlassian.Statuspage/* Promitor.Integrations.Sinks.Atlassian.Statuspage/
COPY Promitor.Integrations.Sinks.OpenTelemetry/* Promitor.Integrations.Sinks.OpenTelemetry/
COPY Promitor.Integrations.Sinks.Prometheus/* Promitor.Integrations.Sinks.Prometheus/
Expand Down
1 change: 1 addition & 0 deletions src/Promitor.Agents.Scraper/Dockerfile.windows
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ COPY Promitor.Integrations.Azure/* Promitor.Integrations.Azure/
COPY Promitor.Integrations.AzureMonitor/* Promitor.Integrations.AzureMonitor/
COPY Promitor.Integrations.AzureStorage/* Promitor.Integrations.AzureStorage/
COPY Promitor.Integrations.LogAnalytics/* Promitor.Integrations.LogAnalytics/
COPY Promitor.Integrations.Sinks.Core/* Promitor.Integrations.Sinks.Core/
COPY Promitor.Integrations.Sinks.Atlassian.Statuspage/* Promitor.Integrations.Sinks.Atlassian.Statuspage/
COPY Promitor.Integrations.Sinks.OpenTelemetry/* Promitor.Integrations.Sinks.OpenTelemetry/
COPY Promitor.Integrations.Sinks.Prometheus/* Promitor.Integrations.Sinks.Prometheus/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ public virtual MetricsDeclaration Get(bool applyDefaults = false, IErrorReporter
metric.LogAnalyticsConfiguration.Aggregation.Interval ??= config.MetricDefaults.Aggregation?.Interval;
}
}

return config;
}

Expand Down
4 changes: 0 additions & 4 deletions src/Promitor.Core/Promitor.Core.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,4 @@
<PackageReference Include="YamlDotNet" Version="13.1.1" />
</ItemGroup>

<ItemGroup>
<Folder Include="Metrics\Sinks\" />
</ItemGroup>

</Project>
Original file line number Diff line number Diff line change
Expand Up @@ -6,28 +6,28 @@
using Microsoft.Extensions.Options;
using Promitor.Core;
using Promitor.Core.Metrics.Sinks;
using Promitor.Core.Scraping.Configuration.Providers.Interfaces;
using Promitor.Integrations.Sinks.Atlassian.Statuspage.Configuration;
using Promitor.Integrations.Sinks.Core;

namespace Promitor.Integrations.Sinks.Atlassian.Statuspage
{
public class AtlassianStatuspageMetricSink : IMetricSink
public class AtlassianStatuspageMetricSink : MetricSink, IMetricSink
{
private readonly ILogger<AtlassianStatuspageMetricSink> _logger;
private readonly IAtlassianStatuspageClient _atlassianStatusPageClient;
private readonly IOptionsMonitor<AtlassianStatusPageSinkConfiguration> _sinkConfiguration;

public MetricSinkType Type { get; } = MetricSinkType.AtlassianStatuspage;
public MetricSinkType Type => MetricSinkType.AtlassianStatuspage;

public AtlassianStatuspageMetricSink(IAtlassianStatuspageClient atlassianStatusPageClient, IOptionsMonitor<AtlassianStatusPageSinkConfiguration> sinkConfiguration, ILogger<AtlassianStatuspageMetricSink> logger)
public AtlassianStatuspageMetricSink(IAtlassianStatuspageClient atlassianStatusPageClient, IMetricsDeclarationProvider metricsDeclarationProvider, IOptionsMonitor<AtlassianStatusPageSinkConfiguration> sinkConfiguration, ILogger<AtlassianStatuspageMetricSink> logger)
: base(metricsDeclarationProvider, logger)
{
Guard.NotNull(atlassianStatusPageClient, nameof(atlassianStatusPageClient));
Guard.NotNull(sinkConfiguration, nameof(sinkConfiguration));
Guard.NotNull(sinkConfiguration.CurrentValue, nameof(sinkConfiguration.CurrentValue));
Guard.NotNull(logger, nameof(logger));

_atlassianStatusPageClient = atlassianStatusPageClient;
_sinkConfiguration = sinkConfiguration;
_logger = logger;
}

public async Task ReportMetricAsync(string metricName, string metricDescription, ScrapeResult scrapeResult)
Expand Down Expand Up @@ -59,7 +59,7 @@ public async Task ReportMetricAsync(string metricName, string metricDescription,
await _atlassianStatusPageClient.ReportMetricAsync(systemMetricMapping.Id, metricValue);
}

_logger.LogTrace("Metric {MetricName} with value {MetricValue} was written to Atlassian Statuspage", metricName, metricValue);
Logger.LogTrace("Metric {MetricName} with value {MetricValue} was written to Atlassian Statuspage", metricName, metricValue);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

<ItemGroup>
<ProjectReference Include="..\Promitor.Core.Scraping\Promitor.Core.Scraping.csproj" />
<ProjectReference Include="..\Promitor.Integrations.Sinks.Core\Promitor.Integrations.Sinks.Core.csproj" />
</ItemGroup>

</Project>
77 changes: 77 additions & 0 deletions src/Promitor.Integrations.Sinks.Core/MetricSink.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
using System;
using System.Collections.Generic;
using System.Linq;
using GuardNet;
using Microsoft.Extensions.Logging;
using Promitor.Core;
using Promitor.Core.Metrics;
using Promitor.Core.Scraping.Configuration.Providers.Interfaces;

namespace Promitor.Integrations.Sinks.Core
{
public class MetricSink
{
protected ILogger Logger { get; }
protected IMetricsDeclarationProvider MetricsDeclarationProvider { get; }

protected MetricSink(IMetricsDeclarationProvider metricsDeclarationProvider, ILogger logger)
{
Guard.NotNull(metricsDeclarationProvider, nameof(metricsDeclarationProvider));
Guard.NotNull(logger, nameof(logger));

MetricsDeclarationProvider = metricsDeclarationProvider;
Logger = logger;
}

public Dictionary<string, string> DetermineLabels(string metricName, ScrapeResult scrapeResult, MeasuredMetric measuredMetric, Func<string, string> mutateLabelName = null)
{
var metricDefinition = MetricsDeclarationProvider.GetPrometheusDefinition(metricName);
var defaultLabels = MetricsDeclarationProvider.GetDefaultLabels();

var labels = new Dictionary<string, string>(scrapeResult.Labels.Select(label =>
{
var labelName = DetermineLabelName(label.Key, mutateLabelName);
return new KeyValuePair<string, string>(labelName, label.Value);
}));

if (measuredMetric.IsDimensional)
{
labels.Add(DetermineLabelName(measuredMetric.DimensionName, mutateLabelName), measuredMetric.DimensionValue);
}

if (metricDefinition?.Labels?.Any() == true)
{
foreach (var customLabel in metricDefinition.Labels)
{
var customLabelKey = DetermineLabelName(customLabel.Key, mutateLabelName);
if (labels.TryGetValue(customLabelKey, out var label))
{
Logger.LogWarning("Custom label {CustomLabelName} was already specified with value '{LabelValue}' instead of '{CustomLabelValue}'. Ignoring...", customLabel.Key, label, customLabel.Value);
continue;
}

labels.Add(customLabelKey, customLabel.Value);
}
}

foreach (var defaultLabel in defaultLabels)
{
var defaultLabelKey = DetermineLabelName(defaultLabel.Key, mutateLabelName);
labels.TryAdd(defaultLabelKey, defaultLabel.Value);
}

// Add the tenant id
var metricsDeclaration = MetricsDeclarationProvider.Get(applyDefaults: true);
labels.TryAdd("tenant_id", metricsDeclaration.AzureMetadata.TenantId);

var orderedLabels = labels.OrderBy(kvp => kvp.Key).ToDictionary(kvp => kvp.Key, kvp => kvp.Value);

return orderedLabels;
}

private static string DetermineLabelName(string originalLabelName, Func<string, string> mutateLabelName)
{
return mutateLabelName!= null ? mutateLabelName.Invoke(originalLabelName) : originalLabelName;
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<RuntimeFrameworkVersion>7.0.7</RuntimeFrameworkVersion>
</PropertyGroup>

<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|AnyCPU'">
<NoWarn>1701;1702;1591</NoWarn>
</PropertyGroup>

<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
<NoWarn>1701;1702;1591</NoWarn>
</PropertyGroup>

<ItemGroup>
<ProjectReference Include="..\Promitor.Core.Scraping\Promitor.Core.Scraping.csproj" />
</ItemGroup>

</Project>
Original file line number Diff line number Diff line change
Expand Up @@ -9,17 +9,20 @@
using Microsoft.Extensions.Logging;
using Promitor.Core;
using Promitor.Core.Metrics.Sinks;
using Promitor.Core.Scraping.Configuration.Providers.Interfaces;
using Promitor.Integrations.Sinks.Core;

namespace Promitor.Integrations.Sinks.OpenTelemetry
{
public class OpenTelemetryCollectorMetricSink : IMetricSink
public class OpenTelemetryCollectorMetricSink : MetricSink, IMetricSink
{
private readonly ILogger<OpenTelemetryCollectorMetricSink> _logger;
private static readonly Meter azureMonitorMeter = new("Promitor.Scraper.Metrics.AzureMonitor", "1.0");

public MetricSinkType Type => MetricSinkType.OpenTelemetryCollector;

public OpenTelemetryCollectorMetricSink(ILogger<OpenTelemetryCollectorMetricSink> logger)
public OpenTelemetryCollectorMetricSink(IMetricsDeclarationProvider metricsDeclarationProvider, ILogger<OpenTelemetryCollectorMetricSink> logger)
: base(metricsDeclarationProvider, logger)
{
Guard.NotNull(logger, nameof(logger));

Expand All @@ -38,7 +41,9 @@ public async Task ReportMetricAsync(string metricName, string metricDescription,
{
var metricValue = measuredMetric.Value ?? 0;

var reportMetricTask = ReportMetricAsync(metricName, metricDescription, metricValue, scrapeResult.Labels);
var metricLabels = DetermineLabels(metricName, scrapeResult, measuredMetric);

var reportMetricTask = ReportMetricAsync(metricName, metricDescription, metricValue, metricLabels);
reportMetricTasks.Add(reportMetricTask);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

<ItemGroup>
<ProjectReference Include="..\Promitor.Core.Scraping\Promitor.Core.Scraping.csproj" />
<ProjectReference Include="..\Promitor.Integrations.Sinks.Core\Promitor.Integrations.Sinks.Core.csproj" />
</ItemGroup>

</Project>
Loading

0 comments on commit 2ae09eb

Please sign in to comment.