[sdk-metrics] Exemplar spec improvements

src/OpenTelemetry.Exporter.Console/ConsoleMetricExporter.cs

@@ -188,30 +188,44 @@ public override ExportResult Export(in Batch<Metric> batch)
  }
 
  var exemplarString = new StringBuilder();
- foreach (var exemplar in metricPoint.GetExemplars())
+ if (metricPoint.TryGetExemplars(out var exemplars))
  {
- if (exemplar.Timestamp != default)
+ foreach (ref readonly var exemplar in exemplars)
  {
- exemplarString.Append("Value: ");
- exemplarString.Append(exemplar.DoubleValue);
- exemplarString.Append(" Timestamp: ");
+ exemplarString.Append("Timestamp: ");
  exemplarString.Append(exemplar.Timestamp.ToString("yyyy-MM-ddTHH:mm:ss.fffffffZ", CultureInfo.InvariantCulture));
- exemplarString.Append(" TraceId: ");
- exemplarString.Append(exemplar.TraceId);
- exemplarString.Append(" SpanId: ");
- exemplarString.Append(exemplar.SpanId);
+ if (metricType.IsDouble())
+ {
+ exemplarString.Append(" Value: ");
+ exemplarString.Append(exemplar.DoubleValue);
+ }
+ else if (metricType.IsLong())
+ {
+ exemplarString.Append(" Value: ");
+ exemplarString.Append(exemplar.LongValue);
+ }
 
- if (exemplar.FilteredTags != null && exemplar.FilteredTags.Count > 0)
+ if (exemplar.TraceId.HasValue)
  {
- exemplarString.Append(" Filtered Tags : ");
+ exemplarString.Append(" TraceId: ");
+ exemplarString.Append(exemplar.TraceId.Value.ToHexString());
+ exemplarString.Append(" SpanId: ");
+ exemplarString.Append(exemplar.SpanId.Value.ToHexString());
+ }
 
- foreach (var tag in exemplar.FilteredTags)
+ bool appendedTagString = false;
+ foreach (var tag in exemplar.FilteredTags)
+ {
+ if (ConsoleTagTransformer.Instance.TryTransformTag(tag, out var result))
  {
- if (ConsoleTagTransformer.Instance.TryTransformTag(tag, out var result))
+ if (!appendedTagString)
  {
- exemplarString.Append(result);
- exemplarString.Append(' ');
+ exemplarString.Append(" Filtered Tags : ");
+ appendedTagString = true;
  }
+
+ exemplarString.Append(result);
+ exemplarString.Append(' ');
  }
  }
 

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/MetricItemExtensions.cs

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 using System.Collections.Concurrent;
+using System.Diagnostics;
 using System.Runtime.CompilerServices;
 using Google.Protobuf;
 using Google.Protobuf.Collections;
@@ -267,37 +268,12 @@ internal static OtlpMetrics.Metric ToOtlpMetric(this Metric metric)
  }
  }
 
- var exemplars = metricPoint.GetExemplars();
- foreach (var examplar in exemplars)
+ if (metricPoint.TryGetExemplars(out var exemplars))
  {
- if (examplar.Timestamp != default)
+ foreach (ref readonly var exemplar in exemplars)
  {
- byte[] traceIdBytes = new byte[16];
- examplar.TraceId?.CopyTo(traceIdBytes);
-
- byte[] spanIdBytes = new byte[8];
- examplar.SpanId?.CopyTo(spanIdBytes);
-
- var otlpExemplar = new OtlpMetrics.Exemplar
- {
- TimeUnixNano = (ulong)examplar.Timestamp.ToUnixTimeNanoseconds(),
- TraceId = UnsafeByteOperations.UnsafeWrap(traceIdBytes),
- SpanId = UnsafeByteOperations.UnsafeWrap(spanIdBytes),
- AsDouble = examplar.DoubleValue,
- };
-
- if (examplar.FilteredTags != null)
- {
- foreach (var tag in examplar.FilteredTags)
- {
- if (OtlpKeyValueTransformer.Instance.TryTransformTag(tag, out var result))
- {
- otlpExemplar.FilteredAttributes.Add(result);
- }
- }
- }
-
- dataPoint.Exemplars.Add(otlpExemplar);
+ dataPoint.Exemplars.Add(
+ ToOtlpExemplar(exemplar.DoubleValue, in exemplar));
  }
  }
 
@@ -379,51 +355,48 @@ private static void AddScopeAttributes(IEnumerable<KeyValuePair<string, object>>
  }
  }
 
- /*
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static OtlpMetrics.Exemplar ToOtlpExemplar(this IExemplar exemplar)
+ private static OtlpMetrics.Exemplar ToOtlpExemplar<T>(T value, in Metrics.Exemplar exemplar)
+ where T : struct
  {
- var otlpExemplar = new OtlpMetrics.Exemplar();
-
- if (exemplar.Value is double doubleValue)
+ var otlpExemplar = new OtlpMetrics.Exemplar
  {
- otlpExemplar.AsDouble = doubleValue;
- }
- else if (exemplar.Value is long longValue)
+ TimeUnixNano = (ulong)exemplar.Timestamp.ToUnixTimeNanoseconds(),
+ };
+
+ if (exemplar.TraceId.HasValue)
  {
- otlpExemplar.AsInt = longValue;
+ byte[] traceIdBytes = new byte[16];
+ exemplar.TraceId.Value.CopyTo(traceIdBytes);
+
+ byte[] spanIdBytes = new byte[8];
+ exemplar.SpanId.Value.CopyTo(spanIdBytes);
+
+ otlpExemplar.TraceId = UnsafeByteOperations.UnsafeWrap(traceIdBytes);
+ otlpExemplar.SpanId = UnsafeByteOperations.UnsafeWrap(spanIdBytes);
  }
- else
+
+ if (typeof(T) == typeof(long))
  {
- // TODO: Determine how we want to handle exceptions here.
- // Do we want to just skip this exemplar and move on?
- // Should we skip recording the whole metric?
- throw new ArgumentException();
+ otlpExemplar.AsInt = (long)(object)value;
  }
-
- otlpExemplar.TimeUnixNano = (ulong)exemplar.Timestamp.ToUnixTimeNanoseconds();
-
- // TODO: Do the TagEnumerationState thing.
- foreach (var tag in exemplar.FilteredTags)
+ else if (typeof(T) == typeof(double))
  {
- otlpExemplar.FilteredAttributes.Add(tag.ToOtlpAttribute());
+ otlpExemplar.AsDouble = (double)(object)value;
  }
-
- if (exemplar.TraceId != default)
+ else
  {
- byte[] traceIdBytes = new byte[16];
- exemplar.TraceId.CopyTo(traceIdBytes);
- otlpExemplar.TraceId = UnsafeByteOperations.UnsafeWrap(traceIdBytes);
+ Debug.Fail("Unexpected type");
+ otlpExemplar.AsDouble = Convert.ToDouble(value);
  }
 
- if (exemplar.SpanId != default)
+ foreach (var tag in exemplar.FilteredTags)
  {
- byte[] spanIdBytes = new byte[8];
- exemplar.SpanId.CopyTo(spanIdBytes);
- otlpExemplar.SpanId = UnsafeByteOperations.UnsafeWrap(spanIdBytes);
+ if (OtlpKeyValueTransformer.Instance.TryTransformTag(tag, out var result))
+ {
+ otlpExemplar.FilteredAttributes.Add(result);
+ }
  }
 
  return otlpExemplar;
  }
- */
 }

src/OpenTelemetry/.publicApi/Experimental/PublicAPI.Unshipped.txt

@@ -12,16 +12,36 @@ OpenTelemetry.Metrics.AlwaysOnExemplarFilter.AlwaysOnExemplarFilter() -> void
 OpenTelemetry.Metrics.Exemplar
 OpenTelemetry.Metrics.Exemplar.DoubleValue.get -> double
 OpenTelemetry.Metrics.Exemplar.Exemplar() -> void
+OpenTelemetry.Metrics.Exemplar.FilteredTags.get -> OpenTelemetry.ReadOnlyFilteredTagCollection
+OpenTelemetry.Metrics.Exemplar.LongValue.get -> long
 OpenTelemetry.Metrics.Exemplar.SpanId.get -> System.Diagnostics.ActivitySpanId?
 OpenTelemetry.Metrics.Exemplar.Timestamp.get -> System.DateTimeOffset
 OpenTelemetry.Metrics.Exemplar.TraceId.get -> System.Diagnostics.ActivityTraceId?
 OpenTelemetry.Metrics.ExemplarFilter
 OpenTelemetry.Metrics.ExemplarFilter.ExemplarFilter() -> void
-OpenTelemetry.Metrics.MetricPoint.GetExemplars() -> OpenTelemetry.Metrics.Exemplar[]!
+OpenTelemetry.Metrics.ExemplarMeasurement<T>
+OpenTelemetry.Metrics.ExemplarMeasurement<T>.ExemplarMeasurement() -> void
+OpenTelemetry.Metrics.ExemplarMeasurement<T>.Tags.get -> System.ReadOnlySpan<System.Collections.Generic.KeyValuePair<string!, object?>>
+OpenTelemetry.Metrics.ExemplarMeasurement<T>.Value.get -> T
+OpenTelemetry.Metrics.MetricPoint.TryGetExemplars(out OpenTelemetry.Metrics.ReadOnlyExemplarCollection? exemplars) -> bool
 OpenTelemetry.Metrics.MetricStreamConfiguration.CardinalityLimit.get -> int?
 OpenTelemetry.Metrics.MetricStreamConfiguration.CardinalityLimit.set -> void
+OpenTelemetry.Metrics.ReadOnlyExemplarCollection
+OpenTelemetry.Metrics.ReadOnlyExemplarCollection.Enumerator
+OpenTelemetry.Metrics.ReadOnlyExemplarCollection.Enumerator.Current.get -> OpenTelemetry.Metrics.Exemplar
+OpenTelemetry.Metrics.ReadOnlyExemplarCollection.Enumerator.Enumerator() -> void
+OpenTelemetry.Metrics.ReadOnlyExemplarCollection.Enumerator.MoveNext() -> bool
+OpenTelemetry.Metrics.ReadOnlyExemplarCollection.GetEnumerator() -> OpenTelemetry.Metrics.ReadOnlyExemplarCollection.Enumerator
+OpenTelemetry.Metrics.ReadOnlyExemplarCollection.ReadOnlyExemplarCollection() -> void
 OpenTelemetry.Metrics.TraceBasedExemplarFilter
 OpenTelemetry.Metrics.TraceBasedExemplarFilter.TraceBasedExemplarFilter() -> void
+OpenTelemetry.ReadOnlyFilteredTagCollection
+OpenTelemetry.ReadOnlyFilteredTagCollection.Enumerator
+OpenTelemetry.ReadOnlyFilteredTagCollection.Enumerator.Current.get -> System.Collections.Generic.KeyValuePair<string!, object?>
+OpenTelemetry.ReadOnlyFilteredTagCollection.Enumerator.Enumerator() -> void
+OpenTelemetry.ReadOnlyFilteredTagCollection.Enumerator.MoveNext() -> bool
+OpenTelemetry.ReadOnlyFilteredTagCollection.GetEnumerator() -> OpenTelemetry.ReadOnlyFilteredTagCollection.Enumerator
+OpenTelemetry.ReadOnlyFilteredTagCollection.ReadOnlyFilteredTagCollection() -> void
 static OpenTelemetry.Logs.LoggerProviderBuilderExtensions.AddProcessor(this OpenTelemetry.Logs.LoggerProviderBuilder! loggerProviderBuilder, OpenTelemetry.BaseProcessor<OpenTelemetry.Logs.LogRecord!>! processor) -> OpenTelemetry.Logs.LoggerProviderBuilder!
 static OpenTelemetry.Logs.LoggerProviderBuilderExtensions.AddProcessor(this OpenTelemetry.Logs.LoggerProviderBuilder! loggerProviderBuilder, System.Func<System.IServiceProvider!, OpenTelemetry.BaseProcessor<OpenTelemetry.Logs.LogRecord!>!>! implementationFactory) -> OpenTelemetry.Logs.LoggerProviderBuilder!
 static OpenTelemetry.Logs.LoggerProviderBuilderExtensions.AddProcessor<T>(this OpenTelemetry.Logs.LoggerProviderBuilder! loggerProviderBuilder) -> OpenTelemetry.Logs.LoggerProviderBuilder!
@@ -38,7 +58,6 @@ static OpenTelemetry.OpenTelemetryBuilderSdkExtensions.WithLogging(this OpenTele
 static OpenTelemetry.Sdk.CreateLoggerProviderBuilder() -> OpenTelemetry.Logs.LoggerProviderBuilder!
 abstract OpenTelemetry.Metrics.ExemplarFilter.ShouldSample(double value, System.ReadOnlySpan<System.Collections.Generic.KeyValuePair<string!, object?>> tags) -> bool
 abstract OpenTelemetry.Metrics.ExemplarFilter.ShouldSample(long value, System.ReadOnlySpan<System.Collections.Generic.KeyValuePair<string!, object?>> tags) -> bool
-OpenTelemetry.Metrics.Exemplar.FilteredTags.get -> System.Collections.Generic.List<System.Collections.Generic.KeyValuePair<string!, object?>>?
 override OpenTelemetry.Metrics.AlwaysOffExemplarFilter.ShouldSample(double value, System.ReadOnlySpan<System.Collections.Generic.KeyValuePair<string!, object?>> tags) -> bool
 override OpenTelemetry.Metrics.AlwaysOffExemplarFilter.ShouldSample(long value, System.ReadOnlySpan<System.Collections.Generic.KeyValuePair<string!, object?>> tags) -> bool
 override OpenTelemetry.Metrics.AlwaysOnExemplarFilter.ShouldSample(double value, System.ReadOnlySpan<System.Collections.Generic.KeyValuePair<string!, object?>> tags) -> bool

src/OpenTelemetry/CHANGELOG.md

@@ -42,6 +42,10 @@
  [IMetricsListener](https://learn.microsoft.com/dotNet/api/microsoft.extensions.diagnostics.metrics.imetricslistener).
  ([#5265](https:/open-telemetry/opentelemetry-dotnet/pull/5265))
 
+* **Experimental (pre-release builds only):** `Exemplar` and `ExemplarReservoir`
+ APIs have been updated to match the OpenTelemetry Specification.
+ ([#5386](https:/open-telemetry/opentelemetry-dotnet/pull/5386))
+
 ## 1.7.0
 
 Released 2023-Dec-08

src/OpenTelemetry/Metrics/AggregatorStore.cs

@@ -11,6 +11,7 @@ namespace OpenTelemetry.Metrics;
 
 internal sealed class AggregatorStore
 {
+ internal readonly HashSet<string>? TagKeysInteresting;
  internal readonly bool OutputDelta;
  internal readonly bool OutputDeltaWithUnusedMetricPointReclaimEnabled;
  internal readonly int CardinalityLimit;
@@ -24,7 +25,6 @@ internal sealed class AggregatorStore
 
  private readonly object lockZeroTags = new();
  private readonly object lockOverflowTag = new();
- private readonly HashSet<string>? tagKeysInteresting;
  private readonly int tagsKeysInterestingCount;
 
  // This holds the reclaimed MetricPoints that are available for reuse.
@@ -84,7 +84,7 @@ internal AggregatorStore(
  this.updateLongCallback = this.UpdateLongCustomTags;
  this.updateDoubleCallback = this.UpdateDoubleCustomTags;
  var hs = new HashSet<string>(metricStreamIdentity.TagKeys, StringComparer.Ordinal);
- this.tagKeysInteresting = hs;
+ this.TagKeysInteresting = hs;
  this.tagsKeysInterestingCount = hs.Count;
  }
 
@@ -1122,9 +1122,9 @@ private int FindMetricAggregatorsCustomTag(ReadOnlySpan<KeyValuePair<string, obj
 
  var storage = ThreadStaticStorage.GetStorage();
 
- Debug.Assert(this.tagKeysInteresting != null, "this.tagKeysInteresting was null");
+ Debug.Assert(this.TagKeysInteresting != null, "this.tagKeysInteresting was null");
 
- storage.SplitToKeysAndValues(tags, tagLength, this.tagKeysInteresting!, out var tagKeysAndValues, out var actualLength);
+ storage.SplitToKeysAndValues(tags, tagLength, this.TagKeysInteresting!, out var tagKeysAndValues, out var actualLength);
 
  // Actual number of tags depend on how many
  // of the incoming tags has user opted to

src/OpenTelemetry/Metrics/Exemplar/AlignedHistogramBucketExemplarReservoir.cs

@@ -8,92 +8,28 @@ namespace OpenTelemetry.Metrics;
 /// <summary>
 /// The AlignedHistogramBucketExemplarReservoir implementation.
 /// </summary>
-internal sealed class AlignedHistogramBucketExemplarReservoir : ExemplarReservoir
+internal sealed class AlignedHistogramBucketExemplarReservoir : FixedSizeExemplarReservoir
 {
- private readonly Exemplar[] runningExemplars;
- private readonly Exemplar[] tempExemplars;
-
- public AlignedHistogramBucketExemplarReservoir(int length)
- {
- this.runningExemplars = new Exemplar[length + 1];
- this.tempExemplars = new Exemplar[length + 1];
- }
-
- public override void Offer(long value, ReadOnlySpan<KeyValuePair<string, object?>> tags, int index = default)
- {
- this.OfferAtBoundary(value, tags, index);
- }
-
- public override void Offer(double value, ReadOnlySpan<KeyValuePair<string, object?>> tags, int index = default)
+ public AlignedHistogramBucketExemplarReservoir(int numberOfBuckets)
+ : base(numberOfBuckets + 1)
  {
- this.OfferAtBoundary(value, tags, index);
  }
 
- public override Exemplar[] Collect(ReadOnlyTagCollection actualTags, bool reset)
+ public override void Offer(in ExemplarMeasurement<long> measurement)
  {
- for (int i = 0; i < this.runningExemplars.Length; i++)
- {
- this.tempExemplars[i] = this.runningExemplars[i];
- if (this.runningExemplars[i].FilteredTags != null)
- {
- // TODO: Better data structure to avoid this Linq.
- // This is doing filtered = alltags - storedtags.
- // TODO: At this stage, this logic is done inside Reservoir.
- // Kinda hard for end users who write own reservoirs.
- // Evaluate if this logic can be moved elsewhere.
- // TODO: The cost is paid irrespective of whether the
- // Exporter supports Exemplar or not. One idea is to
- // defer this until first exporter attempts read.
- this.tempExemplars[i].FilteredTags = this.runningExemplars[i].FilteredTags!.Except(actualTags.KeyAndValues.ToList()).ToList();
- }
-
- if (reset)
- {
- this.runningExemplars[i].Timestamp = default;
- }
- }
+ Debug.Assert(
+ measurement.ExplicitBucketHistogramBucketIndex != -1,
+ "ExplicitBucketHistogramBucketIndex was -1");
 
- return this.tempExemplars;
+ this.UpdateExemplar(measurement.ExplicitBucketHistogramBucketIndex, in measurement);
  }
 
- private void OfferAtBoundary(double value, ReadOnlySpan<KeyValuePair<string, object?>> tags, int index)
+ public override void Offer(in ExemplarMeasurement<double> measurement)
  {
- ref var exemplar = ref this.runningExemplars[index];
- exemplar.Timestamp = DateTimeOffset.UtcNow;
- exemplar.DoubleValue = value;
- exemplar.TraceId = Activity.Current?.TraceId;
- exemplar.SpanId = Activity.Current?.SpanId;
-
- if (tags == default)
- {
- // default tag is used to indicate
- // the special case where all tags provided at measurement
- // recording time are stored.
- // In this case, Exemplars does not have to store any tags.
- // In other words, FilteredTags will be empty.
- return;
- }
-
- if (exemplar.FilteredTags == null)
- {
- exemplar.FilteredTags = new List<KeyValuePair<string, object?>>(tags.Length);
- }
- else
- {
- // Keep the list, but clear contents.
- exemplar.FilteredTags.Clear();
- }
+ Debug.Assert(
+ measurement.ExplicitBucketHistogramBucketIndex != -1,
+ "ExplicitBucketHistogramBucketIndex was -1");
 
- // Though only those tags that are filtered need to be
- // stored, finding filtered list from the full tag list
- // is expensive. So all the tags are stored in hot path (this).
- // During snapshot, the filtered list is calculated.
- // TODO: Evaluate alternative approaches based on perf.
- // TODO: This is not user friendly to Reservoir authors
- // and must be handled as transparently as feasible.
- foreach (var tag in tags)
- {
- exemplar.FilteredTags.Add(tag);
- }
+ this.UpdateExemplar(measurement.ExplicitBucketHistogramBucketIndex, in measurement);
  }
 }