WIP: Revert OTel expiration code (#1143)

go.mod

@@ -68,6 +68,16 @@ require (
  sigs.k8s.io/e2e-framework v0.3.0
 )
 
+replace go.opentelemetry.io/otel => github.com/grafana/opentelemetry-go v1.28.0-grafana.3
+
+replace go.opentelemetry.io/otel/metric => github.com/grafana/opentelemetry-go/metric v1.28.0-grafana.3
+
+replace go.opentelemetry.io/otel/trace => github.com/grafana/opentelemetry-go/trace v1.28.0-grafana.3
+
+replace go.opentelemetry.io/otel/sdk => github.com/grafana/opentelemetry-go/sdk v1.28.0-grafana.3
+
+replace go.opentelemetry.io/otel/sdk/metric => github.com/grafana/opentelemetry-go/sdk/metric v1.28.0-grafana.3
+
 require (
  cloud.google.com/go/compute/metadata v0.4.0 // indirect
  github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.24.0 // indirect

go.sum

@@ -105,6 +105,16 @@ github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWm
 github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/grafana/go-offsets-tracker v0.1.7 h1:2zBQ7iiGzvyXY7LA8kaaSiEqH/Yx82UcfRabbY5aOG4=
 github.com/grafana/go-offsets-tracker v0.1.7/go.mod h1:qcQdu7zlUKIFNUdBJlLyNHuJGW0SKWKjkrN6jtt+jds=
+github.com/grafana/opentelemetry-go v1.28.0-grafana.3 h1:vExZiZKDZTdDi7fP1GG3GOGuoZ0GNu76408tNXfsnD0=
+github.com/grafana/opentelemetry-go v1.28.0-grafana.3/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4=
+github.com/grafana/opentelemetry-go/metric v1.28.0-grafana.3 h1:CfCrKV0S4Lq7yJ1pShUrxaV34EGhUpE+3GW3XyaMbJU=
+github.com/grafana/opentelemetry-go/metric v1.28.0-grafana.3/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s=
+github.com/grafana/opentelemetry-go/sdk v1.28.0-grafana.3 h1:HeU/OFEfs0dPRgXiM7bMjazMtb2Xda/eFg2drL5QY+Q=
+github.com/grafana/opentelemetry-go/sdk v1.28.0-grafana.3/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg=
+github.com/grafana/opentelemetry-go/sdk/metric v1.28.0-grafana.3 h1:UDmKndO9VVARK21dLK37hlTMf2zUdEJfGI24HM4kmnU=
+github.com/grafana/opentelemetry-go/sdk/metric v1.28.0-grafana.3/go.mod h1:cWPjykihLAPvXKi4iZc1dpER3Jdq2Z0YLse3moQUCpg=
+github.com/grafana/opentelemetry-go/trace v1.28.0-grafana.3 h1:TUj00E+urS316FDwvwo5HM4C9rXsZbOo5ekbx7dkXdc=
+github.com/grafana/opentelemetry-go/trace v1.28.0-grafana.3/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k=
 github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY=
@@ -321,8 +331,6 @@ go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.5
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0/go.mod h1:azvtTADFQJA8mX80jIH/akaE7h+dbm/sVuaHqN13w74=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0/go.mod h1:jjdQuTGVsXV4vSs+CJ2qYDeDPf9yIJV23qlIzBm73Vg=
-go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo=
-go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4=
 go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.4.0 h1:zBPZAISA9NOc5cE8zydqDiS0itvg/P/0Hn9m72a5gvM=
 go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.4.0/go.mod h1:gcj2fFjEsqpV3fXuzAA+0Ze1p2/4MJ4T7d77AmkvueQ=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.28.0 h1:U2guen0GhqH8o/G2un8f/aG/y++OuW6MyCo6hT9prXk=
@@ -343,16 +351,8 @@ go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0 h1:EVSnY9JbEEW92bE
 go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0/go.mod h1:Ea1N1QQryNXpCD0I1fdLibBAIpQuBkznMmkdKrapk1Y=
 go.opentelemetry.io/otel/log v0.4.0 h1:/vZ+3Utqh18e8TPjuc3ecg284078KWrR8BRz+PQAj3o=
 go.opentelemetry.io/otel/log v0.4.0/go.mod h1:DhGnQvky7pHy82MIRV43iXh3FlKN8UUKftn0KbLOq6I=
-go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q=
-go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s=
-go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE=
-go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg=
 go.opentelemetry.io/otel/sdk/log v0.4.0 h1:1mMI22L82zLqf6KtkjrRy5BbagOTWdJsqMY/HSqILAA=
 go.opentelemetry.io/otel/sdk/log v0.4.0/go.mod h1:AYJ9FVF0hNOgAVzUG/ybg/QttnXhUePWAupmCqtdESo=
-go.opentelemetry.io/otel/sdk/metric v1.28.0 h1:OkuaKgKrgAbYrrY0t92c+cC+2F6hsFNnCQArXCKlg08=
-go.opentelemetry.io/otel/sdk/metric v1.28.0/go.mod h1:cWPjykihLAPvXKi4iZc1dpER3Jdq2Z0YLse3moQUCpg=
-go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g=
-go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI=
 go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0=
 go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=

pkg/beyla/config.go

@@ -75,7 +75,7 @@ var DefaultConfig = Config{
  },
  // TODO: keep OTEL expiration disabled by default until we address
  // this issue: https:/grafana/beyla/issues/1065
- TTL: 100 * 365 * 24 * time.Hour,
+ TTL: defaultMetricsTTL,
  },
  Traces: otel.TracesConfig{
  Protocol: otel.ProtocolUnset,

pkg/export/otel/expirer.go

@@ -7,6 +7,7 @@ import (
  "time"
 
  "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/metric"
 
  "github.com/grafana/beyla/pkg/export/attributes"
  "github.com/grafana/beyla/pkg/export/expire"
@@ -19,9 +20,7 @@ func plog() *slog.Logger {
 }
 
 type removableMetric[VT any] interface {
- // [TODO otel-removal] uncomment when this is fixed https:/grafana/beyla/issues/1065
- // or when this is released into main OTEL https:/open-telemetry/opentelemetry-specification/pull/4135
- // Remove(context.Context, ...metric.RemoveOption)
+ Remove(context.Context, ...metric.RemoveOption)
 }
 
 // Expirer drops metrics from labels that haven't been updated during a given timeout.
@@ -78,9 +77,7 @@ func (ex *Expirer[Record, Metric, ValType]) ForRecord(r Record, extraAttrs ...at
  // metric might stay visible after 2*TTL time after not being updated
  now := ex.clock()
  if now.Sub(ex.lastExpiration) >= ex.ttl {
- // [TODO otel-removal] uncomment when this is fixed https:/grafana/beyla/issues/1065
- // or when this is released into main OTEL https:/open-telemetry/opentelemetry-specification/pull/4135
- // ex.removeOutdated(ex.ctx)
+ ex.removeOutdated(ex.ctx)
  ex.lastExpiration = now
  }
  recordAttrs, attrValues := ex.recordAttributes(r, extraAttrs...)
@@ -107,36 +104,32 @@ func (ex *Expirer[Record, Metric, ValType]) recordAttributes(m Record, extraAttr
  return attribute.NewSet(keyVals...), vals
 }
 
-// [TODO otel-removal] uncomment when this is fixed https:/grafana/beyla/issues/1065
-// or when this is released into main OTEL https:/open-telemetry/opentelemetry-specification/pull/4135
-// func (ex *Expirer[Record, Metric, ValType]) removeOutdated(ctx context.Context) {
-// for _, attrs := range ex.entries.DeleteExpired() {
-// ex.deleteMetricInstance(ctx, attrs)
-// }
-// }
-//
-// func (ex *Expirer[Record, Metric, ValType]) deleteMetricInstance(ctx context.Context, attrs attribute.Set) {
-// if ex.log.Enabled(ex.ctx, slog.LevelDebug) {
-// ex.logger(attrs).Debug("deleting old OTEL metric")
-// }
-// ex.metric.Remove(ctx, metric.WithAttributeSet(attrs))
-// }
-//
-// func (ex *Expirer[Record, Metric, ValType]) logger(attrs attribute.Set) *slog.Logger {
-// fmtAttrs := make([]any, 0, attrs.Len()*2)
-// for it := attrs.Iter(); it.Next(); {
-// a := it.Attribute()
-// fmtAttrs = append(fmtAttrs, string(a.Key), a.Value.Emit())
-// }
-// return ex.log.With(fmtAttrs...)
-// }
+func (ex *Expirer[Record, Metric, ValType]) removeOutdated(ctx context.Context) {
+ for _, attrs := range ex.entries.DeleteExpired() {
+ ex.deleteMetricInstance(ctx, attrs)
+ }
+}
+
+func (ex *Expirer[Record, Metric, ValType]) deleteMetricInstance(ctx context.Context, attrs attribute.Set) {
+ if ex.log.Enabled(ex.ctx, slog.LevelDebug) {
+ ex.logger(attrs).Debug("deleting old OTEL metric")
+ }
+ ex.metric.Remove(ctx, metric.WithAttributeSet(attrs))
+}
+
+func (ex *Expirer[Record, Metric, ValType]) logger(attrs attribute.Set) *slog.Logger {
+ fmtAttrs := make([]any, 0, attrs.Len()*2)
+ for it := attrs.Iter(); it.Next(); {
+ a := it.Attribute()
+ fmtAttrs = append(fmtAttrs, string(a.Key), a.Value.Emit())
+ }
+ return ex.log.With(fmtAttrs...)
+}
 
 // RemoveAllMetrics is explicitly invoked when the metrics reporter of a given service
 // instance needs to be shut down
-func (ex *Expirer[Record, Metric, ValType]) RemoveAllMetrics(_ context.Context) {
- // [TODO otel-removal] uncomment when this is fixed https:/grafana/beyla/issues/1065
- // or when this is released into main OTEL https:/open-telemetry/opentelemetry-specification/pull/4135
- // for _, attrs := range ex.entries.DeleteAll() {
- // ex.deleteMetricInstance(ctx, attrs)
- // }
+func (ex *Expirer[Record, Metric, ValType]) RemoveAllMetrics(ctx context.Context) {
+ for _, attrs := range ex.entries.DeleteAll() {
+ ex.deleteMetricInstance(ctx, attrs)
+ }
 }

pkg/export/otel/expirer_test.go

@@ -22,9 +22,6 @@ import (
 const timeout = 20 * time.Second
 
 func TestNetMetricsExpiration(t *testing.T) {
- // [TODO otel-removal] unskip when this is fixed https:/grafana/beyla/issues/1065
- // or when this is released into main OTEL https:/open-telemetry/opentelemetry-specification/pull/4135
- t.Skip("otel-removal is temporarily disabled")
  defer restoreEnvAfterExecution()()
  ctx, cancelCtx := context.WithCancel(context.Background())
  defer cancelCtx()
@@ -133,9 +130,6 @@ func TestNetMetricsExpiration(t *testing.T) {
 // (2) by metric set of a given service ID
 // this test verifies case 1
 func TestAppMetricsExpiration_ByMetricAttrs(t *testing.T) {
- // [TODO otel-removal] unskip when this is fixed https:/grafana/beyla/issues/1065
- // or when this is released into main OTEL https:/open-telemetry/opentelemetry-specification/pull/4135
- t.Skip("otel-removal is temporarily disabled")
  defer restoreEnvAfterExecution()()
  ctx, cancelCtx := context.WithCancel(context.Background())
  defer cancelCtx()
@@ -255,9 +249,6 @@ func TestAppMetricsExpiration_ByMetricAttrs(t *testing.T) {
 // (2) by metric set of a given service ID
 // this test verifies case 2
 func TestAppMetricsExpiration_BySvcID(t *testing.T) {
- // [TODO otel-removal] unskip when this is fixed https:/grafana/beyla/issues/1065
- // or when this is released into main OTEL https:/open-telemetry/opentelemetry-specification/pull/4135
- t.Skip("otel-removal is temporarily disabled")
  defer restoreEnvAfterExecution()()
  ctx, cancelCtx := context.WithCancel(context.Background())
  defer cancelCtx()