otlpmetricgrpc: Keep metadata for gRPC in context (#5892)

When using otlpmetricgrpc, we can inject metadata dynamically with
`metadata.AppendToOutgoingContext`. However when specifying headers with
`WithHeaders`, it becomes impossible to add additional metadata.

For example, the code below sends additional metadata in the header
```go
exp := otlpmetricgrpc.New(context.Background())

ctx := context.Background()
ctx = metadata.AppendToOutgoingContext(ctx, "dynamic-key", "value")
exp.Export(ctx, &metricdata.ResourceMetrics{}))
```


But when using `WithHeader` like below, no additional metadata will be
sent
```go
exp := otlpmetricgrpc.New(context.Background(), otlpmetricgrpc.WithHeaders("custom-key", "value"))

ctx := context.Background()
ctx = metadata.AppendToOutgoingContext(ctx, "dynamic-key", "value")
exp.Export(ctx, &metricdata.ResourceMetrics{}))
```

To eliminate this inconsistency, keep the metadata in the context and
send them in the header.

---------

Co-authored-by: Tyler Yahn <[email protected]>

CHANGELOG.md

@@ -22,6 +22,10 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 - Global MeterProvider registration unwraps global instrument Observers, the undocumented Unwrap() methods are now private. (#5881)
 
+### Changed
+
+- `go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc` now keeps the metadata already present in the context when `WithHeaders` is used. (#5892)
+
 <!-- Released section -->
 <!-- Don't change this section unless doing release -->
 

exporters/otlp/otlpmetric/otlpmetricgrpc/client.go

@@ -155,7 +155,12 @@ func (c *client) exportContext(parent context.Context) (context.Context, context
  }
 
  if c.metadata.Len() > 0 {
- ctx = metadata.NewOutgoingContext(ctx, c.metadata)
+ md := c.metadata
+ if outMD, ok := metadata.FromOutgoingContext(ctx); ok {
+ md = metadata.Join(md, outMD)
+ }
+
+ ctx = metadata.NewOutgoingContext(ctx, md)
  }
 
  return ctx, cancel

exporters/otlp/otlpmetric/otlpmetricgrpc/client_test.go

@@ -13,6 +13,7 @@ import (
  "google.golang.org/genproto/googleapis/rpc/errdetails"
  "google.golang.org/grpc"
  "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/metadata"
  "google.golang.org/grpc/status"
  "google.golang.org/protobuf/types/known/durationpb"
 
@@ -207,14 +208,18 @@ func TestConfig(t *testing.T) {
  headers := map[string]string{key: "custom-value"}
  exp, coll := factoryFunc(nil, WithHeaders(headers))
  t.Cleanup(coll.Shutdown)
+
  ctx := context.Background()
+ additionalKey := "additional-custom-header"
+ ctx = metadata.AppendToOutgoingContext(ctx, additionalKey, "additional-value")
  require.NoError(t, exp.Export(ctx, &metricdata.ResourceMetrics{}))
  // Ensure everything is flushed.
  require.NoError(t, exp.Shutdown(ctx))
 
  got := coll.Headers()
  require.Regexp(t, "OTel Go OTLP over gRPC metrics exporter/[01]\\..*", got)
  require.Contains(t, got, key)
+ require.Contains(t, got, additionalKey)
  assert.Equal(t, []string{headers[key]}, got[key])
  })