Skip to content

Commit

Permalink
lambda: use _X_AMZN_TRACE_ID trace context as a Link not parent
Browse files Browse the repository at this point in the history
  • Loading branch information
tsloughter committed Feb 25, 2023
1 parent d9b1211 commit 67d1ded
Show file tree
Hide file tree
Showing 10 changed files with 66 additions and 50 deletions.
6 changes: 5 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,11 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm

### Changed

- `samplers/jaegerremote`: change to use protobuf parser instead of encoding/json to accept enums as strings. (#3183)
- `samplers/jaegerremote`: change to use protobuf parser instead of
encoding/json to accept enums as strings. (#3183)
- `go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-lambda-go/otellambda`
updated to add a Link to the new Span if Trace Context is found in the
environment variable `_X_AMZN_TRACE_ID`, instead of using it as the parent (#3428)

### Fixed

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ go 1.18
require (
github.com/aws/aws-lambda-go v1.37.0
github.com/stretchr/testify v1.8.1
go.opentelemetry.io/contrib/propagators/aws v1.14.0
go.opentelemetry.io/otel v1.13.0
go.opentelemetry.io/otel/trace v1.13.0
)
Expand All @@ -13,6 +14,10 @@ require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/go-logr/logr v1.2.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/kr/pretty v0.1.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
go.opentelemetry.io/otel/sdk v1.13.0 // indirect
golang.org/x/sys v0.4.0 // indirect
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
14 changes: 13 additions & 1 deletion instrumentation/github.com/aws/aws-lambda-go/otellambda/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,11 @@ github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbV
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
Expand All @@ -18,12 +23,19 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
go.opentelemetry.io/contrib/propagators/aws v1.14.0 h1:At9KgnobpQSaC7LYQ2JpDEOxLlMW6EsmLPsCAFfXSNg=
go.opentelemetry.io/contrib/propagators/aws v1.14.0/go.mod h1:KB4fnXEZfSGUC39lmyXfqfuw7D1C8n01nXxsYgKvQhc=
go.opentelemetry.io/otel v1.13.0 h1:1ZAKnNQKwBBxFtww/GwxNUyTf0AxkZzrukO8MeXqe4Y=
go.opentelemetry.io/otel v1.13.0/go.mod h1:FH3RtdZCzRkJYFTCsAKDy9l/XYjMdNv6QrkFFB8DvVg=
go.opentelemetry.io/otel/sdk v1.13.0 h1:BHib5g8MvdqS65yo2vV1s6Le42Hm6rrw08qU6yz5JaM=
go.opentelemetry.io/otel/sdk v1.13.0/go.mod h1:YLKPx5+6Vx/o1TCUYYs+bpymtkmazOMT6zoRrC7AQ7I=
go.opentelemetry.io/otel/trace v1.13.0 h1:CBgRZ6ntv+Amuj1jDsMhZtlAPT6gbyIRdaIzFhfBSdY=
go.opentelemetry.io/otel/trace v1.13.0/go.mod h1:muCvmmO9KKpvuXSf3KKAXXB2ygNYHQ+ZfI5X08d3tds=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18=
golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
24 changes: 22 additions & 2 deletions instrumentation/github.com/aws/aws-lambda-go/otellambda/lambda.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,10 @@ import (
"strings"

"github.com/aws/aws-lambda-go/lambdacontext"

"go.opentelemetry.io/contrib/propagators/aws/xray"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/propagation"
semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
"go.opentelemetry.io/otel/trace"
)
Expand Down Expand Up @@ -56,6 +57,23 @@ func newInstrumentor(opts ...Option) instrumentor {
resAttrs: []attribute.KeyValue{}}
}

func xrayEnvToLinks() []trace.Link {
xrayTraceID := os.Getenv("_X_AMZN_TRACE_ID")

if xrayTraceID != "" {
carrier := propagation.HeaderCarrier{"X-Amzn-Trace-Id": []string{xrayTraceID}}

xrayPropagator := xray.Propagator{}
ctx := xrayPropagator.Extract(context.Background(), carrier)

link := trace.LinkFromContext(ctx, attribute.String("source", "x-ray-env"))

return []trace.Link{link}
}

return []trace.Link{}
}

// Logic to start OTel Tracing.
func (i *instrumentor) tracingBegin(ctx context.Context, eventJSON []byte) (context.Context, trace.Span) {
// Add trace id to context
Expand Down Expand Up @@ -89,7 +107,9 @@ func (i *instrumentor) tracingBegin(ctx context.Context, eventJSON []byte) (cont
attributes = append(attributes, i.resAttrs...)
}

ctx, span = i.tracer.Start(ctx, spanName, trace.WithSpanKind(trace.SpanKindServer), trace.WithAttributes(attributes...))
links := xrayEnvToLinks()

ctx, span = i.tracer.Start(ctx, spanName, trace.WithSpanKind(trace.SpanKindServer), trace.WithAttributes(attributes...), trace.WithLinks(links...))

return ctx, span
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,8 @@ require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/go-logr/logr v1.2.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 // indirect
golang.org/x/sys v0.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
github.com/aws/aws-lambda-go v1.37.0 h1:WXkQ/xhIcXZZ2P5ZBEw+bbAKeCEcb5NtiYpSwVVzIXg=
github.com/aws/aws-lambda-go v1.37.0/go.mod h1:jwFe2KmMsHmffA1X2R09hH6lFzJQxzI8qK17ewzbQMM=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
Expand All @@ -9,6 +10,9 @@ github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbV
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
Expand All @@ -24,10 +28,10 @@ go.opentelemetry.io/otel/sdk v1.13.0 h1:BHib5g8MvdqS65yo2vV1s6Le42Hm6rrw08qU6yz5
go.opentelemetry.io/otel/sdk v1.13.0/go.mod h1:YLKPx5+6Vx/o1TCUYYs+bpymtkmazOMT6zoRrC7AQ7I=
go.opentelemetry.io/otel/trace v1.13.0 h1:CBgRZ6ntv+Amuj1jDsMhZtlAPT6gbyIRdaIzFhfBSdY=
go.opentelemetry.io/otel/trace v1.13.0/go.mod h1:muCvmmO9KKpvuXSf3KKAXXB2ygNYHQ+ZfI5X08d3tds=
golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 h1:h+EGohizhe9XlX18rfpa8k8RAc5XyaeamM+0VHRd4lc=
golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18=
golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ var (
attribute.String("faas.id", "arn:partition:service:region:account-id:resource-type:resource-id"),
attribute.String("cloud.account.id", "account-id")},
Events: nil,
Links: nil,
Links: []sdktrace.Link{sdktrace.Link{SpanContext: trace.SpanContextFromContext(mockContext), Attributes: []attribute.KeyValue{attribute.String("source", "x-ray-env")}}},
Status: sdktrace.Status{},
DroppedAttributes: 0,
DroppedEvents: 0,
Expand Down Expand Up @@ -330,7 +330,7 @@ var (
attribute.String("faas.id", "arn:partition:service:region:account-id:resource-type:resource-id"),
attribute.String("cloud.account.id", "account-id")},
Events: nil,
Links: nil,
Links: []sdktrace.Link{sdktrace.Link{SpanContext: trace.SpanContextFromContext(mockContext), Attributes: []attribute.KeyValue{attribute.String("source", "x-ray-env")}}},
Status: sdktrace.Status{},
DroppedAttributes: 0,
DroppedEvents: 0,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ require (
go.opentelemetry.io/otel v1.13.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.13.0
go.opentelemetry.io/otel/sdk v1.13.0
go.opentelemetry.io/otel/trace v1.13.0
go.opentelemetry.io/proto/otlp v0.19.0
google.golang.org/grpc v1.53.0
)
Expand All @@ -32,6 +31,7 @@ require (
github.com/pmezard/go-difflib v1.0.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.13.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.13.0 // indirect
go.opentelemetry.io/otel/trace v1.13.0 // indirect
golang.org/x/net v0.7.0 // indirect
golang.org/x/sys v0.5.0 // indirect
golang.org/x/text v0.7.0 // indirect
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ package xrayconfig // import "go.opentelemetry.io/contrib/instrumentation/github

import (
"context"
"os"

lambdadetector "go.opentelemetry.io/contrib/detectors/aws/lambda"
"go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-lambda-go/otellambda"
Expand All @@ -26,9 +25,10 @@ import (
sdktrace "go.opentelemetry.io/otel/sdk/trace"
)

// TODO: Currently does nothing but should support pulling the propagated
// trace context from attributes depending on the type of Lambda event
func xrayEventToCarrier([]byte) propagation.TextMapCarrier {
xrayTraceID := os.Getenv("_X_AMZN_TRACE_ID")
return propagation.HeaderCarrier{"X-Amzn-Trace-Id": []string{xrayTraceID}}
return propagation.HeaderCarrier{}
}

// NewTracerProvider returns a TracerProvider configured with an exporter,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,41 +28,11 @@ import (
"go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-lambda-go/otellambda"
"go.opentelemetry.io/contrib/propagators/aws/xray"
"go.opentelemetry.io/otel/propagation"
"go.opentelemetry.io/otel/trace"
v1common "go.opentelemetry.io/proto/otlp/common/v1"
v1resource "go.opentelemetry.io/proto/otlp/resource/v1"
v1trace "go.opentelemetry.io/proto/otlp/trace/v1"
)

func TestEventToCarrier(t *testing.T) {
os.Clearenv()

_ = os.Setenv("_X_AMZN_TRACE_ID", "traceID")
carrier := xrayEventToCarrier([]byte{})

assert.Equal(t, "traceID", carrier.Get("X-Amzn-Trace-Id"))
}

func TestEventToCarrierWithPropagator(t *testing.T) {
os.Clearenv()

_ = os.Setenv("_X_AMZN_TRACE_ID", "Root=1-5759e988-bd862e3fe1be46a994272793;Parent=53995c3f42cd8ad8;Sampled=1")
carrier := xrayEventToCarrier([]byte{})
ctx := xray.Propagator{}.Extract(context.Background(), carrier)

expectedTraceID, _ := trace.TraceIDFromHex("5759e988bd862e3fe1be46a994272793")
expectedSpanID, _ := trace.SpanIDFromHex("53995c3f42cd8ad8")
expectedCtx := trace.ContextWithRemoteSpanContext(context.Background(), trace.NewSpanContext(trace.SpanContextConfig{
TraceID: expectedTraceID,
SpanID: expectedSpanID,
TraceFlags: trace.FlagsSampled,
TraceState: trace.TraceState{},
Remote: true,
}))

assert.Equal(t, expectedCtx, ctx)
}

func setEnvVars() {
_ = os.Setenv("AWS_LAMBDA_FUNCTION_NAME", "testFunction")
_ = os.Setenv("AWS_REGION", "us-texas-1")
Expand All @@ -88,16 +58,16 @@ var (
}
mockContext = xray.Propagator{}.Extract(lambdacontext.NewContext(context.Background(), &mockLambdaContext),
propagation.HeaderCarrier{
"X-Amzn-Trace-Id": []string{"Root=1-5759e988-bd862e3fe1be46a994272793;Parent=53995c3f42cd8ad8;Sampled=1"},
"X-Amzn-Trace-Id": []string{"Root=1-6769e988-bd862e3fe1be46a994272793;Parent=63996c3f42cd8ad8;Sampled=1"},
})

expectedSpans = v1trace.ScopeSpans{
Scope: &v1common.InstrumentationScope{Name: "go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-lambda-go/otellambda", Version: otellambda.SemVersion()},
Spans: []*v1trace.Span{{
TraceId: []byte{0x57, 0x59, 0xe9, 0x88, 0xbd, 0x86, 0x2e, 0x3f, 0xe1, 0xbe, 0x46, 0xa9, 0x94, 0x27, 0x27, 0x93},
TraceId: []byte{0x67, 0x69, 0xe9, 0x88, 0xbd, 0x86, 0x2e, 0x3f, 0xe1, 0xbe, 0x46, 0xa9, 0x94, 0x27, 0x27, 0x93},
SpanId: nil,
TraceState: "",
ParentSpanId: []byte{0x53, 0x99, 0x5c, 0x3f, 0x42, 0xcd, 0x8a, 0xd8},
ParentSpanId: []byte{0x63, 0x99, 0x6c, 0x3f, 0x42, 0xcd, 0x8a, 0xd8},
Name: "testFunction",
Kind: v1trace.Span_SPAN_KIND_SERVER,
StartTimeUnixNano: 0,
Expand All @@ -108,7 +78,7 @@ var (
DroppedAttributesCount: 0,
Events: nil,
DroppedEventsCount: 0,
Links: nil,
Links: []*v1trace.Span_Link{{TraceId: []byte{0x57, 0x59, 0xe9, 0x88, 0xbd, 0x86, 0x2e, 0x3f, 0xe1, 0xbe, 0x46, 0xa9, 0x94, 0x27, 0x27, 0x93}, SpanId: []byte{0x53, 0x99, 0x5c, 0x3f, 0x42, 0xcd, 0x8a, 0xd8}, Attributes: []*v1common.KeyValue{{Key: "source", Value: &v1common.AnyValue{Value: &v1common.AnyValue_StringValue{StringValue: "x-ray-env"}}}}}},
DroppedLinksCount: 0,
Status: &v1trace.Status{Code: v1trace.Status_STATUS_CODE_UNSET},
}},
Expand Down Expand Up @@ -145,7 +115,7 @@ func assertResourceEquals(t *testing.T, expected *v1resource.Resource, actual *v

// ignore timestamps and SpanID since time is obviously variable,
// and SpanID is randomized when using xray IDGenerator.
func assertSpanEqualsIgnoreTimeAndSpanID(t *testing.T, expected *v1trace.ResourceSpans, actual *v1trace.ResourceSpans) {
func assertSpanEqualsIgnoreTimeAndTraceIDAndSpanID(t *testing.T, expected *v1trace.ResourceSpans, actual *v1trace.ResourceSpans) {
assert.Equal(t, expected.ScopeSpans[0].Scope, actual.ScopeSpans[0].Scope)

actualSpan := actual.ScopeSpans[0].Spans[0]
Expand Down Expand Up @@ -189,5 +159,5 @@ func TestWrapEndToEnd(t *testing.T) {

resSpans := mockCollector.getResourceSpans()
assert.Len(t, resSpans, 1)
assertSpanEqualsIgnoreTimeAndSpanID(t, &expectedResourceSpans, resSpans[0])
assertSpanEqualsIgnoreTimeAndTraceIDAndSpanID(t, &expectedResourceSpans, resSpans[0])
}

0 comments on commit 67d1ded

Please sign in to comment.