Skip to content

Commit

Permalink
Add support for tracking Redis client calls for non-Go programs (#891)
Browse files Browse the repository at this point in the history
  • Loading branch information
grcevski authored May 30, 2024
1 parent 33422a4 commit 05ff13a
Show file tree
Hide file tree
Showing 95 changed files with 1,145 additions and 140 deletions.
9 changes: 7 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -282,13 +282,18 @@ oats-test-sql-other-langs: oats-prereq
mkdir -p test/oats/sql_other_langs/$(TEST_OUTPUT)/run
cd test/oats/sql_other_langs && TESTCASE_BASE_PATH=./yaml $(GINKGO) -v -r

.PHONY: oats-test-redis-other-langs
oats-test-redis-other-langs: oats-prereq
mkdir -p test/oats/redis_other_langs/$(TEST_OUTPUT)/run
cd test/oats/redis_other_langs && TESTCASE_BASE_PATH=./yaml $(GINKGO) -v -r

.PHONY: oats-test
oats-test: oats-test-sql oats-test-sql-statement oats-test-sql-other-langs
oats-test: oats-test-sql oats-test-sql-statement oats-test-sql-other-langs oats-test-redis-other-langs
$(MAKE) itest-coverage-data

.PHONY: oats-test-debug
oats-test-debug: oats-prereq
cd test/oats/sql && TESTCASE_BASE_PATH=./yaml TESTCASE_MANUAL_DEBUG=true TESTCASE_TIMEOUT=1h $(GINKGO) -v -r
cd test/oats/sql_statement && TESTCASE_BASE_PATH=./yaml TESTCASE_MANUAL_DEBUG=true TESTCASE_TIMEOUT=1h $(GINKGO) -v -r

.PHONY: drone
drone:
Expand Down
1 change: 1 addition & 0 deletions bpf/http_sock.h
Original file line number Diff line number Diff line change
Expand Up @@ -552,6 +552,7 @@ static __always_inline void handle_unknown_tcp_connection(pid_connection_info_t
bpf_dbg_printk("Sending TCP trace %lx, response length %d", existing, existing->resp_len);

bpf_memcpy(trace, existing, sizeof(tcp_req_t));
bpf_probe_read(trace->rbuf, K_TCP_RES_LEN, u_buf);
bpf_ringbuf_submit(trace, get_flags());
}
bpf_map_delete_elem(&ongoing_tcp_req, pid_conn);
Expand Down
2 changes: 2 additions & 0 deletions bpf/http_types.h
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
#define KPROBES_LARGE_RESPONSE_LEN 100000 // 100K and above we try to track the response actual time with kretprobes

#define K_TCP_MAX_LEN 256
#define K_TCP_RES_LEN 24

#define CONN_INFO_FLAG_TRACE 0x1

Expand Down Expand Up @@ -96,6 +97,7 @@ typedef struct tcp_req {
u64 start_monotime_ns;
u64 end_monotime_ns;
unsigned char buf[K_TCP_MAX_LEN] __attribute__ ((aligned (8))); // ringbuffer memcpy complains unless this is 8 byte aligned
unsigned char rbuf[K_TCP_RES_LEN] __attribute__ ((aligned (8))); // ringbuffer memcpy complains unless this is 8 byte aligned
u32 len;
u32 resp_len;
u8 ssl;
Expand Down
106 changes: 54 additions & 52 deletions docs/sources/metrics.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ The following table describes the exported metrics in both OpenTelemetry and Pro
| Application | `rpc.client.duration` | `rpc_client_duration_seconds` | Histogram | seconds | Duration of GRPC service calls from the client side |
| Application | `rpc.server.duration` | `rpc_server_duration_seconds` | Histogram | seconds | Duration of RPC service calls from the server side |
| Application | `sql.client.duration` | `sql_client_duration_seconds` | Histogram | seconds | Duration of SQL client operations (Experimental) |
| Application | `redis.client.duration` | `redis_client_duration_seconds` | Histogram | seconds | Duration of Redis client operations (Experimental) |
| Network | `beyla.network.flow.bytes` | `beyla_network_flow_bytes` | Counter | bytes | Bytes submitted from a source network endpoint to a destination network endpoint |

Beyla can also export [Span metrics](/docs/tempo/latest/metrics-generator/span_metrics/) and
Expand All @@ -38,58 +39,59 @@ the metrics and attributes are exposed `underscore_notation` when a Prometheus e
In order to hide attributes that are shown by default, or show attributes that are hidden by
default, check the `attributes`->`select` section in the [configuration documentation]({{< relref "./configure/options.md" >}}).

| Metrics | Name | Default |
|----------------------------|-----------------------------|-----------------------------------------------|
| Application (all) | `http.request.method` | shown |
| Application (all) | `http.response.status_code` | shown |
| Application (all) | `http.route` | shown if `routes` configuration is defined |
| Application (all) | `k8s.daemonset.name` | shown if `attributes.kubernetes.enable` |
| Application (all) | `k8s.deployment.name` | shown if `attributes.kubernetes.enable` |
| Application (all) | `k8s.namespace.name` | shown if `attributes.kubernetes.enable` |
| Application (all) | `k8s.node.name` | shown if `attributes.kubernetes.enable` |
| Application (all) | `k8s.pod.name` | shown if `attributes.kubernetes.enable` |
| Application (all) | `k8s.pod.start_time` | shown if `attributes.kubernetes.enable` |
| Application (all) | `k8s.pod.uid` | shown if `attributes.kubernetes.enable` |
| Application (all) | `k8s.replicaset.name` | shown if `attributes.kubernetes.enable` |
| Application (all) | `k8s.statefulset.name` | shown if `attributes.kubernetes.enable` |
| Application (all) | `service.name` | shown |
| Application (all) | `service.namespace` | shown |
| Application (all) | `target.instance` | shown |
| Application (all) | `url.path` | hidden |
| Application (client) | `server.address` | hidden |
| Application (client) | `server.port` | hidden |
| Application `rpc.*` | `rpc.grpc.status_code` | shown |
| Application `rpc.*` | `rpc.method` | shown |
| Application `rpc.*` | `rpc.system` | shown |
| Application (server) | `client.address` | hidden |
| `beyla.network.flow.bytes` | `beyla.ip` | hidden |
| `sql.client.duration` | `db.operation` | shown |
| `sql.client.duration` | `db.statement` | shown |
| `beyla.network.flow.bytes` | `direction` | hidden |
| `beyla.network.flow.bytes` | `dst.address` | hidden |
| `beyla.network.flow.bytes` | `dst.cidr` | shown if the `cidrs` configuration is defined |
| `beyla.network.flow.bytes` | `dst.name` | hidden |
| `beyla.network.flow.bytes` | `dst.port` | hidden |
| `beyla.network.flow.bytes` | `iface` | hidden |
| `beyla.network.flow.bytes` | `k8s.cluster.name` | shown if `attributes.kubernetes.enable` |
| `beyla.network.flow.bytes` | `k8s.dst.name` | hidden |
| `beyla.network.flow.bytes` | `k8s.dst.namespace` | shown if `attributes.kubernetes.enable` |
| `beyla.network.flow.bytes` | `k8s.dst.node.ip` | hidden |
| `beyla.network.flow.bytes` | `k8s.dst.node.name` | hidden |
| `beyla.network.flow.bytes` | `k8s.dst.owner.type` | hidden |
| `beyla.network.flow.bytes` | `k8s.dst.type` | hidden |
| `beyla.network.flow.bytes` | `k8s.dst.owner.name` | shown if `attributes.kubernetes.enable` |
| `beyla.network.flow.bytes` | `k8s.src.name` | hidden |
| `beyla.network.flow.bytes` | `k8s.src.namespace` | shown if `attributes.kubernetes.enable` |
| `beyla.network.flow.bytes` | `k8s.src.node.ip` | hidden |
| `beyla.network.flow.bytes` | `k8s.src.owner.name` | shown if `attributes.kubernetes.enable` |
| `beyla.network.flow.bytes` | `k8s.src.owner.type` | hidden |
| `beyla.network.flow.bytes` | `k8s.src.type` | hidden |
| `beyla.network.flow.bytes` | `src.address` | hidden |
| `beyla.network.flow.bytes` | `src.cidr` | shown if the `cidrs` configuration is defined |
| `beyla.network.flow.bytes` | `src.name` | hidden |
| `beyla.network.flow.bytes` | `src.port` | hidden |
| `beyla.network.flow.bytes` | `transport` | hidden |
| Metrics | Name | Default |
|--------------------------------|-----------------------------|-----------------------------------------------|
| Application (all) | `http.request.method` | shown |
| Application (all) | `http.response.status_code` | shown |
| Application (all) | `http.route` | shown if `routes` configuration is defined |
| Application (all) | `k8s.daemonset.name` | shown if `attributes.kubernetes.enable` |
| Application (all) | `k8s.deployment.name` | shown if `attributes.kubernetes.enable` |
| Application (all) | `k8s.namespace.name` | shown if `attributes.kubernetes.enable` |
| Application (all) | `k8s.node.name` | shown if `attributes.kubernetes.enable` |
| Application (all) | `k8s.pod.name` | shown if `attributes.kubernetes.enable` |
| Application (all) | `k8s.pod.start_time` | shown if `attributes.kubernetes.enable` |
| Application (all) | `k8s.pod.uid` | shown if `attributes.kubernetes.enable` |
| Application (all) | `k8s.replicaset.name` | shown if `attributes.kubernetes.enable` |
| Application (all) | `k8s.statefulset.name` | shown if `attributes.kubernetes.enable` |
| Application (all) | `service.name` | shown |
| Application (all) | `service.namespace` | shown |
| Application (all) | `target.instance` | shown |
| Application (all) | `url.path` | hidden |
| Application (client) | `server.address` | hidden |
| Application (client) | `server.port` | hidden |
| Application `rpc.*` | `rpc.grpc.status_code` | shown |
| Application `rpc.*` | `rpc.method` | shown |
| Application `rpc.*` | `rpc.system` | shown |
| Application (server) | `client.address` | hidden |
| `beyla.network.flow.bytes` | `beyla.ip` | hidden |
| `db.client.operation.duration` | `db.operation.name` | shown |
| `db.client.operation.duration` | `db.collection.name` | hidden |
| `beyla.network.flow.bytes` | `direction` | hidden |
| `beyla.network.flow.bytes` | `dst.address` | hidden |
| `beyla.network.flow.bytes` | `dst.cidr` | shown if the `cidrs` configuration is defined |
| `beyla.network.flow.bytes` | `dst.name` | hidden |
| `beyla.network.flow.bytes` | `dst.port` | hidden |
| `beyla.network.flow.bytes` | `iface` | hidden |
| `beyla.network.flow.bytes` | `k8s.cluster.name` | shown if `attributes.kubernetes.enable` |
| `beyla.network.flow.bytes` | `k8s.dst.name` | hidden |
| `beyla.network.flow.bytes` | `k8s.dst.namespace` | shown if `attributes.kubernetes.enable` |
| `beyla.network.flow.bytes` | `k8s.dst.node.ip` | hidden |
| `beyla.network.flow.bytes` | `k8s.dst.node.name` | hidden |
| `beyla.network.flow.bytes` | `k8s.dst.owner.type` | hidden |
| `beyla.network.flow.bytes` | `k8s.dst.type` | hidden |
| `beyla.network.flow.bytes` | `k8s.dst.owner.name` | shown if `attributes.kubernetes.enable` |
| `beyla.network.flow.bytes` | `k8s.src.name` | hidden |
| `beyla.network.flow.bytes` | `k8s.src.namespace` | shown if `attributes.kubernetes.enable` |
| `beyla.network.flow.bytes` | `k8s.src.node.ip` | hidden |
| `beyla.network.flow.bytes` | `k8s.src.owner.name` | shown if `attributes.kubernetes.enable` |
| `beyla.network.flow.bytes` | `k8s.src.owner.type` | hidden |
| `beyla.network.flow.bytes` | `k8s.src.type` | hidden |
| `beyla.network.flow.bytes` | `src.address` | hidden |
| `beyla.network.flow.bytes` | `src.cidr` | shown if the `cidrs` configuration is defined |
| `beyla.network.flow.bytes` | `src.name` | hidden |
| `beyla.network.flow.bytes` | `src.port` | hidden |
| `beyla.network.flow.bytes` | `transport` | hidden |
| Traces (SQL, Redis) | `db.query.text` | hidden |

## Internal metrics

Expand Down
1 change: 1 addition & 0 deletions pkg/internal/ebpf/common/bpf_bpfel_arm64.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Binary file modified pkg/internal/ebpf/common/bpf_bpfel_arm64.o
Binary file not shown.
1 change: 1 addition & 0 deletions pkg/internal/ebpf/common/bpf_bpfel_x86.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Binary file modified pkg/internal/ebpf/common/bpf_bpfel_x86.o
Binary file not shown.
149 changes: 149 additions & 0 deletions pkg/internal/ebpf/common/redis_detect_transform.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
package ebpfcommon

import (
"bytes"
"strings"

trace2 "go.opentelemetry.io/otel/trace"

"github.com/grafana/beyla/pkg/internal/request"
)

const minRedisFrameLen = 3

func isRedis(buf []uint8) bool {
if len(buf) < minRedisFrameLen {
return false
}

return isRedisOp(buf)
}

// nolint:cyclop
func isRedisOp(buf []uint8) bool {
if len(buf) == 0 {
return false
}
c := buf[0]

switch c {
case '+':
return crlfTerminatedMatch(buf[1:], func(c uint8) bool {
return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '.' || c == ' ' || c == '-' || c == '_'
})
case '-':
return isRedisError(buf[1:])
case ':', '$', '*':
return crlfTerminatedMatch(buf[1:], func(c uint8) bool {
return (c >= '0' && c <= '9')
})
}

return false
}

func isRedisError(buf []uint8) bool {
return bytes.HasPrefix(buf, []byte("ERR ")) ||
bytes.HasPrefix(buf, []byte("WRONGTYPE ")) ||
bytes.HasPrefix(buf, []byte("MOVED ")) ||
bytes.HasPrefix(buf, []byte("ASK ")) ||
bytes.HasPrefix(buf, []byte("BUSY ")) ||
bytes.HasPrefix(buf, []byte("NOSCRIPT ")) ||
bytes.HasPrefix(buf, []byte("CLUSTERDOWN "))
}

func crlfTerminatedMatch(buf []uint8, matches func(c uint8) bool) bool {
cr := false
i := 0
for ; i < len(buf); i++ {
c := buf[i]
if matches(c) {
continue
}
if c == '\r' {
cr = true
break
}

return false
}

if !cr || i >= len(buf)-1 {
return false
}

return buf[i+1] == '\n'
}

func parseRedisRequest(buf string) (string, string, bool) {
lines := strings.Split(buf, "\r\n")

if len(lines) < 2 {
return "", "", false
}

// It's not a command, something else?
if lines[0][0] != '*' {
return "", "", true
}

op := ""
text := ""

read := false
// Skip the first line
for _, l := range lines[1:] {
if len(l) == 0 {
continue
}
if !read {
if isRedisOp([]uint8(l + "\r\n")) {
read = true
} else {
break
}
} else {
if op == "" {
op = l
}
text += l + " "
read = false
}
}

return op, text, true
}

func TCPToRedisToSpan(trace *TCPRequestInfo, op, text string, status int) request.Span {
peer := ""
hostname := ""
hostPort := 0

if trace.ConnInfo.S_port != 0 || trace.ConnInfo.D_port != 0 {
peer, hostname = trace.reqHostInfo()
hostPort = int(trace.ConnInfo.D_port)
}

return request.Span{
Type: request.EventTypeRedisClient,
Method: op,
Path: text,
Peer: peer,
Host: hostname,
HostPort: hostPort,
ContentLength: 0,
RequestStart: int64(trace.StartMonotimeNs),
Start: int64(trace.StartMonotimeNs),
End: int64(trace.EndMonotimeNs),
Status: status,
TraceID: trace2.TraceID(trace.Tp.TraceId),
SpanID: trace2.SpanID(trace.Tp.SpanId),
ParentSpanID: trace2.SpanID(trace.Tp.ParentId),
Flags: trace.Tp.Flags,
Pid: request.PidInfo{
HostPID: trace.Pid.HostPid,
UserPID: trace.Pid.UserPid,
Namespace: trace.Pid.Ns,
},
}
}
29 changes: 29 additions & 0 deletions pkg/internal/ebpf/common/redis_detect_transform_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
package ebpfcommon

import (
"testing"

"github.com/stretchr/testify/assert"
)

type crlfTest struct {
testStr string
result bool
}

func TestCRLFMatching(t *testing.T) {
for _, ts := range []crlfTest{
{testStr: "Not a sql or any known protocol", result: false},
{testStr: "Not a sql or any known protocol\r\n", result: true},
{testStr: "123\r\n", result: false},
{testStr: "\r\n", result: true},
{testStr: "\n", result: false},
{testStr: "\r", result: false},
{testStr: "", result: false},
} {
res := crlfTerminatedMatch([]uint8(ts.testStr), func(c uint8) bool {
return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '.' || c == ' ' || c == '-' || c == '_'
})
assert.Equal(t, res, ts.result)
}
}
Loading

0 comments on commit 05ff13a

Please sign in to comment.