From eee54f7762258b42cc0ba464e1fcc2c41011cd80 Mon Sep 17 00:00:00 2001 From: Ofek Lev Date: Wed, 31 Jul 2024 10:20:18 -0400 Subject: [PATCH] Implement Python API for submitting logs (#26978) --- pkg/collector/python/datadog_agent.go | 22 ++++++++++ pkg/collector/python/init.go | 2 + ...s-from-python-checks-4fa62927ae2b86a2.yaml | 5 +++ rtloader/common/builtins/datadog_agent.c | 43 +++++++++++++++++++ rtloader/common/builtins/datadog_agent.h | 9 ++++ rtloader/include/datadog_agent_rtloader.h | 11 +++++ rtloader/include/rtloader.h | 9 ++++ rtloader/include/rtloader_types.h | 2 + rtloader/rtloader/api.cpp | 5 +++ rtloader/test/datadog_agent/datadog_agent.go | 13 ++++++ .../test/datadog_agent/datadog_agent_test.go | 13 ++++++ rtloader/three/three.cpp | 5 +++ rtloader/three/three.h | 1 + rtloader/two/two.cpp | 5 +++ rtloader/two/two.h | 1 + 15 files changed, 146 insertions(+) create mode 100644 releasenotes/notes/submit-logs-from-python-checks-4fa62927ae2b86a2.yaml diff --git a/pkg/collector/python/datadog_agent.go b/pkg/collector/python/datadog_agent.go index 040995dc0c554..17c4214e49975 100644 --- a/pkg/collector/python/datadog_agent.go +++ b/pkg/collector/python/datadog_agent.go @@ -201,6 +201,28 @@ func ReadPersistentCache(key *C.char) *C.char { return TrackedCString(data) } +// SendLog submits a log for one check instance. +// Indirectly used by the C function `send_log` that's mapped to `datadog_agent.send_log`. +// +//export SendLog +func SendLog(logLine, checkID *C.char) { + line := C.GoString(logLine) + cid := C.GoString(checkID) + + cc, err := getCheckContext() + if err != nil { + log.Errorf("Log submission failed: %s", err) + } + + lr, ok := cc.logReceiver.Get() + if !ok { + log.Error("Log submission failed: no receiver") + return + } + + lr.SendLog(line, cid) +} + var ( // one obfuscator instance is shared across all python checks. It is not threadsafe but that is ok because // the GIL is always locked when calling c code from python which means that the exported functions in this file diff --git a/pkg/collector/python/init.go b/pkg/collector/python/init.go index de57c3d841f83..f3d9901ba508e 100644 --- a/pkg/collector/python/init.go +++ b/pkg/collector/python/init.go @@ -86,6 +86,7 @@ void GetHostname(char **); void GetVersion(char **); void Headers(char **); char * ReadPersistentCache(char *); +void SendLog(char *, char *); void SetCheckMetadata(char *, char *, char *); void SetExternalTags(char *, char *, char **); void WritePersistentCache(char *, char *); @@ -101,6 +102,7 @@ void initDatadogAgentModule(rtloader_t *rtloader) { set_get_hostname_cb(rtloader, GetHostname); set_get_version_cb(rtloader, GetVersion); set_headers_cb(rtloader, Headers); + set_send_log_cb(rtloader, SendLog); set_set_check_metadata_cb(rtloader, SetCheckMetadata); set_set_external_tags_cb(rtloader, SetExternalTags); set_write_persistent_cache_cb(rtloader, WritePersistentCache); diff --git a/releasenotes/notes/submit-logs-from-python-checks-4fa62927ae2b86a2.yaml b/releasenotes/notes/submit-logs-from-python-checks-4fa62927ae2b86a2.yaml new file mode 100644 index 0000000000000..537b71eeafe9a --- /dev/null +++ b/releasenotes/notes/submit-logs-from-python-checks-4fa62927ae2b86a2.yaml @@ -0,0 +1,5 @@ +--- +enhancements: + - | + Implement API that allows Python checks to send logs for + eventual submission. diff --git a/rtloader/common/builtins/datadog_agent.c b/rtloader/common/builtins/datadog_agent.c index 165d962134755..d65e2c991d1bd 100644 --- a/rtloader/common/builtins/datadog_agent.c +++ b/rtloader/common/builtins/datadog_agent.c @@ -16,6 +16,7 @@ static cb_get_hostname_t cb_get_hostname = NULL; static cb_tracemalloc_enabled_t cb_tracemalloc_enabled = NULL; static cb_get_version_t cb_get_version = NULL; static cb_headers_t cb_headers = NULL; +static cb_send_log_t cb_send_log = NULL; static cb_set_check_metadata_t cb_set_check_metadata = NULL; static cb_set_external_tags_t cb_set_external_tags = NULL; static cb_write_persistent_cache_t cb_write_persistent_cache = NULL; @@ -33,6 +34,7 @@ static PyObject *tracemalloc_enabled(PyObject *self, PyObject *args); static PyObject *get_version(PyObject *self, PyObject *args); static PyObject *headers(PyObject *self, PyObject *args, PyObject *kwargs); static PyObject *log_message(PyObject *self, PyObject *args); +static PyObject *send_log(PyObject *self, PyObject *args); static PyObject *set_check_metadata(PyObject *self, PyObject *args); static PyObject *set_external_tags(PyObject *self, PyObject *args); static PyObject *write_persistent_cache(PyObject *self, PyObject *args); @@ -50,6 +52,7 @@ static PyMethodDef methods[] = { { "get_version", get_version, METH_NOARGS, "Get Agent version." }, { "headers", (PyCFunction)headers, METH_VARARGS | METH_KEYWORDS, "Get standard set of HTTP headers." }, { "log", log_message, METH_VARARGS, "Log a message through the agent logger." }, + { "send_log", send_log, METH_VARARGS, "Submit a log for Checks." }, { "set_check_metadata", set_check_metadata, METH_VARARGS, "Send metadata for Checks." }, { "set_external_tags", set_external_tags, METH_VARARGS, "Send external host tags." }, { "write_persistent_cache", write_persistent_cache, METH_VARARGS, "Store a value for a given key." }, @@ -103,6 +106,11 @@ void _set_get_clustername_cb(cb_get_clustername_t cb) cb_get_clustername = cb; } +void _set_send_log_cb(cb_send_log_t cb) +{ + cb_send_log = cb; +} + void _set_set_check_metadata_cb(cb_set_check_metadata_t cb) { cb_set_check_metadata = cb; @@ -411,6 +419,41 @@ static PyObject *log_message(PyObject *self, PyObject *args) Py_RETURN_NONE; } +/*! \fn PyObject *send_log(PyObject *self, PyObject *args) + \brief This function implements the `datadog_agent.send_log` method, sending + a log for eventual submission. + \param self A PyObject* pointer to the `datadog_agent` module. + \param args A PyObject* pointer to a 2-ary tuple containing a log line and the + unique ID of a check instance. + \return A PyObject* pointer to `None`. + + This function is callable as the `datadog_agent.send_log` Python method and + uses the `cb_send_log()` callback to retrieve the value from the agent + with CGO. If the callback has not been set `None` will be returned. +*/ +static PyObject *send_log(PyObject *self, PyObject *args) +{ + // callback must be set + if (cb_send_log == NULL) { + Py_RETURN_NONE; + } + + char *log_line, *check_id; + + PyGILState_STATE gstate = PyGILState_Ensure(); + + // datadog_agent.send_log(log_line, check_id) + if (!PyArg_ParseTuple(args, "ss", &log_line, &check_id)) { + PyGILState_Release(gstate); + return NULL; + } + + PyGILState_Release(gstate); + cb_send_log(log_line, check_id); + + Py_RETURN_NONE; +} + /*! \fn PyObject *set_check_metadata(PyObject *self, PyObject *args) \brief This function implements the `datadog_agent.set_check_metadata` method, updating the value in the cache. diff --git a/rtloader/common/builtins/datadog_agent.h b/rtloader/common/builtins/datadog_agent.h index b51f1a43f35e6..87c55c92e13aa 100644 --- a/rtloader/common/builtins/datadog_agent.h +++ b/rtloader/common/builtins/datadog_agent.h @@ -74,6 +74,14 @@ The callback is expected to be provided by the rtloader caller - in go-context: CGO. */ +/*! \fn void _set_send_log_cb(cb_send_log_t) + \brief Sets a callback to be used by rtloader to allow for submitting a log for a given + check instance. + \param object A function pointer with cb_send_log_t prototype to the callback + function. + + The callback is expected to be provided by the rtloader caller - in go-context: CGO. +*/ /*! \fn void _set_set_check_metadata_cb(cb_set_check_metadata_t) \brief Sets a callback to be used by rtloader to allow setting metadata for a given check instance. @@ -142,6 +150,7 @@ void _set_tracemalloc_enabled_cb(cb_tracemalloc_enabled_t); void _set_get_version_cb(cb_get_version_t); void _set_headers_cb(cb_headers_t); void _set_log_cb(cb_log_t); +void _set_send_log_cb(cb_send_log_t); void _set_set_check_metadata_cb(cb_set_check_metadata_t); void _set_set_external_tags_cb(cb_set_external_tags_t); void _set_write_persistent_cache_cb(cb_write_persistent_cache_t); diff --git a/rtloader/include/datadog_agent_rtloader.h b/rtloader/include/datadog_agent_rtloader.h index 7887a5ef19935..f3bf407596af4 100644 --- a/rtloader/include/datadog_agent_rtloader.h +++ b/rtloader/include/datadog_agent_rtloader.h @@ -489,6 +489,17 @@ DATADOG_AGENT_RTLOADER_API void set_tracemalloc_enabled_cb(rtloader_t *, cb_trac */ DATADOG_AGENT_RTLOADER_API void set_log_cb(rtloader_t *, cb_log_t); +/*! \fn void set_send_log_cb(rtloader_t *, cb_send_log_t) + \brief Sets a callback to be used by rtloader to allow for submitting a log for a given + check instance. + \param rtloader_t A rtloader_t * pointer to the RtLoader instance. + \param object A function pointer with cb_send_log_t prototype to the callback + function. + + The callback is expected to be provided by the rtloader caller - in go-context: CGO. +*/ +DATADOG_AGENT_RTLOADER_API void set_send_log_cb(rtloader_t *, cb_send_log_t); + /*! \fn void set_set_check_metadata_cb(rtloader_t *, cb_set_check_metadata_t) \brief Sets a callback to be used by rtloader to allow setting metadata for a given check instance. diff --git a/rtloader/include/rtloader.h b/rtloader/include/rtloader.h index 51a9ad6e66b95..6ea5f197bd94e 100644 --- a/rtloader/include/rtloader.h +++ b/rtloader/include/rtloader.h @@ -336,6 +336,15 @@ class RtLoader */ virtual void setLogCb(cb_log_t) = 0; + //! sendLogCb member. + /*! + \param A cb_send_log_t function pointer to the CGO callback. + + This allows us to set the relevant CGO callback that will allow for sending a log for + eventual submission for a specific check instance. + */ + virtual void setSendLogCb(cb_send_log_t) = 0; + //! setCheckMetadataCb member. /*! \param A cb_set_check_metadata_t function pointer to the CGO callback. diff --git a/rtloader/include/rtloader_types.h b/rtloader/include/rtloader_types.h index edf72d53cf82c..32094bbd0d252 100644 --- a/rtloader/include/rtloader_types.h +++ b/rtloader/include/rtloader_types.h @@ -123,6 +123,8 @@ typedef void (*cb_get_clustername_t)(char **); typedef bool (*cb_tracemalloc_enabled_t)(void); // (message, level) typedef void (*cb_log_t)(char *, int); +// (log_line, check_id) +typedef void (*cb_send_log_t)(char *, char *); // (check_id, name, value) typedef void (*cb_set_check_metadata_t)(char *, char *, char *); // (hostname, source_type_name, list of tags) diff --git a/rtloader/rtloader/api.cpp b/rtloader/rtloader/api.cpp index 71beaf2e16c16..214ec0144c79d 100644 --- a/rtloader/rtloader/api.cpp +++ b/rtloader/rtloader/api.cpp @@ -527,6 +527,11 @@ void set_log_cb(rtloader_t *rtloader, cb_log_t cb) AS_TYPE(RtLoader, rtloader)->setLogCb(cb); } +void set_send_log_cb(rtloader_t *rtloader, cb_send_log_t cb) +{ + AS_TYPE(RtLoader, rtloader)->setSendLogCb(cb); +} + void set_set_check_metadata_cb(rtloader_t *rtloader, cb_set_check_metadata_t cb) { AS_TYPE(RtLoader, rtloader)->setSetCheckMetadataCb(cb); diff --git a/rtloader/test/datadog_agent/datadog_agent.go b/rtloader/test/datadog_agent/datadog_agent.go index 58c7260598f4b..445d4bafa5794 100644 --- a/rtloader/test/datadog_agent/datadog_agent.go +++ b/rtloader/test/datadog_agent/datadog_agent.go @@ -32,6 +32,7 @@ extern void getHostname(char **); extern bool getTracemallocEnabled(); extern void getVersion(char **); extern void headers(char **); +extern void sendLog(char *, char *); extern void setCheckMetadata(char*, char*, char*); extern void setExternalHostTags(char*, char*, char**); extern void writePersistentCache(char*, char*); @@ -51,6 +52,7 @@ static void initDatadogAgentTests(rtloader_t *rtloader) { set_get_version_cb(rtloader, getVersion); set_headers_cb(rtloader, headers); set_log_cb(rtloader, doLog); + set_send_log_cb(rtloader, sendLog); set_set_check_metadata_cb(rtloader, setCheckMetadata); set_set_external_tags_cb(rtloader, setExternalHostTags); set_write_persistent_cache_cb(rtloader, writePersistentCache); @@ -190,6 +192,17 @@ func doLog(msg *C.char, level C.int) { os.WriteFile(tmpfile.Name(), data, 0644) } +//export sendLog +func sendLog(logLine, checkID *C.char) { + line := C.GoString(logLine) + cid := C.GoString(checkID) + + f, _ := os.OpenFile(tmpfile.Name(), os.O_APPEND|os.O_RDWR|os.O_CREATE, 0666) + defer f.Close() + + f.WriteString(strings.Join([]string{line, cid}, ",")) +} + //export setCheckMetadata func setCheckMetadata(checkID, name, value *C.char) { cid := C.GoString(checkID) diff --git a/rtloader/test/datadog_agent/datadog_agent_test.go b/rtloader/test/datadog_agent/datadog_agent_test.go index 6cacc39696ef6..8f2317f4bbf77 100644 --- a/rtloader/test/datadog_agent/datadog_agent_test.go +++ b/rtloader/test/datadog_agent/datadog_agent_test.go @@ -179,6 +179,19 @@ func TestLog(t *testing.T) { helpers.AssertMemoryUsage(t) } +func TestSendLog(t *testing.T) { + code := ` + datadog_agent.send_log("log line", "postgres:test:12345") + ` + out, err := run(code) + if err != nil { + t.Fatal(err) + } + if out != "log line,postgres:test:12345" { + t.Errorf("Unexpected printed value: '%s'", out) + } +} + func TestSetCheckMetadata(t *testing.T) { code := ` datadog_agent.set_check_metadata("redis:test:12345", "version.raw", "5.0.6") diff --git a/rtloader/three/three.cpp b/rtloader/three/three.cpp index de46ac22d4d2b..725c940f50735 100644 --- a/rtloader/three/three.cpp +++ b/rtloader/three/three.cpp @@ -925,6 +925,11 @@ void Three::setLogCb(cb_log_t cb) _set_log_cb(cb); } +void Three::setSendLogCb(cb_send_log_t cb) +{ + _set_send_log_cb(cb); +} + void Three::setSetCheckMetadataCb(cb_set_check_metadata_t cb) { _set_set_check_metadata_cb(cb); diff --git a/rtloader/three/three.h b/rtloader/three/three.h index 4c37747e6ff38..20cd09415e58e 100644 --- a/rtloader/three/three.h +++ b/rtloader/three/three.h @@ -104,6 +104,7 @@ class Three : public RtLoader void setGetClusternameCb(cb_get_clustername_t); void setGetTracemallocEnabledCb(cb_tracemalloc_enabled_t); void setLogCb(cb_log_t); + void setSendLogCb(cb_send_log_t); void setSetCheckMetadataCb(cb_set_check_metadata_t); void setSetExternalTagsCb(cb_set_external_tags_t); void setWritePersistentCacheCb(cb_write_persistent_cache_t); diff --git a/rtloader/two/two.cpp b/rtloader/two/two.cpp index 38ab2bb5fdd99..22e9110f535ee 100644 --- a/rtloader/two/two.cpp +++ b/rtloader/two/two.cpp @@ -923,6 +923,11 @@ void Two::setLogCb(cb_log_t cb) _set_log_cb(cb); } +void Two::setSendLogCb(cb_send_log_t cb) +{ + _set_send_log_cb(cb); +} + void Two::setSetCheckMetadataCb(cb_set_check_metadata_t cb) { _set_set_check_metadata_cb(cb); diff --git a/rtloader/two/two.h b/rtloader/two/two.h index 9896f7b48a71f..8b0afdb73a39b 100644 --- a/rtloader/two/two.h +++ b/rtloader/two/two.h @@ -102,6 +102,7 @@ class Two : public RtLoader void setGetClusternameCb(cb_get_clustername_t); void setGetTracemallocEnabledCb(cb_tracemalloc_enabled_t); void setLogCb(cb_log_t); + void setSendLogCb(cb_send_log_t); void setSetCheckMetadataCb(cb_set_check_metadata_t); void setSetExternalTagsCb(cb_set_external_tags_t); void setWritePersistentCacheCb(cb_write_persistent_cache_t);