Implement Python API for submitting logs (#26978)

pkg/collector/python/datadog_agent.go

@@ -201,6 +201,28 @@ func ReadPersistentCache(key *C.char) *C.char {
  return TrackedCString(data)
 }
 
+// SendLog submits a log for one check instance.
+// Indirectly used by the C function `send_log` that's mapped to `datadog_agent.send_log`.
+//
+//export SendLog
+func SendLog(logLine, checkID *C.char) {
+ line := C.GoString(logLine)
+ cid := C.GoString(checkID)
+
+ cc, err := getCheckContext()
+ if err != nil {
+ log.Errorf("Log submission failed: %s", err)
+ }
+
+ lr, ok := cc.logReceiver.Get()
+ if !ok {
+ log.Error("Log submission failed: no receiver")
+ return
+ }
+
+ lr.SendLog(line, cid)
+}
+
 var (
  // one obfuscator instance is shared across all python checks. It is not threadsafe but that is ok because
  // the GIL is always locked when calling c code from python which means that the exported functions in this file

pkg/collector/python/init.go

@@ -86,6 +86,7 @@ void GetHostname(char **);
 void GetVersion(char **);
 void Headers(char **);
 char * ReadPersistentCache(char *);
+void SendLog(char *, char *);
 void SetCheckMetadata(char *, char *, char *);
 void SetExternalTags(char *, char *, char **);
 void WritePersistentCache(char *, char *);
@@ -101,6 +102,7 @@ void initDatadogAgentModule(rtloader_t *rtloader) {
  set_get_hostname_cb(rtloader, GetHostname);
  set_get_version_cb(rtloader, GetVersion);
  set_headers_cb(rtloader, Headers);
+ set_send_log_cb(rtloader, SendLog);
  set_set_check_metadata_cb(rtloader, SetCheckMetadata);
  set_set_external_tags_cb(rtloader, SetExternalTags);
  set_write_persistent_cache_cb(rtloader, WritePersistentCache);

releasenotes/notes/submit-logs-from-python-checks-4fa62927ae2b86a2.yaml

@@ -0,0 +1,5 @@
+---
+enhancements:
+ - |
+ Implement API that allows Python checks to send logs for
+ eventual submission.

rtloader/common/builtins/datadog_agent.c

@@ -16,6 +16,7 @@ static cb_get_hostname_t cb_get_hostname = NULL;
 static cb_tracemalloc_enabled_t cb_tracemalloc_enabled = NULL;
 static cb_get_version_t cb_get_version = NULL;
 static cb_headers_t cb_headers = NULL;
+static cb_send_log_t cb_send_log = NULL;
 static cb_set_check_metadata_t cb_set_check_metadata = NULL;
 static cb_set_external_tags_t cb_set_external_tags = NULL;
 static cb_write_persistent_cache_t cb_write_persistent_cache = NULL;
@@ -33,6 +34,7 @@ static PyObject *tracemalloc_enabled(PyObject *self, PyObject *args);
 static PyObject *get_version(PyObject *self, PyObject *args);
 static PyObject *headers(PyObject *self, PyObject *args, PyObject *kwargs);
 static PyObject *log_message(PyObject *self, PyObject *args);
+static PyObject *send_log(PyObject *self, PyObject *args);
 static PyObject *set_check_metadata(PyObject *self, PyObject *args);
 static PyObject *set_external_tags(PyObject *self, PyObject *args);
 static PyObject *write_persistent_cache(PyObject *self, PyObject *args);
@@ -50,6 +52,7 @@ static PyMethodDef methods[] = {
  { "get_version", get_version, METH_NOARGS, "Get Agent version." },
  { "headers", (PyCFunction)headers, METH_VARARGS | METH_KEYWORDS, "Get standard set of HTTP headers." },
  { "log", log_message, METH_VARARGS, "Log a message through the agent logger." },
+ { "send_log", send_log, METH_VARARGS, "Submit a log for Checks." },
  { "set_check_metadata", set_check_metadata, METH_VARARGS, "Send metadata for Checks." },
  { "set_external_tags", set_external_tags, METH_VARARGS, "Send external host tags." },
  { "write_persistent_cache", write_persistent_cache, METH_VARARGS, "Store a value for a given key." },
@@ -103,6 +106,11 @@ void _set_get_clustername_cb(cb_get_clustername_t cb)
  cb_get_clustername = cb;
 }
 
+void _set_send_log_cb(cb_send_log_t cb)
+{
+ cb_send_log = cb;
+}
+
 void _set_set_check_metadata_cb(cb_set_check_metadata_t cb)
 {
  cb_set_check_metadata = cb;
@@ -411,6 +419,41 @@ static PyObject *log_message(PyObject *self, PyObject *args)
  Py_RETURN_NONE;
 }
 
+/*! \fn PyObject *send_log(PyObject *self, PyObject *args)
+ \brief This function implements the `datadog_agent.send_log` method, sending
+ a log for eventual submission.
+ \param self A PyObject* pointer to the `datadog_agent` module.
+ \param args A PyObject* pointer to a 2-ary tuple containing a log line and the
+ unique ID of a check instance.
+ \return A PyObject* pointer to `None`.
+
+ This function is callable as the `datadog_agent.send_log` Python method and
+ uses the `cb_send_log()` callback to retrieve the value from the agent
+ with CGO. If the callback has not been set `None` will be returned.
+*/
+static PyObject *send_log(PyObject *self, PyObject *args)
+{
+ // callback must be set
+ if (cb_send_log == NULL) {
+ Py_RETURN_NONE;
+ }
+
+ char *log_line, *check_id;
+
+ PyGILState_STATE gstate = PyGILState_Ensure();
+
+ // datadog_agent.send_log(log_line, check_id)
+ if (!PyArg_ParseTuple(args, "ss", &log_line, &check_id)) {
+ PyGILState_Release(gstate);
+ return NULL;
+ }
+
+ PyGILState_Release(gstate);
+ cb_send_log(log_line, check_id);
+
+ Py_RETURN_NONE;
+}
+
 /*! \fn PyObject *set_check_metadata(PyObject *self, PyObject *args)
  \brief This function implements the `datadog_agent.set_check_metadata` method, updating
  the value in the cache.

rtloader/common/builtins/datadog_agent.h

@@ -74,6 +74,14 @@
 
  The callback is expected to be provided by the rtloader caller - in go-context: CGO.
 */
+/*! \fn void _set_send_log_cb(cb_send_log_t)
+ \brief Sets a callback to be used by rtloader to allow for submitting a log for a given
+ check instance.
+ \param object A function pointer with cb_send_log_t prototype to the callback
+ function.
+
+ The callback is expected to be provided by the rtloader caller - in go-context: CGO.
+*/
 /*! \fn void _set_set_check_metadata_cb(cb_set_check_metadata_t)
  \brief Sets a callback to be used by rtloader to allow setting metadata for a given
  check instance.
@@ -142,6 +150,7 @@ void _set_tracemalloc_enabled_cb(cb_tracemalloc_enabled_t);
 void _set_get_version_cb(cb_get_version_t);
 void _set_headers_cb(cb_headers_t);
 void _set_log_cb(cb_log_t);
+void _set_send_log_cb(cb_send_log_t);
 void _set_set_check_metadata_cb(cb_set_check_metadata_t);
 void _set_set_external_tags_cb(cb_set_external_tags_t);
 void _set_write_persistent_cache_cb(cb_write_persistent_cache_t);

rtloader/include/datadog_agent_rtloader.h

@@ -489,6 +489,17 @@ DATADOG_AGENT_RTLOADER_API void set_tracemalloc_enabled_cb(rtloader_t *, cb_trac
 */
 DATADOG_AGENT_RTLOADER_API void set_log_cb(rtloader_t *, cb_log_t);
 
+/*! \fn void set_send_log_cb(rtloader_t *, cb_send_log_t)
+ \brief Sets a callback to be used by rtloader to allow for submitting a log for a given
+ check instance.
+ \param rtloader_t A rtloader_t * pointer to the RtLoader instance.
+ \param object A function pointer with cb_send_log_t prototype to the callback
+ function.
+
+ The callback is expected to be provided by the rtloader caller - in go-context: CGO.
+*/
+DATADOG_AGENT_RTLOADER_API void set_send_log_cb(rtloader_t *, cb_send_log_t);
+
 /*! \fn void set_set_check_metadata_cb(rtloader_t *, cb_set_check_metadata_t)
  \brief Sets a callback to be used by rtloader to allow setting metadata for a given
  check instance.

rtloader/include/rtloader.h

@@ -336,6 +336,15 @@ class RtLoader
  */
  virtual void setLogCb(cb_log_t) = 0;
 
+ //! sendLogCb member.
+ /*!
+ \param A cb_send_log_t function pointer to the CGO callback.
+
+ This allows us to set the relevant CGO callback that will allow for sending a log for
+ eventual submission for a specific check instance.
+ */
+ virtual void setSendLogCb(cb_send_log_t) = 0;
+
  //! setCheckMetadataCb member.
  /*!
  \param A cb_set_check_metadata_t function pointer to the CGO callback.

rtloader/include/rtloader_types.h

@@ -123,6 +123,8 @@ typedef void (*cb_get_clustername_t)(char **);
 typedef bool (*cb_tracemalloc_enabled_t)(void);
 // (message, level)
 typedef void (*cb_log_t)(char *, int);
+// (log_line, check_id)
+typedef void (*cb_send_log_t)(char *, char *);
 // (check_id, name, value)
 typedef void (*cb_set_check_metadata_t)(char *, char *, char *);
 // (hostname, source_type_name, list of tags)

rtloader/rtloader/api.cpp

@@ -527,6 +527,11 @@ void set_log_cb(rtloader_t *rtloader, cb_log_t cb)
  AS_TYPE(RtLoader, rtloader)->setLogCb(cb);
 }
 
+void set_send_log_cb(rtloader_t *rtloader, cb_send_log_t cb)
+{
+ AS_TYPE(RtLoader, rtloader)->setSendLogCb(cb);
+}
+
 void set_set_check_metadata_cb(rtloader_t *rtloader, cb_set_check_metadata_t cb)
 {
  AS_TYPE(RtLoader, rtloader)->setSetCheckMetadataCb(cb);

rtloader/test/datadog_agent/datadog_agent.go

@@ -32,6 +32,7 @@ extern void getHostname(char **);
 extern bool getTracemallocEnabled();
 extern void getVersion(char **);
 extern void headers(char **);
+extern void sendLog(char *, char *);
 extern void setCheckMetadata(char*, char*, char*);
 extern void setExternalHostTags(char*, char*, char**);
 extern void writePersistentCache(char*, char*);
@@ -51,6 +52,7 @@ static void initDatadogAgentTests(rtloader_t *rtloader) {
  set_get_version_cb(rtloader, getVersion);
  set_headers_cb(rtloader, headers);
  set_log_cb(rtloader, doLog);
+ set_send_log_cb(rtloader, sendLog);
  set_set_check_metadata_cb(rtloader, setCheckMetadata);
  set_set_external_tags_cb(rtloader, setExternalHostTags);
  set_write_persistent_cache_cb(rtloader, writePersistentCache);
@@ -190,6 +192,17 @@ func doLog(msg *C.char, level C.int) {
  os.WriteFile(tmpfile.Name(), data, 0644)
 }
 
+//export sendLog
+func sendLog(logLine, checkID *C.char) {
+ line := C.GoString(logLine)
+ cid := C.GoString(checkID)
+
+ f, _ := os.OpenFile(tmpfile.Name(), os.O_APPEND|os.O_RDWR|os.O_CREATE, 0666)
+ defer f.Close()
+
+ f.WriteString(strings.Join([]string{line, cid}, ","))
+}
+
 //export setCheckMetadata
 func setCheckMetadata(checkID, name, value *C.char) {
  cid := C.GoString(checkID)

rtloader/test/datadog_agent/datadog_agent_test.go

@@ -179,6 +179,19 @@ func TestLog(t *testing.T) {
  helpers.AssertMemoryUsage(t)
 }
 
+func TestSendLog(t *testing.T) {
+ code := `
+ datadog_agent.send_log("log line", "postgres:test:12345")
+ `
+ out, err := run(code)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if out != "log line,postgres:test:12345" {
+ t.Errorf("Unexpected printed value: '%s'", out)
+ }
+}
+
 func TestSetCheckMetadata(t *testing.T) {
  code := `
  datadog_agent.set_check_metadata("redis:test:12345", "version.raw", "5.0.6")

rtloader/three/three.cpp

@@ -925,6 +925,11 @@ void Three::setLogCb(cb_log_t cb)
  _set_log_cb(cb);
 }
 
+void Three::setSendLogCb(cb_send_log_t cb)
+{
+ _set_send_log_cb(cb);
+}
+
 void Three::setSetCheckMetadataCb(cb_set_check_metadata_t cb)
 {
  _set_set_check_metadata_cb(cb);

rtloader/three/three.h

@@ -104,6 +104,7 @@ class Three : public RtLoader
  void setGetClusternameCb(cb_get_clustername_t);
  void setGetTracemallocEnabledCb(cb_tracemalloc_enabled_t);
  void setLogCb(cb_log_t);
+ void setSendLogCb(cb_send_log_t);
  void setSetCheckMetadataCb(cb_set_check_metadata_t);
  void setSetExternalTagsCb(cb_set_external_tags_t);
  void setWritePersistentCacheCb(cb_write_persistent_cache_t);

rtloader/two/two.cpp

@@ -923,6 +923,11 @@ void Two::setLogCb(cb_log_t cb)
  _set_log_cb(cb);
 }
 
+void Two::setSendLogCb(cb_send_log_t cb)
+{
+ _set_send_log_cb(cb);
+}
+
 void Two::setSetCheckMetadataCb(cb_set_check_metadata_t cb)
 {
  _set_set_check_metadata_cb(cb);

rtloader/two/two.h

@@ -102,6 +102,7 @@ class Two : public RtLoader
  void setGetClusternameCb(cb_get_clustername_t);
  void setGetTracemallocEnabledCb(cb_tracemalloc_enabled_t);
  void setLogCb(cb_log_t);
+ void setSendLogCb(cb_send_log_t);
  void setSetCheckMetadataCb(cb_set_check_metadata_t);
  void setSetExternalTagsCb(cb_set_external_tags_t);
  void setWritePersistentCacheCb(cb_write_persistent_cache_t);