From 8432c1b1282be8bb8b5a9e05c30c664a230776ae Mon Sep 17 00:00:00 2001
From: Chris Berkhout <chris.berkhout@elastic.co>
Date: Thu, 8 Feb 2024 10:08:15 +0100
Subject: [PATCH] [filebeat][threatintel] MISP pagination fixes (#37898)

Update the HTTP JSON input configuration for the Threat Intel module's
misp fileset with pagination fixes that were done earlier in the
Agent-based MISP integration, in these PRs:

- Fix timestamp format sent to API
  https://github.com/elastic/integrations/pull/6482

- Fix duplicate requests for page 1
  https://github.com/elastic/integrations/pull/6495

- Keep the same timestamp for later pages
  https://github.com/elastic/integrations/pull/6649

- Pagination fixes
  https://github.com/elastic/integrations/pull/9073

(cherry picked from commit b7fc69a3ee3211300be9432a622a2ff0d26f45ff)
---
 CHANGELOG.next.asciidoc                       | 17 +++++++++++++
 .../module/threatintel/misp/config/config.yml | 25 ++++++++++++++++---
 2 files changed, 39 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
index f54d7a3fea6..b528272ad29 100644
--- a/CHANGELOG.next.asciidoc
+++ b/CHANGELOG.next.asciidoc
@@ -38,6 +38,23 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Filebeat*
 
+- Fix nil pointer dereference in the httpjson input {pull}37591[37591]
+- [Gcs Input] - Added missing locks for safe concurrency {pull}34914[34914]
+- Fix the ignore_inactive option being ignored in Filebeat's filestream input {pull}34770[34770]
+- Fix TestMultiEventForEOFRetryHandlerInput unit test of CometD input {pull}34903[34903]
+- Add input instance id to request trace filename for httpjson and cel inputs {pull}35024[35024]
+- Fixes "Can only start an input when all related states are finished" error when running under Elastic-Agent {pull}35250[35250] {issue}33653[33653]
+- [system] sync system/auth dataset with system integration 1.29.0. {pull}35581[35581]
+- [GCS Input] - Fixed an issue where bucket_timeout was being applied to the entire bucket poll interval and not individual bucket object read operations. Fixed a map write concurrency issue arising from data races when using a high number of workers. Fixed the flaky tests that were present in the GCS test suit. {pull}35605[35605]
+- Fixed concurrency and flakey tests issue in azure blob storage input. {issue}35983[35983] {pull}36124[36124]
+- Fix panic when sqs input metrics getter is invoked {pull}36101[36101] {issue}36077[36077]
+- Fix handling of Juniper SRX structured data when there is no leading junos element. {issue}36270[36270] {pull}36308[36308]
+- Fix Filebeat Cisco module with missing escape character {issue}36325[36325] {pull}36326[36326]
+- Added a fix for Crowdstrike pipeline handling process arrays {pull}36496[36496]
+- Fix m365_defender cursor value and query building. {pull}37116[37116]
+- Fix TCP/UDP metric queue length parsing base. {pull}37714[37714]
+- Update github.com/lestrrat-go/jwx dependency. {pull}37799[37799]
+- [threatintel] MISP pagination fixes {pull}37898[37898]
 
 *Heartbeat*
 
diff --git a/x-pack/filebeat/module/threatintel/misp/config/config.yml b/x-pack/filebeat/module/threatintel/misp/config/config.yml
index 10fb32c748c..6aa1f5d89fb 100644
--- a/x-pack/filebeat/module/threatintel/misp/config/config.yml
+++ b/x-pack/filebeat/module/threatintel/misp/config/config.yml
@@ -27,8 +27,20 @@ request.transforms:
 {{end}}
 - set:
     target: body.timestamp
-    value: '[[.cursor.timestamp]]'
-    default: '[[ formatDate (now (parseDuration "-{{ .first_interval }}")) "UnixDate" ]]'
+    value: >-
+      [[- if index .cursor "timestamp" -]]
+        [[- .cursor.timestamp -]]
+      [[- else -]]
+        [[- .last_response.url.params.Get "timestamp" -]]
+      [[- end -]]
+    default: '[[ (now (parseDuration "-{{ .first_interval }}")).Unix ]]'
+- set:
+    target: body.order
+    value: timestamp
+- set:
+    # Ignored by MISP, set as a workaround to make it available in response.pagination.
+    target: url.params.timestamp
+    value: '[[.body.timestamp]]'
 
 response.split:
   target: body.response
@@ -46,8 +58,15 @@ response.request_body_on_pagination: true
 response.pagination:
 - set:
     target: body.page
-    value: '[[if (ne (len .last_response.body.response) 0)]][[add .last_response.page 1]][[end]]'
+    # Add 2 because the httpjson page counter is zero-based while the MISP page parameter starts at 1.
+    value: '[[if (ne (len .last_response.body.response) 0)]][[add .last_response.page 2]][[end]]'
     fail_on_template_error: true
+- set:
+    target: body.timestamp
+    value: '[[.last_response.url.params.Get "timestamp"]]'
+- set:
+    target: url.params.timestamp
+    value: '[[.last_response.url.params.Get "timestamp"]]'
 cursor:
   timestamp:
     value: '[[.last_event.Event.timestamp]]'