diff --git a/packages/ti_misp/_dev/deploy/docker/files/config.yml b/packages/ti_misp/_dev/deploy/docker/files/config.yml index a48ec6cf3c8c..482cc08c7d8d 100644 --- a/packages/ti_misp/_dev/deploy/docker/files/config.yml +++ b/packages/ti_misp/_dev/deploy/docker/files/config.yml @@ -9,70 +9,77 @@ rules: - status_code: 200 body: |- { - "response": [{ + "response": [ + { "Event": { - "Attribute": [{ - "Galaxy": [], - "ShadowAttribute": [], - "category": "Payload delivery", - "comment": "filename contect for test event 3", - "deleted": false, - "disable_correlation": false, - "distribution": "5", - "event_id": "3633", - "first_seen": null, - "id": "266263", - "last_seen": null, - "object_id": "0", - "object_relation": null, - "sharing_group_id": "0", - "timestamp": "1621589229", - "to_ids": false, - "type": "filename", - "uuid": "3b322e1a-1dd8-490c-ab96-12e1bc3ee6a3", - "value": "thetestfile.txt" - }], - "EventReport": [], - "Galaxy": [], - "Object": [{ - "Attribute": [{ + "Attribute": [ + { "Galaxy": [], "ShadowAttribute": [], "category": "Payload delivery", - "comment": "", + "comment": "filename content for test event 3", "deleted": false, "disable_correlation": false, "distribution": "5", "event_id": "3633", "first_seen": null, - "id": "266265", + "id": "266263", + "last_seen": null, + "object_id": "0", + "object_relation": null, + "sharing_group_id": "0", + "timestamp": "1621589229", + "to_ids": false, + "type": "filename", + "uuid": "3b322e1a-1dd8-490c-ab96-12e1bc3ee6a3", + "value": "thetestfile.txt" + } + ], + "EventReport": [], + "Galaxy": [], + "Object": [ + { + "Attribute": [ + { + "Galaxy": [], + "ShadowAttribute": [], + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "3633", + "first_seen": null, + "id": "266265", + "last_seen": null, + "object_id": "18207", + "object_relation": "sha256", + "sharing_group_id": "0", + "timestamp": "1621589548", + "to_ids": true, + "type": "sha256", + "uuid": "657c5f2b-9d68-4ff7-a9ad-ab9e6a6c953e", + "value": "f33c27745f2bd87344be790465ef984a972fd539dc83bd4f61d4242c607ef1ee" + } + ], + "ObjectReference": [], + "comment": "File object for event 3", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "3633", + "first_seen": null, + "id": "18207", "last_seen": null, - "object_id": "18207", - "object_relation": "sha256", + "meta-category": "file", + "name": "file", "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "22", "timestamp": "1621589548", - "to_ids": true, - "type": "sha256", - "uuid": "657c5f2b-9d68-4ff7-a9ad-ab9e6a6c953e", - "value": "f33c27745f2bd87344be790465ef984a972fd539dc83bd4f61d4242c607ef1ee" - }], - "ObjectReference": [], - "comment": "File object for event 3", - "deleted": false, - "description": "File object describing a file with meta-information", - "distribution": "5", - "event_id": "3633", - "first_seen": null, - "id": "18207", - "last_seen": null, - "meta-category": "file", - "name": "file", - "sharing_group_id": "0", - "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", - "template_version": "22", - "timestamp": "1621589548", - "uuid": "42a88ad4-6834-46a9-a18b-aff9e078a4ea" - }], + "uuid": "42a88ad4-6834-46a9-a18b-aff9e078a4ea" + } + ], "Org": { "id": "1", "local": true, @@ -85,31 +92,33 @@ rules: "name": "ORGNAME", "uuid": "78acad2d-cc2d-4785-94d6-b428a0070488" }, - "RelatedEvent": [{ - "Event": { - "Org": { - "id": "1", - "name": "ORGNAME", - "uuid": "78acad2d-cc2d-4785-94d6-b428a0070488" - }, - "Orgc": { - "id": "1", - "name": "ORGNAME", - "uuid": "78acad2d-cc2d-4785-94d6-b428a0070488" - }, - "analysis": "0", - "date": "2021-05-21", - "distribution": "1", - "id": "3631", - "info": "Test event 1 just atrributes", - "org_id": "1", - "orgc_id": "1", - "published": false, - "threat_level_id": "1", - "timestamp": "1621588162", - "uuid": "8ca56ae9-3747-4172-93d2-808da1a4eaf3" + "RelatedEvent": [ + { + "Event": { + "Org": { + "id": "1", + "name": "ORGNAME", + "uuid": "78acad2d-cc2d-4785-94d6-b428a0070488" + }, + "Orgc": { + "id": "1", + "name": "ORGNAME", + "uuid": "78acad2d-cc2d-4785-94d6-b428a0070488" + }, + "analysis": "0", + "date": "2021-05-21", + "distribution": "1", + "id": "3631", + "info": "Test event 1 just atrributes", + "org_id": "1", + "orgc_id": "1", + "published": false, + "threat_level_id": "1", + "timestamp": "1621588162", + "uuid": "8ca56ae9-3747-4172-93d2-808da1a4eaf3" + } } - }], + ], "ShadowAttribute": [], "analysis": "0", "attribute_count": "6", @@ -131,30 +140,45 @@ rules: "timestamp": "1621592532", "uuid": "4edb20c7-8175-484d-bdcd-fce6872c1ef3" } - }, + } + ] + } + - path: /events/restSearch + methods: ["POST"] + request_headers: + Authorization: "test" + Content-Type: application/json + request_body: /^{"limit":"10","page":"2","returnFormat":"json","timestamp":"\d+"/ + responses: + - status_code: 200 + body: |- + { + "response": [ { "Event": { - "Attribute": [{ - "Galaxy": [], - "ShadowAttribute": [], - "category": "Network activity", - "comment": "Conext for domain type attribute event 2", - "deleted": false, - "disable_correlation": false, - "distribution": "5", - "event_id": "3632", - "first_seen": null, - "id": "266260", - "last_seen": null, - "object_id": "0", - "object_relation": null, - "sharing_group_id": "0", - "timestamp": "1621588744", - "to_ids": true, - "type": "domain", - "uuid": "a52a1b47-a580-4f33-96ba-939cf9146c9b", - "value": "baddom.madeup.local" - }], + "Attribute": [ + { + "Galaxy": [], + "ShadowAttribute": [], + "category": "Network activity", + "comment": "Conext for domain type attribute event 2", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "3632", + "first_seen": null, + "id": "266260", + "last_seen": null, + "object_id": "0", + "object_relation": null, + "sharing_group_id": "0", + "timestamp": "1621588744", + "to_ids": true, + "type": "domain", + "uuid": "a52a1b47-a580-4f33-96ba-939cf9146c9b", + "value": "baddom.madeup.local" + } + ], "EventReport": [], "Galaxy": [], "Object": [], @@ -170,31 +194,33 @@ rules: "name": "ORGNAME", "uuid": "78acad2d-cc2d-4785-94d6-b428a0070488" }, - "RelatedEvent": [{ - "Event": { - "Org": { - "id": "1", - "name": "ORGNAME", - "uuid": "78acad2d-cc2d-4785-94d6-b428a0070488" - }, - "Orgc": { - "id": "2", - "name": "CIRCL", - "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" - }, - "analysis": "2", - "date": "2018-03-26", - "distribution": "3", - "id": "684", - "info": "OSINT - Forgot About Default Accounts? No Worries, GoScanSSH Didn’t", - "org_id": "1", - "orgc_id": "2", - "published": true, - "threat_level_id": "3", - "timestamp": "1523865236", - "uuid": "5acdb4d0-b534-4713-9612-4a1d950d210f" + "RelatedEvent": [ + { + "Event": { + "Org": { + "id": "1", + "name": "ORGNAME", + "uuid": "78acad2d-cc2d-4785-94d6-b428a0070488" + }, + "Orgc": { + "id": "2", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "analysis": "2", + "date": "2018-03-26", + "distribution": "3", + "id": "684", + "info": "OSINT - Forgot About Default Accounts? No Worries, GoScanSSH Didn’t", + "org_id": "1", + "orgc_id": "2", + "published": true, + "threat_level_id": "3", + "timestamp": "1523865236", + "uuid": "5acdb4d0-b534-4713-9612-4a1d950d210f" + } } - }], + ], "ShadowAttribute": [], "analysis": "0", "attribute_count": "4", @@ -249,55 +275,60 @@ rules: "uuid": "55f6ea5f-fd34-43b8-ac1d-40cb950d210f", "local": false }, - "Attribute": [{ - "id": "12394", - "type": "domain", - "category": "Network activity", - "to_ids": false, - "uuid": "572b4ab3-1af0-4d91-9cd5-07a1c0a8ab16", - "event_id": "22", - "distribution": "5", - "timestamp": "1462454963", - "comment": "", - "sharing_group_id": "0", - "deleted": false, - "disable_correlation": false, - "object_id": "0", - "object_relation": null, - "first_seen": null, - "last_seen": null, - "value": "whatsapp.com", - "Galaxy": [], - "ShadowAttribute": [] - }], + "Attribute": [ + { + "id": "12394", + "type": "domain", + "category": "Network activity", + "to_ids": false, + "uuid": "572b4ab3-1af0-4d91-9cd5-07a1c0a8ab16", + "event_id": "22", + "distribution": "5", + "timestamp": "1462454963", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "0", + "object_relation": null, + "first_seen": null, + "last_seen": null, + "value": "whatsapp.com", + "Galaxy": [], + "ShadowAttribute": [] + } + ], "ShadowAttribute": [], "RelatedEvent": [], "Galaxy": [], "Object": [], "EventReport": [], - "Tag": [{ - "id": "1", - "name": "type:OSINT", - "colour": "#004646", - "exportable": true, - "user_id": "0", - "hide_tag": false, - "numerical_value": null, - "is_galaxy": false, - "is_custom_galaxy": false, - "local": 0 - }, { - "id": "2", - "name": "tlp:green", - "colour": "#339900", - "exportable": true, - "user_id": "0", - "hide_tag": false, - "numerical_value": null, - "is_galaxy": false, - "is_custom_galaxy": false, - "local": 0 - }] + "Tag": [ + { + "id": "1", + "name": "type:OSINT", + "colour": "#004646", + "exportable": true, + "user_id": "0", + "hide_tag": false, + "numerical_value": null, + "is_galaxy": false, + "is_custom_galaxy": false, + "local": 0 + }, + { + "id": "2", + "name": "tlp:green", + "colour": "#339900", + "exportable": true, + "user_id": "0", + "hide_tag": false, + "numerical_value": null, + "is_galaxy": false, + "is_custom_galaxy": false, + "local": 0 + } + ] } } ] @@ -307,7 +338,7 @@ rules: request_headers: Authorization: "test" Content-Type: application/json - request_body: /^{"limit":"10","page":"2","returnFormat":"json","timestamp":"\d+"/ + request_body: /^{"limit":"10","page":"3","returnFormat":"json","timestamp":"\d+"/ responses: - status_code: 200 body: |- diff --git a/packages/ti_misp/changelog.yml b/packages/ti_misp/changelog.yml index aa572144691f..c1b83ef6ae10 100644 --- a/packages/ti_misp/changelog.yml +++ b/packages/ti_misp/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.15.2" + changes: + - description: Prevent duplicate requests for the first page while paginating. + type: bugfix + link: https://github.com/elastic/integrations/pull/6495 - version: "1.15.1" changes: - description: Fix timestamp format sent in query. diff --git a/packages/ti_misp/data_stream/threat/_dev/test/system/test-default-config.yml b/packages/ti_misp/data_stream/threat/_dev/test/system/test-default-config.yml index 9ef767ad9b65..c689b96f1a66 100644 --- a/packages/ti_misp/data_stream/threat/_dev/test/system/test-default-config.yml +++ b/packages/ti_misp/data_stream/threat/_dev/test/system/test-default-config.yml @@ -9,3 +9,5 @@ data_stream: interval: 10m initial_interval: 10m enable_request_tracer: true +assert: + hit_count: 3 diff --git a/packages/ti_misp/data_stream/threat/agent/stream/httpjson.yml.hbs b/packages/ti_misp/data_stream/threat/agent/stream/httpjson.yml.hbs index 51859906b275..33ecc72f4bb9 100644 --- a/packages/ti_misp/data_stream/threat/agent/stream/httpjson.yml.hbs +++ b/packages/ti_misp/data_stream/threat/agent/stream/httpjson.yml.hbs @@ -56,7 +56,8 @@ response.request_body_on_pagination: true response.pagination: - set: target: body.page - value: '[[if (ne (len .last_response.body.response) 0)]][[add .last_response.page 1]][[end]]' + # Add 2 because the httpjson page counter is zero-based while the MISP page parameter starts at 1. + value: '[[if (ne (len .last_response.body.response) 0)]][[add .last_response.page 2]][[end]]' fail_on_template_error: true cursor: timestamp: diff --git a/packages/ti_misp/data_stream/threat_attributes/_dev/test/system/test-default-config.yml b/packages/ti_misp/data_stream/threat_attributes/_dev/test/system/test-default-config.yml index 9ef767ad9b65..d1cb754892fc 100644 --- a/packages/ti_misp/data_stream/threat_attributes/_dev/test/system/test-default-config.yml +++ b/packages/ti_misp/data_stream/threat_attributes/_dev/test/system/test-default-config.yml @@ -9,3 +9,5 @@ data_stream: interval: 10m initial_interval: 10m enable_request_tracer: true +assert: + hit_count: 5 diff --git a/packages/ti_misp/data_stream/threat_attributes/agent/stream/httpjson.yml.hbs b/packages/ti_misp/data_stream/threat_attributes/agent/stream/httpjson.yml.hbs index 4bf8d3728ee5..c0e2a09990e5 100644 --- a/packages/ti_misp/data_stream/threat_attributes/agent/stream/httpjson.yml.hbs +++ b/packages/ti_misp/data_stream/threat_attributes/agent/stream/httpjson.yml.hbs @@ -48,7 +48,8 @@ response.request_body_on_pagination: true response.pagination: - set: target: body.page - value: '[[if (ne (len .last_response.body.response) 0)]][[add .last_response.page 1]][[end]]' + # Add 2 because the httpjson page counter is zero-based while the MISP page parameter starts at 1. + value: '[[if (ne (len .last_response.body.response) 0)]][[add .last_response.page 2]][[end]]' fail_on_template_error: true cursor: timestamp: diff --git a/packages/ti_misp/manifest.yml b/packages/ti_misp/manifest.yml index dfe56468afe7..0a457f13f1a4 100644 --- a/packages/ti_misp/manifest.yml +++ b/packages/ti_misp/manifest.yml @@ -1,6 +1,6 @@ name: ti_misp title: MISP -version: "1.15.1" +version: "1.15.2" release: ga description: Ingest threat intelligence indicators from MISP platform with Elastic Agent. type: integration