From 574ce1ebc6cfeab7b7b6f83566c2f30ea71ad3ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Rebollo=20P=C3=A9rez?= Date: Tue, 16 Jan 2024 18:12:04 +0000 Subject: [PATCH 1/7] feat: include delete package operations in AG Syscollector messages --- .../wazuh_testing/data/syscollector.py | 2 +- .../data/syscollector_parsed_packages.json | 9 ++ .../wazuh_testing/tools/agent_simulator.py | 95 ++++++++++++++++--- 3 files changed, 91 insertions(+), 15 deletions(-) create mode 100644 deps/wazuh_testing/wazuh_testing/data/syscollector_parsed_packages.json diff --git a/deps/wazuh_testing/wazuh_testing/data/syscollector.py b/deps/wazuh_testing/wazuh_testing/data/syscollector.py index f98d5c36b6..3091d849d8 100644 --- a/deps/wazuh_testing/wazuh_testing/data/syscollector.py +++ b/deps/wazuh_testing/wazuh_testing/data/syscollector.py @@ -59,7 +59,7 @@ "format": "", "groups": "editors", "install_time": "", - "item_id": "", + "item_id": "", "location": " ", "multiarch": "null", "name": "", diff --git a/deps/wazuh_testing/wazuh_testing/data/syscollector_parsed_packages.json b/deps/wazuh_testing/wazuh_testing/data/syscollector_parsed_packages.json new file mode 100644 index 0000000000..a41b357172 --- /dev/null +++ b/deps/wazuh_testing/wazuh_testing/data/syscollector_parsed_packages.json @@ -0,0 +1,9 @@ +[ + { + "vendor": "bsdi", + "product": "bsd_os", + "version": "3.1", + "checksum": "66e8140425c4e12c2d35e8267d31deb2853f1f5e", + "item_id": "6bbdd1d1b4337b9cb73b2e6520528b013a9aab0c" + } +] diff --git a/deps/wazuh_testing/wazuh_testing/tools/agent_simulator.py b/deps/wazuh_testing/wazuh_testing/tools/agent_simulator.py index 35874531fa..6c96c12830 100644 --- a/deps/wazuh_testing/wazuh_testing/tools/agent_simulator.py +++ b/deps/wazuh_testing/wazuh_testing/tools/agent_simulator.py @@ -775,15 +775,18 @@ def __init__(self, agent_name, event_types_list, old_format, batch_size, syscoll } self.syscollector_packages_vuln_content = syscollector_packages_vuln_content - self.default_package_data = { - '': 'A low-level cryptographic library', - '': 'x86_64', - '': 'rpm', - '': 'nettle', - '': 'vim', - '': 'Ubuntu Developers ', - '': '2.7.1-9.el7_9' - } + self.packages = [ + { + 'installed': False, + 'description': 'A low-level cryptographic library', + 'architecture': 'x86_64', + 'format': 'rpm', + 'name': 'nettle', + 'source': 'vim', + 'vendor': 'Ubuntu Developers ', + 'version': '2.7.1-9.el7_9' + } + ] self.old_format = old_format self.agent_name = agent_name @@ -791,6 +794,64 @@ def __init__(self, agent_name, event_types_list, old_format, batch_size, syscoll self.syscollector_tag = 'syscollector' self.syscollector_mq = 'd' self.current_id = 1 + self.default_packages_vuln_content = os.path.join(_data_path, 'syscollector_parsed_packages.json') + + self.package_index = 0 + + if self.syscollector_packages_vuln_content: + self.packages = self.init_package_data(self.syscollector_packages_vuln_content) + else: + self.packages = self.init_package_data(self.default_packages_vuln_content) + + def parse_package_template(self, message, package_data): + template_package_fields = { + '': package_data['description'], + '': package_data['architecture'], + '': package_data['format'], + '': package_data['product'], + '': package_data['source'], + '': package_data['vendor'], + '': package_data['version'], + '': package_data['item_id'] + } + + for package_key, package_value in template_package_fields.items(): + message = message.replace(package_key, package_value) + + return message + + def get_package_data(self): + operation = 'INSERTED' if not self.packages[self.package_index]['installed'] else 'DELETED' + print(f"Current package operation: {operation}") + + package_data = self.packages[self.package_index] + + self.package_index = (self.package_index + 1) % len(self.packages) + + return package_data, operation + + def init_package_data(self, packages_file): + """Get package data from a json file. + Returns: + dict: Package data. + """ + with open(os.path.join(_data_path, packages_file), 'r') as fp: + package_data = json.load(fp) + + for package in package_data: + package['installed'] = False + if 'description' not in package: + package['description'] = '' + if 'architecture' not in package: + package['architecture'] = '' + if 'format' not in package: + package['format'] = '' + if 'source' not in package: + package['source'] = '' + if 'item_id' not in package: + package['item_id'] = get_random_string(10) + + return package_data def get_event_template_legacy(self, message_type): """Get syscollector legacy message of the specified type. @@ -831,6 +892,7 @@ def get_event_template(self, message_type): message_operation = operation message_data = {} + package_data = {} if message_type == 'network': message_data = syscollector.SYSCOLLECTOR_NETWORK_IFACE_DELTA_EVENT_TEMPLATE @@ -847,12 +909,22 @@ def get_event_template(self, message_type): elif message_type == 'hotfix': message_data = syscollector.SYSCOLLECTOR_HOTFIX_DELTA_DATA_TEMPLATE + if message_type == 'packages': + print("PACKAGEEEEEEEEEEEES") + package_data, operation = self.get_package_data() + message_operation = operation + message = '{"type": "%s", "data": %s, "operation": "%s"}' % ( message_event_type, re.sub(r'\s', '', json.dumps(message_data)), message_operation ) + if message_type == 'packages': + message = self.parse_package_template(message, package_data) + print(f"Current package index: {self.package_index}") + self.packages[self.package_index]['installed'] = not self.packages[self.package_index]['installed'] + return message def format_event_template(self, template, message_type=None): @@ -869,11 +941,6 @@ def format_event_template(self, template, message_type=None): for variable, value in generics_fields_to_replace: message = message.replace(variable, value) - if message_type == 'packages': - if not self.syscollector_packages_vuln_content: - for package_key, package_value in self.default_package_data.items(): - message = message.replace(package_key, package_value) - final_mesage = f"{self.syscollector_mq}:{self.syscollector_tag}:{message}" return final_mesage From fb82a3b3e5ab4685c7947b1f314b7a736a963d66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Rebollo=20P=C3=A9rez?= Date: Tue, 16 Jan 2024 18:22:55 +0000 Subject: [PATCH 2/7] fix: remove debug messages --- .../wazuh_testing/data/syscollector_parsed_packages.json | 2 -- deps/wazuh_testing/wazuh_testing/tools/agent_simulator.py | 1 - 2 files changed, 3 deletions(-) diff --git a/deps/wazuh_testing/wazuh_testing/data/syscollector_parsed_packages.json b/deps/wazuh_testing/wazuh_testing/data/syscollector_parsed_packages.json index a41b357172..f744a353e9 100644 --- a/deps/wazuh_testing/wazuh_testing/data/syscollector_parsed_packages.json +++ b/deps/wazuh_testing/wazuh_testing/data/syscollector_parsed_packages.json @@ -3,7 +3,5 @@ "vendor": "bsdi", "product": "bsd_os", "version": "3.1", - "checksum": "66e8140425c4e12c2d35e8267d31deb2853f1f5e", - "item_id": "6bbdd1d1b4337b9cb73b2e6520528b013a9aab0c" } ] diff --git a/deps/wazuh_testing/wazuh_testing/tools/agent_simulator.py b/deps/wazuh_testing/wazuh_testing/tools/agent_simulator.py index 6c96c12830..5d6bd1699c 100644 --- a/deps/wazuh_testing/wazuh_testing/tools/agent_simulator.py +++ b/deps/wazuh_testing/wazuh_testing/tools/agent_simulator.py @@ -910,7 +910,6 @@ def get_event_template(self, message_type): message_data = syscollector.SYSCOLLECTOR_HOTFIX_DELTA_DATA_TEMPLATE if message_type == 'packages': - print("PACKAGEEEEEEEEEEEES") package_data, operation = self.get_package_data() message_operation = operation From 53dc3aaa59fc62826debd291a6b3f650a6577c96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Rebollo=20P=C3=A9rez?= Date: Wed, 17 Jan 2024 10:02:12 +0000 Subject: [PATCH 3/7] feat: include content snapshot parsing script --- .../scripts/parse_packages_content.py | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 deps/wazuh_testing/wazuh_testing/scripts/parse_packages_content.py diff --git a/deps/wazuh_testing/wazuh_testing/scripts/parse_packages_content.py b/deps/wazuh_testing/wazuh_testing/scripts/parse_packages_content.py new file mode 100644 index 0000000000..d051a96ab3 --- /dev/null +++ b/deps/wazuh_testing/wazuh_testing/scripts/parse_packages_content.py @@ -0,0 +1,76 @@ +import argparse +import json +import logging + + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger("parse_packages_content.py") + +config = None + + +def parse_packages_content(output_file, packages_file, n_packages): + list_packages = [] + + with open(packages_file) as f: + for line in f: + if len(list_packages) >= n_packages: + break + + if line.strip(): + config = json.loads(line) + + if 'payload' in config and 'containers' in config['payload'] \ + and 'cna' in config['payload']['containers']: + if 'affected' in config['payload']['containers']['cna']: + for affected in config['payload']['containers']['cna']['affected']: + vendor = affected['vendor'] + product = affected['product'] + + for affected_version in affected['versions']: + status, version = affected_version['status'], affected_version['version'] + + if status == 'affected': + list_packages.append({ + 'vendor': vendor, + 'product': product, + 'version': version + }) + else: + logger.debug("No affected found in package: %s", config) + else: + logger.warning("No payload found for package: %s", config['id']) + logger.debug("Package: %s", config) + + with open(output_file, 'w') as f: + json.dump(list_packages, f, indent=4) + + +def main(): + arg_parser = argparse.ArgumentParser() + + arg_parser.add_argument('-p', '--packages', metavar='', type=str, required=True, + help='Packages file', dest='packages_file') + + arg_parser.add_argument('-n', '--n_packages', metavar='', type=int, required=True, + help='Number of packages to parse', dest='n_packages') + + arg_parser.add_argument('-d', '--debug', action='store_true', help='Enable debug mode', dest='debug') + + arg_parser.add_argument('-o', '--output', metavar='', type=str, required=True, + help='Output file', dest='output_file') + + args = arg_parser.parse_args() + + if args.debug: + logger.setLevel(logging.DEBUG) + else: + logger.setLevel(logging.INFO) + + logging.info("Parsing packages content...") + parse_packages_content(args.output_file, args.packages_file, args.n_packages) + logging.info("Packages parsed successfully") + + +if __name__ == '__main__': + main() From 542c492436a2e71f465729ad8cefdd72bf911df9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Rebollo=20P=C3=A9rez?= Date: Wed, 17 Jan 2024 10:03:01 +0000 Subject: [PATCH 4/7] feat: increase default packages to 10 --- .../data/syscollector_parsed_packages.json | 112 +++++++++++++++++- 1 file changed, 111 insertions(+), 1 deletion(-) diff --git a/deps/wazuh_testing/wazuh_testing/data/syscollector_parsed_packages.json b/deps/wazuh_testing/wazuh_testing/data/syscollector_parsed_packages.json index f744a353e9..edb0840c08 100644 --- a/deps/wazuh_testing/wazuh_testing/data/syscollector_parsed_packages.json +++ b/deps/wazuh_testing/wazuh_testing/data/syscollector_parsed_packages.json @@ -2,6 +2,116 @@ { "vendor": "bsdi", "product": "bsd_os", - "version": "3.1", + "version": "3.1" + }, + { + "vendor": "freebsd", + "product": "freebsd", + "version": "1.0" + }, + { + "vendor": "freebsd", + "product": "freebsd", + "version": "1.1" + }, + { + "vendor": "freebsd", + "product": "freebsd", + "version": "1.1.5.1" + }, + { + "vendor": "freebsd", + "product": "freebsd", + "version": "1.2" + }, + { + "vendor": "freebsd", + "product": "freebsd", + "version": "2.0" + }, + { + "vendor": "freebsd", + "product": "freebsd", + "version": "2.0.1" + }, + { + "vendor": "freebsd", + "product": "freebsd", + "version": "2.0.5" + }, + { + "vendor": "freebsd", + "product": "freebsd", + "version": "2.1.5" + }, + { + "vendor": "freebsd", + "product": "freebsd", + "version": "2.1.6" + }, + { + "vendor": "freebsd", + "product": "freebsd", + "version": "2.1.6.1" + }, + { + "vendor": "freebsd", + "product": "freebsd", + "version": "2.1.7" + }, + { + "vendor": "freebsd", + "product": "freebsd", + "version": "2.1.7.1" + }, + { + "vendor": "freebsd", + "product": "freebsd", + "version": "2.2" + }, + { + "vendor": "freebsd", + "product": "freebsd", + "version": "2.2.2" + }, + { + "vendor": "freebsd", + "product": "freebsd", + "version": "2.2.3" + }, + { + "vendor": "freebsd", + "product": "freebsd", + "version": "2.2.4" + }, + { + "vendor": "freebsd", + "product": "freebsd", + "version": "2.2.5" + }, + { + "vendor": "freebsd", + "product": "freebsd", + "version": "2.2.6" + }, + { + "vendor": "freebsd", + "product": "freebsd", + "version": "2.2.8" + }, + { + "vendor": "freebsd", + "product": "freebsd", + "version": "3.0" + }, + { + "vendor": "openbsd", + "product": "openbsd", + "version": "2.3" + }, + { + "vendor": "openbsd", + "product": "openbsd", + "version": "2.4" } ] From be408be25de8117f08d5f1ecf840480d9d271000 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Rebollo=20P=C3=A9rez?= Date: Wed, 17 Jan 2024 10:10:53 +0000 Subject: [PATCH 5/7] style: remove nonused default package list --- .../scripts/parse_packages_content.py | 8 ++-- .../wazuh_testing/tools/agent_simulator.py | 37 +++++++++---------- 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/deps/wazuh_testing/wazuh_testing/scripts/parse_packages_content.py b/deps/wazuh_testing/wazuh_testing/scripts/parse_packages_content.py index d051a96ab3..907d012fa4 100644 --- a/deps/wazuh_testing/wazuh_testing/scripts/parse_packages_content.py +++ b/deps/wazuh_testing/wazuh_testing/scripts/parse_packages_content.py @@ -6,11 +6,10 @@ logging.basicConfig(level=logging.INFO) logger = logging.getLogger("parse_packages_content.py") -config = None - def parse_packages_content(output_file, packages_file, n_packages): list_packages = [] + config = None with open(packages_file) as f: for line in f: @@ -55,11 +54,12 @@ def main(): arg_parser.add_argument('-n', '--n_packages', metavar='', type=int, required=True, help='Number of packages to parse', dest='n_packages') - arg_parser.add_argument('-d', '--debug', action='store_true', help='Enable debug mode', dest='debug') - arg_parser.add_argument('-o', '--output', metavar='', type=str, required=True, help='Output file', dest='output_file') + arg_parser.add_argument('-d', '--debug', action='store_true', help='Enable debug mode', dest='debug') + + args = arg_parser.parse_args() if args.debug: diff --git a/deps/wazuh_testing/wazuh_testing/tools/agent_simulator.py b/deps/wazuh_testing/wazuh_testing/tools/agent_simulator.py index 5d6bd1699c..1b2dfc84c3 100644 --- a/deps/wazuh_testing/wazuh_testing/tools/agent_simulator.py +++ b/deps/wazuh_testing/wazuh_testing/tools/agent_simulator.py @@ -775,18 +775,7 @@ def __init__(self, agent_name, event_types_list, old_format, batch_size, syscoll } self.syscollector_packages_vuln_content = syscollector_packages_vuln_content - self.packages = [ - { - 'installed': False, - 'description': 'A low-level cryptographic library', - 'architecture': 'x86_64', - 'format': 'rpm', - 'name': 'nettle', - 'source': 'vim', - 'vendor': 'Ubuntu Developers ', - 'version': '2.7.1-9.el7_9' - } - ] + self.packages = [] self.old_format = old_format self.agent_name = agent_name @@ -794,16 +783,23 @@ def __init__(self, agent_name, event_types_list, old_format, batch_size, syscoll self.syscollector_tag = 'syscollector' self.syscollector_mq = 'd' self.current_id = 1 - self.default_packages_vuln_content = os.path.join(_data_path, 'syscollector_parsed_packages.json') + self.default_packages_vuln_content = os.path.join(_data_path, 'syscollector_parsed_packages.json') self.package_index = 0 if self.syscollector_packages_vuln_content: - self.packages = self.init_package_data(self.syscollector_packages_vuln_content) + self.packages = self.init_package_list(self.syscollector_packages_vuln_content) else: - self.packages = self.init_package_data(self.default_packages_vuln_content) + self.packages = self.init_package_list(self.default_packages_vuln_content) def parse_package_template(self, message, package_data): + """Parse package template with package data. + Args: + message (str): Syscollector event message. + package_data (dict): Package data. + Returns: + str: Parsed syscollector event message. + """ template_package_fields = { '': package_data['description'], '': package_data['architecture'], @@ -821,16 +817,20 @@ def parse_package_template(self, message, package_data): return message def get_package_data(self): + """Get package data. + Returns: + dict: Package data. + str: Operation (INSERTED or DELETED). + """ + operation = 'INSERTED' if not self.packages[self.package_index]['installed'] else 'DELETED' - print(f"Current package operation: {operation}") package_data = self.packages[self.package_index] - self.package_index = (self.package_index + 1) % len(self.packages) return package_data, operation - def init_package_data(self, packages_file): + def init_package_list(self, packages_file): """Get package data from a json file. Returns: dict: Package data. @@ -921,7 +921,6 @@ def get_event_template(self, message_type): if message_type == 'packages': message = self.parse_package_template(message, package_data) - print(f"Current package index: {self.package_index}") self.packages[self.package_index]['installed'] = not self.packages[self.package_index]['installed'] return message From b1a3c22f30c9b88a201edeed4c6b3d32723f1df4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Rebollo=20P=C3=A9rez?= Date: Wed, 17 Jan 2024 10:12:54 +0000 Subject: [PATCH 6/7] refac: remove intermediate var --- deps/wazuh_testing/wazuh_testing/tools/agent_simulator.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/deps/wazuh_testing/wazuh_testing/tools/agent_simulator.py b/deps/wazuh_testing/wazuh_testing/tools/agent_simulator.py index 1b2dfc84c3..490a18e652 100644 --- a/deps/wazuh_testing/wazuh_testing/tools/agent_simulator.py +++ b/deps/wazuh_testing/wazuh_testing/tools/agent_simulator.py @@ -910,8 +910,7 @@ def get_event_template(self, message_type): message_data = syscollector.SYSCOLLECTOR_HOTFIX_DELTA_DATA_TEMPLATE if message_type == 'packages': - package_data, operation = self.get_package_data() - message_operation = operation + package_data, message_operation = self.get_package_data() message = '{"type": "%s", "data": %s, "operation": "%s"}' % ( message_event_type, From c79b414fa29431b02df2d20cc81ed593937bab7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Rebollo=20P=C3=A9rez?= Date: Wed, 17 Jan 2024 11:25:01 +0000 Subject: [PATCH 7/7] fix: unmatched package index --- deps/wazuh_testing/setup.py | 1 + deps/wazuh_testing/wazuh_testing/tools/agent_simulator.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/deps/wazuh_testing/setup.py b/deps/wazuh_testing/setup.py index f2b2c21098..8ef5064d1b 100644 --- a/deps/wazuh_testing/setup.py +++ b/deps/wazuh_testing/setup.py @@ -30,6 +30,7 @@ 'qa_ctl/provisioning/wazuh_deployment/templates/preloaded_vars.conf.j2', 'data/qactl_conf_validator_schema.json', 'data/all_disabled_ossec.conf', + 'data/syscollector_parsed_packages.json', 'tools/migration_tool/delta_schema.json', 'tools/migration_tool/CVE_JSON_5.0_bundled.json' ] diff --git a/deps/wazuh_testing/wazuh_testing/tools/agent_simulator.py b/deps/wazuh_testing/wazuh_testing/tools/agent_simulator.py index 490a18e652..c1384ca57d 100644 --- a/deps/wazuh_testing/wazuh_testing/tools/agent_simulator.py +++ b/deps/wazuh_testing/wazuh_testing/tools/agent_simulator.py @@ -822,10 +822,11 @@ def get_package_data(self): dict: Package data. str: Operation (INSERTED or DELETED). """ - - operation = 'INSERTED' if not self.packages[self.package_index]['installed'] else 'DELETED' + operation = 'DELETED' if self.packages[self.package_index]['installed'] else 'INSERTED' package_data = self.packages[self.package_index] + + self.packages[self.package_index]['installed'] = not self.packages[self.package_index]['installed'] self.package_index = (self.package_index + 1) % len(self.packages) return package_data, operation @@ -920,7 +921,6 @@ def get_event_template(self, message_type): if message_type == 'packages': message = self.parse_package_template(message, package_data) - self.packages[self.package_index]['installed'] = not self.packages[self.package_index]['installed'] return message