From b843cda62742bf66fbb26accf1b7baf882791a58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Rebollo=20P=C3=A9rez?= Date: Thu, 15 Feb 2024 10:45:40 +0000 Subject: [PATCH 01/10] feat: gather alerts by agent through Indexer request --- .../wazuh_testing/end_to_end/indexer_api.py | 31 +++++++++++++------ .../end_to_end/vulnerability_detector.py | 29 +++++++++++++++++ .../test_vulnerability_detector.py | 24 +++++++++----- 3 files changed, 66 insertions(+), 18 deletions(-) diff --git a/deps/wazuh_testing/wazuh_testing/end_to_end/indexer_api.py b/deps/wazuh_testing/wazuh_testing/end_to_end/indexer_api.py index 441bbacc38..12e5d45d95 100644 --- a/deps/wazuh_testing/wazuh_testing/end_to_end/indexer_api.py +++ b/deps/wazuh_testing/wazuh_testing/end_to_end/indexer_api.py @@ -22,7 +22,7 @@ def get_indexer_values(host_manager: HostManager, credentials: dict = {'user': 'admin', 'password': 'changeme'}, - index: str = 'wazuh-alerts*', greater_than_timestamp=None) -> Dict: + index: str = 'wazuh-alerts*', greater_than_timestamp=None, agent: str = '') -> Dict: """ Get values from the Wazuh Indexer API. @@ -49,26 +49,37 @@ def get_indexer_values(host_manager: HostManager, credentials: dict = {'user': ' } } - if greater_than_timestamp: + if greater_than_timestamp and agent: + query = { + "bool": { + "must": [ + {"range": {"@timestamp": {"gte": f"{greater_than_timestamp}"}}}, + {"match": {"agent.id": f"{agent}"}} + ] + } + } + + data['query'] = query + elif greater_than_timestamp: query = { "bool": { "must": [ - {"match_all": {}}, {"range": {"@timestamp": {"gte": f"{greater_than_timestamp}"}}} ] } } - sort = [ - { - "@timestamp": { - "order": "desc" + data['query'] = query + elif agent: + query = { + "bool": { + "must": [ + {"match": {"agent.name": f"{agent}"}} + ] } - } - ] + } data['query'] = query - data['sort'] = sort param = { 'pretty': 'true', diff --git a/deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py b/deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py index af982e1021..51a314c1f7 100644 --- a/deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py +++ b/deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py @@ -131,6 +131,35 @@ def get_alerts_by_agent(alerts, regex) -> Dict: return alerts_vuln_by_agent +def parse_vulnerability_detector_alerts(alerts) -> Dict: + """ + Get specific alerts by agent. + + Args: + alerts (list): List of alerts. + regex (str): Regular expression to match the alerts. + + Returns: + dict: Dictionary containing the alerts by agent. + """ + vulnerability_detector_alerts = {} + vulnerability_detector_alerts['affected'] = [] + vulnerability_detector_alerts['mitigated'] = [] + + regex_affected = 'CVE.*? affects.*"?' + rege_mitigated = "The .* that affected .* was solved due to a package removal" + + # Parse affected vuln alerts + for alert in alerts: + print(alert) + if re.match(regex_affected, alert['_source']['rule']['description']): + vulnerability_detector_alerts['affected'].append(alert) + elif re.match(rege_mitigated, alert['_source']['rule']['description']): + vulnerability_detector_alerts['mitigated'].append(alert) + + return vulnerability_detector_alerts + + def get_indexed_vulnerabilities_by_agent(indexed_vulnerabilities) -> Dict: """Get indexed vulnerabilities by agent. diff --git a/tests/end_to_end/test_vulnerability_detector/test_vulnerability_detector.py b/tests/end_to_end/test_vulnerability_detector/test_vulnerability_detector.py index 00df98e1fb..e47ac951db 100644 --- a/tests/end_to_end/test_vulnerability_detector/test_vulnerability_detector.py +++ b/tests/end_to_end/test_vulnerability_detector/test_vulnerability_detector.py @@ -47,6 +47,7 @@ import time import ast import datetime +from typing import Generator from wazuh_testing.end_to_end.configuration import backup_configurations, restore_configuration, \ configure_environment, save_indexer_credentials_into_keystore @@ -58,7 +59,8 @@ from wazuh_testing.tools.configuration import load_configuration_template from wazuh_testing.tools.system import HostManager from wazuh_testing.end_to_end.remote_operations_handler import launch_parallel_operations -from wazuh_testing.end_to_end.vulnerability_detector import get_alerts_by_agent, get_indexed_vulnerabilities_by_agent +from wazuh_testing.end_to_end.vulnerability_detector import get_alerts_by_agent, get_indexed_vulnerabilities_by_agent, \ + parse_vulnerability_detector_alerts from wazuh_testing.modules.syscollector import TIMEOUT_SYSCOLLECTOR_SCAN @@ -362,17 +364,22 @@ def test_syscollector_first_scan_alerts(self, request, host_manager, setup_vulne time.sleep(TIMEOUT_PER_AGENT_VULNERABILITY_SCAN) logger.critical("Check agent's vulnerabilities") - alerts_first_scan = get_indexer_values(host_manager, - greater_than_timestamp=setup_vulnerability_tests)['hits']['hits'] - vuln_alerts_by_agent_first_scan = get_alerts_by_agent(alerts_first_scan, 'CVE.*? affects.*"?') - test_result['evidences']['vulnerabilities_alerts_first_scan'] = vuln_alerts_by_agent_first_scan + vuln_by_agent_alerts = {} + for agent in agents_to_check: + agent_all_alerts = get_indexer_values(host_manager, + greater_than_timestamp=setup_vulnerability_tests, agent=agent)['hits']['hits'] + + # Only is expected alert of affected vulnerabilities + vuln_by_agent_alerts[agent] = parse_vulnerability_detector_alerts(agent_all_alerts)['affected'] + + test_result['evidences']['vulnerabilities_alerts_first_scan'] = vuln_by_agent_alerts # Check that it has been triggered vulnerability detector alerts logger.critical("Checking that all agents has been scanned") for agent in agents_to_check: - if agent not in list(vuln_alerts_by_agent_first_scan.keys()) or \ - len(vuln_alerts_by_agent_first_scan[agent]) == 0: + if agent not in list(vuln_by_agent_alerts.keys()) or \ + len(vuln_by_agent_alerts[agent]) == 0: logger.critical(f"Agent {agent} has not been scanned. Continuing with remaining agents") test_result['checks']['all_successfull'] = False test_result['evidences']['agents_not_detected_alerts'].append(agent) @@ -380,7 +387,7 @@ def test_syscollector_first_scan_alerts(self, request, host_manager, setup_vulne results[test_name] = test_result # Store full alert list in global results. It is needed for the next test - results['vulnerabilities_alerts_first_scan'] = vuln_alerts_by_agent_first_scan + results['vulnerabilities_alerts_first_scan'] = vuln_by_agent_alerts if not test_result['checks']['all_successfull']: pytest.fail(f"Some agents has not been scanned: {test_result['evidences']['agents_not_detected_alerts']}." @@ -444,6 +451,7 @@ def test_syscollector_first_scan_index(self, request, host_manager, setup_vulner index_state_first_scan = get_indexer_values(host_manager, index='wazuh-states-vulnerabilities', greater_than_timestamp=setup_vulnerability_tests) index_vulnerabilities_by_agent_first_scan = get_indexed_vulnerabilities_by_agent(index_state_first_scan) + test_result['evidences']['vulnerabilities_index_first_scan'] = index_vulnerabilities_by_agent_first_scan logger.critical("Checking that all agents has been scanned and generated vulnerabilities in the index") From 1db8345f84e70d899ebb8f36a52e73d9730f0857 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Rebollo=20P=C3=A9rez?= Date: Wed, 21 Feb 2024 10:30:38 +0000 Subject: [PATCH 02/10] fix: error in indexer agent filter --- deps/wazuh_testing/wazuh_testing/end_to_end/indexer_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/wazuh_testing/wazuh_testing/end_to_end/indexer_api.py b/deps/wazuh_testing/wazuh_testing/end_to_end/indexer_api.py index 12e5d45d95..5e7705962c 100644 --- a/deps/wazuh_testing/wazuh_testing/end_to_end/indexer_api.py +++ b/deps/wazuh_testing/wazuh_testing/end_to_end/indexer_api.py @@ -54,7 +54,7 @@ def get_indexer_values(host_manager: HostManager, credentials: dict = {'user': ' "bool": { "must": [ {"range": {"@timestamp": {"gte": f"{greater_than_timestamp}"}}}, - {"match": {"agent.id": f"{agent}"}} + {"match": {"agent.name": f"{agent}"}} ] } } From c45a604ce1ba89ad040c1df09c604a25de55792f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Rebollo=20P=C3=A9rez?= Date: Wed, 21 Feb 2024 10:31:00 +0000 Subject: [PATCH 03/10] fix: adapt E2E Vuln tests to new indexer schema --- .../test_vulnerability_detector.py | 69 ++++++++++++------- 1 file changed, 43 insertions(+), 26 deletions(-) diff --git a/tests/end_to_end/test_vulnerability_detector/test_vulnerability_detector.py b/tests/end_to_end/test_vulnerability_detector/test_vulnerability_detector.py index e47ac951db..9d5c5e18a3 100644 --- a/tests/end_to_end/test_vulnerability_detector/test_vulnerability_detector.py +++ b/tests/end_to_end/test_vulnerability_detector/test_vulnerability_detector.py @@ -75,7 +75,7 @@ 'agent': os.path.join(configurations_dir, 'agent.yaml') } vulnerability_detector_logs_dir = os.path.join(current_dir, "logs") -TIMEOUT_PER_AGENT_VULNERABILITY_SCAN = 60 +TIMEOUT_PER_AGENT_VULNERABILITY_SCAN = 120 def load_vulnerability_detector_configurations(host_manager): @@ -197,9 +197,9 @@ def check_vuln_state_consistency(vulnerabilities_alerts, vulnerabilities_states) for vulnerabilities_state in list(vulnerabilities_states.values())[0]: state_agent = vulnerabilities_state['_source']['agent']['name'] - state_cve = vulnerabilities_state['_source']['vulnerability']['id'] - state_package_name = vulnerabilities_state['_source']['package']['name'] - state_package_version = vulnerabilities_state['_source']['package']['version'] + state_cve = vulnerabilities_state['_source']['data']['vulnerability']['cve'] + state_package_name = vulnerabilities_state['_source']['data']['vulnerability']['package']['name'] + state_package_version = vulnerabilities_state['_source']['data']['vulnerability']['package']['version'] indices_vulnerabilities.append({ 'cve': state_cve, @@ -448,16 +448,21 @@ def test_syscollector_first_scan_index(self, request, host_manager, setup_vulner # Check vulnerabilities in the index logger.critical("Checking vulnerabilities in the index") - index_state_first_scan = get_indexer_values(host_manager, index='wazuh-states-vulnerabilities', - greater_than_timestamp=setup_vulnerability_tests) - index_vulnerabilities_by_agent_first_scan = get_indexed_vulnerabilities_by_agent(index_state_first_scan) + vuln_by_agent_index = {} + for agent in agents_to_check: + agent_all_alerts = get_indexer_values(host_manager, + greater_than_timestamp=setup_vulnerability_tests, + agent=agent)['hits']['hits'] + + # Only is expected alert of affected vulnerabilities + vuln_by_agent_index[agent] = parse_vulnerability_detector_alerts(agent_all_alerts)['affected'] - test_result['evidences']['vulnerabilities_index_first_scan'] = index_vulnerabilities_by_agent_first_scan + test_result['evidences']['vulnerabilities_index_first_scan'] = vuln_by_agent_index logger.critical("Checking that all agents has been scanned and generated vulnerabilities in the index") for agent in agents_to_check: - if agent not in list(index_vulnerabilities_by_agent_first_scan.keys()) or \ - len(index_vulnerabilities_by_agent_first_scan[agent]) == 0: + if agent not in list(vuln_by_agent_index.keys()) or \ + len(vuln_by_agent_index[agent]) == 0: logger.critical(f"Agent {agent} has not been scanned. Continuing with remaining agents") test_result['checks']['all_successfull'] = False test_result['evidences']['agents_not_detected_index_vulnerabilities'].append(agent) @@ -465,7 +470,7 @@ def test_syscollector_first_scan_index(self, request, host_manager, setup_vulner results[test_name] = test_result # Store full alert index list in global results. It is needed for the next test - results['vulnerabilities_index_first_scan'] = index_vulnerabilities_by_agent_first_scan + results['vulnerabilities_index_first_scan'] = vuln_by_agent_index if not test_result['checks']['all_successfull']: pytest.fail("Some agents has not been scanned and updated states index:" @@ -674,24 +679,30 @@ def tests_syscollector_first_second_scan_consistency_alerts(self, request, host_ time.sleep(TIMEOUT_PER_AGENT_VULNERABILITY_SCAN) logger.critical("Checking vulnerabilities in the second scan") - alerts_second_scan = get_indexer_values(host_manager, - greater_than_timestamp=setup_vulnerability_tests)['hits']['hits'] - vuln_alerts_by_agent_second_scan = get_alerts_by_agent(alerts_second_scan, 'CVE.*? affects.*"?') + + vuln_by_agent_alert_second_scan = {} + for agent in agents_to_check: + agent_all_alerts = get_indexer_values(host_manager, + greater_than_timestamp=setup_vulnerability_tests, + agent=agent)['hits']['hits'] + # Only is expected alert of affected vulnerabilities + vuln_by_agent_alert_second_scan[agent] = parse_vulnerability_detector_alerts(agent_all_alerts)['affected'] + # Store full alert list in global results. It is needed for the next test - results['vulnerabilities_alerts_second_scan'] = vuln_alerts_by_agent_second_scan - test_result['evidences']['vulnerabilities_alerts_second_scan'] = vuln_alerts_by_agent_second_scan + results['vulnerabilities_alerts_second_scan'] = vuln_by_agent_alert_second_scan + test_result['evidences']['vulnerabilities_alerts_second_scan'] = vuln_by_agent_alert_second_scan alert_present_in_first_scan_not_in_second_scan = [] alert_present_in_second_scan_not_in_first_scan = [] logger.critical("Checking that all agents has been scanned") # Check if the number of agents for each scan is the same - if len(vuln_alerts_by_agent_second_scan.keys()) != len(results['vulnerabilities_alerts_first_scan'].keys()): + if len(vuln_by_agent_alert_second_scan.keys()) != len(results['vulnerabilities_alerts_first_scan'].keys()): test_result['checks']['all_successfull'] = False logging.critical(f"Agents with vulnerabilities changed between scans: " f"First scan: {list(results['vulnerabilities_alerts_first_scan'].keys())}" - f"Second scan: {list(vuln_alerts_by_agent_second_scan.keys())}") + f"Second scan: {list(vuln_by_agent_alert_second_scan.keys())}") test_result['evidences']['agents_different_between_scans'] = \ list(set(list(results['vulnerabilities_alerts_first_scan'].keys())) ^ @@ -699,13 +710,14 @@ def tests_syscollector_first_second_scan_consistency_alerts(self, request, host_ logger.critical("Checking the number of vulnerabilities for each agent") for agent in agents_to_check: - if agent in list(results['vulnerabilities_alerts_first_scan'].keys()): - for alert in list(vuln_alerts_by_agent_second_scan[agent][0]): + if agent in list(results['vulnerabilities_alerts_first_scan'].keys()) and \ + len(results['vulnerabilities_alerts_first_scan'][agent]): + for alert in list(vuln_by_agent_alert_second_scan[agent][0]): if alert not in results['vulnerabilities_alerts_first_scan'][agent][0]: alert_present_in_second_scan_not_in_first_scan.append(alert) for alert in list(results['vulnerabilities_alerts_first_scan'][agent][0]): - if alert not in vuln_alerts_by_agent_second_scan[agent][0]: + if alert not in vuln_by_agent_alert_second_scan[agent][0]: alert_present_in_first_scan_not_in_second_scan.append(alert) logger.critical("Checking that all agents has been scanned") @@ -769,12 +781,17 @@ def tests_syscollector_first_second_scan_consistency_index(self, request, host_m } logger.critical("Checking vulnerabilities in the second scan") - index_state_second_scan = get_indexer_values(host_manager, index='wazuh-states-vulnerabilities', - greater_than_timestamp=setup_vulnerability_tests) - index_vulnerabilities_by_agent_second_scan = get_indexed_vulnerabilities_by_agent(index_state_second_scan) - results['vulnerabilities_index_second_scan'] = index_vulnerabilities_by_agent_second_scan - test_result['evidences']['vulnerabilities_index_second_scan'] = index_vulnerabilities_by_agent_second_scan + vuln_by_agent_index_second_scan = {} + for agent in host_manager.get_group_hosts('agent'): + agent_all_alerts = get_indexer_values(host_manager, + greater_than_timestamp=setup_vulnerability_tests, + agent=agent)['hits']['hits'] + # Only is expected alert of affected vulnerabilities + vuln_by_agent_index_second_scan[agent] = parse_vulnerability_detector_alerts(agent_all_alerts)['affected'] + + results['vulnerabilities_index_second_scan'] = vuln_by_agent_index_second_scan + test_result['evidences']['vulnerabilities_index_second_scan'] = vuln_by_agent_index_second_scan differences = list(set(results['vulnerabilities_index_first_scan']).symmetric_difference( set(results['vulnerabilities_index_second_scan']))) From f1892149799e5c73b7bb416c557b1cac069da36b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Rebollo=20P=C3=A9rez?= Date: Wed, 21 Feb 2024 16:39:43 +0000 Subject: [PATCH 04/10] fix: increase timeout and improve indexer request performance --- .../wazuh_testing/end_to_end/monitoring.py | 24 +++--- .../end_to_end/vulnerability_detector.py | 1 - .../test_vulnerability_detector.py | 80 +++++++++---------- 3 files changed, 50 insertions(+), 55 deletions(-) diff --git a/deps/wazuh_testing/wazuh_testing/end_to_end/monitoring.py b/deps/wazuh_testing/wazuh_testing/end_to_end/monitoring.py index a40476b280..36749bef65 100644 --- a/deps/wazuh_testing/wazuh_testing/end_to_end/monitoring.py +++ b/deps/wazuh_testing/wazuh_testing/end_to_end/monitoring.py @@ -26,7 +26,7 @@ from wazuh_testing.tools.system import HostManager -def monitoring_events_multihost(host_manager: HostManager, monitoring_data: Dict, ignore_error: bool = False, +def monitoring_events_multihost(host_manager: HostManager, monitoring_data: Dict, ignore_error: bool = True, scan_interval: int = 5) -> Dict: """ Monitor events on multiple hosts concurrently. @@ -63,7 +63,7 @@ def monitoring_events_multihost(host_manager: HostManager, monitoring_data: Dict } """ def monitoring_event(host_manager: HostManager, host: str, monitoring_elements: List[Dict], - ignore_error: bool = False, + ignore_error: bool = True, scan_interval: int = 5) -> Dict: """ Monitor the specified elements on a host. @@ -72,7 +72,7 @@ def monitoring_event(host_manager: HostManager, host: str, monitoring_elements: host_manager (HostManager): Host Manager to handle the environment host (str): The target host. monitoring_elements(List): A list of dictionaries containing regex, timeout, and file. - ignore_error: If True, ignore errors and continue monitoring. + ignore_error: If True, ignore TimeoutError and return the result. Raises: TimeoutError: If no match is found within the specified timeout. @@ -97,8 +97,13 @@ def filter_events_by_timestamp(match_events: List) -> List: timestamp_format = "%Y/%m/%d %H:%M:%S" timestamp_format_parameter = "%Y-%m-%dT%H:%M:%S.%f" - timestamp_datetime = datetime.strptime(timestamp_str, timestamp_format) - greater_than_timestamp_formatted = datetime.strptime(greater_than_timestamp, timestamp_format_parameter) + try: + timestamp_datetime = datetime.strptime(timestamp_str, timestamp_format) + greater_than_timestamp_formatted = datetime.strptime(greater_than_timestamp, + timestamp_format_parameter) + except ValueError: + raise ValueError(f"Timestamp format not supported: {timestamp_str}." + 'Do the regex includes the timestamp?') if timestamp_datetime >= greater_than_timestamp_formatted: match_that_fit_timestamp.append(match) @@ -153,15 +158,12 @@ def filter_events_by_timestamp(match_events: List) -> List: with ThreadPoolExecutor() as executor: futures = [] for host, data in monitoring_data.items(): - futures.append(executor.submit(monitoring_event, host_manager, host, data, ignore_error)) + futures.append(executor.submit(monitoring_event, host_manager, host, data, ignore_error, scan_interval)) results = {} for future in as_completed(futures): - try: - result = future.result() - results.update(result) - except Exception as e: - logging.error(f"An error occurred: {e}") + result = future.result() + results.update(result) logging.info(f"Monitoring results: {results}") diff --git a/deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py b/deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py index 51a314c1f7..a046960173 100644 --- a/deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py +++ b/deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py @@ -151,7 +151,6 @@ def parse_vulnerability_detector_alerts(alerts) -> Dict: # Parse affected vuln alerts for alert in alerts: - print(alert) if re.match(regex_affected, alert['_source']['rule']['description']): vulnerability_detector_alerts['affected'].append(alert) elif re.match(rege_mitigated, alert['_source']['rule']['description']): diff --git a/tests/end_to_end/test_vulnerability_detector/test_vulnerability_detector.py b/tests/end_to_end/test_vulnerability_detector/test_vulnerability_detector.py index 9d5c5e18a3..5a1c349692 100644 --- a/tests/end_to_end/test_vulnerability_detector/test_vulnerability_detector.py +++ b/tests/end_to_end/test_vulnerability_detector/test_vulnerability_detector.py @@ -35,10 +35,6 @@ - e2e - vulnerability_detector - tier0 - -ToDo: - - check_vuln_state_consistency: Function to ensure the consitency of the agent's vulnerabilities - - Check if wait_until_vuln_scan_agents_finished function works as expected for the second scan """ import os import pytest @@ -59,8 +55,7 @@ from wazuh_testing.tools.configuration import load_configuration_template from wazuh_testing.tools.system import HostManager from wazuh_testing.end_to_end.remote_operations_handler import launch_parallel_operations -from wazuh_testing.end_to_end.vulnerability_detector import get_alerts_by_agent, get_indexed_vulnerabilities_by_agent, \ - parse_vulnerability_detector_alerts +from wazuh_testing.end_to_end.vulnerability_detector import parse_vulnerability_detector_alerts from wazuh_testing.modules.syscollector import TIMEOUT_SYSCOLLECTOR_SCAN @@ -75,7 +70,7 @@ 'agent': os.path.join(configurations_dir, 'agent.yaml') } vulnerability_detector_logs_dir = os.path.join(current_dir, "logs") -TIMEOUT_PER_AGENT_VULNERABILITY_SCAN = 120 +TIMEOUT_PER_AGENT_VULNERABILITY_SCAN = 200 def load_vulnerability_detector_configurations(host_manager): @@ -125,9 +120,6 @@ def setup_vulnerability_tests(host_manager: HostManager) -> Generator: logger.error("Getting backup of current configurations") hosts_configuration_backup = backup_configurations(host_manager) - logger.error("Save the Wazuh indexer username and password into the Wazuh manager keystore") - save_indexer_credentials_into_keystore(host_manager) - logger.error("Configuring environment") configure_environment(host_manager, load_vulnerability_detector_configurations(host_manager)) @@ -141,6 +133,12 @@ def setup_vulnerability_tests(host_manager: HostManager) -> Generator: logger.error("Restarting managers") host_manager.control_environment('restart', ['manager']) + logger.error("Save the Wazuh indexer username and password into the Wazuh manager keystore") + save_indexer_credentials_into_keystore(host_manager) + + logger.error("Restarting managers") + host_manager.control_environment('restart', ['manager']) + utc_now_timestamp = datetime.datetime.utcnow() # Format the date and time as per the given format @@ -299,14 +297,13 @@ def test_syscollector_first_scan(self, request, host_manager, setup_vulnerabilit results[test_name] = test_result if not test_result['checks']['all_successfull']: - pytest.fail("Some agents has not been scanned:" - f"{test_result['evidences']['agents_not_scanned_first_scan']}." - "Check logs for more information") + logging_message = 'Syscollector scan not started in the following agents:' \ + f"{test_result['evidences']['agents_not_scanned_first_scan']}." + logger.critical(logging_message) + pytest.fail(logging_message) else: logger.critical("All agents has been scanned") - logger.critical(results) - def test_syscollector_first_scan_alerts(self, request, host_manager, setup_vulnerability_tests, get_results): """ description: Validates that the Vulnerability Detector detects vulnerabilities within the environment in the @@ -368,7 +365,8 @@ def test_syscollector_first_scan_alerts(self, request, host_manager, setup_vulne vuln_by_agent_alerts = {} for agent in agents_to_check: agent_all_alerts = get_indexer_values(host_manager, - greater_than_timestamp=setup_vulnerability_tests, agent=agent)['hits']['hits'] + greater_than_timestamp=setup_vulnerability_tests, + agent=agent)['hits']['hits'] # Only is expected alert of affected vulnerabilities vuln_by_agent_alerts[agent] = parse_vulnerability_detector_alerts(agent_all_alerts)['affected'] @@ -390,13 +388,13 @@ def test_syscollector_first_scan_alerts(self, request, host_manager, setup_vulne results['vulnerabilities_alerts_first_scan'] = vuln_by_agent_alerts if not test_result['checks']['all_successfull']: - pytest.fail(f"Some agents has not been scanned: {test_result['evidences']['agents_not_detected_alerts']}." - "Check logs for more information") + logging_message = 'Some agents has not been scanned:' \ + f"{test_result['evidences']['agents_not_detected_alerts']}." + logger.critical(logging_message) + pytest.fail(logging_message) else: logger.critical("All agents has been scanned") - logger.critical(results) - def test_syscollector_first_scan_index(self, request, host_manager, setup_vulnerability_tests, get_results): """ description: Validates that the Vulnerability Detector detects vulnerabilities within the environment in the @@ -450,9 +448,7 @@ def test_syscollector_first_scan_index(self, request, host_manager, setup_vulner logger.critical("Checking vulnerabilities in the index") vuln_by_agent_index = {} for agent in agents_to_check: - agent_all_alerts = get_indexer_values(host_manager, - greater_than_timestamp=setup_vulnerability_tests, - agent=agent)['hits']['hits'] + agent_all_alerts = get_indexer_values(host_manager, greater_than_timestamp=setup_vulnerability_tests, agent=agent, index='wazuh-states-vulnerabilities',)['hits']['hits'] # Only is expected alert of affected vulnerabilities vuln_by_agent_index[agent] = parse_vulnerability_detector_alerts(agent_all_alerts)['affected'] @@ -473,12 +469,13 @@ def test_syscollector_first_scan_index(self, request, host_manager, setup_vulner results['vulnerabilities_index_first_scan'] = vuln_by_agent_index if not test_result['checks']['all_successfull']: - pytest.fail("Some agents has not been scanned and updated states index:" - f"{test_result['evidences']['agents_not_detected_index_vulnerabilities']}.") + logging_message = 'Some agents has not been scanned and generated vulnerabilities in the index:' \ + f"{test_result['evidences']['agents_not_detected_index_vulnerabilities']}." + logger.critical(logging_message) + pytest.fail(logging_message) else: logger.critical("All agents has been scanned and updated states index") - logger.critical(results) def tests_syscollector_vulnerabilities_index_alerts_consistency(self, request, setup_vulnerability_tests, get_results): @@ -540,15 +537,14 @@ def tests_syscollector_vulnerabilities_index_alerts_consistency(self, request, results[test_name] = test_result if not test_result['checks']['all_successfull']: - logger.critical("Index state is not consistent with the alerts") + logging_message = "Index state is not consistent with the alerts" + logger.critical(logging_message) logger.critical(f"Alerts not in states: {test_result['evidences']['alerts_not_in_states']}") logger.critical(f"States not in alerts: {test_result['evidences']['states_not_in_alerts']}") - pytest.fail() + pytest.fail(logging_message) else: logger.critical("Index state is consistent with the alerts") - logger.critical(results) - def test_syscollector_second_scan(self, request, host_manager, setup_vulnerability_tests, get_results): """ description: Validates the initiation of the second Syscollector scans across all agents in the environment. @@ -616,13 +612,12 @@ def test_syscollector_second_scan(self, request, host_manager, setup_vulnerabili results[test_name] = test_result if not test_result['checks']['all_successfull']: - pytest.fail("Syscollector scan not started in the following agents:" - f"{test_name['evidences']['agents_syscollector_second_scan_not_started']}.") + logging_message = "Syscollector scan not started in the following agents:" \ + f"{test_result['evidences']['agents_syscollector_second_scan_not_started']}." + pytest.fail(logging_message) else: logger.critical("Syscollector scan started in all agents") - logger.critical(results) - def tests_syscollector_first_second_scan_consistency_alerts(self, request, host_manager, setup_vulnerability_tests, get_results): """ @@ -688,7 +683,6 @@ def tests_syscollector_first_second_scan_consistency_alerts(self, request, host_ # Only is expected alert of affected vulnerabilities vuln_by_agent_alert_second_scan[agent] = parse_vulnerability_detector_alerts(agent_all_alerts)['affected'] - # Store full alert list in global results. It is needed for the next test results['vulnerabilities_alerts_second_scan'] = vuln_by_agent_alert_second_scan test_result['evidences']['vulnerabilities_alerts_second_scan'] = vuln_by_agent_alert_second_scan @@ -731,12 +725,13 @@ def tests_syscollector_first_second_scan_consistency_alerts(self, request, host_ results[test_name] = test_result if not test_result['checks']['all_successfull']: - pytest.fail("Inconsistencies found between first and second scan." - "Check evidences for more information") + logging_message = "Inconsistencies found between first and second scan." \ + "Check evidences for more information" + logger.critical(logging_message) + pytest.fail(logging_message) else: logger.critical("The number of vulnerabilities is the same between scans") - logger.critical(results) def tests_syscollector_first_second_scan_consistency_index(self, request, host_manager, setup_vulnerability_tests, get_results): @@ -786,6 +781,7 @@ def tests_syscollector_first_second_scan_consistency_index(self, request, host_m for agent in host_manager.get_group_hosts('agent'): agent_all_alerts = get_indexer_values(host_manager, greater_than_timestamp=setup_vulnerability_tests, + index='wazuh-states-vulnerabilities', agent=agent)['hits']['hits'] # Only is expected alert of affected vulnerabilities vuln_by_agent_index_second_scan[agent] = parse_vulnerability_detector_alerts(agent_all_alerts)['affected'] @@ -798,15 +794,15 @@ def tests_syscollector_first_second_scan_consistency_index(self, request, host_m results[test_name] = test_result - if results['vulnerabilities_index_first_scan'] != results['vulnerabilities_index_second_scan']: + if differences: test_result['checks']['all_successfull'] = False results[test_name]['evidences']['vulnerabilities_not_equal_between_scans_indexer'] = differences + logging.critical("Inconsistencies found between first and second scan in the index." + f"Check evidences for more information: {differences}") pytest.fail('The number of vulnerabilities is not the same between scans') else: logger.critical("The number of vulnerabilities is the same between scans") - logger.critical(results) - # ------------------------- @@ -844,7 +840,6 @@ def test_vulnerability_detector_scans_cases(self, setup_vulnerability_tests, req results[request.node.name] = {} results['setup'] = setup_results - hosts_to_ignore = [] for host in setup_results.keys(): @@ -883,6 +878,5 @@ def test_vulnerability_detector_scans_cases(self, setup_vulnerability_tests, req results[test_name] = test_result logger.critical("Final Results") - logger.critical(results) assert success_for_all_agents is True, "Test failed. Check logs for more information" From f94e2a0dcf6f00cba9c35b62ed75081f9c1677cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Rebollo=20P=C3=A9rez?= Date: Thu, 22 Feb 2024 11:51:06 +0000 Subject: [PATCH 05/10] docs: include agent to get_indexer_values Co-authored-by: Julia Magan --- deps/wazuh_testing/wazuh_testing/end_to_end/indexer_api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/deps/wazuh_testing/wazuh_testing/end_to_end/indexer_api.py b/deps/wazuh_testing/wazuh_testing/end_to_end/indexer_api.py index 5e7705962c..32a1a680e0 100644 --- a/deps/wazuh_testing/wazuh_testing/end_to_end/indexer_api.py +++ b/deps/wazuh_testing/wazuh_testing/end_to_end/indexer_api.py @@ -32,6 +32,7 @@ def get_indexer_values(host_manager: HostManager, credentials: dict = {'user': ' {'user': 'admin', 'password': 'changeme'}. index (Optional): The Indexer index name. Defaults to 'wazuh-alerts*'. greater_than_timestamp (Optional): The timestamp to filter the results. Defaults to None. + agent (Optional): The agent name to filter the results. Defaults to ''. Returns: Dict: A dictionary containing the values retrieved from the Indexer API. From 9c2fb39d9da5b0dd0a337706edaaebff0e49e369 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Rebollo=20P=C3=A9rez?= Date: Thu, 22 Feb 2024 11:54:31 +0000 Subject: [PATCH 06/10] docs: remove regex from docstring --- .../wazuh_testing/end_to_end/vulnerability_detector.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py b/deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py index a046960173..2a62241a43 100644 --- a/deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py +++ b/deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py @@ -26,6 +26,10 @@ from wazuh_testing.end_to_end.indexer_api import get_indexer_values +regex_affected = 'CVE.*? affects.*"?' +rege_mitigated = "The .* that affected .* was solved due to a package removal" + + def load_packages_metadata() -> Dict: """ Load packages metadata from the packages.json file. @@ -137,7 +141,6 @@ def parse_vulnerability_detector_alerts(alerts) -> Dict: Args: alerts (list): List of alerts. - regex (str): Regular expression to match the alerts. Returns: dict: Dictionary containing the alerts by agent. @@ -146,9 +149,6 @@ def parse_vulnerability_detector_alerts(alerts) -> Dict: vulnerability_detector_alerts['affected'] = [] vulnerability_detector_alerts['mitigated'] = [] - regex_affected = 'CVE.*? affects.*"?' - rege_mitigated = "The .* that affected .* was solved due to a package removal" - # Parse affected vuln alerts for alert in alerts: if re.match(regex_affected, alert['_source']['rule']['description']): From e49f94aa404991848d82b074c67a8df884e9bfa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Rebollo=20P=C3=A9rez?= Date: Thu, 22 Feb 2024 11:58:44 +0000 Subject: [PATCH 07/10] style: move regexes to regex module --- deps/wazuh_testing/wazuh_testing/end_to_end/regex.py | 6 ++++++ .../end_to_end/vulnerability_detector.py | 12 ++++++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/deps/wazuh_testing/wazuh_testing/end_to_end/regex.py b/deps/wazuh_testing/wazuh_testing/end_to_end/regex.py index b5126c4a2b..cd1c4136d0 100644 --- a/deps/wazuh_testing/wazuh_testing/end_to_end/regex.py +++ b/deps/wazuh_testing/wazuh_testing/end_to_end/regex.py @@ -44,6 +44,12 @@ 'regex': '.*HOST_NAME.*package":.*name":"PACKAGE_NAME".*version":"PACKAGE_VERSION".*"' 'architecture":"ARCHITECTURE.*"cve":"CVE"', 'parameters': ['HOST_NAME', 'CVE', 'PACKAGE_NAME', 'PACKAGE_VERSION', 'ARCHITECTURE'] + }, + 'vuln_affected': { + 'regex': 'CVE.*? affects.*"?' + }, + 'vuln_mitigated': { + 'regex': "The .* that affected .* was solved due to a package removal" } } diff --git a/deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py b/deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py index 2a62241a43..9f46771eaf 100644 --- a/deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py +++ b/deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py @@ -24,10 +24,7 @@ from wazuh_testing.tools.system import HostManager from wazuh_testing.end_to_end.indexer_api import get_indexer_values - - -regex_affected = 'CVE.*? affects.*"?' -rege_mitigated = "The .* that affected .* was solved due to a package removal" +from wazuh_testing.end_to_end.regex import regex_affected, regex_mitigated def load_packages_metadata() -> Dict: @@ -149,11 +146,14 @@ def parse_vulnerability_detector_alerts(alerts) -> Dict: vulnerability_detector_alerts['affected'] = [] vulnerability_detector_alerts['mitigated'] = [] + vuln_affected_regex = regex_affected['regex'] + vuln_mitigated_regex = regex_mitigated['regex'] + # Parse affected vuln alerts for alert in alerts: - if re.match(regex_affected, alert['_source']['rule']['description']): + if re.match(vuln_affected_regex, alert['_source']['rule']['description']): vulnerability_detector_alerts['affected'].append(alert) - elif re.match(rege_mitigated, alert['_source']['rule']['description']): + elif re.match(vuln_mitigated_regex['regex'], alert['_source']['rule']['description']): vulnerability_detector_alerts['mitigated'].append(alert) return vulnerability_detector_alerts From b1397a377f2f3279965c01a47c149f2b3b494291 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Rebollo=20P=C3=A9rez?= Date: Thu, 22 Feb 2024 11:59:59 +0000 Subject: [PATCH 08/10] docs: fix parse vulnerability docstring description Co-authored-by: Julia Magan --- .../wazuh_testing/end_to_end/vulnerability_detector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py b/deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py index 9f46771eaf..c1916dbb40 100644 --- a/deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py +++ b/deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py @@ -134,7 +134,7 @@ def get_alerts_by_agent(alerts, regex) -> Dict: def parse_vulnerability_detector_alerts(alerts) -> Dict: """ - Get specific alerts by agent. + Parse vulnerability detector alerts. Args: alerts (list): List of alerts. From 4c08533990e882e0adf9a4f800714e71a013b3ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Rebollo=20P=C3=A9rez?= Date: Thu, 22 Feb 2024 17:15:12 +0000 Subject: [PATCH 09/10] refac: removed unnused vuln detection functions Co-authored-by: Julia Magan --- .../end_to_end/vulnerability_detector.py | 54 ++----------------- 1 file changed, 3 insertions(+), 51 deletions(-) diff --git a/deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py b/deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py index c1916dbb40..d5184d338f 100644 --- a/deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py +++ b/deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py @@ -6,9 +6,7 @@ Functions: - load_packages_metadata: Load packages metadata from the packages.json file. - check_vuln_state_index: Check vulnerability state index for a host. - - get_alerts_by_agent: Get specific alerts by agent. - get_indexed_vulnerabilities_by_agent: Get indexed vulnerabilities by agent. - - get_vulnerabilities_alerts_indexer: Get vulnerabilities alerts by agent. - check_vuln_alert_indexer: Check vulnerability alerts in the indexer for a host. @@ -24,7 +22,7 @@ from wazuh_testing.tools.system import HostManager from wazuh_testing.end_to_end.indexer_api import get_indexer_values -from wazuh_testing.end_to_end.regex import regex_affected, regex_mitigated +from wazuh_testing.end_to_end.regex import REGEX_PATTERNS def load_packages_metadata() -> Dict: @@ -108,30 +106,6 @@ def check_vuln_state_index(host_manager: HostManager, host: str, package: Dict[s return expected_alerts_not_found -def get_alerts_by_agent(alerts, regex) -> Dict: - """ - Get specific alerts by agent. - - Args: - alerts (list): List of alerts. - regex (str): Regular expression to match the alerts. - - Returns: - dict: Dictionary containing the alerts by agent. - """ - alerts_vuln_by_agent = {} - - for alert in alerts: - if re.match(regex, alert['_source']['rule']['description']): - if 'agent' in alert['_source']: - agent = alert['_source']['agent']['name'] - if agent not in alerts_vuln_by_agent: - alerts_vuln_by_agent[agent] = [] - alerts_vuln_by_agent[agent].append(alert) - - return alerts_vuln_by_agent - - def parse_vulnerability_detector_alerts(alerts) -> Dict: """ Parse vulnerability detector alerts. @@ -146,8 +120,8 @@ def parse_vulnerability_detector_alerts(alerts) -> Dict: vulnerability_detector_alerts['affected'] = [] vulnerability_detector_alerts['mitigated'] = [] - vuln_affected_regex = regex_affected['regex'] - vuln_mitigated_regex = regex_mitigated['regex'] + vuln_affected_regex = REGEX_PATTERNS['vuln_affected']['regex'] + vuln_mitigated_regex = REGEX_PATTERNS['vuln_mitigated']['regex'] # Parse affected vuln alerts for alert in alerts: @@ -180,28 +154,6 @@ def get_indexed_vulnerabilities_by_agent(indexed_vulnerabilities) -> Dict: return vulnerabilities_by_agent -def get_vulnerabilities_alerts_indexer(host_manager: HostManager, greater_than_timestamp: str = "", - vuln_mitigated=False) -> Dict: - """Get vulnerabilities alerts by agent. - - Args: - host_manager (HostManager): An instance of the HostManager class containing information about hosts. - host (str): Host name. - greater_than_timestamp (str): Datetime to filter the vulnerability state index. - vuln_mitigated (bool): Indicates if the vulnerability is mitigated. - - Returns: - dict: Dictionary containing the indexed vulnerabilities by agent. - """ - - indexer_alerts = get_indexer_values(host_manager, greater_than_timestamp=greater_than_timestamp)['hits']['hits'] - - regex_to_match = "CVE.* affects .*" if not vuln_mitigated else \ - "The .* that affected .* was solved due to a package removal" - - return get_alerts_by_agent(indexer_alerts, regex_to_match) - - def check_vuln_alert_indexer(vulnerabilities_alerts: Dict, host: str, package: Dict[str, Dict], current_datetime: str = '') -> List: """ From db1e6de6fa8b1bb0a3ea7e06e52006b913a4804a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Rebollo=20P=C3=A9rez?= Date: Thu, 22 Feb 2024 17:21:13 +0000 Subject: [PATCH 10/10] docs: include 5003 changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index af08aaf6e2..ab90ac0a90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ All notable changes to this project will be documented in this file. ### Fixed +- Fix timeout and performance issues in E2E Vulnerability Detector tests ([#5003](https://github.com/wazuh/wazuh-qa/pull/5003)) \- (Framework) - Fixed Filebeat provisioning role with pre-release and staging URLs ([#4950](https://github.com/wazuh/wazuh-qa/pull/4950)) \- (Framework) - Fix macOS Vulnerability Detection handler provision in E2E tests ([#4948](https://github.com/wazuh/wazuh-qa/pull/4948)) \- (Framework) - Migrate Vulnerability Detection timeouts variables to the waiters module ([#4949](https://github.com/wazuh/wazuh-qa/pull/4949)) \- (Framework)