Merge pull request #5032 from wazuh/fix/5019-initial-scans-fix

Fix Initial scans tests

deps/wazuh_testing/wazuh_testing/end_to_end/remote_operations_handler.py

@@ -28,7 +28,8 @@
 from wazuh_testing.end_to_end.waiters import wait_syscollector_and_vuln_scan
 from wazuh_testing.tools.system import HostManager
 from wazuh_testing.end_to_end.vulnerability_detector import check_vuln_alert_indexer, check_vuln_state_index, \
- load_packages_metadata, get_vulnerabilities_alerts_indexer, get_indexer_values
+ load_packages_metadata, parse_vulnerability_detector_alerts
+from wazuh_testing.end_to_end.indexer_api import get_indexer_values
 
 
 def check_vulnerability_alerts(results: Dict, check_data: Dict, current_datetime: str, host_manager: HostManager,
@@ -37,11 +38,23 @@ def check_vulnerability_alerts(results: Dict, check_data: Dict, current_datetime
  operation: str = 'install') -> None:
 
  # Get all the alerts generated in the timestamp
- vulnerability_alerts = get_vulnerabilities_alerts_indexer(host_manager, current_datetime)
- vulnerability_alerts_mitigated = get_vulnerabilities_alerts_indexer(host_manager, current_datetime, True)
+ vulnerability_alerts = {}
+ vulnerability_alerts_mitigated = {}
+ vulnerability_index = {}
 
- vulnerability_index = get_indexer_values(host_manager, index='wazuh-states-vulnerabilities',
- greater_than_timestamp=current_datetime)['hits']['hits']
+ for agent in host_manager.get_group_hosts('agent'):
+ agent_all_alerts = parse_vulnerability_detector_alerts(get_indexer_values(host_manager,
+ greater_than_timestamp=current_datetime,
+ agent=agent)['hits']['hits'])
+
+ agent_all_vulnerabilities = get_indexer_values(host_manager, greater_than_timestamp=current_datetime,
+ agent=agent,
+ index='wazuh-states-vulnerabilities')['hits']['hits']
+
+ vulnerability_alerts[agent] = agent_all_alerts['affected']
+ vulnerability_alerts_mitigated[agent] = agent_all_alerts['mitigated']
+
+ vulnerability_index[agent] = agent_all_vulnerabilities
 
  results['evidences']['all_alerts_found'] = vulnerability_alerts
  results['evidences']['all_alerts_found_mitigated'] = vulnerability_alerts_mitigated

deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py

@@ -8,6 +8,8 @@
  - check_vuln_state_index: Check vulnerability state index for a host.
  - get_indexed_vulnerabilities_by_agent: Get indexed vulnerabilities by agent.
  - check_vuln_alert_indexer: Check vulnerability alerts in the indexer for a host.
+ - parse_vulnerability_detector_alerts: Parse vulnerability detector alerts.
+ - check_vuln_state_consistency: Check vulnerability state consistency.
 
 
 Copyright (C) 2015, Wazuh Inc.
@@ -22,7 +24,7 @@
 
 from wazuh_testing.tools.system import HostManager
 from wazuh_testing.end_to_end.indexer_api import get_indexer_values
-from wazuh_testing.end_to_end.regex import REGEX_PATTERNS 
+from wazuh_testing.end_to_end.regex import REGEX_PATTERNS
 
 
 def load_packages_metadata() -> Dict:
@@ -127,7 +129,7 @@ def parse_vulnerability_detector_alerts(alerts) -> Dict:
  for alert in alerts:
  if re.match(vuln_affected_regex, alert['_source']['rule']['description']):
  vulnerability_detector_alerts['affected'].append(alert)
- elif re.match(vuln_mitigated_regex['regex'], alert['_source']['rule']['description']):
+ elif re.match(vuln_mitigated_regex, alert['_source']['rule']['description']):
  vulnerability_detector_alerts['mitigated'].append(alert)
 
  return vulnerability_detector_alerts
@@ -204,3 +206,81 @@ def check_vuln_alert_indexer(vulnerabilities_alerts: Dict, host: str, package: D
  'PACKAGE_VERSION': package_version})
 
  return expected_alerts_not_found
+
+
+def check_vuln_state_consistency(vulnerabilities_alerts, vulnerabilities_states):
+ # Get the indexer values
+ alerts_vulnerabilities = {}
+ indices_vulnerabilities = {}
+
+ for agent, vuln_alerts in vulnerabilities_alerts.items():
+ for vuln_alert in vuln_alerts:
+ if agent != vuln_alert['_source']['agent']['name']:
+ logging.critical("Agent name is not the same as the agent in the alert")
+ alert_agent = vuln_alert['_source']['agent']['name']
+ alert_cve = vuln_alert['_source']['data']['vulnerability']['cve']
+ alert_package_version = vuln_alert['_source']['data']['vulnerability']['package']['version']
+ alert_package_name = vuln_alert['_source']['data']['vulnerability']['package']['name']
+
+ if agent not in alerts_vulnerabilities:
+ alerts_vulnerabilities[agent] = []
+
+ alerts_vulnerabilities[agent].append({
+ 'cve': alert_cve,
+ 'agent': alert_agent,
+ 'package_name': alert_package_name,
+ 'package_version': alert_package_version
+ })
+
+ for agent, vulnerabilities in vulnerabilities_states.items():
+ for vulnerability in vulnerabilities:
+ if agent != vulnerability['_source']['agent']['name']:
+ logging.critical("Agent name is not the same as the agent in the vulnerability state")
+
+ state_agent = vulnerability['_source']['agent']['name']
+ state_cve = vulnerability['_source']['vulnerability']['id']
+ state_package_name = vulnerability['_source']['package']['name']
+ state_package_version = vulnerability['_source']['package']['version']
+
+ if agent not in indices_vulnerabilities:
+ indices_vulnerabilities[agent] = []
+
+ indices_vulnerabilities[agent].append({
+ 'cve': state_cve,
+ 'agent': state_agent,
+ 'package_name': state_package_name,
+ 'package_version': state_package_version
+ })
+
+ if vulnerabilities_states.keys() != vulnerabilities_alerts.keys():
+ logging.critical("The number of agents is not the same between alerts and states")
+
+ agents_in_alerts_states = [agent for agent in vulnerabilities_states.keys() \
+ if agent in vulnerabilities_alerts.keys()]
+
+ alerts_not_in_states = []
+ states_not_in_alerts = []
+
+ for agent in agents_in_alerts_states:
+ agent_alerts = [] if agent not in alerts_vulnerabilities else alerts_vulnerabilities[agent]
+ agent_states = [] if agent not in indices_vulnerabilities else indices_vulnerabilities[agent]
+
+ if len(agent_alerts) != len(agent_states):
+ logging.critical(f"The number of alerts is not the same as the number of states for agent {agent}")
+ logging.critical(f"Alerts: {len(agent_alerts)}")
+ logging.critical(f"States: {len(agent_states)}")
+
+ # Check that all alerts are in the index
+ for alert in agent_alerts:
+ if alert not in agent_states:
+ alerts_not_in_states.append(alert)
+
+ # Check that all index states are in the alerts
+ for state in agent_states:
+ if state not in agent_alerts:
+ alerts_not_in_states.append(state)
+
+ return {
+ 'alerts_not_in_states': alerts_not_in_states,
+ 'states_not_in_alerts': states_not_in_alerts
+ }

tests/end_to_end/test_vulnerability_detector/test_vulnerability_detector.py

@@ -55,7 +55,8 @@
 from wazuh_testing.tools.configuration import load_configuration_template
 from wazuh_testing.tools.system import HostManager
 from wazuh_testing.end_to_end.remote_operations_handler import launch_parallel_operations
-from wazuh_testing.end_to_end.vulnerability_detector import parse_vulnerability_detector_alerts
+from wazuh_testing.end_to_end.vulnerability_detector import parse_vulnerability_detector_alerts, \
+ check_vuln_state_consistency
 from wazuh_testing.modules.syscollector import TIMEOUT_SYSCOLLECTOR_SCAN
 
 
@@ -70,6 +71,7 @@
  'agent': os.path.join(configurations_dir, 'agent.yaml')
  }
 vulnerability_detector_logs_dir = os.path.join(current_dir, "logs")
+
 TIMEOUT_PER_AGENT_VULNERABILITY_SCAN = 200
 
 
@@ -176,60 +178,6 @@ def setup_vulnerability_tests(host_manager: HostManager) -> Generator:
  host_manager.control_environment('restart', ['manager'])
 
 
-def check_vuln_state_consistency(vulnerabilities_alerts, vulnerabilities_states):
- # Get the indexer values
- alerts_vulnerabilities = []
- indices_vulnerabilities = []
-
- for alert in list(vulnerabilities_alerts.values())[0]:
- alert_agent = alert['_source']['agent']['name']
- alert_cve = alert['_source']['data']['vulnerability']['cve']
- alert_package_version = alert['_source']['data']['vulnerability']['package']['version']
- alert_package_name = alert['_source']['data']['vulnerability']['package']['name']
- alerts_vulnerabilities.append({
- 'cve': alert_cve,
- 'agent': alert_agent,
- 'package_name': alert_package_name,
- 'package_version': alert_package_version
- })
-
- for vulnerabilities_state in list(vulnerabilities_states.values())[0]:
- state_agent = vulnerabilities_state['_source']['agent']['name']
- state_cve = vulnerabilities_state['_source']['data']['vulnerability']['cve']
- state_package_name = vulnerabilities_state['_source']['data']['vulnerability']['package']['name']
- state_package_version = vulnerabilities_state['_source']['data']['vulnerability']['package']['version']
-
- indices_vulnerabilities.append({
- 'cve': state_cve,
- 'agent': state_agent,
- 'package_name': state_package_name,
- 'package_version': state_package_version
- })
-
- if len(alerts_vulnerabilities) != len(indices_vulnerabilities):
- logger.critical("The number of alerts is not the same as the number of states")
- logger.critical(f"Alerts: {len(alerts_vulnerabilities)}")
- logger.critical(f"States: {len(indices_vulnerabilities)}")
-
- alerts_not_in_states = []
- states_not_in_alerts = []
-
- # Check that all alerts are in the index
- for alert in alerts_vulnerabilities:
- if alert not in indices_vulnerabilities:
- alerts_not_in_states.append(alert)
-
- # Check that all index states are in the alerts
- for state in indices_vulnerabilities:
- if state not in alerts_vulnerabilities:
- states_not_in_alerts.append(state)
-
- return {
- 'alerts_not_in_states': alerts_not_in_states,
- 'states_not_in_alerts': states_not_in_alerts
- }
-
-
 @pytest.mark.filterwarnings('ignore::urllib3.exceptions.InsecureRequestWarning')
 class TestInitialScans():
  results = {}
@@ -304,6 +252,7 @@ def test_syscollector_first_scan(self, request, host_manager, setup_vulnerabilit
  else:
  logger.critical("All agents has been scanned")
 
+
  def test_syscollector_first_scan_alerts(self, request, host_manager, setup_vulnerability_tests, get_results):
  """
  description: Validates that the Vulnerability Detector detects vulnerabilities within the environment in the
@@ -387,6 +336,7 @@ def test_syscollector_first_scan_alerts(self, request, host_manager, setup_vulne
  # Store full alert list in global results. It is needed for the next test
  results['vulnerabilities_alerts_first_scan'] = vuln_by_agent_alerts
 
+
  if not test_result['checks']['all_successfull']:
  logging_message = 'Some agents has not been scanned:' \
  f"{test_result['evidences']['agents_not_detected_alerts']}."
@@ -447,18 +397,20 @@ def test_syscollector_first_scan_index(self, request, host_manager, setup_vulner
  # Check vulnerabilities in the index
  logger.critical("Checking vulnerabilities in the index")
  vuln_by_agent_index = {}
+
  for agent in agents_to_check:
- agent_all_alerts = get_indexer_values(host_manager, greater_than_timestamp=setup_vulnerability_tests, agent=agent, index='wazuh-states-vulnerabilities',)['hits']['hits']
+ agent_all_vulnerabilities = get_indexer_values(host_manager, greater_than_timestamp=setup_vulnerability_tests, agent=agent, index='wazuh-states-vulnerabilities',)['hits']['hits']
+
+ vuln_by_agent_index[agent] = agent_all_vulnerabilities
 
- # Only is expected alert of affected vulnerabilities
- vuln_by_agent_index[agent] = parse_vulnerability_detector_alerts(agent_all_alerts)['affected']
 
  test_result['evidences']['vulnerabilities_index_first_scan'] = vuln_by_agent_index
 
  logger.critical("Checking that all agents has been scanned and generated vulnerabilities in the index")
  for agent in agents_to_check:
  if agent not in list(vuln_by_agent_index.keys()) or \
  len(vuln_by_agent_index[agent]) == 0:
+
  logger.critical(f"Agent {agent} has not been scanned. Continuing with remaining agents")
  test_result['checks']['all_successfull'] = False
  test_result['evidences']['agents_not_detected_index_vulnerabilities'].append(agent)
@@ -779,12 +731,12 @@ def tests_syscollector_first_second_scan_consistency_index(self, request, host_m
 
  vuln_by_agent_index_second_scan = {}
  for agent in host_manager.get_group_hosts('agent'):
- agent_all_alerts = get_indexer_values(host_manager,
+ agent_all_vulnerabilities = get_indexer_values(host_manager,
  greater_than_timestamp=setup_vulnerability_tests,
  index='wazuh-states-vulnerabilities',
  agent=agent)['hits']['hits']
  # Only is expected alert of affected vulnerabilities
- vuln_by_agent_index_second_scan[agent] = parse_vulnerability_detector_alerts(agent_all_alerts)['affected']
+ vuln_by_agent_index_second_scan[agent] = agent_all_vulnerabilities
 
  results['vulnerabilities_index_second_scan'] = vuln_by_agent_index_second_scan
  test_result['evidences']['vulnerabilities_index_second_scan'] = vuln_by_agent_index_second_scan