feat(wazuh/wazuh#17388): add cloudwatch discard regex test and cases

tests/integration/test_aws/data/configuration_template/discard_regex_test_module/configuration_cloudwatch_discard_regex.yaml

@@ -0,0 +1,23 @@
+- sections:
+ - section: wodle
+ attributes:
+ - name: aws-s3
+ elements:
+ - disabled:
+ value: 'no'
+ - service:
+ attributes:
+ - type: SERVICE_TYPE
+ elements:
+ - aws_profile:
+ value: qa
+ - aws_log_groups:
+ value: LOG_GROUP_NAME
+ - only_logs_after:
+ value: ONLY_LOGS_AFTER
+ - regions:
+ value: REGIONS
+ - discard_regex:
+ attributes:
+ - field: DISCARD_FIELD
+ value: DISCARD_REGEX

tests/integration/test_aws/data/test_cases/discard_regex_test_module/cases_cloudwatch_discard_regex.yaml

@@ -0,0 +1,35 @@
+- name: cloudwatch_discard_regex_json
+ description: CloudWatch discard regex configurations
+ configuration_parameters:
+ SERVICE_TYPE: cloudwatchlogs
+ LOG_GROUP_NAME: wazuh-cloudwatchlogs-integration-tests
+ REGIONS: us-east-1
+ DISCARD_FIELD: networkInterfaces.networkInterfaceId
+ DISCARD_REGEX: .*eni-networkInterfaceId*
+ ONLY_LOGS_AFTER: 2023-JUL-03
+ metadata:
+ service_type: cloudwatchlogs
+ log_group_name: wazuh-cloudwatchlogs-integration-tests
+ only_logs_after: 2023-JUL-03
+ discard_field: networkInterfaces.networkInterfaceId
+ discard_regex: .*eni-networkInterfaceId.*
+ regions: us-east-1
+ found_logs: 1
+
+- name: cloudwatch_discard_regex_simple_text
+ description: CloudWatch discard regex configurations
+ configuration_parameters:
+ SERVICE_TYPE: cloudwatchlogs
+ LOG_GROUP_NAME: wazuh-cloudwatchlogs-integration-tests
+ REGIONS: us-east-1
+ DISCARD_FIELD: ''
+ DISCARD_REGEX: .*Test.*
+ ONLY_LOGS_AFTER: 2023-JAN-12
+ metadata:
+ service_type: cloudwatchlogs
+ log_group_name: wazuh-cloudwatchlogs-integration-tests
+ only_logs_after: 2023-JAN-12
+ discard_field: ''
+ discard_regex: .*Test.*
+ regions: us-east-1
+ found_logs: 3

tests/integration/test_aws/test_discard_regex.py

@@ -3,7 +3,7 @@
 import pytest
 from wazuh_testing import T_20, TEMPLATE_DIR, TEST_CASES_DIR, global_parameters
 from wazuh_testing.modules.aws import event_monitor, local_internal_options # noqa: F401
-from wazuh_testing.modules.aws.db_utils import s3_db_exists
+from wazuh_testing.modules.aws.db_utils import s3_db_exists, services_db_exists
 from wazuh_testing.tools.configuration import (
  get_test_cases_data,
  load_configuration_template,
@@ -19,18 +19,18 @@
 TEST_CASES_PATH = os.path.join(TEST_DATA_PATH, TEST_CASES_DIR, MODULE)
 
 # ---------------------------------------------------- TEST_BUCKET_DISCARD_REGEX -------------------------------------------------------
-configurations_path = os.path.join(CONFIGURATIONS_PATH, 'configuration_bucket_discard_regex.yaml')
-cases_path = os.path.join(TEST_CASES_PATH, 'cases_bucket_discard_regex.yaml')
+t0_configurations_path = os.path.join(CONFIGURATIONS_PATH, 'configuration_bucket_discard_regex.yaml')
+t0_cases_path = os.path.join(TEST_CASES_PATH, 'cases_bucket_discard_regex.yaml')
 
-configuration_parameters, configuration_metadata, case_ids = get_test_cases_data(cases_path)
-configurations = load_configuration_template(
- configurations_path, configuration_parameters, configuration_metadata
+t0_configuration_parameters, t0_configuration_metadata, t0_case_ids = get_test_cases_data(t0_cases_path)
+t0_configurations = load_configuration_template(
+ t0_configurations_path, t0_configuration_parameters, t0_configuration_metadata
 )
 
 
 @pytest.mark.tier(level=0)
-@pytest.mark.parametrize('configuration, metadata', zip(configurations, configuration_metadata), ids=case_ids)
-def test_discard_regex(
+@pytest.mark.parametrize('configuration, metadata', zip(t0_configurations, t0_configuration_metadata), ids=t0_case_ids)
+def test_bucket_discard_regex(
  configuration, metadata, load_wazuh_basic_configuration, set_wazuh_configuration, clean_s3_cloudtrail_db,
  configure_local_internal_options_function, truncate_monitored_files, restart_wazuh_function, file_monitoring,
 ):
@@ -86,8 +86,8 @@ def test_discard_regex(
  - Check the expected number of events were forwarded to analysisd.
  - Check the database was created and updated accordingly.
  input_description:
- - The `configuration_discard_regex` file provides the module configuration for this test.
- - The `cases_discard_regex` file provides the test cases.
+ - The `configuration_bucket_discard_regex` file provides the module configuration for this test.
+ - The `cases_bucket_discard_regex` file provides the test cases.
  """
  bucket_name = metadata['bucket_name']
  bucket_type = metadata['bucket_type']
@@ -140,3 +140,123 @@ def test_discard_regex(
  ).result()
 
  assert s3_db_exists()
+
+# ---------------------------------------------------- TEST_CLOUDWATCH_DISCARD_REGEX -------------------------------------------------------
+t1_configurations_path = os.path.join(CONFIGURATIONS_PATH, 'configuration_cloudwatch_discard_regex.yaml')
+t1_cases_path = os.path.join(TEST_CASES_PATH, 'cases_cloudwatch_discard_regex.yaml')
+
+t1_configuration_parameters, t1_configuration_metadata, t1_case_ids = get_test_cases_data(t1_cases_path)
+t1_configurations = load_configuration_template(
+ t1_configurations_path, t1_configuration_parameters, t1_configuration_metadata
+)
+
+
+@pytest.mark.tier(level=0)
+@pytest.mark.parametrize('configuration, metadata', zip(t1_configurations, t1_configuration_metadata), ids=t1_case_ids)
+def test_cloudwatch_discard_regex(
+ configuration, metadata, load_wazuh_basic_configuration, set_wazuh_configuration, clean_aws_services_db,
+ configure_local_internal_options_function, truncate_monitored_files, restart_wazuh_function, file_monitoring,
+):
+ """
+ description: Fetch logs excluding the ones that match with the regex.
+ test_phases:
+ - setup:
+ - Load Wazuh light configuration.
+ - Apply ossec.conf configuration changes according to the configuration template and use case.
+ - Apply custom settings in local_internal_options.conf.
+ - Truncate wazuh logs.
+ - Restart wazuh-manager service to apply configuration changes.
+ - test:
+ - Check in the ossec.log that a line has appeared calling the module with correct parameters.
+ - Check the expected number of events were forwarded to analysisd, only logs stored in the bucket and skips
+ the ones that match with regex.
+ - Check the database was created and updated accordingly.
+ - teardown:
+ - Truncate wazuh logs.
+ - Restore initial configuration, both ossec.conf and local_internal_options.conf.
+ - Delete the uploaded file
+ wazuh_min_version: 4.5.0
+ parameters:
+ - configuration:
+ type: dict
+ brief: Get configurations from the module.
+ - metadata:
+ type: dict
+ brief: Get metadata from the module.
+ - load_wazuh_basic_configuration:
+ type: fixture
+ brief: Load basic wazuh configuration.
+ - set_wazuh_configuration:
+ type: fixture
+ brief: Apply changes to the ossec.conf configuration.
+ - clean_aws_services_db:
+ type: fixture
+ brief: Delete the DB file before and after the test execution.
+ - configure_local_internal_options_function:
+ type: fixture
+ brief: Apply changes to the local_internal_options.conf configuration.
+ - truncate_monitored_files:
+ type: fixture
+ brief: Truncate wazuh logs.
+ - restart_wazuh_daemon_function:
+ type: fixture
+ brief: Restart the wazuh service.
+ - file_monitoring:
+ type: fixture
+ brief: Handle the monitoring of a specified file.
+ assertions:
+ - Check in the log that the module was called with correct parameters.
+ - Check the expected number of events were forwarded to analysisd.
+ - Check the database was created and updated accordingly.
+ input_description:
+ - The `configuration_cloudwatch_discard_regex` file provides the module configuration for this test.
+ - The `cases_cloudwatch_discard_regex` file provides the test cases.
+ """
+ log_group_name = metadata['log_group_name']
+ service_type = metadata['service_type']
+ only_logs_after = metadata['only_logs_after']
+ regions: str = metadata['regions']
+ discard_field = metadata['discard_field']
+ discard_regex = metadata['discard_regex']
+ found_logs = metadata['found_logs']
+
+ pattern = fr'.*The "{discard_regex}" regex found a match in the "{discard_field}" field. The event will be skipped.'
+
+ parameters = [
+ 'wodles/aws/aws-s3',
+ '--service', service_type,
+ '--aws_profile', 'qa',
+ '--only_logs_after', only_logs_after,
+ '--regions', regions,
+ '--aws_log_groups', log_group_name,
+ '--discard-field', discard_field,
+ '--discard-regex', discard_regex,
+ '--debug', '2'
+ ]
+
+
+ # Check AWS module started
+ log_monitor.start(
+ timeout=global_parameters.default_timeout,
+ callback=event_monitor.callback_detect_aws_module_start,
+ error_message='The AWS module did not start as expected',
+ ).result()
+
+ # Check command was called correctly
+ log_monitor.start(
+ timeout=global_parameters.default_timeout,
+ callback=event_monitor.callback_detect_aws_module_called(parameters),
+ error_message='The AWS module was not called with the correct parameters',
+ ).result()
+
+ log_monitor.start(
+ timeout=T_20,
+ callback=event_monitor.callback_detect_event_processed_or_skipped(pattern),
+ error_message=(
+ 'The AWS module did not show the correct message about discard regex or ',
+ 'did not process the expected amount of logs'
+ ),
+ accum_results=found_logs
+ ).result()
+
+ assert services_db_exists()