Add logcollector ignore and restrict IT tests

CHANGELOG.md

@@ -9,6 +9,7 @@ Release report: TBD
 
 ### Added
 
+- Add new tests for logcollector 'ignore' and 'restrict' options ([#3582](https:/wazuh/wazuh-qa/pull/3582)) \- (Tests)
 - Add 'Force reconnect' feature to agent_simulator tool. ([#3111](https:/wazuh/wazuh-qa/pull/3111)) \- (Tools)
 
 ### Changed

deps/wazuh_testing/wazuh_testing/modules/logcollector/__init__.py

@@ -1,5 +1,27 @@
+import sys
+from wazuh_testing.tools.monitoring import LOG_COLLECTOR_DETECTOR_PREFIX, AGENT_DETECTOR_PREFIX
 
-LOG_COLLECTOR_PREFIX = r'.*wazuh-logcollector.*'
+
+# Variables
+LOG_COLLECTOR_PREFIX = AGENT_DETECTOR_PREFIX if sys.platform == 'win32' else LOG_COLLECTOR_DETECTOR_PREFIX
 WINDOWS_AGENT_PREFIX = r'.*wazuh-agent.*'
 MAILD_PREFIX = r'.*wazuh-maild.*'
+
+
+# Error Messages
 GENERIC_CALLBACK_ERROR_COMMAND_MONITORING = 'The expected command monitoring log has not been produced'
+ERR_MSG_UNEXPECTED_IGNORE_EVENT = "Found unexpected 'Ignoring the log <message> due to ignore/restrict config' event"
+
+
+# Local_internal_options
+if sys.platform == 'win32':
+ LOGCOLLECTOR_DEFAULT_LOCAL_INTERNAL_OPTIONS = {
+ 'windows.debug': '2',
+ 'agent.debug': '2'
+ }
+else:
+ LOGCOLLECTOR_DEFAULT_LOCAL_INTERNAL_OPTIONS = {
+ 'logcollector.debug': '2',
+ 'monitord.rotate_log': '0',
+ 'agent.debug': '0',
+ }

deps/wazuh_testing/wazuh_testing/modules/logcollector/event_monitor.py

@@ -1,9 +1,9 @@
 import re
-
-from wazuh_testing import T_30
-from wazuh_testing.modules.logcollector import LOG_COLLECTOR_PREFIX
+import sys
+import pytest
+from wazuh_testing import T_30, T_10, LOG_FILE_PATH
+from wazuh_testing.modules.logcollector import LOG_COLLECTOR_PREFIX, ERR_MSG_UNEXPECTED_IGNORE_EVENT
 from wazuh_testing.tools.monitoring import FileMonitor
-from wazuh_testing import LOG_FILE_PATH
 
 
 def make_logcollector_callback(pattern, prefix=LOG_COLLECTOR_PREFIX, escape=False):
@@ -50,11 +50,13 @@ def check_logcollector_event(file_monitor=None, callback='', error_message=None,
  error_message = f"Could not find this event in {file_to_monitor}: {callback}" if error_message is None else \
  error_message
 
- file_monitor.start(timeout=timeout, update_position=update_position, accum_results=accum_results,
- callback=make_logcollector_callback(callback, prefix, escape), error_message=error_message)
+ result = file_monitor.start(timeout=timeout, update_position=update_position, accum_results=accum_results,
+ callback=make_logcollector_callback(callback, prefix, escape),
+ error_message=error_message).result()
+ return result
 
 
-def check_analyzing_file(file, error_message, prefix, file_monitor=None):
+def check_analyzing_file(file, prefix, error_message=None, file_monitor=None):
  """Create a callback to detect if logcollector is monitoring a file.
 
  Args:
@@ -63,13 +65,17 @@ def check_analyzing_file(file, error_message, prefix, file_monitor=None):
  prefix (str): Daemon that generates the error log.
  file_monitor (FileMonitor): Log monitor.
  """
+ if error_message is None:
+ error_message = f"Did not receive the expected 'Analyzing file: {file}' event"
+
  check_logcollector_event(file_monitor=file_monitor, timeout=T_30,
- callback=fr".*Analyzing file: '{re.escape(file)}'.*",
+ callback=fr".*Analyzing file: '{file}'.*",
  error_message=error_message, prefix=prefix)
 
 
-def check_syslog_messages(message, error_message, prefix, file_monitor=None, timeout=T_30, escape=False):
+def check_syslog_message(message, prefix, error_message=None, file_monitor=None, timeout=T_30, escape=False):
  """Create a callback to detect "DEBUG: Read <number> lines from command <command>" debug line.
+
  Args:
  message (str): Command to be monitored.
  error_message (str): Error message.
@@ -78,6 +84,51 @@ def check_syslog_messages(message, error_message, prefix, file_monitor=None, tim
  timeout (int): Timeout to check the log.
  escape (bool): Flag to escape special characters in the pattern.
  """
- callback_msg = fr"DEBUG: Reading syslog message: '{message}'"
+ if error_message is None:
+ error_message = f"Did not receive the expected 'Reading syslog message: {message}' event"
+
+ callback_msg = fr".*DEBUG: Reading syslog message: '{message}'.*"
+
  check_logcollector_event(file_monitor=file_monitor, timeout=timeout, callback=callback_msg,
  error_message=error_message, prefix=prefix, escape=escape)
+
+
+def check_ignore_restrict_message(message, regex, tag, prefix, error_message=None, file_monitor=None, timeout=T_10,
+ escape=False):
+ """Create a callback to detect "DEBUG: Ignoring the log ... due to config" debug line.
+
+ Args:
+ message (str): Command to be monitored.
+ regex (str): regex pattern configured to ignore or restrict to.
+ tag (str): string with the configured tag. Values: 'ignore' or 'restrict'
+ error_message (str): Error message.
+ prefix (str): Daemon that generates the error log.
+ file_monitor (FileMonitor): Log monitor.
+ timeout (int): Timeout to check the log.
+ escape (bool): Flag to escape special characters in the pattern.
+
+ Returns: True if the expected message has been found, False otherwise.
+ """
+ if error_message is None:
+ error_message = f"Did not receive the expected 'Ignoring the log line: {message} due to {tag} config' event"
+
+ callback_msg = fr"Ignoring the log line '{message}' due to {tag} config: '{regex}'"
+
+ return check_logcollector_event(file_monitor=file_monitor, timeout=timeout, callback=callback_msg,
+ error_message=error_message, prefix=prefix, escape=escape)
+
+
+def check_ignore_restrict_message_not_found(message, regex, tag, prefix):
+ """Check that an unexpected "Ignoring the log line..." event does not appear and a log is not ignored when it
+ does not match the regex.
+
+ Args:
+ message (str): Message to be monitored.
+ regex (str): regex pattern configured to ignore or restrict to.
+ tag (str): string with the configured tag. Values: 'ignore' or 'restrict'
+ prefix (str): Daemon that generates the error log.
+ """
+ log_found = False
+ with pytest.raises(TimeoutError):
+ log_found = check_ignore_restrict_message(message=message, regex=regex, tag=tag, prefix=prefix)
+ assert log_found is False, ERR_MSG_UNEXPECTED_IGNORE_EVENT

tests/integration/conftest.py

@@ -1017,10 +1017,14 @@ def set_wazuh_configuration(configuration):
  conf.write_wazuh_conf(backup_config)
 
 
-@pytest.fixture(scope='function')
+@pytest.fixture()
 def truncate_monitored_files():
  """Truncate all the log files and json alerts files before and after the test execution"""
- log_files = [LOG_FILE_PATH, ALERT_FILE_PATH]
+
+ if 'agent' in get_service():
+ log_files = [LOG_FILE_PATH]
+ else:
+ log_files = [LOG_FILE_PATH, ALERT_FILE_PATH]
 
  for log_file in log_files:
  truncate_file(log_file)

tests/integration/test_logcollector/test_log_filter_options/data/configuration_template/configuration_ignore_multiple_regex.yaml

@@ -0,0 +1,33 @@
+- sections:
+ - section: localfile
+ elements:
+ - log_format:
+ value: syslog
+ - location:
+ value: LOCATION
+ - ignore:
+ value: REGEX_1
+ - ignore:
+ value: REGEX_2
+
+ - section: sca
+ elements:
+ - enabled:
+ value: 'no'
+
+ - section: rootcheck
+ elements:
+ - disabled:
+ value: 'yes'
+
+ - section: syscheck
+ elements:
+ - disabled:
+ value: 'yes'
+
+ - section: wodle
+ attributes:
+ - name: syscollector
+ elements:
+ - disabled:
+ value: 'yes'

tests/integration/test_logcollector/test_log_filter_options/data/configuration_template/configuration_restrict_ignore_regex_values.yaml

@@ -0,0 +1,37 @@
+- sections:
+ - section: localfile
+ elements:
+ - log_format:
+ value: syslog
+ - location:
+ value: LOCATION
+ - restrict:
+ value: RESTRICT_REGEX
+ attributes:
+ - type: RESTRICT_TYPE
+ - ignore:
+ value: IGNORE_REGEX
+ attributes:
+ - type: IGNORE_TYPE
+
+ - section: sca
+ elements:
+ - enabled:
+ value: 'no'
+
+ - section: rootcheck
+ elements:
+ - disabled:
+ value: 'yes'
+
+ - section: syscheck
+ elements:
+ - disabled:
+ value: 'yes'
+
+ - section: wodle
+ attributes:
+ - name: syscollector
+ elements:
+ - disabled:
+ value: 'yes'

tests/integration/test_logcollector/test_log_filter_options/data/configuration_template/configuration_restrict_multiple_regex.yaml

@@ -0,0 +1,33 @@
+- sections:
+ - section: localfile
+ elements:
+ - log_format:
+ value: syslog
+ - location:
+ value: LOCATION
+ - restrict:
+ value: REGEX_1
+ - restrict:
+ value: REGEX_2
+
+ - section: sca
+ elements:
+ - enabled:
+ value: 'no'
+
+ - section: rootcheck
+ elements:
+ - disabled:
+ value: 'yes'
+
+ - section: syscheck
+ elements:
+ - disabled:
+ value: 'yes'
+
+ - section: wodle
+ attributes:
+ - name: syscollector
+ elements:
+ - disabled:
+ value: 'yes'

tests/integration/test_logcollector/test_log_filter_options/data/test_cases/cases_ignore_multiple_regex.yaml

@@ -0,0 +1,43 @@
+- name: Log match - Two ignore tags - Match first tag
+ description: Test two Ignore tags, with matching log first tag
+ configuration_parameters:
+ REGEX_1: .+regex1
+ REGEX_2: .+regex2
+ metadata:
+ regex1: .+regex1
+ regex2: .+regex2
+ log_sample: "Nov 10 12:19:04 localhost sshd: log matches regex1"
+ matches: regex1
+
+- name: Log match - Two ignore tags - Match both tags
+ description: Test two Ignore tags, with matching log both tags
+ configuration_parameters:
+ REGEX_1: .+regex1
+ REGEX_2: .+regex2
+ metadata:
+ regex1: .+regex1
+ regex2: .+regex2
+ log_sample: "Nov 10 12:19:04 localhost sshd: log matches regex1 regex2"
+ matches: regex1 regex2
+
+- name: Log match - Two ignore tags - Match second tag
+ description: Test two Ignore tags, with matching log second tag
+ configuration_parameters:
+ REGEX_1: .+regex1
+ REGEX_2: .+regex2
+ metadata:
+ regex1: .+regex1
+ regex2: .+regex2
+ log_sample: "Nov 10 12:19:04 localhost sshd: log matches regex2"
+ matches: regex2
+
+- name: No match - Two ignore tags
+ description: Test two Ignore tags, with no matches
+ configuration_parameters:
+ REGEX_1: .+regex1
+ REGEX_2: .+regex2
+ metadata:
+ regex1: .+regex1
+ regex2: .+regex2
+ log_sample: "Nov 10 12:19:04 localhost sshd: log does not matches"
+ matches: no match

tests/integration/test_logcollector/test_log_filter_options/data/test_cases/cases_restrict_ignore_regex_values.yaml

@@ -0,0 +1,83 @@
+# Test PCRE2
+- name: Matches with restrict tag - Ignore and restrict tags with PCRE2 regex
+ description: Test Restrict + Ignore tags both with PCRE2 regex. Log matches restrict
+ configuration_parameters:
+ RESTRICT_REGEX: .*restrict
+ IGNORE_REGEX: .*ignore
+ RESTRICT_TYPE: PCRE2
+ IGNORE_TYPE: PCRE2
+ metadata:
+ restrict_regex: .*restrict
+ ignore_regex: .*ignore
+ log_sample: "Nov 10 12:19:04 localhost sshd: log matches restrict"
+ matches: restrict
+
+# Test osregex
+- name: Matches with ignore tag - Ignore and restrict tags with osregex regex
+ description: Test Restrict + Ignore tags both with osregex regex. Log matches ignore
+ configuration_parameters:
+ RESTRICT_REGEX: \.restrict
+ IGNORE_REGEX: \.ignore
+ RESTRICT_TYPE: osregex
+ IGNORE_TYPE: osregex
+ metadata:
+ restrict_regex: \\.restrict
+ ignore_regex: \\.ignore
+ log_sample: "Nov 10 12:19:04 localhost sshd: log matches ignore"
+ matches: ignore
+
+# Test osmatch
+- name: Log match - Ignore and restrict tags with osmatch regex
+ description: Test Restrict + Ignore tags both with osmatch regex. Log matches both
+ configuration_parameters:
+ RESTRICT_REGEX: restrict$
+ IGNORE_REGEX: ignore
+ RESTRICT_TYPE: osmatch
+ IGNORE_TYPE: osmatch
+ metadata:
+ restrict_regex: restrict\$
+ ignore_regex: ignore
+ log_sample: "Nov 10 12:19:04 localhost sshd: log matches ignore restrict"
+ matches: ignore - restrict
+
+# Test pcre2 + osregex
+- name: Matches with restrict tag - Ignore and restrict tags with pcre2 and osregex regex
+ description: Test Restrict + Ignore tags (pcre2+osregex). Log matches restrict
+ configuration_parameters:
+ RESTRICT_REGEX: .*restrict
+ IGNORE_REGEX: \.ignore
+ RESTRICT_TYPE: pcre2
+ IGNORE_TYPE: osregex
+ metadata:
+ restrict_regex: .*restrict
+ ignore_regex: \\.ignore
+ log_sample: "Nov 10 12:19:04 localhost sshd: log matches restrict"
+ matches: restrict
+
+# Test pcre2 + osmatch
+- name: Matches with ignore tag - Ignore and restrict tags with pcre2 and osmatch regex
+ description: Test Restrict + Ignore tags (pcre2+osmatch). Log matches ignore
+ configuration_parameters:
+ RESTRICT_REGEX: .*restrict
+ IGNORE_REGEX: ignore
+ RESTRICT_TYPE: pcre2
+ IGNORE_TYPE: osmatch
+ metadata:
+ restrict_regex: .*restrict
+ ignore_regex: ignore
+ log_sample: "Nov 10 12:19:04 localhost sshd: log matches ignore"
+ matches: ignore
+
+# Test osmatch + osregex
+- name: Log match - Ignore and restrict tags with osregex and osmatch regex
+ description: Test Restrict + Ignore tags (osregex+osmatch). Log matches both
+ configuration_parameters:
+ RESTRICT_REGEX: \.restrict
+ IGNORE_REGEX: ignore
+ RESTRICT_TYPE: osregex
+ IGNORE_TYPE: osmatch
+ metadata:
+ restrict_regex: \\.restrict
+ ignore_regex: ignore
+ log_sample: "Nov 10 12:19:04 localhost sshd: log matches ignore restrict"
+ matches: ignore - restrict