Server Access integration tests (#3771)

* feat(#3344): Add basic test for aws server access integration * feat(#3344): Add data generator for Server Access type * feat(#3344): Add remove_from_bucket test for aws SA integration * feat(#3344): Add schema for ServerAccess table * feat(#3344): Add only_logs_after tests for aws SA integration * feat(#3344): Add path tests for aws SA integration * feat(#3344): Add discard_regex tests for aws FA integration * feat(#3344): Add only_logs_after tier 1 for aws SA integration * feat(#3344): Use table_name instead of bucket_type * style(#3344): Apply suggestions from code review Co-authored-by: Facundo Dalmau <[email protected]> --------- Co-authored-by: Facundo Dalmau <[email protected]>
wazuh · Mar 13, 2023 · 2a604bc · 2a604bc
1 parent 95a58e5
commit 2a604bc
Show file tree

Hide file tree

Showing 13 changed files with 177 additions and 7 deletions.
diff --git a/deps/wazuh_testing/wazuh_testing/modules/aws/constants.py b/deps/wazuh_testing/wazuh_testing/modules/aws/constants.py
@@ -13,6 +13,7 @@
 FLOW_LOG_ID = "fl-0754d951c16f517fa"
 CONFIG = 'Config'
 ELASTIC_LOAD_BALANCING = 'elasticloadbalancing'
+SERVER_ACCESS_TABLE_NAME = 's3_server_access'
 
 EVENT_TIME_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
 PATH_DATE_FORMAT = '%Y/%m/%d'
@@ -41,3 +42,4 @@
 GUARD_DUTY_TYPE = 'guardduty'
 NATIVE_GUARD_DUTY_TYPE = 'native-guardduty'
 WAF_TYPE = 'waf'
+SERVER_ACCESS = 'server_access'
diff --git a/deps/wazuh_testing/wazuh_testing/modules/aws/data_generator.py b/deps/wazuh_testing/wazuh_testing/modules/aws/data_generator.py
@@ -1050,6 +1050,57 @@ def get_data_sample(self) -> str:
  )
 
 
+class ServerAccessDataGenerator(DataGenerator):
+ BASE_PATH = ''
+ BASE_FILE_NAME = ''
+
+ def get_filename(self) -> str:
+ """Return the filename in the server access format.
+
+ Example:
+ <prefix>/
+
+ Returns:
+ str: Synthetic filename.
+ """
+ now = datetime.utcnow()
+ date_format = '%Y-%m-%d-%H-%M-%S'
+ name = f'{now.strftime(date_format)}-{get_random_string(16).upper()}'
+ return join(self.BASE_PATH, name)
+
+ def get_data_sample(self) -> str:
+ """Return a sample of data according to the server access format.
+
+ Returns:
+ str: Synthetic data.
+ """
+ data = []
+
+ for _ in range(5):
+ data.append(
+ [
+ str(uuid4()), 'wazuh-server-access-integration-tests',
+ datetime.utcnow().strftime('[%d/%b/%Y:%H:%M:%S %z]'), get_random_ip(),
+ f"arn:aws:iam::{cons.RANDOM_ACCOUNT_ID}:user/fake.user", get_random_string(16).upper(),
+ 'REST.GET.WEBSITE', '-', 'GET, /wazuh-server-access-integration-tests?website= HTTP/1.1',
+ '404', 'NoSuchWebsiteConfiguration', '343', '-', '85', '-', '-',
+ (
+ 'S3Console/0.4, aws-internal/3 aws-sdk-java/1.11.991'
+ 'Linux/4.9.230-0.1.ac.224.84.332.metal1.x86_64'
+ 'OpenJDK_64-Bit_Server_VM/25.282-b08 java/1.8.0_282 vendor/Oracle_Corporation'
+ 'cfg/retry-mode/legacy'
+ ),
+ '-', str(uuid4()), 'SigV4', 'ECDHE-RSA-AES128-GCM-SHA256', 'AuthHeader', 's3.amazonaws.com',
+ 'TLSv1.2'
+
+ ]
+ )
+ buffer = StringIO()
+ csv.writer(buffer, delimiter=" ").writerows(data)
+
+ return buffer.getvalue()
+
+
 # Maps bucket type with corresponding data generator
 buckets_data_mapping = {
  cons.CLOUD_TRAIL_TYPE: CloudTrailDataGenerator,
@@ -1064,6 +1115,7 @@ def get_data_sample(self) -> str:
  cons.GUARD_DUTY_TYPE: GuardDutyDataGenerator,
  cons.NATIVE_GUARD_DUTY_TYPE: NativeGuardDutyDataGenerator,
  cons.WAF_TYPE: WAFDataGenerator,
+ cons.SERVER_ACCESS: ServerAccessDataGenerator
 }
 
 

diff --git a/deps/wazuh_testing/wazuh_testing/modules/aws/db_utils.py b/deps/wazuh_testing/wazuh_testing/modules/aws/db_utils.py
@@ -12,7 +12,8 @@
  NLB_TYPE,
  S3_CLOUDTRAIL_DB_PATH,
  VPC_FLOW_TYPE,
- WAF_TYPE
+ WAF_TYPE,
+ SERVER_ACCESS_TABLE_NAME
 )
 
 SELECT_QUERY_TEMPLATE = 'SELECT * FROM {table_name}'
@@ -41,6 +42,10 @@
  'S3WAFRow', 'bucket_path aws_account_id log_key processed_date created_date'
 )
 
+S3ServerAccessRow = namedtuple(
+ 'S3ServerAccessRow', 'bucket_path aws_account_id log_key processed_date created_date'
+)
+
 s3_rows_map = {
  CLOUD_TRAIL_TYPE: S3CloudTrailRow,
  VPC_FLOW_TYPE: S3VPCFlowRow,
@@ -49,7 +54,8 @@
  NLB_TYPE: S3ALBRow,
  CUSTOM_TYPE: S3CustomRow,
  GUARD_DUTY_TYPE: S3GuardDutyRow,
- WAF_TYPE: S3WAFRow
+ WAF_TYPE: S3WAFRow,
+ SERVER_ACCESS_TABLE_NAME: S3ServerAccessRow
 }
 
 

diff --git a/deps/wazuh_testing/wazuh_testing/tools/utils.py b/deps/wazuh_testing/wazuh_testing/tools/utils.py
@@ -170,7 +170,7 @@ def get_host_name():
 
 def validate_interval_format(interval):
  """Validate that the interval passed has the format in which the last digit is a letter from those passed and
- the other characters are between 0-9"""
+ the other characters are between 0-9."""
  if interval == '':
  return False
  if interval[-1] not in ['s', 'm', 'h', 'd', 'w', 'y'] or not isinstance(int(interval[0:-1]), numbers.Number):

diff --git a/tests/integration/test_aws/data/test_cases/basic_test_module/cases_defaults.yaml b/tests/integration/test_aws/data/test_cases/basic_test_module/cases_defaults.yaml
@@ -105,3 +105,12 @@
  metadata:
  bucket_type: waf
  bucket_name: wazuh-waf-integration-tests
+
+- name: server_access_defaults
+ description: Server Access default configurations
+ configuration_parameters:
+ BUCKET_TYPE: server_access
+ BUCKET_NAME: wazuh-server-access-integration-tests
+ metadata:
+ bucket_type: server_access
+ bucket_name: wazuh-server-access-integration-tests
diff --git a/...s/integration/test_aws/data/test_cases/discard_regex_test_module/cases_discard_regex.yaml b/...s/integration/test_aws/data/test_cases/discard_regex_test_module/cases_discard_regex.yaml
@@ -189,3 +189,19 @@
  discard_regex: ALLOW
  found_logs: 3
  skipped_logs: 1
+
+- name: server_access_discard_regex
+ description: Server Access discard regex configurations
+ configuration_parameters:
+ BUCKET_TYPE: server_access
+ BUCKET_NAME: wazuh-server-access-integration-tests
+ DISCARD_FIELD: http_status
+ DISCARD_REGEX: '200'
+ metadata:
+ bucket_type: server_access
+ bucket_name: wazuh-server-access-integration-tests
+ only_logs_after: 2022-NOV-20
+ discard_field: http_status
+ discard_regex: '200'
+ found_logs: 3
+ skipped_logs: 1
diff --git a/...ntegration/test_aws/data/test_cases/only_logs_after_test_module/cases_multiple_calls.yaml b/...ntegration/test_aws/data/test_cases/only_logs_after_test_module/cases_multiple_calls.yaml
@@ -91,3 +91,10 @@
  metadata:
  bucket_type: waf
  bucket_name: wazuh-waf-integration-tests
+
+- name: server_access_only_logs_after_multiple_calls
+ description: Server Access only_logs_after multiple calls configurations
+ configuration_parameters:
+ metadata:
+ bucket_type: server_access
+ bucket_name: wazuh-server-access-integration-tests
diff --git a/...tion/test_aws/data/test_cases/only_logs_after_test_module/cases_with_only_logs_after.yaml b/...tion/test_aws/data/test_cases/only_logs_after_test_module/cases_with_only_logs_after.yaml
@@ -141,3 +141,16 @@
  bucket_name: wazuh-waf-integration-tests
  only_logs_after: 2022-NOV-20
  expected_results: 3
+
+- name: server_access_with_only_logs_after
+ description: Server Access only logs after configurations
+ configuration_parameters:
+ BUCKET_TYPE: server_access
+ BUCKET_NAME: wazuh-server-access-integration-tests
+ ONLY_LOGS_AFTER: 2022-NOV-20
+ metadata:
+ bucket_type: server_access
+ bucket_name: wazuh-server-access-integration-tests
+ only_logs_after: 2022-NOV-20
+ expected_results: 3
+ table_name: s3_server_access
diff --git a/...n/test_aws/data/test_cases/only_logs_after_test_module/cases_without_only_logs_after.yaml b/...n/test_aws/data/test_cases/only_logs_after_test_module/cases_without_only_logs_after.yaml
@@ -117,3 +117,14 @@
  bucket_type: waf
  bucket_name: wazuh-waf-integration-tests
  expected_results: 1
+
+- name: server_access_without_only_logs_after
+ description: Server Access only logs after configurations
+ configuration_parameters:
+ BUCKET_TYPE: server_access
+ BUCKET_NAME: wazuh-server-access-integration-tests
+ metadata:
+ bucket_type: server_access
+ bucket_name: wazuh-server-access-integration-tests
+ expected_results: 1
+ table_name: s3_server_access
diff --git a/tests/integration/test_aws/data/test_cases/path_test_module/cases_path.yaml b/tests/integration/test_aws/data/test_cases/path_test_module/cases_path.yaml
@@ -465,3 +465,45 @@
  only_logs_after: 2022-NOV-20
  path: inexistent_prefix
  expected_results: 0
+
+- name: server_access_path_with_data
+ description: Server Access path configurations
+ configuration_parameters:
+ BUCKET_TYPE: server_access
+ BUCKET_NAME: wazuh-server-access-integration-tests
+ PATH: test_prefix
+ metadata:
+ bucket_type: server_access
+ bucket_name: wazuh-server-access-integration-tests
+ only_logs_after: 2022-NOV-20
+ path: test_prefix
+ expected_results: 1
+ table_name: s3_server_access
+
+- name: server_access_path_without_data
+ description: Server Access path configurations
+ configuration_parameters:
+ BUCKET_TYPE: server_access
+ BUCKET_NAME: wazuh-server-access-integration-tests
+ PATH: empty_prefix
+ metadata:
+ bucket_type: server_access
+ bucket_name: wazuh-server-access-integration-tests
+ only_logs_after: 2022-NOV-20
+ path: empty_prefix
+ expected_results: 0
+ table_name: s3_server_access
+
+- name: server_access_inexistent_path
+ description: Server Access path configurations
+ configuration_parameters:
+ BUCKET_TYPE: server_access
+ BUCKET_NAME: wazuh-server-access-integration-tests
+ PATH: inexistent_prefix
+ metadata:
+ bucket_type: server_access
+ bucket_name: wazuh-server-access-integration-tests
+ only_logs_after: 2022-NOV-20
+ path: inexistent_prefix
+ expected_results: 0
+ table_name: s3_server_access
diff --git a/...ion/test_aws/data/test_cases/remove_from_bucket_test_module/cases_remove_from_bucket.yaml b/...ion/test_aws/data/test_cases/remove_from_bucket_test_module/cases_remove_from_bucket.yaml
@@ -105,3 +105,12 @@
  metadata:
  bucket_type: waf
  bucket_name: wazuh-waf-integration-tests
+
+- name: server_access_remove_from_bucket
+ description: Server Access remove from bucket configurations
+ configuration_parameters:
+ BUCKET_TYPE: server_access
+ BUCKET_NAME: wazuh-server-access-integration-tests
+ metadata:
+ bucket_type: server_access
+ bucket_name: wazuh-server-access-integration-tests
diff --git a/tests/integration/test_aws/test_only_logs_after.py b/tests/integration/test_aws/test_only_logs_after.py
@@ -106,6 +106,7 @@ def test_without_only_logs_after(
  bucket_name = metadata['bucket_name']
  bucket_type = metadata['bucket_type']
  expected_results = metadata['expected_results']
+ table_name = metadata.get('table_name', bucket_type)
 
  parameters = [
  'wodles/aws/aws-s3',
@@ -138,7 +139,7 @@ def test_without_only_logs_after(
 
  assert s3_db_exists()
 
- data = get_s3_db_row(table_name=bucket_type)
+ data = get_s3_db_row(table_name=table_name)
 
  assert bucket_name in data.bucket_path
  assert metadata['uploaded_file'] == data.log_key
@@ -218,6 +219,7 @@ def test_with_only_logs_after(
  bucket_type = metadata['bucket_type']
  only_logs_after = metadata['only_logs_after']
  expected_results = metadata['expected_results']
+ table_name = metadata.get('table_name', bucket_type)
 
  parameters = [
  'wodles/aws/aws-s3',
@@ -250,7 +252,7 @@ def test_with_only_logs_after(
 
  assert s3_db_exists()
 
- for row in get_multiple_s3_db_row(table_name=bucket_type):
+ for row in get_multiple_s3_db_row(table_name=table_name):
  assert bucket_name in row.bucket_path
  assert (
  datetime.strptime(only_logs_after, '%Y-%b-%d') < datetime.strptime(str(row.created_date), '%Y%m%d')

diff --git a/tests/integration/test_aws/test_path.py b/tests/integration/test_aws/test_path.py
@@ -98,6 +98,7 @@ def test_path(
  only_logs_after = metadata['only_logs_after']
  path = metadata['path']
  expected_results = metadata['expected_results']
+ table_name = metadata.get('table_name', bucket_type)
  pattern = fr".*WARNING: Bucket: - No files were found in '{bucket_name}/{path}/'. No logs will be processed.\n+"
 
  parameters = [
@@ -146,8 +147,8 @@ def test_path(
  assert s3_db_exists()
 
  if expected_results:
- data = get_s3_db_row(table_name=bucket_type)
+ data = get_s3_db_row(table_name=table_name)
  assert f"{bucket_name}/{path}/" == data.bucket_path
  assert data.log_key.startswith(f"{path}/")
  else:
- assert not table_exists_or_has_values(table_name=bucket_type)
+ assert not table_exists_or_has_values(table_name=table_name)