Fix get mappings view API incorrectly returning ecs path (opensearch-…

…project#867) * add logic and integ tests to not add duplicate to log-types-config index Signed-off-by: Joanne Wang <[email protected]> * change naming for existingFieldMapping and change contains to equals Signed-off-by: Joanne Wang <[email protected]> --------- Signed-off-by: Joanne Wang <[email protected]>
jowg-amazon · Mar 14, 2024 · 840a8bf · 840a8bf
1 parent 1b948ac
commit 840a8bf
Show file tree

Hide file tree

Showing 3 changed files with 148 additions and 23 deletions.
diff --git a/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java b/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java
@@ -348,15 +348,26 @@ private List<FieldMappingDoc> mergeFieldMappings(List<FieldMappingDoc> existingF
  List<FieldMappingDoc> newFieldMappings = new ArrayList<>();
  fieldMappingDocs.forEach( newFieldMapping -> {
  Optional<FieldMappingDoc> foundFieldMappingDoc = Optional.empty();
- for (FieldMappingDoc e: existingFieldMappings) {
- if (e.getRawField().equals(newFieldMapping.getRawField())) {
+ for (FieldMappingDoc existingFieldMapping: existingFieldMappings) {
+ if (existingFieldMapping.getRawField().equals(newFieldMapping.getRawField())) {
  if ((
- e.get(defaultSchemaField) != null && newFieldMapping.get(defaultSchemaField) != null &&
- e.get(defaultSchemaField).equals(newFieldMapping.get(defaultSchemaField))
+ existingFieldMapping.get(defaultSchemaField) != null && newFieldMapping.get(defaultSchemaField) != null &&
+ existingFieldMapping.get(defaultSchemaField).equals(newFieldMapping.get(defaultSchemaField))
  ) || (
- e.get(defaultSchemaField) == null && newFieldMapping.get(defaultSchemaField) == null
+ existingFieldMapping.get(defaultSchemaField) == null && newFieldMapping.get(defaultSchemaField) == null
  )) {
- foundFieldMappingDoc = Optional.of(e);
+ foundFieldMappingDoc = Optional.of(existingFieldMapping);
+ }
+ // Grabs the right side of the ID with "|" as the delimiter if present representing the ecs field from predefined mappings
+ // Additional check to see if raw field path + log type combination is already in existingFieldMappings so a new one is not indexed
+ } else {
+ String id = existingFieldMapping.getId();
+ int indexOfPipe = id.indexOf("|");
+ if (indexOfPipe != -1) {
+ String ecsIdField = id.substring(indexOfPipe + 1);
+ if (ecsIdField.equals(newFieldMapping.getRawField()) && existingFieldMapping.getLogTypes().containsAll(newFieldMapping.getLogTypes())) {
+ foundFieldMappingDoc = Optional.of(existingFieldMapping);
+ }
  }
  }
  }

diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java
@@ -239,8 +239,8 @@ public static String randomRule() {
  "level: high";
  }
 
- public static String randomNullRule() {
- return "title: null field\n" +
+ public static String randomRuleWithRawField() {
+ return "title: Remote Encrypting File System Abuse\n" +
  "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" +
  "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" +
  "references:\n" +
@@ -258,18 +258,50 @@ public static String randomNullRule() {
  "logsource:\n" +
  " product: rpc_firewall\n" +
  " category: application\n" +
- " definition: 'Requirements: install and apply the RPC Firew all to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" +
+ " definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" +
  "detection:\n" +
  " selection:\n" +
- " EventID: 22\n" +
- " RecordNumber: null\n" +
+ " eventName: testinghere\n" +
  " condition: selection\n" +
  "falsepositives:\n" +
  " - Legitimate usage of remote file encryption\n" +
  "level: high";
  }
 
- public static String randomRuleForMappingView(String field) {
+ public static String randomRuleWithNotCondition() {
+ return "title: Remote Encrypting File System Abuse\n" +
+ "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" +
+ "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" +
+ "references:\n" +
+ " - https://attack.mitre.org/tactics/TA0008/\n" +
+ " - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" +
+ " - https:/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" +
+ " - https:/zeronetworks/rpcfirewall\n" +
+ " - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" +
+ "tags:\n" +
+ " - attack.defense_evasion\n" +
+ "status: experimental\n" +
+ "author: Sagie Dulce, Dekel Paz\n" +
+ "date: 2022/01/01\n" +
+ "modified: 2022/01/01\n" +
+ "logsource:\n" +
+ " product: rpc_firewall\n" +
+ " category: application\n" +
+ " definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" +
+ "detection:\n" +
+ " selection1:\n" +
+ " AccountType: TestAccountType\n" +
+ " selection2:\n" +
+ " AccountName: TestAccountName\n" +
+ " selection3:\n" +
+ " EventID: 22\n" +
+ " condition: (not selection1 and not selection2) and selection3\n" +
+ "falsepositives:\n" +
+ " - Legitimate usage of remote file encryption\n" +
+ "level: high";
+ }
+
+ public static String randomRuleWithNotConditionBoolAndNum() {
  return "title: Remote Encrypting File System Abuse\n" +
  "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" +
  "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" +
@@ -290,8 +322,42 @@ public static String randomRuleForMappingView(String field) {
  " category: application\n" +
  " definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" +
  "detection:\n" +
+ " selection1:\n" +
+ " Initiated: \"false\"\n" +
+ " selection2:\n" +
+ " AccountName: TestAccountName\n" +
+ " selection3:\n" +
+ " EventID: 21\n" +
+ " condition: not selection1 and not selection3\n" +
+ "falsepositives:\n" +
+ " - Legitimate usage of remote file encryption\n" +
+ "level: high";
+ }
+
+ public static String randomNullRule() {
+ return "title: null field\n" +
+ "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" +
+ "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" +
+ "references:\n" +
+ " - https://attack.mitre.org/tactics/TA0008/\n" +
+ " - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" +
+ " - https:/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" +
+ " - https:/zeronetworks/rpcfirewall\n" +
+ " - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" +
+ "tags:\n" +
+ " - attack.defense_evasion\n" +
+ "status: experimental\n" +
+ "author: Sagie Dulce, Dekel Paz\n" +
+ "date: 2022/01/01\n" +
+ "modified: 2022/01/01\n" +
+ "logsource:\n" +
+ " product: rpc_firewall\n" +
+ " category: application\n" +
+ " definition: 'Requirements: install and apply the RPC Firew all to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" +
+ "detection:\n" +
  " selection:\n" +
- " "+ field + ": 'ACL'\n" +
+ " EventID: 22\n" +
+ " RecordNumber: null\n" +
  " condition: selection\n" +
  "falsepositives:\n" +
  " - Legitimate usage of remote file encryption\n" +
@@ -1635,31 +1701,31 @@ public static String windowsIndexMappingOnlyNumericAndText() {
  " }";
  }
 
- public static String randomDocWithNullField() {
-  return "{\n" +
-  "\"@timestamp\":\"2020-02-04T14:59:39.343541+00:00\",\n" +
+
+ public static String randomDoc(int severity, int version, String opCode) {
+ String doc = "{\n" +
  "\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" +
  "\"HostName\":\"EC2AMAZ-EPO7HKA\",\n" +
  "\"Keywords\":\"9223372036854775808\",\n" +
- "\"SeverityValue\":2,\n" +
+ "\"SeverityValue\":%s,\n" +
  "\"Severity\":\"INFO\",\n" +
  "\"EventID\":22,\n" +
  "\"SourceName\":\"Microsoft-Windows-Sysmon\",\n" +
  "\"ProviderGuid\":\"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}\",\n" +
- "\"Version\":5,\n" +
+ "\"Version\":%s,\n" +
  "\"TaskValue\":22,\n" +
  "\"OpcodeValue\":0,\n" +
- "\"RecordNumber\":null,\n" +
+ "\"RecordNumber\":9532,\n" +
  "\"ExecutionProcessID\":1996,\n" +
  "\"ExecutionThreadID\":2616,\n" +
  "\"Channel\":\"Microsoft-Windows-Sysmon/Operational\",\n" +
- "\"Domain\":\"NTAUTHORITY\",\n" +
+ "\"Domain\":\"NT AUTHORITY\",\n" +
  "\"AccountName\":\"SYSTEM\",\n" +
  "\"UserID\":\"S-1-5-18\",\n" +
  "\"AccountType\":\"User\",\n" +
  "\"Message\":\"Dns query:\\r\\nRuleName: \\r\\nUtcTime: 2020-02-04 14:59:38.349\\r\\nProcessGuid: {b3c285a4-3cda-5dc0-0000-001077270b00}\\r\\nProcessId: 1904\\r\\nQueryName: EC2AMAZ-EPO7HKA\\r\\nQueryStatus: 0\\r\\nQueryResults: 172.31.46.38;\\r\\nImage: C:\\\\Program Files\\\\nxlog\\\\nxlog.exe\",\n" +
  "\"Category\":\"Dns query (rule: DnsQuery)\",\n" +
- "\"Opcode\":\"Info\",\n" +
+ "\"Opcode\":\"%s\",\n" +
  "\"UtcTime\":\"2020-02-04 14:59:38.349\",\n" +
  "\"ProcessGuid\":\"{b3c285a4-3cda-5dc0-0000-001077270b00}\",\n" +
  "\"ProcessId\":\"1904\",\"QueryName\":\"EC2AMAZ-EPO7HKA\",\"QueryStatus\":\"0\",\n" +
@@ -1671,9 +1737,11 @@ public static String randomDocWithNullField() {
  "\"CommandLine\": \"eachtest\",\n" +
  "\"Initiated\": \"true\"\n" +
  "}";
+ return String.format(Locale.ROOT, doc, severity, version, opCode);
+
  }
 
- public static String randomDoc(int severity,  int version, String opCode) {
+ public static String randomDocForNotCondition(int severity, int version, String opCode) {
  String doc = "{\n" +
  "\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" +
  "\"HostName\":\"EC2AMAZ-EPO7HKA\",\n" +
@@ -1691,7 +1759,6 @@ public static String randomDoc(int severity, int version, String opCode) {
  "\"ExecutionThreadID\":2616,\n" +
  "\"Channel\":\"Microsoft-Windows-Sysmon/Operational\",\n" +
  "\"Domain\":\"NT AUTHORITY\",\n" +
- "\"AccountName\":\"SYSTEM\",\n" +
  "\"UserID\":\"S-1-5-18\",\n" +
  "\"AccountType\":\"User\",\n" +
  "\"Message\":\"Dns query:\\r\\nRuleName: \\r\\nUtcTime: 2020-02-04 14:59:38.349\\r\\nProcessGuid: {b3c285a4-3cda-5dc0-0000-001077270b00}\\r\\nProcessId: 1904\\r\\nQueryName: EC2AMAZ-EPO7HKA\\r\\nQueryStatus: 0\\r\\nQueryResults: 172.31.46.38;\\r\\nImage: C:\\\\Program Files\\\\nxlog\\\\nxlog.exe\",\n" +
@@ -1783,6 +1850,7 @@ public static String randomVpcFlowDoc() {
  " \"srcport\": 9000,\n" +
  " \"dstport\": 8000,\n" +
  " \"severity_id\": \"-1\",\n" +
+ " \"id.orig_h\": \"1.2.3.4\",\n" +
  " \"class_name\": \"Network Activity\"\n" +
  "}";
  }

diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java
@@ -439,6 +439,52 @@ public void testOCSFCloudtrailGetMappingsViewApi() throws IOException {
  assertEquals(24, unmappedFieldAliases.size());
  }
 
+ @SuppressWarnings("unchecked")
+ public void testOCSFCloudtrailGetMappingsViewApiWithCustomRule() throws IOException {
+ String index = createTestIndex("cloudtrail", ocsfCloudtrailMappings());
+
+ Request request = new Request("GET", SecurityAnalyticsPlugin.MAPPINGS_VIEW_BASE_URI);
+ // both req params and req body are supported
+ request.addParameter("index_name", index);
+ request.addParameter("rule_topic", "cloudtrail");
+ Response response = client().performRequest(request);
+ assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode());
+ Map<String, Object> respMap = responseAsMap(response);
+ // Verify alias mappings
+ Map<String, Object> props = (Map<String, Object>) respMap.get("properties");
+ Assert.assertEquals(18, props.size());
+ // Verify unmapped index fields
+ List<String> unmappedIndexFields = (List<String>) respMap.get("unmapped_index_fields");
+ assertEquals(20, unmappedIndexFields.size());
+ // Verify unmapped field aliases
+ List<String> unmappedFieldAliases = (List<String>) respMap.get("unmapped_field_aliases");
+ assertEquals(25, unmappedFieldAliases.size());
+
+ // create a cloudtrail rule with a raw field
+ String rule = randomRuleWithRawField();
+ Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "cloudtrail"),
+ new StringEntity(rule), new BasicHeader("Content-Type", "application/json"));
+ Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse));
+
+ // check the mapping view API again to ensure it's the same after rule is created
+ Response response2 = client().performRequest(request);
+ assertEquals(HttpStatus.SC_OK, response2.getStatusLine().getStatusCode());
+ Map<String, Object> respMap2 = responseAsMap(response2);
+ // Verify alias mappings
+ Map<String, Object> props2 = (Map<String, Object>) respMap2.get("properties");
+ Assert.assertEquals(18, props2.size());
+ // Verify unmapped index fields
+ List<String> unmappedIndexFields2 = (List<String>) respMap2.get("unmapped_index_fields");
+ assertEquals(20, unmappedIndexFields2.size());
+ // Verify unmapped field aliases
+ List<String> unmappedFieldAliases2 = (List<String>) respMap2.get("unmapped_field_aliases");
+ assertEquals(25, unmappedFieldAliases2.size());
+ // Verify that first response and second response are the same after rule was indexed
+ assertEquals(props, props2);
+ assertEquals(unmappedIndexFields, unmappedIndexFields2);
+ assertEquals(unmappedFieldAliases, unmappedFieldAliases2);
+ }
+
  @SuppressWarnings("unchecked")
  public void testOCSFVpcflowGetMappingsViewApi() throws IOException {
  String index = createTestIndex("vpcflow", ocsfVpcflowMappings());