From 872b14fe33756c50a249419164bc99e2641f85a7 Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Wed, 7 Feb 2024 17:36:40 -0800 Subject: [PATCH 01/15] test design Signed-off-by: Joanne Wang --- .../rules/backend/OSQueryBackend.java | 12 +- .../rules/backend/QueryBackend.java | 37 ++++- .../securityanalytics/TestHelpers.java | 62 +++++++++ .../resthandler/DetectorMonitorRestApiIT.java | 119 +++++++++++++--- .../rules/backend/QueryBackendTests.java | 128 +++++++++++++++++- 5 files changed, 330 insertions(+), 28 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java index 2d1763a43..399cb790e 100644 --- a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java +++ b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java @@ -57,6 +57,8 @@ public class OSQueryBackend extends QueryBackend { private String notToken; + private String existsToken; + private String escapeChar; private String wildcardMulti; @@ -119,6 +121,7 @@ public OSQueryBackend(Map fieldMappings, boolean collectErrors, this.orToken = "OR"; this.andToken = "AND"; this.notToken = "NOT"; + this.existsToken = "_exists_"; this.escapeChar = "\\"; this.wildcardMulti = "*"; this.wildcardSingle = "?"; @@ -250,7 +253,7 @@ public Object convertConditionNot(ConditionNOT condition) { return String.format(Locale.getDefault(), groupExpression, this.notToken + this.tokenSeparator + this.convertConditionGroup(argType)); } else if (arg.getLeft().isMiddle()) { ConditionType argType = new ConditionType(Either.right(Either.left(arg.getLeft().getMiddle()))); - return String.format(Locale.getDefault(), groupExpression, this.notToken + this.tokenSeparator + this.convertCondition(argType).toString()); + return String.format(Locale.getDefault(), groupExpression, this.convertConditionFieldEqValNot(argType).toString() + this.notToken + this.tokenSeparator + this.convertCondition(argType).toString()); } else { ConditionType argType = new ConditionType(Either.right(Either.right(arg.getLeft().get()))); return String.format(Locale.getDefault(), groupExpression, this.notToken + this.tokenSeparator + this.convertCondition(argType).toString()); @@ -269,9 +272,14 @@ public Object convertConditionFieldEqValStr(ConditionFieldEqualsValueExpression String expr = "%s" + this.eqToken + " " + (containsWildcard? this.reQuote: this.strQuote) + "%s" + (containsWildcard? this.reQuote: this.strQuote); String field = getFinalField(condition.getField()); - ruleQueryFields.put(field, Map.of("type", "text", "analyzer", "rule_analyzer")); + ruleQueryFields.put(field, Map.of("type", "text", "analyzer", "rule_analyzer")); //check this return String.format(Locale.getDefault(), expr, field, this.convertValueStr(value)); } + @Override + public Object convertExistsFieldStr(ConditionFieldEqualsValueExpression condition) { + String field = getFinalField(condition.getField()); + return this.existsToken + this.eqToken + " " + field + tokenSeparator + this.andToken + this.tokenSeparator; + } @Override public Object convertConditionFieldEqValNum(ConditionFieldEqualsValueExpression condition) { diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java index c63dce05d..2799ad7e1 100644 --- a/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java +++ b/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java @@ -128,7 +128,8 @@ public Object convertCondition(ConditionType conditionType) throws SigmaValueErr } } else if (conditionType.isConditionNOT()) { return this.convertConditionNot(conditionType.getConditionNOT()); - } else if (conditionType.isEqualsValueExpression()) { + } else if (conditionType.isEqualsValueExpression()) { // would HAVE to be done here... + // add a check to see if it should be done, then call antoher method to add them together else, return as normal BUT the check needs to see if top parent is NOT return this.convertConditionFieldEqVal(conditionType.getEqualsValueExpression()); } else if (conditionType.isValueExpression()) { return this.convertConditionVal(conditionType.getValueExpression()); @@ -136,6 +137,13 @@ public Object convertCondition(ConditionType conditionType) throws SigmaValueErr throw new IllegalArgumentException("Unexpected data type in condition parse tree"); } } + public Object convertConditionFieldEqValNot(ConditionType conditionType) throws SigmaValueError { + if (conditionType.isEqualsValueExpression()) { + return this.convertConditionFieldEqValNOT(conditionType.getEqualsValueExpression()); + } else { + throw new IllegalArgumentException("Unexpected data type in condition parse tree"); + } + } public boolean decideConvertConditionAsInExpression(Either condition) { if ((!this.convertOrAsIn && condition.isRight()) || (!this.convertAndAsIn && condition.isLeft())) { @@ -214,6 +222,31 @@ else if (condition.getValue() instanceof SigmaQueryExpression) { } } + public Object convertConditionFieldEqValNOT(ConditionFieldEqualsValueExpression condition) throws SigmaValueError { + if (condition.getValue() instanceof SigmaString) { + return this.convertExistsFieldStr(condition); + } else if (condition.getValue() instanceof SigmaNumber) { + return this.convertExistsFieldStr(condition); + } else if (condition.getValue() instanceof SigmaBool) { + return this.convertExistsFieldStr(condition); + } else if (condition.getValue() instanceof SigmaRegularExpression) { + return this.convertExistsFieldStr(condition); + } else if (condition.getValue() instanceof SigmaCIDRExpression) { + return this.convertExistsFieldStr(condition); + } else if (condition.getValue() instanceof SigmaCompareExpression) { + return this.convertExistsFieldStr(condition); + } else if (condition.getValue() instanceof SigmaNull) { + return this.convertExistsFieldStr(condition); + }/* TODO: below methods will be supported when Sigma Expand Modifier is supported. + else if (condition.getValue() instanceof SigmaQueryExpression) { + return this.convertConditionFieldEqValQueryExpr(condition); + }*/ else if (condition.getValue() instanceof SigmaExpansion) { + return this.convertExistsFieldStr(condition); + } else { + throw new IllegalArgumentException("Unexpected value type class in condition parse tree: " + condition.getValue().getClass().getName()); + } + } + public abstract Object convertConditionFieldEqValStr(ConditionFieldEqualsValueExpression condition) throws SigmaValueError; public abstract Object convertConditionFieldEqValNum(ConditionFieldEqualsValueExpression condition); @@ -228,6 +261,8 @@ else if (condition.getValue() instanceof SigmaQueryExpression) { public abstract Object convertConditionFieldEqValNull(ConditionFieldEqualsValueExpression condition); + public abstract Object convertExistsFieldStr(ConditionFieldEqualsValueExpression condition); + /* public abstract Object convertConditionFieldEqValQueryExpr(ConditionFieldEqualsValueExpression condition);*/ public Object convertConditionFieldEqValQueryExpansion(ConditionFieldEqualsValueExpression condition) { diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index d907b797c..20ed1025f 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -494,6 +494,68 @@ public static String randomRuleWithDateKeywords() { "level: high"; } + public static String randomRuleWithNotConditionInvalidField() { + return "title: Remote Encrypting File System Abuse\n" + + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + + "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" + + "references:\n" + + " - https://attack.mitre.org/tactics/TA0008/\n" + + " - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" + + " - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" + + " - https://github.com/zeronetworks/rpcfirewall\n" + + " - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" + + "tags:\n" + + " - attack.defense_evasion\n" + + "status: experimental\n" + + "author: Sagie Dulce, Dekel Paz\n" + + "date: 2022/01/01\n" + + "modified: 2022/01/01\n" + + "logsource:\n" + + " product: rpc_firewall\n" + + " category: application\n" + + " definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" + + "detection:\n" + + " selection1:\n" + + " randomField: value1\n" + + " selection2:\n" + + " EventID: 21\n" + + " condition: not selection1 and selection2\n" + + "falsepositives:\n" + + " - Legitimate usage of remote file encryption\n" + + "level: high"; + } + + public static String randomRuleWithNotConditionValidField() { + return "title: Remote Encrypting File System Abuse\n" + + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + + "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" + + "references:\n" + + " - https://attack.mitre.org/tactics/TA0008/\n" + + " - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" + + " - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" + + " - https://github.com/zeronetworks/rpcfirewall\n" + + " - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" + + "tags:\n" + + " - attack.defense_evasion\n" + + "status: experimental\n" + + "author: Sagie Dulce, Dekel Paz\n" + + "date: 2022/01/01\n" + + "modified: 2022/01/01\n" + + "logsource:\n" + + " product: rpc_firewall\n" + + " category: application\n" + + " definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" + + "detection:\n" + + " selection1:\n" + + " EventID: 10\n" + + " selection2:\n" + + " Severity: INFO\n" + + " condition: not selection1 and selection2\n" + + "falsepositives:\n" + + " - Legitimate usage of remote file encryption\n" + + "level: high"; + } + public static String countAggregationTestRule() { return " title: Test\n" + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java index 3a11300ee..1b127b207 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java @@ -37,28 +37,7 @@ import static java.util.Collections.emptyList; -import static org.opensearch.securityanalytics.TestHelpers.cloudtrailOcsfMappings; -import static org.opensearch.securityanalytics.TestHelpers.randomAggregationRule; -import static org.opensearch.securityanalytics.TestHelpers.randomCloudtrailAggrRule; -import static org.opensearch.securityanalytics.TestHelpers.randomCloudtrailAggrRuleWithDotFields; -import static org.opensearch.securityanalytics.TestHelpers.randomCloudtrailAggrRuleWithEcsFields; -import static org.opensearch.securityanalytics.TestHelpers.randomCloudtrailDoc; -import static org.opensearch.securityanalytics.TestHelpers.randomCloudtrailOcsfDoc; -import static org.opensearch.securityanalytics.TestHelpers.randomDetector; -import static org.opensearch.securityanalytics.TestHelpers.randomDetectorType; -import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputs; -import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputsAndTriggers; -import static org.opensearch.securityanalytics.TestHelpers.randomDoc; -import static org.opensearch.securityanalytics.TestHelpers.randomIndex; -import static org.opensearch.securityanalytics.TestHelpers.randomRule; -import static org.opensearch.securityanalytics.TestHelpers.randomRuleWithKeywords; -import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; -import static org.opensearch.securityanalytics.TestHelpers.randomRuleWithStringKeywords; -import static org.opensearch.securityanalytics.TestHelpers.randomDocOnlyNumericAndDate; -import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMappingOnlyNumericAndDate; -import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMappingOnlyNumericAndText; -import static org.opensearch.securityanalytics.TestHelpers.randomRuleWithDateKeywords; -import static org.opensearch.securityanalytics.TestHelpers.randomDocOnlyNumericAndText; +import static org.opensearch.securityanalytics.TestHelpers.*; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE; public class DetectorMonitorRestApiIT extends SecurityAnalyticsRestTestCase { @@ -2139,6 +2118,102 @@ public void testCreateDetectorWithCloudtrailAggrRuleWithEcsFields() throws IOExc assertEquals(1, getFindingsBody.get("total_findings")); } + public void testCreateDetectorWithNotCondition_verifyFindings_success() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response createMappingResponse = client().performRequest(createMappingRequest); + + assertEquals(HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode()); + + // Create random doc rule + String randomDocRuleId = createRule(randomRuleWithNotConditionInvalidField()); + String randomDocRuleId2 = createRule(randomRuleWithNotConditionValidField()); + + DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(randomDocRuleId), new DetectorRule(randomDocRuleId2)), + emptyList()); + Detector detector = randomDetectorWithInputs(List.of(input)); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map updateResponseBody = asMap(createResponse); + String detectorId = updateResponseBody.get("_id").toString(); + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + + // Verify newly created doc level monitor + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + Map detectorAsMap = (Map) hit.getSourceAsMap().get("detector"); + List monitorIds = ((List) (detectorAsMap).get("monitor_id")); + + assertEquals(1, monitorIds.size()); + + String monitorId = monitorIds.get(0); + String monitorType = ((Map) entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + monitorId))).get("monitor")).get("monitor_type"); + + assertEquals(MonitorType.DOC_LEVEL_MONITOR.getValue(), monitorType); + + // Verify rules + request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + + assertEquals(2, response.getHits().getTotalHits().value); + + // Verify findings + indexDoc(index, "1", randomDoc(2, 5, "Test")); + indexDoc(index, "2", randomDoc(3, 5, "Test")); + + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + // Verify 1 custom rule matches + assertEquals(1, noOfSigmaRuleMatches); + + Map params = new HashMap<>(); + params.put("detector_id", detectorId); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + + assertNotNull(getFindingsBody); + // When doc level monitor is being applied one finding is generated per document + assertEquals(2, getFindingsBody.get("total_findings")); + + List> findings = (List) getFindingsBody.get("findings"); + List foundDocIds = new ArrayList<>(); + for (Map finding : findings) { + Set aggRulesFinding = ((List>) finding.get("queries")).stream().map(it -> it.get("id").toString()).collect( + Collectors.toSet()); + + List findingDocs = (List) finding.get("related_doc_ids"); + Assert.assertEquals(1, findingDocs.size()); + foundDocIds.addAll(findingDocs); + } + assertTrue(Arrays.asList("1", "2").containsAll(foundDocIds)); + } + private static void assertRuleMonitorFinding(Map executeResults, String ruleId, int expectedDocCount, List expectedTriggerResult) { List> buckets = ((List>) (((Map) ((Map) ((Map) ((List) ((Map) executeResults.get("input_results")).get("results")).get(0)).get("aggregations")).get("result_agg")).get("buckets"))); Integer docCount = buckets.stream().mapToInt(it -> (Integer) it.get("doc_count")).sum(); diff --git a/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java b/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java index aff11d913..9ea75c4b5 100644 --- a/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java +++ b/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java @@ -711,10 +711,132 @@ public void testConvertNot() throws IOException, SigmaError { " category: test_category\n" + " product: test_product\n" + " detection:\n" + - " sel:\n" + + " sel1:\n" + + " - resp_mime_types|contains: 'dosexec'\n" + + " - c-uri|endswith: '.exe'\n" + + " sel2:\n" + + " fieldB: value2\n" + + " condition: not (sel1 or sel2)", false)); + Assert.assertEquals("(_exists_: fieldA AND NOT fieldA: \"value1\")", queries.get(0).toString()); + } + + public void testConvertNotWithAnd() throws IOException, SigmaError { + OSQueryBackend queryBackend = testBackend(); + List queries = queryBackend.convertRule(SigmaRule.fromYaml( + " title: Test\n" + + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " selection:\n" + + " EventType: SetValue\n" + + " TargetObject|endswith: '\\Software\\Microsoft\\WAB\\DLLPath'\n" + + " filter:\n" + + " Details: '%CommonProgramFiles%\\System\\wab32.dll'\n" + + " condition: selection and not filter", false)); + Assert.assertEquals("((EventType: \"SetValue\") AND (TargetObject: *\\\\Software\\\\Microsoft\\\\WAB\\\\DLLPath)) AND ((NOT Details: \"%CommonProgramFiles%\\\\System\\\\wab32.dll\" AND _exists_: \"Details\"))", queries.get(0).toString()); + } + + public void testConvertNotWithOrAndList() throws IOException, SigmaError { + OSQueryBackend queryBackend = testBackend(); + List queries = queryBackend.convertRule(SigmaRule.fromYaml( + " title: Test\n" + + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel1:\n" + + " fieldA1: valueA1\n" + + " fieldA2: valueA2\n" + + " fieldA3: valueA3\n" + + " sel2:\n" + + " fieldB: value2\n" + + " sel3:\n" + + " - resp_mime_types|contains: 'dosexec'\n" + + " - c-uri|endswith: '.exe'\n" + + " condition: not sel1 or not sel2", false)); + Assert.assertEquals("(NOT (fieldA: \"value1\"))", queries.get(0).toString()); + } + + public void testConvertNotWithNumAndBool() throws IOException, SigmaError { + OSQueryBackend queryBackend = testBackend(); + List queries = queryBackend.convertRule(SigmaRule.fromYaml( + " title: Test\n" + + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel1:\n" + + " fieldA: 1\n" + + " sel2:\n" + + " fieldB: true\n" + + " condition: not sel1 and not sel2", false)); + Assert.assertEquals("(NOT ((\"test1\") OR (\"test2\")))", queries.get(0).toString()); + } + + public void testConvertNotWithNull() throws IOException, SigmaError { + OSQueryBackend queryBackend = testBackend(); + List queries = queryBackend.convertRule(SigmaRule.fromYaml( + " title: Test\n" + + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel1:\n" + + " fieldA: null\n" + + " sel2:\n" + + " fieldB: true\n" + + " condition: not sel1", false)); + Assert.assertEquals("(NOT ((\"test1\") OR (\"test2\")))", queries.get(0).toString()); + } + + public void testConvertNotWithKeywords() throws IOException, SigmaError { + OSQueryBackend queryBackend = testBackend(); + List queries = queryBackend.convertRule(SigmaRule.fromYaml( + " title: Test\n" + + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel1:\n" + " fieldA: value1\n" + - " condition: not sel", false)); - Assert.assertEquals("(NOT fieldA: \"value1\")", queries.get(0).toString()); + " sel2:\n" + + " fieldB: value2\n" + + " keywords:\n" + + " - test1\n" + + " - test2\n" + + " condition: not keywords", false)); + Assert.assertEquals("(NOT ((\"test1\") OR (\"test2\")))", queries.get(0).toString()); } public void testConvertPrecedence() throws IOException, SigmaError { From 2ac748b4d44f847582e234a618715a1c8f1e25b7 Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Thu, 8 Feb 2024 09:40:33 -0800 Subject: [PATCH 02/15] working version Signed-off-by: Joanne Wang --- .../rules/backend/OSQueryBackend.java | 97 +++++++++++++------ .../rules/backend/QueryBackend.java | 84 ++++++---------- .../rules/backend/QueryBackendTests.java | 23 ++--- 3 files changed, 112 insertions(+), 92 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java index 399cb790e..ca8b2c949 100644 --- a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java +++ b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java @@ -148,15 +148,15 @@ public OSQueryBackend(Map fieldMappings, boolean collectErrors, } @Override - public Object convertConditionAsInExpression(Either condition) { + public Object convertConditionAsInExpression(Either condition, Boolean isNot, Boolean applyDeMorgans) { if (condition.isLeft()) { - return this.convertConditionAnd(condition.getLeft()); + return this.convertConditionAnd(condition.getLeft(), isNot, applyDeMorgans); } - return this.convertConditionOr(condition.get()); + return this.convertConditionOr(condition.get(), isNot, applyDeMorgans); } @Override - public Object convertConditionAnd(ConditionAND condition) { + public Object convertConditionAnd(ConditionAND condition, Boolean isNot, Boolean applyDeMorgans) { try { StringBuilder queryBuilder = new StringBuilder(); StringBuilder joiner = new StringBuilder(); @@ -174,21 +174,29 @@ public Object convertConditionAnd(ConditionAND condition) { ConditionType argType = arg.getLeft().getLeft().getClass().equals(ConditionAND.class)? new ConditionType(Either.left(AnyOneOf.leftVal((ConditionAND) arg.getLeft().getLeft()))): (arg.getLeft().getLeft().getClass().equals(ConditionOR.class)? new ConditionType(Either.left(AnyOneOf.middleVal((ConditionOR) arg.getLeft().getLeft()))): new ConditionType(Either.left(AnyOneOf.rightVal((ConditionNOT) arg.getLeft().getLeft())))); - converted = this.convertConditionGroup(argType); + converted = this.convertConditionGroup(argType, isNot,applyDeMorgans ); } else if (arg.getLeft().isMiddle()) { - converted = this.convertConditionGroup(new ConditionType(Either.right(Either.left(arg.getLeft().getMiddle())))); + converted = this.convertConditionGroup(new ConditionType(Either.right(Either.left(arg.getLeft().getMiddle()))), isNot, applyDeMorgans); } else if (arg.getLeft().isRight()) { - converted = this.convertConditionGroup(new ConditionType(Either.right(Either.right(arg.getLeft().get())))); + converted = this.convertConditionGroup(new ConditionType(Either.right(Either.right(arg.getLeft().get()))), isNot, applyDeMorgans); } - if (converted != null) { + if (applyDeMorgans) { +// String convertedWithNot = this.notToken + this.tokenSeparator + converted; + if (!first) { + String andJoiner = this.tokenSeparator + this.orToken + this.tokenSeparator; + queryBuilder.append(andJoiner).append(converted); + } else { + queryBuilder.append(converted); + first = false; + } + } else { if (!first) { queryBuilder.append(joiner).append(converted); } else { queryBuilder.append(converted); first = false; } - } } } @@ -199,7 +207,7 @@ public Object convertConditionAnd(ConditionAND condition) { } @Override - public Object convertConditionOr(ConditionOR condition) { + public Object convertConditionOr(ConditionOR condition, Boolean isNot, Boolean applyDeMorgans) { // if it's parent is NOT, then need to apply de morgans, another bool for deMorgans try { StringBuilder queryBuilder = new StringBuilder(); StringBuilder joiner = new StringBuilder(); @@ -217,21 +225,31 @@ public Object convertConditionOr(ConditionOR condition) { ConditionType argType = arg.getLeft().getLeft().getClass().equals(ConditionAND.class)? new ConditionType(Either.left(AnyOneOf.leftVal((ConditionAND) arg.getLeft().getLeft()))): (arg.getLeft().getLeft().getClass().equals(ConditionOR.class)? new ConditionType(Either.left(AnyOneOf.middleVal((ConditionOR) arg.getLeft().getLeft()))): new ConditionType(Either.left(AnyOneOf.rightVal((ConditionNOT) arg.getLeft().getLeft())))); - converted = this.convertConditionGroup(argType); + converted = this.convertConditionGroup(argType, isNot, applyDeMorgans); } else if (arg.getLeft().isMiddle()) { - converted = this.convertConditionGroup(new ConditionType(Either.right(Either.left(arg.getLeft().getMiddle())))); + converted = this.convertConditionGroup(new ConditionType(Either.right(Either.left(arg.getLeft().getMiddle()))), isNot, applyDeMorgans); } else if (arg.getLeft().isRight()) { - converted = this.convertConditionGroup(new ConditionType(Either.right(Either.right(arg.getLeft().get())))); + converted = this.convertConditionGroup(new ConditionType(Either.right(Either.right(arg.getLeft().get()))), isNot, applyDeMorgans); } - if (converted != null) { - if (!first) { - queryBuilder.append(joiner).append(converted); + if (converted != null) { // need to add some logic here to check if deMorgans is true, if it is then change the function + if (applyDeMorgans){ +// String convertedWithNot = this.notToken + this.tokenSeparator + converted; + if (!first) { + String andJoiner = this.tokenSeparator + this.andToken + this.tokenSeparator; + queryBuilder.append(andJoiner).append(converted); + } else { + queryBuilder.append(converted); + first = false; + } } else { - queryBuilder.append(converted); - first = false; + if (!first) { + queryBuilder.append(joiner).append(converted); + } else { + queryBuilder.append(converted); + first = false; + } } - } } } @@ -242,21 +260,21 @@ public Object convertConditionOr(ConditionOR condition) { } @Override - public Object convertConditionNot(ConditionNOT condition) { + public Object convertConditionNot(ConditionNOT condition, Boolean isNot, Boolean applyDeMorgans) { Either, String> arg = condition.getArgs().get(0); try { if (arg.isLeft()) { - if (arg.getLeft().isLeft()) { + if (arg.getLeft().isLeft()) { // if it every comes into HERE... THEN we will need to apply De Morgans ConditionType argType = arg.getLeft().getLeft().getClass().equals(ConditionAND.class) ? new ConditionType(Either.left(AnyOneOf.leftVal((ConditionAND) arg.getLeft().getLeft()))) : (arg.getLeft().getLeft().getClass().equals(ConditionOR.class) ? new ConditionType(Either.left(AnyOneOf.middleVal((ConditionOR) arg.getLeft().getLeft()))) : new ConditionType(Either.left(AnyOneOf.rightVal((ConditionNOT) arg.getLeft().getLeft())))); - return String.format(Locale.getDefault(), groupExpression, this.notToken + this.tokenSeparator + this.convertConditionGroup(argType)); + return String.format(Locale.getDefault(), groupExpression, this.convertConditionGroup(argType, true, true)); // mark TRUE that conditionNot is seen } else if (arg.getLeft().isMiddle()) { ConditionType argType = new ConditionType(Either.right(Either.left(arg.getLeft().getMiddle()))); - return String.format(Locale.getDefault(), groupExpression, this.convertConditionFieldEqValNot(argType).toString() + this.notToken + this.tokenSeparator + this.convertCondition(argType).toString()); + return String.format(Locale.getDefault(), groupExpression, this.notToken + this.tokenSeparator + this.convertCondition(argType, true, applyDeMorgans).toString()); // mark TRUE that conditionNot is seen } else { ConditionType argType = new ConditionType(Either.right(Either.right(arg.getLeft().get()))); - return String.format(Locale.getDefault(), groupExpression, this.notToken + this.tokenSeparator + this.convertCondition(argType).toString()); + return String.format(Locale.getDefault(), groupExpression, this.notToken + this.tokenSeparator + this.convertCondition(argType, true, applyDeMorgans).toString()); // mark TRUE that conditionNot is seen } } } catch (Exception ex) { @@ -266,19 +284,40 @@ public Object convertConditionNot(ConditionNOT condition) { } @Override - public Object convertConditionFieldEqValStr(ConditionFieldEqualsValueExpression condition) throws SigmaValueError { + public Object convertConditionFieldEqValStr(ConditionFieldEqualsValueExpression condition, Boolean isNot, Boolean applyDeMorgans) throws SigmaValueError { SigmaString value = (SigmaString) condition.getValue(); boolean containsWildcard = value.containsWildcard(); String expr = "%s" + this.eqToken + " " + (containsWildcard? this.reQuote: this.strQuote) + "%s" + (containsWildcard? this.reQuote: this.strQuote); + String exprWithDeMorgans = this.notToken + " " + "%s" + this.eqToken + " " + (containsWildcard? this.reQuote: this.strQuote) + "%s" + (containsWildcard? this.reQuote: this.strQuote); String field = getFinalField(condition.getField()); ruleQueryFields.put(field, Map.of("type", "text", "analyzer", "rule_analyzer")); //check this - return String.format(Locale.getDefault(), expr, field, this.convertValueStr(value)); +// String combinedExpr = "%s" + " " + this.andToken + " " + "%s"; +// String baseExpr = String.format(Locale.getDefault(), expr, field, this.convertValueStr(value)); +// +// // check and modify the query if ancestor is ConditionNot +// if (isNot) { +// String existsExpr = this.existsToken + this.eqToken + " " + field; +// try{ +// return String.format(Locale.getDefault(), combinedExpr, baseExpr, existsExpr); +// } +// catch (Exception ex) { +// throw new NotImplementedException("Type mismatch: strings cannot be combine"); +// } +// } +// else { +// return baseExpr; +// } + if (applyDeMorgans) { + return String.format(Locale.getDefault(), exprWithDeMorgans, field, this.convertValueStr(value)); + } else { + return String.format(Locale.getDefault(), expr, field, this.convertValueStr(value)); + } } @Override public Object convertExistsFieldStr(ConditionFieldEqualsValueExpression condition) { String field = getFinalField(condition.getField()); - return this.existsToken + this.eqToken + " " + field + tokenSeparator + this.andToken + this.tokenSeparator; + return tokenSeparator + this.andToken + this.tokenSeparator + this.existsToken + this.eqToken + " " + field; } @Override @@ -426,8 +465,8 @@ private boolean comparePrecedence(ConditionType outer, ConditionType inner) { return idxInner <= precedence.indexOf(outerClass); } - private Object convertConditionGroup(ConditionType condition) throws SigmaValueError { - return String.format(Locale.getDefault(), groupExpression, this.convertCondition(condition)); + private Object convertConditionGroup(ConditionType condition, Boolean isNot, Boolean applyDeMorgans) throws SigmaValueError { + return String.format(Locale.getDefault(), groupExpression, this.convertCondition(condition, isNot, applyDeMorgans)); } private Object convertValueStr(SigmaString s) throws SigmaValueError { diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java index 2799ad7e1..4e3b3b55a 100644 --- a/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java +++ b/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java @@ -85,15 +85,15 @@ public List convertRule(SigmaRule rule) throws SigmaError { Object query; if (conditionItem instanceof ConditionAND) { - query = this.convertCondition(new ConditionType(Either.left(AnyOneOf.leftVal((ConditionAND) conditionItem)))); + query = this.convertCondition(new ConditionType(Either.left(AnyOneOf.leftVal((ConditionAND) conditionItem))), false, false); } else if (conditionItem instanceof ConditionOR) { - query = this.convertCondition(new ConditionType(Either.left(AnyOneOf.middleVal((ConditionOR) conditionItem)))); + query = this.convertCondition(new ConditionType(Either.left(AnyOneOf.middleVal((ConditionOR) conditionItem))), false, false); } else if (conditionItem instanceof ConditionNOT) { - query = this.convertCondition(new ConditionType(Either.left(AnyOneOf.rightVal((ConditionNOT) conditionItem)))); + query = this.convertCondition(new ConditionType(Either.left(AnyOneOf.rightVal((ConditionNOT) conditionItem))), true, false); } else if (conditionItem instanceof ConditionFieldEqualsValueExpression) { - query = this.convertCondition(new ConditionType(Either.right(Either.left((ConditionFieldEqualsValueExpression) conditionItem)))); + query = this.convertCondition(new ConditionType(Either.right(Either.left((ConditionFieldEqualsValueExpression) conditionItem))), false, false); } else { - query = this.convertCondition(new ConditionType(Either.right(Either.right((ConditionValueExpression) conditionItem)))); + query = this.convertCondition(new ConditionType(Either.right(Either.right((ConditionValueExpression) conditionItem))), false, false); } queries.add(query); if (aggItem != null) { @@ -113,37 +113,38 @@ public List convertRule(SigmaRule rule) throws SigmaError { return queries; } - public Object convertCondition(ConditionType conditionType) throws SigmaValueError { + public Object convertCondition(ConditionType conditionType, boolean isNot, boolean applyDeMorgans) throws SigmaValueError { if (conditionType.isConditionOR()) { if (this.decideConvertConditionAsInExpression(Either.right(conditionType.getConditionOR()))) { - return this.convertConditionAsInExpression(Either.right(conditionType.getConditionOR())); + return this.convertConditionAsInExpression(Either.right(conditionType.getConditionOR()), isNot, applyDeMorgans ); } else { - return this.convertConditionOr(conditionType.getConditionOR()); + return this.convertConditionOr(conditionType.getConditionOR(), isNot, applyDeMorgans); } } else if (conditionType.isConditionAND()) { if (this.decideConvertConditionAsInExpression(Either.left(conditionType.getConditionAND()))) { - return this.convertConditionAsInExpression(Either.left(conditionType.getConditionAND())); + return this.convertConditionAsInExpression(Either.left(conditionType.getConditionAND()), isNot, applyDeMorgans); } else { - return this.convertConditionAnd(conditionType.getConditionAND()); + return this.convertConditionAnd(conditionType.getConditionAND(), isNot, applyDeMorgans); } } else if (conditionType.isConditionNOT()) { - return this.convertConditionNot(conditionType.getConditionNOT()); + return this.convertConditionNot(conditionType.getConditionNOT(), isNot, applyDeMorgans); } else if (conditionType.isEqualsValueExpression()) { // would HAVE to be done here... // add a check to see if it should be done, then call antoher method to add them together else, return as normal BUT the check needs to see if top parent is NOT - return this.convertConditionFieldEqVal(conditionType.getEqualsValueExpression()); + if (isNot){ + return String.format(Locale.getDefault(), this.convertConditionFieldEqVal(conditionType.getEqualsValueExpression(), isNot, applyDeMorgans).toString() + + this.convertConditionFieldEqValNOT(conditionType.getEqualsValueExpression()).toString()); + } else { + return this.convertConditionFieldEqVal(conditionType.getEqualsValueExpression(), isNot, applyDeMorgans); + } } else if (conditionType.isValueExpression()) { return this.convertConditionVal(conditionType.getValueExpression()); } else { throw new IllegalArgumentException("Unexpected data type in condition parse tree"); } } - public Object convertConditionFieldEqValNot(ConditionType conditionType) throws SigmaValueError { - if (conditionType.isEqualsValueExpression()) { - return this.convertConditionFieldEqValNOT(conditionType.getEqualsValueExpression()); - } else { - throw new IllegalArgumentException("Unexpected data type in condition parse tree"); - } - } +// public Object convertConditionFieldEqValNot(ConditionFieldEqualsValueExpression condition, Boolean isNot) throws SigmaValueError { +// return this.convertConditionFieldEqVal(condition, isNot).toString() + this.convertConditionFieldEqValNOT(condition).toString(); +// } public boolean decideConvertConditionAsInExpression(Either condition) { if ((!this.convertOrAsIn && condition.isRight()) || (!this.convertAndAsIn && condition.isLeft())) { @@ -189,17 +190,17 @@ public void resetQueryFields() { } } - public abstract Object convertConditionAsInExpression(Either condition); + public abstract Object convertConditionAsInExpression(Either condition, Boolean isNot, Boolean applyDeMorgans); - public abstract Object convertConditionAnd(ConditionAND condition); + public abstract Object convertConditionAnd(ConditionAND condition, Boolean isNot, Boolean applyDeMorgans); - public abstract Object convertConditionOr(ConditionOR condition); + public abstract Object convertConditionOr(ConditionOR condition, Boolean isNot, Boolean applyDeMorgans); - public abstract Object convertConditionNot(ConditionNOT condition); + public abstract Object convertConditionNot(ConditionNOT condition, Boolean isNot, Boolean applyDeMorgans); - public Object convertConditionFieldEqVal(ConditionFieldEqualsValueExpression condition) throws SigmaValueError { + public Object convertConditionFieldEqVal(ConditionFieldEqualsValueExpression condition, Boolean isNot, Boolean applyDeMorgans) throws SigmaValueError { if (condition.getValue() instanceof SigmaString) { - return this.convertConditionFieldEqValStr(condition); + return this.convertConditionFieldEqValStr(condition, isNot, applyDeMorgans); } else if (condition.getValue() instanceof SigmaNumber) { return this.convertConditionFieldEqValNum(condition); } else if (condition.getValue() instanceof SigmaBool) { @@ -216,38 +217,17 @@ public Object convertConditionFieldEqVal(ConditionFieldEqualsValueExpression con else if (condition.getValue() instanceof SigmaQueryExpression) { return this.convertConditionFieldEqValQueryExpr(condition); }*/ else if (condition.getValue() instanceof SigmaExpansion) { - return this.convertConditionFieldEqValQueryExpansion(condition); + return this.convertConditionFieldEqValQueryExpansion(condition, isNot, applyDeMorgans); } else { throw new IllegalArgumentException("Unexpected value type class in condition parse tree: " + condition.getValue().getClass().getName()); } } - public Object convertConditionFieldEqValNOT(ConditionFieldEqualsValueExpression condition) throws SigmaValueError { - if (condition.getValue() instanceof SigmaString) { - return this.convertExistsFieldStr(condition); - } else if (condition.getValue() instanceof SigmaNumber) { - return this.convertExistsFieldStr(condition); - } else if (condition.getValue() instanceof SigmaBool) { - return this.convertExistsFieldStr(condition); - } else if (condition.getValue() instanceof SigmaRegularExpression) { - return this.convertExistsFieldStr(condition); - } else if (condition.getValue() instanceof SigmaCIDRExpression) { - return this.convertExistsFieldStr(condition); - } else if (condition.getValue() instanceof SigmaCompareExpression) { - return this.convertExistsFieldStr(condition); - } else if (condition.getValue() instanceof SigmaNull) { - return this.convertExistsFieldStr(condition); - }/* TODO: below methods will be supported when Sigma Expand Modifier is supported. - else if (condition.getValue() instanceof SigmaQueryExpression) { - return this.convertConditionFieldEqValQueryExpr(condition); - }*/ else if (condition.getValue() instanceof SigmaExpansion) { - return this.convertExistsFieldStr(condition); - } else { - throw new IllegalArgumentException("Unexpected value type class in condition parse tree: " + condition.getValue().getClass().getName()); - } + public Object convertConditionFieldEqValNOT(ConditionFieldEqualsValueExpression condition) { + return this.convertExistsFieldStr(condition); } - public abstract Object convertConditionFieldEqValStr(ConditionFieldEqualsValueExpression condition) throws SigmaValueError; + public abstract Object convertConditionFieldEqValStr(ConditionFieldEqualsValueExpression condition, Boolean isNot, Boolean applyDeMorgans) throws SigmaValueError; public abstract Object convertConditionFieldEqValNum(ConditionFieldEqualsValueExpression condition); @@ -265,14 +245,14 @@ else if (condition.getValue() instanceof SigmaQueryExpression) { /* public abstract Object convertConditionFieldEqValQueryExpr(ConditionFieldEqualsValueExpression condition);*/ - public Object convertConditionFieldEqValQueryExpansion(ConditionFieldEqualsValueExpression condition) { + public Object convertConditionFieldEqValQueryExpansion(ConditionFieldEqualsValueExpression condition, Boolean isNot, Boolean applyDeMorgans) { List, String>> args = new ArrayList<>(); for (SigmaType sigmaType: ((SigmaExpansion) condition.getValue()).getValues()) { args.add(Either.left(AnyOneOf.middleVal(new ConditionFieldEqualsValueExpression(condition.getField(), sigmaType)))); } ConditionOR conditionOR = new ConditionOR(false, args); - return this.convertConditionOr(conditionOR); + return this.convertConditionOr(conditionOR, isNot, applyDeMorgans); } public Object convertConditionVal(ConditionValueExpression condition) throws SigmaValueError { diff --git a/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java b/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java index 9ea75c4b5..8bb54f68e 100644 --- a/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java +++ b/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java @@ -712,12 +712,13 @@ public void testConvertNot() throws IOException, SigmaError { " product: test_product\n" + " detection:\n" + " sel1:\n" + - " - resp_mime_types|contains: 'dosexec'\n" + - " - c-uri|endswith: '.exe'\n" + + " - Opcode: Info\n" + + " - Severity: INFO\n" + " sel2:\n" + - " fieldB: value2\n" + - " condition: not (sel1 or sel2)", false)); - Assert.assertEquals("(_exists_: fieldA AND NOT fieldA: \"value1\")", queries.get(0).toString()); + " Severity: value2\n" + + " Severity2: value3\n" + + " condition: not (sel1 and sel2)", false)); + Assert.assertEquals("((NOT Keywords: \"value1\" AND _exists_: Keywords)) AND ((NOT Severity: \"value2\" AND _exists_: Severity))", queries.get(0).toString()); } public void testConvertNotWithAnd() throws IOException, SigmaError { @@ -740,7 +741,7 @@ public void testConvertNotWithAnd() throws IOException, SigmaError { " filter:\n" + " Details: '%CommonProgramFiles%\\System\\wab32.dll'\n" + " condition: selection and not filter", false)); - Assert.assertEquals("((EventType: \"SetValue\") AND (TargetObject: *\\\\Software\\\\Microsoft\\\\WAB\\\\DLLPath)) AND ((NOT Details: \"%CommonProgramFiles%\\\\System\\\\wab32.dll\" AND _exists_: \"Details\"))", queries.get(0).toString()); + Assert.assertEquals("((EventType: \"SetValue\") AND (TargetObject: *\\\\Software\\\\Microsoft\\\\WAB\\\\DLLPath)) AND ((_exists_: Details AND NOT Details: \"%CommonProgramFiles%\\\\System\\\\wab32.dll\"))", queries.get(0).toString()); } public void testConvertNotWithOrAndList() throws IOException, SigmaError { @@ -766,7 +767,7 @@ public void testConvertNotWithOrAndList() throws IOException, SigmaError { " sel3:\n" + " - resp_mime_types|contains: 'dosexec'\n" + " - c-uri|endswith: '.exe'\n" + - " condition: not sel1 or not sel2", false)); + " condition: (sel1 and sel2) or sel3", false)); Assert.assertEquals("(NOT (fieldA: \"value1\"))", queries.get(0).toString()); } @@ -788,8 +789,8 @@ public void testConvertNotWithNumAndBool() throws IOException, SigmaError { " fieldA: 1\n" + " sel2:\n" + " fieldB: true\n" + - " condition: not sel1 and not sel2", false)); - Assert.assertEquals("(NOT ((\"test1\") OR (\"test2\")))", queries.get(0).toString()); + " condition: not (sel1 and sel2)", false)); + Assert.assertEquals("((_exists_: fieldA AND NOT fieldA: 1)) AND ((_exists_: mappedB AND NOT mappedB: true))", queries.get(0).toString()); } public void testConvertNotWithNull() throws IOException, SigmaError { @@ -811,7 +812,7 @@ public void testConvertNotWithNull() throws IOException, SigmaError { " sel2:\n" + " fieldB: true\n" + " condition: not sel1", false)); - Assert.assertEquals("(NOT ((\"test1\") OR (\"test2\")))", queries.get(0).toString()); + Assert.assertEquals("(_exists_: fieldA AND NOT fieldA: (NOT [* TO *]))", queries.get(0).toString()); } public void testConvertNotWithKeywords() throws IOException, SigmaError { @@ -862,7 +863,7 @@ public void testConvertPrecedence() throws IOException, SigmaError { " sel4:\n" + " fieldD: value5\n" + " condition: (sel1 or sel2) and not (sel3 and sel4)", false)); - Assert.assertEquals("((fieldA: \"value1\") OR (mappedB: \"value2\")) AND ((NOT ((fieldC: \"value4\") AND (fieldD: \"value5\"))))", queries.get(0).toString()); + Assert.assertEquals("((fieldA: \"value1\") OR (mappedB: \"value2\")) AND ((((NOT fieldC: \"value4\" AND _exists_: fieldC) OR (NOT fieldD: \"value5\" AND _exists_: fieldD))))", queries.get(0).toString()); } public void testConvertMultiConditions() throws IOException, SigmaError { From d0831f224d27cda537651023083a574a36068b94 Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Thu, 8 Feb 2024 15:55:16 -0800 Subject: [PATCH 03/15] cleaning up Signed-off-by: Joanne Wang --- .../rules/backend/OSQueryBackend.java | 125 ++++++++---------- .../rules/backend/QueryBackend.java | 63 +++++---- .../securityanalytics/TestHelpers.java | 6 +- .../resthandler/DetectorMonitorRestApiIT.java | 1 + .../rules/backend/QueryBackendTests.java | 44 +++++- 5 files changed, 133 insertions(+), 106 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java index ca8b2c949..5bcdcbf8e 100644 --- a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java +++ b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java @@ -4,6 +4,8 @@ */ package org.opensearch.securityanalytics.rules.backend; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.opensearch.OpenSearchParseException; import org.opensearch.common.UUIDs; import org.opensearch.core.common.bytes.BytesReference; @@ -48,7 +50,6 @@ import java.util.Map; public class OSQueryBackend extends QueryBackend { - private String tokenSeparator; private String orToken; @@ -148,15 +149,15 @@ public OSQueryBackend(Map fieldMappings, boolean collectErrors, } @Override - public Object convertConditionAsInExpression(Either condition, Boolean isNot, Boolean applyDeMorgans) { + public Object convertConditionAsInExpression(Either condition, boolean isConditionNot, boolean applyDeMorgans) { if (condition.isLeft()) { - return this.convertConditionAnd(condition.getLeft(), isNot, applyDeMorgans); + return this.convertConditionAnd(condition.getLeft(), isConditionNot, applyDeMorgans); } - return this.convertConditionOr(condition.get(), isNot, applyDeMorgans); + return this.convertConditionOr(condition.get(), isConditionNot, applyDeMorgans); } @Override - public Object convertConditionAnd(ConditionAND condition, Boolean isNot, Boolean applyDeMorgans) { + public Object convertConditionAnd(ConditionAND condition, boolean isConditionNot, boolean applyDeMorgans) { try { StringBuilder queryBuilder = new StringBuilder(); StringBuilder joiner = new StringBuilder(); @@ -174,23 +175,23 @@ public Object convertConditionAnd(ConditionAND condition, Boolean isNot, Boolean ConditionType argType = arg.getLeft().getLeft().getClass().equals(ConditionAND.class)? new ConditionType(Either.left(AnyOneOf.leftVal((ConditionAND) arg.getLeft().getLeft()))): (arg.getLeft().getLeft().getClass().equals(ConditionOR.class)? new ConditionType(Either.left(AnyOneOf.middleVal((ConditionOR) arg.getLeft().getLeft()))): new ConditionType(Either.left(AnyOneOf.rightVal((ConditionNOT) arg.getLeft().getLeft())))); - converted = this.convertConditionGroup(argType, isNot,applyDeMorgans ); + converted = this.convertConditionGroup(argType, isConditionNot,applyDeMorgans ); } else if (arg.getLeft().isMiddle()) { - converted = this.convertConditionGroup(new ConditionType(Either.right(Either.left(arg.getLeft().getMiddle()))), isNot, applyDeMorgans); + converted = this.convertConditionGroup(new ConditionType(Either.right(Either.left(arg.getLeft().getMiddle()))), isConditionNot, applyDeMorgans); } else if (arg.getLeft().isRight()) { - converted = this.convertConditionGroup(new ConditionType(Either.right(Either.right(arg.getLeft().get()))), isNot, applyDeMorgans); + converted = this.convertConditionGroup(new ConditionType(Either.right(Either.right(arg.getLeft().get()))), isConditionNot, applyDeMorgans); } - if (applyDeMorgans) { -// String convertedWithNot = this.notToken + this.tokenSeparator + converted; - if (!first) { - String andJoiner = this.tokenSeparator + this.orToken + this.tokenSeparator; - queryBuilder.append(andJoiner).append(converted); - } else { - queryBuilder.append(converted); - first = false; + if (converted != null) { + // if applyDeMorgans is true, then use OR instead of AND + if (applyDeMorgans) { + joiner.setLength(0); // clear the joiner to convert it to OR + if (this.tokenSeparator.equals(this.andToken)) { + joiner.append(this.orToken); + } else { + joiner.append(this.tokenSeparator).append(this.orToken).append(this.tokenSeparator); + } } - } else { if (!first) { queryBuilder.append(joiner).append(converted); } else { @@ -207,7 +208,7 @@ public Object convertConditionAnd(ConditionAND condition, Boolean isNot, Boolean } @Override - public Object convertConditionOr(ConditionOR condition, Boolean isNot, Boolean applyDeMorgans) { // if it's parent is NOT, then need to apply de morgans, another bool for deMorgans + public Object convertConditionOr(ConditionOR condition, boolean isConditionNot, boolean applyDeMorgans) { try { StringBuilder queryBuilder = new StringBuilder(); StringBuilder joiner = new StringBuilder(); @@ -225,56 +226,55 @@ public Object convertConditionOr(ConditionOR condition, Boolean isNot, Boolean a ConditionType argType = arg.getLeft().getLeft().getClass().equals(ConditionAND.class)? new ConditionType(Either.left(AnyOneOf.leftVal((ConditionAND) arg.getLeft().getLeft()))): (arg.getLeft().getLeft().getClass().equals(ConditionOR.class)? new ConditionType(Either.left(AnyOneOf.middleVal((ConditionOR) arg.getLeft().getLeft()))): new ConditionType(Either.left(AnyOneOf.rightVal((ConditionNOT) arg.getLeft().getLeft())))); - converted = this.convertConditionGroup(argType, isNot, applyDeMorgans); + converted = this.convertConditionGroup(argType, isConditionNot, applyDeMorgans); } else if (arg.getLeft().isMiddle()) { - converted = this.convertConditionGroup(new ConditionType(Either.right(Either.left(arg.getLeft().getMiddle()))), isNot, applyDeMorgans); + converted = this.convertConditionGroup(new ConditionType(Either.right(Either.left(arg.getLeft().getMiddle()))), isConditionNot, applyDeMorgans); } else if (arg.getLeft().isRight()) { - converted = this.convertConditionGroup(new ConditionType(Either.right(Either.right(arg.getLeft().get()))), isNot, applyDeMorgans); + converted = this.convertConditionGroup(new ConditionType(Either.right(Either.right(arg.getLeft().get()))), isConditionNot, applyDeMorgans); } - if (converted != null) { // need to add some logic here to check if deMorgans is true, if it is then change the function - if (applyDeMorgans){ -// String convertedWithNot = this.notToken + this.tokenSeparator + converted; - if (!first) { - String andJoiner = this.tokenSeparator + this.andToken + this.tokenSeparator; - queryBuilder.append(andJoiner).append(converted); + if (converted != null) { + // if applyDeMorgans is true, then use AND instead of OR + if (applyDeMorgans) { + joiner.setLength(0); // clear the joiner to convert it to AND + if (this.tokenSeparator.equals(this.orToken)) { + joiner.append(this.andToken); } else { - queryBuilder.append(converted); - first = false; + joiner.append(this.tokenSeparator).append(this.andToken).append(this.tokenSeparator); } + } + + if (!first) { + queryBuilder.append(joiner).append(converted); } else { - if (!first) { - queryBuilder.append(joiner).append(converted); - } else { - queryBuilder.append(converted); - first = false; - } + queryBuilder.append(converted); + first = false; } } } } return queryBuilder.toString(); } catch (Exception ex) { - throw new NotImplementedException("Operator 'and' not supported by the backend"); + throw new NotImplementedException("Operator 'or' not supported by the backend"); } } @Override - public Object convertConditionNot(ConditionNOT condition, Boolean isNot, Boolean applyDeMorgans) { + public Object convertConditionNot(ConditionNOT condition, boolean isConditionNot, boolean applyDeMorgans) { Either, String> arg = condition.getArgs().get(0); try { if (arg.isLeft()) { - if (arg.getLeft().isLeft()) { // if it every comes into HERE... THEN we will need to apply De Morgans + if (arg.getLeft().isLeft()) { ConditionType argType = arg.getLeft().getLeft().getClass().equals(ConditionAND.class) ? new ConditionType(Either.left(AnyOneOf.leftVal((ConditionAND) arg.getLeft().getLeft()))) : (arg.getLeft().getLeft().getClass().equals(ConditionOR.class) ? new ConditionType(Either.left(AnyOneOf.middleVal((ConditionOR) arg.getLeft().getLeft()))) : new ConditionType(Either.left(AnyOneOf.rightVal((ConditionNOT) arg.getLeft().getLeft())))); - return String.format(Locale.getDefault(), groupExpression, this.convertConditionGroup(argType, true, true)); // mark TRUE that conditionNot is seen + return String.format(Locale.getDefault(), groupExpression, this.convertConditionGroup(argType, true, true)); } else if (arg.getLeft().isMiddle()) { ConditionType argType = new ConditionType(Either.right(Either.left(arg.getLeft().getMiddle()))); - return String.format(Locale.getDefault(), groupExpression, this.notToken + this.tokenSeparator + this.convertCondition(argType, true, applyDeMorgans).toString()); // mark TRUE that conditionNot is seen + return String.format(Locale.getDefault(), groupExpression, this.notToken + this.tokenSeparator + this.convertCondition(argType, true, applyDeMorgans).toString()); } else { ConditionType argType = new ConditionType(Either.right(Either.right(arg.getLeft().get()))); - return String.format(Locale.getDefault(), groupExpression, this.notToken + this.tokenSeparator + this.convertCondition(argType, true, applyDeMorgans).toString()); // mark TRUE that conditionNot is seen + return String.format(Locale.getDefault(), groupExpression, this.notToken + this.tokenSeparator + this.convertCondition(argType, true, applyDeMorgans).toString()); } } } catch (Exception ex) { @@ -284,40 +284,25 @@ public Object convertConditionNot(ConditionNOT condition, Boolean isNot, Boolean } @Override - public Object convertConditionFieldEqValStr(ConditionFieldEqualsValueExpression condition, Boolean isNot, Boolean applyDeMorgans) throws SigmaValueError { + public Object convertExistsField(ConditionFieldEqualsValueExpression condition) { + String field = getFinalField(condition.getField()); + return String.format(Locale.getDefault(),tokenSeparator + this.andToken + this.tokenSeparator + this.existsToken + this.eqToken + " " + field); + } + + @Override + public Object convertConditionFieldEqValStr(ConditionFieldEqualsValueExpression condition, boolean isConditionNot, boolean applyDeMorgans) throws SigmaValueError { SigmaString value = (SigmaString) condition.getValue(); boolean containsWildcard = value.containsWildcard(); String expr = "%s" + this.eqToken + " " + (containsWildcard? this.reQuote: this.strQuote) + "%s" + (containsWildcard? this.reQuote: this.strQuote); - String exprWithDeMorgans = this.notToken + " " + "%s" + this.eqToken + " " + (containsWildcard? this.reQuote: this.strQuote) + "%s" + (containsWildcard? this.reQuote: this.strQuote); + String exprWithDeMorgansApplied = this.notToken + " " + "%s" + this.eqToken + " " + (containsWildcard? this.reQuote: this.strQuote) + "%s" + (containsWildcard? this.reQuote: this.strQuote); String field = getFinalField(condition.getField()); - ruleQueryFields.put(field, Map.of("type", "text", "analyzer", "rule_analyzer")); //check this -// String combinedExpr = "%s" + " " + this.andToken + " " + "%s"; -// String baseExpr = String.format(Locale.getDefault(), expr, field, this.convertValueStr(value)); -// -// // check and modify the query if ancestor is ConditionNot -// if (isNot) { -// String existsExpr = this.existsToken + this.eqToken + " " + field; -// try{ -// return String.format(Locale.getDefault(), combinedExpr, baseExpr, existsExpr); -// } -// catch (Exception ex) { -// throw new NotImplementedException("Type mismatch: strings cannot be combine"); -// } -// } -// else { -// return baseExpr; -// } + ruleQueryFields.put(field, Map.of("type", "text", "analyzer", "rule_analyzer")); + String convertedExpr = String.format(Locale.getDefault(), expr, field, this.convertValueStr(value)); if (applyDeMorgans) { - return String.format(Locale.getDefault(), exprWithDeMorgans, field, this.convertValueStr(value)); - } else { - return String.format(Locale.getDefault(), expr, field, this.convertValueStr(value)); + convertedExpr = String.format(Locale.getDefault(), exprWithDeMorgansApplied, this.existsToken, field); } - } - @Override - public Object convertExistsFieldStr(ConditionFieldEqualsValueExpression condition) { - String field = getFinalField(condition.getField()); - return tokenSeparator + this.andToken + this.tokenSeparator + this.existsToken + this.eqToken + " " + field; + return convertedExpr; } @Override @@ -465,8 +450,8 @@ private boolean comparePrecedence(ConditionType outer, ConditionType inner) { return idxInner <= precedence.indexOf(outerClass); } - private Object convertConditionGroup(ConditionType condition, Boolean isNot, Boolean applyDeMorgans) throws SigmaValueError { - return String.format(Locale.getDefault(), groupExpression, this.convertCondition(condition, isNot, applyDeMorgans)); + private Object convertConditionGroup(ConditionType condition, boolean isConditionNot, boolean applyDeMorgans) throws SigmaValueError { + return String.format(Locale.getDefault(), groupExpression, this.convertCondition(condition, isConditionNot, applyDeMorgans)); } private Object convertValueStr(SigmaString s) throws SigmaValueError { diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java index 4e3b3b55a..542bbfd9b 100644 --- a/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java +++ b/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java @@ -4,6 +4,8 @@ */ package org.opensearch.securityanalytics.rules.backend; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.opensearch.commons.alerting.aggregation.bucketselectorext.BucketSelectorExtAggregationBuilder; import org.opensearch.search.aggregations.AggregationBuilder; import org.opensearch.securityanalytics.rules.aggregation.AggregationItem; @@ -31,6 +33,8 @@ import org.opensearch.securityanalytics.rules.utils.AnyOneOf; import org.opensearch.securityanalytics.rules.utils.Either; import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.securityanalytics.threatIntel.DetectorThreatIntelService; +import org.opensearch.securityanalytics.util.RuleIndices; import org.yaml.snakeyaml.LoaderOptions; import org.yaml.snakeyaml.Yaml; import org.yaml.snakeyaml.constructor.SafeConstructor; @@ -48,6 +52,7 @@ public abstract class QueryBackend { + private static final Logger log = LogManager.getLogger(QueryBackend.class); private boolean convertOrAsIn; private boolean convertAndAsIn; private boolean collectErrors; @@ -96,6 +101,8 @@ public List convertRule(SigmaRule rule) throws SigmaError { query = this.convertCondition(new ConditionType(Either.right(Either.right((ConditionValueExpression) conditionItem))), false, false); } queries.add(query); + log.debug("converted query"); + log.debug(query); if (aggItem != null) { aggItem.setTimeframe(rule.getDetection().getTimeframe()); queries.add(convertAggregation(aggItem)); @@ -113,28 +120,29 @@ public List convertRule(SigmaRule rule) throws SigmaError { return queries; } - public Object convertCondition(ConditionType conditionType, boolean isNot, boolean applyDeMorgans) throws SigmaValueError { + public Object convertCondition(ConditionType conditionType, boolean isConditionNot, boolean applyDeMorgans) throws SigmaValueError { if (conditionType.isConditionOR()) { if (this.decideConvertConditionAsInExpression(Either.right(conditionType.getConditionOR()))) { - return this.convertConditionAsInExpression(Either.right(conditionType.getConditionOR()), isNot, applyDeMorgans ); + return this.convertConditionAsInExpression(Either.right(conditionType.getConditionOR()), isConditionNot, applyDeMorgans ); } else { - return this.convertConditionOr(conditionType.getConditionOR(), isNot, applyDeMorgans); + return this.convertConditionOr(conditionType.getConditionOR(), isConditionNot, applyDeMorgans); } } else if (conditionType.isConditionAND()) { if (this.decideConvertConditionAsInExpression(Either.left(conditionType.getConditionAND()))) { - return this.convertConditionAsInExpression(Either.left(conditionType.getConditionAND()), isNot, applyDeMorgans); + return this.convertConditionAsInExpression(Either.left(conditionType.getConditionAND()), isConditionNot, applyDeMorgans); } else { - return this.convertConditionAnd(conditionType.getConditionAND(), isNot, applyDeMorgans); + return this.convertConditionAnd(conditionType.getConditionAND(), isConditionNot, applyDeMorgans); } } else if (conditionType.isConditionNOT()) { - return this.convertConditionNot(conditionType.getConditionNOT(), isNot, applyDeMorgans); - } else if (conditionType.isEqualsValueExpression()) { // would HAVE to be done here... - // add a check to see if it should be done, then call antoher method to add them together else, return as normal BUT the check needs to see if top parent is NOT - if (isNot){ - return String.format(Locale.getDefault(), this.convertConditionFieldEqVal(conditionType.getEqualsValueExpression(), isNot, applyDeMorgans).toString() + - this.convertConditionFieldEqValNOT(conditionType.getEqualsValueExpression()).toString()); + return this.convertConditionNot(conditionType.getConditionNOT(), isConditionNot, applyDeMorgans); + } else if (conditionType.isEqualsValueExpression()) { + // add a check to see if it should be done, then call another method to add them together else, return as normal BUT the check needs to see if top parent is NOT + if (isConditionNot){ + String baseString = this.convertConditionFieldEqVal(conditionType.getEqualsValueExpression(), isConditionNot, applyDeMorgans).toString(); + String addExists = this.convertConditionFieldEqValNOT(conditionType.getEqualsValueExpression()).toString(); + return String.format(Locale.getDefault(), ("%s"+ "%s"), baseString, addExists); } else { - return this.convertConditionFieldEqVal(conditionType.getEqualsValueExpression(), isNot, applyDeMorgans); + return this.convertConditionFieldEqVal(conditionType.getEqualsValueExpression(), isConditionNot, applyDeMorgans); } } else if (conditionType.isValueExpression()) { return this.convertConditionVal(conditionType.getValueExpression()); @@ -142,9 +150,10 @@ public Object convertCondition(ConditionType conditionType, boolean isNot, boole throw new IllegalArgumentException("Unexpected data type in condition parse tree"); } } -// public Object convertConditionFieldEqValNot(ConditionFieldEqualsValueExpression condition, Boolean isNot) throws SigmaValueError { -// return this.convertConditionFieldEqVal(condition, isNot).toString() + this.convertConditionFieldEqValNOT(condition).toString(); -// } + + public Object convertConditionFieldEqValNot(ConditionFieldEqualsValueExpression condition, boolean isConditionNot, boolean applyDeMorgans) throws SigmaValueError { + return this.convertConditionFieldEqVal(condition, isConditionNot, applyDeMorgans).toString() + this.convertConditionFieldEqValNOT(condition).toString(); + } public boolean decideConvertConditionAsInExpression(Either condition) { if ((!this.convertOrAsIn && condition.isRight()) || (!this.convertAndAsIn && condition.isLeft())) { @@ -190,17 +199,17 @@ public void resetQueryFields() { } } - public abstract Object convertConditionAsInExpression(Either condition, Boolean isNot, Boolean applyDeMorgans); + public abstract Object convertConditionAsInExpression(Either condition, boolean isConditionNot, boolean applyDeMorgans); - public abstract Object convertConditionAnd(ConditionAND condition, Boolean isNot, Boolean applyDeMorgans); + public abstract Object convertConditionAnd(ConditionAND condition, boolean isConditionNot, boolean applyDeMorgans); - public abstract Object convertConditionOr(ConditionOR condition, Boolean isNot, Boolean applyDeMorgans); + public abstract Object convertConditionOr(ConditionOR condition, boolean isConditionNot, boolean applyDeMorgans); - public abstract Object convertConditionNot(ConditionNOT condition, Boolean isNot, Boolean applyDeMorgans); + public abstract Object convertConditionNot(ConditionNOT condition, boolean isConditionNot, boolean applyDeMorgans); - public Object convertConditionFieldEqVal(ConditionFieldEqualsValueExpression condition, Boolean isNot, Boolean applyDeMorgans) throws SigmaValueError { + public Object convertConditionFieldEqVal(ConditionFieldEqualsValueExpression condition, boolean isConditionNot, boolean applyDeMorgans) throws SigmaValueError { if (condition.getValue() instanceof SigmaString) { - return this.convertConditionFieldEqValStr(condition, isNot, applyDeMorgans); + return this.convertConditionFieldEqValStr(condition, isConditionNot, applyDeMorgans); } else if (condition.getValue() instanceof SigmaNumber) { return this.convertConditionFieldEqValNum(condition); } else if (condition.getValue() instanceof SigmaBool) { @@ -217,17 +226,17 @@ public Object convertConditionFieldEqVal(ConditionFieldEqualsValueExpression con else if (condition.getValue() instanceof SigmaQueryExpression) { return this.convertConditionFieldEqValQueryExpr(condition); }*/ else if (condition.getValue() instanceof SigmaExpansion) { - return this.convertConditionFieldEqValQueryExpansion(condition, isNot, applyDeMorgans); + return this.convertConditionFieldEqValQueryExpansion(condition, isConditionNot, applyDeMorgans); } else { throw new IllegalArgumentException("Unexpected value type class in condition parse tree: " + condition.getValue().getClass().getName()); } } public Object convertConditionFieldEqValNOT(ConditionFieldEqualsValueExpression condition) { - return this.convertExistsFieldStr(condition); + return this.convertExistsField(condition); } - public abstract Object convertConditionFieldEqValStr(ConditionFieldEqualsValueExpression condition, Boolean isNot, Boolean applyDeMorgans) throws SigmaValueError; + public abstract Object convertConditionFieldEqValStr(ConditionFieldEqualsValueExpression condition, boolean isConditionNot, boolean applyDeMorgans) throws SigmaValueError; public abstract Object convertConditionFieldEqValNum(ConditionFieldEqualsValueExpression condition); @@ -241,18 +250,18 @@ public Object convertConditionFieldEqValNOT(ConditionFieldEqualsValueExpression public abstract Object convertConditionFieldEqValNull(ConditionFieldEqualsValueExpression condition); - public abstract Object convertExistsFieldStr(ConditionFieldEqualsValueExpression condition); + public abstract Object convertExistsField(ConditionFieldEqualsValueExpression condition); /* public abstract Object convertConditionFieldEqValQueryExpr(ConditionFieldEqualsValueExpression condition);*/ - public Object convertConditionFieldEqValQueryExpansion(ConditionFieldEqualsValueExpression condition, Boolean isNot, Boolean applyDeMorgans) { + public Object convertConditionFieldEqValQueryExpansion(ConditionFieldEqualsValueExpression condition, boolean isConditionNot, boolean applyDeMorgans) { List, String>> args = new ArrayList<>(); for (SigmaType sigmaType: ((SigmaExpansion) condition.getValue()).getValues()) { args.add(Either.left(AnyOneOf.middleVal(new ConditionFieldEqualsValueExpression(condition.getField(), sigmaType)))); } ConditionOR conditionOR = new ConditionOR(false, args); - return this.convertConditionOr(conditionOR, isNot, applyDeMorgans); + return this.convertConditionOr(conditionOR, isConditionNot, applyDeMorgans); } public Object convertConditionVal(ConditionValueExpression condition) throws SigmaValueError { diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index 20ed1025f..5f4ef846f 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -547,10 +547,10 @@ public static String randomRuleWithNotConditionValidField() { " definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" + "detection:\n" + " selection1:\n" + - " EventID: 10\n" + + " AccountName: asdf\n" + " selection2:\n" + - " Severity: INFO\n" + - " condition: not selection1 and selection2\n" + + " Severity: asdf\n" + + " condition: not selection1\n" + "falsepositives:\n" + " - Legitimate usage of remote file encryption\n" + "level: high"; diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java index 1b127b207..25944b660 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java @@ -2190,6 +2190,7 @@ public void testCreateDetectorWithNotCondition_verifyFindings_success() throws I Map executeResults = entityAsMap(executeResponse); int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); // Verify 1 custom rule matches + assertEquals(1, noOfSigmaRuleMatches); Map params = new HashMap<>(); diff --git a/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java b/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java index 8bb54f68e..97e091748 100644 --- a/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java +++ b/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java @@ -712,13 +712,45 @@ public void testConvertNot() throws IOException, SigmaError { " product: test_product\n" + " detection:\n" + " sel1:\n" + - " - Opcode: Info\n" + - " - Severity: INFO\n" + + " Opcode: Info\n" + " sel2:\n" + " Severity: value2\n" + - " Severity2: value3\n" + - " condition: not (sel1 and sel2)", false)); - Assert.assertEquals("((NOT Keywords: \"value1\" AND _exists_: Keywords)) AND ((NOT Severity: \"value2\" AND _exists_: Severity))", queries.get(0).toString()); + " condition: not (sel1 or sel2)", false)); + Assert.assertEquals("(((NOT Opcode: \"Info\" AND _exists_: Opcode) AND (NOT Severity: \"value2\" AND _exists_: Severity)))", queries.get(0).toString()); + } + + public void testConvertNotTesting() throws IOException, SigmaError { + OSQueryBackend queryBackend = testBackend(); + List queries = queryBackend.convertRule(SigmaRule.fromYaml( + " title: Test\n" + + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " selection1:\n" + + " CommandLine|endswith: '.cpl'\n" + + " filter:\n" + + " CommandLine|contains\n" + + " - '\\System32\\'\n" + + " - '%System%'\n" + + " fp1_igfx:\n" + + " CommandLine|contains|all:\n" + + " - 'regsvr32 '\n" + + " - ' /s '\n" + + " - 'igfxCPL.cpl'\n" + + " selection2:\n" + + " Image|endswith: '\\reg.exe'\n" + + " CommandLine|contains: 'add'\n" + + " selection3:\n" + + " CommandLine|contains: 'CurrentVersion\\Control Panel\\CPLs'\n" + + " condition: (selection1 and not filter and not fp1_igfx) or (selection2 and selection3)", false)); + Assert.assertEquals("(((NOT Opcode: \"Info\" AND _exists_: Opcode) AND (NOT Severity: \"value2\" AND _exists_: Severity)))", queries.get(0).toString()); } public void testConvertNotWithAnd() throws IOException, SigmaError { @@ -741,7 +773,7 @@ public void testConvertNotWithAnd() throws IOException, SigmaError { " filter:\n" + " Details: '%CommonProgramFiles%\\System\\wab32.dll'\n" + " condition: selection and not filter", false)); - Assert.assertEquals("((EventType: \"SetValue\") AND (TargetObject: *\\\\Software\\\\Microsoft\\\\WAB\\\\DLLPath)) AND ((_exists_: Details AND NOT Details: \"%CommonProgramFiles%\\\\System\\\\wab32.dll\"))", queries.get(0).toString()); + Assert.assertEquals("((EventType: \"SetValue\") AND (TargetObject: *\\\\Software\\\\Microsoft\\\\WAB\\\\DLLPath)) AND ((NOT Details: \"%CommonProgramFiles%\\\\System\\\\wab32.dll\" AND _exists_: Details))", queries.get(0).toString()); } public void testConvertNotWithOrAndList() throws IOException, SigmaError { From 27333eca92dc5824a1006f836f67fcddab1d8ed0 Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Fri, 9 Feb 2024 13:10:25 -0800 Subject: [PATCH 04/15] testing Signed-off-by: Joanne Wang --- .../rules/backend/OSQueryBackend.java | 2 +- .../rules/backend/QueryBackend.java | 7 +- .../securityanalytics/TestHelpers.java | 93 ++++++------------- .../resthandler/DetectorMonitorRestApiIT.java | 63 +++++-------- 4 files changed, 56 insertions(+), 109 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java index 5bcdcbf8e..e0a603d64 100644 --- a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java +++ b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java @@ -235,7 +235,7 @@ public Object convertConditionOr(ConditionOR condition, boolean isConditionNot, if (converted != null) { // if applyDeMorgans is true, then use AND instead of OR - if (applyDeMorgans) { + if (applyDeMorgans && !isConditionNot) { joiner.setLength(0); // clear the joiner to convert it to AND if (this.tokenSeparator.equals(this.orToken)) { joiner.append(this.andToken); diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java index 542bbfd9b..52c026140 100644 --- a/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java +++ b/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java @@ -101,6 +101,7 @@ public List convertRule(SigmaRule rule) throws SigmaError { query = this.convertCondition(new ConditionType(Either.right(Either.right((ConditionValueExpression) conditionItem))), false, false); } queries.add(query); +// queries.add("(_exists_: Keywords)"); log.debug("converted query"); log.debug(query); if (aggItem != null) { @@ -140,7 +141,7 @@ public Object convertCondition(ConditionType conditionType, boolean isConditionN if (isConditionNot){ String baseString = this.convertConditionFieldEqVal(conditionType.getEqualsValueExpression(), isConditionNot, applyDeMorgans).toString(); String addExists = this.convertConditionFieldEqValNOT(conditionType.getEqualsValueExpression()).toString(); - return String.format(Locale.getDefault(), ("%s"+ "%s"), baseString, addExists); + return String.format(Locale.getDefault(), ("%s" + "%s"), baseString, addExists); } else { return this.convertConditionFieldEqVal(conditionType.getEqualsValueExpression(), isConditionNot, applyDeMorgans); } @@ -151,10 +152,6 @@ public Object convertCondition(ConditionType conditionType, boolean isConditionN } } - public Object convertConditionFieldEqValNot(ConditionFieldEqualsValueExpression condition, boolean isConditionNot, boolean applyDeMorgans) throws SigmaValueError { - return this.convertConditionFieldEqVal(condition, isConditionNot, applyDeMorgans).toString() + this.convertConditionFieldEqValNOT(condition).toString(); - } - public boolean decideConvertConditionAsInExpression(Either condition) { if ((!this.convertOrAsIn && condition.isRight()) || (!this.convertAndAsIn && condition.isLeft())) { return false; diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index 5f4ef846f..ce1a026fd 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -259,6 +259,35 @@ public static String randomRule() { "level: high"; } + public static String randomRuleWithNot() { + return "title: Remote Encrypting File System Abuse\n" + + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + + "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" + + "references:\n" + + " - https://attack.mitre.org/tactics/TA0008/\n" + + " - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" + + " - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" + + " - https://github.com/zeronetworks/rpcfirewall\n" + + " - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" + + "tags:\n" + + " - attack.defense_evasion\n" + + "status: experimental\n" + + "author: Sagie Dulce, Dekel Paz\n" + + "date: 2022/01/01\n" + + "modified: 2022/01/01\n" + + "logsource:\n" + + " product: rpc_firewall\n" + + " category: application\n" + + " definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" + + "detection:\n" + + " selection:\n" + + " EventID: 21\n" + + " condition: not selection\n" + + "falsepositives:\n" + + " - Legitimate usage of remote file encryption\n" + + "level: high"; + } + public static String randomNullRule() { return "title: null field\n" + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + @@ -494,68 +523,6 @@ public static String randomRuleWithDateKeywords() { "level: high"; } - public static String randomRuleWithNotConditionInvalidField() { - return "title: Remote Encrypting File System Abuse\n" + - "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + - "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" + - "references:\n" + - " - https://attack.mitre.org/tactics/TA0008/\n" + - " - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" + - " - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" + - " - https://github.com/zeronetworks/rpcfirewall\n" + - " - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" + - "tags:\n" + - " - attack.defense_evasion\n" + - "status: experimental\n" + - "author: Sagie Dulce, Dekel Paz\n" + - "date: 2022/01/01\n" + - "modified: 2022/01/01\n" + - "logsource:\n" + - " product: rpc_firewall\n" + - " category: application\n" + - " definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" + - "detection:\n" + - " selection1:\n" + - " randomField: value1\n" + - " selection2:\n" + - " EventID: 21\n" + - " condition: not selection1 and selection2\n" + - "falsepositives:\n" + - " - Legitimate usage of remote file encryption\n" + - "level: high"; - } - - public static String randomRuleWithNotConditionValidField() { - return "title: Remote Encrypting File System Abuse\n" + - "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + - "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" + - "references:\n" + - " - https://attack.mitre.org/tactics/TA0008/\n" + - " - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" + - " - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" + - " - https://github.com/zeronetworks/rpcfirewall\n" + - " - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" + - "tags:\n" + - " - attack.defense_evasion\n" + - "status: experimental\n" + - "author: Sagie Dulce, Dekel Paz\n" + - "date: 2022/01/01\n" + - "modified: 2022/01/01\n" + - "logsource:\n" + - " product: rpc_firewall\n" + - " category: application\n" + - " definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" + - "detection:\n" + - " selection1:\n" + - " AccountName: asdf\n" + - " selection2:\n" + - " Severity: asdf\n" + - " condition: not selection1\n" + - "falsepositives:\n" + - " - Legitimate usage of remote file encryption\n" + - "level: high"; - } - public static String countAggregationTestRule() { return " title: Test\n" + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + @@ -2494,7 +2461,7 @@ public static List randomLowerCaseStringList() { stringList.add(randomLowerCaseString()); return stringList; } - + public static XContentParser parser(String xc) throws IOException { XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, xc); parser.nextToken(); diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java index 25944b660..86f74a3e8 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java @@ -1000,7 +1000,7 @@ public void testCreateDetector_verifyWorkflowCreation_success_WithoutGroupByRule String testOpCode = "Test"; String maxRuleId = createRule(randomAggregationRule("max", " > 3", testOpCode)); - String randomDocRuleId = createRule(randomRule()); + String randomDocRuleId = createRule(randomRuleWithNot()); List detectorRules = List.of(new DetectorRule(maxRuleId), new DetectorRule(randomDocRuleId)); DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), detectorRules, emptyList()); @@ -2135,62 +2135,56 @@ public void testCreateDetectorWithNotCondition_verifyFindings_success() throws I assertEquals(HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode()); - // Create random doc rule - String randomDocRuleId = createRule(randomRuleWithNotConditionInvalidField()); - String randomDocRuleId2 = createRule(randomRuleWithNotConditionValidField()); - - DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(randomDocRuleId), new DetectorRule(randomDocRuleId2)), + // Create random custom doc rule with NOT condition + String randomDocRuleId = createRule(randomRuleWithNot()); + DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(randomDocRuleId)), emptyList()); Detector detector = randomDetectorWithInputs(List.of(input)); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); - assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); - - Map updateResponseBody = asMap(createResponse); - String detectorId = updateResponseBody.get("_id").toString(); String request = "{\n" + " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId + "\"\n" + + " \"match_all\":{\n" + " }\n" + " }\n" + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); - // Verify newly created doc level monitor - List hits = executeSearch(Detector.DETECTORS_INDEX, request); - SearchHit hit = hits.get(0); - Map detectorAsMap = (Map) hit.getSourceAsMap().get("detector"); - List monitorIds = ((List) (detectorAsMap).get("monitor_id")); - - assertEquals(1, monitorIds.size()); - - String monitorId = monitorIds.get(0); - String monitorType = ((Map) entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + monitorId))).get("monitor")).get("monitor_type"); + assertEquals(1, response.getHits().getTotalHits().value); - assertEquals(MonitorType.DOC_LEVEL_MONITOR.getValue(), monitorType); + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + Map responseBody = asMap(createResponse); - // Verify rules + String detectorId = responseBody.get("_id").toString(); request = "{\n" + " \"query\" : {\n" + - " \"match_all\":{\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + Map detectorMap = (HashMap) (hit.getSourceAsMap().get("detector")); + List inputArr = (List) detectorMap.get("inputs"); - assertEquals(2, response.getHits().getTotalHits().value); + assertEquals(1, ((Map>) inputArr.get(0)).get("detector_input").get("custom_rules").size()); + + List monitorIds = ((List) (detectorMap).get("monitor_id")); + assertEquals(1, monitorIds.size()); + + String monitorId = monitorIds.get(0); // Verify findings indexDoc(index, "1", randomDoc(2, 5, "Test")); indexDoc(index, "2", randomDoc(3, 5, "Test")); - Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); Map executeResults = entityAsMap(executeResponse); int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); - // Verify 1 custom rule matches + // Verify 1 custom rule assertEquals(1, noOfSigmaRuleMatches); Map params = new HashMap<>(); @@ -2202,17 +2196,6 @@ public void testCreateDetectorWithNotCondition_verifyFindings_success() throws I // When doc level monitor is being applied one finding is generated per document assertEquals(2, getFindingsBody.get("total_findings")); - List> findings = (List) getFindingsBody.get("findings"); - List foundDocIds = new ArrayList<>(); - for (Map finding : findings) { - Set aggRulesFinding = ((List>) finding.get("queries")).stream().map(it -> it.get("id").toString()).collect( - Collectors.toSet()); - - List findingDocs = (List) finding.get("related_doc_ids"); - Assert.assertEquals(1, findingDocs.size()); - foundDocIds.addAll(findingDocs); - } - assertTrue(Arrays.asList("1", "2").containsAll(foundDocIds)); } private static void assertRuleMonitorFinding(Map executeResults, String ruleId, int expectedDocCount, List expectedTriggerResult) { From 1ba01173e27cc5573e5899d432f06db7dfed8e07 Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Fri, 9 Feb 2024 16:48:27 -0800 Subject: [PATCH 05/15] working version Signed-off-by: Joanne Wang --- .../rules/backend/OSQueryBackend.java | 8 ++-- .../rules/backend/QueryBackend.java | 5 +- .../securityanalytics/TestHelpers.java | 46 ++++++++++++++++++- .../resthandler/DetectorMonitorRestApiIT.java | 6 +-- 4 files changed, 54 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java index e0a603d64..b220dd9de 100644 --- a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java +++ b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java @@ -235,7 +235,7 @@ public Object convertConditionOr(ConditionOR condition, boolean isConditionNot, if (converted != null) { // if applyDeMorgans is true, then use AND instead of OR - if (applyDeMorgans && !isConditionNot) { + if (applyDeMorgans) { joiner.setLength(0); // clear the joiner to convert it to AND if (this.tokenSeparator.equals(this.orToken)) { joiner.append(this.andToken); @@ -286,7 +286,7 @@ public Object convertConditionNot(ConditionNOT condition, boolean isConditionNot @Override public Object convertExistsField(ConditionFieldEqualsValueExpression condition) { String field = getFinalField(condition.getField()); - return String.format(Locale.getDefault(),tokenSeparator + this.andToken + this.tokenSeparator + this.existsToken + this.eqToken + " " + field); + return String.format(Locale.getDefault(),tokenSeparator + this.andToken + this.tokenSeparator + this.existsToken + this.eqToken + " _exists_" + field); } @Override @@ -299,8 +299,8 @@ public Object convertConditionFieldEqValStr(ConditionFieldEqualsValueExpression String field = getFinalField(condition.getField()); ruleQueryFields.put(field, Map.of("type", "text", "analyzer", "rule_analyzer")); String convertedExpr = String.format(Locale.getDefault(), expr, field, this.convertValueStr(value)); - if (applyDeMorgans) { - convertedExpr = String.format(Locale.getDefault(), exprWithDeMorgansApplied, this.existsToken, field); + if (applyDeMorgans) { // reformat this? + convertedExpr = String.format(Locale.getDefault(), exprWithDeMorgansApplied, field, this.convertValueStr(value)); } return convertedExpr; } diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java index 52c026140..a4ca5e6db 100644 --- a/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java +++ b/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java @@ -101,7 +101,6 @@ public List convertRule(SigmaRule rule) throws SigmaError { query = this.convertCondition(new ConditionType(Either.right(Either.right((ConditionValueExpression) conditionItem))), false, false); } queries.add(query); -// queries.add("(_exists_: Keywords)"); log.debug("converted query"); log.debug(query); if (aggItem != null) { @@ -138,9 +137,11 @@ public Object convertCondition(ConditionType conditionType, boolean isConditionN return this.convertConditionNot(conditionType.getConditionNOT(), isConditionNot, applyDeMorgans); } else if (conditionType.isEqualsValueExpression()) { // add a check to see if it should be done, then call another method to add them together else, return as normal BUT the check needs to see if top parent is NOT - if (isConditionNot){ + if (isConditionNot) { String baseString = this.convertConditionFieldEqVal(conditionType.getEqualsValueExpression(), isConditionNot, applyDeMorgans).toString(); String addExists = this.convertConditionFieldEqValNOT(conditionType.getEqualsValueExpression()).toString(); + log.error("I AM HERE"); + log.error(String.format(Locale.getDefault(), ("%s" + "%s"), baseString, addExists)); return String.format(Locale.getDefault(), ("%s" + "%s"), baseString, addExists); } else { return this.convertConditionFieldEqVal(conditionType.getEqualsValueExpression(), isConditionNot, applyDeMorgans); diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index ce1a026fd..4ad6861ad 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -281,8 +281,12 @@ public static String randomRuleWithNot() { " definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" + "detection:\n" + " selection:\n" + - " EventID: 21\n" + - " condition: not selection\n" + + " AccountType: whatever\n" + + " selection2:\n" + + " AccountName: SYSTEM\n" + + " selection3:\n" + + " EventID: 22\n" + + " condition: (not selection or selection2) and selection3\n" + "falsepositives:\n" + " - Legitimate usage of remote file encryption\n" + "level: high"; @@ -1730,6 +1734,44 @@ public static String randomDoc(int severity, int version, String opCode) { } + public static String randomDocWithoutAccountName(int severity, int version, String opCode) { + String doc = "{\n" + + "\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" + + "\"HostName\":\"EC2AMAZ-EPO7HKA\",\n" + + "\"Keywords\":\"9223372036854775808\",\n" + + "\"SeverityValue\":%s,\n" + + "\"Severity\":\"INFO\",\n" + + "\"EventID\":22,\n" + + "\"SourceName\":\"Microsoft-Windows-Sysmon\",\n" + + "\"ProviderGuid\":\"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}\",\n" + + "\"Version\":%s,\n" + + "\"TaskValue\":22,\n" + + "\"OpcodeValue\":0,\n" + + "\"RecordNumber\":9532,\n" + + "\"ExecutionProcessID\":1996,\n" + + "\"ExecutionThreadID\":2616,\n" + + "\"Channel\":\"Microsoft-Windows-Sysmon/Operational\",\n" + + "\"Domain\":\"NT AUTHORITY\",\n" + + "\"UserID\":\"S-1-5-18\",\n" + + "\"AccountType\":\"User\",\n" + + "\"Message\":\"Dns query:\\r\\nRuleName: \\r\\nUtcTime: 2020-02-04 14:59:38.349\\r\\nProcessGuid: {b3c285a4-3cda-5dc0-0000-001077270b00}\\r\\nProcessId: 1904\\r\\nQueryName: EC2AMAZ-EPO7HKA\\r\\nQueryStatus: 0\\r\\nQueryResults: 172.31.46.38;\\r\\nImage: C:\\\\Program Files\\\\nxlog\\\\nxlog.exe\",\n" + + "\"Category\":\"Dns query (rule: DnsQuery)\",\n" + + "\"Opcode\":\"%s\",\n" + + "\"UtcTime\":\"2020-02-04 14:59:38.349\",\n" + + "\"ProcessGuid\":\"{b3c285a4-3cda-5dc0-0000-001077270b00}\",\n" + + "\"ProcessId\":\"1904\",\"QueryName\":\"EC2AMAZ-EPO7HKA\",\"QueryStatus\":\"0\",\n" + + "\"QueryResults\":\"172.31.46.38;\",\n" + + "\"Image\":\"C:\\\\Program Files\\\\nxlog\\\\regsvr32.exe\",\n" + + "\"EventReceivedTime\":\"2020-02-04T14:59:40.780905+00:00\",\n" + + "\"SourceModuleName\":\"in\",\n" + + "\"SourceModuleType\":\"im_msvistalog\",\n" + + "\"CommandLine\": \"eachtest\",\n" + + "\"Initiated\": \"true\"\n" + + "}"; + return String.format(Locale.ROOT, doc, severity, version, opCode); + + } + public static String randomDocOnlyNumericAndDate(int severity, int version, String opCode) { String doc = "{\n" + "\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" + diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java index 86f74a3e8..83470c084 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java @@ -2178,7 +2178,8 @@ public void testCreateDetectorWithNotCondition_verifyFindings_success() throws I // Verify findings indexDoc(index, "1", randomDoc(2, 5, "Test")); - indexDoc(index, "2", randomDoc(3, 5, "Test")); + indexDoc(index, "2", randomDocWithoutAccountName(3, 5, "Test")); + indexDoc(index, "3", randomDoc(3, 5, "Test")); Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); Map executeResults = entityAsMap(executeResponse); @@ -2194,8 +2195,7 @@ public void testCreateDetectorWithNotCondition_verifyFindings_success() throws I assertNotNull(getFindingsBody); // When doc level monitor is being applied one finding is generated per document - assertEquals(2, getFindingsBody.get("total_findings")); - + assertEquals(3, getFindingsBody.get("total_findings")); } private static void assertRuleMonitorFinding(Map executeResults, String ruleId, int expectedDocCount, List expectedTriggerResult) { From 20086b2327cdda72bde436a3e1acf55e305eb533 Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Fri, 9 Feb 2024 17:07:19 -0800 Subject: [PATCH 06/15] working version Signed-off-by: Joanne Wang --- .../securityanalytics/rules/backend/OSQueryBackend.java | 2 +- .../resthandler/DetectorMonitorRestApiIT.java | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java index b220dd9de..db9d80cf8 100644 --- a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java +++ b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java @@ -299,7 +299,7 @@ public Object convertConditionFieldEqValStr(ConditionFieldEqualsValueExpression String field = getFinalField(condition.getField()); ruleQueryFields.put(field, Map.of("type", "text", "analyzer", "rule_analyzer")); String convertedExpr = String.format(Locale.getDefault(), expr, field, this.convertValueStr(value)); - if (applyDeMorgans) { // reformat this? + if (applyDeMorgans) { convertedExpr = String.format(Locale.getDefault(), exprWithDeMorgansApplied, field, this.convertValueStr(value)); } return convertedExpr; diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java index 83470c084..b46339fdf 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java @@ -2196,6 +2196,15 @@ public void testCreateDetectorWithNotCondition_verifyFindings_success() throws I assertNotNull(getFindingsBody); // When doc level monitor is being applied one finding is generated per document assertEquals(3, getFindingsBody.get("total_findings")); + + List> findings = (List) getFindingsBody.get("findings"); + List foundDocIds = new ArrayList<>(); + for (Map finding : findings) { + List findingDocs = (List) finding.get("related_doc_ids"); + Assert.assertEquals(1, findingDocs.size()); + foundDocIds.addAll(findingDocs); + } + assertTrue(Arrays.asList("1", "2", "3").containsAll(foundDocIds)); } private static void assertRuleMonitorFinding(Map executeResults, String ruleId, int expectedDocCount, List expectedTriggerResult) { From 9a01b04bd4cfaa182700f485874890522b8d0892 Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Fri, 9 Feb 2024 17:20:39 -0800 Subject: [PATCH 07/15] refactored querybackend Signed-off-by: Joanne Wang --- .../rules/backend/OSQueryBackend.java | 2 +- .../rules/backend/QueryBackend.java | 50 ++++++++----------- 2 files changed, 22 insertions(+), 30 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java index db9d80cf8..b97f777d2 100644 --- a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java +++ b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java @@ -290,7 +290,7 @@ public Object convertExistsField(ConditionFieldEqualsValueExpression condition) } @Override - public Object convertConditionFieldEqValStr(ConditionFieldEqualsValueExpression condition, boolean isConditionNot, boolean applyDeMorgans) throws SigmaValueError { + public Object convertConditionFieldEqValStr(ConditionFieldEqualsValueExpression condition, boolean applyDeMorgans) throws SigmaValueError { SigmaString value = (SigmaString) condition.getValue(); boolean containsWildcard = value.containsWildcard(); String expr = "%s" + this.eqToken + " " + (containsWildcard? this.reQuote: this.strQuote) + "%s" + (containsWildcard? this.reQuote: this.strQuote); diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java index a4ca5e6db..61da76506 100644 --- a/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java +++ b/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java @@ -6,8 +6,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.commons.alerting.aggregation.bucketselectorext.BucketSelectorExtAggregationBuilder; -import org.opensearch.search.aggregations.AggregationBuilder; import org.opensearch.securityanalytics.rules.aggregation.AggregationItem; import org.opensearch.securityanalytics.rules.backend.OSQueryBackend.AggregationQueries; import org.opensearch.securityanalytics.rules.condition.ConditionAND; @@ -33,22 +31,8 @@ import org.opensearch.securityanalytics.rules.utils.AnyOneOf; import org.opensearch.securityanalytics.rules.utils.Either; import org.apache.commons.lang3.tuple.Pair; -import org.opensearch.securityanalytics.threatIntel.DetectorThreatIntelService; -import org.opensearch.securityanalytics.util.RuleIndices; -import org.yaml.snakeyaml.LoaderOptions; -import org.yaml.snakeyaml.Yaml; -import org.yaml.snakeyaml.constructor.SafeConstructor; - -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; + +import java.util.*; public abstract class QueryBackend { @@ -138,11 +122,7 @@ public Object convertCondition(ConditionType conditionType, boolean isConditionN } else if (conditionType.isEqualsValueExpression()) { // add a check to see if it should be done, then call another method to add them together else, return as normal BUT the check needs to see if top parent is NOT if (isConditionNot) { - String baseString = this.convertConditionFieldEqVal(conditionType.getEqualsValueExpression(), isConditionNot, applyDeMorgans).toString(); - String addExists = this.convertConditionFieldEqValNOT(conditionType.getEqualsValueExpression()).toString(); - log.error("I AM HERE"); - log.error(String.format(Locale.getDefault(), ("%s" + "%s"), baseString, addExists)); - return String.format(Locale.getDefault(), ("%s" + "%s"), baseString, addExists); + return this.convertConditionFieldEqValNot(conditionType, isConditionNot, applyDeMorgans); } else { return this.convertConditionFieldEqVal(conditionType.getEqualsValueExpression(), isConditionNot, applyDeMorgans); } @@ -153,6 +133,15 @@ public Object convertCondition(ConditionType conditionType, boolean isConditionN } } + public String convertConditionFieldEqValNot(ConditionType conditionType, boolean isConditionNot, boolean applyDeMorgans) throws SigmaValueError { + String baseString = this.convertConditionFieldEqVal(conditionType.getEqualsValueExpression(), isConditionNot, applyDeMorgans).toString(); + String addExists = this.convertExistsField(conditionType.getEqualsValueExpression()).toString(); +// log.error("I AM HERE"); +// log.error(String.format(Locale.getDefault(), ("%s" + "%s"), baseString, addExists)); + return String.format(Locale.getDefault(), ("%s" + "%s"), baseString, addExists); + } + + public boolean decideConvertConditionAsInExpression(Either condition) { if ((!this.convertOrAsIn && condition.isRight()) || (!this.convertAndAsIn && condition.isLeft())) { return false; @@ -207,7 +196,7 @@ public void resetQueryFields() { public Object convertConditionFieldEqVal(ConditionFieldEqualsValueExpression condition, boolean isConditionNot, boolean applyDeMorgans) throws SigmaValueError { if (condition.getValue() instanceof SigmaString) { - return this.convertConditionFieldEqValStr(condition, isConditionNot, applyDeMorgans); + return this.convertConditionFieldEqValStr(condition, applyDeMorgans); } else if (condition.getValue() instanceof SigmaNumber) { return this.convertConditionFieldEqValNum(condition); } else if (condition.getValue() instanceof SigmaBool) { @@ -230,11 +219,11 @@ else if (condition.getValue() instanceof SigmaQueryExpression) { } } - public Object convertConditionFieldEqValNOT(ConditionFieldEqualsValueExpression condition) { - return this.convertExistsField(condition); - } +// public Object convertConditionFieldEqValNOT(ConditionFieldEqualsValueExpression condition) { +// return this.convertExistsField(condition); +// } - public abstract Object convertConditionFieldEqValStr(ConditionFieldEqualsValueExpression condition, boolean isConditionNot, boolean applyDeMorgans) throws SigmaValueError; + public abstract Object convertConditionFieldEqValStr(ConditionFieldEqualsValueExpression condition, boolean applyDeMorgans) throws SigmaValueError; public abstract Object convertConditionFieldEqValNum(ConditionFieldEqualsValueExpression condition); @@ -250,7 +239,10 @@ public Object convertConditionFieldEqValNOT(ConditionFieldEqualsValueExpression public abstract Object convertExistsField(ConditionFieldEqualsValueExpression condition); -/* public abstract Object convertConditionFieldEqValQueryExpr(ConditionFieldEqualsValueExpression condition);*/ +// public abstract String convertConditionFieldEqValNot(ConditionType conditionType, boolean isConditionNot, boolean applyDeMorgans) throws SigmaValueError; + + + /* public abstract Object convertConditionFieldEqValQueryExpr(ConditionFieldEqualsValueExpression condition);*/ public Object convertConditionFieldEqValQueryExpansion(ConditionFieldEqualsValueExpression condition, boolean isConditionNot, boolean applyDeMorgans) { List, String>> args = new ArrayList<>(); From 034097bdcd30540b3539b86c409e64e58b97815a Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Mon, 12 Feb 2024 08:41:21 -0800 Subject: [PATCH 08/15] working on tests Signed-off-by: Joanne Wang --- .../rules/backend/OSQueryBackend.java | 66 +++++++++++++++---- .../rules/backend/QueryBackend.java | 60 ++++++++--------- .../rules/backend/QueryBackendTests.java | 56 ++++++++++------ 3 files changed, 119 insertions(+), 63 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java index b97f777d2..b4e3d60a6 100644 --- a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java +++ b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java @@ -306,45 +306,67 @@ public Object convertConditionFieldEqValStr(ConditionFieldEqualsValueExpression } @Override - public Object convertConditionFieldEqValNum(ConditionFieldEqualsValueExpression condition) { + public Object convertConditionFieldEqValNum(ConditionFieldEqualsValueExpression condition, boolean applyDeMorgans) { String field = getFinalField(condition.getField()); SigmaNumber number = (SigmaNumber) condition.getValue(); ruleQueryFields.put(field, number.getNumOpt().isLeft()? Collections.singletonMap("type", "integer"): Collections.singletonMap("type", "float")); - + if (applyDeMorgans) { + return this.notToken + " " +field + this.eqToken + " " + condition.getValue(); + } return field + this.eqToken + " " + condition.getValue(); } @Override - public Object convertConditionFieldEqValBool(ConditionFieldEqualsValueExpression condition) { + public Object convertConditionFieldEqValBool(ConditionFieldEqualsValueExpression condition, boolean applyDeMorgans) { String field = getFinalField(condition.getField()); ruleQueryFields.put(field, Collections.singletonMap("type", "boolean")); - + if (applyDeMorgans) { + return this.notToken + " " + field + this.eqToken + " " + ((SigmaBool) condition.getValue()).isaBoolean(); + } return field + this.eqToken + " " + ((SigmaBool) condition.getValue()).isaBoolean(); } - public Object convertConditionFieldEqValNull(ConditionFieldEqualsValueExpression condition) { + public Object convertConditionFieldEqValNull(ConditionFieldEqualsValueExpression condition, boolean applyDeMorgans) { String field = getFinalField(condition.getField()); ruleQueryFields.put(field, Map.of("type", "text", "analyzer", "rule_analyzer")); + String exprWithDeMorgansApplied = this.notToken + " " + this.fieldNullExpression; + if (applyDeMorgans) { + return String.format(Locale.getDefault(), exprWithDeMorgansApplied, field); + } return String.format(Locale.getDefault(), this.fieldNullExpression, field); } @Override - public Object convertConditionFieldEqValRe(ConditionFieldEqualsValueExpression condition) { + public Object convertConditionFieldEqValRe(ConditionFieldEqualsValueExpression condition, boolean applyDeMorgans) { String field = getFinalField(condition.getField()); ruleQueryFields.put(field, Map.of("type", "text", "analyzer", "rule_analyzer")); + String exprWithDeMorgansApplied = this.notToken + " " + this.reExpression; + if (applyDeMorgans) { + return String.format(Locale.getDefault(), exprWithDeMorgansApplied, field, convertValueRe((SigmaRegularExpression) condition.getValue())); + } return String.format(Locale.getDefault(), this.reExpression, field, convertValueRe((SigmaRegularExpression) condition.getValue())); } @Override - public Object convertConditionFieldEqValCidr(ConditionFieldEqualsValueExpression condition) { + public Object convertConditionFieldEqValCidr(ConditionFieldEqualsValueExpression condition, boolean applyDeMorgans) { String field = getFinalField(condition.getField()); ruleQueryFields.put(field, Map.of("type", "text", "analyzer", "rule_analyzer")); + String exprWithDeMorgansApplied = this.notToken + " " + this.cidrExpression; + if (applyDeMorgans) { + return String.format(Locale.getDefault(), exprWithDeMorgansApplied, field, convertValueCidr((SigmaCIDRExpression) condition.getValue())); + } return String.format(Locale.getDefault(), this.cidrExpression, field, convertValueCidr((SigmaCIDRExpression) condition.getValue())); } @Override - public Object convertConditionFieldEqValOpVal(ConditionFieldEqualsValueExpression condition) { + public Object convertConditionFieldEqValOpVal(ConditionFieldEqualsValueExpression condition, boolean applyDeMorgans) { + String exprWithDeMorgansApplied = this.notToken + " " + this.compareOpExpression; + if (applyDeMorgans) { + return String.format(Locale.getDefault(), exprWithDeMorgansApplied, this.getMappedField(condition.getField()), + compareOperators.get(((SigmaCompareExpression) condition.getValue()).getOp()), ((SigmaCompareExpression) condition.getValue()).getNumber().toString()); + } + return String.format(Locale.getDefault(), this.compareOpExpression, this.getMappedField(condition.getField()), compareOperators.get(((SigmaCompareExpression) condition.getValue()).getOp()), ((SigmaCompareExpression) condition.getValue()).getNumber().toString()); } @@ -362,20 +384,36 @@ public Object convertConditionFieldEqValQueryExpr(ConditionFieldEqualsValueExpre }*/ @Override - public Object convertConditionValStr(ConditionValueExpression condition) throws SigmaValueError { + public Object convertConditionValStr(ConditionValueExpression condition, boolean applyDeMorgans) throws SigmaValueError { SigmaString value = (SigmaString) condition.getValue(); boolean containsWildcard = value.containsWildcard(); - return String.format(Locale.getDefault(), (containsWildcard? this.unboundWildcardExpression: this.unboundValueStrExpression), this.convertValueStr((SigmaString) condition.getValue())); + String exprWithDeMorgansApplied = this.notToken + " " + "%s"; + + String conditionValStr = String.format(Locale.getDefault(), (containsWildcard? this.unboundWildcardExpression: this.unboundValueStrExpression), this.convertValueStr((SigmaString) condition.getValue())); + if (applyDeMorgans) { + conditionValStr = String.format(Locale.getDefault(), exprWithDeMorgansApplied, conditionValStr); + } + return conditionValStr; } @Override - public Object convertConditionValNum(ConditionValueExpression condition) { - return String.format(Locale.getDefault(), this.unboundValueNumExpression, condition.getValue().toString()); + public Object convertConditionValNum(ConditionValueExpression condition, boolean applyDeMorgans) { + String exprWithDeMorgansApplied = this.notToken + " " + "%s"; + String conditionValNum = String.format(Locale.getDefault(), String.format(Locale.getDefault(), this.unboundValueNumExpression, condition.getValue().toString())); + if (applyDeMorgans) { + conditionValNum = String.format(Locale.getDefault(), exprWithDeMorgansApplied, conditionValNum); + } + return conditionValNum; } @Override - public Object convertConditionValRe(ConditionValueExpression condition) { - return String.format(Locale.getDefault(), this.unboundReExpression, convertValueRe((SigmaRegularExpression) condition.getValue())); + public Object convertConditionValRe(ConditionValueExpression condition, boolean applyDeMorgans) { + String exprWithDeMorgansApplied = this.notToken + " " + "%s"; + String conditionValStr = String.format(Locale.getDefault(), this.unboundReExpression, convertValueRe((SigmaRegularExpression) condition.getValue())); + if (applyDeMorgans) { + conditionValStr = String.format(Locale.getDefault(), exprWithDeMorgansApplied, conditionValStr); + } + return conditionValStr; } // TODO: below methods will be supported when Sigma Expand Modifier is supported. diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java index 61da76506..fd484950f 100644 --- a/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java +++ b/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java @@ -85,8 +85,8 @@ public List convertRule(SigmaRule rule) throws SigmaError { query = this.convertCondition(new ConditionType(Either.right(Either.right((ConditionValueExpression) conditionItem))), false, false); } queries.add(query); - log.debug("converted query"); - log.debug(query); +// log.debug("converted query"); +// log.debug(query); if (aggItem != null) { aggItem.setTimeframe(rule.getDetection().getTimeframe()); queries.add(convertAggregation(aggItem)); @@ -127,13 +127,21 @@ public Object convertCondition(ConditionType conditionType, boolean isConditionN return this.convertConditionFieldEqVal(conditionType.getEqualsValueExpression(), isConditionNot, applyDeMorgans); } } else if (conditionType.isValueExpression()) { - return this.convertConditionVal(conditionType.getValueExpression()); + return this.convertConditionVal(conditionType.getValueExpression(), applyDeMorgans); } else { throw new IllegalArgumentException("Unexpected data type in condition parse tree"); } } public String convertConditionFieldEqValNot(ConditionType conditionType, boolean isConditionNot, boolean applyDeMorgans) throws SigmaValueError { +// String baseString; +// String exprWithDeMorgansApplied = "NOT " + "%s"; +// if (applyDeMorgans) { +// baseString = String.format(Locale.getDefault(), exprWithDeMorgansApplied, this.convertConditionFieldEqVal(conditionType.getEqualsValueExpression(), isConditionNot, applyDeMorgans).toString()); +// } else { +// baseString = this.convertConditionFieldEqVal(conditionType.getEqualsValueExpression(), isConditionNot, applyDeMorgans).toString(); +// } + String baseString = this.convertConditionFieldEqVal(conditionType.getEqualsValueExpression(), isConditionNot, applyDeMorgans).toString(); String addExists = this.convertExistsField(conditionType.getEqualsValueExpression()).toString(); // log.error("I AM HERE"); @@ -141,7 +149,6 @@ public String convertConditionFieldEqValNot(ConditionType conditionType, boolean return String.format(Locale.getDefault(), ("%s" + "%s"), baseString, addExists); } - public boolean decideConvertConditionAsInExpression(Either condition) { if ((!this.convertOrAsIn && condition.isRight()) || (!this.convertAndAsIn && condition.isLeft())) { return false; @@ -198,17 +205,17 @@ public Object convertConditionFieldEqVal(ConditionFieldEqualsValueExpression con if (condition.getValue() instanceof SigmaString) { return this.convertConditionFieldEqValStr(condition, applyDeMorgans); } else if (condition.getValue() instanceof SigmaNumber) { - return this.convertConditionFieldEqValNum(condition); + return this.convertConditionFieldEqValNum(condition, applyDeMorgans); } else if (condition.getValue() instanceof SigmaBool) { - return this.convertConditionFieldEqValBool(condition); + return this.convertConditionFieldEqValBool(condition, applyDeMorgans); } else if (condition.getValue() instanceof SigmaRegularExpression) { - return this.convertConditionFieldEqValRe(condition); + return this.convertConditionFieldEqValRe(condition, applyDeMorgans); } else if (condition.getValue() instanceof SigmaCIDRExpression) { - return this.convertConditionFieldEqValCidr(condition); + return this.convertConditionFieldEqValCidr(condition, applyDeMorgans); } else if (condition.getValue() instanceof SigmaCompareExpression) { - return this.convertConditionFieldEqValOpVal(condition); + return this.convertConditionFieldEqValOpVal(condition, applyDeMorgans); } else if (condition.getValue() instanceof SigmaNull) { - return this.convertConditionFieldEqValNull(condition); + return this.convertConditionFieldEqValNull(condition, applyDeMorgans); }/* TODO: below methods will be supported when Sigma Expand Modifier is supported. else if (condition.getValue() instanceof SigmaQueryExpression) { return this.convertConditionFieldEqValQueryExpr(condition); @@ -219,29 +226,22 @@ else if (condition.getValue() instanceof SigmaQueryExpression) { } } -// public Object convertConditionFieldEqValNOT(ConditionFieldEqualsValueExpression condition) { -// return this.convertExistsField(condition); -// } - public abstract Object convertConditionFieldEqValStr(ConditionFieldEqualsValueExpression condition, boolean applyDeMorgans) throws SigmaValueError; - public abstract Object convertConditionFieldEqValNum(ConditionFieldEqualsValueExpression condition); + public abstract Object convertConditionFieldEqValNum(ConditionFieldEqualsValueExpression condition, boolean applyDeMorgans); - public abstract Object convertConditionFieldEqValBool(ConditionFieldEqualsValueExpression condition); + public abstract Object convertConditionFieldEqValBool(ConditionFieldEqualsValueExpression condition, boolean applyDeMorgans); - public abstract Object convertConditionFieldEqValRe(ConditionFieldEqualsValueExpression condition); + public abstract Object convertConditionFieldEqValRe(ConditionFieldEqualsValueExpression condition, boolean applyDeMorgans); - public abstract Object convertConditionFieldEqValCidr(ConditionFieldEqualsValueExpression condition); + public abstract Object convertConditionFieldEqValCidr(ConditionFieldEqualsValueExpression condition, boolean applyDeMorgans); - public abstract Object convertConditionFieldEqValOpVal(ConditionFieldEqualsValueExpression condition); + public abstract Object convertConditionFieldEqValOpVal(ConditionFieldEqualsValueExpression condition, boolean applyDeMorgans); - public abstract Object convertConditionFieldEqValNull(ConditionFieldEqualsValueExpression condition); + public abstract Object convertConditionFieldEqValNull(ConditionFieldEqualsValueExpression condition, boolean applyDeMorgans); public abstract Object convertExistsField(ConditionFieldEqualsValueExpression condition); -// public abstract String convertConditionFieldEqValNot(ConditionType conditionType, boolean isConditionNot, boolean applyDeMorgans) throws SigmaValueError; - - /* public abstract Object convertConditionFieldEqValQueryExpr(ConditionFieldEqualsValueExpression condition);*/ public Object convertConditionFieldEqValQueryExpansion(ConditionFieldEqualsValueExpression condition, boolean isConditionNot, boolean applyDeMorgans) { @@ -254,15 +254,15 @@ public Object convertConditionFieldEqValQueryExpansion(ConditionFieldEqualsValue return this.convertConditionOr(conditionOR, isConditionNot, applyDeMorgans); } - public Object convertConditionVal(ConditionValueExpression condition) throws SigmaValueError { + public Object convertConditionVal(ConditionValueExpression condition, boolean applyDeMorgans) throws SigmaValueError { if (condition.getValue() instanceof SigmaString) { - return this.convertConditionValStr(condition); + return this.convertConditionValStr(condition, applyDeMorgans); } else if (condition.getValue() instanceof SigmaNumber) { - return this.convertConditionValNum(condition); + return this.convertConditionValNum(condition, applyDeMorgans); } else if (condition.getValue() instanceof SigmaBool) { throw new SigmaValueError("Boolean values can't appear as standalone value without a field name."); } else if (condition.getValue() instanceof SigmaRegularExpression) { - return this.convertConditionValRe(condition); + return this.convertConditionValRe(condition, applyDeMorgans); }/* else if (condition.getValue() instanceof SigmaCIDRExpression) { throw new SigmaValueError("CIDR values can't appear as standalone value without a field name."); } else if (condition.getValue() instanceof SigmaQueryExpression) { @@ -272,11 +272,11 @@ public Object convertConditionVal(ConditionValueExpression condition) throws Sig } } - public abstract Object convertConditionValStr(ConditionValueExpression condition) throws SigmaValueError; + public abstract Object convertConditionValStr(ConditionValueExpression condition, boolean applyDeMorgans) throws SigmaValueError; - public abstract Object convertConditionValNum(ConditionValueExpression condition); + public abstract Object convertConditionValNum(ConditionValueExpression condition, boolean applyDeMorgans); - public abstract Object convertConditionValRe(ConditionValueExpression condition); + public abstract Object convertConditionValRe(ConditionValueExpression condition, boolean applyDeMorgans); /* public abstract Object convertConditionValQueryExpr(ConditionValueExpression condition);*/ diff --git a/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java b/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java index 97e091748..3c8e6bf59 100644 --- a/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java +++ b/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java @@ -698,6 +698,26 @@ public void testConvertOr() throws IOException, SigmaError { } public void testConvertNot() throws IOException, SigmaError { + OSQueryBackend queryBackend = testBackend(); + List queries = queryBackend.convertRule(SigmaRule.fromYaml( + " title: Test\n" + + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel:\n" + + " fieldA: value1\n" + + " condition: not sel", false)); + Assert.assertEquals("(NOT fieldA: \"value1\" AND _exists_: _exists_fieldA)", queries.get(0).toString()); + } + + public void testConvertNotWithParenthesis() throws IOException, SigmaError { OSQueryBackend queryBackend = testBackend(); List queries = queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + @@ -716,10 +736,10 @@ public void testConvertNot() throws IOException, SigmaError { " sel2:\n" + " Severity: value2\n" + " condition: not (sel1 or sel2)", false)); - Assert.assertEquals("(((NOT Opcode: \"Info\" AND _exists_: Opcode) AND (NOT Severity: \"value2\" AND _exists_: Severity)))", queries.get(0).toString()); + Assert.assertEquals("(((NOT Opcode: \"Info\" AND _exists_: _exists_Opcode) AND (NOT Severity: \"value2\" AND _exists_: _exists_Severity)))", queries.get(0).toString()); } - public void testConvertNotTesting() throws IOException, SigmaError { + public void testConvertNotComplicatedExpression() throws IOException, SigmaError { OSQueryBackend queryBackend = testBackend(); List queries = queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + @@ -750,7 +770,7 @@ public void testConvertNotTesting() throws IOException, SigmaError { " selection3:\n" + " CommandLine|contains: 'CurrentVersion\\Control Panel\\CPLs'\n" + " condition: (selection1 and not filter and not fp1_igfx) or (selection2 and selection3)", false)); - Assert.assertEquals("(((NOT Opcode: \"Info\" AND _exists_: Opcode) AND (NOT Severity: \"value2\" AND _exists_: Severity)))", queries.get(0).toString()); + Assert.assertEquals("(((NOT Opcode: \"Info\" AND _exists_: _exists_Opcode) AND (NOT Severity: \"value2\" AND _exists_: _exists_Severity)))", queries.get(0).toString()); } public void testConvertNotWithAnd() throws IOException, SigmaError { @@ -773,7 +793,7 @@ public void testConvertNotWithAnd() throws IOException, SigmaError { " filter:\n" + " Details: '%CommonProgramFiles%\\System\\wab32.dll'\n" + " condition: selection and not filter", false)); - Assert.assertEquals("((EventType: \"SetValue\") AND (TargetObject: *\\\\Software\\\\Microsoft\\\\WAB\\\\DLLPath)) AND ((NOT Details: \"%CommonProgramFiles%\\\\System\\\\wab32.dll\" AND _exists_: Details))", queries.get(0).toString()); + Assert.assertEquals("((EventType: \"SetValue\") AND (TargetObject: *\\\\Software\\\\Microsoft\\\\WAB\\\\DLLPath)) AND ((NOT Details: \"%CommonProgramFiles%\\\\System\\\\wab32.dll\" AND _exists_: _exists_Details))", queries.get(0).toString()); } public void testConvertNotWithOrAndList() throws IOException, SigmaError { @@ -791,16 +811,14 @@ public void testConvertNotWithOrAndList() throws IOException, SigmaError { " product: test_product\n" + " detection:\n" + " sel1:\n" + - " fieldA1: valueA1\n" + - " fieldA2: valueA2\n" + - " fieldA3: valueA3\n" + - " sel2:\n" + - " fieldB: value2\n" + + " field1: valueA1\n" + + " field2: valueA2\n" + + " field3: valueA3\n" + " sel3:\n" + " - resp_mime_types|contains: 'dosexec'\n" + " - c-uri|endswith: '.exe'\n" + - " condition: (sel1 and sel2) or sel3", false)); - Assert.assertEquals("(NOT (fieldA: \"value1\"))", queries.get(0).toString()); + " condition: not sel1 or sel3", false)); + Assert.assertEquals("((((NOT field1: \"valueA1\" AND _exists_: _exists_field1) OR (NOT field2: \"valueA2\" AND _exists_: _exists_field2) OR (NOT field3: \"valueA3\" AND _exists_: _exists_field3)))) OR ((resp_mime_types: *dosexec*) OR (c-uri: *.exe))", queries.get(0).toString()); } public void testConvertNotWithNumAndBool() throws IOException, SigmaError { @@ -818,11 +836,11 @@ public void testConvertNotWithNumAndBool() throws IOException, SigmaError { " product: test_product\n" + " detection:\n" + " sel1:\n" + - " fieldA: 1\n" + + " field1: 1\n" + " sel2:\n" + - " fieldB: true\n" + - " condition: not (sel1 and sel2)", false)); - Assert.assertEquals("((_exists_: fieldA AND NOT fieldA: 1)) AND ((_exists_: mappedB AND NOT mappedB: true))", queries.get(0).toString()); + " field2: true\n" + + " condition: not sel1 and not sel2", false)); + Assert.assertEquals("((NOT field1: 1 AND _exists_: _exists_field1)) AND ((NOT field2: true AND _exists_: _exists_field2))", queries.get(0).toString()); } public void testConvertNotWithNull() throws IOException, SigmaError { @@ -844,7 +862,7 @@ public void testConvertNotWithNull() throws IOException, SigmaError { " sel2:\n" + " fieldB: true\n" + " condition: not sel1", false)); - Assert.assertEquals("(_exists_: fieldA AND NOT fieldA: (NOT [* TO *]))", queries.get(0).toString()); + Assert.assertEquals("(NOT fieldA: (NOT [* TO *]) AND _exists_: _exists_fieldA)", queries.get(0).toString()); } public void testConvertNotWithKeywords() throws IOException, SigmaError { @@ -867,9 +885,9 @@ public void testConvertNotWithKeywords() throws IOException, SigmaError { " fieldB: value2\n" + " keywords:\n" + " - test1\n" + - " - test2\n" + + " - 123\n" + " condition: not keywords", false)); - Assert.assertEquals("(NOT ((\"test1\") OR (\"test2\")))", queries.get(0).toString()); + Assert.assertEquals("(((NOT \"test1\") AND (NOT \"123\")))", queries.get(0).toString()); } public void testConvertPrecedence() throws IOException, SigmaError { @@ -895,7 +913,7 @@ public void testConvertPrecedence() throws IOException, SigmaError { " sel4:\n" + " fieldD: value5\n" + " condition: (sel1 or sel2) and not (sel3 and sel4)", false)); - Assert.assertEquals("((fieldA: \"value1\") OR (mappedB: \"value2\")) AND ((((NOT fieldC: \"value4\" AND _exists_: fieldC) OR (NOT fieldD: \"value5\" AND _exists_: fieldD))))", queries.get(0).toString()); + Assert.assertEquals("((fieldA: \"value1\") OR (mappedB: \"value2\")) AND ((((NOT fieldC: \"value4\" AND _exists_: _exists_fieldC) OR (NOT fieldD: \"value5\" AND _exists_: _exists_fieldD))))", queries.get(0).toString()); } public void testConvertMultiConditions() throws IOException, SigmaError { From 0769096d2235e468620a9d0bb1a2091ee951670d Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Mon, 12 Feb 2024 09:29:54 -0800 Subject: [PATCH 09/15] fixed alerting and finding tests Signed-off-by: Joanne Wang --- .../securityanalytics/TestHelpers.java | 40 +++++++++++++++++++ .../securityanalytics/alerts/AlertsIT.java | 14 +------ .../securityanalytics/findings/FindingIT.java | 9 +---- 3 files changed, 44 insertions(+), 19 deletions(-) diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index 4ad6861ad..82f9ea507 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -1911,6 +1911,46 @@ public static String randomDoc() { "}"; } + public static String randomNetworkDoc() { + return "{\n" + + "\"@timestamp\":\"2020-02-04T14:59:39.343541+00:00\",\n" + + "\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" + + "\"HostName\":\"EC2AMAZ-EPO7HKA\",\n" + + "\"Keywords\":\"9223372036854775808\",\n" + + "\"SeverityValue\":2,\n" + + "\"Severity\":\"INFO\",\n" + + "\"EventID\":22,\n" + + "\"SourceName\":\"Microsoft-Windows-Sysmon\",\n" + + "\"SourceIp\":\"1.2.3.4\",\n" + + "\"ProviderGuid\":\"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}\",\n" + + "\"Version\":5,\n" + + "\"TaskValue\":22,\n" + + "\"OpcodeValue\":0,\n" + + "\"RecordNumber\":9532,\n" + + "\"ExecutionProcessID\":1996,\n" + + "\"ExecutionThreadID\":2616,\n" + + "\"Channel\":\"Microsoft-Windows-Sysmon/Operational\",\n" + + "\"Domain\":\"NTAUTHORITY\",\n" + + "\"AccountName\":\"SYSTEM\",\n" + + "\"UserID\":\"S-1-5-18\",\n" + + "\"AccountType\":\"User\",\n" + + "\"Message\":\"Dns query:\\r\\nRuleName: \\r\\nUtcTime: 2020-02-04 14:59:38.349\\r\\nProcessGuid: {b3c285a4-3cda-5dc0-0000-001077270b00}\\r\\nProcessId: 1904\\r\\nQueryName: EC2AMAZ-EPO7HKA\\r\\nQueryStatus: 0\\r\\nQueryResults: 172.31.46.38;\\r\\nImage: C:\\\\Program Files\\\\nxlog\\\\nxlog.exe\",\n" + + "\"Category\":\"Dns query (rule: DnsQuery)\",\n" + + "\"Opcode\":\"Info\",\n" + + "\"UtcTime\":\"2020-02-04 14:59:38.349\",\n" + + "\"ProcessGuid\":\"{b3c285a4-3cda-5dc0-0000-001077270b00}\",\n" + + "\"ProcessId\":\"1904\",\"QueryName\":\"EC2AMAZ-EPO7HKA\",\"QueryStatus\":\"0\",\n" + + "\"QueryResults\":\"172.31.46.38;\",\n" + + "\"Image\":\"C:\\\\Program Files\\\\nxlog\\\\regsvr32.exe\",\n" + + "\"EventReceivedTime\":\"2020-02-04T14:59:40.780905+00:00\",\n" + + "\"SourceModuleName\":\"in\",\n" + + "\"SourceModuleType\":\"im_msvistalog\",\n" + + "\"CommandLine\": \"eachtest\",\n" + + "\"id.orig_h\": \"123.12.123.12\",\n" + + "\"Initiated\": \"true\"\n" + + "}"; + } + public static String randomCloudtrailAggrDoc(String eventType, String accountId) { return "{\n" + " \"AccountName\": \"" + accountId + "\",\n" + diff --git a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java index fbd091595..b3b3a940d 100644 --- a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java +++ b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java @@ -35,17 +35,7 @@ import org.opensearch.securityanalytics.model.DetectorRule; import org.opensearch.securityanalytics.model.DetectorTrigger; -import static org.opensearch.securityanalytics.TestHelpers.netFlowMappings; -import static org.opensearch.securityanalytics.TestHelpers.randomAction; -import static org.opensearch.securityanalytics.TestHelpers.randomDetectorType; -import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputsAndThreatIntel; -import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputsAndTriggers; -import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithTriggers; -import static org.opensearch.securityanalytics.TestHelpers.randomDoc; -import static org.opensearch.securityanalytics.TestHelpers.randomDocWithIpIoc; -import static org.opensearch.securityanalytics.TestHelpers.randomIndex; -import static org.opensearch.securityanalytics.TestHelpers.randomRule; -import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; +import static org.opensearch.securityanalytics.TestHelpers.*; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_INDEX_MAX_AGE; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_MAX_DOCS; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_RETENTION_PERIOD; @@ -545,7 +535,7 @@ public void testGetAlerts_byDetectorType_multipleDetectors_success() throws IOEx String monitorId2 = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); indexDoc(index1, "1", randomDoc()); - indexDoc(index2, "1", randomDoc()); + indexDoc(index2, "1", randomNetworkDoc()); // execute monitor 1 Response executeResponse = executeAlertingMonitor(monitorId1, Collections.emptyMap()); Map executeResults = entityAsMap(executeResponse); diff --git a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java index 3b7ca3c0a..c9da0e9fa 100644 --- a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java +++ b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java @@ -27,12 +27,7 @@ import org.opensearch.securityanalytics.model.DetectorRule; import org.opensearch.securityanalytics.model.DetectorTrigger; -import static org.opensearch.securityanalytics.TestHelpers.netFlowMappings; -import static org.opensearch.securityanalytics.TestHelpers.randomDetectorType; -import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithTriggers; -import static org.opensearch.securityanalytics.TestHelpers.randomDoc; -import static org.opensearch.securityanalytics.TestHelpers.randomIndex; -import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; +import static org.opensearch.securityanalytics.TestHelpers.*; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_INDEX_MAX_AGE; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_MAX_DOCS; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_RETENTION_PERIOD; @@ -234,7 +229,7 @@ public void testGetFindings_byDetectorType_success() throws IOException { String monitorId2 = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); indexDoc(index1, "1", randomDoc()); - indexDoc(index2, "1", randomDoc()); + indexDoc(index2, "1", randomNetworkDoc()); // execute monitor 1 Response executeResponse = executeAlertingMonitor(monitorId1, Collections.emptyMap()); Map executeResults = entityAsMap(executeResponse); From 5ddaab62681a7dc27d4e0130de3d33e93f12b237 Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Mon, 12 Feb 2024 09:55:29 -0800 Subject: [PATCH 10/15] fix correlation tests Signed-off-by: Joanne Wang --- .../java/org/opensearch/securityanalytics/TestHelpers.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index 82f9ea507..1ff8b640c 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -1968,6 +1968,7 @@ public static String randomVpcFlowDoc() { " \"srcport\": 9000,\n" + " \"dstport\": 8000,\n" + " \"severity_id\": \"-1\",\n" + + " \"id.orig_h\": \"1.2.3.4\",\n" + " \"class_name\": \"Network Activity\"\n" + "}"; } @@ -1976,7 +1977,9 @@ public static String randomAdLdapDoc() { return "{\n" + " \"azure.platformlogs.result_type\": 50126,\n" + " \"azure.signinlogs.result_description\": \"Invalid username or password or Invalid on-premises username or password.\",\n" + + " \"azure.signinlogs.properties.user_id\": \"TESTUSER\",\n" + " \"azure.signinlogs.props.user_id\": \"DEYSUBHO\"\n" + + "}"; } From 444472d47cb5790457b05f41ebe46a7c2b3108e2 Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Mon, 12 Feb 2024 10:11:10 -0800 Subject: [PATCH 11/15] working all tests Signed-off-by: Joanne Wang --- .../java/org/opensearch/securityanalytics/TestHelpers.java | 5 ++--- .../correlation/CorrelationEngineRestApiIT.java | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index 1ff8b640c..b5608f573 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -1977,9 +1977,8 @@ public static String randomAdLdapDoc() { return "{\n" + " \"azure.platformlogs.result_type\": 50126,\n" + " \"azure.signinlogs.result_description\": \"Invalid username or password or Invalid on-premises username or password.\",\n" + - " \"azure.signinlogs.properties.user_id\": \"TESTUSER\",\n" + - " \"azure.signinlogs.props.user_id\": \"DEYSUBHO\"\n" + - +// " \"azure.signinlogs.properties.user_id\": \"TESTUSER\",\n" + + " \"azure.signinlogs.properties.user_id\": \"DEYSUBHO\"\n" + "}"; } diff --git a/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java index 149f8fd34..a4cdb6d1c 100644 --- a/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java @@ -1109,7 +1109,7 @@ private String createAdLdapDetector(String indexName) throws IOException { " \"partial\": true,\n" + " \"alias_mappings\": {\n" + " \"properties\": {\n" + - " \"azure-signinlogs-properties-user_id\": {\n" + + " \"azure.signinlogs.properties.user_id\": {\n" + " \"path\": \"azure.signinlogs.props.user_id\",\n" + " \"type\": \"alias\"\n" + " },\n" + From fa2b9ae1951208ee313ece135d88666f9f2f907d Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Mon, 12 Feb 2024 10:24:04 -0800 Subject: [PATCH 12/15] moved test and changed alias for adldap Signed-off-by: Joanne Wang --- .../securityanalytics/TestHelpers.java | 3 +- .../securityanalytics/findings/FindingIT.java | 97 ++++++++++++++++++- .../resthandler/DetectorMonitorRestApiIT.java | 89 ----------------- 3 files changed, 94 insertions(+), 95 deletions(-) diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index b5608f573..087403c80 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -1977,8 +1977,7 @@ public static String randomAdLdapDoc() { return "{\n" + " \"azure.platformlogs.result_type\": 50126,\n" + " \"azure.signinlogs.result_description\": \"Invalid username or password or Invalid on-premises username or password.\",\n" + -// " \"azure.signinlogs.properties.user_id\": \"TESTUSER\",\n" + - " \"azure.signinlogs.properties.user_id\": \"DEYSUBHO\"\n" + + " \"azure.signinlogs.props.user_id\": \"DEYSUBHO\"\n" + "}"; } diff --git a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java index c9da0e9fa..89aecdf31 100644 --- a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java +++ b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java @@ -6,15 +6,13 @@ package org.opensearch.securityanalytics.findings; import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.stream.Collectors; import org.apache.hc.core5.http.HttpStatus; import org.junit.Assert; import org.junit.Ignore; +import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; @@ -22,11 +20,13 @@ import org.opensearch.search.SearchHit; import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; +import org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig; import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.model.DetectorInput; import org.opensearch.securityanalytics.model.DetectorRule; import org.opensearch.securityanalytics.model.DetectorTrigger; +import static java.util.Collections.emptyList; import static org.opensearch.securityanalytics.TestHelpers.*; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_INDEX_MAX_AGE; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_MAX_DOCS; @@ -395,6 +395,95 @@ public void testGetFindings_rolloverByMaxDoc_success() throws IOException, Inter restoreAlertsFindingsIMSettings(); } + public void testCreateDetectorWithNotCondition_verifyFindings_success() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response createMappingResponse = client().performRequest(createMappingRequest); + + assertEquals(HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode()); + + // Create random custom doc rule with NOT condition + String randomDocRuleId = createRule(randomRuleWithNot()); + DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(randomDocRuleId)), + emptyList()); + Detector detector = randomDetectorWithInputs(List.of(input)); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + + assertEquals(1, response.getHits().getTotalHits().value); + + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + Map responseBody = asMap(createResponse); + + String detectorId = responseBody.get("_id").toString(); + request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + Map detectorMap = (HashMap) (hit.getSourceAsMap().get("detector")); + List inputArr = (List) detectorMap.get("inputs"); + + assertEquals(1, ((Map>) inputArr.get(0)).get("detector_input").get("custom_rules").size()); + + List monitorIds = ((List) (detectorMap).get("monitor_id")); + assertEquals(1, monitorIds.size()); + + String monitorId = monitorIds.get(0); + + // Verify findings + indexDoc(index, "1", randomDoc(2, 5, "Test")); + indexDoc(index, "2", randomDocWithoutAccountName(3, 5, "Test")); + indexDoc(index, "3", randomDoc(3, 5, "Test")); + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + + // Verify 1 custom rule + assertEquals(1, noOfSigmaRuleMatches); + + Map params = new HashMap<>(); + params.put("detector_id", detectorId); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + + assertNotNull(getFindingsBody); + // When doc level monitor is being applied one finding is generated per document + assertEquals(3, getFindingsBody.get("total_findings")); + + List> findings = (List) getFindingsBody.get("findings"); + List foundDocIds = new ArrayList<>(); + for (Map finding : findings) { + List findingDocs = (List) finding.get("related_doc_ids"); + Assert.assertEquals(1, findingDocs.size()); + foundDocIds.addAll(findingDocs); + } + assertTrue(Arrays.asList("1", "2", "3").containsAll(foundDocIds)); + } + public void testGetFindings_rolloverByMaxDoc_short_retention_success() throws IOException, InterruptedException { updateClusterSetting(FINDING_HISTORY_ROLLOVER_PERIOD.getKey(), "1s"); updateClusterSetting(FINDING_HISTORY_MAX_DOCS.getKey(), "1"); diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java index b46339fdf..65c2c60db 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java @@ -2118,95 +2118,6 @@ public void testCreateDetectorWithCloudtrailAggrRuleWithEcsFields() throws IOExc assertEquals(1, getFindingsBody.get("total_findings")); } - public void testCreateDetectorWithNotCondition_verifyFindings_success() throws IOException { - String index = createTestIndex(randomIndex(), windowsIndexMapping()); - - // Execute CreateMappingsAction to add alias mapping for index - Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); - // both req params and req body are supported - createMappingRequest.setJsonEntity( - "{ \"index_name\":\"" + index + "\"," + - " \"rule_topic\":\"" + randomDetectorType() + "\", " + - " \"partial\":true" + - "}" - ); - - Response createMappingResponse = client().performRequest(createMappingRequest); - - assertEquals(HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode()); - - // Create random custom doc rule with NOT condition - String randomDocRuleId = createRule(randomRuleWithNot()); - DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(randomDocRuleId)), - emptyList()); - Detector detector = randomDetectorWithInputs(List.of(input)); - - Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); - - String request = "{\n" + - " \"query\" : {\n" + - " \"match_all\":{\n" + - " }\n" + - " }\n" + - "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); - - assertEquals(1, response.getHits().getTotalHits().value); - - assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); - Map responseBody = asMap(createResponse); - - String detectorId = responseBody.get("_id").toString(); - request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId + "\"\n" + - " }\n" + - " }\n" + - "}"; - List hits = executeSearch(Detector.DETECTORS_INDEX, request); - SearchHit hit = hits.get(0); - Map detectorMap = (HashMap) (hit.getSourceAsMap().get("detector")); - List inputArr = (List) detectorMap.get("inputs"); - - assertEquals(1, ((Map>) inputArr.get(0)).get("detector_input").get("custom_rules").size()); - - List monitorIds = ((List) (detectorMap).get("monitor_id")); - assertEquals(1, monitorIds.size()); - - String monitorId = monitorIds.get(0); - - // Verify findings - indexDoc(index, "1", randomDoc(2, 5, "Test")); - indexDoc(index, "2", randomDocWithoutAccountName(3, 5, "Test")); - indexDoc(index, "3", randomDoc(3, 5, "Test")); - - Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); - Map executeResults = entityAsMap(executeResponse); - int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); - - // Verify 1 custom rule - assertEquals(1, noOfSigmaRuleMatches); - - Map params = new HashMap<>(); - params.put("detector_id", detectorId); - Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); - Map getFindingsBody = entityAsMap(getFindingsResponse); - - assertNotNull(getFindingsBody); - // When doc level monitor is being applied one finding is generated per document - assertEquals(3, getFindingsBody.get("total_findings")); - - List> findings = (List) getFindingsBody.get("findings"); - List foundDocIds = new ArrayList<>(); - for (Map finding : findings) { - List findingDocs = (List) finding.get("related_doc_ids"); - Assert.assertEquals(1, findingDocs.size()); - foundDocIds.addAll(findingDocs); - } - assertTrue(Arrays.asList("1", "2", "3").containsAll(foundDocIds)); - } - private static void assertRuleMonitorFinding(Map executeResults, String ruleId, int expectedDocCount, List expectedTriggerResult) { List> buckets = ((List>) (((Map) ((Map) ((Map) ((List) ((Map) executeResults.get("input_results")).get("results")).get(0)).get("aggregations")).get("result_agg")).get("buckets"))); Integer docCount = buckets.stream().mapToInt(it -> (Integer) it.get("doc_count")).sum(); From ec8d4c54db89b108f06d5c7b03ad6b36770c9d9f Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Mon, 12 Feb 2024 11:35:22 -0800 Subject: [PATCH 13/15] added more tests Signed-off-by: Joanne Wang --- .../securityanalytics/TestHelpers.java | 45 +++- .../securityanalytics/findings/FindingIT.java | 216 +++++++++++++++++- .../resthandler/DetectorMonitorRestApiIT.java | 25 +- 3 files changed, 271 insertions(+), 15 deletions(-) diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index 087403c80..2902dbaa7 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -259,7 +259,7 @@ public static String randomRule() { "level: high"; } - public static String randomRuleWithNot() { + public static String randomRuleWithNotCondition() { return "title: Remote Encrypting File System Abuse\n" + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" + @@ -280,13 +280,46 @@ public static String randomRuleWithNot() { " category: application\n" + " definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" + "detection:\n" + - " selection:\n" + - " AccountType: whatever\n" + + " selection1:\n" + + " AccountType: TestAccountType\n" + " selection2:\n" + - " AccountName: SYSTEM\n" + + " AccountName: TestAccountName\n" + " selection3:\n" + " EventID: 22\n" + - " condition: (not selection or selection2) and selection3\n" + + " condition: (not selection1 and not selection2) and selection3\n" + + "falsepositives:\n" + + " - Legitimate usage of remote file encryption\n" + + "level: high"; + } + + public static String randomRuleWithNotConditionBoolAndNum() { + return "title: Remote Encrypting File System Abuse\n" + + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + + "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" + + "references:\n" + + " - https://attack.mitre.org/tactics/TA0008/\n" + + " - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" + + " - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" + + " - https://github.com/zeronetworks/rpcfirewall\n" + + " - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" + + "tags:\n" + + " - attack.defense_evasion\n" + + "status: experimental\n" + + "author: Sagie Dulce, Dekel Paz\n" + + "date: 2022/01/01\n" + + "modified: 2022/01/01\n" + + "logsource:\n" + + " product: rpc_firewall\n" + + " category: application\n" + + " definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" + + "detection:\n" + + " selection1:\n" + + " Initiated: \"false\"\n" + + " selection2:\n" + + " AccountName: TestAccountName\n" + + " selection3:\n" + + " EventID: 21\n" + + " condition: not selection1 and not selection3\n" + "falsepositives:\n" + " - Legitimate usage of remote file encryption\n" + "level: high"; @@ -1734,7 +1767,7 @@ public static String randomDoc(int severity, int version, String opCode) { } - public static String randomDocWithoutAccountName(int severity, int version, String opCode) { + public static String randomDocForNotCondition(int severity, int version, String opCode) { String doc = "{\n" + "\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" + "\"HostName\":\"EC2AMAZ-EPO7HKA\",\n" + diff --git a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java index 89aecdf31..b7975593a 100644 --- a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java +++ b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java @@ -11,11 +11,11 @@ import org.apache.hc.core5.http.HttpStatus; import org.junit.Assert; -import org.junit.Ignore; import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; +import org.opensearch.commons.alerting.model.Monitor; import org.opensearch.core.rest.RestStatus; import org.opensearch.search.SearchHit; import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; @@ -27,7 +27,17 @@ import org.opensearch.securityanalytics.model.DetectorTrigger; import static java.util.Collections.emptyList; -import static org.opensearch.securityanalytics.TestHelpers.*; +import static org.opensearch.securityanalytics.TestHelpers.netFlowMappings; +import static org.opensearch.securityanalytics.TestHelpers.randomDetectorType; +import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithTriggers; +import static org.opensearch.securityanalytics.TestHelpers.randomDoc; +import static org.opensearch.securityanalytics.TestHelpers.randomIndex; +import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; +import static org.opensearch.securityanalytics.TestHelpers.randomRuleWithNotConditionBoolAndNum; +import static org.opensearch.securityanalytics.TestHelpers.randomNetworkDoc; +import static org.opensearch.securityanalytics.TestHelpers.randomDocForNotCondition; +import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputs; +import static org.opensearch.securityanalytics.TestHelpers.randomRuleWithNotCondition; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_INDEX_MAX_AGE; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_MAX_DOCS; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_RETENTION_PERIOD; @@ -412,8 +422,201 @@ public void testCreateDetectorWithNotCondition_verifyFindings_success() throws I assertEquals(HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode()); + // Create random doc rule + String randomDocRuleId = createRule(randomRuleWithNotCondition()); + List prepackagedRules = getRandomPrePackagedRules(); + DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(randomDocRuleId)), + prepackagedRules.stream().map(DetectorRule::new).collect(Collectors.toList())); + Detector detector = randomDetectorWithInputs(List.of(input)); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map updateResponseBody = asMap(createResponse); + String detectorId = updateResponseBody.get("_id").toString(); + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + + // Verify newly created doc level monitor + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + Map detectorAsMap = (Map) hit.getSourceAsMap().get("detector"); + List monitorIds = ((List) (detectorAsMap).get("monitor_id")); + + assertEquals(1, monitorIds.size()); + + String monitorId = monitorIds.get(0); + String monitorType = ((Map) entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + monitorId))).get("monitor")).get("monitor_type"); + + assertEquals(Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue(), monitorType); + + // Verify rules + request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + + assertEquals(6, response.getHits().getTotalHits().value); + + // Verify findings + indexDoc(index, "1", randomDoc(2, 5, "Test")); + indexDoc(index, "2", randomDoc(3, 5, "Test")); + + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + // Verify 5 prepackaged rules and 1 custom rule + assertEquals(6, noOfSigmaRuleMatches); + + Map params = new HashMap<>(); + params.put("detector_id", detectorId); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + + assertNotNull(getFindingsBody); + // When doc level monitor is being applied one finding is generated per document + assertEquals(2, getFindingsBody.get("total_findings")); + + Set docRuleIds = new HashSet<>(prepackagedRules); + docRuleIds.add(randomDocRuleId); + + List> findings = (List) getFindingsBody.get("findings"); + List foundDocIds = new ArrayList<>(); + for (Map finding : findings) { + Set aggRulesFinding = ((List>) finding.get("queries")).stream().map(it -> it.get("id").toString()).collect( + Collectors.toSet()); + + assertTrue(docRuleIds.containsAll(aggRulesFinding)); + + List findingDocs = (List) finding.get("related_doc_ids"); + Assert.assertEquals(1, findingDocs.size()); + foundDocIds.addAll(findingDocs); + } + assertTrue(Arrays.asList("1", "2").containsAll(foundDocIds)); + } + + public void testCreateDetectorWithNotCondition_verifyFindings_success_boolAndNum() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response createMappingResponse = client().performRequest(createMappingRequest); + + assertEquals(HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode()); + + // Create random custom doc rule with NOT condition + String randomDocRuleId = createRule(randomRuleWithNotConditionBoolAndNum()); + DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(randomDocRuleId)), + emptyList()); + Detector detector = randomDetectorWithInputs(List.of(input)); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + + assertEquals(1, response.getHits().getTotalHits().value); + + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + Map responseBody = asMap(createResponse); + + String detectorId = responseBody.get("_id").toString(); + request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + Map detectorMap = (HashMap) (hit.getSourceAsMap().get("detector")); + List inputArr = (List) detectorMap.get("inputs"); + + assertEquals(1, ((Map>) inputArr.get(0)).get("detector_input").get("custom_rules").size()); + + List monitorIds = ((List) (detectorMap).get("monitor_id")); + assertEquals(1, monitorIds.size()); + + String monitorId = monitorIds.get(0); + + // Verify findings + indexDoc(index, "1", randomDoc(2, 5, "Test")); + indexDoc(index, "2", randomDoc(2, 5, "Test")); + + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + + // Verify 1 custom rule + assertEquals(1, noOfSigmaRuleMatches); + + Map params = new HashMap<>(); + params.put("detector_id", detectorId); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + + assertNotNull(getFindingsBody); + // When doc level monitor is being applied one finding is generated per document + assertEquals(2, getFindingsBody.get("total_findings")); + + List> findings = (List) getFindingsBody.get("findings"); + List foundDocIds = new ArrayList<>(); + for (Map finding : findings) { + List findingDocs = (List) finding.get("related_doc_ids"); + Assert.assertEquals(1, findingDocs.size()); + foundDocIds.addAll(findingDocs); + } + assertTrue(Arrays.asList("1", "2").containsAll(foundDocIds)); + } + + /* + Create a detector with custom rules that include a "not" condition in the sigma rule. + Insert two test documents one matching the rule and one without the field matching the condition to generate only one finding + */ + public void testCreateDetectorWithNotCondition_verifyFindingsAndNoFindings_success() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response createMappingResponse = client().performRequest(createMappingRequest); + + assertEquals(HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode()); + // Create random custom doc rule with NOT condition - String randomDocRuleId = createRule(randomRuleWithNot()); + String randomDocRuleId = createRule(randomRuleWithNotCondition()); DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(randomDocRuleId)), emptyList()); Detector detector = randomDetectorWithInputs(List.of(input)); @@ -455,8 +658,7 @@ public void testCreateDetectorWithNotCondition_verifyFindings_success() throws I // Verify findings indexDoc(index, "1", randomDoc(2, 5, "Test")); - indexDoc(index, "2", randomDocWithoutAccountName(3, 5, "Test")); - indexDoc(index, "3", randomDoc(3, 5, "Test")); + indexDoc(index, "4", randomDoc(2, 5, "Test")); Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); Map executeResults = entityAsMap(executeResponse); @@ -472,7 +674,7 @@ public void testCreateDetectorWithNotCondition_verifyFindings_success() throws I assertNotNull(getFindingsBody); // When doc level monitor is being applied one finding is generated per document - assertEquals(3, getFindingsBody.get("total_findings")); + assertEquals(2, getFindingsBody.get("total_findings")); List> findings = (List) getFindingsBody.get("findings"); List foundDocIds = new ArrayList<>(); @@ -481,7 +683,7 @@ public void testCreateDetectorWithNotCondition_verifyFindings_success() throws I Assert.assertEquals(1, findingDocs.size()); foundDocIds.addAll(findingDocs); } - assertTrue(Arrays.asList("1", "2", "3").containsAll(foundDocIds)); + assertTrue(Arrays.asList("1", "4").containsAll(foundDocIds)); } public void testGetFindings_rolloverByMaxDoc_short_retention_success() throws IOException, InterruptedException { diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java index 65c2c60db..3a11300ee 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java @@ -37,7 +37,28 @@ import static java.util.Collections.emptyList; -import static org.opensearch.securityanalytics.TestHelpers.*; +import static org.opensearch.securityanalytics.TestHelpers.cloudtrailOcsfMappings; +import static org.opensearch.securityanalytics.TestHelpers.randomAggregationRule; +import static org.opensearch.securityanalytics.TestHelpers.randomCloudtrailAggrRule; +import static org.opensearch.securityanalytics.TestHelpers.randomCloudtrailAggrRuleWithDotFields; +import static org.opensearch.securityanalytics.TestHelpers.randomCloudtrailAggrRuleWithEcsFields; +import static org.opensearch.securityanalytics.TestHelpers.randomCloudtrailDoc; +import static org.opensearch.securityanalytics.TestHelpers.randomCloudtrailOcsfDoc; +import static org.opensearch.securityanalytics.TestHelpers.randomDetector; +import static org.opensearch.securityanalytics.TestHelpers.randomDetectorType; +import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputs; +import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputsAndTriggers; +import static org.opensearch.securityanalytics.TestHelpers.randomDoc; +import static org.opensearch.securityanalytics.TestHelpers.randomIndex; +import static org.opensearch.securityanalytics.TestHelpers.randomRule; +import static org.opensearch.securityanalytics.TestHelpers.randomRuleWithKeywords; +import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; +import static org.opensearch.securityanalytics.TestHelpers.randomRuleWithStringKeywords; +import static org.opensearch.securityanalytics.TestHelpers.randomDocOnlyNumericAndDate; +import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMappingOnlyNumericAndDate; +import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMappingOnlyNumericAndText; +import static org.opensearch.securityanalytics.TestHelpers.randomRuleWithDateKeywords; +import static org.opensearch.securityanalytics.TestHelpers.randomDocOnlyNumericAndText; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE; public class DetectorMonitorRestApiIT extends SecurityAnalyticsRestTestCase { @@ -1000,7 +1021,7 @@ public void testCreateDetector_verifyWorkflowCreation_success_WithoutGroupByRule String testOpCode = "Test"; String maxRuleId = createRule(randomAggregationRule("max", " > 3", testOpCode)); - String randomDocRuleId = createRule(randomRuleWithNot()); + String randomDocRuleId = createRule(randomRule()); List detectorRules = List.of(new DetectorRule(maxRuleId), new DetectorRule(randomDocRuleId)); DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), detectorRules, emptyList()); From 5efdc79978fad8a4e771e13b59b1679c20b2f3e3 Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Mon, 12 Feb 2024 12:23:27 -0800 Subject: [PATCH 14/15] cleanup code Signed-off-by: Joanne Wang --- .../rules/backend/OSQueryBackend.java | 4 +-- .../rules/backend/QueryBackend.java | 34 ++++++++----------- .../securityanalytics/alerts/AlertsIT.java | 13 ++++++- .../securityanalytics/findings/FindingIT.java | 12 ++++++- .../rules/backend/QueryBackendTests.java | 11 ++---- 5 files changed, 42 insertions(+), 32 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java index b4e3d60a6..58069042a 100644 --- a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java +++ b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java @@ -4,8 +4,6 @@ */ package org.opensearch.securityanalytics.rules.backend; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.opensearch.OpenSearchParseException; import org.opensearch.common.UUIDs; import org.opensearch.core.common.bytes.BytesReference; @@ -286,7 +284,7 @@ public Object convertConditionNot(ConditionNOT condition, boolean isConditionNot @Override public Object convertExistsField(ConditionFieldEqualsValueExpression condition) { String field = getFinalField(condition.getField()); - return String.format(Locale.getDefault(),tokenSeparator + this.andToken + this.tokenSeparator + this.existsToken + this.eqToken + " _exists_" + field); + return String.format(Locale.getDefault(),tokenSeparator + this.andToken + this.tokenSeparator + this.existsToken + this.eqToken + this.tokenSeparator + this.existsToken + field); } @Override diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java index fd484950f..2c56a2c6a 100644 --- a/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java +++ b/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java @@ -4,8 +4,6 @@ */ package org.opensearch.securityanalytics.rules.backend; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.opensearch.securityanalytics.rules.aggregation.AggregationItem; import org.opensearch.securityanalytics.rules.backend.OSQueryBackend.AggregationQueries; import org.opensearch.securityanalytics.rules.condition.ConditionAND; @@ -31,12 +29,22 @@ import org.opensearch.securityanalytics.rules.utils.AnyOneOf; import org.opensearch.securityanalytics.rules.utils.Either; import org.apache.commons.lang3.tuple.Pair; - -import java.util.*; +import org.yaml.snakeyaml.LoaderOptions; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.constructor.SafeConstructor; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; public abstract class QueryBackend { - - private static final Logger log = LogManager.getLogger(QueryBackend.class); private boolean convertOrAsIn; private boolean convertAndAsIn; private boolean collectErrors; @@ -85,8 +93,6 @@ public List convertRule(SigmaRule rule) throws SigmaError { query = this.convertCondition(new ConditionType(Either.right(Either.right((ConditionValueExpression) conditionItem))), false, false); } queries.add(query); -// log.debug("converted query"); -// log.debug(query); if (aggItem != null) { aggItem.setTimeframe(rule.getDetection().getTimeframe()); queries.add(convertAggregation(aggItem)); @@ -120,7 +126,7 @@ public Object convertCondition(ConditionType conditionType, boolean isConditionN } else if (conditionType.isConditionNOT()) { return this.convertConditionNot(conditionType.getConditionNOT(), isConditionNot, applyDeMorgans); } else if (conditionType.isEqualsValueExpression()) { - // add a check to see if it should be done, then call another method to add them together else, return as normal BUT the check needs to see if top parent is NOT + // check to see if conditionNot is an ancestor of the parse tree, otherwise return as normal if (isConditionNot) { return this.convertConditionFieldEqValNot(conditionType, isConditionNot, applyDeMorgans); } else { @@ -134,18 +140,8 @@ public Object convertCondition(ConditionType conditionType, boolean isConditionN } public String convertConditionFieldEqValNot(ConditionType conditionType, boolean isConditionNot, boolean applyDeMorgans) throws SigmaValueError { -// String baseString; -// String exprWithDeMorgansApplied = "NOT " + "%s"; -// if (applyDeMorgans) { -// baseString = String.format(Locale.getDefault(), exprWithDeMorgansApplied, this.convertConditionFieldEqVal(conditionType.getEqualsValueExpression(), isConditionNot, applyDeMorgans).toString()); -// } else { -// baseString = this.convertConditionFieldEqVal(conditionType.getEqualsValueExpression(), isConditionNot, applyDeMorgans).toString(); -// } - String baseString = this.convertConditionFieldEqVal(conditionType.getEqualsValueExpression(), isConditionNot, applyDeMorgans).toString(); String addExists = this.convertExistsField(conditionType.getEqualsValueExpression()).toString(); -// log.error("I AM HERE"); -// log.error(String.format(Locale.getDefault(), ("%s" + "%s"), baseString, addExists)); return String.format(Locale.getDefault(), ("%s" + "%s"), baseString, addExists); } diff --git a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java index b3b3a940d..347fb66f1 100644 --- a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java +++ b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java @@ -35,7 +35,18 @@ import org.opensearch.securityanalytics.model.DetectorRule; import org.opensearch.securityanalytics.model.DetectorTrigger; -import static org.opensearch.securityanalytics.TestHelpers.*; +import static org.opensearch.securityanalytics.TestHelpers.netFlowMappings; +import static org.opensearch.securityanalytics.TestHelpers.randomAction; +import static org.opensearch.securityanalytics.TestHelpers.randomDetectorType; +import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputsAndThreatIntel; +import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputsAndTriggers; +import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithTriggers; +import static org.opensearch.securityanalytics.TestHelpers.randomDoc; +import static org.opensearch.securityanalytics.TestHelpers.randomDocWithIpIoc; +import static org.opensearch.securityanalytics.TestHelpers.randomNetworkDoc; +import static org.opensearch.securityanalytics.TestHelpers.randomIndex; +import static org.opensearch.securityanalytics.TestHelpers.randomRule; +import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_INDEX_MAX_AGE; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_MAX_DOCS; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_RETENTION_PERIOD; diff --git a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java index b7975593a..1f7d112de 100644 --- a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java +++ b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java @@ -6,11 +6,19 @@ package org.opensearch.securityanalytics.findings; import java.io.IOException; -import java.util.*; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.HashSet; +import java.util.ArrayList; +import java.util.Arrays; import java.util.stream.Collectors; import org.apache.hc.core5.http.HttpStatus; import org.junit.Assert; +import org.junit.Ignore; import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Request; import org.opensearch.client.Response; @@ -658,6 +666,8 @@ public void testCreateDetectorWithNotCondition_verifyFindingsAndNoFindings_succe // Verify findings indexDoc(index, "1", randomDoc(2, 5, "Test")); + indexDoc(index, "2", randomDocForNotCondition(2, 5, "Test")); + indexDoc(index, "3", randomDocForNotCondition(2, 5, "Test")); indexDoc(index, "4", randomDoc(2, 5, "Test")); Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); diff --git a/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java b/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java index 3c8e6bf59..2fc31e794 100644 --- a/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java +++ b/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java @@ -756,7 +756,7 @@ public void testConvertNotComplicatedExpression() throws IOException, SigmaError " selection1:\n" + " CommandLine|endswith: '.cpl'\n" + " filter:\n" + - " CommandLine|contains\n" + + " CommandLine|contains:\n" + " - '\\System32\\'\n" + " - '%System%'\n" + " fp1_igfx:\n" + @@ -764,13 +764,8 @@ public void testConvertNotComplicatedExpression() throws IOException, SigmaError " - 'regsvr32 '\n" + " - ' /s '\n" + " - 'igfxCPL.cpl'\n" + - " selection2:\n" + - " Image|endswith: '\\reg.exe'\n" + - " CommandLine|contains: 'add'\n" + - " selection3:\n" + - " CommandLine|contains: 'CurrentVersion\\Control Panel\\CPLs'\n" + - " condition: (selection1 and not filter and not fp1_igfx) or (selection2 and selection3)", false)); - Assert.assertEquals("(((NOT Opcode: \"Info\" AND _exists_: _exists_Opcode) AND (NOT Severity: \"value2\" AND _exists_: _exists_Severity)))", queries.get(0).toString()); + " condition: selection1 and not filter and not fp1_igfx", false)); + Assert.assertEquals("((CommandLine: *.cpl) AND ((((NOT CommandLine: *\\\\System32\\\\* AND _exists_: _exists_CommandLine) AND (NOT CommandLine: *%System%* AND _exists_: _exists_CommandLine))))) AND ((((NOT CommandLine: *regsvr32_ws_* AND _exists_: _exists_CommandLine) OR (NOT CommandLine: *_ws_\\/s_ws_* AND _exists_: _exists_CommandLine) OR (NOT CommandLine: *igfxCPL.cpl* AND _exists_: _exists_CommandLine))))", queries.get(0).toString()); } public void testConvertNotWithAnd() throws IOException, SigmaError { From e774d029cb8c5260b71435a38b69625e10327780 Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Thu, 29 Feb 2024 15:01:30 -0800 Subject: [PATCH 15/15] remove exists flag Signed-off-by: Joanne Wang --- .../rules/backend/OSQueryBackend.java | 2 +- .../rules/backend/QueryBackendTests.java | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java index 58069042a..9c8ccc87a 100644 --- a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java +++ b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java @@ -284,7 +284,7 @@ public Object convertConditionNot(ConditionNOT condition, boolean isConditionNot @Override public Object convertExistsField(ConditionFieldEqualsValueExpression condition) { String field = getFinalField(condition.getField()); - return String.format(Locale.getDefault(),tokenSeparator + this.andToken + this.tokenSeparator + this.existsToken + this.eqToken + this.tokenSeparator + this.existsToken + field); + return String.format(Locale.getDefault(),tokenSeparator + this.andToken + this.tokenSeparator + this.existsToken + this.eqToken + this.tokenSeparator + field); } @Override diff --git a/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java b/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java index 2fc31e794..f56cdc6d9 100644 --- a/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java +++ b/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java @@ -714,7 +714,7 @@ public void testConvertNot() throws IOException, SigmaError { " sel:\n" + " fieldA: value1\n" + " condition: not sel", false)); - Assert.assertEquals("(NOT fieldA: \"value1\" AND _exists_: _exists_fieldA)", queries.get(0).toString()); + Assert.assertEquals("(NOT fieldA: \"value1\" AND _exists_: fieldA)", queries.get(0).toString()); } public void testConvertNotWithParenthesis() throws IOException, SigmaError { @@ -736,7 +736,7 @@ public void testConvertNotWithParenthesis() throws IOException, SigmaError { " sel2:\n" + " Severity: value2\n" + " condition: not (sel1 or sel2)", false)); - Assert.assertEquals("(((NOT Opcode: \"Info\" AND _exists_: _exists_Opcode) AND (NOT Severity: \"value2\" AND _exists_: _exists_Severity)))", queries.get(0).toString()); + Assert.assertEquals("(((NOT Opcode: \"Info\" AND _exists_: Opcode) AND (NOT Severity: \"value2\" AND _exists_: Severity)))", queries.get(0).toString()); } public void testConvertNotComplicatedExpression() throws IOException, SigmaError { @@ -765,7 +765,9 @@ public void testConvertNotComplicatedExpression() throws IOException, SigmaError " - ' /s '\n" + " - 'igfxCPL.cpl'\n" + " condition: selection1 and not filter and not fp1_igfx", false)); - Assert.assertEquals("((CommandLine: *.cpl) AND ((((NOT CommandLine: *\\\\System32\\\\* AND _exists_: _exists_CommandLine) AND (NOT CommandLine: *%System%* AND _exists_: _exists_CommandLine))))) AND ((((NOT CommandLine: *regsvr32_ws_* AND _exists_: _exists_CommandLine) OR (NOT CommandLine: *_ws_\\/s_ws_* AND _exists_: _exists_CommandLine) OR (NOT CommandLine: *igfxCPL.cpl* AND _exists_: _exists_CommandLine))))", queries.get(0).toString()); + Assert.assertEquals("((CommandLine: *.cpl) AND ((((NOT CommandLine: *\\\\System32\\\\* AND _exists_: CommandLine) AND " + + "(NOT CommandLine: *%System%* AND _exists_: CommandLine))))) AND ((((NOT CommandLine: *regsvr32_ws_* AND _exists_: CommandLine) OR " + + "(NOT CommandLine: *_ws_\\/s_ws_* AND _exists_: CommandLine) OR (NOT CommandLine: *igfxCPL.cpl* AND _exists_: CommandLine))))", queries.get(0).toString()); } public void testConvertNotWithAnd() throws IOException, SigmaError { @@ -788,7 +790,7 @@ public void testConvertNotWithAnd() throws IOException, SigmaError { " filter:\n" + " Details: '%CommonProgramFiles%\\System\\wab32.dll'\n" + " condition: selection and not filter", false)); - Assert.assertEquals("((EventType: \"SetValue\") AND (TargetObject: *\\\\Software\\\\Microsoft\\\\WAB\\\\DLLPath)) AND ((NOT Details: \"%CommonProgramFiles%\\\\System\\\\wab32.dll\" AND _exists_: _exists_Details))", queries.get(0).toString()); + Assert.assertEquals("((EventType: \"SetValue\") AND (TargetObject: *\\\\Software\\\\Microsoft\\\\WAB\\\\DLLPath)) AND ((NOT Details: \"%CommonProgramFiles%\\\\System\\\\wab32.dll\" AND _exists_: Details))", queries.get(0).toString()); } public void testConvertNotWithOrAndList() throws IOException, SigmaError { @@ -813,7 +815,7 @@ public void testConvertNotWithOrAndList() throws IOException, SigmaError { " - resp_mime_types|contains: 'dosexec'\n" + " - c-uri|endswith: '.exe'\n" + " condition: not sel1 or sel3", false)); - Assert.assertEquals("((((NOT field1: \"valueA1\" AND _exists_: _exists_field1) OR (NOT field2: \"valueA2\" AND _exists_: _exists_field2) OR (NOT field3: \"valueA3\" AND _exists_: _exists_field3)))) OR ((resp_mime_types: *dosexec*) OR (c-uri: *.exe))", queries.get(0).toString()); + Assert.assertEquals("((((NOT field1: \"valueA1\" AND _exists_: field1) OR (NOT field2: \"valueA2\" AND _exists_: field2) OR (NOT field3: \"valueA3\" AND _exists_: field3)))) OR ((resp_mime_types: *dosexec*) OR (c-uri: *.exe))", queries.get(0).toString()); } public void testConvertNotWithNumAndBool() throws IOException, SigmaError { @@ -835,7 +837,7 @@ public void testConvertNotWithNumAndBool() throws IOException, SigmaError { " sel2:\n" + " field2: true\n" + " condition: not sel1 and not sel2", false)); - Assert.assertEquals("((NOT field1: 1 AND _exists_: _exists_field1)) AND ((NOT field2: true AND _exists_: _exists_field2))", queries.get(0).toString()); + Assert.assertEquals("((NOT field1: 1 AND _exists_: field1)) AND ((NOT field2: true AND _exists_: field2))", queries.get(0).toString()); } public void testConvertNotWithNull() throws IOException, SigmaError { @@ -857,7 +859,7 @@ public void testConvertNotWithNull() throws IOException, SigmaError { " sel2:\n" + " fieldB: true\n" + " condition: not sel1", false)); - Assert.assertEquals("(NOT fieldA: (NOT [* TO *]) AND _exists_: _exists_fieldA)", queries.get(0).toString()); + Assert.assertEquals("(NOT fieldA: (NOT [* TO *]) AND _exists_: fieldA)", queries.get(0).toString()); } public void testConvertNotWithKeywords() throws IOException, SigmaError { @@ -908,7 +910,7 @@ public void testConvertPrecedence() throws IOException, SigmaError { " sel4:\n" + " fieldD: value5\n" + " condition: (sel1 or sel2) and not (sel3 and sel4)", false)); - Assert.assertEquals("((fieldA: \"value1\") OR (mappedB: \"value2\")) AND ((((NOT fieldC: \"value4\" AND _exists_: _exists_fieldC) OR (NOT fieldD: \"value5\" AND _exists_: _exists_fieldD))))", queries.get(0).toString()); + Assert.assertEquals("((fieldA: \"value1\") OR (mappedB: \"value2\")) AND ((((NOT fieldC: \"value4\" AND _exists_: fieldC) OR (NOT fieldD: \"value5\" AND _exists_: fieldD))))", queries.get(0).toString()); } public void testConvertMultiConditions() throws IOException, SigmaError {