diff --git a/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java b/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java index 459f523b7..7cb8d90af 100644 --- a/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java +++ b/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java @@ -4,22 +4,15 @@ */ package org.opensearch.securityanalytics.config.monitors; -import java.util.List; -import java.util.stream.Collectors; -import org.opensearch.common.inject.Inject; -import org.opensearch.securityanalytics.logtype.LogTypeService; -import org.opensearch.securityanalytics.model.Detector; - -import java.util.Arrays; import java.util.HashMap; import java.util.Locale; import java.util.Map; -import org.opensearch.securityanalytics.model.LogType; public class DetectorMonitorConfig { public static final String OPENSEARCH_SAP_RULE_INDEX_TEMPLATE = ".opensearch-sap-detectors-queries-index-template"; + public static final String OPENSEARCH_SAP_ERROR_INDEX = ".opensearch-sap-error-history"; public static String getRuleIndex(String logType) { return String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries", logType); diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java index e6dea9947..bb06e29f2 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java @@ -98,18 +98,11 @@ import org.opensearch.securityanalytics.rules.exceptions.SigmaError; import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; import org.opensearch.securityanalytics.threatIntel.DetectorThreatIntelService; -import org.opensearch.securityanalytics.util.DetectorIndices; -import org.opensearch.securityanalytics.util.DetectorUtils; -import org.opensearch.securityanalytics.util.IndexUtils; -import org.opensearch.securityanalytics.util.MonitorService; -import org.opensearch.securityanalytics.util.RuleIndices; -import org.opensearch.securityanalytics.util.RuleTopicIndices; -import org.opensearch.securityanalytics.util.SecurityAnalyticsException; -import org.opensearch.securityanalytics.util.WorkflowService; +import org.opensearch.securityanalytics.util.*; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; - +import java.io.IOException; import java.time.Instant; import java.util.ArrayList; import java.util.Collection; @@ -118,7 +111,6 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; @@ -135,6 +127,8 @@ public class TransportIndexDetectorAction extends HandledTransportAction() { @@ -1522,9 +1519,15 @@ private void onOperation(IndexResponse response, Detector detector) { } private void onFailures(Exception t) { + log.info("Exception failures while creating detector: {}", t.toString()); if (counter.compareAndSet(false, true)) { finishHim(null, t); } + try { + initErrorHistoryIndexAndAddErrors(request, t.toString()); + } catch (IOException e) { + log.info("Create SAP error index exception", e); + } } private void finishHim(Detector detector, Exception t) { @@ -1573,6 +1576,50 @@ private Map mapMonitorIds(List monitorResp } } + private void initErrorHistoryIndexAndAddErrors(IndexDetectorRequest request, String exceptionMessage) throws IOException { + + if (!errorHistoryIndex.errorHistoryIndexExists()) { + errorHistoryIndex.initErrorsHistoryIndex(new ActionListener<>() { + @Override + public void onResponse(CreateIndexResponse response) { + errorHistoryIndex.onCreateMappingsResponse(response); + try { + errorHistoryIndex.addErrorsToSAPHistoryIndex(request, exceptionMessage, indexTimeout, new ActionListener<>() { + @Override + public void onResponse(IndexResponse indexResponse) { + log.info("Successfully updated error history index"); + } + @Override + public void onFailure(Exception ex) { + log.info("Exception while inserting a document in error history index: {}", ex); + } + + }); + } catch (IOException e) { + log.info("Exception while inserting a document in error history index: {}", e); + } + } + @Override + public void onFailure(Exception e) { + log.debug("Exception while creating error history index: {}", e); + } + }); + } else { + errorHistoryIndex.addErrorsToSAPHistoryIndex(request, exceptionMessage, indexTimeout, new ActionListener<>() { + @Override + public void onResponse(IndexResponse indexResponse) { + log.info("Successfully updated error history index"); + } + @Override + public void onFailure(Exception ex) { + log.debug("Exception while creating error history index: {}", ex); + } + + }); + } + } + + private void setFilterByEnabled(boolean filterByEnabled) { this.filterByEnabled = filterByEnabled; } diff --git a/src/main/java/org/opensearch/securityanalytics/util/ErrorsHistoryIndex.java b/src/main/java/org/opensearch/securityanalytics/util/ErrorsHistoryIndex.java new file mode 100644 index 000000000..9defbe13d --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/util/ErrorsHistoryIndex.java @@ -0,0 +1,106 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.util; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.admin.indices.create.CreateIndexResponse; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.index.IndexResponse; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.client.Client; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.UUIDs; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.securityanalytics.action.IndexDetectorRequest; +import org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig; +import org.opensearch.securityanalytics.logtype.LogTypeService; +import org.opensearch.securityanalytics.model.Detector; +import org.opensearch.threadpool.ThreadPool; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.time.Instant; +import java.util.Objects; +import java.util.Locale; + + public class ErrorsHistoryIndex { + + private static final Logger log = LogManager.getLogger(ErrorsHistoryIndex.class); + + private final Client client; + + private final ClusterService clusterService; + + private final ThreadPool threadPool; + + private final LogTypeService logTypeService; + + public ErrorsHistoryIndex(LogTypeService logTypeService, Client client, ClusterService clusterService, ThreadPool threadPool) { + this.client = client; + this.clusterService = clusterService; + this.threadPool = threadPool; + this.logTypeService = logTypeService; + } + + public static String errorsHistoryIndexMappings() throws IOException { + return new String(Objects.requireNonNull(RuleIndices.class.getClassLoader().getResourceAsStream("mappings/errorsHistory.json")).readAllBytes(), Charset.defaultCharset()); + } + public void initErrorsHistoryIndex(ActionListener actionListener) throws IOException { + Settings errorHistoryIndexSettings = Settings.builder() + .put("index.hidden", true) + .build(); + CreateIndexRequest indexRequest = new CreateIndexRequest(DetectorMonitorConfig.OPENSEARCH_SAP_ERROR_INDEX) + .mapping(errorsHistoryIndexMappings()) + .settings(errorHistoryIndexSettings); + client.admin().indices().create(indexRequest, actionListener); + } + + public void addErrorsToSAPHistoryIndex(IndexDetectorRequest request, String exception, TimeValue indexTimeout, ActionListener actionListener) throws IOException { + Detector detector = request.getDetector(); + String ruleTopic = detector.getDetectorType(); + String indexName = DetectorMonitorConfig.OPENSEARCH_SAP_ERROR_INDEX; + Instant timestamp = detector.getLastUpdateTime(); + String detectorId = detector.getId(); + String operation = detectorId.isEmpty() ? "CREATE_DETECTOR" : "UPDATE_DETECTOR"; + String user = detector.getUser() == null ? "user" : detector.getUser().getName(); + + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + builder.field("detectorId", detectorId); + builder.field("exception", exception); + builder.field("timestamp", timestamp); + builder.field("logType", ruleTopic); + builder.field("operation", operation); + builder.field("user", user); + builder.endObject(); + IndexRequest indexRequest = new IndexRequest(indexName) + .id(UUIDs.base64UUID()) + .source(builder) + .timeout(indexTimeout) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + client.index(indexRequest, actionListener); + } + public void onCreateMappingsResponse(CreateIndexResponse response) { + if (response.isAcknowledged()) { + log.info(String.format(Locale.getDefault(), "Created %s with mappings.", DetectorMonitorConfig.OPENSEARCH_SAP_ERROR_INDEX)); + } else { + log.error(String.format(Locale.getDefault(), "Create %s mappings call not acknowledged.", DetectorMonitorConfig.OPENSEARCH_SAP_ERROR_INDEX)); + throw new OpenSearchStatusException(String.format(Locale.getDefault(), "Create %s mappings call not acknowledged", Detector.DETECTORS_INDEX), RestStatus.INTERNAL_SERVER_ERROR); + } + } + + public boolean errorHistoryIndexExists() { + ClusterState clusterState = clusterService.state(); + return clusterState.getRoutingTable().hasIndex(DetectorMonitorConfig.OPENSEARCH_SAP_ERROR_INDEX); + } + } diff --git a/src/main/resources/mappings/errorsHistory.json b/src/main/resources/mappings/errorsHistory.json new file mode 100644 index 000000000..0d3498f95 --- /dev/null +++ b/src/main/resources/mappings/errorsHistory.json @@ -0,0 +1,26 @@ +{ + "dynamic": "strict", + "_meta" : { + "schema_version": 1 + }, + "properties": { + "detectorId": { + "type": "keyword" + }, + "exception": { + "type": "text" + }, + "timestamp": { + "type": "text" + }, + "operation": { + "type": "keyword" + }, + "logType": { + "type": "keyword" + }, + "user": { + "type": "text" + } + } +} diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java index 541925dd5..2728f1c35 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java @@ -8,22 +8,24 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; - +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.UUID; import org.apache.hc.core5.http.ContentType; import org.apache.hc.core5.http.HttpEntity; import org.apache.hc.core5.http.HttpStatus; import org.apache.hc.core5.http.io.entity.StringEntity; import org.apache.hc.core5.http.message.BasicHeader; import org.junit.Assert; -import org.junit.Ignore; import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Request; import org.opensearch.client.Response; -import org.opensearch.common.settings.Settings; import org.opensearch.client.ResponseException; +import org.opensearch.common.settings.Settings; import org.opensearch.commons.alerting.model.IntervalSchedule; import org.opensearch.commons.alerting.model.Monitor.MonitorType; -import org.opensearch.commons.alerting.model.ScheduledJob; import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.search.SearchHit; @@ -35,14 +37,9 @@ import org.opensearch.securityanalytics.model.DetectorRule; import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.Locale; -import java.util.Map; import java.util.stream.Collectors; import org.opensearch.securityanalytics.model.DetectorTrigger; - -import static org.junit.Assert.assertNotNull; +import org.opensearch.test.rest.OpenSearchRestTestCase; import static org.opensearch.securityanalytics.TestHelpers.*; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE; @@ -377,17 +374,19 @@ public void testCreatingADetectorWithMultipleIndices() throws IOException { Assert.assertEquals(findings.size(), 2); } - public void testCreatingADetectorWithIndexNotExists() throws IOException { + public void testCreatingADetectorWithIndexNotExists() throws IOException, InterruptedException { Detector detector = randomDetectorWithTriggers(getRandomPrePackagedRules(), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of(), List.of()))); try { makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); } catch (ResponseException ex) { Assert.assertEquals(404, ex.getResponse().getStatusLine().getStatusCode()); + Thread.sleep(40000); + // validate SAP history index if it is created an populated correctly + checkIfSAPErrorIndexExistsAndPopulated("no such index"); } } - - public void testCreatingADetectorWithNonExistingCustomRule() throws IOException { +public void CreatingADetectorWithNonExistingCustomRule() throws IOException, InterruptedException { String index = createTestIndex(randomIndex(), windowsIndexMapping()); // Execute CreateMappingsAction to add alias mapping for index @@ -402,7 +401,7 @@ public void testCreatingADetectorWithNonExistingCustomRule() throws IOException Response response = client().performRequest(createMappingRequest); assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); - DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(java.util.UUID.randomUUID().toString())), + DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(UUID.randomUUID().toString())), getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList())); Detector detector = randomDetectorWithInputs(List.of(input)); @@ -410,6 +409,9 @@ public void testCreatingADetectorWithNonExistingCustomRule() throws IOException makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); } catch (ResponseException ex) { Assert.assertEquals(404, ex.getResponse().getStatusLine().getStatusCode()); + Thread.sleep(30000); + // validate SAP history index if it is created an populated correctly + checkIfSAPErrorIndexExistsAndPopulated("Custom Rule Index not found"); } } @@ -418,7 +420,7 @@ public void testCreatingADetectorWithNonExistingCustomRule() throws IOException * 2. Detector without rules and monitors created successfully * @throws IOException */ - public void testCreateDetectorWithoutRules() throws IOException { + public void testCreateDetectorWithoutRules() throws IOException, InterruptedException { String index = createTestIndex(randomIndex(), windowsIndexMapping()); // Execute CreateMappingsAction to add alias mapping for index @@ -442,6 +444,36 @@ public void testCreateDetectorWithoutRules() throws IOException { } catch (ResponseException ex) { Assert.assertEquals(400, ex.getResponse().getStatusLine().getStatusCode()); assertTrue(ex.getMessage().contains("Detector cannot be created as no compatible rules were provided")); + Thread.sleep(30000); + // validate SAP history index if it is created an populated correctly + checkIfSAPErrorIndexExistsAndPopulated("no compatible rules"); + + } + } + + private static void checkIfSAPErrorIndexExistsAndPopulated(String exceptionMessage) throws IOException { + String indexName = ".opensearch-sap-error-history"; + Boolean searchErrResp = OpenSearchRestTestCase.indexExists(indexName); + + // Validate index creation + assertTrue(searchErrResp); + Map searchResponse = OpenSearchRestTestCase.getAsMap("/" + indexName + "/_search"); + assertNotNull(searchResponse); + assertTrue(searchResponse.containsKey("hits")); + Map hits = (Map) searchResponse.get("hits"); + assertTrue(hits.containsKey("hits")); + List> hitList = (List>) hits.get("hits"); + assertTrue(hitList.size() > 0); + + // Iterate through each hit + for (Map hit : hitList) { + Map source = (Map) hit.get("_source"); + assertNotNull(source); + + // Validate the "exception" field in each hit + assertTrue(source.containsKey("exception")); + String exception = (String) source.get("exception"); + assertTrue(exception.contains(exceptionMessage)); } } @@ -857,7 +889,7 @@ public void testUpdateADetector() throws IOException { Assert.assertEquals(6, response.getHits().getTotalHits().value); } - public void testUpdateANonExistingDetector() throws IOException { + public void testUpdateANonExistingDetector() throws IOException, InterruptedException { String index = createTestIndex(randomIndex(), windowsIndexMapping()); // Execute CreateMappingsAction to add alias mapping for index @@ -875,21 +907,27 @@ public void testUpdateANonExistingDetector() throws IOException { Detector updatedDetector = randomDetectorWithInputs(List.of(input)); try { - makeRequest(client(), "PUT", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + java.util.UUID.randomUUID(), Collections.emptyMap(), toHttpEntity(updatedDetector)); + makeRequest(client(), "PUT", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + UUID.randomUUID(), Collections.emptyMap(), toHttpEntity(updatedDetector)); } catch (ResponseException ex) { Assert.assertEquals(404, ex.getResponse().getStatusLine().getStatusCode()); + Thread.sleep(30000); + // validate SAP history index if it is created an populated correctly + checkIfSAPErrorIndexExistsAndPopulated("not found"); } } - public void testUpdateADetectorWithIndexNotExists() throws IOException { + public void testUpdateADetectorWithIndexNotExists() throws IOException, InterruptedException { DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(), getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList())); Detector updatedDetector = randomDetectorWithInputs(List.of(input)); try { - makeRequest(client(), "PUT", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + java.util.UUID.randomUUID(), Collections.emptyMap(), toHttpEntity(updatedDetector)); + makeRequest(client(), "PUT", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + UUID.randomUUID(), Collections.emptyMap(), toHttpEntity(updatedDetector)); } catch (ResponseException ex) { Assert.assertEquals(404, ex.getResponse().getStatusLine().getStatusCode()); + Thread.sleep(30000); + // validate SAP history index if it is created an populated correctly + checkIfSAPErrorIndexExistsAndPopulated("no such index"); } } @@ -1279,7 +1317,7 @@ public void testDeletingADetector_oneDetectorType_multiple_ruleTopicIndex() thro public void testDeletingANonExistingDetector() throws IOException { try { - makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + java.util.UUID.randomUUID(), Collections.emptyMap(), null); + makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + UUID.randomUUID(), Collections.emptyMap(), null); } catch (ResponseException ex) { Assert.assertEquals(404, ex.getResponse().getStatusLine().getStatusCode()); }