[Backport to 2.x] Backport to 2.x #803, #918, and #914 (#930)

* get all findings as part of findings API enhancement (#803) * get all findings as part of findings API enhancement Signed-off-by: Riya Saxena <[email protected]> * findingsAPI feature enhancements (address comments to prev PR) Signed-off-by: Riya Saxena <[email protected]> * findingsAPI feature enhancements (address comments to prev PR) Signed-off-by: Riya Saxena <[email protected]> * added support for param in Finding API Signed-off-by: Riya Saxena <[email protected]> * added detectionType as param for Findings API enhancements Signed-off-by: Riya Saxena <[email protected]> * added few tests to validate findings by params Signed-off-by: Riya Saxena <[email protected]> * added test for searchString param in FindingsAPI Signed-off-by: Riya Saxena <[email protected]> * adding addiional params findingIds, startTime and endTime as findings API enhancement Signed-off-by: Riya Saxena <[email protected]> * added params in getFindingsByDetectorId func * changed the startTime and endTime req input format * fix merge conflixt * fix integ test failures in findings API * fix integ tests * fix integ tests for findings Signed-off-by: Subhobrata Dey <[email protected]> --------- Signed-off-by: Riya Saxena <[email protected]> Signed-off-by: Riya <[email protected]> Signed-off-by: Subhobrata Dey <[email protected]> Co-authored-by: Subhobrata Dey <[email protected]> * fix integ test (#918) Signed-off-by: Joanne Wang <[email protected]> * Feature findings api enhancements (#914) * get all findings as part of findings API enhancement Signed-off-by: Riya Saxena <[email protected]> * findingsAPI feature enhancements (address comments to prev PR) Signed-off-by: Riya Saxena <[email protected]> * findingsAPI feature enhancements (address comments to prev PR) Signed-off-by: Riya Saxena <[email protected]> * added support for param in Finding API Signed-off-by: Riya Saxena <[email protected]> * added detectionType as param for Findings API enhancements Signed-off-by: Riya Saxena <[email protected]> * added few tests to validate findings by params Signed-off-by: Riya Saxena <[email protected]> * added test for searchString param in FindingsAPI Signed-off-by: Riya Saxena <[email protected]> * adding addiional params findingIds, startTime and endTime as findings API enhancement Signed-off-by: Riya Saxena <[email protected]> * added params in getFindingsByDetectorId func * changed the startTime and endTime req input format * fix merge conflixt * fix integ test failures in findings API * fix integ tests * refactored the logic Signed-off-by: Riya Saxena <[email protected]> * remove unused imports * address the pr comments Signed-off-by: Riya Saxena <[email protected]> * address pr comments Signed-off-by: Riya Saxena <[email protected]> * SA integ tests fix * SA integ tests fix * fix integ tests for findings Signed-off-by: Subhobrata Dey <[email protected]> * fix conflixt errors Signed-off-by: Riya Saxena <[email protected]> * fix conflixt errors Signed-off-by: Riya Saxena <[email protected]> * fix conflixt errors Signed-off-by: Riya Saxena <[email protected]> * fix conflixt errors Signed-off-by: Riya Saxena <[email protected]> * fix integ tests Signed-off-by: Riya Saxena <[email protected]> * fix integ tests Signed-off-by: Riya Saxena <[email protected]> * fix integ tests Signed-off-by: Riya Saxena <[email protected]> * fix flaky integ tests Signed-off-by: Riya Saxena <[email protected]> * address pr comments Signed-off-by: Riya Saxena <[email protected]> --------- Signed-off-by: Riya Saxena <[email protected]> Signed-off-by: Riya <[email protected]> Signed-off-by: Subhobrata Dey <[email protected]> Co-authored-by: Subhobrata Dey <[email protected]> --------- Signed-off-by: Riya Saxena <[email protected]> Signed-off-by: Riya <[email protected]> Signed-off-by: Subhobrata Dey <[email protected]> Signed-off-by: Joanne Wang <[email protected]> Co-authored-by: Riya <[email protected]> Co-authored-by: Subhobrata Dey <[email protected]>
opensearch-project · Mar 15, 2024 · f63c351 · f63c351
1 parent fbccff4
commit f63c351
Show file tree

Hide file tree

Showing 11 changed files with 884 additions and 90 deletions.
diff --git a/src/main/java/org/opensearch/securityanalytics/action/GetFindingsRequest.java b/src/main/java/org/opensearch/securityanalytics/action/GetFindingsRequest.java
@@ -5,6 +5,8 @@
 package org.opensearch.securityanalytics.action;
 
 import java.io.IOException;
+import java.time.Instant;
+import java.util.List;
 import java.util.Locale;
 import org.opensearch.action.ActionRequest;
 import org.opensearch.action.ActionRequestValidationException;
@@ -18,9 +20,14 @@
 
 public class GetFindingsRequest extends ActionRequest {
 
+ private List<String> findingIds;
+ private Instant startTime;
+ private Instant endTime;
  private String logType;
  private String detectorId;
  private Table table;
+ private String severity;
+ private String detectionType;
 
  public static final String DETECTOR_ID = "detector_id";
 
@@ -32,22 +39,36 @@ public GetFindingsRequest(StreamInput sin) throws IOException {
  this(
  sin.readOptionalString(),
  sin.readOptionalString(),
- Table.readFrom(sin)
+ Table.readFrom(sin),
+ sin.readOptionalString(),
+ sin.readOptionalString(),
+ sin.readOptionalStringList(),
+ sin.readOptionalInstant(),
+ sin.readOptionalInstant()
  );
  }
 
- public GetFindingsRequest(String detectorId, String logType, Table table) {
+ public GetFindingsRequest(String detectorId, String logType, Table table, String severity, String detectionType, List<String> findingIds, Instant startTime, Instant endTime) {
  this.detectorId = detectorId;
  this.logType = logType;
  this.table = table;
+ this.severity = severity;
+ this.detectionType = detectionType;
+ this.findingIds = findingIds;
+ this.startTime = startTime;
+ this.endTime = endTime;
  }
 
  @Override
  public ActionRequestValidationException validate() {
  ActionRequestValidationException validationException = null;
- if ((detectorId == null || detectorId.length() == 0) && logType == null) {
+ if (detectorId != null && detectorId.length() == 0) {
+ validationException = addValidationError(String.format(Locale.getDefault(),
+ "detector_id is missing"),
+ validationException);
+ } else if(startTime != null && endTime != null && startTime.isAfter(endTime)) {
  validationException = addValidationError(String.format(Locale.getDefault(),
- "At least one of detector type or detector id needs to be passed", DETECTOR_ID),
+ "startTime should be less than endTime"),
  validationException);
  }
  return validationException;
@@ -58,17 +79,42 @@ public void writeTo(StreamOutput out) throws IOException {
  out.writeOptionalString(detectorId);
  out.writeOptionalString(logType);
  table.writeTo(out);
+ out.writeOptionalString(severity);
+ out.writeOptionalString(detectionType);
+ out.writeOptionalStringCollection(findingIds);
+ out.writeOptionalInstant(startTime);
+ out.writeOptionalInstant(endTime);
  }
 
  public String getDetectorId() {
  return detectorId;
  }
 
+ public String getSeverity() {
+ return severity;
+ }
+
+ public String getDetectionType() {
+ return detectionType;
+ }
+
  public String getLogType() {
  return logType;
  }
 
  public Table getTable() {
  return table;
  }
+
+ public List<String> getFindingIds() {
+ return findingIds;
+ }
+
+ public Instant getStartTime() {
+ return startTime;
+ }
+
+ public Instant getEndTime() {
+ return endTime;
+ }
 }
diff --git a/src/main/java/org/opensearch/securityanalytics/findings/FindingsService.java b/src/main/java/org/opensearch/securityanalytics/findings/FindingsService.java
@@ -4,6 +4,7 @@
  */
 package org.opensearch.securityanalytics.findings;
 
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -12,6 +13,7 @@
 import java.util.stream.Collectors;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.lucene.search.join.ScoreMode;
 import org.opensearch.OpenSearchStatusException;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.client.Client;
@@ -21,6 +23,11 @@
 import org.opensearch.commons.alerting.model.FindingWithDocs;
 import org.opensearch.commons.alerting.model.Table;
 import org.opensearch.core.rest.RestStatus;
+import org.opensearch.index.query.BoolQueryBuilder;
+import org.opensearch.index.query.PrefixQueryBuilder;
+import org.opensearch.index.query.NestedQueryBuilder;
+import org.opensearch.index.query.QueryBuilder;
+import org.opensearch.index.query.QueryBuilders;
 import org.opensearch.securityanalytics.action.FindingDto;
 import org.opensearch.securityanalytics.action.GetDetectorAction;
 import org.opensearch.securityanalytics.action.GetDetectorRequest;
@@ -52,7 +59,12 @@ public FindingsService(Client client) {
  * @param table group of search related parameters
  * @param listener ActionListener to get notified on response or error
  */
- public void getFindingsByDetectorId(String detectorId, Table table, ActionListener<GetFindingsResponse> listener ) {
+ public void getFindingsByDetectorId(String detectorId, Table table, String severity,
+ String detectionType,
+ List<String> findingIds,
+ Instant startTime,
+ Instant endTime,
+ ActionListener<GetFindingsResponse> listener ) {
  this.client.execute(GetDetectorAction.INSTANCE, new GetDetectorRequest(detectorId, -3L), new ActionListener<>() {
 
  @Override
@@ -102,6 +114,11 @@ public void onFailure(Exception e) {
  new ArrayList<>(monitorToDetectorMapping.keySet()),
  DetectorMonitorConfig.getAllFindingsIndicesPattern(detector.getDetectorType()),
  table,
+ severity,
+ detectionType,
+ findingIds,
+ startTime,
+ endTime,
  getFindingsResponseListener
  );
  }
@@ -126,18 +143,21 @@ public void getFindingsByMonitorIds(
  List<String> monitorIds,
  String findingIndexName,
  Table table,
+ String severity,
+ String detectionType,
+ List<String> findingIds,
+ Instant startTime,
+ Instant endTime,
  ActionListener<GetFindingsResponse> listener
  ) {
-
+ BoolQueryBuilder queryBuilder = getBoolQueryBuilder(detectionType, severity, findingIds, startTime, endTime);
  org.opensearch.commons.alerting.action.GetFindingsRequest req =
  new org.opensearch.commons.alerting.action.GetFindingsRequest(
  null,
  table,
  null,
- findingIndexName,
- monitorIds
+ findingIndexName, monitorIds, queryBuilder
  );
-
  AlertingPluginInterface.INSTANCE.getFindings((NodeClient) client, req, new ActionListener<>() {
  @Override
  public void onResponse(
@@ -163,6 +183,59 @@ public void onFailure(Exception e) {
 
  }
 
+ private static BoolQueryBuilder getBoolQueryBuilder(String detectionType, String severity, List<String> findingIds, Instant startTime, Instant endTime) {
+ // Construct the query within the search source builder
+ BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
+
+ if (detectionType != null && !detectionType.isBlank()) {
+ QueryBuilder nestedQuery;
+ if (detectionType.equalsIgnoreCase("threat")) {
+ nestedQuery = QueryBuilders.boolQuery().filter(
+ new PrefixQueryBuilder("queries.id", "threat_intel_")
+ );
+ } else {
+ nestedQuery = QueryBuilders.boolQuery().mustNot(
+ new PrefixQueryBuilder("queries.id", "threat_intel_")
+ );
+ }
+
+ // Create a nested query builder
+ NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery(
+ "queries",
+ nestedQuery,
+ ScoreMode.None
+ );
+
+ // Add the nested query to the bool query
+ boolQueryBuilder.must(nestedQueryBuilder);
+ }
+
+ if (findingIds != null && !findingIds.isEmpty()) {
+ boolQueryBuilder.filter(QueryBuilders.termsQuery("id", findingIds));
+ }
+
+
+ if (startTime != null && endTime != null) {
+ long startTimeMillis = startTime.toEpochMilli();
+ long endTimeMillis = endTime.toEpochMilli();
+ QueryBuilder timeRangeQuery = QueryBuilders.rangeQuery("timestamp")
+ .from(startTimeMillis) // Greater than or equal to start time
+ .to(endTimeMillis); // Less than or equal to end time
+ boolQueryBuilder.filter(timeRangeQuery);
+ }
+
+ if (severity != null) {
+ boolQueryBuilder.must(QueryBuilders.nestedQuery(
+ "queries",
+ QueryBuilders.boolQuery().should(
+ QueryBuilders.matchQuery("queries.tags", severity)
+ ),
+ ScoreMode.None
+ ));
+ }
+ return boolQueryBuilder;
+ }
+
  void setIndicesAdminClient(Client client) {
  this.client = client;
  }
@@ -171,6 +244,11 @@ public void getFindings(
  List<Detector> detectors,
  String logType,
  Table table,
+ String severity,
+ String detectionType,
+ List<String> findingIds,
+ Instant startTime,
+ Instant endTime,
  ActionListener<GetFindingsResponse> listener
  ) {
  if (detectors.size() == 0) {
@@ -195,6 +273,11 @@ public void getFindings(
  allMonitorIds,
  DetectorMonitorConfig.getAllFindingsIndicesPattern(logType),
  table,
+ severity,
+ detectionType,
+ findingIds,
+ startTime,
+ endTime,
  new ActionListener<>() {
  @Override
  public void onResponse(GetFindingsResponse getFindingsResponse) {

diff --git a/src/main/java/org/opensearch/securityanalytics/resthandler/RestGetFindingsAction.java b/src/main/java/org/opensearch/securityanalytics/resthandler/RestGetFindingsAction.java
@@ -5,6 +5,10 @@
 package org.opensearch.securityanalytics.resthandler;
 
 import java.io.IOException;
+import java.time.DateTimeException;
+import java.time.Instant;
+import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
 import org.opensearch.client.node.NodeClient;
@@ -40,6 +44,35 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
  int size = request.paramAsInt("size", 20);
  int startIndex = request.paramAsInt("startIndex", 0);
  String searchString = request.param("searchString", "");
+ String severity = request.param("severity", null);
+ String detectionType = request.param("detectionType", null);
+ List<String> findingIds = null;
+ if (request.param("findingIds") != null) {
+ findingIds = Arrays.asList(request.param("findingIds").split(","));
+ }
+ Instant startTime = null;
+ String startTimeParam = request.param("startTime");
+ if (startTimeParam != null && !startTimeParam.isEmpty()) {
+ try {
+ startTime = Instant.ofEpochMilli(Long.parseLong(startTimeParam));
+ } catch (NumberFormatException | NullPointerException | DateTimeException e) {
+ // Handle the parsing error
+ // For example, log the error or provide a default value
+ startTime = Instant.now(); // Default value or fallback
+ }
+ }
+
+ Instant endTime = null;
+ String endTimeParam = request.param("endTime");
+ if (endTimeParam != null && !endTimeParam.isEmpty()) {
+ try {
+ endTime = Instant.ofEpochMilli(Long.parseLong(endTimeParam));
+ } catch (NumberFormatException | NullPointerException | DateTimeException e) {
+ // Handle the parsing error
+ // For example, log the error or provide a default value
+ endTime = Instant.now(); // Default value or fallback
+ }
+ }
 
  Table table = new Table(
  sortOrder,
@@ -53,7 +86,12 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
  GetFindingsRequest req = new GetFindingsRequest(
  detectorId,
  detectorType,
- table
+ table,
+ severity,
+ detectionType,
+ findingIds,
+ startTime,
+ endTime
  );
 
  return channel -> client.execute(