[BUG] Unable to initialize .opendistro_security index with remote-store feature enabled.

[rishabh6788]

Describe the bug
I am trying to set-up a secure multi-node cluster with remote-store enabled but it is failing to bring up the OS process while trying to create the .opendistro_security index. I am getting following error:

[2023-06-19T18:42:05,625][INFO ][o.o.p.PluginsService     ] [ip-10-0-5-117.ec2.internal] PluginService:onIndexModule index:[.opendistro_security/X10sm8tySnq5AndLv_ICiA]
[2023-06-19T18:42:05,635][WARN ][o.o.i.c.IndicesClusterStateService] [ip-10-0-5-117.ec2.internal] [.opendistro_security][0] marking and sending shard failed due to [failed to create shard]
java.lang.IllegalArgumentException: Repository should be created before creating index with remote_store enabled setting
        at org.opensearch.index.store.RemoteSegmentStoreDirectoryFactory.newDirectory(RemoteSegmentStoreDirectoryFactory.java:62) ~[opensearch-2.9.0.jar:2.9.0]
        at org.opensearch.index.IndexService.createShard(IndexService.java:475) ~[opensearch-2.9.0.jar:2.9.0]
        at org.opensearch.indices.IndicesService.createShard(IndicesService.java:951) ~[opensearch-2.9.0.jar:2.9.0]
        at org.opensearch.indices.IndicesService.createShard(IndicesService.java:210) ~[opensearch-2.9.0.jar:2.9.0]
        at org.opensearch.indices.cluster.IndicesClusterStateService.createShard(IndicesClusterStateService.java:662) [opensearch-2.9.0.jar:2.9.0]
        at org.opensearch.indices.cluster.IndicesClusterStateService.createOrUpdateShards(IndicesClusterStateService.java:639) [opensearch-2.9.0.jar:2.9.0]
        at org.opensearch.indices.cluster.IndicesClusterStateService.applyClusterState(IndicesClusterStateService.java:296) [opensearch-2.9.0.jar:2.9.0]
        at org.opensearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:606) [opensearch-2.9.0.jar:2.9.0]
        at org.opensearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:593) [opensearch-2.9.0.jar:2.9.0]
        at org.opensearch.cluster.service.ClusterApplierService.applyChanges(ClusterApplierService.java:561) [opensearch-2.9.0.jar:2.9.0]
        at org.opensearch.cluster.service.ClusterApplierService.runTask(ClusterApplierService.java:484) [opensearch-2.9.0.jar:2.9.0]
        at org.opensearch.cluster.service.ClusterApplierService$UpdateTask.run(ClusterApplierService.java:186) [opensearch-2.9.0.jar:2.9.0]
        at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:747) [opensearch-2.9.0.jar:2.9.0]
        at org.opensearch.common.util.concurrent.PrioritizedOpenSearchThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedOpenSearchThreadPoolExecutor.java:282) [opensearch-2.9.0.jar:2.9.0]
        at org.opensearch.common.util.concurrent.PrioritizedOpenSearchThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedOpenSearchThreadPoolExecutor.java:245) [opensearch-2.9.0.jar:2.9.0]
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]
        at java.lang.Thread.run(Thread.java:833) [?:?]
Caused by: org.opensearch.repositories.RepositoryMissingException: [opensearch-infra-stack-test-us-east-1-secure-repo] missing

As per my understanding all the system indices are supposed to have DOCUMENT type replication setting so not sure why the security index is being registered as remote-store index.

Below is my opensearch.yml setting:

cluster.initial_cluster_manager_nodes:
  - seed
discovery.seed_providers: ec2
network.host: 0.0.0.0
discovery.ec2.tag.Name: >-
  opensearch-infra-stack-test-us-east-1-secure/seedNodeAsg,opensearch-infra-stack-test-us-east-1-secure/managerNodeAsg
 
node.roles:
  - data
  - ingest
 
cluster.remote_store.repository: opensearch-infra-stack-test-us-east-1-secure-repo
opensearch.experimental.feature.remote_store.enabled: 'true'
cluster.remote_store.enabled: 'true'
opensearch.experimental.feature.segment_replication_experimental.enabled: 'true'
cluster.indices.replication.strategy: SEGMENT
 
 
######## Start OpenSearch Security Demo Configuration ########
# WARNING: revise all the lines below before you go into production
plugins.security.ssl.transport.pemcert_filepath: esnode.pem
plugins.security.ssl.transport.pemkey_filepath: esnode-key.pem
plugins.security.ssl.transport.pemtrustedcas_filepath: root-ca.pem
plugins.security.ssl.transport.enforce_hostname_verification: false
plugins.security.ssl.http.enabled: true
plugins.security.ssl.http.pemcert_filepath: esnode.pem
plugins.security.ssl.http.pemkey_filepath: esnode-key.pem
plugins.security.ssl.http.pemtrustedcas_filepath: root-ca.pem
plugins.security.allow_unsafe_democertificates: true
plugins.security.allow_default_init_securityindex: true
plugins.security.authcz.admin_dn:
  - CN=kirk,OU=client,O=client,L=test, C=de
 
plugins.security.audit.type: internal_opensearch
plugins.security.enable_snapshot_restore_privilege: true
plugins.security.check_snapshot_restore_write_privileges: true
plugins.security.restapi.roles_enabled: ["all_access", "security_rest_api_access"]
plugins.security.system_indices.enabled: true
plugins.security.system_indices.indices: [".plugins-ml-model-group", ".plugins-ml-model", ".plugins-ml-task", ".opendistro-alerting-config", ".opendistro-alerting-alert*", ".opendistro-anomaly-results*", ".opendistro-anomaly-detector*", ".opendistro-anomaly-checkpoints", ".opendistro-anomaly-detection-state", ".opendistro-reports-*", ".opensearch-notifications-*", ".opensearch-notebooks", ".opensearch-observability", ".ql-datasources", ".opendistro-asynchronous-search-response*", ".replication-metadata-store", ".opensearch-knn-models"]
node.max_local_storage_nodes: 3

To Reproduce
Steps to reproduce the behavior:

1. Create a remote-store enabled multi-node cluster with security.

Expected behavior
The cluster should come up and the system indices should be created without any issue.

Plugins
Please list all plugins currently enabled.

Screenshots
If applicable, add screenshots to help explain your problem.

Host/Environment (please complete the following information):

• OS: [e.g. iOS]
• Version [e.g. 22]

Additional context
Add any other context about the problem here.

[rishabh6788]

The remote-store is working as expected with security disabled.

[sachinpkale]

If we create a cluster by disabling remote store settings, what is the replication type of .opendistro_security index? DOCUMENT or SEGMENT?

[rishabh6788]

I created a 2.9 cluster with only segment replication enabled and the .opendistro_security index replication type is DOCUMENT.

sngri@3c06304c0d45 opensearch-cluster-cdk % curl -X GET "https://opens-clust-1KXORCC7F0TCF-10d58231468f105e.elb.us-east-1.amazonaws.com/.opendistro_security/_settings?include_defaults&pretty" -ku admin:admin
{
  ".opendistro_security" : {
    "settings" : {
      "index" : {
        "replication" : {
          "type" : "DOCUMENT"
        },

@sachinpkale

[mch2]

There are three things going on here.

1. The remote store paths are implicitly overwriting the replication strategy, leading to unexpected behavior - [Segment Replication + Remote Storage] Remove implicit replication strategy change when configuring remote storage. #8162.
2. If a system index is configured to use remote storage, it breaks because the index is wired up before a remote store repository (stacktrace above).
3. Today system indices are not supported with segment replication. Given remote store can only be enabled with segrep, they would also not be supported for remote storage. - [BUG - Segment Replication + Remote Store] - Support segment replication for system indices #8182

[mch2]

I think we need to fix all three of these bugs for GA of remote storage. I would expect system indices to also be backed up.

[mch2]

@sachinpkale We would like to remove the restriction on system indices for segrep. I spent some time trying to solve 2 above and wire up the repository before the index. The registration for a repo is only done via API and not through yml settings. Given we want to use the default remote store repo, we can't require plugins to register their own repositories before attempting index creation. We either need to provide the ability to set config for a repository through yml and do it before plugins are registered or default to node-node until the repo is registered, but that itself is also not currently supported.

[ashking94]

Created an opensearch cluster using cdk and it is able to create system index with remote enabled settings -

➜  opensearch-cluster-cdk git:(main) curl https://<LB-URL>/\*/_settings\?pretty 
{
  ".opensearch-observability" : {
    "settings" : {
      "index" : {
        "replication" : {
          "type" : "SEGMENT"
        },
        "number_of_shards" : "1",
        "remote_store" : {
          "translog" : {
            "repository" : "opensearch-infra-stack-remote-store-secure-repo"
          },
          "enabled" : "true",
          "segment" : {
            "repository" : "opensearch-infra-stack-remote-store-secure-repo"
          }
        },
        "auto_expand_replicas" : "0-2",
        "provided_name" : ".opensearch-observability",
        "creation_date" : "1694071416491",
        "number_of_replicas" : "2",
        "uuid" : "Q3HHJu0iSaunjsFFSf0HyA",
        "version" : {
          "created" : "136317827"
        }
      }
    }
  },
  ".plugins-ml-config" : {
    "settings" : {
      "index" : {
        "replication" : {
          "type" : "SEGMENT"
        },
        "number_of_shards" : "1",
        "remote_store" : {
          "translog" : {
            "repository" : "opensearch-infra-stack-remote-store-secure-repo"
          },
          "enabled" : "true",
          "segment" : {
            "repository" : "opensearch-infra-stack-remote-store-secure-repo"
          }
        },
        "provided_name" : ".plugins-ml-config",
        "creation_date" : "1694071425987",
        "number_of_replicas" : "1",
        "uuid" : "zUpf0dqHS3yvnJHOLRjlXA",
        "version" : {
          "created" : "136317827"
        }
      }
    }
  },
  "security-auditlog-2023.09.07" : {
    "settings" : {
      "index" : {
        "replication" : {
          "type" : "SEGMENT"
        },
        "number_of_shards" : "1",
        "remote_store" : {
          "translog" : {
            "repository" : "opensearch-infra-stack-remote-store-secure-repo"
          },
          "enabled" : "true",
          "segment" : {
            "repository" : "opensearch-infra-stack-remote-store-secure-repo"
          }
        },
        "provided_name" : "security-auditlog-2023.09.07",
        "creation_date" : "1694071930597",
        "number_of_replicas" : "1",
        "uuid" : "0sRUbxmjTi-_tKaG3y9-YQ",
        "version" : {
          "created" : "136317827"
        }
      }
    }
  },
  ".opendistro_security" : {
    "settings" : {
      "index" : {
        "replication" : {
          "type" : "SEGMENT"
        },
        "number_of_shards" : "1",
        "remote_store" : {
          "translog" : {
            "repository" : "opensearch-infra-stack-remote-store-secure-repo"
          },
          "enabled" : "true",
          "segment" : {
            "repository" : "opensearch-infra-stack-remote-store-secure-repo"
          }
        },
        "auto_expand_replicas" : "0-all",
        "provided_name" : ".opendistro_security",
        "creation_date" : "1694071416075",
        "number_of_replicas" : "2",
        "uuid" : "KLz_n9vqTJuq_HYPbDeyWg",
        "version" : {
          "created" : "136317827"
        }
      }
    }
  }
}