ILM: only allow to configure an index suffix.

Change implementation to only allow configuring an
`apm-server.ilm.setup.mapping.*.index_suffix` rather than the whole
rollover alias.
This change intends to make the ILM setup less error prone and to make
the transition to the new index_templates and data streams easier.

Follow up on elastic#3826

_meta/beat.yml

@@ -384,23 +384,23 @@ apm-server:
 
  # The configured event types, policies and rollover_aliases will be merged with
  # the default setup. Only configure the mappings that should be customized.
- # NOTICE: when customizing `rollover_alias`, it is strongly recommended to keep
- # the prefix "apm-%{[observer.version]}" unchanged;
- # in case the prefix is changed, `setup.template.name` and `setup.template.pattern`
- # need to be changed accordingly to match the created indices
+ # Index names will always start with `apm-%{[observer.version]}-%{[event.type]} (e.g. apm-7.9.0-span)
+ # When configuring an `index-suffix` you can configure a custom part to be added after the fixed index part
+ # NOTE: When customizing `setup.template.name` and `setup.template.pattern`, ensure they are aligned with
+ # the ILM indices.
  #mapping:
  #- event_type: "error"
  # policy_name: "apm-rollover-30-days"
- # rollover_alias: "apm-%{[observer.version]}-error"
+ # index-suffix: ""
  #- event_type: "span"
  # policy_name: "apm-rollover-30-days"
- # rollover_alias: "apm-%{[observer.version]}-span"
+ # index-suffix: ""
  #- event_type: "transaction"
  # policy_name: "apm-rollover-30-days"
- # rollover_alias: "apm-%{[observer.version]}-transaction"
+ # index-suffix: ""
  #- event_type: "metric"
  # policy_name: "apm-rollover-30-days"
- # rollover_alias: "apm-%{[observer.version]}-metric"
+ # index-suffix: ""
 
  # Configured policies are added to pre-defined default policies.
  # If a policy with the same name as a default policy is configured, the configured policy overwrites the default policy.

apm-server.docker.yml

@@ -384,23 +384,23 @@ apm-server:
 
  # The configured event types, policies and rollover_aliases will be merged with
  # the default setup. Only configure the mappings that should be customized.
- # NOTICE: when customizing `rollover_alias`, it is strongly recommended to keep
- # the prefix "apm-%{[observer.version]}" unchanged;
- # in case the prefix is changed, `setup.template.name` and `setup.template.pattern`
- # need to be changed accordingly to match the created indices
+ # Index names will always start with `apm-%{[observer.version]}-%{[event.type]} (e.g. apm-7.9.0-span)
+ # When configuring an `index-suffix` you can configure a custom part to be added after the fixed index part
+ # NOTE: When customizing `setup.template.name` and `setup.template.pattern`, ensure they are aligned with
+ # the ILM indices.
  #mapping:
  #- event_type: "error"
  # policy_name: "apm-rollover-30-days"
- # rollover_alias: "apm-%{[observer.version]}-error"
+ # index-suffix: ""
  #- event_type: "span"
  # policy_name: "apm-rollover-30-days"
- # rollover_alias: "apm-%{[observer.version]}-span"
+ # index-suffix: ""
  #- event_type: "transaction"
  # policy_name: "apm-rollover-30-days"
- # rollover_alias: "apm-%{[observer.version]}-transaction"
+ # index-suffix: ""
  #- event_type: "metric"
  # policy_name: "apm-rollover-30-days"
- # rollover_alias: "apm-%{[observer.version]}-metric"
+ # index-suffix: ""
 
  # Configured policies are added to pre-defined default policies.
  # If a policy with the same name as a default policy is configured, the configured policy overwrites the default policy.

apm-server.yml

@@ -384,23 +384,23 @@ apm-server:
 
  # The configured event types, policies and rollover_aliases will be merged with
  # the default setup. Only configure the mappings that should be customized.
- # NOTICE: when customizing `rollover_alias`, it is strongly recommended to keep
- # the prefix "apm-%{[observer.version]}" unchanged;
- # in case the prefix is changed, `setup.template.name` and `setup.template.pattern`
- # need to be changed accordingly to match the created indices
+ # Index names will always start with `apm-%{[observer.version]}-%{[event.type]} (e.g. apm-7.9.0-span)
+ # When configuring an `index-suffix` you can configure a custom part to be added after the fixed index part
+ # NOTE: When customizing `setup.template.name` and `setup.template.pattern`, ensure they are aligned with
+ # the ILM indices.
  #mapping:
  #- event_type: "error"
  # policy_name: "apm-rollover-30-days"
- # rollover_alias: "apm-%{[observer.version]}-error"
+ # index-suffix: ""
  #- event_type: "span"
  # policy_name: "apm-rollover-30-days"
- # rollover_alias: "apm-%{[observer.version]}-span"
+ # index-suffix: ""
  #- event_type: "transaction"
  # policy_name: "apm-rollover-30-days"
- # rollover_alias: "apm-%{[observer.version]}-transaction"
+ # index-suffix: ""
  #- event_type: "metric"
  # policy_name: "apm-rollover-30-days"
- # rollover_alias: "apm-%{[observer.version]}-metric"
+ # index-suffix: ""
 
  # Configured policies are added to pre-defined default policies.
  # If a policy with the same name as a default policy is configured, the configured policy overwrites the default policy.

idxmgmt/ilm/config.go

@@ -56,9 +56,10 @@ type Policies map[string]Policy
 
 //Mapping binds together an ILM policy's name and body with an event type
 type Mapping struct {
- EventType string `config:"event_type"`
- PolicyName string `config:"policy_name"`
- RolloverAlias string `config:"rollover_alias"`
+ EventType string `config:"event_type"`
+ PolicyName string `config:"policy_name"`
+ IndexSuffix string `config:"index_suffix"`
+ Index string `config:"-"`
 }
 
 // Policy contains information about an ILM policy and the name
@@ -86,11 +87,11 @@ func NewConfig(info beat.Info, cfg *libcommon.Config) (Config, error) {
  // replace variable rollover_alias parts with beat information if available
  // otherwise fail as the full alias needs to be known during setup.
  for et, m := range config.Setup.Mappings {
- rolloverAlias, err := applyStaticFmtstr(info, m.RolloverAlias)
+ idx, err := applyStaticFmtstr(info, m.Index)
  if err != nil {
- return Config{}, errors.Wrap(err, "variable part of rollover_alias cannot be resolved")
+ return Config{}, errors.Wrap(err, "variable part of index suffix cannot be resolved")
  }
- m.RolloverAlias = rolloverAlias
+ m.Index = idx
  config.Setup.Mappings[et] = m
  }
  if len(config.Setup.Policies) == 0 {
@@ -121,12 +122,12 @@ func (m *Mappings) Unpack(cfg *libcommon.Config) error {
  if mapping.PolicyName == "" {
  mapping.PolicyName = existing.PolicyName
  }
- if mapping.RolloverAlias == "" {
- mapping.RolloverAlias = existing.RolloverAlias
- }
+ mapping.Index = existing.Index
+ }
+ if mapping.IndexSuffix != "" {
+ mapping.Index = fmt.Sprintf("%s-%s", mapping.Index, mapping.IndexSuffix)
  }
  (*m)[mapping.EventType] = mapping
-
  }
  return nil
 }
@@ -158,9 +159,6 @@ func validate(c *Config) error {
  if m.PolicyName == "" {
  return errors.New("empty policy_name not supported for ILM setup")
  }
- if m.RolloverAlias == "" {
- return errors.New("empty rollover_alias not supported for ILM setup")
- }
  if !c.Setup.RequirePolicy {
  // `require_policy=false` indicates that policies are set up outside
  // the APM Server, therefore do not throw an error here.
@@ -214,7 +212,7 @@ func defaultMappings() map[string]Mapping {
  m := map[string]Mapping{}
  for _, et := range common.EventTypes {
  m[et] = Mapping{EventType: et, PolicyName: defaultPolicyName,
- RolloverAlias: fmt.Sprintf("%s-%s", common.APMPrefix, et)}
+ Index: fmt.Sprintf("%s-%s", common.APMPrefix, et)}
  }
  return m
 }
@@ -259,7 +257,7 @@ func (c *Config) conditionalIndices() []map[string]interface{} {
  common.ConditionalSourcemapIndex(),
  }
  for _, m := range c.Setup.Mappings {
- conditions = append(conditions, common.Condition(m.EventType, m.RolloverAlias))
+ conditions = append(conditions, common.Condition(m.EventType, m.Index))
  }
  return conditions
 }

idxmgmt/ilm/config_test.go

@@ -117,21 +117,21 @@ func TestConfig_Valid(t *testing.T) {
 
  expected Config
  }{
- {name: "new policy and rollover_alias",
- cfg: `{"setup":{"mapping":[{"event_type":"span","policy_name":"spanPolicy"},{"event_type":"metric","rollover_alias":"apm-metric"}],"policies":[{"name":"spanPolicy","policy":{"phases":{"foo":{}}}}]}}`,
+ {name: "new policy and index suffix",
+ cfg: `{"setup":{"mapping":[{"event_type":"span","policy_name":"spanPolicy"},{"event_type":"metric","index_suffix":"production"},{"event_type":"error","index_suffix":"%{[observer.name]}"}],"policies":[{"name":"spanPolicy","policy":{"phases":{"foo":{}}}}]}}`,
  expected: Config{Mode: libilm.ModeAuto,
  Setup: Setup{Enabled: true, Overwrite: false, RequirePolicy: true,
  Mappings: map[string]Mapping{
  "error": {EventType: "error", PolicyName: defaultPolicyName,
- RolloverAlias: "apm-9.9.9-error"},
+ Index: "apm-9.9.9-error-mockapm", IndexSuffix: "%{[observer.name]}"},
  "span": {EventType: "span", PolicyName: "spanPolicy",
- RolloverAlias: "apm-9.9.9-span"},
+ Index: "apm-9.9.9-span"},
  "transaction": {EventType: "transaction", PolicyName: defaultPolicyName,
- RolloverAlias: "apm-9.9.9-transaction"},
+ Index: "apm-9.9.9-transaction"},
  "metric": {EventType: "metric", PolicyName: defaultPolicyName,
- RolloverAlias: "apm-metric"},
+ Index: "apm-9.9.9-metric-production", IndexSuffix: "production"},
  "profile": {EventType: "profile", PolicyName: defaultPolicyName,
- RolloverAlias: "apm-9.9.9-profile"},
+ Index: "apm-9.9.9-profile"},
  },
  Policies: map[string]Policy{
  defaultPolicyName: defaultPolicies()[defaultPolicyName],
@@ -160,7 +160,7 @@ func TestConfig_Valid(t *testing.T) {
  Mappings: func() map[string]Mapping {
  m := defaultMappingsResolved(mockBeatInfo)
  m["error"] = Mapping{EventType: "error", PolicyName: "errorPolicy",
- RolloverAlias: "apm-9.9.9-error"}
+ Index: "apm-9.9.9-error"}
  return m
  }(),
  Policies: defaultPolicies(),
@@ -188,9 +188,9 @@ func TestConfig_Invalid(t *testing.T) {
  {name: "invalid policy",
  cfg: `{"setup":{"mapping":[{"event_type":"span","policy_name":"xyz"}]}}`,
  errMsg: "policy 'xyz' not configured"},
- {name: "invalid rollover_alias",
- cfg: `{"setup":{"mapping":[{"event_type":"span","rollover_alias":"apm-%{[foo.version]}"}]}}`,
- errMsg: "rollover_alias cannot be resolved"},
+ {name: "invalid index suffix",
+ cfg: `{"setup":{"mapping":[{"event_type":"span","index_suffix":"%{[foo.version]}"}]}}`,
+ errMsg: "index suffix cannot be resolved"},
  } {
  t.Run(tc.name, func(t *testing.T) {
  _, err := NewConfig(mockBeatInfo, common.MustNewConfigFrom(tc.cfg))
@@ -203,7 +203,7 @@ func TestConfig_Invalid(t *testing.T) {
 func defaultMappingsResolved(info beat.Info) map[string]Mapping {
  m := defaultMappings()
  for k, v := range m {
- v.RolloverAlias = fmt.Sprintf("apm-%s-%s", info.Version, k)
+ v.Index = fmt.Sprintf("apm-%s-%s", info.Version, k)
  m[k] = v
  }
  return m

idxmgmt/ilm/supporter_factory.go

@@ -42,7 +42,7 @@ func MakeDefaultSupporter(
 
  for _, m := range ilmConfig.Setup.Mappings {
  policy := ilmConfig.Setup.Policies[m.PolicyName]
- supporter := libilm.NewStdSupport(log, mode, libilm.Alias{Name: m.RolloverAlias, Pattern: pattern},
+ supporter := libilm.NewStdSupport(log, mode, libilm.Alias{Name: m.Index, Pattern: pattern},
  libilm.Policy{Name: policy.Name, Body: policy.Body}, ilmConfig.Setup.Overwrite, true)
  supporters = append(supporters, supporter)
  }

tests/system/config/apm-server.yml.j2

@@ -159,12 +159,12 @@ apm-server:
  ilm.setup.enabled: {{ ilm_setup_enabled }}
  {% endif %}
 
- {% if ilm_aliases %}
+ {% if ilm_custom_suffix %}
  ilm.setup.mapping:
  - event_type: "error"
- rollover_alias: "apm-error-custom"
+ index_suffix: "custom"
  - event_type: "transaction"
- rollover_alias: "apm-transaction-custom"
+ index_suffix: "foo"
  {% endif %}
 
  {% if ilm_policies %}

tests/system/test_setup_index_management.py

@@ -304,15 +304,15 @@ def test_ilm_loaded(self):
 
 @integration_test
 class TestILMCustomAlias(ElasticTest):
- config_overrides = {"ilm_aliases": True}
+ config_overrides = {"ilm_custom_suffix": True}
 
  def setUp(self):
  super(TestILMCustomAlias, self).setUp()
  self.idxmgmt = IdxMgmt(self.es, index_name)
 
  def test_ilm_loaded(self):
  # aliasnames configured in apm-server.yml.j2
- aliases = ["apm-error-custom", "apm-transaction-custom",
+ aliases = [index_error + "-custom", index_transaction + "-foo",
  index_metric, index_profile, index_span]
  self.idxmgmt.wait_until_created(templates=[index_name] + aliases, aliases=aliases)
  self.idxmgmt.assert_event_template(with_ilm=True, event_templates=aliases)