Inject HTTPS_PROXY/HTTP_PROXY env var into endpoint elasticsearch o…

…utput units as proxy_url (#5044)

Inject proxy_url value into endpoint's elasticsearch output configuration, and enpoint/apm's fleet configuration if the attribute is missing and HTTPS_PROXY/HTTP_PROXY env var is set.

The first host value is used to determine if the HTTPS_PROXY, or HTTP_PROXY value is injected.
If that can't be used to determine then the HTTPS_PROXY is preferred.
No Injection occurs if the proxy_url key exists, proxy_disable:  true is set, or the env vars are empty.

(cherry picked from commit 097787f)

changelog/fragments/1720034703-Inject-proxy_url-into-endpoint-elasticsearch-output.yaml

@@ -0,0 +1,35 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: enhancement
+
+# Change summary; a 80ish characters long description of the change.
+summary: Inject proxy_url into endpoint elasticsearch output and fleet config
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+description: |
+ Inject a proxy_url value into endpoint's elasticsearch output and endpoint and APM's fleet configuration if there are no existing keys as part of the policy, but the agent default proxy servers are set by enviroment variables.
+ The 1st host in the output list is used to determine if the HTTPS_PROXY or HTTP_PROXY value is used, if it can't be determined this way the HTTPS_PROXY is used over HTTP_PROXY.
+ Injection will be skipped if the policy sets proxy_disable to true, the proxy_url key is set with an empty value, or the proxy from the env var is empty.
+
+# Affected component; a word indicating the component this changeset affects.
+component:
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https:/elastic/elastic-agent/pull/5044
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+issue: https:/elastic/elastic-agent/issues/2602

internal/pkg/agent/application/application.go

@@ -175,6 +175,7 @@ func New(
  compModifiers = append(compModifiers, FleetServerComponentModifier(cfg.Fleet.Server),
  InjectFleetConfigComponentModifier(cfg.Fleet, agentInfo),
  EndpointSignedComponentModifier(),
+ InjectProxyEndpointModifier(),
  )
 
  // TODO: stop using global state

internal/pkg/agent/application/fleet_server_bootstrap.go

@@ -125,6 +125,10 @@ func InjectFleetConfigComponentModifier(fleetCfg *configuration.FleetAgentConfig
  for _, host := range hostsStr {
  fleetHosts = append(fleetHosts, host)
  }
+ // add host val to hosts array for use in injectProxyURL
+ if host, ok := cfg["host"].(string); ok && host != "" {
+ hostsStr = append(hostsStr, host)
+ }
 
  for i, comp := range comps {
  if comp.InputSpec != nil && (comp.InputSpec.InputType == endpoint || comp.InputSpec.InputType == apmServer) {
@@ -146,6 +150,8 @@ func InjectFleetConfigComponentModifier(fleetCfg *configuration.FleetAgentConfig
 
  // Inject agent log level
  injectAgentLoggingLevel(m, agentInfo)
+ // Inject proxy_url
+ injectProxyURL(m, hostsStr)
  }
  }
  unitCfg, err := component.ExpectedConfig(unitCfgMap)

internal/pkg/agent/application/inject_proxy_component_modifier.go

@@ -0,0 +1,106 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package application
+
+import (
+ "os"
+ "strings"
+
+ "github.com/elastic/elastic-agent-client/v7/pkg/client"
+ "github.com/elastic/elastic-agent/internal/pkg/agent/application/coordinator"
+ "github.com/elastic/elastic-agent/pkg/component"
+)
+
+// InjectProxyEndpointModifier injects a proxy_url value into endpoint's output config if one is not set.
+//
+// This will inject a proxy_url if there is no existing key and the agent has a proxy set through an environment variable.
+// The URL used is the HTTPS_PROXY env var. If that's not set the HTTP_PROXY env var is used.
+// If there are no env vars set, or the unit's config has `proxy_disable: true`, nothing is injected
+// If the output config has `proxy_url: ""`, it will not be overwritten.
+func InjectProxyEndpointModifier() coordinator.ComponentsModifier {
+ return func(comps []component.Component, _ map[string]interface{}) ([]component.Component, error) {
+ for i, comp := range comps {
+ if comp.InputSpec != nil && comp.InputSpec.InputType == endpoint {
+ for j, unit := range comp.Units {
+ if unit.Type == client.UnitTypeOutput && unit.Config.Type == elasticsearch {
+ unitCfgMap := unit.Config.Source.AsMap()
+
+ // convert unitCfgMap["hosts"] to []string
+ var hosts []string
+ if hArr, ok := unitCfgMap["hosts"]; ok {
+ if arr, ok := hArr.([]interface{}); ok {
+ for _, v := range arr {
+ host, ok := v.(string)
+ if !ok {
+ continue
+ }
+ hosts = append(hosts, host)
+ }
+ }
+ }
+
+ injectProxyURL(unitCfgMap, hosts)
+ unitCfg, err := component.ExpectedConfig(unitCfgMap)
+ if err != nil {
+ return nil, err
+ }
+ unit.Config = unitCfg
+ }
+ comp.Units[j] = unit
+ }
+ }
+ comps[i] = comp
+ }
+ return comps, nil
+ }
+}
+
+// injectProxyURL will inject the a proxy_url into the passed map if there is no existing key and there is an appropriate proxy through defined as an env var.
+//
+// The 1st item of the passed hosts list is checked to see if it starts with https or http and the corresponding proxy var is used.
+// Nothing is injected if the *_PROXY env var is empty, the map contains proxy_url: "", or the map has proxy_disable: true.
+// If no hosts are passed, then the HTTPS_PROXY value is used over the HTTP_PROXY value if it's defined.
+func injectProxyURL(m map[string]interface{}, hosts []string) {
+ if m == nil {
+ return
+ }
+ // return if m already has a proxy
+ if _, ok := m["proxy_url"]; ok {
+ return
+ }
+ // Check if proxy_disable is part of config and true
+ disabled, disabledDefined := m["proxy_disable"]
+ if disabledDefined {
+ val, ok := disabled.(bool)
+ if ok && val {
+ return
+ }
+ }
+
+ var proxyURL string
+ matched := false
+ // If hosts are specified, check the 1st to see if HTTPS or HTTP is used to determine proxy
+ if len(hosts) > 0 {
+ if strings.HasPrefix(hosts[0], "https://") {
+ matched = true
+ proxyURL = os.Getenv("HTTPS_PROXY")
+ } else if strings.HasPrefix(hosts[0], "http://") {
+ matched = true
+ proxyURL = os.Getenv("HTTP_PROXY")
+ }
+ }
+ // if no hosts are specified, or it could not match a host prefix prefer HTTPS_PROXY over HTTP_PROXY
+ if proxyURL == "" && !matched {
+ proxyURL = os.Getenv("HTTPS_PROXY")
+ if proxyURL == "" {
+ proxyURL = os.Getenv("HTTP_PROXY")
+ }
+ }
+ // No proxy defined
+ if proxyURL == "" {
+ return
+ }
+ m["proxy_url"] = proxyURL
+}