Remove testing/environments (bp #5035) (#5042)

* Remove testing/environments (#5035)

* Remove testing/environments

* docker-compose.yml: define services locally

We only use the snapshot services, and we customise
the services heavily; let's just define them in our
own docker-compose.yml and drop the dependency on
testing/environments.

* Make target to check docker images are up to date

`make check-docker-compose` checks if the images in
docker-compose.yml are up to date with the most
recent snapshot version for the current branch.

* docker-compose.yml: wait for yellow status

Change the Elasticsearch healthcheck to wait
for the status to become yellow, rather than
green. Yellow means primaries are assigned,
but one or more replicas are not. We don't
care about replicas in system tests.

(cherry picked from commit 70911de)

# Conflicts:
#	testing/environments/docker/elasticsearch_kerberos/Dockerfile
#	testing/environments/latest.yml
#	testing/environments/snapshot-oss.yml
#	testing/environments/snapshot.yml

* Delete testing/environments directory

* Update image versions in docker-compose.yml

Co-authored-by: Andrew Wilkins <[email protected]>

Makefile

@@ -72,7 +72,7 @@ SYSTEM_TEST_TARGET?=./tests/system
 PYTEST_OPTIONS?=--timeout=90 --durations=20 --junit-xml=build/TEST-system.xml
 
 .PHONY: check-full
-check-full: update check golint staticcheck
+check-full: update check golint staticcheck check-docker-compose
 
 .PHONY: check-approvals
 check-approvals: $(APPROVALS)
@@ -193,7 +193,6 @@ update-beats-module:
  $(GO) get -d -u $(BEATS_MODULE)@$(BEATS_VERSION) && $(GO) mod tidy
  diff -u .go-version $$($(GO) list -m -f {{.Dir}} $(BEATS_MODULE))/.go-version \
  || { code=$$?; echo ".go-version out of sync with Beats"; exit $$code; }
- rsync -crv --delete $$($(GO) list -m -f {{.Dir}} $(BEATS_MODULE))/testing/environments testing/
 
 ##############################################################################
 # Kibana synchronisation.
@@ -233,6 +232,10 @@ ifndef CHECK_HEADERS_DISABLED
  @$(GOLICENSER) -d -exclude build -license Elastic x-pack
 endif
 
+.PHONY: check-docker-compose
+check-docker-compose: $(PYTHON_BIN)
+ @PATH=$(PYTHON_BIN):$(PATH) ./script/check_docker_compose.sh $(BEATS_VERSION)
+
 .PHONY: check-package format-package build-package
 check-package: $(ELASTICPACKAGE)
  @for x in apmpackage/apm/*; do (cd $$x; echo "Checking $$x"; $(CURDIR)/$(ELASTICPACKAGE) check); done

docker-compose.yml

@@ -32,12 +32,18 @@ services:
  kibana: { condition: service_healthy }
 
  elasticsearch:
+ image: docker.elastic.co/elasticsearch/elasticsearch:7.13.0-SNAPSHOT
  ports:
  - 9200:9200
- extends:
- file: ./testing/environments/${TESTING_ENVIRONMENT:-snapshot}.yml
- service: elasticsearch
+ healthcheck:
+ test: ["CMD-SHELL", "curl -s http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=500ms"]
+ retries: 300
+ interval: 1s
  environment:
+ - "ES_JAVA_OPTS=-Xms1g -Xmx1g"
+ - "network.host="
+ - "transport.host=127.0.0.1"
+ - "http.host=0.0.0.0"
  - "cluster.routing.allocation.disk.threshold_enabled=false"
  - "discovery.type=single-node"
  - "xpack.security.authc.anonymous.roles=remote_monitoring_collector"
@@ -54,11 +60,13 @@ services:
  - "./testing/docker/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles"
 
  kibana:
+ image: docker.elastic.co/kibana/kibana:7.13.0-SNAPSHOT
  ports:
  - 5601:5601
- extends:
- file: ./testing/environments/${TESTING_ENVIRONMENT:-snapshot}.yml
- service: kibana
+ healthcheck:
+ test: ["CMD-SHELL", "curl -s http://localhost:5601/api/status | grep -q 'Looking good'"]
+ retries: 300
+ interval: 1s
  environment:
  STATUS_ALLOWANONYMOUS: "true"
  ELASTICSEARCH_URL: elasticsearch:9200
@@ -72,6 +80,7 @@ services:
  XPACK_FLEET_AGENTS_TLSCHECKDISABLED: "true"
  XPACK_FLEET_REGISTRYURL: "http://package-registry:8080"
  depends_on:
+ elasticsearch: { condition: service_healthy }
  package-registry: { condition: service_healthy }
 
  package-registry:

script/check_docker_compose.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+#
+# This checks that image versions defined in docker-compose.yml are
+# up to date for the given branch name (master, 7.x, 7.13, etc.)
+#
+# Example usage: ./check_docker_compose.sh 7.x
+set -e
+
+SDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+BRANCH=$*
+LATEST_SNAPSHOT_VERSION=$($SDIR/latest_snapshot_version.py $BRANCH)
+
+# Check docker.elastic.co images listed in docker-compose.yml are up to date.
+# Ignore any images that don't end with "-SNAPSHOT", such as package-registry.
+IMAGES=$(grep 'image: docker.elastic.co.*-SNAPSHOT' $SDIR/../docker-compose.yml | sed 's/.*image: \(.*\)/\1/')
+for IMAGE in $IMAGES; do
+ IMAGE_TAG=$(echo "$IMAGE" | cut -d: -f2)
+ if [ "$IMAGE_TAG" = "$LATEST_SNAPSHOT_VERSION" ]; then
+ printf "docker-compose.yml: image %s up to date (latest '%s' snapshot version %s)\n" "$IMAGE" "$BRANCH" "$LATEST_SNAPSHOT_VERSION"
+ else
+ printf "docker-compose.yml: image %s is out of date (latest '%s' snapshot version is %s)\n" "$IMAGE" "$BRANCH" "$LATEST_SNAPSHOT_VERSION"
+ exit 1
+ fi
+done

script/latest_snapshot_version.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+#
+# Find the latest snapshot version for the specified branch.
+#
+# Example usage: ./latest_snapshot_version.py 7.x
+
+import argparse
+import requests
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('branch', type=str)
+ args = parser.parse_args()
+
+ r = requests.get('https://snapshots.elastic.co/latest/{}.json'.format(args.branch))
+ r.raise_for_status()
+ print(r.json()['version'])
+
+
+if __name__ == '__main__':
+ main()