Skip to content

Commit

Permalink
Merge pull request #20472 from maribu/docker/pin/version
Browse files Browse the repository at this point in the history
makefiles/docker.inc.mk: Pin riotbuild version with BUILD_IN_DOCKER=1
  • Loading branch information
maribu authored Jun 8, 2024
2 parents ce8f89b + 17dcb97 commit b6696e0
Show file tree
Hide file tree
Showing 3 changed files with 157 additions and 5 deletions.
24 changes: 24 additions & 0 deletions dist/tools/buildsystem_sanity_check/check.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
: "${RIOTBASE:="$(cd "$(dirname "$0")/../../../" || exit; pwd)"}"

: "${RIOTTOOLS:=${RIOTBASE}/dist/tools}"
: "${RIOTMAKE:=${RIOTBASE}/makefiles}"
# not running shellcheck with -x in the CI --> disable SC1091
# shellcheck disable=SC1091
. "${RIOTTOOLS}"/ci/github_annotate.sh
Expand Down Expand Up @@ -380,6 +381,28 @@ check_tests_application_path() {
find tests/ -type f "${patterns[@]}" | error_with_message "Invalid application path in tests/"
}

check_pinned_docker_version_is_up_to_date() {
local pinned_digest
local pinned_repo_digest
local upstream_digest
local upstream_repo_digest
pinned_digest="$(awk '/^DOCKER_TESTED_IMAGE_ID := (.*)$/ { print substr($0, index($0, $3)); exit }' "$RIOTMAKE/docker.inc.mk")"
pinned_repo_digest="$(awk '/^DOCKER_TESTED_IMAGE_REPO_DIGEST := (.*)$/ { print substr($0, index($0, $3)); exit }' "$RIOTMAKE/docker.inc.mk")"
# not using docker and jq here but a python script to not have to install
# more stuff for the static test docker image
IFS=' ' read -r upstream_digest upstream_repo_digest <<< "$("$RIOTTOOLS/buildsystem_sanity_check/get_dockerhub_digests.py" "riot/riotbuild")"

if [ "$pinned_digest" != "$upstream_digest" ]; then
git -C "${RIOTBASE}" grep -n '^DOCKER_TESTED_IMAGE_ID :=' "$RIOTMAKE/docker.inc.mk" \
| error_with_message "Update docker image SHA256 to ${upstream_digest}"
fi

if [ "$pinned_repo_digest" != "$upstream_repo_digest" ]; then
git -C "${RIOTBASE}" grep -n '^DOCKER_TESTED_IMAGE_REPO_DIGEST :=' "$RIOTMAKE/docker.inc.mk" \
| error_with_message "Update manifest digest to ${upstream_repo_digest}"
fi
}

error_on_input() {
! grep ''
}
Expand All @@ -402,6 +425,7 @@ all_checks() {
check_no_riot_config
check_stderr_null
check_tests_application_path
check_pinned_docker_version_is_up_to_date
}

if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
Expand Down
91 changes: 91 additions & 0 deletions dist/tools/buildsystem_sanity_check/get_dockerhub_digests.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
#!/usr/bin/env python3
"""
Command line utility to get the image sha256 sum and the manifest sha256 sum
of the latest image at dockerhub.
"""
import http.client
import json
import sys


def get_docker_token(repo):
"""
Get an API access token for the docker registry
:param repo: the repository the API token should be valid for
:type repo: str
:return: the access token to use
:rtype: str
"""
conn = http.client.HTTPSConnection("auth.docker.io")
conn.request("GET",
f"/token?service=registry.docker.io&scope=repository:{repo}:pull",
headers={"Accept": "*/*"})
resp = conn.getresponse()
if resp.status != 200:
raise Exception(f"Tried to get a docker token, but auth.docker.io "
f"replied with {resp.status} {resp.reason}")
decoded = json.loads(resp.read())
conn.close()
return decoded["token"]


def get_manifest(repo, tag="latest", token=None):
"""
Get the manifest of the given repo
:param repo: The repository to get the latest manifest of
:type repo: str
:param tag: The tag to get the manifest of. (Default: "latest")
:type tag: str
:param token: The authorization token to use for the given repo.
(Default: get a fresh one.)
:type token: str
:return: the parsed manifast
:rtype: dict
"""
token = get_docker_token(repo) if token is None else token
conn = http.client.HTTPSConnection("index.docker.io")
hdrs = {
"Accept": "application/vnd.docker.distribution.manifest.v2+json",
"Authorization": f"Bearer {token}"
}
conn.request("GET", f"/v2/{repo}/manifests/{tag}", headers=hdrs)
resp = conn.getresponse()
if resp.status != 200:
raise Exception(f"Tried to get a docker manifest, but "
f"index.docker.io replied with {resp.status} "
f"{resp.reason}")
repo_digest = resp.getheader("ETag")[len("\"sha256:"):-1]
decoded = json.loads(resp.read())
conn.close()
return (decoded, repo_digest)


def get_upstream_digests(repo, tag="latest", token=None):
"""
Get the SHA256 hash of the latest image of the given repo at dockerhub
as string of hex digests
:param repo: The repository to get the hash from
:type repo: str
:param tag: The tag to get the manifest of. (Default: "latest")
:type tag: str
:param token: The authorization token to use for the given repo.
(Default: get a fresh one.)
:type token: str
:return: A 2-tuple of the image digest and the repo digest
:rtype: (str, str)
"""
token = get_docker_token(repo) if token is None else token
manifest, repo_digest = get_manifest(repo, tag=tag, token=token)
digest = manifest["config"]["digest"]
return (digest[len("sha256:"):], repo_digest)


if __name__ == '__main__':
if len(sys.argv) != 2:
sys.exit(f"Usage {sys.argv[0]} <REPO_NAME>")

digest, repo_digest = get_upstream_digests(sys.argv[1])
print(f"{digest} {repo_digest}")
47 changes: 42 additions & 5 deletions makefiles/docker.inc.mk
Original file line number Diff line number Diff line change
@@ -1,6 +1,22 @@
export DOCKER_IMAGE ?= docker.io/riot/riotbuild:latest
# This *MUST* be updated in lock-step with the riotbuild image in
# https:/RIOT-OS/riotdocker. The idea is that when checking out
# a random RIOT merge commit, `make BUILD_IN_DOCKER=1` should always succeed.
DOCKER_TESTED_IMAGE_ID := f5951bc41dfface6cac869181d703e62cbdd3b7976b0946130a38f2e658000b3
DOCKER_TESTED_IMAGE_REPO_DIGEST := 75dec511ba26424987a26bdee5ac2f94d5f4928d79b627d1620b9d2391aab3e1

DOCKER_PULL_IDENTIFIER := docker.io/riot/riotbuild@sha256:$(DOCKER_TESTED_IMAGE_REPO_DIGEST)
DOCKER_IMAGE_DEFAULT := sha256:$(DOCKER_TESTED_IMAGE_ID)
DOCKER_AUTO_PULL ?= 1
export DOCKER_IMAGE ?= $(DOCKER_IMAGE_DEFAULT)
export DOCKER_BUILD_ROOT ?= /data/riotbuild
DOCKER_RIOTBASE ?= $(DOCKER_BUILD_ROOT)/riotbase

# These targets need to be run before docker can be run
DEPS_FOR_RUNNING_DOCKER :=

# Overwrite if you want to use `docker` with sudo
DOCKER ?= docker

# List of Docker-enabled make goals
export DOCKER_MAKECMDGOALS_POSSIBLE = \
all \
Expand All @@ -19,6 +35,25 @@ else
export INSIDE_DOCKER := 0
endif

ifeq (0:1,$(INSIDE_DOCKER):$(BUILD_IN_DOCKER))
ifeq ($(DOCKER_IMAGE),$(DOCKER_IMAGE_DEFAULT))
IMAGE_PRESENT:=$(shell $(DOCKER) image inspect $(DOCKER_IMAGE) 2>/dev/null >/dev/null && echo 1 || echo 0)
ifeq (0,$(IMAGE_PRESENT))
$(warning Required docker image $(DOCKER_IMAGE) not installed)
ifeq (1,$(DOCKER_AUTO_PULL))
$(info Pulling required image automatically. You can disable this with DOCKER_AUTO_PULL=0)
DEPS_FOR_RUNNING_DOCKER += docker-pull
else
$(info Building with latest available riotbuild image. You can pull the correct image automatically with DOCKER_AUTO_PULL=1)
# The currently set DOCKER_IMAGE is not locally available, and the
# user opted out to automatically pull it. Fall back to the
# latest (locally) available riot/riotbuild image instead.
export DOCKER_IMAGE := docker.io/riot/riotbuild:latest
endif
endif
endif
endif

# Default target for building inside a Docker container if nothing was given
export DOCKER_MAKECMDGOALS ?= all
# List of all exported environment variables that shall be passed on to the
Expand Down Expand Up @@ -115,8 +150,6 @@ DOCKER_OVERRIDE_CMDLINE_AUTO := $(foreach varname,$(DOCKER_ENV_VARS), \
))
DOCKER_OVERRIDE_CMDLINE += $(strip $(DOCKER_OVERRIDE_CMDLINE_AUTO))

# Overwrite if you want to use `docker` with sudo
DOCKER ?= docker
_docker_is_podman = $(shell $(DOCKER) --version | grep podman 2>/dev/null)

# Set default run flags:
Expand Down Expand Up @@ -189,7 +222,6 @@ define dir_is_outside_riotbase
$(filter $(abspath $1)/,$(patsubst $(RIOTBASE)/%,%,$(abspath $1)/))
endef


# Mapping of directores inside docker
#
# Return the path of directories from the host within the container
Expand Down Expand Up @@ -342,13 +374,18 @@ docker_run_make = \
-w '$(DOCKER_APPDIR)' '$2' \
$(MAKE) $(DOCKER_OVERRIDE_CMDLINE) $4 $1

# This target pulls the docker image required for BUILD_IN_DOCKER
.PHONY: docker-pull
docker-pull:
$(DOCKER) pull '$(DOCKER_PULL_IDENTIFIER)'

# This will execute `make $(DOCKER_MAKECMDGOALS)` inside a Docker container.
# We do not push the regular $(MAKECMDGOALS) to the container's make command in
# order to only perform building inside the container and defer executing any
# extra commands such as flashing or debugging until after leaving the
# container.
# The `flash`, `term`, `debugserver` etc. targets usually require access to
# hardware which may not be reachable from inside the container.
..in-docker-container:
..in-docker-container: $(DEPS_FOR_RUNNING_DOCKER)
@$(COLOR_ECHO) '$(COLOR_GREEN)Launching build container using image "$(DOCKER_IMAGE)".$(COLOR_RESET)'
$(call docker_run_make,$(DOCKER_MAKECMDGOALS),$(DOCKER_IMAGE),,$(DOCKER_MAKE_ARGS))

0 comments on commit b6696e0

Please sign in to comment.