Enable docker image caching for the deb build

.github/workflows/release-artifacts.yml

@@ -44,12 +44,43 @@ jobs:
  distro: ${{ fromJson(needs.get-distros.outputs.distros) }}
 
  steps:
- - uses: actions/checkout@v2
+ - name: Checkout
+ uses: actions/checkout@v2
  with:
  path: src
- - uses: actions/setup-python@v2
- - run: ./src/scripts-dev/build_debian_packages "${{ matrix.distro }}"
- - uses: actions/upload-artifact@v2
+
+ - name: Set up Docker Buildx
+ id: buildx
+ uses: docker/setup-buildx-action@v1
+ with:
+ install: true
+
+ - name: Set up docker layer caching
+ uses: actions/cache@v2
+ with:
+ path: /tmp/.buildx-cache
+ key: ${{ runner.os }}-buildx-${{ github.sha }}
+ restore-keys: |
+ ${{ runner.os }}-buildx-
+
+ - name: Set up python
+ uses: actions/setup-python@v2
+
+ - name: Build the packages
+ # see https:/docker/build-push-action/issues/252
+ # for the cache magic here
+ run: |
+ ./src/scripts-dev/build_debian_packages \
+ --docker-build-arg=--cache-from=type=local,src=/tmp/.buildx-cache \
+ --docker-build-arg=--cache-to=type=local,mode=max,dest=/tmp/.buildx-cache-new \
+ --docker-build-arg=--progress=plain \
+ --docker-build-arg=--load \
+ "${{ matrix.distro }}"
+ rm -rf /tmp/.buildx-cache
+ mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+
+ - name: Upload debs as artifacts
+ uses: actions/upload-artifact@v2
  with:
  name: debs
  path: debs/*

changelog.d/10431.misc

@@ -0,0 +1 @@
+Use a docker image cache for the prerequisites for the debian package build.

scripts-dev/build_debian_packages

@@ -17,6 +17,7 @@ import subprocess
 import sys
 import threading
 from concurrent.futures import ThreadPoolExecutor
+from typing import Optional, Sequence
 
 DISTS = (
  "debian:buster",
@@ -39,8 +40,11 @@ projdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 
 
 class Builder(object):
- def __init__(self, redirect_stdout=False):
+ def __init__(
+ self, redirect_stdout=False, docker_build_args: Optional[Sequence[str]] = None
+ ):
  self.redirect_stdout = redirect_stdout
+ self._docker_build_args = tuple(docker_build_args or ())
  self.active_containers = set()
  self._lock = threading.Lock()
  self._failed = False
@@ -79,8 +83,8 @@ class Builder(object):
  stdout = None
 
  # first build a docker image for the build environment
- subprocess.check_call(
- [
+ build_args = (
+ (
  "docker",
  "build",
  "--tag",
@@ -89,8 +93,13 @@ class Builder(object):
  "distro=" + dist,
  "-f",
  "docker/Dockerfile-dhvirtualenv",
- "docker",
- ],
+ )
+ + self._docker_build_args
+ + ("docker",)
+ )
+
+ subprocess.check_call(
+ build_args,
  stdout=stdout,
  stderr=subprocess.STDOUT,
  cwd=projdir,
@@ -147,9 +156,7 @@ class Builder(object):
  self.active_containers.remove(c)
 
 
-def run_builds(dists, jobs=1, skip_tests=False):
- builder = Builder(redirect_stdout=(jobs > 1))
-
+def run_builds(builder, dists, jobs=1, skip_tests=False):
  def sig(signum, _frame):
  print("Caught SIGINT")
  builder.kill_containers()
@@ -180,6 +187,11 @@ if __name__ == "__main__":
  action="store_true",
  help="skip running tests after building",
  )
+ parser.add_argument(
+ "--docker-build-arg",
+ action="append",
+ help="specify an argument to pass to docker build",
+ )
  parser.add_argument(
  "--show-dists-json",
  action="store_true",
@@ -195,4 +207,12 @@ if __name__ == "__main__":
  if args.show_dists_json:
  print(json.dumps(DISTS))
  else:
- run_builds(dists=args.dist, jobs=args.jobs, skip_tests=args.no_check)
+ builder = Builder(
+ redirect_stdout=(args.jobs > 1), docker_build_args=args.docker_build_arg
+ )
+ run_builds(
+ builder,
+ dists=args.dist,
+ jobs=args.jobs,
+ skip_tests=args.no_check,
+ )