Overhaul binary build process and support newer GDAL/GEOS/PROJ (#53)

* Switches to using GitHub Actions to build and upload the packages to S3,
  instead of the packages being built in containers locally.
* Fixes the errors seen when building `libkml` (#51) by switching to
  using `libkml` from the Ubuntu APT repository instead. As a side
  effect this means the final GDAL package is smaller since it no longer
  includes unnecessary libkml related headers.
* Switches to using CMake instead of autoconf/make, since newer
  GDAL/GEOS/PROJ no longer support autoconf.
* Rewrites the build scripts to accept an input version, so new versions
  can be built without having to copy-paste-commit additional scripts.
* Adds testing of the built packages against the run image, to ensure
  all dynamically linked libraries can be found.
* Enables GDAL's GEOS related features, by ensuring GEOS is available
  during the GDAL build.
* Documents the newly built and available package versions, which are:
  * GDAL: 3.6.4, 3.7.3, 3.8.5, 3.9.0
  * GEOS: 3.10.6, 3.11.3, 3.12.1
  * PROJ: 9.4.0

For now, the buildpack default versions of GDAL/GEOS/PROJ remain the
same, as anyone wanting the new versions needs to explicitly request the
new version using the `*_VERSION` env vars:
https:/heroku/heroku-geo-buildpack#default-versions

See also:
https://gdal.org/development/building_from_source.html
https://libgeos.org/usage/download/#build-from-source
https://proj.org/en/stable/install.html#compilation-and-installation-from-source-code

Fixes #51.
Fixes #52.

.github/workflows/build_package.yml

@@ -0,0 +1,57 @@
+name: Build and upload package
+run-name: "Build and upload ${{ inputs.package }} ${{ inputs.version }}${{ inputs.dry_run && ' (dry run)' || '' }}"
+
+on:
+ workflow_dispatch:
+ inputs:
+ package:
+ description: "Package to build"
+ type: choice
+ options:
+ - GDAL
+ - GEOS
+ - PROJ
+ required: true
+ version:
+ description: "Version of the package (X.Y.Z)"
+ type: string
+ required: true
+ dry_run:
+ description: "Skip uploading to S3 (dry run)"
+ type: boolean
+ default: false
+ required: false
+
+permissions:
+ contents: read
+
+env:
+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ AWS_DEFAULT_REGION: "us-east-1"
+ S3_BUCKET: "heroku-buildpack-geo"
+
+jobs:
+ build:
+ runs-on: ubuntu-latest
+ strategy:
+ fail-fast: false
+ matrix:
+ stack_version: ["20", "22"]
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v4
+ - name: Build Docker image
+ run: make buildenv STACK_VERSION=${{ matrix.stack_version }}
+ - name: Compile and package ${{ inputs.package }} ${{ inputs.version }}
+ run: |
+ BUILD_SCRIPT="./$(echo '${{ inputs.package }}' | tr '[:upper:]' '[:lower:]').sh"
+ docker run --rm -v "${PWD}/upload:/tmp/upload" geo-buildenv-${{ matrix.stack_version }} "${BUILD_SCRIPT}" '${{ inputs.version }}'
+ - name: Test package
+ run: |
+ RUN_IMAGE='heroku/heroku:${{ matrix.stack_version }}'
+ ARCHIVE_FILEPATH='/upload/heroku-${{ matrix.stack_version }}/${{ inputs.package }}/${{ inputs.package }}-${{ inputs.version }}.tar.gz'
+ docker run --rm -v "${PWD}/upload:/upload:ro" -v "${PWD}/builds:/builds:ro" "${RUN_IMAGE}" /builds/test_package.sh "${ARCHIVE_FILEPATH}"
+ - name: Upload package to S3
+ if: (!inputs.dry_run)
+ run: aws s3 sync ./upload "s3://${S3_BUCKET}"

.github/workflows/ci.yml

@@ -16,8 +16,8 @@ jobs:
  steps:
  - name: Checkout
  uses: actions/checkout@v4
- - run: shellcheck -x bin/*
- - run: find . -name "*.sh" -exec shellcheck -x {} \;
+ - name: Run ShellCheck
+  run: shellcheck --check-sourced --color=always bin/* builds/*.sh tests.sh
 
  test:
  runs-on: ubuntu-latest
@@ -28,4 +28,5 @@ jobs:
  steps:
  - name: Checkout
  uses: actions/checkout@v4
- - run: make test STACK_VERSION='${{ matrix.stack_version }}'
+ - name: Run tests
+ run: make test STACK_VERSION='${{ matrix.stack_version }}'

.gitignore

@@ -0,0 +1,2 @@
+upload/
+.DS_Store

Makefile

@@ -1,5 +1,5 @@
 # These targets are not files
-.PHONY: compile test build-heroku-20 build-heroku-22
+.PHONY: compile test buildenv
 
 STACK_VERSION ?= 22
 STACK := heroku-$(STACK_VERSION)
@@ -21,28 +21,15 @@ test:
  @docker run --rm --platform=$(PLATFORM) -v "$(PWD):/buildpack:ro" -e "STACK=$(STACK)" "$(BASE_BUILD_IMAGE)" /buildpack/tests.sh
  @echo
 
-build-heroku-20:
- @echo "Creating build environment (heroku-20)..."
+buildenv:
+ @echo "Creating build environment using: STACK_VERSION=$(STACK_VERSION)"
+ @echo "To use a different stack, run: 'make buildenv STACK_VERSION=NN'"
  @echo
- @docker build --pull -f "$(shell pwd)/builds/Dockerfile-heroku-20" -t buildenv-heroku-20 .
+ @docker build --pull --platform=$(PLATFORM) --build-arg="STACK_VERSION=$(STACK_VERSION)" -t "geo-buildenv-$(STACK_VERSION)" ./builds/
  @echo
  @echo "Usage..."
  @echo
- @echo " $$ export S3_BUCKET='heroku-geo-buildpack' # Optional unless deploying"
- @echo " $$ export AWS_ACCESS_KEY_ID=foo AWS_SECRET_ACCESS_KEY=bar # Optional unless deploying"
- @echo " $$ ./builds/gdal/gdal-<version>.sh"
+ @echo ' $$ docker run --rm -it -v "$${PWD}/upload:/tmp/upload" geo-buildenv-$(STACK_VERSION) ./proj.sh X.Y.Z'
+ @echo ' $$ docker run --rm -it -v "$${PWD}/upload:/tmp/upload" geo-buildenv-$(STACK_VERSION) ./geos.sh X.Y.Z'
+ @echo ' $$ docker run --rm -it -v "$${PWD}/upload:/tmp/upload" geo-buildenv-$(STACK_VERSION) ./gdal.sh X.Y.Z'
  @echo
- @docker run -e STACK="heroku-20" -e S3_BUCKET -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -it --rm buildenv-heroku-20
-
-build-heroku-22:
- @echo "Creating build environment (heroku-22)..."
- @echo
- @docker build --pull -f "$(shell pwd)/builds/Dockerfile-heroku-22" -t buildenv-heroku-22 .
- @echo
- @echo "Usage..."
- @echo
- @echo " $$ export S3_BUCKET='heroku-geo-buildpack' # Optional unless deploying"
- @echo " $$ export AWS_ACCESS_KEY_ID=foo AWS_SECRET_ACCESS_KEY=bar # Optional unless deploying"
- @echo " $$ ./builds/gdal/gdal-<version>.sh"
- @echo
- @docker run -e STACK="heroku-22" -e S3_BUCKET -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -it --rm buildenv-heroku-22

README.md

@@ -2,7 +2,7 @@ Heroku Buildpack: Geo
 =====================
 
 Heroku Buildpack Geo is a [Heroku buildpack](http://devcenter.heroku.com/articles/buildpacks) that
-installs the Geo/GIS libraries [GDAL](https://gdal.org/), [GEOS](https://trac.osgeo.org/geos/) and [PROJ](https://proj.org/)
+installs the Geo/GIS libraries [GDAL](https://gdal.org/), [GEOS](https://libgeos.org/) and [PROJ](https://proj.org/)
 
 It can be used to get [GeoDjango](https://docs.djangoproject.com/en/stable/ref/contrib/gis/) or [RGeo](https:/rgeo/rgeo) running on Heroku.
 
@@ -40,10 +40,23 @@ Available Versions
 ------------------
 
 - GDAL:
- - Heroku-20: `2.4.0`, `2.4.2`, `3.5.0`
- - Heroku-22: `3.5.0`
-- GEOS: `3.7.2`, `3.10.2`
-- PROJ: `5.2.0`, `8.2.1`
+ - `2.4.0` (Heroku-20 only)
+ - `2.4.2` (Heroku-20 only)
+ - `3.5.0`
+ - `3.6.4`
+ - `3.7.3`
+ - `3.8.5`
+ - `3.9.0`
+- GEOS:
+ - `3.7.2`
+ - `3.10.2`
+ - `3.10.6`
+ - `3.11.3`
+ - `3.12.1`
+- PROJ:
+ - `5.2.0`
+ - `8.2.1`
+ - `9.4.0`
 
 Migrating from heroku/python GEO support
 ----------------------------------------

builds/Dockerfile

@@ -0,0 +1,19 @@
+ARG STACK_VERSION="24"
+FROM heroku/heroku:${STACK_VERSION}-build
+
+ARG STACK_VERSION
+ENV STACK="heroku-${STACK_VERSION}"
+
+# For Heroku-24 and newer, the build image sets a non-root default `USER`.
+USER root
+
+RUN apt-get update --error-on=any \
+ && apt-get install -y --no-install-recommends \
+ libkml-dev \
+ libsqlite3-dev \
+ sqlite3 \
+ && rm -rf /var/lib/apt/lists/*
+
+ADD . /heroku-geo-buildpack
+
+WORKDIR "/heroku-geo-buildpack"

builds/gdal.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# shellcheck source=builds/utils.sh
+source "$(dirname "${BASH_SOURCE[0]}")/utils.sh"
+
+VERSION="${1:?"Error: The GDAL version to build must be specified as the first argument."}"
+
+SRC_DIR="$(mktemp -d)"
+INSTALL_DIR="$(mktemp -d)"
+ARCHIVE_DIR="/tmp/upload/${STACK}/GDAL"
+
+CONCURRENCY="$(nproc)"
+
+echo "Building GDAL ${VERSION} for ${STACK}..."
+
+set -o xtrace
+
+# GDAL 3.0.0+ requires PROJ at build time.
+vendor_dependency "PROJ" "9.4.0" "${INSTALL_DIR}"
+
+# The optional GEOS features require that GEOS be available at build time.
+vendor_dependency "GEOS" "3.12.1" "${INSTALL_DIR}"
+
+# The optional KML features require that libkml be installed. The libkml headers and libs were
+# installed in the build environment for this script using APT, but the libs won't be present
+# in the run image, so we have to vendor them (and some transitive deps) in the package.
+# We don't build libkml from source since its last official release is from 2015 and broken:
+# https:/heroku/heroku-geo-buildpack/issues/51
+cp /lib/x86_64-linux-gnu/{libkml,libminizip,liburiparser}* "${INSTALL_DIR}/lib"
+
+# Ensure vendored dependencies can be found.
+export CPATH="${INSTALL_DIR}/include"
+export LIBRARY_PATH="${INSTALL_DIR}/lib"
+export LD_LIBRARY_PATH="${INSTALL_DIR}/lib"
+
+cd "${SRC_DIR}"
+curl -sSf --retry 3 --retry-connrefused --connect-timeout 10 --max-time 60 "https://download.osgeo.org/gdal/${VERSION}/gdal-${VERSION}.tar.gz" \
+ | tar --gzip --extract --strip-components=1 --directory .
+
+# https://gdal.org/development/building_from_source.html
+mkdir build
+cd build
+cmake \
+ -DBUILD_TESTING=OFF \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \
+ ..
+cmake --build . -j "${CONCURRENCY}"
+cmake --build . --target install -j "${CONCURRENCY}"
+
+# Strip binaries to reduce package size. Files that are executable but not binaries (eg scripts)
+# have to be explicitly skipped otherwise `strip` will give 'file format not recognized' errors.
+find "${INSTALL_DIR}" -type f \( -executable -o -name '*.so*' \) ! \( -name 'gdal-config' -o -name 'geos-config' -o -name '*.la' \) -print -exec strip --strip-unneeded '{}' +
+
+mkdir -p "${ARCHIVE_DIR}"
+TAR_FILEPATH="${ARCHIVE_DIR}/GDAL-${VERSION}.tar"
+tar --create --format=pax --sort=name --file="${TAR_FILEPATH}" --directory="${INSTALL_DIR}" .
+gzip --best "${TAR_FILEPATH}"