Poetry: select olddeps using poetry

.ci/scripts/test_old_deps.sh

@@ -1,6 +1,9 @@
 #!/usr/bin/env bash
-# this script is run by GitHub Actions in a plain `focal` container; it installs the
-# minimal requirements for tox and hands over to the py3-old tox environment.
+# this script is run by GitHub Actions in a plain `focal` container; it
+# - installs the minimal system requirements, and poetry;
+# - patches the project definition file to refer to old versions only;
+# - creates a venv with these old versions using poetry; and finally
+# - invokes `trial` to run the tests with old deps.
 
 # Prevent tzdata from asking for user input
 export DEBIAN_FRONTEND=noninteractive
@@ -9,12 +12,70 @@ set -ex
 
 apt-get update
 apt-get install -y \
- python3 python3-dev python3-pip python3-venv \
- libxml2-dev libxslt-dev xmlsec1 zlib1g-dev tox libjpeg-dev libwebp-dev
+ python3 python3-dev python3-pip python3-venv pipx \
+ libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev
 
 export LANG="C.UTF-8"
 
 # Prevent virtualenv from auto-updating pip to an incompatible version
 export VIRTUALENV_NO_DOWNLOAD=1
 
-exec tox -e py3-old
+# TODO: in the future, we could use an implementation of
+# https:/python-poetry/poetry/issues/3527
+# https:/pypa/pip/issues/8085
+# to select the lowest possible versions, rather than resorting to this sed script.
+
+# Patch the project definitions in-place:
+# - Replace all lower and tilde bounds with exact bounds
+# - Make the pyopenssl 17.0, which is the oldest version that works with
+# a `cryptography` compiled against OpenSSL 1.1.
+# - Delete all lines referring to psycopg2 --- so no testing of postgres support.
+# - Omit systemd: we're not logging to journal here.
+
+# TODO: also replace caret bounds, see https://python-poetry.org/docs/dependency-specification/#version-constraints
+# We don't use these yet, but IIRC they are the default bound used when you `poetry add`.
+# The sed expression 's/\^/==/g' ought to do the trick. But it would also change
+# `python = "^3.7"` to `python = "==3.7", which would mean we fail because olddeps
+# runs on 3.8 (#12343).
+
+sed -i \
+ -e "s/[~>]=/==/g" \
+ -e "/psycopg2/d" \
+ -e 's/pyOpenSSL = "==16.0.0"/pyOpenSSL = "==17.0.0"/' \
+ -e '/systemd/d' \
+ pyproject.toml
+
+# Use poetry to do the installation. This ensures that the versions are all mutually
+# compatible (as far the package metadata declares, anyway); pip's package resolver
+# is more lax.
+#
+# Rather than `poetry install --no-dev`, we drop all dev dependencies from the
+# toml file. This means we don't have to ensure compatibility between old deps and
+# dev tools.
+
+pip install --user toml
+
+REMOVE_DEV_DEPENDENCIES="
+import toml
+with open('pyproject.toml', 'r') as f:
+ data = toml.loads(f.read())
+
+del data['tool']['poetry']['dev-dependencies']
+
+with open('pyproject.toml', 'w') as f:
+ toml.dump(data, f)
+"
+python3 -c "$REMOVE_DEV_DEPENDENCIES"
+
+pipx install poetry==1.1.12
+~/.local/bin/poetry lock
+
+echo "::group::Patched pyproject.toml"
+cat pyproject.toml
+echo "::endgroup::"
+echo "::group::Lockfile after patch"
+cat poetry.lock
+echo "::endgroup::"
+
+~/.local/bin/poetry install -E "all test"
+~/.local/bin/poetry run trial --jobs=2 tests

.github/workflows/tests.yml

@@ -128,18 +128,19 @@ jobs:
  || true
 
  trial-olddeps:
+ # Note: sqlite only; no postgres
  if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
  needs: linting-done
  runs-on: ubuntu-latest
  steps:
  - uses: actions/checkout@v2
  - name: Test with old deps
  uses: docker://ubuntu:focal # For old python and sqlite
+ # Note: focal seems to be using 3.8, but the oldest is 3.7?
+ # See https:/matrix-org/synapse/issues/12343
  with:
  workdir: /github/workspace
  entrypoint: .ci/scripts/test_old_deps.sh
- env:
- TRIAL_FLAGS: "--jobs=2"
  - name: Dump logs
  # Logs are most useful when the command fails, always include them.
  if: ${{ always() }}

changelog.d/12407.misc

@@ -0,0 +1 @@
+Run the olddeps CI job using Poetry.

tox.ini

@@ -105,29 +105,7 @@ commands =
 # )
 usedevelop=true
 
-# A test suite for the oldest supported versions of Python libraries, to catch
-# any uses of APIs not available in them.
-[testenv:py3-old]
-skip_install = true
-usedevelop = false
-deps =
- Automat == 0.8.0
- lxml
- # markupsafe 2.1 introduced a change that breaks Jinja 2.x. Since we depend on
- # Jinja >= 2.9, it means this test suite will fail if markupsafe >= 2.1 is installed.
- markupsafe < 2.1
- {[base]deps}
-
-commands =
- # Make all greater-thans equals so we test the oldest version of our direct
- # dependencies, but make the pyopenssl 17.0, which can work against an
- # OpenSSL 1.1 compiled cryptography (as older ones don't compile on Travis).
- /bin/sh -c 'python -m synapse.python_dependencies | sed -e "s/>=/==/g" -e "/psycopg2/d" -e "s/pyopenssl==16.0.0/pyopenssl==17.0.0/" | xargs -d"\n" pip install'
-
- # Install Synapse itself. This won't update any libraries.
- pip install -e ".[test]"
 
- {[testenv]commands}
 
 [testenv:benchmark]
 deps =