Add a stub Rust crate

.ci/scripts/test_old_deps.sh → .ci/scripts/prepare_old_deps.sh

@@ -5,18 +5,8 @@
 # - creates a venv with these old versions using poetry; and finally
 # - invokes `trial` to run the tests with old deps.
 
-# Prevent tzdata from asking for user input
-export DEBIAN_FRONTEND=noninteractive
-
 set -ex
 
-apt-get update
-apt-get install -y \
- python3 python3-dev python3-pip python3-venv pipx \
- libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev
-
-export LANG="C.UTF-8"
-
 # Prevent virtualenv from auto-updating pip to an incompatible version
 export VIRTUALENV_NO_DOWNLOAD=1
 
@@ -55,7 +45,7 @@ sed -i \
 # toml file. This means we don't have to ensure compatibility between old deps and
 # dev tools.
 
-pip install --user toml
+pip install toml wheel
 
 REMOVE_DEV_DEPENDENCIES="
 import toml
@@ -69,15 +59,12 @@ with open('pyproject.toml', 'w') as f:
 "
 python3 -c "$REMOVE_DEV_DEPENDENCIES"
 
-pipx install poetry==1.1.14
-~/.local/bin/poetry lock
+pip install poetry==1.2.0
+poetry lock
 
 echo "::group::Patched pyproject.toml"
 cat pyproject.toml
 echo "::endgroup::"
 echo "::group::Lockfile after patch"
 cat poetry.lock
 echo "::endgroup::"
-
-~/.local/bin/poetry install -E "all test"
-~/.local/bin/poetry run trial --jobs=2 tests

.dockerignore

@@ -4,8 +4,12 @@
 # things to include
 !docker
 !synapse
+!rust
 !README.rst
 !pyproject.toml
 !poetry.lock
+!build_rust.py
+
+rust/target
 
 **/__pycache__

.github/workflows/release-artifacts.yml

@@ -15,7 +15,7 @@ on:
 concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true
- 
+
 permissions:
  contents: write
 
@@ -89,17 +89,50 @@ jobs:
  name: debs
  path: debs/*
 
- build-sdist:
- name: "Build pypi distribution files"
- uses: "matrix-org/backend-meta/.github/workflows/packaging.yml@v1"
+ build_wheels:
+ name: Build wheels on ${{ matrix.os }}
+ runs-on: ${{ matrix.os }}
+ strategy:
+ matrix:
+ os: [ubuntu-20.04, macos-10.15]
+
+ steps:
+ - uses: actions/checkout@v3
+
+ - uses: actions/setup-python@v3
+
+ - name: Install cibuildwheel
+ run: python -m pip install cibuildwheel==2.9.0 poetry==1.2.0
+
+ - name: Build wheels
+ run: python -m cibuildwheel --output-dir wheelhouse
+ env:
+ # Skip testing for platforms which various libraries don't have wheels
+ # for, and so need extra build deps.
+ CIBW_TEST_SKIP: pp39-* *i686* *musl* pp37-macosx*
+
+ - uses: actions/upload-artifact@v3
+ with:
+ name: Wheel
+ path: ./wheelhouse/*.whl
+
+ - name: Build sdist
+ if: matrix.os == 'ubuntu' && matrix.python-version == 'cp310'
+ run: python -m build --sdist
+
+ - uses: actions/upload-artifact@v2
+ with:
+ name: Sdist
+ path: dist/*.tar.gz
+
 
  # if it's a tag, create a release and attach the artifacts to it
  attach-assets:
  name: "Attach assets to release"
  if: ${{ !failure() && !cancelled() && startsWith(github.ref, 'refs/tags/') }}
  needs:
  - build-debs
- - build-sdist
+ - build_wheels
  runs-on: ubuntu-latest
  steps:
  - name: Download all workflow run artifacts

.github/workflows/tests.yml

@@ -150,16 +150,47 @@ jobs:
  # Note: sqlite only; no postgres
  if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
  needs: linting-done
- runs-on: ubuntu-latest
+ runs-on: ubuntu-20.04
  steps:
  - uses: actions/checkout@v2
- - name: Test with old deps
- uses: docker://ubuntu:focal # For old python and sqlite
- # Note: focal seems to be using 3.8, but the oldest is 3.7?
- # See https:/matrix-org/synapse/issues/12343
+ - name: Install Rust
+ uses: actions-rs/toolchain@v1
+ with:
+ toolchain: stable
+ override: true
+
+ # There aren't wheels for some of the older deps, so we need to install
+ # their build dependencies
+ - run: |
+ sudo apt-get -qq install build-essential libffi-dev python-dev \
+ libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev
+
+ - uses: actions/setup-python@v4
+ with:
+ python-version: '3.7'
+
+ # Calculating the old-deps actually takes a bunch of time, so we cache the
+ # pyproject.toml / poetry.lock.
+ - uses: actions/cache@v3
+ id: cache-poetry-old-deps
+ name: Cache poetry.lock
  with:
- workdir: /github/workspace
- entrypoint: .ci/scripts/test_old_deps.sh
+ path: |
+ poetry.lock
+ pyproject.toml
+ key: poetry-old-deps2-${{ hashFiles('pyproject.toml') }}
+ - name: Prepare old deps
+ if: steps.cache-poetry-old-deps.outputs.cache-hit != 'true'
+ run: .ci/scripts/prepare_old_deps.sh
+
+ # We only now install poetry so that `setup-python-poetry` caches the
+ # right poetry.lock's dependencies.
+ - uses: matrix-org/setup-python-poetry@v1
+ with:
+ python-version: '3.7'
+ extras: "all test"
+
+ - run: poetry run trial -j 2 tests
  - name: Dump logs
  # Logs are most useful when the command fails, always include them.
  if: ${{ always() }}
@@ -252,6 +283,11 @@ jobs:
  - uses: actions/checkout@v2
  - name: Prepare test blacklist
  run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers
+ - name: Install Rust
+ uses: actions-rs/toolchain@v1
+ with:
+ toolchain: stable
+ override: true
  - name: Run SyTest
  run: /bootstrap.sh synapse
  working-directory: /src
@@ -358,6 +394,12 @@ jobs:
  with:
  path: synapse
 
+ - name: Install latest nightly
+ uses: actions-rs/toolchain@v1
+ with:
+ toolchain: stable
+ override: true
+
  - name: Prepare Complement's Prerequisites
  run: synapse/.ci/scripts/setup_complement_prerequisites.sh
 

.gitignore

@@ -60,3 +60,10 @@ book/
 # complement
 /complement-*
 /master.tar.gz
+
+# rust
+/target/
+/synapse/*.so
+
+# Poetry will create a setup.py, which we don't want to include.
+/setup.py

Cargo.toml

@@ -0,0 +1,5 @@
+# We make the whole Synapse folder a workspace so that we can run `cargo`
+# commands from the root (rather than having to cd into rust/).
+
+[workspace]
+members = ["rust"]

build_rust.py

@@ -0,0 +1,19 @@
+# A build script for poetry that adds the rust extension.
+
+import os
+
+from setuptools_rust import Binding, RustExtension
+
+
+def build(setup_kwargs):
+ original_project_dir = os.path.dirname(os.path.realpath(__file__))
+ cargo_toml_path = os.path.join(original_project_dir, "rust", "Cargo.toml")
+
+ extension = RustExtension(
+ target="synapse.synapse_rust",
+ path=cargo_toml_path,
+ binding=Binding.PyO3,
+ py_limited_api=True,
+ )
+ setup_kwargs.setdefault("rust_extensions", []).append(extension)
+ setup_kwargs["zip_safe"] = False

changelog.d/12595.misc

@@ -0,0 +1 @@
+Add a stub Rust crate.

changelog.d/13707.misc

@@ -0,0 +1 @@
+Update trial old deps CI to use poetry 1.2.0.

debian/build_virtualenv

@@ -61,7 +61,7 @@ dh_virtualenv \
  --extras="all,systemd,test" \
  --requirements="exported_requirements.txt"
 
-PACKAGE_BUILD_DIR="debian/matrix-synapse-py3"
+PACKAGE_BUILD_DIR="$(pwd)/debian/matrix-synapse-py3"
 VIRTUALENV_DIR="${PACKAGE_BUILD_DIR}${DH_VIRTUALENV_INSTALL_ROOT}/matrix-synapse"
 TARGET_PYTHON="${VIRTUALENV_DIR}/bin/python"
 
@@ -78,9 +78,14 @@ case "$DEB_BUILD_OPTIONS" in
 
  cp -r tests "$tmpdir"
 
+ # To avoid pulling in the unbuilt Synapse in the local directory
+ pushd /
+
  PYTHONPATH="$tmpdir" \
  "${TARGET_PYTHON}" -m twisted.trial --reporter=text -j2 tests
 
+ popd
+
  ;;
 esac
 

debian/changelog

@@ -12,11 +12,15 @@ matrix-synapse-py3 (1.66.0) stable; urgency=medium
 
 matrix-synapse-py3 (1.66.0~rc2+nmu1) UNRELEASED; urgency=medium
 
+ [ Jörg Behrmann ]
  * Update debhelper to compatibility level 12.
  * Drop the preinst script stopping synapse.
  * Allocate a group for the system user.
  * Change dpkg-statoverride to --force-statoverride-add.
 
+ [ Erik Johnston ]
+ * Disable `dh_auto_configure` as it broke during Rust build.
+
  -- Jörg Behrmann <[email protected]> Tue, 23 Aug 2022 17:17:00 +0100
 
 matrix-synapse-py3 (1.66.0~rc2) stable; urgency=medium

debian/rules

@@ -12,6 +12,8 @@ override_dh_installsystemd:
 # we don't really want to strip the symbols from our object files.
 override_dh_strip:
 
+override_dh_auto_configure:
+
 # many libraries pulled from PyPI have allocatable sections after
 # non-allocatable ones on which dwz errors out. For those without the issue the
 # gains are only marginal
@@ -49,6 +51,7 @@ override_dh_shlibdeps:
  -X site-packages/PIL/.libs \
  -X site-packages/Pillow.libs \
  -X site-packages/psycopg2
+ # TODO: Do we need to add our rust package here?
 
 override_dh_virtualenv:
  ./debian/build_virtualenv

docker/Dockerfile

@@ -92,11 +92,20 @@ RUN \
  libxml++2.6-dev \
  libxslt1-dev \
  openssl \
- rustc \
  zlib1g-dev \
  git \
+ curl \
  && rm -rf /var/lib/apt/lists/*
 
+
+# Install rust and ensure its in the PATH
+ENV RUSTUP_HOME=/rust
+ENV CARGO_HOME=/cargo
+ENV PATH=/cargo/bin:/rust/bin:$PATH
+RUN mkdir /rust /cargo
+
+RUN curl -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --default-toolchain stable
+
 # To speed up rebuilds, install all of the dependencies before we copy over
 # the whole synapse project, so that this layer in the Docker cache can be
 # used while you develop on the source
@@ -108,8 +117,9 @@ RUN --mount=type=cache,target=/root/.cache/pip \
 
 # Copy over the rest of the synapse source code.
 COPY synapse /synapse/synapse/
+COPY rust /synapse/rust/
 # ... and what we need to `pip install`.
-COPY pyproject.toml README.rst /synapse/
+COPY pyproject.toml README.rst build_rust.py /synapse/
 
 # Repeat of earlier build argument declaration, as this is a new build stage.
 ARG TEST_ONLY_IGNORE_POETRY_LOCKFILE

docker/Dockerfile-dhvirtualenv

@@ -72,6 +72,7 @@ RUN apt-get update -qq -o Acquire::Languages=none \
  && env DEBIAN_FRONTEND=noninteractive apt-get install \
  -yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io \
  build-essential \
+ curl \
  debhelper \
  devscripts \
  libsystemd-dev \
@@ -85,6 +86,15 @@ RUN apt-get update -qq -o Acquire::Languages=none \
  libpq-dev \
  xmlsec1
 
+# Install rust and ensure its in the PATH
+ENV RUSTUP_HOME=/rust
+ENV CARGO_HOME=/cargo
+ENV PATH=/cargo/bin:/rust/bin:$PATH
+RUN mkdir /rust /cargo
+
+RUN curl -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --default-toolchain stable
+
+
 COPY --from=builder /dh-virtualenv_1.2.2-1_all.deb /
 
 # install dhvirtualenv. Update the apt cache again first, in case we got a

docker/complement/conf/start_for_complement.sh

@@ -104,6 +104,8 @@ openssl x509 -in /conf/server.tls.crt -noout -text | grep DNS:
 export SYNAPSE_TLS_CERT=/conf/server.tls.crt
 export SYNAPSE_TLS_KEY=/conf/server.tls.key
 
+pip install -U setuptools
+
 # Run the script that writes the necessary config files and starts supervisord, which in turn
 # starts everything else
 exec /configure_workers_and_start.py

docs/deprecation_policy.md

@@ -18,6 +18,12 @@ documented at [https://endoflife.date/python](https://endoflife.date/python) and
 [https://endoflife.date/postgresql](https://endoflife.date/postgresql).
 
 
+A Rust compiler is required to build Synapse from source. For any given release
+the minimum required version may be bumped up to the latest released Rust
+version, and so people building from source must be able to fetch the latest
+version of Rust (e.g. by using [rustup](https://rustup.rs/)).
+
+
 Context
 -------
 
@@ -31,3 +37,9 @@ long process.
 By following the upstream support life cycles Synapse can ensure that its
 dependencies continue to get security patches, while not requiring system admins
 to constantly update their platform dependencies to the latest versions.
+
+For Rust, the situation is a bit different given that a) the Rust foundation
+does not generally support older Rust versions, and b) the library ecosystem
+generally bump their minimum support Rust versions frequently. In general,
+Synapse will unlikely ever require the absolute latest Rust version, but
+introducing a formal policy is hard given the constraints of the ecosystem.