Move complement setup stuff into the Synapse repo

.github/workflows/tests.yml

@@ -361,27 +361,11 @@ jobs:
  (wget -O - "https:/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
  done
 
- # Build initial Synapse image
- - run: docker build -t matrixdotorg/synapse:latest -f docker/Dockerfile .
- working-directory: synapse
- env:
- DOCKER_BUILDKIT: 1
-
- # Build a ready-to-run Synapse image based on the initial image above.
- # This new image includes a config file, keys for signing and TLS, and
- # other settings to make it suitable for testing under Complement.
- - run: docker build -t complement-synapse -f Synapse.Dockerfile .
- working-directory: complement/dockerfiles
-
- # Run Complement
  - run: |
  set -o pipefail
- go test -v -json -tags synapse_blacklist,msc2716,msc3030 ./tests/... 2>&1 | gotestfmt
+ COMPLEMENT_DIR=complement ./scripts-dev/complement.sh -json 2>&1 | gotestfmt
  shell: bash
  name: Run Complement Tests
- env:
- COMPLEMENT_BASE_IMAGE: complement-synapse:latest
- working-directory: complement
 
  # a job which marks all the other jobs as complete, thus allowing PRs to be merged.
  tests-done:

changelog.d/12404.misc

@@ -0,0 +1 @@
+Add files used to build the Docker image used for complement testing into the Synapse repository.

docker/complement/Dockerfile

@@ -0,0 +1,22 @@
+# A dockerfile which builds an image suitable for testing Synapse under
+# complement.
+
+ARG SYNAPSE_VERSION=latest
+
+FROM matrixdotorg/synapse:${SYNAPSE_VERSION}
+
+ENV SERVER_NAME=localhost
+
+COPY conf/* /conf/
+
+# generate a signing key
+RUN generate_signing_key -o /conf/server.signing.key
+
+WORKDIR /data
+
+EXPOSE 8008 8448
+
+ENTRYPOINT ["/conf/start.sh"]
+
+HEALTHCHECK --start-period=5s --interval=1s --timeout=1s \
+ CMD curl -fSs http://localhost:8008/health || exit 1

docker/complement/README.md

@@ -0,0 +1 @@
+Stuff for building the docker image used for testing under complement.

docker/complement/conf/homeserver.yaml

@@ -0,0 +1,122 @@
+## Server ##
+
+server_name: SERVER_NAME
+log_config: /conf/log_config.yaml
+report_stats: False
+signing_key_path: /conf/server.signing.key
+trusted_key_servers: []
+enable_registration: true
+enable_registration_without_verification: true
+
+## Listeners ##
+
+tls_certificate_path: /conf/server.tls.crt
+tls_private_key_path: /conf/server.tls.key
+bcrypt_rounds: 4
+registration_shared_secret: complement
+
+listeners:
+ - port: 8448
+ bind_addresses: ['::']
+ type: http
+ tls: true
+ resources:
+ - names: [federation]
+
+ - port: 8008
+ bind_addresses: ['::']
+ type: http
+
+ resources:
+ - names: [client]
+
+## Database ##
+
+database:
+ name: "sqlite3"
+ args:
+ # We avoid /data, as it is a volume and is not transferred when the container is committed,
+ # which is a fundamental necessity in complement.
+ database: "/conf/homeserver.db"
+
+## Federation ##
+
+
+# disable verification of federation certificates
+#
+# TODO: this is temporary; see
+# https:/matrix-org/synapse/issues/11803
+federation_verify_certificates: false
+
+# trust certs signed by the complement CA
+federation_custom_ca_list:
+- /complement/ca/ca.crt
+
+# unblacklist RFC1918 addresses
+ip_range_blacklist: []
+
+# Disable server rate-limiting
+rc_federation:
+ window_size: 1000
+ sleep_limit: 10
+ sleep_delay: 500
+ reject_limit: 99999
+ concurrent: 3
+
+rc_message:
+ per_second: 9999
+ burst_count: 9999
+
+rc_registration:
+ per_second: 9999
+ burst_count: 9999
+
+rc_login:
+ address:
+ per_second: 9999
+ burst_count: 9999
+ account:
+ per_second: 9999
+ burst_count: 9999
+ failed_attempts:
+ per_second: 9999
+ burst_count: 9999
+
+rc_admin_redaction:
+ per_second: 9999
+ burst_count: 9999
+
+rc_joins:
+ local:
+ per_second: 9999
+ burst_count: 9999
+ remote:
+ per_second: 9999
+ burst_count: 9999
+
+federation_rr_transactions_per_room_per_second: 9999
+
+## API Configuration ##
+
+# A list of application service config files to use
+#
+app_service_config_files:
+AS_REGISTRATION_FILES 
+
+## Experimental Features ##
+
+experimental_features:
+ # Enable spaces support
+ spaces_enabled: true
+ # Enable history backfilling support
+ msc2716_enabled: true
+ # server-side support for partial state in /send_join
+ msc3706_enabled: true
+ # Enable jump to date endpoint
+ msc3030_enabled: true
+
+server_notices:
+ system_mxid_localpart: _server
+ system_mxid_display_name: "Server Alert"
+ system_mxid_avatar_url: ""
+ room_name: "Server Alert"

docker/complement/conf/log_config.yaml

@@ -0,0 +1,24 @@
+version: 1
+
+formatters:
+ precise:
+ format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
+
+filters:
+ context:
+ (): synapse.logging.context.LoggingContextFilter
+ request: ""
+
+handlers:
+ console:
+ class: logging.StreamHandler
+ formatter: precise
+ filters: [context]
+ # log to stdout, for easier use with 'docker logs'
+ stream: 'ext://sys.stdout'
+
+root:
+ level: INFO
+ handlers: [console]
+
+disable_existing_loggers: false

docker/complement/conf/start.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+
+set -e
+
+sed -i "s/SERVER_NAME/${SERVER_NAME}/g" /conf/homeserver.yaml
+
+# Add the application service registration files to the homeserver.yaml config
+for filename in /complement/appservice/*.yaml; do
+ [ -f "$filename" ] || break
+
+ as_id=$(basename "$filename" .yaml)
+
+ # Insert the path to the registration file and the AS_REGISTRATION_FILES marker after 
+ # so we can add the next application service in the next iteration of this for loop
+ sed -i "s/AS_REGISTRATION_FILES/ - \/complement\/appservice\/${as_id}.yaml\nAS_REGISTRATION_FILES/g" /conf/homeserver.yaml
+done
+# Remove the AS_REGISTRATION_FILES entry
+sed -i "s/AS_REGISTRATION_FILES//g" /conf/homeserver.yaml
+
+# generate an ssl key and cert for the server, signed by the complement CA
+openssl genrsa -out /conf/server.tls.key 2048
+
+openssl req -new -key /conf/server.tls.key -out /conf/server.tls.csr \
+ -subj "/CN=${SERVER_NAME}"
+openssl x509 -req -in /conf/server.tls.csr \
+ -CA /complement/ca/ca.crt -CAkey /complement/ca/ca.key -set_serial 1 \
+ -out /conf/server.tls.crt
+
+exec python -m synapse.app.homeserver -c /conf/homeserver.yaml "$@"
+

scripts-dev/complement.sh

@@ -50,25 +50,18 @@ if [[ -n "$WORKERS" ]]; then
 
  export COMPLEMENT_BASE_IMAGE=complement-synapse-workers
  COMPLEMENT_DOCKERFILE=SynapseWorkers.Dockerfile
+
  # And provide some more configuration to complement.
- export COMPLEMENT_CA=true
  export COMPLEMENT_SPAWN_HS_TIMEOUT_SECS=25
 else
  export COMPLEMENT_BASE_IMAGE=complement-synapse
- COMPLEMENT_DOCKERFILE=Synapse.Dockerfile
+ COMPLEMENT_DOCKERFILE=Dockerfile
 fi
 
 # Build the Complement image from the Synapse image we just built.
-docker build -t $COMPLEMENT_BASE_IMAGE -f "$COMPLEMENT_DIR/dockerfiles/$COMPLEMENT_DOCKERFILE" "$COMPLEMENT_DIR/dockerfiles"
-
-cd "$COMPLEMENT_DIR"
-
-EXTRA_COMPLEMENT_ARGS=""
-if [[ -n "$1" ]]; then
- # A test name regex has been set, supply it to Complement
- EXTRA_COMPLEMENT_ARGS+="-run $1 "
-fi
+docker build -t $COMPLEMENT_BASE_IMAGE -f "docker/complement/$COMPLEMENT_DOCKERFILE" "docker/complement"
 
 # Run the tests!
 echo "Images built; running complement"
-go test -v -tags synapse_blacklist,msc2716,msc3030 -count=1 $EXTRA_COMPLEMENT_ARGS ./tests/...
+cd "$COMPLEMENT_DIR"
+go test -v -tags synapse_blacklist,msc2716,msc3030 -count=1 "$@" ./tests/...