diff --git a/scripts/offlinepi/README.md b/scripts/offlinepi/README.md new file mode 100644 index 0000000000000..505ff6376952a --- /dev/null +++ b/scripts/offlinepi/README.md @@ -0,0 +1,50 @@ +# offlinepi + +Utilities for managing an offline version of PyPI. + +## Usage + +Record PyPI responses during a command: + +``` +./offlinepi record +``` + +Replay PyPI responses during a command: + +``` +./offlinepi replay +``` + +### Example + +Record server interactions during Puffin's tests: + +``` +./offlinepi record cargo test --features pypi -- --test-threads=1 +``` + +**Note**: Recording tests without parallelism is helpful for reliable replays. + +Then, run it again using replayed responses: + +``` +./offlinepi replay cargo test --features pypi +``` + +## TLS Certificates + +In order to record HTTPS requests, the certificate generated by mitmproxy must be installed. +See [the mitmproxy certificate documentation](https://docs.mitmproxy.org/stable/concepts-certificates/) for details. + +## Implementation + +[mitmproxy](https://mitmproxy.org/) is used to record and replay responses. + +The proxy is temporarily created for the execution of the provided command. + +The command _must_ respect the `HTTP_PROXY` and `HTTPS_PROXY` environment variables. + +Response recording is limited to `pypi.org` and `files.pythonhosted.org`. + +Responses are written to `responses.dat` in the `offlinepi` project root. diff --git a/scripts/offlinepi/offlinepi b/scripts/offlinepi/offlinepi new file mode 100755 index 0000000000000..f8c665b78ef3c --- /dev/null +++ b/scripts/offlinepi/offlinepi @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# +# Run a command, recording or replaying interaction with the PyPI server. +# +# Usage: +# +# offlinepi +# + +projectroot=$(realpath "$(dirname "$0")") +responsefile=$projectroot/responses.dat + +mode=$1 +shift + +if [ -z "$mode" ]; then + echo 'A mode must be provided e.g. `offlinepi record ...`' + exit 1 +fi + +if [[ "${mode}" != @(record|replay) ]]; then + echo "Invalid mode \"$mode\"; expected either \"record\" or \"replay\"." + exit 1 +fi + +if $projectroot/offlinepi-healthcheck; then + echo "Proxy is already running at localhost:8080" + echo "Aborted!" + exit 1 +fi + +echo "Starting proxy server to $mode responses..." +$projectroot/offlinepi-$mode $responsefile& +PROXY_PID=$! + +if ! $projectroot/offlinepi-wait $PROXY_PID; then + echo "Server failed to start!" + echo "Aborted!" + $projectroot/offlinepi-stop $PROXY_PID + exit 1 +fi + +export HTTP_PROXY=http://localhost:8080 +export HTTPS_PROXY=https://localhost:8080 + +echo "Running provided command..." +"$@" + +echo "Stopping proxy server..." +$projectroot/offlinepi-stop $PROXY_PID \ No newline at end of file diff --git a/scripts/offlinepi/offlinepi-healthcheck b/scripts/offlinepi/offlinepi-healthcheck new file mode 100755 index 0000000000000..57c45346ef78b --- /dev/null +++ b/scripts/offlinepi/offlinepi-healthcheck @@ -0,0 +1,11 @@ +#!/usr/bin/env sh +# +# Check if the proxy is running. +# +# Usage: +# offlinepi-healthcheck + +exec curl --output /dev/null --silent --head --fail --proxy 127.0.0.1:8080 http://mitm.it + +# TODO(zanieb): We could consider looking at the response to determine if a _different_ proxy is being used. +# TODO(zanieb): This could take a configurable host and port diff --git a/scripts/offlinepi/offlinepi-record b/scripts/offlinepi/offlinepi-record new file mode 100755 index 0000000000000..0b1760122dc3d --- /dev/null +++ b/scripts/offlinepi/offlinepi-record @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# +# Start a proxy that records client server interactions to a file. +# +# Usage: +# +# offlinepi-record + +path=$1 +shift + +if [ -z "$path" ]; then + echo 'A recording path must be provided.' + exit 1 +fi + +if [ ! -z "$*" ]; then + echo "Unexpected extra arguments: $*" + exit 1 +fi + +# Remove the file before starting +rm $path 2> /dev/null + +# N.B. Additional options must be added _before_ the filter string +exec mitmdump \ + -w $path \ + --set stream_large_bodies=1000m \ + "~d pypi.org|files.pythonhosted.org|mitm.it" + +# stream_large_bodies: must be set to a large value or large responses will not be recorded +# resulting in an unexpected file endings during replays +# ~d: only interactions with package index domains should be recorded +# we also allow `mitm.it` so healthchecks succeed when replaying \ No newline at end of file diff --git a/scripts/offlinepi/offlinepi-replay b/scripts/offlinepi/offlinepi-replay new file mode 100755 index 0000000000000..36712d5d9214f --- /dev/null +++ b/scripts/offlinepi/offlinepi-replay @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# +# Start a proxy that replays server responses from a recording. +# Unknown responses will result in a 500. +# Each response can only be replayed once or it will be treated as unknown. +# +# Usage: +# +# offlinepi-start-replay + +path=$1 +shift + +if [ -z "$path" ]; then + echo 'A recording path must be provided.' + exit 1 +fi + +if [ ! -z "$*" ]; then + echo "Unexpected extra arguments: $*" + exit 1 +fi + +exec mitmdump --server-replay $path \ + --server-replay-extra 500 \ + --set connection_strategy=lazy + +# server-replay-extra: configures behavior when a response is unknown. +# connection_stategy: lazy is required to replay offline diff --git a/scripts/offlinepi/offlinepi-stop b/scripts/offlinepi/offlinepi-stop new file mode 100755 index 0000000000000..00944e618ce61 --- /dev/null +++ b/scripts/offlinepi/offlinepi-stop @@ -0,0 +1,8 @@ +#!/usr/bin/env sh +# Stops the proxy at the given PID. + +pid=$1 + +kill $pid 2> /dev/null +wait $pid 2> /dev/null +echo "Done!" diff --git a/scripts/offlinepi/offlinepi-wait b/scripts/offlinepi/offlinepi-wait new file mode 100755 index 0000000000000..f54bab3e05379 --- /dev/null +++ b/scripts/offlinepi/offlinepi-wait @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# +# Waits for the proxy to be ready. +# +# Usage: +# +# offlinepi-wait-ready + +projectroot=$(realpath "$(dirname "$0")") + +pid=$1 +shift + +if [ -z "$pid" ]; then + echo 'A PID must be provided.' + exit 1 +fi + +if [ ! -z "$*" ]; then + echo "Unexpected extra arguments: $*" + exit 1 +fi + + +# Wait until the server is ready +until $($projectroot/offlinepi-healthcheck); do + if ! kill -0 $pid 2> /dev/null; then + exit 1 + fi + sleep 1 +done