Merge #817

817: Enable usage of other toolchains in images r=Alexhuszagh a=Emilgardis Makes the following work ```toml # Cross.toml [build] default-target = "x86_64-unknown-linux-musl" [target."x86_64-unknown-linux-musl"] image.name = "alpine:edge" image.toolchain = ["x86_64-unknown-linux-musl"] pre-build = ["apk add --no-cache gcc musl-dev"] ``` ```rust // src/main.rs fn main() { println!("Hello, world!"); } ``` ``` > cross run [..snip] Finished dev [unoptimized + debuginfo] target(s) in 0.24s Running `/target/x86_64-unknown-linux-musl/debug/cross-minimal` Hello, world! ``` and also ```toml [target.x86_64-unknown-linux-gnu] pre-build = [ "apt-get update && apt-get install -y libc6 g++-x86-64-linux-gnu libc6-dev-amd64-cross", ] [target.x86_64-unknown-linux-gnu.env] passthrough = [ "CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=x86_64-linux-gnu-gcc", "CC_x86_64_unknown_linux_gnu=x86_64-linux-gnu-gcc", "CXX_x86_64_unknown_linux_gnu=x86_64-linux-gnu-g++", ] [target.x86_64-unknown-linux-gnu.image] name = "ubuntu:20.04" toolchain = ["aarch64-unknown-linux-gnu"] ``` ``` cross build --target x86_64-unknown-linux-gnu -v + cargo metadata --format-version 1 --filter-platform x86_64-unknown-linux-gnu + rustc --print sysroot + /usr/local/bin/docker + /usr/local/bin/docker version '-f "{{ .Server.Os }},{{ .Server.Arch }}"' + rustup --verbose toolchain list + rustup --verbose target list --toolchain stable-aarch64-unknown-linux-gnu + rustup --verbose component list --toolchain stable-aarch64-unknown-linux-gnu + "/Users/emil/workspace/cross-tests/basic" /usr/local/bin/docker build --platform linux/arm64 --label 'org.cross-rs.for-cross-target=x86_64-unknown-linux-gnu' --label 'org.cross-rs.runs-with=aarch64-unknown-linux-gnu' --label 'org.cross-rs.workspace_root=/Users/emil/workspace/cross-tests/basic' --tag cross-custom-basic:x86_64-unknown-linux-gnu-50dfa-pre-build --build-arg 'CROSS_CMD=apt-get update && apt-get install -y libc6 g++-x86-64-linux-gnu libc6-dev-amd64-cross' --build-arg 'CROSS_DEB_ARCH=amd64' --file /Users/emil/workspace/cross-tests/basic/target/x86_64-unknown-linux-gnu/Dockerfile.x86_64-unknown-linux-gnu-custom . [+] Building 10.0s (6/6) FINISHED => [internal] load build definition from Dockerfile.x86_64-unknown-linux-gnu-custom 0.0s => => transferring dockerfile: 166B 0.0s => [internal] load .dockerignore 0.0s => => transferring context: 2B 0.0s => [internal] load metadata for docker.io/library/ubuntu:20.04 0.6s => CACHED [1/2] FROM docker.io/library/ubuntu:20.04@sha256:fd92c36d3cb9b1d027c4d2a72c6bf0125da82425fc2ca 0.0s => [2/2] RUN eval "apt-get update && apt-get install -y libc6 g++-x86-64-linux-gnu libc6-dev-amd64-cross 8.8s => exporting to image 0.5s => => exporting layers 0.5s => => writing image sha256:92f3cf51cc16c7b176c9bc09ce28ef83c029881e9c5017589eb8372bf6fff8d3 0.0s => => naming to docker.io/library/cross-custom-basic:x86_64-unknown-linux-gnu-50dfa-pre-build 0.0s + /usr/local/bin/docker run --userns host --platform linux/arm64 -e 'CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=x86_64-linux-gnu-gcc' -e 'CC_x86_64_unknown_linux_gnu=x86_64-linux-gnu-gcc' -e 'CXX_x86_64_unknown_linux_gnu=x86_64-linux-gnu-g++' -e 'PKG_CONFIG_ALLOW_CROSS=1' -e 'XARGO_HOME=/xargo' -e 'CARGO_HOME=/cargo' -e 'CARGO_TARGET_DIR=/target' -e 'CROSS_RUNNER=' -e TERM -e 'USER=emil' --rm --user 501:20 -v /Users/emil/.xargo:/xargo:Z -v /Users/emil/.cargo:/cargo:Z -v /cargo/bin -v /Users/emil/workspace/cross-tests/basic:/project:Z -v /Users/emil/.rustup/toolchains/stable-aarch64-unknown-linux-gnu:/rust:Z,ro -v /Users/emil/workspace/cross-tests/basic/target:/target:Z -w /project -i -t cross-custom-basic:x86_64-unknown-linux-gnu-50dfa-pre-build sh -c 'PATH=$PATH:/rust/bin cargo build --target x86_64-unknown-linux-gnu -v' Fresh basic v0.1.0 (/project) Finished dev [unoptimized + debuginfo] target(s) in 0.47s basic on master [?] is 📦 v0.1.0 via 🦀 v1.61.0 took 12s ❯ file target/x86_64-unknown-linux-gnu/debug/basic target/x86_64-unknown-linux-gnu/debug/basic: ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /lib64/ld-linux-x86-64.so.2, BuildID[sha1]=5716ca6c6514e255e9a29eeeeac3d9cb4fb43c0a, for GNU/Linux 3.2.0, with debug_info, not stripped basic on master [?] is 📦 v0.1.0 via 🦀 v1.61.0 ❯ uname -a Darwin Emils-MacBook-Pro.local 21.5.0 Darwin Kernel Version 21.5.0: Tue Apr 26 21:08:37 PDT 2022; root:xnu-8020.121.3~4/RELEASE_ARM64_T6000 arm64 ``` also makes `cross +nightly-2022-01-01` possible. this is a requirement for #751 Co-authored-by: Emil Gardström <[email protected]>
cross-rs · Jul 15, 2022 · a4da896 · a4da896
2 parents 86e3f62 + 86a550d
commit a4da896
Show file tree

Hide file tree

Showing 36 changed files with 2,261 additions and 589 deletions.
diff --git a/.changes/817.json b/.changes/817.json
@@ -0,0 +1,11 @@
+[
+ {
+ "description": "Images can now specify a certain toolchain via `target.{target}.image.toolchain`",
+ "breaking": true,
+ "type": "changed"
+ },
+ {
+ "description": "made `cross +channel` parsing more compliant to parsing a toolchain",
+ "type": "fixed"
+ }
+]
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -333,15 +333,14 @@ jobs:
 
  # we should always have an artifact from a previous build.
  remote:
- needs: [shellcheck, test, check]
+ needs: [test, check]
  runs-on: ubuntu-latest
  if: github.actor == 'bors[bot]'
  steps:
  - uses: actions/checkout@v3
  - uses: ./.github/actions/setup-rust
 
  - name: LLVM instrument coverage
- id: remote-cov
  uses: ./.github/actions/cargo-llvm-cov
  with:
  name: integration-remote
@@ -353,15 +352,14 @@ jobs:
  shell: bash
 
  bisect:
- needs: [shellcheck, test, check]
+ needs: [test, check]
  runs-on: ubuntu-latest
  if: github.actor == 'bors[bot]'
  steps:
  - uses: actions/checkout@v3
  - uses: ./.github/actions/setup-rust
 
  - name: LLVM instrument coverage
- id: bisect-cov
  uses: ./.github/actions/cargo-llvm-cov
  with:
  name: integration-bisect
@@ -372,16 +370,40 @@ jobs:
  run: ./ci/test-bisect.sh
  shell: bash
 
+ foreign:
+ needs: [test, check]
+ runs-on: ubuntu-latest
+ if: github.actor == 'bors[bot]'
+ steps:
+ - uses: actions/checkout@v3
+ - uses: ./.github/actions/setup-rust
+
+ - name: LLVM instrument coverage
+ uses: ./.github/actions/cargo-llvm-cov
+ with:
+ name: integration-bisect
+ - name: Set up QEMU
+ uses: docker/setup-qemu-action@v2
+ with:
+ platforms: arm64
+ - name: Set up docker buildx
+ uses: docker/setup-buildx-action@v2
+ id: buildx
+ with:
+ install: true
+ - name: Run Foreign toolchain test
+ run: ./ci/test-foreign-toolchain.sh
+ shell: bash
+
  docker-in-docker:
- needs: [shellcheck, test, check]
+ needs: [test, check]
  runs-on: ubuntu-latest
  if: github.actor == 'bors[bot]'
  steps:
  - uses: actions/checkout@v3
  - uses: ./.github/actions/setup-rust
 
  - name: LLVM instrument coverage
- id: docker-in-docker-cov
  uses: ./.github/actions/cargo-llvm-cov
  with:
  name: integration-docker-in-docker
@@ -405,7 +427,7 @@ jobs:
  github-token: ${{ secrets.GITHUB_TOKEN }}
 
  conclusion:
- needs: [shellcheck, fmt, clippy, test, generate-matrix, build, publish, check, remote, bisect, docker-in-docker]
+ needs: [shellcheck, fmt, clippy, test, generate-matrix, build, publish, check, remote, bisect, docker-in-docker, foreign]
  if: always()
  runs-on: ubuntu-latest
  steps:

diff --git a/Cargo.toml b/Cargo.toml
@@ -10,8 +10,6 @@ version = "0.2.4"
 edition = "2021"
 include = [
  "src/**/*",
- "docker/Dockerfile.*",
- "docker/*.sh",
  "docs/*.md",
  "Cargo.toml",
  "Cargo.lock",

diff --git a/README.md b/README.md
@@ -46,11 +46,11 @@ One of these container engines is required. If both are installed, `cross` will
 default to `docker`.
 
 - [Docker]. Note that on Linux non-sudo users need to be in the `docker` group.
- Read the official [post-installation steps][post]. Requires version 1.24 or later.
+ Read the official [post-installation steps][post]. Requires version 20.10 (API 1.40) or later.
 
 [post]: https://docs.docker.com/install/linux/linux-postinstall/
 
-- [Podman]. Requires version 1.6.3 or later.
+- [Podman]. Requires version 3.4.0 or later.
 
 ## Installation
 

diff --git a/ci/shared.sh b/ci/shared.sh
@@ -1,5 +1,21 @@
 #!/usr/bin/env bash
 
+ci_dir=$(dirname "${BASH_SOURCE[0]}")
+ci_dir=$(realpath "${ci_dir}")
+PROJECT_HOME=$(dirname "${ci_dir}")
+export PROJECT_HOME
+CARGO_TMP_DIR="${PROJECT_HOME}/target/tmp"
+export CARGO_TMP_DIR
+
+if [[ -n "${CROSS_CONTAINER_ENGINE}" ]]; then
+ CROSS_ENGINE="${CROSS_CONTAINER_ENGINE}"
+elif command -v docker >/dev/null 2>&1; then
+ CROSS_ENGINE=docker
+else
+ CROSS_ENGINE=podman
+fi
+export CROSS_ENGINE
+
 function retry {
  local tries="${TRIES-5}"
  local timeout="${TIMEOUT-1}"
@@ -21,3 +37,14 @@ function retry {
 
  return ${exit_code}
 }
+
+function mkcargotemp {
+ local td=
+ td="$CARGO_TMP_DIR"/$(mktemp -u "${@}" | xargs basename)
+ mkdir -p "$td"
+ echo '# Cargo.toml
+ [workspace]
+ members = ["'"$(basename "$td")"'"]
+ ' > "$CARGO_TMP_DIR"/Cargo.toml
+ echo "$td"
+}
diff --git a/ci/test-bisect.sh b/ci/test-bisect.sh
@@ -13,7 +13,7 @@ fi
 ci_dir=$(dirname "${BASH_SOURCE[0]}")
 ci_dir=$(realpath "${ci_dir}")
 . "${ci_dir}"/shared.sh
-project_home=$(dirname "${ci_dir}")
+
 
 main() {
  local td=
@@ -22,7 +22,7 @@ main() {
  retry cargo fetch
  cargo build
  cargo install cargo-bisect-rustc --debug
- export CROSS="${project_home}/target/debug/cross"
+ export CROSS="${PROJECT_HOME}/target/debug/cross"
 
  td="$(mktemp -d)"
  git clone --depth 1 https:/cross-rs/rust-cpp-hello-word "${td}"

diff --git a/ci/test-cross-image.sh b/ci/test-cross-image.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# shellcheck disable=SC2086
+# shellcheck disable=SC2086,SC1091,SC1090
 
 set -x
 set -eo pipefail
@@ -20,14 +20,17 @@ if [[ -z "${CROSS_TARGET_CROSS_IMAGE}" ]]; then
  CROSS_TARGET_CROSS_IMAGE="ghcr.io/cross-rs/cross:main"
 fi
 
+ci_dir=$(dirname "${BASH_SOURCE[0]}")
+ci_dir=$(realpath "${ci_dir}")
+. "${ci_dir}"/shared.sh
 
 main() {
 
  docker run --rm -e TARGET -e "CROSS_TARGET_${TARGET_UPPER//-/_}_IMAGE" \
  -v /var/run/docker.sock:/var/run/docker.sock \
  "${CROSS_TARGET_CROSS_IMAGE}" sh -c '
 #!/usr/bin/env sh
-td="$(mktemp -d)"
+td="$(mkcargotemp -d)"
 git clone --depth 1 https:/cross-rs/rust-cpp-hello-word "${td}"
 cd "${td}"
 cross run --target "${TARGET}"

diff --git a/ci/test-docker-in-docker.sh b/ci/test-docker-in-docker.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# shellcheck disable=SC1004
+# shellcheck disable=SC1004,SC1091,SC1090
 
 # test to see that running docker-in-docker works
 
@@ -18,17 +18,17 @@ if [[ "${IMAGE}" ]]; then
  export "CROSS_TARGET_${TARGET_UPPER//-/_}_IMAGE"="${IMAGE}"
 fi
 
-source=$(dirname "${BASH_SOURCE[0]}")
-source=$(realpath "${source}")
-home=$(dirname "${source}")
+ci_dir=$(dirname "${BASH_SOURCE[0]}")
+ci_dir=$(realpath "${ci_dir}")
+. "${ci_dir}"/shared.sh
 
 main() {
- docker run -v "${home}":"${home}" -w "${home}" \
+ docker run -v "${PROJECT_HOME}":"${PROJECT_HOME}" -w "${PROJECT_HOME}" \
  --rm -e TARGET -e RUSTFLAGS -e RUST_TEST_THREADS \
  -e LLVM_PROFILE_FILE -e CARGO_INCREMENTAL \
  -e "CROSS_TARGET_${TARGET_UPPER//-/_}_IMAGE" \
  -v /var/run/docker.sock:/var/run/docker.sock \
- docker:18.09-dind sh -c '
+ docker:20.10-dind sh -c '
 #!/usr/bin/env sh
 set -x
 set -euo pipefail

diff --git a/ci/test-foreign-toolchain.sh b/ci/test-foreign-toolchain.sh
@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1091,SC1090
+
+# test to see that foreign toolchains work
+
+set -x
+set -eo pipefail
+
+ci_dir=$(dirname "${BASH_SOURCE[0]}")
+ci_dir=$(realpath "${ci_dir}")
+. "${ci_dir}"/shared.sh
+
+main() {
+ local td=
+
+ retry cargo fetch
+ cargo build
+ export CROSS="${PROJECT_HOME}/target/debug/cross"
+
+ td="$(mkcargotemp -d)"
+
+ pushd "${td}"
+ cargo init --bin --name foreign_toolchain
+ # shellcheck disable=SC2016
+ echo '# Cross.toml
+[build]
+default-target = "x86_64-unknown-linux-musl"
+
+[target."x86_64-unknown-linux-musl"]
+image.name = "alpine:edge"
+image.toolchain = ["x86_64-unknown-linux-musl"]
+pre-build = ["apk add --no-cache gcc musl-dev"]' >"${CARGO_TMP_DIR}"/Cross.toml
+
+ "$CROSS" run -v
+
+ local tmp_basename
+ tmp_basename=$(basename "${CARGO_TMP_DIR}")
+ "${CROSS_ENGINE}" images --format '{{.Repository}}:{{.Tag}}' --filter 'label=org.cross-rs.for-cross-target' | grep "cross-custom-${tmp_basename}" | xargs -t "${CROSS_ENGINE}" rmi
+
+ echo '# Cross.toml
+[build]
+default-target = "x86_64-unknown-linux-gnu"
+
+[target.x86_64-unknown-linux-gnu]
+pre-build = [
+ "apt-get update && apt-get install -y libc6 g++-x86-64-linux-gnu libc6-dev-amd64-cross",
+]
+
+[target.x86_64-unknown-linux-gnu.env]
+passthrough = [
+ "CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=x86_64-linux-gnu-gcc",
+ "CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER=/qemu-runner x86_64",
+ "CC_x86_64_unknown_linux_gnu=x86_64-linux-gnu-gcc",
+ "CXX_x86_64_unknown_linux_gnu=x86_64-linux-gnu-g++",
+]
+
+[target.x86_64-unknown-linux-gnu.image]
+name = "ubuntu:20.04"
+toolchain = ["aarch64-unknown-linux-gnu"]
+ ' >"${CARGO_TMP_DIR}"/Cross.toml
+
+ "$CROSS" build -v
+
+ "${CROSS_ENGINE}" images --format '{{.Repository}}:{{.Tag}}' --filter 'label=org.cross-rs.for-cross-target' | grep "cross-custom-${tmp_basename}" | xargs "${CROSS_ENGINE}" rmi
+
+ popd
+
+ rm -rf "${td}"
+}
+
+main
diff --git a/ci/test-remote.sh b/ci/test-remote.sh
@@ -14,15 +14,14 @@ fi
 ci_dir=$(dirname "${BASH_SOURCE[0]}")
 ci_dir=$(realpath "${ci_dir}")
 . "${ci_dir}"/shared.sh
-project_home=$(dirname "${ci_dir}")
 
 main() {
  local err=
 
  retry cargo fetch
  cargo build
- export CROSS="${project_home}/target/debug/cross"
- export CROSS_UTIL="${project_home}/target/debug/cross-util"
+ export CROSS="${PROJECT_HOME}/target/debug/cross"
+ export CROSS_UTIL="${PROJECT_HOME}/target/debug/cross-util"
 
  # if the create volume fails, ensure it exists.
  if ! err=$("${CROSS_UTIL}" volumes create 2>&1 >/dev/null); then
@@ -40,7 +39,7 @@ main() {
 
 cross_test_cpp() {
  local td=
- td="$(mktemp -d)"
+ td="$(mkcargotemp -d)"
 
  git clone --depth 1 https:/cross-rs/rust-cpp-hello-word "${td}"