Linux arm build (#1465)

* Temporary cancel build step while I figure out some stuff.

* Empty container?

* Confirm JS actions are disallowed.

* They are disallowed.

* Run single step in container.

* Was it really running IN the container?

* Run script.

* need to get the path right.

* Basic cargo builds with a target and setup.

* Setup cargo target.

* Check if we're on Linux correctly.

* Fix toolchain

* Is jemalloc the culprit?

* Do I need build-essential?

* Forget target for now, try build.

* cache key calculation

* Syntax fix.

* Correct cache key conditional

* Use build.sh for everything.

* Fix conditional

* Make cache-key computable

* set -x and address some shellcheck issues.

* Update vendor_download.sh to download arm assets.

* Guard script arguments. Remove some old commentary.

* My guards worked too well...

* Run git config update just before build.

* Change release verification to include arm

* Only unit-tests.

* Spec fix for building with arm.

* aarch target.

* hail mary rust build

* One more target.

* Does installing musl-dev work?

* use sudo?

* Try moving rust build into script.

* More scripts.

* syntax error.

* Try with cc arg

* revert me

* Corrections.

* env vars

* Revert "Try with cc arg"

This reverts commit 7c92287.

* Try cross.

* Revert "Try cross."

This reverts commit fb866a8.

* try using build-arm.sh

* Non-interactive rustup.

* Enable the environment.

* Switch to lib-snippets. Configure link flags for Cargo.

* path fix I hope

* Propagate project files correctly.

* conditionalize extra-source-files.

* Print some env info.

* Always save the cabal cache.

* Commentary.

* Fix cargo test. more debug output

* temporarily disable cabal install check.

* Help embeddedbinaries be findable when cabal is not running in root dir.

* Temporarily disable diagnose.

* Bring back cabal install check.

* Revert "Bring back cabal install check."

This reverts commit c56b7ce.

* Cache key perms.

* Permissions.

* Try cleaning up.

* Print home where cabal stuff goes.

* echo paths

* store-dir

* Revert "store-dir"

This reverts commit 2379f28.

* Logic for LinuxARM cabal store.

* MOre cabal store junk

* Dummy cache test.

* Try adding an additional rust-cache directory.

* Re-enable other matrix builds.

* Restore integration tests.

* Update create release.

* Try putting the cabal install check back.

* Remove extra ls

* Revert "Try putting the cabal install check back."

This reverts commit 1e34a3b.

* Bring back non linux arm cargo tests and diagnose checks.

* Install nextest.

* Don't need extra tooling on linux arm

* Fix condition.

* Cabal install check condition.

* Cache OS.

* Sha256 sum everything.

* A little cleanup.

* Fix quoting.

* Insert arm generated bundles.

* Add support for linux/arm64 to install-latest.sh

* Only shasum fossa binaries.

* Fix arch_version check so it doesn't trigger on macos.

* Attempt to re-enable jemalloc on linux x86_64.

* Remove dead code.

* Revert conditional extra-source-files.

* Final cleanup.

* Update changelog.

* jemalloc linux arm?

* Better integrate jemalloc on arm.

* Fix duplicated tests.

* Use /runner/. $HOME is not set correctly.

* Fix script oops.

.cargo/config.toml

@@ -0,0 +1,2 @@
+[target.aarch64-unknown-linux-musl]
+rustflags = ["-C", "link-arg=-lgcc"]

.github/workflows/scripts/build-arm.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env sh
+
+set -e
+set -x
+
+if [ $# -lt 2 ] ; then
+ echo "Usage: ./build-arm.sh <RUNNER OS> <PROJECT FILE> [FEATURES]"
+ exit 1
+fi
+
+RUNNER_OS=$1
+PROJECT_FILE=$2
+FEATURES=$3
+
+# Install rust tooling
+curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal
+. "/home/runner/.cargo/env"
+
+rustc -V
+cargo -V
+
+# Build Rust binaries
+cargo build ${FEATURES:+--features $FEATURES} --release
+
+cargo test --release
+
+# Validate that diagnose runs
+cargo run --release --bin diagnose -- walk --trace-spans none --trace-level info
+
+# Now do Haskell things
+./.github/workflows/scripts/build.sh "$RUNNER_OS" "$PROJECT_FILE"

.github/workflows/scripts/build.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env sh
+
+set -e
+set -x
+
+if [ $# -lt 2 ] ; then
+ echo "Usage: ./build.sh <RUNNER OS> <PROJECT FILE>"
+ exit 1
+fi
+
+RUNNER_OS=$1
+PROJECT_FILE=$2
+
+make build-test-data
+
+# Does this need to run in the container?
+# It normally runs outside.
+git config --global --add safe.directory "$GITHUB_WORKSPACE"
+
+# With dist-newstyle caches:
+# Cabal mainly knows to recompile based on changes to files.
+# Tagging in git doesn't reliably change a file in a fixed location that cabal/GHC can track to indicate that there's a new tag.
+# For our release process, we merge to master, which builds (and may store a dist-newstyle cache), then push a release tag.
+# During the tag build, cabal/GHC may not realize that they have to rebuild the Version.hs file because the tag is invisible to it.
+# This line adds a comment to our version source file to prompt cabal/GHC to rebuild Version.hs.
+echo "{- $GITHUB_RUN_ID -}" >> src/App/Version.hs
+cabal update
+cabal build --project-file="$PROJECT_FILE" all
+cabal test --project-file="$PROJECT_FILE" unit-tests
+
+# Todo: Bring this back.
+# Test cabal-install.
+# This check ensures that QuickImport can use spectrometer as a library.
+# if [ "$RUNNER_OS" = 'Linux' ] ; then
+# cabal install --overwrite-policy=always --project-file="$PROJECT_FILE" --ghc-options="-Wwarn"
+# fi

.github/workflows/scripts/compute_cache_key.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env sh
+
+set -e
+set -x
+
+if [ $# -lt 2 ] ; then
+ echo "Usage: ./compute_cache_key <RUNNER OS> <PROJECT FILE>"
+ exit 1
+fi
+
+RUNNER_OS=$1
+PROJECT_FILE=$2
+
+cabal --project-file="$PROJECT_FILE" update
+cabal --project-file="$PROJECT_FILE" build --dry-run
+jq '."install-plan"[]."id"' < dist-newstyle/cache/plan.json | sort > /tmp/cabal-cache-key
+echo "Install plan:"
+cat /tmp/cabal-cache-key
+
+if [ "$RUNNER_OS" = "macOS" ]; then
+ PLAN_SUM=$(shasum -a256 /tmp/cabal-cache-key)
+else
+ PLAN_SUM=$(sha256sum /tmp/cabal-cache-key)
+fi
+
+CABAL_CACHE_KEY="$(echo "$PLAN_SUM" | awk '{print $1}')"
+export CABAL_CACHE_KEY
+echo "Cabal cache key: $CABAL_CACHE_KEY"
+echo "cabal-cache-key=$CABAL_CACHE_KEY" >> "$GITHUB_OUTPUT"
+
+echo "Home dotfiles in container: $(ls -al ~/.*)"
+
+# Cleanup. Restoring this cache seems to fail if the directory already exists.
+rm -rf dist-newstyle

Cargo.toml

@@ -17,3 +17,7 @@ resolver = "2"
 [profile.release]
 lto = "fat"
 strip = true
+
+# There is additional configuration in .cargo/config.toml.
+# It is used for building on aarch64-unknown-linux-musl.
+# More context on the split here: https:/rust-lang/cargo/issues/12738

Changelog.md

@@ -1,5 +1,9 @@
 # FOSSA CLI Changelog
 
+## 3.9.32
+
+- Platform Support: Add a binary for ARM64 Linux environments. ([#1465](https:/fossas/fossa-cli/pull/1465))
+
 ## 3.9.31
 
 - Resolve an issue parsing toml configuration files. ([#1459](https:/fossas/fossa-cli/pull/1459))

extlib/millhone/Cargo.toml

@@ -12,7 +12,7 @@ tikv-jemallocator = { version = "0.5.4", optional = true }
 clap = { version = "4.3.21", features = ["derive", "env", "cargo"] }
 stable-eyre = "0.2.2"
 srclib = { version = "*", git = "https:/fossas/foundation-libs" }
-snippets = { version = "0.1.3", git = "https:/fossas/foundation-libs", features = ["lang-all"] }
+snippets = { version = "0.1.3", tag = "v0.1.3", git = "https:/fossas/lib-snippets", features = ["lang-all"] }
 traceconf = { git = "https:/fossas/foundation-libs", version = "1.1.0" }
 serde = { version = "1.0.183", features = ["derive"] }
 thiserror = "1.0.46"

install-latest.sh

@@ -66,6 +66,7 @@ is_supported_platform() {
  darwin/amd64) found=0 ;;
  darwin/arm64) found=0 ;;
  linux/amd64) found=0 ;;
+ linux/arm64) found=0 ;;
  esac
  return $found
 }
@@ -119,7 +120,16 @@ adjust_arch() {
  # adjust archive name based on ARCH
  true
 }
-
+arch_version_check() {
+ # TODO: Make this version correct before merging.
+ if [ "${OS}/${ARCH}" = "linux/arm64" ] && version_less_than "${VERSION}" '3.9.32'
+ then
+ echo "There is no linux/arm64 binary for version $VERSION."
+ echo "Please select a version that is at least version 3.9.32"
+ exit 1
+ fi
+ true
+}
 cat /dev/null <<EOF
 ------------------------------------------------------------------------
 https:/client9/shlib - portable posix shell functions
@@ -482,6 +492,7 @@ adjust_os
 
 adjust_arch
 
+arch_version_check
 
 NAME=$(get_binary_name)
 TARBALL=${NAME}.${FORMAT}

src/Data/FileEmbed/Extra.hs

@@ -1,16 +1,17 @@
 module Data.FileEmbed.Extra (
  embedFile',
  embedFileIfExists,
-) where
+)
+where
 
 import Control.Exception (try)
 import Control.Monad (when)
-import Data.FileEmbed (embedFile)
+import Data.FileEmbed (embedFile, makeRelativeToProject)
 import Data.Foldable (traverse_)
 import Data.Maybe (isJust)
 import Language.Haskell.TH (reportWarning, runIO)
 import Language.Haskell.TH.Syntax (Exp (LitE), Lit (..), Q)
-import Path (parseRelFile)
+import Path (parseAbsFile)
 import Path.IO (doesFileExist)
 import System.Directory (getCurrentDirectory, listDirectory)
 import System.Environment (lookupEnv)
@@ -19,7 +20,12 @@ import System.FilePath (isRelative, splitPath, takeDirectory)
 embedFileIfExists :: FilePath -> Q Exp
 embedFileIfExists inputPath = do
  skipEmbedEnvVar <- runIO $ lookupEnv "FOSSA_SKIP_EMBED_FILE_IN_HLS"
- case (skipEmbedEnvVar, parseRelFile inputPath) of
+ -- During some commands (like cabal install) the working directory may not
+ -- be the same as the root of this project.
+ -- makeRelativeToProject takes a relative path and roots it in the highest directory
+ -- a .cabal file is found and returns an absolute path.
+ inputPath' <- makeRelativeToProject inputPath
+ case (skipEmbedEnvVar, parseAbsFile inputPath') of
  (Just _, _) -> do
  pure (LitE $ StringL "")
  (_, Just path) -> do

test/App/Fossa/PathDependencySpec.hs

@@ -11,6 +11,7 @@ import Control.Algebra (Has)
 import Control.Effect.FossaApiClient (FossaApiClientF (..), PackageRevision (PackageRevision))
 import Data.Flag (toFlag)
 import Data.Set qualified as Set
+import Data.String.Conversion (toText)
 import Data.Text (Text)
 import DepTypes (
  DepEnvironment (EnvTesting),
@@ -41,10 +42,11 @@ absPathOfSpec = describe "absPathOfSpec" $ do
  mkPathSpec cwd "../" isAbsDir
  mkPathSpec cwd "./" isAbsDir
 
- mkPathSpec cwd "../fossa-cli" isAbsDir
- mkPathSpec cwd "../fossa-cli/" isAbsDir
- mkPathSpec cwd "../fossa-cli/test" isAbsDir
- mkPathSpec cwd "../fossa-cli/test/" isAbsDir
+ let currentPathRel = "../" <> (toText . toFilePath . dirname $ cwd)
+ mkPathSpec cwd currentPathRel isAbsDir
+ mkPathSpec cwd (currentPathRel <> "/") isAbsDir
+ mkPathSpec cwd (currentPathRel <> "/test") isAbsDir
+ mkPathSpec cwd (currentPathRel <> "/test/") isAbsDir
 
  mkPathSpec cwd "./test" isAbsDir
  mkPathSpec cwd "./test/" isAbsDir

vendor_download.sh

@@ -51,11 +51,20 @@ case "$(uname -s)" in
  ;;
 
  Linux)
- ASSET_POSTFIX="linux"
- THEMIS_ASSET_POSTFIX="linux-amd64"
- LERNIE_ASSET_POSTFIX="x86_64-linux"
- ;;
+ case "$(uname -m)" in
+ aarch64)
+ ASSET_POSTFIX="linux"
+ THEMIS_ASSET_POSTFIX="linux-arm64"
+ LERNIE_ASSET_POSTFIX="aarch64-linux"
+ ;;
 
+ *)
+ ASSET_POSTFIX="linux"
+ THEMIS_ASSET_POSTFIX="linux-amd64"
+ LERNIE_ASSET_POSTFIX="x86_64-linux"
+ ;;
+ esac
+ ;;
  *)
  echo "Warn: Assuming $(uname -s) is Windows"
  ASSET_POSTFIX="windows.exe"