From 6785e86ca3251f21db5646628e3c681f588356d0 Mon Sep 17 00:00:00 2001 From: Steffen Siering Date: Thu, 24 Jan 2019 01:47:31 +0100 Subject: [PATCH] Introduce mage and enable arm testing via docker+qemu (#32) We introduce support for testing other environments here. For now we only add ARM (32bit) support to our tests. The Travis configs are updated to cross-compile and run the testsuite on ARM, via qemu. I've been able to reproduce the panic on atomic access, which has been fixed in #31 with these tests + some false assumption in an unit test has been fixed as well. As the build and testing environment has become somewhat more complicated due to the need to cross compile and run/test via qemu, we introduce mage here. We hope for the full test suite and cross-combile/test support to be executable via Windows, Linux, and Darwin. We still target x86_64 dev environments, though. --- .gitignore | 3 + .travis.yml | 42 +++- dev-tools/lib/mage/gotool/go.go | 255 ++++++++++++++++++++++++ dev-tools/lib/mage/mgenv/mgenv.go | 102 ++++++++++ dev-tools/lib/mage/xbuild/docker.go | 74 +++++++ dev-tools/lib/mage/xbuild/xbuild.go | 71 +++++++ file.go | 21 +- file_test.go | 3 +- magefile.go | 288 ++++++++++++++++++++++++++++ 9 files changed, 844 insertions(+), 15 deletions(-) create mode 100644 dev-tools/lib/mage/gotool/go.go create mode 100644 dev-tools/lib/mage/mgenv/mgenv.go create mode 100644 dev-tools/lib/mage/xbuild/docker.go create mode 100644 dev-tools/lib/mage/xbuild/xbuild.go create mode 100644 magefile.go diff --git a/.gitignore b/.gitignore index a1338d6..8674653 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,6 @@ # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 .glide/ + +# build directory +/build/ diff --git a/.travis.yml b/.travis.yml index cd4d963..e082899 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,16 +22,43 @@ env: jobs: include: # try to cross compile to untested OSes - - env: TARGET=openbsd + - name: XBuild OpenBSD + env: TARGET=openbsd go: $GO_CROSS_VERSION script: eval $GO_CHECK_CROSS_SCRIPT - - env: TARGET=netbsd + - name: XBuild NetBSD + env: TARGET=netbsd go: $GO_CROSS_VERSION script: eval $GO_CHECK_CROSS_SCRIPT - - env: TARGET=freebsd + - name: XBuild FreeBSD + env: TARGET=freebsd go: $GO_CROSS_VERSION script: eval $GO_CHECK_CROSS_SCRIPT + - name: 32Bit ARM go1.10 + env: [BUILD_OS=linux, BUILD_ARCH=arm] + go: '1.10' + services: [docker] + before_install: + - docker run --rm --privileged multiarch/qemu-user-static:register --reset + - | + go get -u -d github.com/magefile/mage + (cd $GOPATH/src/github.com/magefile/mage; go run bootstrap.go) + script: + - mage -v test + + - name: 32Bit ARM go1.11 + env: [BUILD_OS=linux, BUILD_ARCH=arm] + go: '1.11' + services: [docker] + before_install: + - docker run --rm --privileged multiarch/qemu-user-static:register --reset + - | + go get -u -d github.com/magefile/mage + (cd $GOPATH/src/github.com/magefile/mage; go run bootstrap.go) + script: + - mage -v test + # Check we're testing the correct commit (Snippet from: https://github.com/travis-ci/travis-ci/issues/7459#issuecomment-287040521) before_install: - | @@ -39,6 +66,11 @@ before_install: echo "Commit $(git rev-parse HEAD) doesn't match expected commit $TRAVIS_COMMIT" exit 1 fi + - | + # install mage + go get -u -d github.com/magefile/mage + (cd $GOPATH/src/github.com/magefile/mage; go run bootstrap.go) -script: | - go test -cover -v ./... +script: +- go env +- mage -v test diff --git a/dev-tools/lib/mage/gotool/go.go b/dev-tools/lib/mage/gotool/go.go new file mode 100644 index 0000000..ed82eb7 --- /dev/null +++ b/dev-tools/lib/mage/gotool/go.go @@ -0,0 +1,255 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package gotool + +import ( + "os" + "strings" + + "github.com/magefile/mage/mg" + "github.com/magefile/mage/sh" +) + +// Args holds parameters, environment variables and flag information used to +// pass to the go tool. +type Args struct { + extra map[string]string // extra flags one can pass to the command + env map[string]string + flags map[string]string + pos []string +} + +// ArgOpt is a functional option adding info to Args once executed. +type ArgOpt func(args *Args) + +type goTest func(opts ...ArgOpt) error + +// Test runs `go test` and provides optionals for adding command line arguments. +var Test goTest = runGoTest + +// ListProjectPackages lists all packages in the current project +func ListProjectPackages() ([]string, error) { + return ListPackages("./...") +} + +// ListPackages calls `go list` for every package spec given. +func ListPackages(pkgs ...string) ([]string, error) { + return getLines(callGo(nil, "list", pkgs...)) +} + +// ListTestFiles lists all go and cgo test files available in a package. +func ListTestFiles(pkg string) ([]string, error) { + const tmpl = `{{ range .TestGoFiles }}{{ printf "%s\n" . }}{{ end }}` + + `{{ range .XTestGoFiles }}{{ printf "%s\n" . }}{{ end }}` + + return getLines(callGo(nil, "list", "-f", tmpl, pkg)) +} + +// HasTests returns true if the given package contains test files. +func HasTests(pkg string) (bool, error) { + files, err := ListTestFiles(pkg) + if err != nil { + return false, err + } + return len(files) > 0, nil +} + +func (goTest) WithCoverage(to string) ArgOpt { + return combine(flagArg("-cover", ""), flagArgIf("-test.coverprofile", to)) +} +func (goTest) Short(b bool) ArgOpt { return flagBoolIf("-test.short", b) } +func (goTest) Use(bin string) ArgOpt { return extraArgIf("use", bin) } +func (goTest) OS(os string) ArgOpt { return envArgIf("GOOS", os) } +func (goTest) ARCH(arch string) ArgOpt { return envArgIf("GOARCH", arch) } +func (goTest) Create() ArgOpt { return flagArg("-c", "") } +func (goTest) Out(path string) ArgOpt { return flagArg("-o", path) } +func (goTest) Package(path string) ArgOpt { return posArg(path) } +func (goTest) Verbose() ArgOpt { return flagArg("-test.v", "") } +func runGoTest(opts ...ArgOpt) error { + args := buildArgs(opts) + if bin := args.Val("use"); bin != "" { + flags := map[string]string{} + for k, v := range args.flags { + if strings.HasPrefix(k, "-test.") { + flags[k] = v + } + } + + useArgs := &Args{} + *useArgs = *args + useArgs.flags = flags + + _, err := sh.Exec(useArgs.env, os.Stdout, os.Stderr, bin, useArgs.build()...) + return err + } + + return runVGo("test", args) +} + +func getLines(out string, err error) ([]string, error) { + if err != nil { + return nil, err + } + + lines := strings.Split(out, "\n") + res := lines[:0] + for _, line := range lines { + line = strings.TrimSpace(line) + if len(line) > 0 { + res = append(res, line) + } + } + + return res, nil +} + +func callGo(env map[string]string, cmd string, opts ...string) (string, error) { + args := []string{cmd} + args = append(args, opts...) + return sh.OutputWith(env, mg.GoCmd(), args...) +} + +func runVGo(cmd string, args *Args) error { + return execGoWith(func(env map[string]string, cmd string, args ...string) error { + _, err := sh.Exec(env, os.Stdout, os.Stderr, cmd, args...) + return err + }, cmd, args) +} + +func runGo(cmd string, args *Args) error { + return execGoWith(sh.RunWith, cmd, args) +} + +func execGoWith( + fn func(map[string]string, string, ...string) error, + cmd string, args *Args, +) error { + cliArgs := []string{cmd} + cliArgs = append(cliArgs, args.build()...) + return fn(args.env, mg.GoCmd(), cliArgs...) +} + +func posArg(value string) ArgOpt { + return func(a *Args) { a.Add(value) } +} + +func extraArg(k, v string) ArgOpt { + return func(a *Args) { a.Extra(k, v) } +} + +func extraArgIf(k, v string) ArgOpt { + if v == "" { + return nil + } + return extraArg(k, v) +} + +func envArg(k, v string) ArgOpt { + return func(a *Args) { a.Env(k, v) } +} + +func envArgIf(k, v string) ArgOpt { + if v == "" { + return nil + } + return envArg(k, v) +} + +func flagArg(flag, value string) ArgOpt { + return func(a *Args) { a.Flag(flag, value) } +} + +func flagArgIf(flag, value string) ArgOpt { + if value == "" { + return nil + } + return flagArg(flag, value) +} + +func flagBoolIf(flag string, b bool) ArgOpt { + if b { + return flagArg(flag, "") + } + return nil +} + +func combine(opts ...ArgOpt) ArgOpt { + return func(a *Args) { + for _, opt := range opts { + if opt != nil { + opt(a) + } + } + } +} + +func buildArgs(opts []ArgOpt) *Args { + a := &Args{} + combine(opts...)(a) + return a +} + +// Extra sets a special k/v pair to be interpreted by the execution function. +func (a *Args) Extra(k, v string) { + if a.extra == nil { + a.extra = map[string]string{} + } + a.extra[k] = v +} + +// Val returns a special functions value for a given key. +func (a *Args) Val(k string) string { + if a.extra == nil { + return "" + } + return a.extra[k] +} + +// Env sets an environmant variable to be passed to the child process on exec. +func (a *Args) Env(k, v string) { + if a.env == nil { + a.env = map[string]string{} + } + a.env[k] = v +} + +// Flag adds a flag to be passed to the child process on exec. +func (a *Args) Flag(flag, value string) { + if a.flags == nil { + a.flags = map[string]string{} + } + a.flags[flag] = value +} + +// Add adds a positional argument to be passed to the child process on exec. +func (a *Args) Add(p string) { + a.pos = append(a.pos, p) +} + +func (a *Args) build() []string { + args := make([]string, 0, 2*len(a.flags)+len(a.pos)) + for k, v := range a.flags { + args = append(args, k) + if v != "" { + args = append(args, v) + } + } + + args = append(args, a.pos...) + return args +} diff --git a/dev-tools/lib/mage/mgenv/mgenv.go b/dev-tools/lib/mage/mgenv/mgenv.go new file mode 100644 index 0000000..b065a7d --- /dev/null +++ b/dev-tools/lib/mage/mgenv/mgenv.go @@ -0,0 +1,102 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgenv + +import ( + "fmt" + "os" + "sort" + "strconv" +) + +// Var holds an environment variables name, default value and doc string. +type Var struct { + name string + other string + doc string +} + +var envVars = map[string]Var{} +var envKeys []string + +func makeVar(name, other, doc string) Var { + if v, exists := envVars[name]; exists { + return v + } + + v := Var{name, other, doc} + envVars[name] = v + envKeys = append(envKeys, name) + sort.Strings(envKeys) + return v +} + +// MakeEnv builds a dictionary of defined environment variables, such that +// these can be passed to other processes (e.g. providers) +func MakeEnv() map[string]string { + m := make(map[string]string, len(envVars)) + for k, v := range envVars { + m[k] = v.Get() + } + return m +} + +// Keys returns the keys of registered environment variables. The keys returned +// are sorted. +// Note: The returned slice must not be changed or appended to. +func Keys() []string { + return envKeys +} + +// Find returns a registered Var by name. +func Find(name string) (Var, bool) { + v, ok := envVars[name] + return v, ok +} + +// String registers an environment variable and reads the current contents. +func String(name, other, doc string) string { + v := makeVar(name, other, doc) + return v.Get() +} + +// Bool registers an environment variable and interprets the current variable as bool. +func Bool(name string, other bool, doc string) bool { + v := makeVar(name, fmt.Sprint(other), doc) + b, err := strconv.ParseBool(v.Get()) + return err == nil && b +} + +// Name returns the environment variables name +func (v Var) Name() string { return v.name } + +// Default returns the environment variables default value as string. +func (v Var) Default() string { return v.other } + +// Doc returns the doc-string. +func (v Var) Doc() string { return v.doc } + +// Get reads an environment variable. Get returns the default value if the +// variable is not present or empty. +func (v Var) Get() string { + val := os.Getenv(v.name) + if val == "" { + return v.Default() + } + return val +} diff --git a/dev-tools/lib/mage/xbuild/docker.go b/dev-tools/lib/mage/xbuild/docker.go new file mode 100644 index 0000000..9c4f8f1 --- /dev/null +++ b/dev-tools/lib/mage/xbuild/docker.go @@ -0,0 +1,74 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package xbuild + +import ( + "fmt" + + "github.com/magefile/mage/sh" +) + +// DockerImage provides based on downloadable docker images. +type DockerImage struct { + Image string + Workdir string + Volumes map[string]string + Env map[string]string +} + +// Build pulls the required image. +func (p DockerImage) Build() error { + return sh.Run("docker", "pull", p.Image) +} + +// Run executes the command in a temporary docker container. The container is +// deleted after its execution. +func (p DockerImage) Run(env map[string]string, cmdAndArgs ...string) error { + spec := []string{"run", "--rm", "-i", "-t"} + for k, v := range mergeEnv(p.Env, env) { + spec = append(spec, "-e", fmt.Sprintf("%v=%v", k, v)) + } + for k, v := range p.Volumes { + spec = append(spec, "-v", fmt.Sprintf("%v:%v", k, v)) + } + if w := p.Workdir; w != "" { + spec = append(spec, "-w", w) + } + + spec = append(spec, p.Image) + for _, v := range cmdAndArgs { + if v != "" { + spec = append(spec, v) + } + } + + return sh.RunV("docker", spec...) +} + +func mergeEnv(a, b map[string]string) map[string]string { + merged := make(map[string]string, len(a)+len(b)) + copyEnv(merged, a) + copyEnv(merged, b) + return merged +} + +func copyEnv(to, from map[string]string) { + for k, v := range from { + to[k] = v + } +} diff --git a/dev-tools/lib/mage/xbuild/xbuild.go b/dev-tools/lib/mage/xbuild/xbuild.go new file mode 100644 index 0000000..d9699cc --- /dev/null +++ b/dev-tools/lib/mage/xbuild/xbuild.go @@ -0,0 +1,71 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package xbuild + +import ( + "fmt" + + "github.com/magefile/mage/mg" +) + +// Registry of available cross build environment providers. +type Registry struct { + table map[OSArch]Provider +} + +// Provider defines available functionality all cross build providers MUST implement. +type Provider interface { + // Build the environment + Build() error + + // Run command within environment. + Run(env map[string]string, cmdAndArgs ...string) error +} + +// OSArch tuple. +type OSArch struct { + OS string + Arch string +} + +// NewRegistry creates a new Regsitry. +func NewRegistry(tbl map[OSArch]Provider) *Registry { + return &Registry{tbl} +} + +// Find finds a provider by OS and Architecture name. +// Returns error if no provider can be found. +func (r *Registry) Find(os, arch string) (Provider, error) { + p := r.table[OSArch{os, arch}] + if p == nil { + return nil, fmt.Errorf("No provider for %v:%v defined", os, arch) + } + return p, nil +} + +// With calls fn with a provider matching the requires OS and ARCH. Returns +// and error if no provider can be found or function itself errors. +func (r *Registry) With(os, arch string, fn func(Provider) error) error { + p, err := r.Find(os, arch) + if err != nil { + return err + } + + mg.Deps(p.Build) + return fn(p) +} diff --git a/file.go b/file.go index f227a34..65f3580 100644 --- a/file.go +++ b/file.go @@ -75,6 +75,17 @@ const ( minRequiredFileSize = initSize ) +var maxMmapSize uint + +func init() { + if math.MaxUint32 == maxUint { + maxMmapSize = 2 * sz1GB + } else { + tmp := uint64(0x1FFFFFFFFFFF) + maxMmapSize = uint(tmp) + } +} + // Open opens or creates a new transactional file. // Open tries to create the file, if the file does not exist yet. Returns an // error if file access fails, file can not be locked or file meta pages are @@ -642,14 +653,6 @@ func readMeta(f vfs.File, off int64) (metaPage, reason) { // That is, exponential grows with values of 64KB, 128KB, 512KB, 1024KB, and so on. // Once 1GB is reached, the mmaped area is always a multiple of 1GB. func computeMmapSize(minSize, maxSize, pageSize uint) (uint, reason) { - var maxMapSize uint - if math.MaxUint32 == maxUint { - maxMapSize = 2 * sz1GB - } else { - tmp := uint64(0x1FFFFFFFFFFF) - maxMapSize = uint(tmp) - } - if maxSize != 0 { // return maxSize as multiple of pages. Round downwards in case maxSize // is not multiple of pages @@ -678,7 +681,7 @@ func computeMmapSize(minSize, maxSize, pageSize uint) (uint, reason) { // allocate number of 1GB blocks to fulfill minSize sz := ((minSize + (sz1GB - 1)) / sz1GB) * sz1GB - if sz > maxMapSize { + if sz > maxMmapSize { return 0, raiseInvalidParamf("mmap size of %v bytes is too large", sz) } diff --git a/file_test.go b/file_test.go index 248bed1..4a0db1f 100644 --- a/file_test.go +++ b/file_test.go @@ -82,7 +82,8 @@ func TestTxFile(t *testing.T) { title := fmt.Sprintf("min=%v,max=%v,expected=%v", min, max, expected) assert.Run(title, func(assert *assertions) { - if expected > uint64(maxUint) { + fmt.Printf("maxUint: %v, expected: %v\n", maxUint, expected) + if expected > uint64(maxMmapSize) { assert.Skip("unsatisfyable tests on 32bit system") } diff --git a/magefile.go b/magefile.go new file mode 100644 index 0000000..7ea7530 --- /dev/null +++ b/magefile.go @@ -0,0 +1,288 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//+build mage + +package main + +import ( + "errors" + "fmt" + "os" + "path" + "path/filepath" + "runtime" + "strings" + + "github.com/magefile/mage/mg" + "github.com/magefile/mage/sh" + + "github.com/elastic/go-txfile/dev-tools/lib/mage/gotool" + "github.com/elastic/go-txfile/dev-tools/lib/mage/mgenv" + "github.com/elastic/go-txfile/dev-tools/lib/mage/xbuild" +) + +// Info namespace is used to print additional docs, help messages, and other info. +type Info mg.Namespace + +// Prepare namespace is used to prepare/download/build common depenendencies for other tasks to run. +type Prepare mg.Namespace + +// Check runs pre-build checks on the environment and source code. (e.g. linters) +type Check mg.Namespace + +// Build namespace defines the set of build targets +type Build mg.Namespace + +const buildHome = "build" + +// environment variables +var ( + envBuildOS = mgenv.String("BUILD_OS", runtime.GOOS, "(string) set compiler target GOOS") + envBuildArch = mgenv.String("BUILD_ARCH", runtime.GOARCH, "(string) set compiler target GOARCH") + envTestUseBin = mgenv.Bool("TEST_USE_BIN", false, "(bool) reuse prebuild test binary when running tests") + envTestShort = mgenv.Bool("TEST_SHORT", false, "(bool) run tests with -short flag") + envFailFast = mgenv.Bool("FAIL_FAST", false, "(bool) do not run other tasks on failure") +) + +var xProviders = xbuild.NewRegistry(map[xbuild.OSArch]xbuild.Provider{ + xbuild.OSArch{"linux", "arm"}: &xbuild.DockerImage{ + Image: "balenalib/revpi-core-3-alpine-golang:latest-edge-build", + Workdir: "/go/src/github.com/elastic/go-txfile", + Volumes: map[string]string{ + filepath.Join(os.Getenv("GOPATH"), "src"): "/go/src", + }, + }, +}) + +// targets + +// Env prints environment info +func (Info) Env() { + printTitle("Mage environment variables") + for _, k := range mgenv.Keys() { + v, _ := mgenv.Find(k) + fmt.Printf("%v=%v\n", k, v.Get()) + } + fmt.Println() + + printTitle("Go info") + sh.RunV(mg.GoCmd(), "env") + fmt.Println() + + printTitle("docker info") + sh.RunV("docker", "version") +} + +// Vars prints the list of registered environment variables +func (Info) Vars() { + for _, k := range mgenv.Keys() { + v, _ := mgenv.Find(k) + fmt.Printf("%v=%v : %v\n", k, v.Default(), v.Doc()) + } +} + +// All runs all Prepare tasks +func (Prepare) All() { mg.Deps(Prepare.Dirs) } + +// Dirs creates requires build directories for storing artifacts +func (Prepare) Dirs() error { return mkdir("build") } + +// Lint runs golint +func (Check) Lint() error { + return errors.New("TODO: implement me") +} + +// Clean removes build artifacts +func Clean() error { + return sh.Rm(buildHome) +} + +// Mage builds the magefile binary for reuse +func (Build) Mage() error { + mg.Deps(Prepare.Dirs) + + goos := envBuildOS + goarch := envBuildArch + out := filepath.Join(buildHome, fmt.Sprintf("mage-%v-%v", goos, goarch)) + return sh.Run("mage", "-f", "-goos="+goos, "-goarch="+goarch, "-compile", out) +} + +// Test builds the per package unit test executables. +func (Build) Test() error { + mg.Deps(Prepare.Dirs) + + return withList(gotool.ListProjectPackages, failFastEach, func(pkg string) error { + tst := gotool.Test + return tst( + tst.OS(envBuildOS), + tst.ARCH(envBuildArch), + tst.Create(), + tst.WithCoverage(""), + tst.Out(path.Join(buildHome, pkg, path.Base(pkg))), + tst.Package(pkg), + ) + }) +} + +// Test runs the unit tests. +func Test() error { + mg.Deps(Prepare.Dirs) + + if crossBuild() { + return withXProvider(func(p xbuild.Provider) error { + mg.Deps(Build.Mage, Build.Test) + + env := mgenv.MakeEnv() + env["TEST_USE_BIN"] = "true" + return p.Run(env, "./build/mage-linux-arm", useIf("-v", mg.Verbose()), "test") + }) + } + + return withList(gotool.ListProjectPackages, failFastEach, func(pkg string) error { + fmt.Println("Test:", pkg) + if b, err := gotool.HasTests(pkg); !b { + fmt.Printf("Skipping %v: No tests found\n", pkg) + return err + } + + home := path.Join(buildHome, pkg) + if err := mkdir(home); err != nil { + return err + } + + tst := gotool.Test + bin := path.Join(home, path.Base(pkg)) + return tst( + tst.Use(useIf(bin, existsFile(bin) && envTestUseBin)), + tst.WithCoverage(path.Join(home, "cover.out")), + tst.Short(envTestShort), + tst.Out(bin), + tst.Package(pkg), + tst.Verbose(), + ) + }) +} + +// helpers + +func withList( + gen func() ([]string, error), + mode func(...func() error) error, + fn func(string) error, +) error { + list, err := gen() + if err != nil { + return err + } + + ops := make([]func() error, len(list)) + for i, v := range list { + v := v + ops[i] = func() error { return fn(v) } + } + + return mode(ops...) +} + +func useIf(s string, b bool) string { + if b { + return s + } + return "" +} + +func existsFile(path string) bool { + fi, err := os.Stat(path) + return err == nil && fi.Mode().IsRegular() +} + +func mkdirs(paths ...string) error { + for _, p := range paths { + if err := mkdir(p); err != nil { + return err + } + } + return nil +} + +func mkdir(path string) error { + return os.MkdirAll(path, os.ModeDir|0700) +} + +func failFastEach(ops ...func() error) error { + mode := each + if envFailFast { + mode = and + } + return mode(ops...) +} + +func each(ops ...func() error) error { + var errs []error + for _, op := range ops { + if err := op(); err != nil { + errs = append(errs, err) + } + } + return makeErrs(errs) +} + +func and(ops ...func() error) error { + for _, op := range ops { + if err := op(); err != nil { + return err + } + } + return nil +} + +type multiErr []error + +func makeErrs(errs []error) error { + if len(errs) == 0 { + return nil + } + return multiErr(errs) +} + +func (m multiErr) Error() string { + var bld strings.Builder + for _, err := range m { + if bld.Len() > 0 { + bld.WriteByte('\n') + bld.WriteString(err.Error()) + } + } + return bld.String() +} + +func printTitle(s string) { + fmt.Println(s) + for range s { + fmt.Print("=") + } + fmt.Println() +} + +func crossBuild() bool { + return envBuildArch != runtime.GOARCH || envBuildOS != runtime.GOOS +} + +func withXProvider(fn func(p xbuild.Provider) error) error { + return xProviders.With(envBuildOS, envBuildArch, fn) +}