Add --tf-download-url configuration option

cmd/server.go

@@ -72,6 +72,7 @@ const (
  SlackTokenFlag = "slack-token"
  SSLCertFileFlag = "ssl-cert-file"
  SSLKeyFileFlag = "ssl-key-file"
+ TFDownloadURLFlag = "tf-download-url"
  TFEHostnameFlag = "tfe-hostname"
  TFETokenFlag = "tfe-token"
  WriteGitCredsFlag = "write-git-creds"
@@ -86,6 +87,7 @@ const (
  DefaultGitlabHostname = "gitlab.com"
  DefaultLogLevel = "info"
  DefaultPort = 4141
+ DefaultTFDownloadURL = "https://releases.hashicorp.com"
  DefaultTFEHostname = "app.terraform.io"
 )
 
@@ -202,6 +204,10 @@ var stringFlags = map[string]stringFlag{
  SSLKeyFileFlag: {
  description: fmt.Sprintf("File containing x509 private key matching --%s.", SSLCertFileFlag),
  },
+ TFDownloadURLFlag: {
+ description: "The URL to download Terraform from.",
+ defaultValue: DefaultTFDownloadURL,
+ },
  TFEHostnameFlag: {
  description: "Hostname of your Terraform Enterprise installation. If using Terraform Cloud no need to set.",
  defaultValue: DefaultTFEHostname,
@@ -454,6 +460,9 @@ func (s *ServerCmd) setDefaults(c *server.UserConfig) {
  if c.Port == 0 {
  c.Port = DefaultPort
  }
+ if c.TFDownloadURL == "" {
+ c.TFDownloadURL = DefaultTFDownloadURL
+ }
  if c.TFEHostname == "" {
  c.TFEHostname = DefaultTFEHostname
  }

cmd/server_test.go

@@ -392,6 +392,7 @@ func TestExecute_Defaults(t *testing.T) {
  Equals(t, "", passedConfig.SlackToken)
  Equals(t, "", passedConfig.SSLCertFile)
  Equals(t, "", passedConfig.SSLKeyFile)
+ Equals(t, "https://releases.hashicorp.com", passedConfig.TFDownloadURL)
  Equals(t, "app.terraform.io", passedConfig.TFEHostname)
  Equals(t, "", passedConfig.TFEToken)
  Equals(t, false, passedConfig.WriteGitCreds)
@@ -515,6 +516,7 @@ func TestExecute_Flags(t *testing.T) {
  cmd.SlackTokenFlag: "slack-token",
  cmd.SSLCertFileFlag: "cert-file",
  cmd.SSLKeyFileFlag: "key-file",
+ cmd.TFDownloadURLFlag: "https://my-hostname.com",
  cmd.TFEHostnameFlag: "my-hostname",
  cmd.TFETokenFlag: "my-token",
  cmd.WriteGitCredsFlag: true,
@@ -554,6 +556,7 @@ func TestExecute_Flags(t *testing.T) {
  Equals(t, "slack-token", passedConfig.SlackToken)
  Equals(t, "cert-file", passedConfig.SSLCertFile)
  Equals(t, "key-file", passedConfig.SSLKeyFile)
+ Equals(t, "https://my-hostname.com", passedConfig.TFDownloadURL)
  Equals(t, "my-hostname", passedConfig.TFEHostname)
  Equals(t, "my-token", passedConfig.TFEToken)
  Equals(t, true, passedConfig.WriteGitCreds)
@@ -594,6 +597,7 @@ require-mergeable: true
 slack-token: slack-token
 ssl-cert-file: cert-file
 ssl-key-file: key-file
+tf-download-url: https://my-hostname.com
 tfe-hostname: my-hostname
 tfe-token: my-token
 write-git-creds: true
@@ -637,6 +641,7 @@ write-git-creds: true
  Equals(t, "slack-token", passedConfig.SlackToken)
  Equals(t, "cert-file", passedConfig.SSLCertFile)
  Equals(t, "key-file", passedConfig.SSLKeyFile)
+ Equals(t, "https://my-hostname.com", passedConfig.TFDownloadURL)
  Equals(t, "my-hostname", passedConfig.TFEHostname)
  Equals(t, "my-token", passedConfig.TFEToken)
  Equals(t, true, passedConfig.WriteGitCreds)
@@ -676,6 +681,7 @@ require-approval: true
 slack-token: slack-token
 ssl-cert-file: cert-file
 ssl-key-file: key-file
+tf-download-url: https://my-hostname.com
 tfe-hostname: my-hostname
 tfe-token: my-token
 write-git-creds: true
@@ -716,6 +722,7 @@ write-git-creds: true
  "SLACK_TOKEN": "override-slack-token",
  "SSL_CERT_FILE": "override-cert-file",
  "SSL_KEY_FILE": "override-key-file",
+ "TF_DOWNLOAD_URL": "https://override-my-hostname.com",
  "TFE_HOSTNAME": "override-my-hostname",
  "TFE_TOKEN": "override-my-token",
  "WRITE_GIT_CREDS": "false",
@@ -759,6 +766,7 @@ write-git-creds: true
  Equals(t, "override-slack-token", passedConfig.SlackToken)
  Equals(t, "override-cert-file", passedConfig.SSLCertFile)
  Equals(t, "override-key-file", passedConfig.SSLKeyFile)
+ Equals(t, "https://override-my-hostname.com", passedConfig.TFDownloadURL)
  Equals(t, "override-my-hostname", passedConfig.TFEHostname)
  Equals(t, "override-my-token", passedConfig.TFEToken)
  Equals(t, false, passedConfig.WriteGitCreds)
@@ -799,6 +807,7 @@ require-mergeable: true
 slack-token: slack-token
 ssl-cert-file: cert-file
 ssl-key-file: key-file
+tf-download-url: https://my-hostname.com
 tfe-hostname: my-hostname
 tfe-token: my-token
 write-git-creds: true
@@ -838,6 +847,7 @@ write-git-creds: true
  cmd.SlackTokenFlag: "override-slack-token",
  cmd.SSLCertFileFlag: "override-cert-file",
  cmd.SSLKeyFileFlag: "override-key-file",
+ cmd.TFDownloadURLFlag: "https://override-my-hostname.com",
  cmd.TFEHostnameFlag: "override-my-hostname",
  cmd.TFETokenFlag: "override-my-token",
  cmd.WriteGitCredsFlag: false,
@@ -875,6 +885,7 @@ write-git-creds: true
  Equals(t, "override-slack-token", passedConfig.SlackToken)
  Equals(t, "override-cert-file", passedConfig.SSLCertFile)
  Equals(t, "override-key-file", passedConfig.SSLKeyFile)
+ Equals(t, "https://override-my-hostname.com", passedConfig.TFDownloadURL)
  Equals(t, "override-my-hostname", passedConfig.TFEHostname)
  Equals(t, "override-my-token", passedConfig.TFEToken)
  Equals(t, false, passedConfig.WriteGitCreds)
@@ -917,6 +928,7 @@ func TestExecute_FlagEnvVarOverride(t *testing.T) {
  "SLACK_TOKEN": "slack-token",
  "SSL_CERT_FILE": "cert-file",
  "SSL_KEY_FILE": "key-file",
+ "TF_DOWNLOAD_URL": "https://my-hostname.com",
  "TFE_HOSTNAME": "my-hostname",
  "TFE_TOKEN": "my-token",
  "WRITE_GIT_CREDS": "true",
@@ -964,6 +976,7 @@ func TestExecute_FlagEnvVarOverride(t *testing.T) {
  cmd.SlackTokenFlag: "override-slack-token",
  cmd.SSLCertFileFlag: "override-cert-file",
  cmd.SSLKeyFileFlag: "override-key-file",
+ cmd.TFDownloadURLFlag: "https://override-my-hostname.com",
  cmd.TFEHostnameFlag: "override-my-hostname",
  cmd.TFETokenFlag: "override-my-token",
  cmd.WriteGitCredsFlag: false,
@@ -1003,6 +1016,7 @@ func TestExecute_FlagEnvVarOverride(t *testing.T) {
  Equals(t, "override-slack-token", passedConfig.SlackToken)
  Equals(t, "override-cert-file", passedConfig.SSLCertFile)
  Equals(t, "override-key-file", passedConfig.SSLKeyFile)
+ Equals(t, "https://override-my-hostname.com", passedConfig.TFDownloadURL)
  Equals(t, "override-my-hostname", passedConfig.TFEHostname)
  Equals(t, "override-my-token", passedConfig.TFEToken)
  Equals(t, false, passedConfig.WriteGitCreds)

runatlantis.io/docs/server-configuration.md

@@ -421,7 +421,15 @@ Values are chosen in this order:
  atlantis server --ssl-cert-file="/etc/ssl/private/my-cert.key"
  ```
  File containing x509 private key matching `--ssl-cert-file`.
-
+
+* ### `--tf-download-url`
+ ```bash
+ atlantis server --tf-download-url="https://releases.company.com"
+ ```
+ An alternative URL to download Terraform Versions if they are missing. Useful in an airgapped
+ environment where releases.hashicorp.com is not available. Directory structure of the custom
+ endpoint should match that of releases.hashicorp.com.
+
 * ### `--tfe-hostname`
  ```bash
  atlantis server --tfe-hostname="my-terraform-enterprise.company.com"

server/events/terraform/terraform_client.go

@@ -56,7 +56,8 @@ type DefaultClient struct {
  // with another binary, ex. echo.
  overrideTF string
  // downloader downloads terraform versions.
- downloader Downloader
+ downloader Downloader
+ downloadURL string
  // versions maps from the string representation of a tf version (ex. 0.11.10)
  // to the absolute path of that binary on disk (if it exists).
  // Use versionsLock to control access.
@@ -80,8 +81,6 @@ const (
  // binDirName is the name of the directory inside our data dir where
  // we download terraform binaries.
  binDirName = "bin"
- // releasesURL is the base url to download terraform from.
- releasesURL = "https://releases.hashicorp.com"
 )
 
 // versionRegex extracts the version from `terraform version` output.
@@ -107,6 +106,7 @@ func NewClient(
  tfeHostname string,
  defaultVersionStr string,
  defaultVersionFlagName string,
+ tfDownloadURL string,
  tfDownloader Downloader) (*DefaultClient, error) {
  var finalDefaultVersion *version.Version
  var localVersion *version.Version
@@ -145,7 +145,7 @@ func NewClient(
  // Since ensureVersion might end up downloading terraform,
  // we call it asynchronously so as to not delay server startup.
  versionsLock.Lock()
- _, err := ensureVersion(log, tfDownloader, versions, defaultVersion, binDir)
+ _, err := ensureVersion(log, tfDownloader, versions, defaultVersion, binDir, tfDownloadURL)
  versionsLock.Unlock()
  if err != nil {
  log.Err("could not download terraform %s", defaultVersion.String())
@@ -176,6 +176,7 @@ func NewClient(
  terraformPluginCacheDir: cacheDir,
  binDir: binDir,
  downloader: tfDownloader,
+ downloadURL: tfDownloadURL,
  versionsLock: &versionsLock,
  versions: versions,
  }, nil
@@ -192,6 +193,11 @@ func (c *DefaultClient) TerraformBinDir() string {
  return c.binDir
 }
 
+//TerraformDownloadURL returns the URL where we download Terraform binaries.
+func (c *DefaultClient) TerraformDownloadURL() string {
+ return c.downloadURL
+}
+
 // See Client.EnsureVersion.
 func (c *DefaultClient) EnsureVersion(log *logging.SimpleLogger, v *version.Version) error {
  if v == nil {
@@ -200,7 +206,7 @@ func (c *DefaultClient) EnsureVersion(log *logging.SimpleLogger, v *version.Vers
 
  var err error
  c.versionsLock.Lock()
- _, err = ensureVersion(log, c.downloader, c.versions, v, c.binDir)
+ _, err = ensureVersion(log, c.downloader, c.versions, v, c.binDir, c.downloadURL)
  c.versionsLock.Unlock()
  if err != nil {
  return err
@@ -245,7 +251,7 @@ func (c *DefaultClient) prepCmd(log *logging.SimpleLogger, v *version.Version, w
  } else {
  var err error
  c.versionsLock.Lock()
- binPath, err = ensureVersion(log, c.downloader, c.versions, v, c.binDir)
+ binPath, err = ensureVersion(log, c.downloader, c.versions, v, c.binDir, c.downloadURL)
  c.versionsLock.Unlock()
  if err != nil {
  return "", nil, err
@@ -390,7 +396,7 @@ func MustConstraint(v string) version.Constraints {
 
 // ensureVersion returns the path to a terraform binary of version v.
 // It will download this version if we don't have it.
-func ensureVersion(log *logging.SimpleLogger, dl Downloader, versions map[string]string, v *version.Version, binDir string) (string, error) {
+func ensureVersion(log *logging.SimpleLogger, dl Downloader, versions map[string]string, v *version.Version, binDir string, downloadURL string) (string, error) {
  if binPath, ok := versions[v.String()]; ok {
  return binPath, nil
  }
@@ -411,9 +417,8 @@ func ensureVersion(log *logging.SimpleLogger, dl Downloader, versions map[string
  versions[v.String()] = dest
  return dest, nil
  }
-
- log.Info("could not find terraform version %s in PATH or %s, downloading from %s", v.String(), binDir, releasesURL)
- urlPrefix := fmt.Sprintf("%s/terraform/%s/terraform_%s", releasesURL, v.String(), v.String())
+ log.Info("could not find terraform version %s in PATH or %s, downloading from %s", v.String(), binDir, downloadURL)
+ urlPrefix := fmt.Sprintf("%s/terraform/%s/terraform_%s", downloadURL, v.String(), v.String())
  binURL := fmt.Sprintf("%s_%s_%s.zip", urlPrefix, runtime.GOOS, runtime.GOARCH)
  checksumURL := fmt.Sprintf("%s_SHA256SUMS", urlPrefix)
  if err := dl.GetFile(dest, fmt.Sprintf("%s?checksum=file:%s", binURL, checksumURL)); err != nil {

server/events/terraform/terraform_client_test.go

@@ -68,7 +68,7 @@ is 0.11.13. You can update by downloading from www.terraform.io/downloads.html
  Ok(t, err)
  defer tempSetEnv(t, "PATH", fmt.Sprintf("%s:%s", tmp, os.Getenv("PATH")))()
 
- c, err := terraform.NewClient(nil, tmp, "", "", "", cmd.DefaultTFVersionFlag, nil)
+ c, err := terraform.NewClient(nil, tmp, "", "", "", cmd.DefaultTFVersionFlag, cmd.TFDownloadURLFlag, nil)
  Ok(t, err)
 
  Ok(t, err)
@@ -96,7 +96,7 @@ is 0.11.13. You can update by downloading from www.terraform.io/downloads.html
  Ok(t, err)
  defer tempSetEnv(t, "PATH", fmt.Sprintf("%s:%s", tmp, os.Getenv("PATH")))()
 
- c, err := terraform.NewClient(nil, tmp, "", "", "0.11.10", cmd.DefaultTFVersionFlag, nil)
+ c, err := terraform.NewClient(nil, tmp, "", "", "0.11.10", cmd.DefaultTFVersionFlag, cmd.TFDownloadURLFlag, nil)
  Ok(t, err)
 
  Ok(t, err)
@@ -116,7 +116,7 @@ func TestNewClient_NoTF(t *testing.T) {
  // Set PATH to only include our empty directory.
  defer tempSetEnv(t, "PATH", tmp)()
 
- _, err := terraform.NewClient(nil, tmp, "", "", "", cmd.DefaultTFVersionFlag, nil)
+ _, err := terraform.NewClient(nil, tmp, "", "", "", cmd.DefaultTFVersionFlag, cmd.TFDownloadURLFlag, nil)
  ErrEquals(t, "terraform not found in $PATH. Set --default-tf-version or download terraform from https://www.terraform.io/downloads.html", err)
 }
 
@@ -133,7 +133,7 @@ func TestNewClient_DefaultTFFlagInPath(t *testing.T) {
  Ok(t, err)
  defer tempSetEnv(t, "PATH", fmt.Sprintf("%s:%s", tmp, os.Getenv("PATH")))()
 
- c, err := terraform.NewClient(nil, tmp, "", "", "0.11.10", cmd.DefaultTFVersionFlag, nil)
+ c, err := terraform.NewClient(nil, tmp, "", "", "0.11.10", cmd.DefaultTFVersionFlag, cmd.TFDownloadURLFlag, nil)
  Ok(t, err)
 
  Ok(t, err)
@@ -157,7 +157,7 @@ func TestNewClient_DefaultTFFlagInBinDir(t *testing.T) {
  Ok(t, err)
  defer tempSetEnv(t, "PATH", fmt.Sprintf("%s:%s", tmp, os.Getenv("PATH")))()
 
- c, err := terraform.NewClient(logging.NewNoopLogger(), tmp, "", "", "0.11.10", cmd.DefaultTFVersionFlag, nil)
+ c, err := terraform.NewClient(logging.NewNoopLogger(), tmp, "", "", "0.11.10", cmd.DefaultTFVersionFlag, cmd.TFDownloadURLFlag, nil)
  Ok(t, err)
 
  Ok(t, err)
@@ -183,12 +183,12 @@ func TestNewClient_DefaultTFFlagDownload(t *testing.T) {
  err := ioutil.WriteFile(params[0].(string), []byte("#!/bin/sh\necho '\nTerraform v0.11.10\n'"), 0755)
  return []pegomock.ReturnValue{err}
  })
- c, err := terraform.NewClient(nil, tmp, "", "", "0.11.10", cmd.DefaultTFVersionFlag, mockDownloader)
+ c, err := terraform.NewClient(nil, tmp, "", "", "0.11.10", cmd.DefaultTFVersionFlag, cmd.TFDownloadURLFlag, mockDownloader)
  Ok(t, err)
 
  Ok(t, err)
  Equals(t, "0.11.10", c.DefaultVersion().String())
- baseURL := "https://releases.hashicorp.com/terraform/0.11.10"
+ baseURL := fmt.Sprintf("%s/terraform/0.11.10", cmd.TFDownloadURLFlag)
  expURL := fmt.Sprintf("%s/terraform_0.11.10_%s_%s.zip?checksum=file:%s/terraform_0.11.10_SHA256SUMS",
  baseURL,
  runtime.GOOS,
@@ -207,7 +207,7 @@ func TestNewClient_DefaultTFFlagDownload(t *testing.T) {
 func TestNewClient_BadVersion(t *testing.T) {
  tmp, cleanup := TempDir(t)
  defer cleanup()
- _, err := terraform.NewClient(nil, tmp, "", "", "malformed", cmd.DefaultTFVersionFlag, nil)
+ _, err := terraform.NewClient(nil, tmp, "", "", "malformed", cmd.DefaultTFVersionFlag, cmd.TFDownloadURLFlag, nil)
  ErrEquals(t, "Malformed version: malformed", err)
 }
 
@@ -219,7 +219,7 @@ func TestRunCommandWithVersion_DLsTF(t *testing.T) {
 
  mockDownloader := mocks.NewMockDownloader()
  // Set up our mock downloader to write a fake tf binary when it's called.
- baseURL := "https://releases.hashicorp.com/terraform/99.99.99"
+ baseURL := fmt.Sprintf("%s/terraform/99.99.99", cmd.TFDownloadURLFlag)
  expURL := fmt.Sprintf("%s/terraform_99.99.99_%s_%s.zip?checksum=file:%s/terraform_99.99.99_SHA256SUMS",
  baseURL,
  runtime.GOOS,
@@ -230,7 +230,7 @@ func TestRunCommandWithVersion_DLsTF(t *testing.T) {
  return []pegomock.ReturnValue{err}
  })
 
- c, err := terraform.NewClient(nil, tmp, "", "", "0.11.10", cmd.DefaultTFVersionFlag, mockDownloader)
+ c, err := terraform.NewClient(nil, tmp, "", "", "0.11.10", cmd.DefaultTFVersionFlag, cmd.TFDownloadURLFlag, mockDownloader)
  Ok(t, err)
  Equals(t, "0.11.10", c.DefaultVersion().String())
 
@@ -249,7 +249,7 @@ func TestEnsureVersion_downloaded(t *testing.T) {
 
  mockDownloader := mocks.NewMockDownloader()
 
- c, err := terraform.NewClient(nil, tmp, "", "", "0.11.10", cmd.DefaultTFVersionFlag, mockDownloader)
+ c, err := terraform.NewClient(nil, tmp, "", "", "0.11.10", cmd.DefaultTFVersionFlag, cmd.TFDownloadURLFlag, mockDownloader)
  Ok(t, err)
 
  Equals(t, "0.11.10", c.DefaultVersion().String())
@@ -261,7 +261,7 @@ func TestEnsureVersion_downloaded(t *testing.T) {
 
  Ok(t, err)
 
- baseURL := "https://releases.hashicorp.com/terraform/99.99.99"
+ baseURL := fmt.Sprintf("%s/terraform/99.99.99", cmd.TFDownloadURLFlag)
  expURL := fmt.Sprintf("%s/terraform_99.99.99_%s_%s.zip?checksum=file:%s/terraform_99.99.99_SHA256SUMS",
  baseURL,
  runtime.GOOS,

server/events_controller_e2e_test.go

@@ -400,7 +400,7 @@ func setupE2E(t *testing.T, repoDir string) (server.EventsController, *vcsmocks.
  GithubUser: "github-user",
  GitlabUser: "gitlab-user",
  }
- terraformClient, err := terraform.NewClient(logger, dataDir, "", "", "", "default-tf-version", &NoopTFDownloader{})
+ terraformClient, err := terraform.NewClient(logger, dataDir, "", "", "", "tfdownloadurl", "default-tf-version", &NoopTFDownloader{})
  Ok(t, err)
  boltdb, err := db.New(dataDir)
  Ok(t, err)

server/server.go

@@ -211,6 +211,7 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) {
  userConfig.DataDir,
  userConfig.TFEToken,
  userConfig.TFEHostname,
+ userConfig.TFDownloadURL,
  userConfig.DefaultTFVersion,
  config.DefaultTFVersionFlag,
  &terraform.DefaultDownloader{})

server/user_config.go

@@ -44,6 +44,7 @@ type UserConfig struct {
  SlackToken string `mapstructure:"slack-token"`
  SSLCertFile string `mapstructure:"ssl-cert-file"`
  SSLKeyFile string `mapstructure:"ssl-key-file"`
+ TFDownloadURL string `mapstructure:"tf-download-url"`
  TFEHostname string `mapstructure:"tfe-hostname"`
  TFEToken string `mapstructure:"tfe-token"`
  DefaultTFVersion string `mapstructure:"default-tf-version"`