[SPIKE- DO NOT MERGE]Adds venv sh from cloud

Dockerfile

@@ -12,7 +12,10 @@ RUN apt-get -y update && apt-get -y upgrade && \
  apt-get -y install && apt-get -y upgrade && \
  apt-get -y install software-properties-common && \
  apt-get -y install curl && \
- apt-get -y install iputils-ping &&\
+ apt-get -y install iputils-ping \
+ squashfs-tools \
+ python3-venv \
+ jq && \
  apt-get -y install git libpq-dev openssh-client openssl && \
  apt-get -y autoremove && \
  rm -rf /var/lib/apt/lists/*
@@ -43,3 +46,16 @@ RUN pip install --force-reinstall MarkupSafe==2.0.1 # TODO: find better fix for
 
 COPY ./dbt_server /usr/src/app/dbt_server
 COPY ./dbt_worker /usr/src/app/dbt_worker
+
+# dbt virtual environments
+#
+RUN mkdir -p /venv
+RUN --mount=type=ssh \
+ --mount=type=secret,id=GITHUB_TOKEN \
+ # TODO: MAKE TOKEN AVAILABLE TO SCRIPT
+ export GITHUB_TOKEN={{token}} && \
+ /usr/src/app/bash/install-venv.sh
+# no longer needed
+RUN apt remove -y squashfs-tools
+
+RUN echo "venvs:" && ls -al /venv

bash/install-venv.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+#
+# installs dbt-core virtual environments for use by cloud
+#
+# NOTE: this script is meant to be run within a dockerfile build.
+#
+# NOTE: an appropriate GITHUB_TOKEN must be defined in the environment.
+
+VENV_ROOT=/venv
+
+set -eo pipefail
+
+TAG_NAME="latest"
+# alternatively, specify an actual tag in order to pin the versions being used
+# TAG_NAME=tags/<tag number here>
+
+RELEASE=$(curl -s -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${GITHUB_TOKEN}" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/repos/dbt-labs/venv-cache/releases/${TAG_NAME})
+echo ${RELEASE}
+
+# This step is just to capture any failures which might happen below,
+# which would otherwise silently fail via the process substitution
+# used at the bottom of the while loop:
+# https://www.gnu.org/software/bash/manual/html_node/Process-Substitution.html
+# In the event of a problem, we want a failure to be loud here,
+# causing CI to fail.
+if ! echo ${RELEASE} | jq -r ".assets[] | [.url, .name] | @tsv"; then
+ echo "The GITHUB_TOKEN environment variable is probably missing or invalid."
+ exit 1
+fi
+
+while IFS=$'\t' read -r uri name; do
+ echo "retrieving name: ${name} uri: ${uri}"
+ fullname=${VENV_ROOT}/${name}
+ # tell curl to fail on any 400 status or above
+ if ! curl --fail -H "Accept: application/octet-stream" -H "Authorization: Bearer ${GITHUB_TOKEN}" -H "X-GitHub-Api-Version: 2022-11-28" -o ${fullname} -L ${uri}; then
+ echo "Failed to curl asset: $?: ${uri}"
+ exit 1
+ fi
+ extension="${fullname##*.}"
+ if [ "${extension}" == "sqsh" ]; then
+ dirname=${fullname%.*}
+ unsquashfs -d ${dirname} ${fullname}
+ rm -f ${fullname}
+ fi
+done < <(echo ${RELEASE} | jq -r ".assets[] | [.url, .name] | @tsv")