E2E Tests for Astro Deploy Action #140
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E Tests for Astro Deploy Action | |
| on: | |
| push: | |
| branches: | |
| - main | |
| workflow_dispatch: | |
| inputs: | |
| workspace_id: | |
| description: "Workspace ID" | |
| required: false | |
| default: "" | |
| org_id: | |
| description: "Organization ID" | |
| required: false | |
| default: "" | |
| astronomer_host: | |
| description: "Astronomer Host" | |
| required: false | |
| default: "" | |
| token: | |
| description: "API Token" | |
| required: false | |
| default: "" | |
| env: | |
| ASTRO_API_TOKEN: ${{ secrets.ASTRO_API_TOKEN }} | |
| jobs: | |
| redact-inputs: | |
| name: Redact Inputs | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Redact Inputs | |
| run: | | |
| # add-mask will redact the value of the input, but will still log the value, | |
| # hence using a known workaround: https:/actions/runner/issues/643 | |
| WORKSPACE_ID=$(jq -r '.inputs.workspace_id' $GITHUB_EVENT_PATH) | |
| echo ::add-mask::$WORKSPACE_ID | |
| ORG_ID=$(jq -r '.inputs.org_id' $GITHUB_EVENT_PATH) | |
| echo ::add-mask::$ORG_ID | |
| ASTRONOMER_HOST=$(jq -r '.inputs.astronomer_host' $GITHUB_EVENT_PATH) | |
| echo ::add-mask::$ASTRONOMER_HOST | |
| TOKEN=$(jq -r '.inputs.token' $GITHUB_EVENT_PATH) | |
| echo ::add-mask::$TOKEN | |
| # Create test deployments would use the template files and generate deployments with unique names | |
| create-test-deployments: | |
| name: Create Deployment | |
| needs: redact-inputs | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| deployment: [deployment.yaml] | |
| outputs: | |
| DEPLOYMENT_ID: ${{ steps.create-deployment.outputs.DEPLOYMENT_ID }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Get Astro Environment Info | |
| id: get-astro-env-info | |
| uses: ./.github/workflows/e2e/get_astro_env_info | |
| with: | |
| input_workspace_id: ${{ github.event.inputs.workspace_id }} | |
| input_organization_id: ${{ github.event.inputs.org_id }} | |
| input_astronomer_host: ${{ github.event.inputs.astronomer_host }} | |
| input_astro_api_token: ${{ github.event.inputs.token }} | |
| secret_workspace_id: ${{ secrets.WORKSPACE_ID }} | |
| secret_organization_id: ${{ secrets.ORGANIZATION_ID }} | |
| secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }} | |
| secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} | |
| - name: Install dependencies | |
| run: | | |
| sudo add-apt-repository ppa:rmescandon/yq | |
| sudo apt update | |
| sudo apt install yq -y | |
| curl -sSL https://install.astronomer.io | sudo bash -s | |
| astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| astro workspace switch ${{ steps.get-astro-env-info.outputs.workspace_id }} | |
| # Generate a random test_uid and set it as an environment variable, so we can append it to the deployment name | |
| test_uid=$(uuidgen) | |
| echo "test_uid=$test_uid" >> $GITHUB_ENV | |
| - name: Create Deployment | |
| id: create-deployment | |
| run: | | |
| # Update the deployment name to maintain uniqueness | |
| sed -i "s| name:.*| name: deploy-action-e2e-${{ env.test_uid }}|g" e2e-setup/deployment-templates/${{ matrix.deployment }} | |
| DEPLOYMENT_ID=$(astro deployment create --deployment-file e2e-setup/deployment-templates/${{ matrix.deployment }} | yq e '.deployment.metadata.deployment_id' -) | |
| echo "DEPLOYMENT_ID=$DEPLOYMENT_ID" >> $GITHUB_OUTPUT | |
| default-deploy-tests: | |
| name: Default Deploy Test | |
| runs-on: ubuntu-latest | |
| needs: [create-test-deployments] | |
| strategy: | |
| max-parallel: 1 | |
| matrix: | |
| deployment_id: | |
| ["${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}"] | |
| deploy_type: [dags, image-and-dags] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Get Astro Environment Info | |
| id: get-astro-env-info | |
| uses: ./.github/workflows/e2e/get_astro_env_info | |
| with: | |
| input_workspace_id: ${{ github.event.inputs.workspace_id }} | |
| input_organization_id: ${{ github.event.inputs.org_id }} | |
| input_astronomer_host: ${{ github.event.inputs.astronomer_host }} | |
| input_astro_api_token: ${{ github.event.inputs.token }} | |
| secret_workspace_id: ${{ secrets.WORKSPACE_ID }} | |
| secret_organization_id: ${{ secrets.ORGANIZATION_ID }} | |
| secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }} | |
| secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} | |
| - name: Mock git commands | |
| run: | | |
| # if deploy_type is dags, then use dags-deploy-git.sh as mock otherwise use image-deploy-git.sh as mock | |
| if [[ "${{ matrix.deploy_type }}" == "dags" ]]; then | |
| mv e2e-setup/mocks/default-dag-deploy-git.sh /usr/local/bin/git | |
| else | |
| mv e2e-setup/mocks/default-image-deploy-git.sh /usr/local/bin/git | |
| fi | |
| chmod +x /usr/local/bin/git | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get install jq | |
| # we need to pre-install the CLI to set the context | |
| curl -sSL https://install.astronomer.io | sudo bash -s | |
| - name: Setup astro project in base folder | |
| run: | | |
| # copy the astro project to the base folder | |
| cp -r e2e-setup/astro-project/ . | |
| - name: Set CLI context | |
| run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Get Deployment Info Before Test | |
| id: get-deployment-before | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| deployment_id: ${{ matrix.deployment_id }} | |
| organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} | |
| astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} | |
| astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Run Deploy Action | |
| uses: ./ | |
| with: | |
| deployment-id: ${{ matrix.deployment_id }} | |
| workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} | |
| - name: Get Deployment Info After Test | |
| id: get-deployment-after | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| deployment_id: ${{ matrix.deployment_id }} | |
| organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} | |
| astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} | |
| astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Validate Deploy Action | |
| uses: ./.github/workflows/e2e/validate_deployment | |
| with: | |
| is_dag_only_deploy: ${{ matrix.deploy_type == 'dags' }} | |
| dag_tarball_version_before: ${{ steps.get-deployment-before.outputs.desired_dag_tarball_version }} | |
| image_version_before: ${{ steps.get-deployment-before.outputs.desired_image_version }} | |
| dag_tarball_version_after: ${{ steps.get-deployment-after.outputs.desired_dag_tarball_version }} | |
| image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }} | |
| # DAG Deploy test would test the DAG only deploy functionality in deploy action | |
| dag-deploy-test: | |
| name: DAG Deploy Test | |
| runs-on: ubuntu-latest | |
| needs: [default-deploy-tests, create-test-deployments] | |
| strategy: | |
| matrix: | |
| deployment_id: | |
| ["${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}"] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Get Astro Environment Info | |
| id: get-astro-env-info | |
| uses: ./.github/workflows/e2e/get_astro_env_info | |
| with: | |
| input_workspace_id: ${{ github.event.inputs.workspace_id }} | |
| input_organization_id: ${{ github.event.inputs.org_id }} | |
| input_astronomer_host: ${{ github.event.inputs.astronomer_host }} | |
| input_astro_api_token: ${{ github.event.inputs.token }} | |
| secret_workspace_id: ${{ secrets.WORKSPACE_ID }} | |
| secret_organization_id: ${{ secrets.ORGANIZATION_ID }} | |
| secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }} | |
| secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} | |
| - name: Mock git commands | |
| run: | | |
| mv e2e-setup/mocks/dag-deploy-git.sh /usr/local/bin/git | |
| chmod +x /usr/local/bin/git | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get install jq | |
| # we need to pre-install the CLI to set the context | |
| curl -sSL https://install.astronomer.io | sudo bash -s | |
| - name: Set CLI context | |
| run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Get Deployment Info Before Test | |
| id: get-deployment-before | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| deployment_id: ${{ matrix.deployment_id }} | |
| organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} | |
| astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} | |
| astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Deploy to Astro | |
| uses: ./ | |
| with: | |
| deployment-id: ${{ matrix.deployment_id }} | |
| workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} | |
| parse: true | |
| root-folder: e2e-setup/astro-project | |
| - name: Get Deployment Info After Test | |
| id: get-deployment-after | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| deployment_id: ${{ matrix.deployment_id }} | |
| organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} | |
| astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} | |
| astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Validate Deploy Action | |
| uses: ./.github/workflows/e2e/validate_deployment | |
| with: | |
| is_dag_only_deploy: true | |
| dag_tarball_version_before: ${{ steps.get-deployment-before.outputs.desired_dag_tarball_version }} | |
| image_version_before: ${{ steps.get-deployment-before.outputs.desired_image_version }} | |
| dag_tarball_version_after: ${{ steps.get-deployment-after.outputs.desired_dag_tarball_version }} | |
| image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }} | |
| pytests-test: | |
| name: Pytest Test | |
| runs-on: ubuntu-latest | |
| needs: [dag-deploy-test, create-test-deployments] | |
| strategy: | |
| matrix: | |
| deployment_id: | |
| ["${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}"] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Get Astro Environment Info | |
| id: get-astro-env-info | |
| uses: ./.github/workflows/e2e/get_astro_env_info | |
| with: | |
| input_workspace_id: ${{ github.event.inputs.workspace_id }} | |
| input_organization_id: ${{ github.event.inputs.org_id }} | |
| input_astronomer_host: ${{ github.event.inputs.astronomer_host }} | |
| input_astro_api_token: ${{ github.event.inputs.token }} | |
| secret_workspace_id: ${{ secrets.WORKSPACE_ID }} | |
| secret_organization_id: ${{ secrets.ORGANIZATION_ID }} | |
| secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }} | |
| secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get install jq | |
| # we need to pre-install the CLI to set the context | |
| curl -sSL https://install.astronomer.io | sudo bash -s | |
| - name: Set CLI context | |
| run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Mock git commands for DAG Deploy | |
| run: | | |
| # set to dag deploy git mock, with deploy-type set to image-and-dags to test that it will do a image deploy | |
| mv e2e-setup/mocks/dag-deploy-git.sh /usr/local/bin/git | |
| chmod +x /usr/local/bin/git | |
| - name: Get Deployment Info Before Test | |
| id: get-deployment-before | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| deployment_id: ${{ matrix.deployment_id }} | |
| organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} | |
| astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} | |
| astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Run Pytest | |
| uses: ./ | |
| with: | |
| deployment-id: ${{ matrix.deployment_id }} | |
| workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} | |
| pytest: true | |
| action: deploy | |
| deploy-type: image-and-dags | |
| root-folder: e2e-setup/astro-project | |
| - name: Get Deployment Info After Test | |
| id: get-deployment-after | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| deployment_id: ${{ matrix.deployment_id }} | |
| organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} | |
| astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} | |
| astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Validate Deploy Action | |
| uses: ./.github/workflows/e2e/validate_deployment | |
| with: | |
| dag_tarball_version_before: ${{ steps.get-deployment-before.outputs.desired_dag_tarball_version }} | |
| image_version_before: ${{ steps.get-deployment-before.outputs.desired_image_version }} | |
| dag_tarball_version_after: ${{ steps.get-deployment-after.outputs.desired_dag_tarball_version }} | |
| image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }} | |
| infer-deploy: | |
| name: Infer Deploy Test | |
| runs-on: ubuntu-latest | |
| needs: [pytests-test, create-test-deployments] | |
| strategy: | |
| max-parallel: 1 | |
| matrix: | |
| deployment_id: | |
| ["${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}"] | |
| deploy_type: [dags, image-and-dags] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Get Astro Environment Info | |
| id: get-astro-env-info | |
| uses: ./.github/workflows/e2e/get_astro_env_info | |
| with: | |
| input_workspace_id: ${{ github.event.inputs.workspace_id }} | |
| input_organization_id: ${{ github.event.inputs.org_id }} | |
| input_astronomer_host: ${{ github.event.inputs.astronomer_host }} | |
| input_astro_api_token: ${{ github.event.inputs.token }} | |
| secret_workspace_id: ${{ secrets.WORKSPACE_ID }} | |
| secret_organization_id: ${{ secrets.ORGANIZATION_ID }} | |
| secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }} | |
| secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} | |
| - name: Mock git commands | |
| run: | | |
| # if deploy_type is dags, then use dags-deploy-git.sh as mock otherwise use image-deploy-git.sh as mock | |
| if [[ "${{ matrix.deploy_type }}" == "dags" ]]; then | |
| mv e2e-setup/mocks/dag-deploy-git.sh /usr/local/bin/git | |
| else | |
| mv e2e-setup/mocks/image-deploy-git.sh /usr/local/bin/git | |
| fi | |
| chmod +x /usr/local/bin/git | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get install jq | |
| # we need to pre-install the CLI to set the context | |
| curl -sSL https://install.astronomer.io | sudo bash -s | |
| - name: Set CLI context | |
| run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Get Deployment Info Before Test | |
| id: get-deployment-before | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| deployment_id: ${{ matrix.deployment_id }} | |
| organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} | |
| astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} | |
| astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Run Deploy Action | |
| uses: ./ | |
| with: | |
| deployment-id: ${{ matrix.deployment_id }} | |
| workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} | |
| parse: true | |
| action: deploy | |
| root-folder: e2e-setup/astro-project | |
| - name: Get Deployment Info After Test | |
| id: get-deployment-after | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| deployment_id: ${{ matrix.deployment_id }} | |
| organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} | |
| astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} | |
| astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Validate Deploy Action | |
| uses: ./.github/workflows/e2e/validate_deployment | |
| with: | |
| is_dag_only_deploy: ${{ matrix.deploy_type == 'dags' }} | |
| dag_tarball_version_before: ${{ steps.get-deployment-before.outputs.desired_dag_tarball_version }} | |
| image_version_before: ${{ steps.get-deployment-before.outputs.desired_image_version }} | |
| dag_tarball_version_after: ${{ steps.get-deployment-after.outputs.desired_dag_tarball_version }} | |
| image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }} | |
| no-deploy-test: | |
| name: No Deploy Test | |
| runs-on: ubuntu-latest | |
| needs: [infer-deploy, create-test-deployments] | |
| strategy: | |
| matrix: | |
| deployment_id: | |
| ["${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}"] | |
| deploy_type: [infer, dbt, dags, image-and-dags] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Get Astro Environment Info | |
| id: get-astro-env-info | |
| uses: ./.github/workflows/e2e/get_astro_env_info | |
| with: | |
| input_workspace_id: ${{ github.event.inputs.workspace_id }} | |
| input_organization_id: ${{ github.event.inputs.org_id }} | |
| input_astronomer_host: ${{ github.event.inputs.astronomer_host }} | |
| input_astro_api_token: ${{ github.event.inputs.token }} | |
| secret_workspace_id: ${{ secrets.WORKSPACE_ID }} | |
| secret_organization_id: ${{ secrets.ORGANIZATION_ID }} | |
| secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }} | |
| secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} | |
| - name: Setup dependencies | |
| run: | | |
| sudo apt-get install jq | |
| # we need to pre-install the CLI to set the context | |
| curl -sSL https://install.astronomer.io | sudo bash -s | |
| - name: Set CLI context | |
| run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Mock git commands for DAG Deploy | |
| run: | | |
| # set to no deploy git mock, to simulate no interested file change scenario | |
| mv e2e-setup/mocks/no-deploy-git.sh /usr/local/bin/git | |
| chmod +x /usr/local/bin/git | |
| - name: Get Deployment Info Before Test | |
| id: get-deployment-before | |
| if: ${{ matrix.deploy_type != 'dbt' }} | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| deployment_id: ${{ matrix.deployment_id }} | |
| organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} | |
| astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} | |
| astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Run Deploy Action | |
| uses: ./ | |
| with: | |
| deployment-id: ${{ matrix.deployment_id }} | |
| workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} | |
| action: deploy | |
| deploy-type: ${{ matrix.deploy_type }} | |
| root-folder: ${{ matrix.deploy_type == 'dbt' && 'e2e-setup/dbt' || 'e2e-setup/astro-project' }} | |
| - name: Get Deployment Info After Test | |
| id: get-deployment-after | |
| if: ${{ matrix.deploy_type != 'dbt' }} | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| deployment_id: ${{ matrix.deployment_id }} | |
| organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} | |
| astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} | |
| astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Validate no image or dag deploy happened | |
| if: ${{ matrix.deploy_type != 'dbt' }} | |
| uses: ./.github/workflows/e2e/validate_deployment | |
| with: | |
| is_no_deploy: true | |
| dag_tarball_version_before: ${{ steps.get-deployment-before.outputs.desired_dag_tarball_version }} | |
| image_version_before: ${{ steps.get-deployment-before.outputs.desired_image_version }} | |
| dag_tarball_version_after: ${{ steps.get-deployment-after.outputs.desired_dag_tarball_version }} | |
| image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }} | |
| - name: Get DBT bundle info | |
| id: get-dbt-bundle-info | |
| if: ${{ matrix.deploy_type == 'dbt' }} | |
| uses: ./.github/workflows/e2e/get_bundle_info | |
| with: | |
| deployment_id: ${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }} | |
| organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} | |
| astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} | |
| astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Validate no DBT deploy happened | |
| id: validate-dbt-deploy-action | |
| if: ${{ matrix.deploy_type == 'dbt' }} | |
| shell: bash | |
| run: | | |
| if [[ "${{ steps.get-dbt-bundle-info.outputs.desired_bundle_version }}" != "" ]]; then | |
| echo "DBT Deploy Action validation failed: DBT bundle found, but there should be no deploy" | |
| exit 1 | |
| fi | |
| echo "DBT Deploy Action validation succeeded: DBT bundle not found" | |
| exit 0 | |
| custom-docker-image-test: | |
| name: Custom Docker Image Test | |
| runs-on: ubuntu-latest | |
| needs: [no-deploy-test, create-test-deployments] | |
| strategy: | |
| matrix: | |
| deployment_id: | |
| ["${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}"] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Get Astro Environment Info | |
| id: get-astro-env-info | |
| uses: ./.github/workflows/e2e/get_astro_env_info | |
| with: | |
| input_workspace_id: ${{ github.event.inputs.workspace_id }} | |
| input_organization_id: ${{ github.event.inputs.org_id }} | |
| input_astronomer_host: ${{ github.event.inputs.astronomer_host }} | |
| input_astro_api_token: ${{ github.event.inputs.token }} | |
| secret_workspace_id: ${{ secrets.WORKSPACE_ID }} | |
| secret_organization_id: ${{ secrets.ORGANIZATION_ID }} | |
| secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }} | |
| secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get install jq | |
| # we need to pre-install the CLI to set the context | |
| curl -sSL https://install.astronomer.io | sudo bash -s | |
| - name: Build Docker image | |
| run: cd e2e-setup/astro-project && docker build -t custom-image:latest . | |
| - name: Set CLI context | |
| run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Get Deployment Info Before Test | |
| id: get-deployment-before | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| deployment_id: ${{ matrix.deployment_id }} | |
| organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} | |
| astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} | |
| astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Deploy to Astro | |
| uses: ./ | |
| with: | |
| deployment-id: ${{ matrix.deployment_id }} | |
| workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} | |
| image-name: custom-image:latest | |
| action: deploy | |
| deploy-type: image-and-dags | |
| root-folder: e2e-setup/astro-project | |
| - name: Get Deployment Info After Test | |
| id: get-deployment-after | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| deployment_id: ${{ matrix.deployment_id }} | |
| organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} | |
| astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} | |
| astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Validate Deploy Action | |
| uses: ./.github/workflows/e2e/validate_deployment | |
| with: | |
| dag_tarball_version_before: ${{ steps.get-deployment-before.outputs.desired_dag_tarball_version }} | |
| image_version_before: ${{ steps.get-deployment-before.outputs.desired_image_version }} | |
| dag_tarball_version_after: ${{ steps.get-deployment-after.outputs.desiredDagTarballVersion }} | |
| image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }} | |
| # DBT Deploy test would test the DAG only deploy functionality in deploy action | |
| dbt-deploy-test: | |
| name: DBT Deploy Test | |
| runs-on: ubuntu-latest | |
| needs: [custom-docker-image-test, create-test-deployments] | |
| strategy: | |
| matrix: | |
| deployment_id: | |
| ["${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}"] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Get Astro Environment Info | |
| id: get-astro-env-info | |
| uses: ./.github/workflows/e2e/get_astro_env_info | |
| with: | |
| input_workspace_id: ${{ github.event.inputs.workspace_id }} | |
| input_organization_id: ${{ github.event.inputs.org_id }} | |
| input_astronomer_host: ${{ github.event.inputs.astronomer_host }} | |
| input_astro_api_token: ${{ github.event.inputs.token }} | |
| secret_workspace_id: ${{ secrets.WORKSPACE_ID }} | |
| secret_organization_id: ${{ secrets.ORGANIZATION_ID }} | |
| secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }} | |
| secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} | |
| - name: Mock git commands | |
| run: | | |
| mv e2e-setup/mocks/dbt-deploy-git.sh /usr/local/bin/git | |
| chmod +x /usr/local/bin/git | |
| - name: Install dependencies | |
| id: setup | |
| run: | | |
| sudo apt-get install jq | |
| # we need to pre-install the CLI to set the context | |
| curl -sSL https://install.astronomer.io | sudo bash -s | |
| test_start_time=$(date -u +"%Y-%m-%dT%H:%M:%S.%NZ") | |
| echo "test_start_time=$test_start_time" >> $GITHUB_OUTPUT | |
| - name: Set CLI context | |
| run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: DBT Deploy to Astro | |
| uses: ./ | |
| with: | |
| deployment-id: ${{ matrix.deployment_id }} | |
| workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} | |
| parse: true | |
| root-folder: e2e-setup/dbt | |
| deploy-type: dbt | |
| description: "test-dbt-deploy-action" | |
| - name: Get DBT bundle info | |
| id: get-dbt-bundle-info | |
| uses: ./.github/workflows/e2e/get_bundle_info | |
| with: | |
| deployment_id: ${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }} | |
| organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} | |
| astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} | |
| astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Validate DBT Deploy Action | |
| id: validate-dbt-deploy-action | |
| shell: bash | |
| run: | | |
| if [[ "${{ steps.get-dbt-bundle-info.outputs.desired_bundle_version }}" == "" ]]; then | |
| echo "DBT Deploy Action validation failed: DBT bundle not found" | |
| exit 1 | |
| fi | |
| updated_at=$(date -u -d "${{ steps.get-dbt-bundle-info.outputs.updated_at }}" +"%s.%N") | |
| test_start_time=$(date -u -d "${{ steps.setup.outputs.test_start_time }}" +"%s.%N") | |
| if [[ $(echo "$updated_at > $test_start_time" | bc -l) ]]; then | |
| echo "DBT Deploy Action validation succeeded: DBT bundle found" | |
| exit 0 | |
| fi | |
| echo "DBT Deploy Action validation failed: DBT bundle was not updated" | |
| exit 1 | |
| # Deployment preview tests can run in parallel to above tests since it operates on a different deployment, so it won't interfere with the above tests | |
| deployment-preview-test: | |
| name: Deployment Preview Test | |
| runs-on: ubuntu-latest | |
| needs: [create-test-deployments] | |
| strategy: | |
| matrix: | |
| deployment_id: | |
| ["${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}"] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Get Astro Environment Info | |
| id: get-astro-env-info | |
| uses: ./.github/workflows/e2e/get_astro_env_info | |
| with: | |
| input_workspace_id: ${{ github.event.inputs.workspace_id }} | |
| input_organization_id: ${{ github.event.inputs.org_id }} | |
| input_astronomer_host: ${{ github.event.inputs.astronomer_host }} | |
| input_astro_api_token: ${{ github.event.inputs.token }} | |
| secret_workspace_id: ${{ secrets.WORKSPACE_ID }} | |
| secret_organization_id: ${{ secrets.ORGANIZATION_ID }} | |
| secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }} | |
| secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} | |
| - name: Install dependencies | |
| id: setup | |
| run: | | |
| sudo apt-get install jq | |
| # we need to pre-install the CLI to set the context | |
| curl -sSL https://install.astronomer.io | sudo bash -s | |
| test_start_time=$(date -u +"%Y-%m-%dT%H:%M:%S.%NZ") | |
| echo "test_start_time=$test_start_time" >> $GITHUB_OUTPUT | |
| - name: Set CLI context | |
| run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Create Deployment Preview | |
| id: create-deployment-preview | |
| uses: ./ | |
| with: | |
| deployment-id: ${{ matrix.deployment_id }} | |
| workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} | |
| action: create-deployment-preview | |
| preview-name: test-preview-${{ matrix.deployment_id }} | |
| root-folder: e2e-setup/astro-project | |
| # TODO: allow copying airflow resources, once we add wait in the deploy action for deployment preview to be healthy | |
| copy-connections: false | |
| copy-airflow-variables: false | |
| copy-pools: false | |
| - name: Get Deployment Info After Test | |
| id: get-deployment-after-create-preview | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| deployment_id: ${{ steps.create-deployment-preview.outputs.preview-id }} | |
| organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} | |
| astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} | |
| astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Validate Deploy Action | |
| id: validate-deployment-preview-create | |
| uses: ./.github/workflows/e2e/validate_deployment | |
| with: | |
| dag_tarball_version_before: "" | |
| image_version_before: "12.1.1" | |
| dag_tarball_version_after: ${{ steps.get-deployment-after-create-preview.outputs.desired_dag_tarball_version }} | |
| image_version_after: ${{ steps.get-deployment-after-create-preview.outputs.desired_image_version }} | |
| - name: Mock git commands | |
| run: | | |
| mv e2e-setup/mocks/image-deploy-git.sh /usr/local/bin/git | |
| chmod +x /usr/local/bin/git | |
| - name: Get Deployment Info Before Test | |
| id: get-deployment-before-deploy | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| deployment_id: ${{ steps.create-deployment-preview.outputs.preview-id }} | |
| organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} | |
| astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} | |
| astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Deploy to Deployment Preview | |
| id: deployment-preview-deploy | |
| uses: ./ | |
| with: | |
| deployment-id: ${{ steps.create-deployment-preview.outputs.preview-id }} | |
| workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} | |
| action: deploy-deployment-preview | |
| deploy-type: image-and-dags | |
| root-folder: e2e-setup/astro-project | |
| preview-name: test-preview-${{ matrix.deployment_id }} | |
| - name: Get Deployment Info After Test | |
| id: get-deployment-after-deploy-preview | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| deployment_id: ${{ steps.deployment-preview-deploy.outputs.preview-id }} | |
| organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} | |
| astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} | |
| astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Validate Deploy Action | |
| id: validate-deployment-preview-deploy | |
| uses: ./.github/workflows/e2e/validate_deployment | |
| with: | |
| dag_tarball_version_before: ${{ steps.get-deployment-before-deploy.outputs.desired_dag_tarball_version }} | |
| image_version_before: ${{ steps.get-deployment-before-deploy.outputs.desired_image_version }} | |
| dag_tarball_version_after: ${{ steps.get-deployment-after-deploy-preview.outputs.desired_dag_tarball_version }} | |
| image_version_after: ${{ steps.get-deployment-after-deploy-preview.outputs.desired_image_version }} | |
| - name: Mock git commands for DAG Deploy | |
| run: | | |
| mv e2e-setup/mocks/dag-deploy-git.sh /usr/local/bin/git | |
| chmod +x /usr/local/bin/git | |
| - name: Get Deployment Info Before DAG Deploy Test | |
| id: get-deployment-before-dag-deploy | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| deployment_id: ${{ steps.create-deployment-preview.outputs.preview-id }} | |
| organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} | |
| astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} | |
| astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: DAG Deploy to Astro | |
| id: deployment-preview-dag-deploy | |
| uses: ./ | |
| with: | |
| deployment-id: ${{ steps.create-deployment-preview.outputs.preview-id }} | |
| workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} | |
| action: deploy-deployment-preview | |
| parse: true | |
| deploy-type: dags-only | |
| root-folder: e2e-setup/astro-project | |
| preview-name: test-preview-${{ matrix.deployment_id }} | |
| - name: Get Deployment Info After DAG Deploy Test | |
| id: get-deployment-after-dag-deploy | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| deployment_id: ${{ steps.deployment-preview-dag-deploy.outputs.preview-id }} | |
| organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} | |
| astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} | |
| astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Validate DAG Deploy Action | |
| id: validate-dag-deploy-action | |
| uses: ./.github/workflows/e2e/validate_deployment | |
| with: | |
| is_dag_only_deploy: true | |
| dag_tarball_version_before: ${{ steps.get-deployment-before-dag-deploy.outputs.desired_dag_tarball_version }} | |
| image_version_before: ${{ steps.get-deployment-before-dag-deploy.outputs.desired_image_version }} | |
| dag_tarball_version_after: ${{ steps.get-deployment-after-dag-deploy.outputs.desired_dag_tarball_version }} | |
| image_version_after: ${{ steps.get-deployment-after-dag-deploy.outputs.desired_image_version }} | |
| - name: Mock git commands for DBT Deploy | |
| run: | | |
| mv e2e-setup/mocks/dbt-deploy-git.sh /usr/local/bin/git | |
| chmod +x /usr/local/bin/git | |
| - name: DBT Deploy to Astro | |
| id: deployment-preview-dbt-deploy | |
| uses: ./ | |
| with: | |
| deployment-id: ${{ steps.create-deployment-preview.outputs.preview-id }} | |
| workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} | |
| action: deploy-deployment-preview | |
| deploy-type: dbt | |
| description: "test-dbt-deploy-action" | |
| root-folder: e2e-setup/dbt | |
| preview-name: test-preview-${{ matrix.deployment_id }} | |
| - name: Get DBT bundle info | |
| id: get-dbt-bundle-info | |
| uses: ./.github/workflows/e2e/get_bundle_info | |
| with: | |
| deployment_id: ${{ steps.deployment-preview-dag-deploy.outputs.preview-id }} | |
| organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} | |
| astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} | |
| astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Validate DBT Deploy Action | |
| id: validate-dbt-deploy-action | |
| shell: bash | |
| run: | | |
| if [[ "${{ steps.get-dbt-bundle-info.outputs.desired_bundle_version }}" == "" ]]; then | |
| echo "DBT Deploy Action validation failed: DBT bundle not found" | |
| exit 1 | |
| fi | |
| updated_at=$(date -u -d "${{ steps.get-dbt-bundle-info.outputs.updated_at }}" +"%s.%N") | |
| test_start_time=$(date -u -d "${{ steps.setup.outputs.test_start_time }}" +"%s.%N") | |
| if [[ $(echo "$updated_at > $test_start_time" | bc -l) ]]; then | |
| echo "DBT Deploy Action validation succeeded: DBT bundle found" | |
| exit 0 | |
| fi | |
| echo "DBT Deploy Action validation failed: DBT bundle was not updated" | |
| exit 1 | |
| - name: Delete Deployment Preview | |
| id: delete-deployment-preview | |
| # ensure that we always try delete the deployment preview even if the previous steps fail | |
| if: always() | |
| uses: ./ | |
| with: | |
| deployment-id: ${{ steps.create-deployment-preview.outputs.preview-id }} | |
| workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} | |
| action: delete-deployment-preview | |
| preview-name: test-preview-${{ matrix.deployment_id }} | |
| - name: Validate Deployment Preview Deletion | |
| id: validate-deployment-preview-deletion | |
| # ensure that we always try delete the deployment preview even if the previous steps fail | |
| if: always() | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| expected_status_code: 404 | |
| deployment_id: ${{ steps.create-deployment-preview.outputs.preview-id }} | |
| organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} | |
| astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} | |
| astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| delete-test-deployments: | |
| name: Delete Deployment | |
| runs-on: ubuntu-latest | |
| needs: | |
| [ | |
| create-test-deployments, | |
| dag-deploy-test, | |
| pytests-test, | |
| custom-docker-image-test, | |
| dbt-deploy-test, | |
| deployment-preview-test, | |
| ] | |
| # ensure that we always try delete the deployment even if any of the tests fail | |
| if: always() | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Get Astro Environment Info | |
| id: get-astro-env-info | |
| uses: ./.github/workflows/e2e/get_astro_env_info | |
| with: | |
| input_workspace_id: ${{ github.event.inputs.workspace_id }} | |
| input_organization_id: ${{ github.event.inputs.org_id }} | |
| input_astronomer_host: ${{ github.event.inputs.astronomer_host }} | |
| input_astro_api_token: ${{ github.event.inputs.token }} | |
| secret_workspace_id: ${{ secrets.WORKSPACE_ID }} | |
| secret_organization_id: ${{ secrets.ORGANIZATION_ID }} | |
| secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }} | |
| secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} | |
| - name: Install Astro CLI and set context | |
| run: | | |
| curl -sSL https://install.astronomer.io | sudo bash -s | |
| astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| astro workspace switch ${{ steps.get-astro-env-info.outputs.workspace_id }} | |
| - name: Delete Deployment | |
| run: | | |
| astro deployment delete -f ${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }} |