Upgrade v14

.dockerignore

@@ -1,6 +1,11 @@
 **/node_modules/
-datahub-frontend/build/
-metadata-ingestion/venv/
+*/build/
+*/*/build/
+**/venv/
+**/.tox/
+**/.mypy_cache/
+**/.pytest_cache/
+**/__pycache__/
 out
 **/*.class
 # Have to copy gradle/wrapper/gradle-wrapper.jar, can't exclude ALL jars

.github/actions/ci-optimization/action.yml

@@ -0,0 +1,92 @@
+name: 'Identify CI Optimizations'
+description: 'Determine if code changes are specific to certain modules.'
+
+outputs:
+ frontend-only:
+ description: "Frontend only change"
+ value: ${{ steps.filter.outputs.frontend == 'true' && steps.filter.outputs.ingestion == 'false' && steps.filter.outputs.backend == 'false' }}
+ ingestion-only:
+ description: "Ingestion only change"
+ value: ${{ steps.filter.outputs.frontend == 'false' && steps.filter.outputs.ingestion == 'true' && steps.filter.outputs.backend == 'false' }}
+ backend-only:
+ description: "Backend only change"
+ value: ${{ steps.filter.outputs.frontend == 'false' && steps.filter.outputs.ingestion == 'false' && steps.filter.outputs.backend == 'true' }}
+ backend-change:
+ description: "Backend code has changed"
+ value: ${{ steps.filter.outputs.backend == 'true' }}
+ ingestion-change:
+ description: "Ingestion code has changed"
+ value: ${{ steps.filter.outputs.ingestion == 'true' }}
+ ingestion-base-change:
+ description: "Ingestion base image docker image has changed"
+ value: ${{ steps.filter.outputs.ingestion-base == 'true' }}
+ frontend-change:
+ description: "Frontend code has changed"
+ value: ${{ steps.filter.outputs.frontend == 'true' }}
+ docker-change:
+ description: "Docker code has changed"
+ value: ${{ steps.filter.outputs.docker == 'true' }}
+ kafka-setup-change:
+ description: "Kafka setup docker change"
+ value: ${{ steps.filter.outputs.kafka-setup == 'true' }}
+ mysql-setup-change:
+ description: "Mysql setup docker change"
+ value: ${{ steps.filter.outputs.mysql-setup == 'true' }}
+ postgres-setup-change:
+ description: "Postgres setup docker change"
+ value: ${{ steps.filter.outputs.postgres-setup == 'true' }}
+ elasticsearch-setup-change:
+ description: "Elasticsearch setup docker change"
+ value: ${{ steps.filter.outputs.elasticsearch-setup == 'true' }}
+ smoke-test-change:
+ description: "Smoke test change"
+ value: ${{ steps.filter.outputs.smoke-test == 'true' }}
+runs:
+ using: "composite"
+ steps:
+ - uses: dorny/paths-filter@v2
+ id: filter
+ with:
+ filters: |
+ frontend:
+ - "datahub-frontend/**"
+ - "datahub-web-react/**"
+ - "smoke-test/tests/cypress/**"
+ - "docker/datahub-frontend/**"
+ ingestion:
+ - "metadata-ingestion-modules/**"
+ - "metadata-ingestion/**"
+ - "metadata-models/**"
+ - "smoke-test/**"
+ - "docker/datahub-ingestion**"
+ ingestion-base:
+ - "docker/datahub-ingestion-base/**"
+ docker:
+ - "docker/**"
+ backend:
+ - ".github/**"
+ - "metadata-models/**"
+ - "datahub-upgrade/**"
+ - "entity-registry/**"
+ - "li-utils/**"
+ - "metadata-auth/**"
+ - "metadata-dao-impl/**"
+ - "metadata-events/**"
+ - "metadata-io/**"
+ - "metadata-jobs/**"
+ - "metadata-service/**"
+ - "metadata-utils/**"
+ - "metadata-operation-context/**"
+ - "datahub-graphql-core/**"
+ - "smoke-test/**"
+ - "docker/**"
+ kafka-setup:
+ - "docker/kafka-setup/**"
+ mysql-setup:
+ - "docker/mysql-setup/**"
+ postgres-setup:
+ - "docker/postgres-setup/**"
+ elasticsearch-setup:
+ - "docker/elasticsearch-setup/**"
+ smoke-test:
+ - "smoke-test/**"

.github/actions/docker-custom-build-and-push/action.yml

@@ -20,7 +20,7 @@ inputs:
  required: false
 
  images:
- # e.g. linkedin/datahub-gms
+ # e.g. acryldata/datahub-gms
  description: "List of Docker images to use as base name for tags"
  required: true
  build-args:
@@ -55,7 +55,7 @@ runs:
 
  # Code for testing the build when not pushing to Docker Hub.
  - name: Build and Load image for testing (if not publishing)
- uses: docker/build-push-action@v3
+ uses: docker/build-push-action@v5
  if: ${{ inputs.publish != 'true' }}
  with:
  context: ${{ inputs.context }}
@@ -70,27 +70,36 @@ runs:
  push: false
  cache-from: type=registry,ref=${{ steps.docker_meta.outputs.tags }}
  cache-to: type=inline
+ - name: Single Tag
+ if: ${{ inputs.publish != 'true' }}
+ shell: bash
+ run: |
+ TAGS="""
+ ${{ steps.docker_meta.outputs.tags }}
+ """
+ echo "SINGLE_TAG=$(echo $TAGS | tr '\n' ' ' | awk -F' ' '{ print $1 }')" >> $GITHUB_OUTPUT
+ id: single_tag
  - name: Upload image locally for testing (if not publishing)
  uses: ishworkh/docker-image-artifact-upload@v1
  if: ${{ inputs.publish != 'true' }}
  with:
- image: ${{ steps.docker_meta.outputs.tags }}
+ image: ${{ steps.single_tag.outputs.SINGLE_TAG }}
 
  # Code for building multi-platform images and pushing to Docker Hub.
  - name: Set up QEMU
- uses: docker/setup-qemu-action@v2
+ uses: docker/setup-qemu-action@v3
  if: ${{ inputs.publish == 'true' }}
  - name: Set up Docker Buildx
- uses: docker/setup-buildx-action@v2
+ uses: docker/setup-buildx-action@v3
  if: ${{ inputs.publish == 'true' }}
  - name: Login to DockerHub
- uses: docker/login-action@v2
+ uses: docker/login-action@v3
  if: ${{ inputs.publish == 'true' }}
  with:
  username: ${{ inputs.username }}
  password: ${{ inputs.password }}
  - name: Build and Push Multi-Platform image
- uses: docker/build-push-action@v3
+ uses: docker/build-push-action@v5
  if: ${{ inputs.publish == 'true' }}
  with:
  context: ${{ inputs.context }}

.github/scripts/check_policies.py

@@ -20,7 +20,7 @@
  elif urn == "urn:li:dataHubPolicy:editor-platform-policy":
  editor_platform_policy_privileges = policy["info"]["privileges"]
  elif urn == "urn:li:dataHubPolicy:7":
- all_user_platform_policy_privilges = policy["info"]["privileges"]
+ all_user_platform_policy_privileges = policy["info"]["privileges"]
  try:
  doc_type = policy["info"]["type"]
  privileges = policy["info"]["privileges"]
@@ -54,10 +54,22 @@
 )
 assert len(diff_policies) == 0, f"Missing privileges for root user are {diff_policies}"
 
-diff_policies = set(editor_platform_policy_privileges).difference(
- set(all_user_platform_policy_privilges)
-)
-assert "MANAGE_POLICIES" not in all_user_platform_policy_privilges
-assert (
- len(diff_policies) == 0
-), f"Missing privileges for all user policies are {diff_policies}"
+# All users privileges checks
+assert "MANAGE_POLICIES" not in all_user_platform_policy_privileges
+assert "MANAGE_USERS_AND_GROUPS" not in all_user_platform_policy_privileges
+assert "MANAGE_SECRETS" not in all_user_platform_policy_privileges
+assert "MANAGE_USER_CREDENTIALS" not in all_user_platform_policy_privileges
+assert "MANAGE_ACCESS_TOKENS" not in all_user_platform_policy_privileges
+assert "EDIT_ENTITY" not in all_user_platform_policy_privileges
+assert "DELETE_ENTITY" not in all_user_platform_policy_privileges
+
+# Editor checks
+assert "MANAGE_POLICIES" not in editor_platform_policy_privileges
+assert "MANAGE_USERS_AND_GROUPS" not in editor_platform_policy_privileges
+assert "MANAGE_SECRETS" not in editor_platform_policy_privileges
+assert "MANAGE_USER_CREDENTIALS" not in editor_platform_policy_privileges
+assert "MANAGE_ACCESS_TOKENS" not in editor_platform_policy_privileges
+# These don't prevent a user from modifying entities they are an asset owner of, i.e. their own profile info
+assert "EDIT_CONTACT_INFO" not in editor_platform_policy_privileges
+assert "EDIT_USER_PROFILE" not in editor_platform_policy_privileges
+assert "EDIT_ENTITY_OWNERS" not in editor_platform_policy_privileges

.github/scripts/check_python_package.py

@@ -0,0 +1,33 @@
+import setuptools
+import os
+
+folders = ["./smoke-test/tests"]
+
+for folder in folders:
+ print(f"Checking folder {folder}")
+ packages = [i for i in setuptools.find_packages(folder) if "cypress" not in i]
+ namespace_packages = [
+ i for i in setuptools.find_namespace_packages(folder) if "cypress" not in i
+ ]
+
+ print("Packages found:", packages)
+ print("Namespace packages found:", namespace_packages)
+
+ in_packages_not_namespace = set(packages) - set(namespace_packages)
+ in_namespace_not_packages = set(namespace_packages) - set(packages)
+
+ if in_packages_not_namespace:
+ print(f"Packages not in namespace packages: {in_packages_not_namespace}")
+ if in_namespace_not_packages:
+ print(f"Namespace packages not in packages: {in_namespace_not_packages}")
+ for pkg in in_namespace_not_packages:
+ pkg_path = os.path.join(folder, pkg.replace(".", os.path.sep))
+ print(f"Contents of {pkg_path}:")
+ print(os.listdir(pkg_path))
+
+ assert (
+ len(in_packages_not_namespace) == 0
+ ), f"Found packages in {folder} that are not in namespace packages: {in_packages_not_namespace}"
+ assert (
+ len(in_namespace_not_packages) == 0
+ ), f"Found namespace packages in {folder} that are not in packages: {in_namespace_not_packages}"

.github/scripts/docker_helpers.sh

@@ -12,29 +12,29 @@ export SHORT_SHA=$(get_short_sha)
 echo "SHORT_SHA: $SHORT_SHA"
 
 function get_tag {
- echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
+ echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g'),${SHORT_SHA}
 }
 
 function get_tag_slim {
- echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-slim\,${SHORT_SHA}-slim,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-slim,g')
+ echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-slim,g" -e 's,refs/tags/\(.*\),\1-slim,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-slim,g'),${SHORT_SHA}-slim
 }
 
 function get_tag_full {
- echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-full\,${SHORT_SHA}-full,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g')
+ echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-full,g" -e 's,refs/tags/\(.*\),\1-full,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g'),${SHORT_SHA}-full
 }
 
 function get_python_docker_release_v {
- echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},0.0.0+docker.${SHORT_SHA},g" -e 's,refs/tags/v\(.*\),\1+docker,g' -e 's,refs/pull/\([0-9]*\).*,0.0.0+docker.pr\1,g')
+ echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},1!0.0.0+docker.${SHORT_SHA},g" -e 's,refs/tags/v\(.*\),1!\1+docker,g' -e 's,refs/pull/\([0-9]*\).*,1!0.0.0+docker.pr\1,g')
 }
 
 function get_unique_tag {
  echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
 }
 
 function get_unique_tag_slim {
- echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA}-slim,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-slim,g')
+ echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA}-slim,g" -e 's,refs/tags/\(.*\),\1-slim,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-slim,g')
 }
 
 function get_unique_tag_full {
- echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA}-full,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g')
-}
+ echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA}-full,g" -e 's,refs/tags/\(.*\),\1-full,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g')
+}

.github/workflows/airflow-plugin.yml

@@ -12,7 +12,7 @@ on:
  branches:
  - "**"
  paths:
- - ".github/**"
+ - ".github/workflows/airflow-plugin.yml"
  - "metadata-ingestion-modules/airflow-plugin/**"
  - "metadata-ingestion/**"
  - "metadata-models/**"
@@ -32,32 +32,45 @@ jobs:
  strategy:
  matrix:
  include:
+ # Note: this should be kept in sync with tox.ini.
  - python-version: "3.8"
  extra_pip_requirements: "apache-airflow~=2.1.4"
  extra_pip_extras: plugin-v1
  - python-version: "3.8"
  extra_pip_requirements: "apache-airflow~=2.2.4"
  extra_pip_extras: plugin-v1
  - python-version: "3.10"
- extra_pip_requirements: "apache-airflow~=2.4.0"
+ extra_pip_requirements: "apache-airflow~=2.4.3"
+ extra_pip_extras: plugin-v2,test-airflow24
+ - python-version: "3.10"
+ extra_pip_requirements: 'apache-airflow~=2.6.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-3.10.txt'
  extra_pip_extras: plugin-v2
  - python-version: "3.10"
- extra_pip_requirements: "apache-airflow~=2.6.0"
+ extra_pip_requirements: 'apache-airflow~=2.7.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-3.10.txt'
  extra_pip_extras: plugin-v2
  - python-version: "3.10"
- extra_pip_requirements: "apache-airflow>=2.7.0"
+ extra_pip_requirements: 'apache-airflow~=2.8.1 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.10.txt'
+ extra_pip_extras: plugin-v2
+ - python-version: "3.11"
+ extra_pip_requirements: 'apache-airflow~=2.9.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.11.txt'
  extra_pip_extras: plugin-v2
  fail-fast: false
  steps:
- - uses: actions/checkout@v3
+ - name: Set up JDK 17
+ uses: actions/setup-java@v3
+ with:
+ distribution: "zulu"
+ java-version: 17
+ - uses: gradle/actions/setup-gradle@v3
+ - uses: acryldata/sane-checkout-action@v3
  - uses: actions/setup-python@v4
  with:
  python-version: ${{ matrix.python-version }}
  cache: "pip"
  - name: Install dependencies
  run: ./metadata-ingestion/scripts/install_deps.sh
  - name: Install airflow package and test (extras ${{ matrix.extra_pip_requirements }})
- run: ./gradlew -Pextra_pip_requirements='${{ matrix.extra_pip_requirements }}' -Pextra_pip_extras='${{ matrix.extra_pip_extras }}' :metadata-ingestion-modules:airflow-plugin:lint :metadata-ingestion-modules:airflow-plugin:testQuick
+ run: ./gradlew -Pextra_pip_requirements='${{ matrix.extra_pip_requirements }}' -Pextra_pip_extras='${{ matrix.extra_pip_extras }}' :metadata-ingestion-modules:airflow-plugin:build
  - name: pip freeze show list installed
  if: always()
  run: source metadata-ingestion-modules/airflow-plugin/venv/bin/activate && pip freeze
@@ -69,15 +82,16 @@ jobs:
  **/build/reports/tests/test/**
  **/build/test-results/test/**
  **/junit.*.xml
+ !**/binary/**
  - name: Upload coverage to Codecov
  if: always()
  uses: codecov/codecov-action@v3
  with:
  token: ${{ secrets.CODECOV_TOKEN }}
  directory: .
  fail_ci_if_error: false
- flags: airflow-${{ matrix.python-version }}-${{ matrix.extraPythonRequirement }}
- name: pytest-airflow
+ flags: airflow,airflow-${{ matrix.extra_pip_extras }}
+ name: pytest-airflow-${{ matrix.python-version }}-${{ matrix.extra_pip_requirements }}
  verbose: true
 
  event-file: