ci: make wheel builds more robust (#8815)

datahub-project · Sep 12, 2023 · 449cc9b · 449cc9b
1 parent f7fee74
commit 449cc9b
Show file tree

Hide file tree

Showing 5 changed files with 22 additions and 15 deletions.
diff --git a/docs-website/sphinx/Makefile b/docs-website/sphinx/Makefile
@@ -22,7 +22,7 @@ $(VENV_SENTINEL): requirements.txt
  $(VENV_DIR)/bin/pip install -r requirements.txt
  touch $(VENV_SENTINEL)
 
-.PHONY: help html doctest linkcheck clean serve md
+.PHONY: help html doctest linkcheck clean clean_all serve md
 
 # Not using Python's http.server because it enables caching headers.
 serve:
@@ -35,3 +35,6 @@ md: html
 # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
 html doctest linkcheck clean: venv Makefile
  @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+clean_all: clean
+ -rm -rf $(VENV_DIR)
diff --git a/docs-website/sphinx/requirements.txt b/docs-website/sphinx/requirements.txt
@@ -1,4 +1,4 @@
--e ../../metadata-ingestion[datahub-rest]
+-e ../../metadata-ingestion[datahub-rest,sql-parsing]
 beautifulsoup4==4.11.2
 Sphinx==6.1.3
 sphinx-click==4.4.0

diff --git a/docs-website/yarn.lock b/docs-website/yarn.lock
@@ -2986,6 +2986,13 @@
  dependencies:
  "@types/node" "*"
 
+"@types/websocket@^1.0.3":
+ version "1.0.6"
+ resolved "https://registry.yarnpkg.com/@types/websocket/-/websocket-1.0.6.tgz#ec8dce5915741632ac3a4b1f951b6d4156e32d03"
+ integrity sha512-JXkliwz93B2cMWOI1ukElQBPN88vMg3CruvW4KVSKpflt3NyNCJImnhIuB/f97rG7kakqRJGFiwkA895Kn02Dg==
+ dependencies:
+ "@types/node" "*"
+
 "@types/ws@^8.5.5":
  version "8.5.5"
  resolved "https://registry.yarnpkg.com/@types/ws/-/ws-8.5.5.tgz#af587964aa06682702ee6dcbc7be41a80e4b28eb"
@@ -7053,7 +7060,6 @@ node-forge@^1:
  resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3"
  integrity sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==
 
-
 node-gyp-build@^4.3.0:
  version "4.6.1"
  resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.6.1.tgz#24b6d075e5e391b8d5539d98c7fc5c210cac8a3e"
@@ -9903,6 +9909,10 @@ use-sidecar@^1.1.2:
  detect-node-es "^1.1.0"
  tslib "^2.0.0"
 
+use-sync-external-store@^1.2.0:
+ version "1.2.0"
+ resolved "https://registry.yarnpkg.com/use-sync-external-store/-/use-sync-external-store-1.2.0.tgz#7dbefd6ef3fe4e767a0cf5d7287aacfb5846928a"
+ integrity sha512-eEgnFxGQ1Ife9bzYs6VLi8/4X6CObHMw9Qr9tPY43iKwsPw8xE8+EFsf/2cFZ5S3esXgpWgtSCtLNS41F+sKPA==
 
 utf-8-validate@^5.0.2:
  version "5.0.10"
@@ -9911,12 +9921,6 @@ utf-8-validate@^5.0.2:
  dependencies:
  node-gyp-build "^4.3.0"
 
-use-sync-external-store@^1.2.0:
- version "1.2.0"
- resolved "https://registry.yarnpkg.com/use-sync-external-store/-/use-sync-external-store-1.2.0.tgz#7dbefd6ef3fe4e767a0cf5d7287aacfb5846928a"
- integrity sha512-eEgnFxGQ1Ife9bzYs6VLi8/4X6CObHMw9Qr9tPY43iKwsPw8xE8+EFsf/2cFZ5S3esXgpWgtSCtLNS41F+sKPA==
-
-
 util-deprecate@^1.0.1, util-deprecate@^1.0.2, util-deprecate@~1.0.1:
  version "1.0.2"
  resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"

diff --git a/metadata-ingestion-modules/airflow-plugin/build.gradle b/metadata-ingestion-modules/airflow-plugin/build.gradle
@@ -110,14 +110,14 @@ task testFull(type: Exec, dependsOn: [testQuick, installDevTest]) {
  commandLine 'bash', '-x', '-c',
  "source ${venv_name}/bin/activate && pytest -m 'not slow_integration' -vv --continue-on-collection-errors --junit-xml=junit.full.xml"
 }
-task buildWheel(type: Exec, dependsOn: [install]) {
- commandLine 'bash', '-c', "source ${venv_name}/bin/activate && " + 'pip install build && RELEASE_VERSION="\${RELEASE_VERSION:-0.0.0.dev1}" RELEASE_SKIP_TEST=1 RELEASE_SKIP_UPLOAD=1 ./scripts/release.sh'
-}
 
 task cleanPythonCache(type: Exec) {
  commandLine 'bash', '-c',
  "find src -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete -o -type d -empty -delete"
 }
+task buildWheel(type: Exec, dependsOn: [install, cleanPythonCache]) {
+ commandLine 'bash', '-c', "source ${venv_name}/bin/activate && " + 'pip install build && RELEASE_VERSION="\${RELEASE_VERSION:-0.0.0.dev1}" RELEASE_SKIP_TEST=1 RELEASE_SKIP_UPLOAD=1 ./scripts/release.sh'
+}
 
 build.dependsOn install
 check.dependsOn lint

diff --git a/metadata-ingestion/build.gradle b/metadata-ingestion/build.gradle
@@ -185,16 +185,16 @@ task specGen(type: Exec, dependsOn: [codegen, installDevTest]) {
 task docGen(type: Exec, dependsOn: [codegen, installDevTest, specGen]) {
  commandLine 'bash', '-c', "source ${venv_name}/bin/activate && ./scripts/docgen.sh"
 }
-task buildWheel(type: Exec, dependsOn: [install, codegen]) {
- commandLine 'bash', '-c', "source ${venv_name}/bin/activate && " + 'pip install build && RELEASE_VERSION="\${RELEASE_VERSION:-0.0.0.dev1}" RELEASE_SKIP_TEST=1 RELEASE_SKIP_UPLOAD=1 ./scripts/release.sh'
-}
 
 
 
 task cleanPythonCache(type: Exec) {
  commandLine 'bash', '-c',
  "find src tests -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete -o -type d -empty -delete"
 }
+task buildWheel(type: Exec, dependsOn: [install, codegen, cleanPythonCache]) {
+ commandLine 'bash', '-c', "source ${venv_name}/bin/activate && " + 'pip install build && RELEASE_VERSION="\${RELEASE_VERSION:-0.0.0.dev1}" RELEASE_SKIP_TEST=1 RELEASE_SKIP_UPLOAD=1 ./scripts/release.sh'
+}
 
 build.dependsOn install
 check.dependsOn lint