From 4cf967426b3fbf914c800cd79f937cc51ac2a236 Mon Sep 17 00:00:00 2001 From: Daniel Mateus Pires Date: Thu, 15 Apr 2021 10:24:27 +0100 Subject: [PATCH 01/14] =?UTF-8?q?=F0=9F=94=A8=20Extend=20git=20package=20c?= =?UTF-8?q?ontract=20and=20signatures=20to=20pass=20`subdirectory`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/dbt/clients/git.py | 13 +++++++++---- core/dbt/contracts/project.py | 1 + core/dbt/deps/git.py | 15 +++++++++------ 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/core/dbt/clients/git.py b/core/dbt/clients/git.py index 583c8a09faf..db8bacaf4a5 100644 --- a/core/dbt/clients/git.py +++ b/core/dbt/clients/git.py @@ -11,7 +11,7 @@ def _is_commit(revision: str) -> bool: return bool(re.match(r"\b[0-9a-f]{40}\b", revision)) -def clone(repo, cwd, dirname=None, remove_git_dir=False, revision=None): +def clone(repo, cwd, dirname=None, remove_git_dir=False, revision=None, subdirectory=None): has_revision = revision is not None is_commit = _is_commit(revision or "") @@ -84,11 +84,16 @@ def remove_remote(cwd): def clone_and_checkout(repo, cwd, dirname=None, remove_git_dir=False, - revision=None): + revision=None, subdirectory=None): exists = None try: - _, err = clone(repo, cwd, dirname=dirname, - remove_git_dir=remove_git_dir) + _, err = clone( + repo, + cwd, + dirname=dirname, + remove_git_dir=remove_git_dir, + subdirectory=subdirectory, + ) except dbt.exceptions.CommandResultError as exc: err = exc.stderr.decode('utf-8') exists = re.match("fatal: destination path '(.+)' already exists", err) diff --git a/core/dbt/contracts/project.py b/core/dbt/contracts/project.py index 18ba456b510..b3649cc17ab 100644 --- a/core/dbt/contracts/project.py +++ b/core/dbt/contracts/project.py @@ -70,6 +70,7 @@ class GitPackage(Package): git: str revision: Optional[RawVersion] = None warn_unpinned: Optional[bool] = None + subdirectory: Optional[str] = None def get_revisions(self) -> List[str]: if self.revision is None: diff --git a/core/dbt/deps/git.py b/core/dbt/deps/git.py index caed8df7164..3f4989cf6b2 100644 --- a/core/dbt/deps/git.py +++ b/core/dbt/deps/git.py @@ -1,6 +1,6 @@ import os import hashlib -from typing import List +from typing import List, Optional from dbt.clients import git, system from dbt.config import Project @@ -37,11 +37,12 @@ def source_type(self) -> str: class GitPinnedPackage(GitPackageMixin, PinnedPackage): def __init__( - self, git: str, revision: str, warn_unpinned: bool = True + self, git: str, revision: str, warn_unpinned: bool = True, subdirectory: Optional[str] = None ) -> None: super().__init__(git) self.revision = revision self.warn_unpinned = warn_unpinned + self.subdirectory = subdirectory self._checkout_name = md5sum(self.git) def get_version(self): @@ -69,7 +70,7 @@ def _checkout(self): try: dir_ = git.clone_and_checkout( self.git, get_downloads_path(), revision=self.revision, - dirname=self._checkout_name + dirname=self._checkout_name, subdirectory=self.subdirectory ) except ExecutableError as exc: if exc.cmd and exc.cmd[0] == 'git': @@ -107,11 +108,12 @@ def install(self, project, renderer): class GitUnpinnedPackage(GitPackageMixin, UnpinnedPackage[GitPinnedPackage]): def __init__( - self, git: str, revisions: List[str], warn_unpinned: bool = True + self, git: str, revisions: List[str], warn_unpinned: bool = True, subdirectory: Optional[str] = None ) -> None: super().__init__(git) self.revisions = revisions self.warn_unpinned = warn_unpinned + self.subdirectory = subdirectory @classmethod def from_contract( @@ -122,7 +124,7 @@ def from_contract( # we want to map None -> True warn_unpinned = contract.warn_unpinned is not False return cls(git=contract.git, revisions=revisions, - warn_unpinned=warn_unpinned) + warn_unpinned=warn_unpinned, subdirectory=contract.subdirectory) def all_names(self) -> List[str]: if self.git.endswith('.git'): @@ -140,6 +142,7 @@ def incorporate( git=self.git, revisions=self.revisions + other.revisions, warn_unpinned=warn_unpinned, + subdirectory=self.subdirectory, ) def resolved(self) -> GitPinnedPackage: @@ -153,5 +156,5 @@ def resolved(self) -> GitPinnedPackage: return GitPinnedPackage( git=self.git, revision=requested.pop(), - warn_unpinned=self.warn_unpinned + warn_unpinned=self.warn_unpinned, subdirectory=self.subdirectory ) From 032d2b278737c6b3d7cb58d1e3f49864fd78b94d Mon Sep 17 00:00:00 2001 From: Daniel Mateus Pires Date: Thu, 15 Apr 2021 11:46:20 +0100 Subject: [PATCH 02/14] Add sparse checkout logic --- core/dbt/clients/git.py | 18 ++++++++++++++++-- core/dbt/deps/base.py | 2 ++ core/dbt/deps/git.py | 3 +++ core/dbt/task/deps.py | 5 ++++- 4 files changed, 25 insertions(+), 3 deletions(-) diff --git a/core/dbt/clients/git.py b/core/dbt/clients/git.py index db8bacaf4a5..616b42eb865 100644 --- a/core/dbt/clients/git.py +++ b/core/dbt/clients/git.py @@ -4,6 +4,7 @@ from dbt.clients.system import run_cmd, rmdir from dbt.logger import GLOBAL_LOGGER as logger import dbt.exceptions +from packaging import version def _is_commit(revision: str) -> bool: @@ -16,6 +17,17 @@ def clone(repo, cwd, dirname=None, remove_git_dir=False, revision=None, subdirec is_commit = _is_commit(revision or "") clone_cmd = ['git', 'clone', '--depth', '1'] + if subdirectory: + logger.debug(' Subdirectory specified: {}, using sparse checkout.'.format(subdirectory)) + out, _ = run_cmd(cwd, ['git', '--version'], env={'LC_ALL': 'C'}) + git_version = version.parse(re.search(r"\d+\.\d+\.\d+", out.decode("utf-8")).group(0)) + if not git_version >= version.parse("2.25.0"): + # 2.25.0 introduces --sparse + raise RuntimeError( + "Please update your git version to pull a dbt package " + "from a subdirectory: your version is {}, >= 2.25.0 needed".format(git_version) + ) + clone_cmd.extend(['--filter=blob:none', '--sparse']) if has_revision and not is_commit: clone_cmd.extend(['--branch', revision]) @@ -24,9 +36,11 @@ def clone(repo, cwd, dirname=None, remove_git_dir=False, revision=None, subdirec if dirname is not None: clone_cmd.append(dirname) - result = run_cmd(cwd, clone_cmd, env={'LC_ALL': 'C'}) + if subdirectory: + run_cmd(os.path.join(cwd, dirname), ['git', 'sparse-checkout', 'set', subdirectory]) + if remove_git_dir: rmdir(os.path.join(dirname, '.git')) @@ -125,4 +139,4 @@ def clone_and_checkout(repo, cwd, dirname=None, remove_git_dir=False, start_sha[:7], end_sha[:7]) else: logger.debug(' Checked out at {}.', end_sha[:7]) - return directory + return os.path.join(directory, subdirectory) diff --git a/core/dbt/deps/base.py b/core/dbt/deps/base.py index 0baa03ed86d..37eb37ce40a 100644 --- a/core/dbt/deps/base.py +++ b/core/dbt/deps/base.py @@ -93,6 +93,8 @@ def get_installation_path(self, project, renderer): dest_dirname = self.get_project_name(project, renderer) return os.path.join(project.modules_path, dest_dirname) + def get_subdirectory(self): + return None SomePinned = TypeVar('SomePinned', bound=PinnedPackage) SomeUnpinned = TypeVar('SomeUnpinned', bound='UnpinnedPackage') diff --git a/core/dbt/deps/git.py b/core/dbt/deps/git.py index 3f4989cf6b2..b15eda4eaea 100644 --- a/core/dbt/deps/git.py +++ b/core/dbt/deps/git.py @@ -48,6 +48,9 @@ def __init__( def get_version(self): return self.revision + def get_subdirectory(self): + return self.subdirectory + def nice_version_name(self): if self.revision == 'HEAD': return 'HEAD (default revision)' diff --git a/core/dbt/task/deps.py b/core/dbt/task/deps.py index f6979295cb9..c93dd680204 100644 --- a/core/dbt/task/deps.py +++ b/core/dbt/task/deps.py @@ -59,8 +59,11 @@ def run(self): for package in final_deps: logger.info('Installing {}', package) package.install(self.config, renderer) - logger.info(' Installed from {}\n', + logger.info(' Installed from {}', package.nice_version_name()) + if package.get_subdirectory(): + logger.info(' and subdirectory {}\n', + package.get_subdirectory()) self.track_package_install( package_name=package.name, From 82527fe688b947b4c76fc47926c64eb7c0536e1e Mon Sep 17 00:00:00 2001 From: Daniel Mateus Pires Date: Thu, 15 Apr 2021 12:04:13 +0100 Subject: [PATCH 03/14] =?UTF-8?q?=E2=9C=85=20Add=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/rpc/test_deps.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/test/rpc/test_deps.py b/test/rpc/test_deps.py index d1c1bfa514a..e2b3713c4a5 100644 --- a/test/rpc/test_deps.py +++ b/test/rpc/test_deps.py @@ -87,10 +87,19 @@ def test_rpc_deps_packages(project_root, profiles_root, dbt_profile, unique_sche @pytest.mark.supported('postgres') def test_rpc_deps_git(project_root, profiles_root, dbt_profile, unique_schema): - packages = [{ - 'git': 'https://github.com/fishtown-analytics/dbt-utils.git', - 'revision': '0.5.0' - }] + packages = [ + { + 'git': 'https://github.com/fishtown-analytics/dbt-utils.git', + 'revision': '0.5.0' + }, + # TODO: Change me to something that fishtown-analytics manages! + # here I just moved the dbt_utils code into a subdirectory + { + 'git': 'https://github.com/dmateusp/dbt-utils.git', + 'revision': 'dmateusp/move_dbt_utils_to_subdir', + 'subdirectory': 'dbt_projects/dbt_utils' + }, + ] # if you use a bad URL, git thinks it's a private repo and prompts for auth bad_packages = [{ 'git': 'https://github.com/fishtown-analytics/dbt-utils.git', From db1f5e90519c26193e772b3504e11c5838489216 Mon Sep 17 00:00:00 2001 From: Daniel Mateus Pires Date: Thu, 15 Apr 2021 12:04:28 +0100 Subject: [PATCH 04/14] =?UTF-8?q?=F0=9F=A7=B9=20Lint?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/dbt/deps/base.py | 1 + core/dbt/deps/git.py | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/core/dbt/deps/base.py b/core/dbt/deps/base.py index 37eb37ce40a..0c9cd4b3752 100644 --- a/core/dbt/deps/base.py +++ b/core/dbt/deps/base.py @@ -96,6 +96,7 @@ def get_installation_path(self, project, renderer): def get_subdirectory(self): return None + SomePinned = TypeVar('SomePinned', bound=PinnedPackage) SomeUnpinned = TypeVar('SomeUnpinned', bound='UnpinnedPackage') diff --git a/core/dbt/deps/git.py b/core/dbt/deps/git.py index b15eda4eaea..c1470794c5b 100644 --- a/core/dbt/deps/git.py +++ b/core/dbt/deps/git.py @@ -37,7 +37,11 @@ def source_type(self) -> str: class GitPinnedPackage(GitPackageMixin, PinnedPackage): def __init__( - self, git: str, revision: str, warn_unpinned: bool = True, subdirectory: Optional[str] = None + self, + git: str, + revision: str, + warn_unpinned: bool = True, + subdirectory: Optional[str] = None, ) -> None: super().__init__(git) self.revision = revision @@ -111,7 +115,11 @@ def install(self, project, renderer): class GitUnpinnedPackage(GitPackageMixin, UnpinnedPackage[GitPinnedPackage]): def __init__( - self, git: str, revisions: List[str], warn_unpinned: bool = True, subdirectory: Optional[str] = None + self, + git: str, + revisions: List[str], + warn_unpinned: bool = True, + subdirectory: Optional[str] = None, ) -> None: super().__init__(git) self.revisions = revisions From b808648b42a37e9a68deed32860989f9259e9e44 Mon Sep 17 00:00:00 2001 From: Daniel Mateus Pires Date: Thu, 15 Apr 2021 12:13:41 +0100 Subject: [PATCH 05/14] =?UTF-8?q?=E2=9C=8F=EF=B8=8F=20Update=20CHANGELOG?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index dfc7f4a41f6..14b6fcce7b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ ### Features - Support commit hashes in dbt deps package revision ([#3268](https://github.com/fishtown-analytics/dbt/issues/3268), [#3270](https://github.com/fishtown-analytics/dbt/pull/3270)) +- Add optional `subdirectory` key to install dbt packages that are not hosted at the root of a Git repository ([#275](https://github.com/fishtown-analytics/dbt/issues/275), [#3267](https://github.com/fishtown-analytics/dbt/pull/3267)) - Add optional configs for `require_partition_filter` and `partition_expiration_days` in BigQuery ([#1843](https://github.com/fishtown-analytics/dbt/issues/1843), [#2928](https://github.com/fishtown-analytics/dbt/pull/2928)) - Fix for EOL SQL comments prevent entire line execution ([#2731](https://github.com/fishtown-analytics/dbt/issues/2731), [#2974](https://github.com/fishtown-analytics/dbt/pull/2974)) - Add optional `merge_update_columns` config to specify columns to update for `merge` statements in BigQuery and Snowflake ([#1862](https://github.com/fishtown-analytics/dbt/issues/1862), [#3100](https://github.com/fishtown-analytics/dbt/pull/3100)) From 1987374139d1dbe258773013f41b23b8c4d320b4 Mon Sep 17 00:00:00 2001 From: Daniel Mateus Pires Date: Thu, 15 Apr 2021 13:57:09 +0100 Subject: [PATCH 06/14] =?UTF-8?q?=F0=9F=90=9B=20Make=20os.path.join=20safe?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/dbt/clients/git.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/dbt/clients/git.py b/core/dbt/clients/git.py index 616b42eb865..ba3bf2e1330 100644 --- a/core/dbt/clients/git.py +++ b/core/dbt/clients/git.py @@ -39,7 +39,7 @@ def clone(repo, cwd, dirname=None, remove_git_dir=False, revision=None, subdirec result = run_cmd(cwd, clone_cmd, env={'LC_ALL': 'C'}) if subdirectory: - run_cmd(os.path.join(cwd, dirname), ['git', 'sparse-checkout', 'set', subdirectory]) + run_cmd(os.path.join(cwd, dirname or ''), ['git', 'sparse-checkout', 'set', subdirectory]) if remove_git_dir: rmdir(os.path.join(dirname, '.git')) @@ -139,4 +139,4 @@ def clone_and_checkout(repo, cwd, dirname=None, remove_git_dir=False, start_sha[:7], end_sha[:7]) else: logger.debug(' Checked out at {}.', end_sha[:7]) - return os.path.join(directory, subdirectory) + return os.path.join(directory, subdirectory or '') From 1767b53c7cee84de2c2b5c5e0904ccb26382cebb Mon Sep 17 00:00:00 2001 From: Daniel Mateus Pires Date: Thu, 15 Apr 2021 14:40:19 +0100 Subject: [PATCH 07/14] Use a test-container with an updated `git` version --- .circleci/config.yml | 8 ++++++-- Dockerfile.test | 3 +++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4303a729645..75990cae1ad 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,7 +2,9 @@ version: 2.1 jobs: unit: docker: &test_only - - image: fishtownanalytics/test-container:11 + # TODO: Ask for someone at fishtown to publish + # a new version of the test container + - image: dmateusp/dbt-test-container environment: DBT_INVOCATION_ENV: circle DOCKER_TEST_DATABASE_HOST: "database" @@ -37,7 +39,9 @@ jobs: destination: dist integration-postgres: docker: - - image: fishtownanalytics/test-container:11 + # TODO: Ask for someone at fishtown to publish + # a new version of the test container + - image: dmateusp/dbt-test-container environment: DBT_INVOCATION_ENV: circle DOCKER_TEST_DATABASE_HOST: "database" diff --git a/Dockerfile.test b/Dockerfile.test index 5771fdef30b..26d18cf00e9 100644 --- a/Dockerfile.test +++ b/Dockerfile.test @@ -3,6 +3,9 @@ FROM ubuntu:18.04 ENV DEBIAN_FRONTEND noninteractive RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + software-properties-common \ + && add-apt-repository ppa:git-core/ppa -y \ && apt-get dist-upgrade -y \ && apt-get install -y --no-install-recommends \ netcat \ From 7be1a48c516139d35d602e271d36b1043d0fb7b1 Mon Sep 17 00:00:00 2001 From: Daniel Mateus Pires Date: Thu, 15 Apr 2021 15:59:34 +0100 Subject: [PATCH 08/14] =?UTF-8?q?=F0=9F=94=A8=20Fix=20integration=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/rpc/test_deps.py | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/test/rpc/test_deps.py b/test/rpc/test_deps.py index e2b3713c4a5..959cf9fee6e 100644 --- a/test/rpc/test_deps.py +++ b/test/rpc/test_deps.py @@ -87,19 +87,10 @@ def test_rpc_deps_packages(project_root, profiles_root, dbt_profile, unique_sche @pytest.mark.supported('postgres') def test_rpc_deps_git(project_root, profiles_root, dbt_profile, unique_schema): - packages = [ - { - 'git': 'https://github.com/fishtown-analytics/dbt-utils.git', - 'revision': '0.5.0' - }, - # TODO: Change me to something that fishtown-analytics manages! - # here I just moved the dbt_utils code into a subdirectory - { - 'git': 'https://github.com/dmateusp/dbt-utils.git', - 'revision': 'dmateusp/move_dbt_utils_to_subdir', - 'subdirectory': 'dbt_projects/dbt_utils' - }, - ] + packages = [{ + 'git': 'https://github.com/fishtown-analytics/dbt-utils.git', + 'revision': '0.5.0' + }] # if you use a bad URL, git thinks it's a private repo and prompts for auth bad_packages = [{ 'git': 'https://github.com/fishtown-analytics/dbt-utils.git', @@ -120,3 +111,20 @@ def test_rpc_deps_git_commit(project_root, profiles_root, dbt_profile, unique_sc 'revision': 'b736cf6' }] deps_with_packages(packages, bad_packages, project_root, profiles_root, unique_schema) + + +def test_rpc_deps_git_subdir(project_root, profiles_root, dbt_profile, unique_schema): + # TODO: Change me to something that fishtown-analytics manages! + # here I just moved the dbt_utils code into a subdirectory + packages = [{ + 'git': 'https://github.com/dmateusp/dbt-utils.git', + 'revision': 'dmateusp/0.5.0/move_dbt_utils_to_subdir', + 'subdirectory': 'dbt_projects/dbt_utils' + }] + # + bad_packages = [{ + 'git': 'https://github.com/fishtown-analytics/dbt-utils.git', + 'revision': '0.5.0', + 'subdirectory': 'path/to/nonexistent/dir', + }] + deps_with_packages(packages, bad_packages, project_root, profiles_root, unique_schema) From db40aaf7cfe77982aaf1d2f8e88165c086e9cc19 Mon Sep 17 00:00:00 2001 From: Daniel Mateus Pires Date: Mon, 19 Apr 2021 12:11:57 +0100 Subject: [PATCH 09/14] =?UTF-8?q?=F0=9F=93=96=20Update=20CHANGELOG=20contr?= =?UTF-8?q?ibutors=20to=20include=20this=20PR?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 14b6fcce7b8..5587c3a0d36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,8 +52,12 @@ Contributors: - [@techytushar](https://github.com/techytushar) ([#3158](https://github.com/fishtown-analytics/dbt/pull/3158)) - [@cgopalan](https://github.com/cgopalan) ([#3165](https://github.com/fishtown-analytics/dbt/pull/3165), [#3182](https://github.com/fishtown-analytics/dbt/pull/3182)) - [@fux](https://github.com/fuchsst) ([#3241](https://github.com/fishtown-analytics/dbt/issues/3241)) +<<<<<<< HEAD - [@dmateusp](https://github.com/dmateusp) ([#3270](https://github.com/fishtown-analytics/dbt/pull/3270)) - [@arzavj](https://github.com/arzavj) ([3106](https://github.com/fishtown-analytics/dbt/pull/3106)) +======= +- [@dmateusp](https://github.com/dmateusp) ([#3270](https://github.com/fishtown-analytics/dbt/pull/3270), [#3267](https://github.com/fishtown-analytics/dbt/pull/3267)) +>>>>>>> 591ad65d (📖 Update CHANGELOG contributors to include this PR) ## dbt 0.19.1 (March 31, 2021) From f50c5f195ac35dd32dddb94daf4f3ed75e15857f Mon Sep 17 00:00:00 2001 From: Daniel Mateus Pires Date: Mon, 19 Apr 2021 14:40:24 +0100 Subject: [PATCH 10/14] =?UTF-8?q?=F0=9F=A7=AA=20Parameterize=20the=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/rpc/test_deps.py | 129 ++++++++++++++++++++++++------------------ 1 file changed, 75 insertions(+), 54 deletions(-) diff --git a/test/rpc/test_deps.py b/test/rpc/test_deps.py index 959cf9fee6e..2ac3f7f01ef 100644 --- a/test/rpc/test_deps.py +++ b/test/rpc/test_deps.py @@ -72,59 +72,80 @@ def deps_with_packages(packages, bad_packages, project_dir, profiles_dir, schema querier.is_result(querier.async_wait(tok1)) +@pytest.mark.parametrize( + "packages, bad_packages", + # from dbt hub + [( + [{ + 'package': 'fishtown-analytics/dbt_utils', + 'version': '0.5.0', + }], + # wrong package name + [{ + 'package': 'fishtown-analytics/dbt_util', + 'version': '0.5.0', + }], + ), + # from git release/tag/branch + ( + [{ + 'git': 'https://github.com/fishtown-analytics/dbt-utils.git', + 'revision': '0.5.0', + }], + # if you use a bad URL, git thinks it's a private repo and prompts for auth + [{ + 'git': 'https://github.com/fishtown-analytics/dbt-utils.git', + 'revision': 'not-a-real-revision', + }], + ), + # from git commit + ( + [{ + 'git': 'https://github.com/fishtown-analytics/dbt-utils.git', + 'revision': 'b736cf6acdbf80d2de69b511a51c8d7fe214ee79', + }], + # don't use short commits + [{ + 'git': 'https://github.com/fishtown-analytics/dbt-utils.git', + 'revision': 'b736cf6', + }], + ), + # from git release and subdirectory + ( + # TODO: Change me to something that fishtown-analytics manages! + # here I just moved the dbt_utils code into a subdirectory + [{ + 'git': 'https://github.com/dmateusp/dbt-utils.git', + 'revision': 'dmateusp/0.5.0/move_dbt_utils_to_subdir', + 'subdirectory': 'dbt_projects/dbt_utils', + }], + [{ + 'git': 'https://github.com/fishtown-analytics/dbt-utils.git', + 'revision': '0.5.0', + 'subdirectory': 'path/to/nonexistent/dir', + }], + ), + # from git commit and subdirectory + ( + [{ + 'git': 'https://github.com/fishtown-analytics/dbt-utils.git', + 'revision': 'f4f84e9110db26aba22f756abbae9f1f8dbb15da', + 'subdirectory': 'dbt_projects/dbt_utils', + }], + [{ + 'git': 'https://github.com/fishtown-analytics/dbt-utils.git', + 'revision': 'f4f84e9110db26aba22f756abbae9f1f8dbb15da', + 'subdirectory': 'path/to/nonexistent/dir', + }], + )], + ids=[ + "from dbt hub", + "from git release/tag/branch", + "from git commit", + "from git release and subdirectory", + "from git commit and subdirectory", + ], +) @pytest.mark.supported('postgres') -def test_rpc_deps_packages(project_root, profiles_root, dbt_profile, unique_schema): - packages = [{ - 'package': 'fishtown-analytics/dbt_utils', - 'version': '0.5.0', - }] - bad_packages = [{ - 'package': 'fishtown-analytics/dbt_util', - 'version': '0.5.0', - }] - deps_with_packages(packages, bad_packages, project_root, profiles_root, unique_schema) - - -@pytest.mark.supported('postgres') -def test_rpc_deps_git(project_root, profiles_root, dbt_profile, unique_schema): - packages = [{ - 'git': 'https://github.com/fishtown-analytics/dbt-utils.git', - 'revision': '0.5.0' - }] - # if you use a bad URL, git thinks it's a private repo and prompts for auth - bad_packages = [{ - 'git': 'https://github.com/fishtown-analytics/dbt-utils.git', - 'revision': 'not-a-real-revision' - }] - deps_with_packages(packages, bad_packages, project_root, profiles_root, unique_schema) - - -@pytest.mark.supported('postgres') -def test_rpc_deps_git_commit(project_root, profiles_root, dbt_profile, unique_schema): - packages = [{ - 'git': 'https://github.com/fishtown-analytics/dbt-utils.git', - 'revision': 'b736cf6acdbf80d2de69b511a51c8d7fe214ee79' - }] - # don't use short commits - bad_packages = [{ - 'git': 'https://github.com/fishtown-analytics/dbt-utils.git', - 'revision': 'b736cf6' - }] - deps_with_packages(packages, bad_packages, project_root, profiles_root, unique_schema) - - -def test_rpc_deps_git_subdir(project_root, profiles_root, dbt_profile, unique_schema): - # TODO: Change me to something that fishtown-analytics manages! - # here I just moved the dbt_utils code into a subdirectory - packages = [{ - 'git': 'https://github.com/dmateusp/dbt-utils.git', - 'revision': 'dmateusp/0.5.0/move_dbt_utils_to_subdir', - 'subdirectory': 'dbt_projects/dbt_utils' - }] - # - bad_packages = [{ - 'git': 'https://github.com/fishtown-analytics/dbt-utils.git', - 'revision': '0.5.0', - 'subdirectory': 'path/to/nonexistent/dir', - }] +def test_rpc_deps_packages(project_root, profiles_root, dbt_profile, unique_schema, packages, bad_packages): deps_with_packages(packages, bad_packages, project_root, profiles_root, unique_schema) From 87250bbd786e95ffdfbb803a96d830bb891a2458 Mon Sep 17 00:00:00 2001 From: Daniel Mateus Pires Date: Wed, 28 Apr 2021 10:25:45 +0100 Subject: [PATCH 11/14] Use new test-container published by @kwigley (contains more recent version of git) --- .circleci/config.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 75990cae1ad..ecee1f49b3d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,9 +2,7 @@ version: 2.1 jobs: unit: docker: &test_only - # TODO: Ask for someone at fishtown to publish - # a new version of the test container - - image: dmateusp/dbt-test-container + - image: fishtownanalytics/test-container:12 environment: DBT_INVOCATION_ENV: circle DOCKER_TEST_DATABASE_HOST: "database" @@ -39,9 +37,7 @@ jobs: destination: dist integration-postgres: docker: - # TODO: Ask for someone at fishtown to publish - # a new version of the test container - - image: dmateusp/dbt-test-container + - image: fishtownanalytics/test-container:12 environment: DBT_INVOCATION_ENV: circle DOCKER_TEST_DATABASE_HOST: "database" From a1838881550680c332dd02d1b69fc98ee8a28583 Mon Sep 17 00:00:00 2001 From: Daniel Mateus Pires Date: Wed, 28 Apr 2021 10:26:24 +0100 Subject: [PATCH 12/14] Use repositories managed by fishtown --- test/rpc/test_deps.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/test/rpc/test_deps.py b/test/rpc/test_deps.py index 2ac3f7f01ef..bf472d71e55 100644 --- a/test/rpc/test_deps.py +++ b/test/rpc/test_deps.py @@ -112,16 +112,14 @@ def deps_with_packages(packages, bad_packages, project_dir, profiles_dir, schema ), # from git release and subdirectory ( - # TODO: Change me to something that fishtown-analytics manages! - # here I just moved the dbt_utils code into a subdirectory [{ - 'git': 'https://github.com/dmateusp/dbt-utils.git', - 'revision': 'dmateusp/0.5.0/move_dbt_utils_to_subdir', - 'subdirectory': 'dbt_projects/dbt_utils', + 'git': 'https://github.com/fishtown-analytics/dbt-labs-experimental-features.git', + 'revision': '0.0.1', + 'subdirectory': 'materialized-views', }], [{ - 'git': 'https://github.com/fishtown-analytics/dbt-utils.git', - 'revision': '0.5.0', + 'git': 'https://github.com/fishtown-analytics/dbt-labs-experimental-features.git', + 'revision': '0.0.1', 'subdirectory': 'path/to/nonexistent/dir', }], ), From e1234e00c94e76abf4592a09ac61d8a533f4bc70 Mon Sep 17 00:00:00 2001 From: Daniel Mateus Pires Date: Wed, 28 Apr 2021 10:32:09 +0100 Subject: [PATCH 13/14] =?UTF-8?q?=F0=9F=A7=98=E2=80=8D=E2=99=82=EF=B8=8F?= =?UTF-8?q?=20Merge=20the=20CHANGELOG?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5587c3a0d36..f2c7dd698e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,12 +52,9 @@ Contributors: - [@techytushar](https://github.com/techytushar) ([#3158](https://github.com/fishtown-analytics/dbt/pull/3158)) - [@cgopalan](https://github.com/cgopalan) ([#3165](https://github.com/fishtown-analytics/dbt/pull/3165), [#3182](https://github.com/fishtown-analytics/dbt/pull/3182)) - [@fux](https://github.com/fuchsst) ([#3241](https://github.com/fishtown-analytics/dbt/issues/3241)) -<<<<<<< HEAD - [@dmateusp](https://github.com/dmateusp) ([#3270](https://github.com/fishtown-analytics/dbt/pull/3270)) - [@arzavj](https://github.com/arzavj) ([3106](https://github.com/fishtown-analytics/dbt/pull/3106)) -======= - [@dmateusp](https://github.com/dmateusp) ([#3270](https://github.com/fishtown-analytics/dbt/pull/3270), [#3267](https://github.com/fishtown-analytics/dbt/pull/3267)) ->>>>>>> 591ad65d (📖 Update CHANGELOG contributors to include this PR) ## dbt 0.19.1 (March 31, 2021) From 405d524e19616ed54bab3646449f66d8ed357b5d Mon Sep 17 00:00:00 2001 From: Daniel Mateus Pires Date: Wed, 28 Apr 2021 10:34:08 +0100 Subject: [PATCH 14/14] =?UTF-8?q?=F0=9F=A4=A6=E2=80=8D=E2=99=82=EF=B8=8F?= =?UTF-8?q?=20Remove=20repetition=20of=20my=20contribution=20on=20the=20CH?= =?UTF-8?q?ANGELOG?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f2c7dd698e0..9756aa323a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,7 +52,6 @@ Contributors: - [@techytushar](https://github.com/techytushar) ([#3158](https://github.com/fishtown-analytics/dbt/pull/3158)) - [@cgopalan](https://github.com/cgopalan) ([#3165](https://github.com/fishtown-analytics/dbt/pull/3165), [#3182](https://github.com/fishtown-analytics/dbt/pull/3182)) - [@fux](https://github.com/fuchsst) ([#3241](https://github.com/fishtown-analytics/dbt/issues/3241)) -- [@dmateusp](https://github.com/dmateusp) ([#3270](https://github.com/fishtown-analytics/dbt/pull/3270)) - [@arzavj](https://github.com/arzavj) ([3106](https://github.com/fishtown-analytics/dbt/pull/3106)) - [@dmateusp](https://github.com/dmateusp) ([#3270](https://github.com/fishtown-analytics/dbt/pull/3270), [#3267](https://github.com/fishtown-analytics/dbt/pull/3267))