diff --git a/.github/scripts/smoke_test.sh b/.github/scripts/smoke_test.sh index 13631c1..23521ce 100755 --- a/.github/scripts/smoke_test.sh +++ b/.github/scripts/smoke_test.sh @@ -143,7 +143,7 @@ EOF # Main test_type=$1 test_ssl="${6:-false}" -profiles_path=~/.dbt/profiles.yml +profiles_path=~/.dbt/profiles.yml if [ $test_type == softwareUP ]; then test_user=$2 diff --git a/dbt/adapters/dremio/connections.py b/dbt/adapters/dremio/connections.py index 82f2360..273429b 100644 --- a/dbt/adapters/dremio/connections.py +++ b/dbt/adapters/dremio/connections.py @@ -216,7 +216,7 @@ def create_catalog(self, database, schema): credentials = connection.credentials api_parameters = connection.handle.get_parameters() - if database == "@" + credentials.UID: + if database == ("@" + credentials.UID): logger.debug("Database is default: creating folders only") else: self._create_space(database, api_parameters) diff --git a/dbt/adapters/dremio/credentials.py b/dbt/adapters/dremio/credentials.py index e3e0e1a..040b3a5 100644 --- a/dbt/adapters/dremio/credentials.py +++ b/dbt/adapters/dremio/credentials.py @@ -20,60 +20,65 @@ @dataclass class DremioCredentials(Credentials): - environment: Optional[str] - database: Optional[str] - schema: Optional[str] - datalake: Optional[str] - root_path: Optional[str] + environment: Optional[str] = None + UID: Optional[str] = None + PWD: Optional[str] = None + pat: Optional[str] = None + datalake: Optional[str] = None + root_path: Optional[str] = None + database: Optional[str] = None + schema: Optional[str] = None cloud_project_id: Optional[str] = None cloud_host: Optional[str] = None software_host: Optional[str] = None - UID: Optional[str] = None - PWD: Optional[str] = None port: Optional[int] = 9047 # for rest endpoint use_ssl: Optional[bool] = True - pat: Optional[str] = None - additional_parameters: Optional[str] = None _ALIASES = { + # Only terms on right-side will be used going forward. + "username": "UID", # backwards compatibility with existing profiles "user": "UID", - "username": "UID", - "pass": "PWD", "password": "PWD", - "server": "host", - "track": "environment", - "space": "database", - "folder": "schema", - "materialization_database": "datalake", - "materialization_schema": "root_path", + "object_storage_source": "datalake", + "object_storage_path": "root_path", + "dremio_space": "database", + "dremio_space_folder": "schema", } + _DEFAULT_OBJECT_STORAGE_SOURCE = "$scratch" + _SPACE_NAME_PLACEHOLDER = "@user" + @property def type(self): return "dremio" @property - def unique_field(self): - return self.host + def aliases(self): + return self._ALIASES def _connection_keys(self): # return an iterator of keys to pretty-print in 'dbt debug' - # raise NotImplementedError - return ( - "driver", "cloud_host", "cloud_project_id", "software_host", "port", + "use_ssl", + "environment", + # These are aliased... "UID", + "root_path", + "datalake", "database", "schema", - "additional_parameters", - "datalake", - "root_path", - "environment", - "use_ssl", + # ...by these. Output these to ensure they match + # what they alias. + "user", # -> UID + "username", # -> UID + "object_storage_source", # -> datalake + "object_storage_path", # -> root_path + "dremio_space", # -> database + "dremio_space_folder", # -> schema ) @classmethod @@ -83,26 +88,31 @@ def __pre_deserialize__(cls, data): data["cloud_host"] = None if "software_host" not in data: data["software_host"] = None + if "database" not in data: data["database"] = None if "schema" not in data: data["schema"] = None + if "datalake" not in data: data["datalake"] = None if "root_path" not in data: data["root_path"] = None - if "environment" not in data: - data["environment"] = None + if "pat" not in data: data["pat"] = None + + if "environment" not in data: + data["environment"] = None + return data def __post_init__(self): - if self.database is None: - self.database = "@" + self.UID - if self.schema is None: - self.schema = DremioRelation.no_schema if self.datalake is None: - self.datalake = "$scratch" + self.datalake = self._DEFAULT_OBJECT_STORAGE_SOURCE if self.root_path is None: self.root_path = DremioRelation.no_schema + if self.database is None or self.database == self._SPACE_NAME_PLACEHOLDER: + self.database = f"@{self.UID}" + if self.schema is None: + self.schema = DremioRelation.no_schema diff --git a/dbt/include/dremio/profile_template.yml b/dbt/include/dremio/profile_template.yml index 7ee5fd6..2715bcb 100644 --- a/dbt/include/dremio/profile_template.yml +++ b/dbt/include/dremio/profile_template.yml @@ -16,7 +16,7 @@ fixed: type: dremio use_ssl: true prompts: - _choose_cloud_or_software_with_passowrd_or_software_with_pat: + _choose_cloud_or_software_with_password_or_software_with_pat: dremio_cloud: cloud_host: default: api.dremio.cloud @@ -54,6 +54,18 @@ prompts: use_ssl: default: false hint: 'use encrypted connection' + object_storage_source: + default: '$scratch' + hint: 'object storage source for seeds, tables, etc. [dbt alias: datalake]' + object_storage_path: + default: 'no_schema' + hint: 'object storage path [dbt alias: schema]' + dremio_space: + default: '@user' + hint: 'space for creating views [dbt alias: database]' + dremio_space_folder: + default: 'no_schema' + hint: 'dremio space folder [dbt alias: root_path]' threads: hint: '1 or more' type: 'int' diff --git a/tests/component/test_profile_template.py b/tests/component/test_profile_template.py new file mode 100644 index 0000000..338f156 --- /dev/null +++ b/tests/component/test_profile_template.py @@ -0,0 +1,139 @@ +# Copyright (C) 2022 Dremio Corporation + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pytest +from typing import Dict +import os + +from dbt.config.profile import read_profile +from dbt.adapters.dremio.credentials import DremioCredentials + +# Tests require manual setup before executing. +# +# Prior to running these tests, create three dbt projects: +# +# 1. `dbt init test_cloud_options` +# - accept all default options +# - provide any value for mandatory options +# +# 2. `dbt init test_sw_up_options` +# - accept all default options +# - provide any value for mandatory options +# +# 3. `dbt init test_sw_pat_options` +# - accept all default options +# - provide any value for mandatory options +# +# These tests assumes there exists a $HOME/.dbt/profiles.yml +# file containing these three dbt projects. +class TestProfileTemplate: + # non-OS specific + PROFILE_DIRECTORY = os.path.expanduser("~") + "/.dbt/" + + # These projects must exist in the profile.yml file. All defaults must be selected. + _TEST_CLOUD_PROFILE_PROJECT = "test_cloud_options" + _TEST_SOFTWARE_USER_PASSWORD_PROFILE_PROJECT = "test_sw_up_options" + _TEST_SOFTWARE_PAT_PROFILE_PROJECT = "test_sw_pat_options" + + _PASSWORD_AUTH_PROFILE_OPTIONS_WITH_DEFAULTS = {"password": None} + _PAT_AUTH_PROFILE_OPTIONS_WITH_DEFAULTS = {"pat": None} + _COMMON_PROFILE_OPTIONS_WITH_DEFAULTS = { + "user": None, + "object_storage_source": "$scratch", + "object_storage_path": "no_schema", + "dremio_space": "@user", + "dremio_space_folder": "no_schema", + "threads": 1, + } + _DREMIO_CLOUD_PROFILE_SPECIFIC_OPTIONS_WITH_DEFAULTS = { + "cloud_host": "api.dremio.cloud", + "cloud_project_id": None, + "use_ssl": True, + } + _DREMIO_SW_PROFILE_SPECIFIC_OPTIONS_WITH_DEFAULTS = { + "software_host": None, + "port": 9047, + "use_ssl": False, + } + + _DREMIO_CLOUD_PROFILE_OPTIONS_WITH_DEFAULTS = ( + _COMMON_PROFILE_OPTIONS_WITH_DEFAULTS + | _DREMIO_CLOUD_PROFILE_SPECIFIC_OPTIONS_WITH_DEFAULTS + | _PAT_AUTH_PROFILE_OPTIONS_WITH_DEFAULTS + ) + _DREMIO_SOFTWARE_USERNAME_PASSWORD_PROFILE_OPTIONS = ( + _COMMON_PROFILE_OPTIONS_WITH_DEFAULTS + | _DREMIO_SW_PROFILE_SPECIFIC_OPTIONS_WITH_DEFAULTS + | _PASSWORD_AUTH_PROFILE_OPTIONS_WITH_DEFAULTS + ) + _DREMIO_SOFTWARE_PAT_PROFILE_OPTIONS = ( + _COMMON_PROFILE_OPTIONS_WITH_DEFAULTS + | _DREMIO_SW_PROFILE_SPECIFIC_OPTIONS_WITH_DEFAULTS + | _PAT_AUTH_PROFILE_OPTIONS_WITH_DEFAULTS + ) + + _PROFILE_OPTIONS_ALIASES = { + "username": "UID", + "user": "UID", + "password": "PWD", + "object_storage_source": "datalake", + "object_storage_path": "root_path", + "dremio_space": "database", + "dremio_space_folder": "schema", + } + + def test_cloud_options(self) -> None: + self._test_project_profile_options( + self._get_dbt_test_project_dict(self._TEST_CLOUD_PROFILE_PROJECT), + self._DREMIO_CLOUD_PROFILE_OPTIONS_WITH_DEFAULTS, + ) + + def test_software_username_password_options(self) -> None: + self._test_project_profile_options( + self._get_dbt_test_project_dict( + self._TEST_SOFTWARE_USER_PASSWORD_PROFILE_PROJECT + ), + self._DREMIO_SOFTWARE_USERNAME_PASSWORD_PROFILE_OPTIONS, + ) + + def test_software_pat_options(self) -> None: + self._test_project_profile_options( + self._get_dbt_test_project_dict(self._TEST_SOFTWARE_PAT_PROFILE_PROJECT), + self._DREMIO_SOFTWARE_PAT_PROFILE_OPTIONS, + ) + + def test_aliases(self) -> None: + credentials = DremioCredentials() + credential_option_aliases = credentials.aliases + for option, alias in self._PROFILE_OPTIONS_ALIASES.items(): + assert credential_option_aliases[option] is not None + assert alias == credential_option_aliases[option] + + @pytest.mark.skip + def _get_dbt_test_project_dict(self, dbt_test_project_name: str) -> Dict[str, any]: + # read_profile returns dictionary with the following layout: + # { : { 'outputs': { 'dev' } : { ] } } + profile_dictionary = read_profile(TestProfileTemplate.PROFILE_DIRECTORY) + return profile_dictionary.get(dbt_test_project_name).get("outputs").get("dev") + + @pytest.mark.skip + def _test_project_profile_options( + self, test_project: Dict[str, any], test_options: Dict[str, any] + ) -> None: + + assert test_project is not None + + for option in test_options: + assert test_project[option] is not None + if test_options[option] is not None: + assert test_project[option] == test_options[option]