From 346c4b51e2f0b134f382d5960690dc88f383da95 Mon Sep 17 00:00:00 2001 From: Abhishek Singh Date: Mon, 1 May 2023 12:33:17 -0700 Subject: [PATCH 1/7] Preparing for release v1.5.0 --- .github/workflows/oracle-xe-adapter-tests.yml | 2 +- .gitignore | 1 + Makefile | 2 +- dbt/adapters/oracle/__version__.py | 2 +- requirements.txt | 4 ++-- requirements_dev.txt | 2 +- setup.cfg | 8 ++++---- setup.py | 8 ++++---- tox.ini | 2 +- 9 files changed, 16 insertions(+), 15 deletions(-) diff --git a/.github/workflows/oracle-xe-adapter-tests.yml b/.github/workflows/oracle-xe-adapter-tests.yml index 58424aa..5790805 100644 --- a/.github/workflows/oracle-xe-adapter-tests.yml +++ b/.github/workflows/oracle-xe-adapter-tests.yml @@ -48,7 +48,7 @@ jobs: - name: Install dbt-oracle with core dependencies run: | python -m pip install --upgrade pip - pip install pytest dbt-tests-adapter==1.4.5 + pip install pytest dbt-tests-adapter==1.5.0 pip install -r requirements.txt pip install -e . diff --git a/.gitignore b/.gitignore index d1c0213..a5d0763 100644 --- a/.gitignore +++ b/.gitignore @@ -145,3 +145,4 @@ doc/build.gitbak .venv1.2/ .venv1.3/ .venv1.4/ +.venv1.5/ diff --git a/Makefile b/Makefile index 4dbfa0d..35a5f39 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Configuration variables -VERSION=1.4.2 +VERSION=1.5.0 PROJ_DIR?=$(shell pwd) VENV_DIR?=${PROJ_DIR}/.bldenv BUILD_DIR=${PROJ_DIR}/build diff --git a/dbt/adapters/oracle/__version__.py b/dbt/adapters/oracle/__version__.py index 2fed148..7497b56 100644 --- a/dbt/adapters/oracle/__version__.py +++ b/dbt/adapters/oracle/__version__.py @@ -14,4 +14,4 @@ See the License for the specific language governing permissions and limitations under the License. """ -version = "1.4.5" +version = "1.5.0" diff --git a/requirements.txt b/requirements.txt index aaa2032..bf8d997 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -dbt-core==1.4.5 +dbt-core==1.5.0 cx_Oracle==8.3.0 -oracledb==1.2.2 +oracledb==1.3.1 diff --git a/requirements_dev.txt b/requirements_dev.txt index cadc816..b45a51c 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -6,4 +6,4 @@ tox coverage twine pytest -dbt-tests-adapter==1.4.5 \ No newline at end of file +dbt-tests-adapter==1.5.0 diff --git a/setup.cfg b/setup.cfg index fc0667c..d520483 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = dbt-oracle -version = 1.4.2 +version = 1.5.0 description = dbt (data build tool) adapter for the Oracle database long_description = file: README.md long_description_content_type = text/markdown @@ -33,12 +33,12 @@ zip_safe = False packages = find: include_package_data = True install_requires = - dbt-core==1.4.5 + dbt-core==1.5.0 cx_Oracle==8.3.0 - oracledb==1.2.2 + oracledb==1.3.1 test_suite=tests test_requires = - dbt-tests-adapter==1.4.5 + dbt-tests-adapter==1.5.0 pytest scripts = bin/create-pem-from-p12 diff --git a/setup.py b/setup.py index a747332..858e787 100644 --- a/setup.py +++ b/setup.py @@ -32,13 +32,13 @@ requirements = [ - "dbt-core==1.4.5", + "dbt-core==1.5.0", "cx_Oracle==8.3.0", - "oracledb==1.2.2" + "oracledb==1.3.1" ] test_requirements = [ - "dbt-tests-adapter==1.4.5", + "dbt-tests-adapter==1.5.0", "pytest" ] @@ -52,7 +52,7 @@ url = 'https://github.com/oracle/dbt-oracle' -VERSION = '1.4.2' +VERSION = '1.5.0' setup( author="Oracle", python_requires='>=3.7.2', diff --git a/tox.ini b/tox.ini index 9ec8920..cfeda42 100644 --- a/tox.ini +++ b/tox.ini @@ -15,7 +15,7 @@ passenv = deps = -rrequirements.txt - dbt-tests-adapter==1.4.5 + dbt-tests-adapter==1.5.0 pytest commands = pytest From 2cacd511c0fee0a3ebf07bb257ec9e4262fa26ed Mon Sep 17 00:00:00 2001 From: Abhishek Singh Date: Mon, 1 May 2023 12:44:18 -0700 Subject: [PATCH 2/7] Fix ImportError: cannot import name 'raise_compiler_error' from 'dbt.exceptions' --- dbt/adapters/oracle/impl.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dbt/adapters/oracle/impl.py b/dbt/adapters/oracle/impl.py index 8e378df..8c0ca98 100644 --- a/dbt/adapters/oracle/impl.py +++ b/dbt/adapters/oracle/impl.py @@ -32,7 +32,6 @@ from dbt.contracts.graph.manifest import Manifest from dbt.events import AdapterLogger -from dbt.exceptions import raise_compiler_error from dbt.utils import filter_null_values from dbt.adapters.oracle.keyword_catalog import KEYWORDS From d24fdc2df25c18579cf180714e5178ababc9a4bd Mon Sep 17 00:00:00 2001 From: Abhishek Singh Date: Wed, 3 May 2023 16:56:58 -0700 Subject: [PATCH 3/7] Added changes for model contracts --- Makefile | 2 +- dbt/adapters/oracle/connections.py | 33 +++ dbt/adapters/oracle/impl.py | 31 +- dbt/include/oracle/macros/adapters.sql | 44 ++- .../adapter/constraints/__init__.py | 0 .../adapter/constraints/fixtures.py | 276 ++++++++++++++++++ .../adapter/constraints/test_constraints.py | 192 ++++++++++++ 7 files changed, 575 insertions(+), 3 deletions(-) create mode 100644 tests/functional/adapter/constraints/__init__.py create mode 100644 tests/functional/adapter/constraints/fixtures.py create mode 100644 tests/functional/adapter/constraints/test_constraints.py diff --git a/Makefile b/Makefile index 35a5f39..f5d6747 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ PROJ_DIR?=$(shell pwd) VENV_DIR?=${PROJ_DIR}/.bldenv BUILD_DIR=${PROJ_DIR}/build DIST_DIR=${PROJ_DIR}/dist -PYTHON_3=python3.8 +PYTHON_3=python3.9 clean_venv: diff --git a/dbt/adapters/oracle/connections.py b/dbt/adapters/oracle/connections.py index 2150ce6..a975359 100644 --- a/dbt/adapters/oracle/connections.py +++ b/dbt/adapters/oracle/connections.py @@ -33,6 +33,35 @@ logger = AdapterLogger("oracle") +DATATYPES = { + "DB_TYPE_BFILE": "BFILE", + "DB_TYPE_BINARY_DOUBLE": "BINARY_DOUBLE", + "DB_TYPE_BINARY_FLOAT": "BINARY_FLOAT", + "DB_TYPE_BINARY_INTEGER": "BINARY_INTEGER", + "DB_TYPE_BLOB": "BLOB", + "DB_TYPE_BOOLEAN": "BOOLEAN", + "DB_TYPE_CHAR": "CHAR", + "DB_TYPE_CLOB": "CLOB", + "DB_TYPE_DATE": "DATE", + "DB_TYPE_INTERVAL_DS": "INTERVAL DAY TO SECOND", + "DB_TYPE_INTERVAL_YM": "INTERVAL YEAR TO MONTH", + "DB_TYPE_JSON": "JSON", + "DB_TYPE_LONG": "LONG", + "DB_TYPE_LONG_NVARCHAR": "LONG NVARCHAR", + "DB_TYPE_LONG_RAW": "LONG RAW", + "DB_TYPE_NCHAR": "NCHAR", + "DB_TYPE_NCLOB": "NCLOB", + "DB_TYPE_NUMBER": "NUMBER", + "DB_TYPE_NVARCHAR": "NVARCHAR2", + "DB_TYPE_OBJECT": "OBJECT", + "DB_TYPE_RAW": "RAW", + "DB_TYPE_ROWID": "ROWID", + "DB_TYPE_TIMESTAMP": "TIMESTAMP", + "DB_TYPE_TIMESTAMP_LTZ": "TIMESTAMP WITH LOCAL TZ", + "DB_TYPE_TIMESTAMP_TZ": "TIMESTAMP WITH TZ", + "DB_TYPE_VARCHAR": "VARCHAR2" +} + class OracleConnectionMethod(enum.Enum): HOST = 1 TNS = 2 @@ -284,3 +313,7 @@ def add_begin_query(self): connection = self.get_thread_connection() cursor = connection.handle.cursor return connection, cursor + + @classmethod + def data_type_code_to_name(cls, type_code) -> str: + return DATATYPES[type_code.name] diff --git a/dbt/adapters/oracle/impl.py b/dbt/adapters/oracle/impl.py index 8c0ca98..215f32e 100644 --- a/dbt/adapters/oracle/impl.py +++ b/dbt/adapters/oracle/impl.py @@ -18,18 +18,23 @@ Optional, List, Set ) from itertools import chain +from typing import ( + Any, + Callable, + Dict) import agate import dbt.exceptions from dbt.adapters.base.relation import BaseRelation, InformationSchema -from dbt.adapters.base.impl import GET_CATALOG_MACRO_NAME +from dbt.adapters.base.impl import GET_CATALOG_MACRO_NAME, ConstraintSupport from dbt.adapters.sql import SQLAdapter from dbt.adapters.base.meta import available from dbt.adapters.oracle import OracleAdapterConnectionManager from dbt.adapters.oracle.column import OracleColumn from dbt.adapters.oracle.relation import OracleRelation from dbt.contracts.graph.manifest import Manifest +from dbt.contracts.graph.nodes import ConstraintType from dbt.events import AdapterLogger from dbt.utils import filter_null_values @@ -78,6 +83,14 @@ class OracleAdapter(SQLAdapter): Relation = OracleRelation Column = OracleColumn + CONSTRAINT_SUPPORT = { + ConstraintType.check: ConstraintSupport.ENFORCED, + ConstraintType.not_null: ConstraintSupport.ENFORCED, + ConstraintType.unique: ConstraintSupport.ENFORCED, + ConstraintType.primary_key: ConstraintSupport.ENFORCED, + ConstraintType.foreign_key: ConstraintSupport.NOT_ENFORCED, + } + def debug_query(self) -> None: self.execute("select 1 as id from dual") @@ -316,3 +329,19 @@ def quote_seed_column( def valid_incremental_strategies(self): return ["append", "merge"] + + @available + @classmethod + def render_raw_columns_constraints(cls, raw_columns: Dict[str, Dict[str, Any]]) -> List: + rendered_column_constraints = [] + + for v in raw_columns.values(): + rendered_column_constraint = [f"{v['name']}"] + for con in v.get("constraints", None): + constraint = cls._parse_column_constraint(con) + c = cls.process_parsed_constraint(constraint, cls.render_column_constraint) + if c is not None: + rendered_column_constraint.append(c) + rendered_column_constraints.append(" ".join(rendered_column_constraint)) + + return rendered_column_constraints diff --git a/dbt/include/oracle/macros/adapters.sql b/dbt/include/oracle/macros/adapters.sql index d0476a1..924c854 100644 --- a/dbt/include/oracle/macros/adapters.sql +++ b/dbt/include/oracle/macros/adapters.sql @@ -25,6 +25,39 @@ {{ return(load_result('get_columns_in_query').table.columns | map(attribute='name') | list) }} {% endmacro %} +{% macro oracle__get_empty_subquery_sql(select_sql) %} + select * from ( + {{ select_sql }} + ) dbt_sbq_tmp + where 1 = 0 and rownum < 1 +{% endmacro %} + +{% macro oracle__get_empty_schema_sql(columns) %} + {%- set col_err = [] -%} + select + {% for i in columns %} + {%- set col = columns[i] -%} + {%- if col['data_type'] is not defined -%} + {{ col_err.append(col['name']) }} + {%- endif -%} + cast(null as {{ col['data_type'] }}) as {{ col['name'] }}{{ ", " if not loop.last }} + {%- endfor -%} + {# Override for Oracle #} + from dual + {%- if (col_err | length) > 0 -%} + {{ exceptions.column_type_missing(column_names=col_err) }} + {%- endif -%} +{% endmacro %} + +{% macro oracle__get_select_subquery(sql) %} + select + {% for column in model['columns'] %} + {{ column }}{{ ", " if not loop.last }} + {% endfor %} + from ( + {{ sql }} + ) model_subq +{%- endmacro %} {% macro oracle__create_schema(relation, schema_name) -%} {% if relation.database -%} @@ -107,12 +140,18 @@ {%- set sql_header = config.get('sql_header', none) -%} {%- set parallel = config.get('parallel', none) -%} {%- set compression_clause = config.get('table_compression_clause', none) -%} + {%- set contract_config = config.get('contract') -%} {{ sql_header if sql_header is not none }} create {% if temporary -%} global temporary {%- endif %} table {{ relation.include(schema=(not temporary)) }} + {%- if contract_config.enforced -%} + {{ get_assert_columns_equivalent(sql) }} + {{ get_table_columns_and_constraints() }} + {%- set sql = get_select_subquery(sql) %} + {% endif %} {% if temporary -%} on commit preserve rows {%- endif %} {% if not temporary -%} {% if parallel %} parallel {{ parallel }}{% endif %} @@ -124,7 +163,10 @@ {%- endmacro %} {% macro oracle__create_view_as(relation, sql) -%} {%- set sql_header = config.get('sql_header', none) -%} - + {%- set contract_config = config.get('contract') -%} + {%- if contract_config.enforced -%} + {{ get_assert_columns_equivalent(sql) }} + {%- endif %} {{ sql_header if sql_header is not none }} create or replace view {{ relation }} as {{ sql }} diff --git a/tests/functional/adapter/constraints/__init__.py b/tests/functional/adapter/constraints/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/functional/adapter/constraints/fixtures.py b/tests/functional/adapter/constraints/fixtures.py new file mode 100644 index 0000000..fb4cd4d --- /dev/null +++ b/tests/functional/adapter/constraints/fixtures.py @@ -0,0 +1,276 @@ +import pytest + +import dbt.tests.adapter.constraints.fixtures + +# model columns data types different to schema definitions +my_model_data_type_sql = """ +{{{{ + config( + materialized = "table" + ) +}}}} + +select + {sql_value} as wrong_data_type_column_name from dual +""" +dbt.tests.adapter.constraints.fixtures.my_model_data_type_sql = my_model_data_type_sql + +my_model_wrong_order_sql = """ +{{ + config( + materialized = "table" + ) +}} + +select + 'blue' as color, + 1 as id, + TO_DATE('2019-01-01', 'YYYY-MM-DD') as date_day + from dual +""" + +my_model_wrong_name_sql = """ +{{ + config( + materialized = "table" + ) +}} + +select + 'blue' as color, + 1 as error, + TO_DATE('2019-01-01', 'YYYY-MM-DD') as date_day + from dual +""" + +my_model_view_wrong_order_sql = """ +{{ + config( + materialized = "view" + ) +}} + +select + 'blue' as color, + 1 as id, + TO_DATE('2019-01-01', 'YYYY-MM-DD') as date_day + from dual +""" + +my_model_view_wrong_name_sql = """ +{{ + config( + materialized = "view" + ) +}} + +select + 'blue' as color, + 1 as error, + TO_DATE('2019-01-01', 'YYYY-MM-DD') as date_day + from dual +""" + +my_model_incremental_wrong_order_sql = """ +{{ + config( + materialized = "incremental", + on_schema_change='append_new_columns' + ) +}} + +select + 'blue' as color, + 1 as id, + TO_DATE('2019-01-01', 'YYYY-MM-DD') as date_day + from dual +""" + +my_model_incremental_wrong_name_sql = """ +{{ + config( + materialized = "incremental", + on_schema_change='append_new_columns' + ) +}} + +select + 'blue' as color, + 1 as error, + TO_DATE('2019-01-01', 'YYYY-MM-DD') as date_day + from dual +""" + +my_model_sql = """ +{{ + config( + materialized = "table" + ) +}} + +select + 1 as id, + 'blue' as color, + TO_DATE('2019-01-01', 'YYYY-MM-DD') as date_day + from dual +""" + +# model breaking constraints +my_model_with_nulls_sql = """ +{{ + config( + materialized = "table" + ) +}} + +select + -- null value for 'id' + cast(null as {{ dbt.type_int() }}) as id, + -- change the color as well (to test rollback) + 'red' as color, + TO_DATE('2019-01-01', 'YYYY-MM-DD') as date_day + from dual +""" + +my_incremental_model_sql = """ +{{ + config( + materialized = "incremental", + on_schema_change='append_new_columns' + ) +}} + +select + 1 as id, + 'blue' as color, + TO_DATE('2019-01-01', 'YYYY-MM-DD') as date_day + from dual +""" + +my_model_incremental_with_nulls_sql = """ +{{ + config( + materialized = "incremental", + on_schema_change='append_new_columns' ) +}} + +select + -- null value for 'id' + cast(null as {{ dbt.type_int() }}) as id, + -- change the color as well (to test rollback) + 'red' as color, + TO_DATE('2019-01-01', 'YYYY-MM-DD') as date_day + from dual +""" + +model_schema_yml = """ +version: 2 +models: + - name: my_model + config: + contract: + enforced: true + columns: + - name: id + quote: true + data_type: integer + description: hello + constraints: + - type: not_null + - type: primary_key + - type: check + expression: (id > 0) + tests: + - unique + - name: color + data_type: char + - name: date_day + data_type: date + - name: my_model_error + config: + contract: + enforced: true + columns: + - name: id + data_type: integer + description: hello + constraints: + - type: not_null + - type: primary_key + - type: check + expression: (id > 0) + tests: + - unique + - name: color + data_type: char + - name: date_day + data_type: date + - name: my_model_wrong_order + config: + contract: + enforced: true + columns: + - name: id + data_type: integer + description: hello + constraints: + - type: not_null + - type: primary_key + - type: check + expression: (id > 0) + tests: + - unique + - name: color + data_type: char + - name: date_day + data_type: date + - name: my_model_wrong_name + config: + contract: + enforced: true + columns: + - name: id + data_type: integer + description: hello + constraints: + - type: not_null + - type: primary_key + - type: check + expression: (id > 0) + tests: + - unique + - name: color + data_type: char + - name: date_day + data_type: date +""" + +constrained_model_schema_yml = """ +version: 2 +models: + - name: my_model + config: + contract: + enforced: true + constraints: + - type: check + expression: (id > 0) + - type: primary_key + columns: [ id ] + - type: unique + columns: [ color, date_day ] + name: strange_uniqueness_requirement + columns: + - name: id + quote: true + data_type: integer + description: hello + constraints: + - type: not_null + tests: + - unique + - name: color + data_type: char + - name: date_day + data_type: date +""" \ No newline at end of file diff --git a/tests/functional/adapter/constraints/test_constraints.py b/tests/functional/adapter/constraints/test_constraints.py new file mode 100644 index 0000000..f93dad4 --- /dev/null +++ b/tests/functional/adapter/constraints/test_constraints.py @@ -0,0 +1,192 @@ +import pytest + +from .fixtures import (model_schema_yml, + constrained_model_schema_yml, + my_model_sql, + my_model_with_nulls_sql, + my_model_wrong_name_sql, + my_model_wrong_order_sql, + my_model_view_wrong_name_sql, + my_model_view_wrong_order_sql, + my_model_incremental_wrong_name_sql, + my_model_incremental_wrong_order_sql, + my_model_incremental_with_nulls_sql, + my_incremental_model_sql) + +from dbt.tests.adapter.constraints.test_constraints import ( + BaseTableConstraintsColumnsEqual, + BaseViewConstraintsColumnsEqual, + BaseIncrementalConstraintsColumnsEqual, + BaseConstraintsRuntimeDdlEnforcement, + BaseIncrementalConstraintsRuntimeDdlEnforcement, + BaseConstraintsRollback, + BaseIncrementalConstraintsRollback, + BaseModelConstraintsRuntimeEnforcement +) + +_expected_sql_oracle = """ +create table ( + id not null primary key check (id > 0), + color, + date_day +) as select + id, + color, + date_day from + ( + select + 'blue' as color, + 1 as id, + TO_DATE('2019-01-01', 'YYYY-MM-DD') as date_day + from dual + ) model_subq +""" + + +class OracleColumnsEqualSetup: + + @pytest.fixture + def string_type(self): + return "CHAR" + + @pytest.fixture + def int_type(self): + return "NUMBER" + + @pytest.fixture + def data_types(self, schema_int_type, int_type, string_type): + # sql_column_value, schema_data_type, error_data_type + return [ + ["1", schema_int_type, int_type], + ["'1'", string_type, string_type], + ["TO_DATE('2019-01-01', 'YYYY-MM-DD')", 'date', "DATE"]] + + +class TestOracleTableConstraintsColumnsEqual(OracleColumnsEqualSetup, BaseTableConstraintsColumnsEqual): + + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_wrong_order.sql": my_model_wrong_order_sql, + "my_model_wrong_name.sql": my_model_wrong_name_sql, + "constraints_schema.yml": model_schema_yml, + } + + +class TestOracleViewConstraintsColumnsEqual(OracleColumnsEqualSetup, BaseViewConstraintsColumnsEqual): + + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_wrong_order.sql": my_model_view_wrong_order_sql, + "my_model_wrong_name.sql": my_model_view_wrong_name_sql, + "constraints_schema.yml": model_schema_yml, + } + + +class TestOracleIncrementalConstraintsColumnsEqual(OracleColumnsEqualSetup, BaseIncrementalConstraintsColumnsEqual): + + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_wrong_order.sql": my_model_incremental_wrong_order_sql, + "my_model_wrong_name.sql": my_model_incremental_wrong_name_sql, + "constraints_schema.yml": model_schema_yml, + } + + +class TestOracleTableConstraintsDdlEnforcement(BaseConstraintsRuntimeDdlEnforcement): + + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_wrong_order_sql, + "constraints_schema.yml": model_schema_yml, + } + + @pytest.fixture(scope="class") + def expected_sql(self): + return _expected_sql_oracle + + +class TestOracleIncrementalConstraintsDdlEnforcement(BaseIncrementalConstraintsRuntimeDdlEnforcement): + + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_incremental_wrong_order_sql, + "constraints_schema.yml": model_schema_yml, + } + + @pytest.fixture(scope="class") + def expected_sql(self): + return _expected_sql_oracle + + +class TestOracleTableConstraintsRollback(BaseConstraintsRollback): + + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_sql, + "constraints_schema.yml": model_schema_yml, + } + + @pytest.fixture(scope="class") + def null_model_sql(self): + return my_model_with_nulls_sql + + @pytest.fixture(scope="class") + def expected_error_messages(self): + return ["ORA-01400: cannot insert NULL into"] + + +class TestOracleIncrementalConstraintsRollback(BaseIncrementalConstraintsRollback): + + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_incremental_model_sql, + "constraints_schema.yml": model_schema_yml, + } + + @pytest.fixture(scope="class") + def null_model_sql(self): + return my_model_incremental_with_nulls_sql + + @pytest.fixture(scope="class") + def expected_error_messages(self): + return ["ORA-01400: cannot insert NULL into"] + + +class TestOracleModelConstraintsRuntimeEnforcement(BaseModelConstraintsRuntimeEnforcement): + + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_sql, + "constraints_schema.yml": constrained_model_schema_yml, + } + + @pytest.fixture(scope="class") + def expected_sql(self): + return """ +create table ( + id not null, + color, + date_day, + check (id > 0), + primary key (id), + constraint strange_uniqueness_requirement unique (color, date_day) +) as select + id, + color, + date_day from + ( + select + 1 as id, + 'blue' as color, + TO_DATE('2019-01-01', 'YYYY-MM-DD') as date_day + from dual + ) model_subq +""" \ No newline at end of file From 368d54831e54fcc56b633bea5d4b70f97391e47b Mon Sep 17 00:00:00 2001 From: Abhishek Singh Date: Mon, 8 May 2023 17:40:55 -0700 Subject: [PATCH 4/7] Added more test cases --- dbt/adapters/oracle/connections.py | 4 +- dbt/adapters/oracle/impl.py | 2 +- .../adapter/constraints/__init__.py | 16 +++++ .../adapter/constraints/fixtures.py | 16 +++++ .../adapter/constraints/test_constraints.py | 40 +++++++---- .../adapter/simple_seed/__init__.py | 16 +++++ .../adapter/simple_seed/test_simple_seed.py | 33 +++++++++ tests/functional/adapter/test_caching.py | 70 +++++++++++++++++++ 8 files changed, 182 insertions(+), 15 deletions(-) create mode 100644 tests/functional/adapter/simple_seed/__init__.py create mode 100644 tests/functional/adapter/simple_seed/test_simple_seed.py create mode 100644 tests/functional/adapter/test_caching.py diff --git a/dbt/adapters/oracle/connections.py b/dbt/adapters/oracle/connections.py index a975359..8921473 100644 --- a/dbt/adapters/oracle/connections.py +++ b/dbt/adapters/oracle/connections.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2022, Oracle and/or its affiliates. +Copyright (c) 2023, Oracle and/or its affiliates. Copyright (c) 2020, Vitor Avancini Licensed under the Apache License, Version 2.0 (the "License"); @@ -117,7 +117,7 @@ def type(self): @property def unique_field(self): - return self.database + return self.database or self.user def _connection_keys(self) -> Tuple[str]: """ diff --git a/dbt/adapters/oracle/impl.py b/dbt/adapters/oracle/impl.py index 215f32e..55999e4 100644 --- a/dbt/adapters/oracle/impl.py +++ b/dbt/adapters/oracle/impl.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2022, Oracle and/or its affiliates. +Copyright (c) 2023, Oracle and/or its affiliates. Copyright (c) 2020, Vitor Avancini Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/tests/functional/adapter/constraints/__init__.py b/tests/functional/adapter/constraints/__init__.py index e69de29..7cd3b08 100644 --- a/tests/functional/adapter/constraints/__init__.py +++ b/tests/functional/adapter/constraints/__init__.py @@ -0,0 +1,16 @@ +""" +Copyright (c) 2023, Oracle and/or its affiliates. +Copyright (c) 2020, Vitor Avancini + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" \ No newline at end of file diff --git a/tests/functional/adapter/constraints/fixtures.py b/tests/functional/adapter/constraints/fixtures.py index fb4cd4d..bff9bfe 100644 --- a/tests/functional/adapter/constraints/fixtures.py +++ b/tests/functional/adapter/constraints/fixtures.py @@ -1,3 +1,19 @@ +""" +Copyright (c) 2023, Oracle and/or its affiliates. +Copyright (c) 2020, Vitor Avancini + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" import pytest import dbt.tests.adapter.constraints.fixtures diff --git a/tests/functional/adapter/constraints/test_constraints.py b/tests/functional/adapter/constraints/test_constraints.py index f93dad4..0bd023d 100644 --- a/tests/functional/adapter/constraints/test_constraints.py +++ b/tests/functional/adapter/constraints/test_constraints.py @@ -1,17 +1,33 @@ +""" +Copyright (c) 2023, Oracle and/or its affiliates. +Copyright (c) 2020, Vitor Avancini + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" import pytest -from .fixtures import (model_schema_yml, - constrained_model_schema_yml, - my_model_sql, - my_model_with_nulls_sql, - my_model_wrong_name_sql, - my_model_wrong_order_sql, - my_model_view_wrong_name_sql, - my_model_view_wrong_order_sql, - my_model_incremental_wrong_name_sql, - my_model_incremental_wrong_order_sql, - my_model_incremental_with_nulls_sql, - my_incremental_model_sql) +from tests.functional.adapter.constraints.fixtures import (model_schema_yml, + constrained_model_schema_yml, + my_model_sql, + my_model_with_nulls_sql, + my_model_wrong_name_sql, + my_model_wrong_order_sql, + my_model_view_wrong_name_sql, + my_model_view_wrong_order_sql, + my_model_incremental_wrong_name_sql, + my_model_incremental_wrong_order_sql, + my_model_incremental_with_nulls_sql, + my_incremental_model_sql) from dbt.tests.adapter.constraints.test_constraints import ( BaseTableConstraintsColumnsEqual, diff --git a/tests/functional/adapter/simple_seed/__init__.py b/tests/functional/adapter/simple_seed/__init__.py new file mode 100644 index 0000000..7cd3b08 --- /dev/null +++ b/tests/functional/adapter/simple_seed/__init__.py @@ -0,0 +1,16 @@ +""" +Copyright (c) 2023, Oracle and/or its affiliates. +Copyright (c) 2020, Vitor Avancini + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" \ No newline at end of file diff --git a/tests/functional/adapter/simple_seed/test_simple_seed.py b/tests/functional/adapter/simple_seed/test_simple_seed.py new file mode 100644 index 0000000..a8825ce --- /dev/null +++ b/tests/functional/adapter/simple_seed/test_simple_seed.py @@ -0,0 +1,33 @@ +""" +Copyright (c) 2023, Oracle and/or its affiliates. +Copyright (c) 2020, Vitor Avancini + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + +import pytest + +from dbt.tests.adapter.simple_seed.test_seed import SeedConfigBase +from dbt.tests.util import run_dbt + + +class TestSimpleBigSeedBatched(SeedConfigBase): + @pytest.fixture(scope="class") + def seeds(self): + seed_data = ["seed_id"] + seed_data.extend([str(i) for i in range(20_000)]) + return {"big_batched_seed.csv": "\n".join(seed_data)} + + def test_big_batched_seed(self, project): + seed_results = run_dbt(["seed"]) + assert len(seed_results) == 1 diff --git a/tests/functional/adapter/test_caching.py b/tests/functional/adapter/test_caching.py new file mode 100644 index 0000000..e2ba9cd --- /dev/null +++ b/tests/functional/adapter/test_caching.py @@ -0,0 +1,70 @@ +""" +Copyright (c) 2023, Oracle and/or its affiliates. +Copyright (c) 2020, Vitor Avancini + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" +import pytest + +from dbt.tests.adapter.caching.test_caching import ( + BaseCachingLowercaseModel, + BaseCachingUppercaseModel, + BaseCachingSelectedSchemaOnly, +) + +model_sql = """ +{{ + config( + materialized='table' + ) +}} +select 1 as id from dual +""" + +another_schema_model_sql = """ +{{ + config( + materialized='table', + schema='another_schema' + ) +}} +select 1 as id from dual +""" + + +class TestCachingLowerCaseModel(BaseCachingLowercaseModel): + + @pytest.fixture(scope="class") + def models(self): + return { + "model.sql": model_sql, + } + + +class TestCachingUppercaseModel(BaseCachingUppercaseModel): + + @pytest.fixture(scope="class") + def models(self): + return { + "MODEL.sql": model_sql, + } + + +class TestCachingSelectedSchemaOnly(BaseCachingSelectedSchemaOnly): + + @pytest.fixture(scope="class") + def models(self): + return { + "model.sql": model_sql, + "another_schema_model.sql": another_schema_model_sql, + } From 409b58c1fa405dbbbcae37b025efe77673a750fd Mon Sep 17 00:00:00 2001 From: Abhishek Singh Date: Mon, 8 May 2023 18:18:24 -0700 Subject: [PATCH 5/7] Added missing newline EOF and fixed caching test case --- dbt/adapters/oracle/impl.py | 2 +- .../adapter/constraints/__init__.py | 2 +- .../adapter/constraints/fixtures.py | 2 +- .../adapter/constraints/test_constraints.py | 2 +- .../adapter/simple_seed/__init__.py | 2 +- tests/functional/adapter/test_caching.py | 31 ++++++++++++++++++- 6 files changed, 35 insertions(+), 6 deletions(-) diff --git a/dbt/adapters/oracle/impl.py b/dbt/adapters/oracle/impl.py index 55999e4..ebab47b 100644 --- a/dbt/adapters/oracle/impl.py +++ b/dbt/adapters/oracle/impl.py @@ -88,7 +88,7 @@ class OracleAdapter(SQLAdapter): ConstraintType.not_null: ConstraintSupport.ENFORCED, ConstraintType.unique: ConstraintSupport.ENFORCED, ConstraintType.primary_key: ConstraintSupport.ENFORCED, - ConstraintType.foreign_key: ConstraintSupport.NOT_ENFORCED, + ConstraintType.foreign_key: ConstraintSupport.ENFORCED, } def debug_query(self) -> None: diff --git a/tests/functional/adapter/constraints/__init__.py b/tests/functional/adapter/constraints/__init__.py index 7cd3b08..86eee3b 100644 --- a/tests/functional/adapter/constraints/__init__.py +++ b/tests/functional/adapter/constraints/__init__.py @@ -13,4 +13,4 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -""" \ No newline at end of file +""" diff --git a/tests/functional/adapter/constraints/fixtures.py b/tests/functional/adapter/constraints/fixtures.py index bff9bfe..a9280ab 100644 --- a/tests/functional/adapter/constraints/fixtures.py +++ b/tests/functional/adapter/constraints/fixtures.py @@ -289,4 +289,4 @@ data_type: char - name: date_day data_type: date -""" \ No newline at end of file +""" diff --git a/tests/functional/adapter/constraints/test_constraints.py b/tests/functional/adapter/constraints/test_constraints.py index 0bd023d..30bfacf 100644 --- a/tests/functional/adapter/constraints/test_constraints.py +++ b/tests/functional/adapter/constraints/test_constraints.py @@ -205,4 +205,4 @@ def expected_sql(self): TO_DATE('2019-01-01', 'YYYY-MM-DD') as date_day from dual ) model_subq -""" \ No newline at end of file +""" diff --git a/tests/functional/adapter/simple_seed/__init__.py b/tests/functional/adapter/simple_seed/__init__.py index 7cd3b08..86eee3b 100644 --- a/tests/functional/adapter/simple_seed/__init__.py +++ b/tests/functional/adapter/simple_seed/__init__.py @@ -13,4 +13,4 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -""" \ No newline at end of file +""" diff --git a/tests/functional/adapter/test_caching.py b/tests/functional/adapter/test_caching.py index e2ba9cd..9bf3c4f 100644 --- a/tests/functional/adapter/test_caching.py +++ b/tests/functional/adapter/test_caching.py @@ -17,10 +17,12 @@ import pytest from dbt.tests.adapter.caching.test_caching import ( + BaseCachingTest, BaseCachingLowercaseModel, BaseCachingUppercaseModel, BaseCachingSelectedSchemaOnly, ) +from dbt.tests.util import run_dbt model_sql = """ {{ @@ -42,7 +44,34 @@ """ -class TestCachingLowerCaseModel(BaseCachingLowercaseModel): +class OracleBaseCaching(BaseCachingTest): + + def run_and_inspect_cache(self, project, run_args=None): + run_dbt(run_args) + + # the cache was empty at the start of the run. + # the model materialization returned an unquoted relation and added to the cache. + adapter = project.adapter + assert len(adapter.cache.relations) == 1 + relation = list(adapter.cache.relations).pop() + # assert relation.schema == project.test_schema + assert relation.schema == project.test_schema.lower() + + # on the second run, dbt will find a relation in the database during cache population. + # this relation will be quoted, because list_relations_without_caching (by default) uses + # quote_policy = {"database": True, "schema": True, "identifier": True} + # when adding relations to the cache. + run_dbt(run_args) + adapter = project.adapter + assert len(adapter.cache.relations) == 1 + second_relation = list(adapter.cache.relations).pop() + + # perform a case-insensitive + quote-insensitive comparison + for key in ["database", "schema", "identifier"]: + assert getattr(relation, key).lower() == getattr(second_relation, key).lower() + + +class TestCachingLowerCaseModel(OracleBaseCaching): @pytest.fixture(scope="class") def models(self): From cff67cbdc53c2b9275e2c89a68a4b999a75e4f38 Mon Sep 17 00:00:00 2001 From: Abhishek Singh Date: Mon, 8 May 2023 18:24:35 -0700 Subject: [PATCH 6/7] Fixed cahcing test case --- tests/functional/adapter/test_caching.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/functional/adapter/test_caching.py b/tests/functional/adapter/test_caching.py index 9bf3c4f..5f95d39 100644 --- a/tests/functional/adapter/test_caching.py +++ b/tests/functional/adapter/test_caching.py @@ -16,12 +16,8 @@ """ import pytest -from dbt.tests.adapter.caching.test_caching import ( - BaseCachingTest, - BaseCachingLowercaseModel, - BaseCachingUppercaseModel, - BaseCachingSelectedSchemaOnly, -) +from dbt.tests.adapter.caching.test_caching import BaseCachingTest + from dbt.tests.util import run_dbt model_sql = """ @@ -80,7 +76,7 @@ def models(self): } -class TestCachingUppercaseModel(BaseCachingUppercaseModel): +class TestCachingUppercaseModel(OracleBaseCaching): @pytest.fixture(scope="class") def models(self): @@ -89,7 +85,7 @@ def models(self): } -class TestCachingSelectedSchemaOnly(BaseCachingSelectedSchemaOnly): +class TestCachingSelectedSchemaOnly(OracleBaseCaching): @pytest.fixture(scope="class") def models(self): @@ -97,3 +93,8 @@ def models(self): "model.sql": model_sql, "another_schema_model.sql": another_schema_model_sql, } + + def test_cache(self, project): + # this should only cache the schema containing the selected model + run_args = ["--cache-selected-only", "run", "--select", "model"] + self.run_and_inspect_cache(project, run_args) From 7b78c5993665b5a9d942c1b6ff99975b4806beba Mon Sep 17 00:00:00 2001 From: Abhishek Singh Date: Tue, 9 May 2023 11:39:06 -0700 Subject: [PATCH 7/7] Updated copyright notice --- tests/functional/adapter/constraints/__init__.py | 1 - tests/functional/adapter/constraints/fixtures.py | 1 - tests/functional/adapter/constraints/test_constraints.py | 1 - tests/functional/adapter/simple_seed/__init__.py | 1 - tests/functional/adapter/simple_seed/test_simple_seed.py | 1 - tests/functional/adapter/test_caching.py | 1 - 6 files changed, 6 deletions(-) diff --git a/tests/functional/adapter/constraints/__init__.py b/tests/functional/adapter/constraints/__init__.py index 86eee3b..310bcac 100644 --- a/tests/functional/adapter/constraints/__init__.py +++ b/tests/functional/adapter/constraints/__init__.py @@ -1,6 +1,5 @@ """ Copyright (c) 2023, Oracle and/or its affiliates. -Copyright (c) 2020, Vitor Avancini Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/tests/functional/adapter/constraints/fixtures.py b/tests/functional/adapter/constraints/fixtures.py index a9280ab..fe49d40 100644 --- a/tests/functional/adapter/constraints/fixtures.py +++ b/tests/functional/adapter/constraints/fixtures.py @@ -1,6 +1,5 @@ """ Copyright (c) 2023, Oracle and/or its affiliates. -Copyright (c) 2020, Vitor Avancini Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/tests/functional/adapter/constraints/test_constraints.py b/tests/functional/adapter/constraints/test_constraints.py index 30bfacf..cc517de 100644 --- a/tests/functional/adapter/constraints/test_constraints.py +++ b/tests/functional/adapter/constraints/test_constraints.py @@ -1,6 +1,5 @@ """ Copyright (c) 2023, Oracle and/or its affiliates. -Copyright (c) 2020, Vitor Avancini Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/tests/functional/adapter/simple_seed/__init__.py b/tests/functional/adapter/simple_seed/__init__.py index 86eee3b..310bcac 100644 --- a/tests/functional/adapter/simple_seed/__init__.py +++ b/tests/functional/adapter/simple_seed/__init__.py @@ -1,6 +1,5 @@ """ Copyright (c) 2023, Oracle and/or its affiliates. -Copyright (c) 2020, Vitor Avancini Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/tests/functional/adapter/simple_seed/test_simple_seed.py b/tests/functional/adapter/simple_seed/test_simple_seed.py index a8825ce..740d257 100644 --- a/tests/functional/adapter/simple_seed/test_simple_seed.py +++ b/tests/functional/adapter/simple_seed/test_simple_seed.py @@ -1,6 +1,5 @@ """ Copyright (c) 2023, Oracle and/or its affiliates. -Copyright (c) 2020, Vitor Avancini Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/tests/functional/adapter/test_caching.py b/tests/functional/adapter/test_caching.py index 5f95d39..2c17891 100644 --- a/tests/functional/adapter/test_caching.py +++ b/tests/functional/adapter/test_caching.py @@ -1,6 +1,5 @@ """ Copyright (c) 2023, Oracle and/or its affiliates. -Copyright (c) 2020, Vitor Avancini Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License.