diff --git a/.gitignore b/.gitignore index 968536a770a..9b97c91563b 100644 --- a/.gitignore +++ b/.gitignore @@ -47,6 +47,9 @@ coverage.xml .hypothesis/ test.env +# Mypy +.mypy_cache/ + # Translations *.mo *.pot diff --git a/core/dbt/contracts/project.py b/core/dbt/contracts/project.py index cc9bda84bd1..e8ed83d53b1 100644 --- a/core/dbt/contracts/project.py +++ b/core/dbt/contracts/project.py @@ -142,7 +142,7 @@ class ProfileConfig(HyphenatedJsonSchemaMixin, Replaceable): config: UserConfig threads: int # TODO: make this a dynamic union of some kind? - credentials: Optional[Any] + credentials: Optional[Dict[str, Any]] @dataclass diff --git a/core/dbt/task/compile.py b/core/dbt/task/compile.py index 42ff1ce9a41..90dcbbef79b 100644 --- a/core/dbt/task/compile.py +++ b/core/dbt/task/compile.py @@ -1,6 +1,7 @@ import os import signal import threading +from typing import Union, List, Dict, Any from dbt.adapters.factory import get_adapter from dbt.clients.jinja import extract_toplevel_blocks @@ -162,3 +163,26 @@ def handle_request(self, name, sql, macros=None): self._raise_set_error() return self.node_results[0].to_dict() + + +class RemoteCompileProjectTask(CompileTask, RemoteCallable): + METHOD_NAME = 'compile_project' + + def __init__(self, args, config, manifest): + super().__init__(args, config) + self.manifest = manifest.deepcopy(config=config) + + def load_manifest(self): + # we started out with a manifest! + pass + + def handle_request( + self, + models: Union[None, str, List[str]] = None, + exclude: Union[None, str, List[str]] = None, + ) -> Dict[str, List[Any]]: + self.args.models = self._listify(models) + self.args.exclude = self._listify(exclude) + + results = self.run() + return {'results': [r.to_dict() for r in results]} diff --git a/core/dbt/task/rpc_server.py b/core/dbt/task/rpc_server.py index f106314a09d..506f798f020 100644 --- a/core/dbt/task/rpc_server.py +++ b/core/dbt/task/rpc_server.py @@ -7,8 +7,12 @@ from dbt.logger import RPC_LOGGER as logger from dbt.task.base import ConfiguredTask -from dbt.task.compile import CompileTask, RemoteCompileTask -from dbt.task.run import RemoteRunTask +from dbt.task.compile import ( + CompileTask, RemoteCompileTask, RemoteCompileProjectTask +) +from dbt.task.run import RemoteRunTask, RemoteRunProjectTask +from dbt.task.seed import RemoteSeedProjectTask +from dbt.task.test import RemoteTestProjectTask from dbt.utils import JSONEncoder from dbt import rpc @@ -19,7 +23,11 @@ def __init__(self, args, config, tasks=None): # compile locally self.manifest = self._compile_manifest() self.task_manager = rpc.TaskManager() - tasks = tasks or [RemoteCompileTask, RemoteRunTask] + tasks = tasks or [ + RemoteCompileTask, RemoteCompileProjectTask, + RemoteRunTask, RemoteRunProjectTask, + RemoteSeedProjectTask, RemoteTestProjectTask + ] for cls in tasks: task = cls(args, config, self.manifest) self.task_manager.add_task_handler(task) diff --git a/core/dbt/task/run.py b/core/dbt/task/run.py index be1e2306601..deeaeb81b60 100644 --- a/core/dbt/task/run.py +++ b/core/dbt/task/run.py @@ -1,7 +1,6 @@ -from __future__ import print_function - import functools import time +from typing import Union, List, Dict, Any from dbt.logger import GLOBAL_LOGGER as logger from dbt.node_types import NodeType, RunHookType @@ -19,6 +18,7 @@ from dbt.compilation import compile_node from dbt.task.compile import CompileTask, RemoteCompileTask +from dbt.task.runnable import RemoteCallable from dbt.utils import get_nodes_by_tags @@ -183,3 +183,26 @@ class RemoteRunTask(RemoteCompileTask, RunTask): def get_runner_type(self): return RPCExecuteRunner + + +class RemoteRunProjectTask(RunTask, RemoteCallable): + METHOD_NAME = 'run_project' + + def __init__(self, args, config, manifest): + super().__init__(args, config) + self.manifest = manifest.deepcopy(config=config) + + def load_manifest(self): + # we started out with a manifest! + pass + + def handle_request( + self, + models: Union[None, str, List[str]] = None, + exclude: Union[None, str, List[str]] = None, + ) -> Dict[str, List[Any]]: + self.args.models = self._listify(models) + self.args.exclude = self._listify(exclude) + + results = self.run() + return {'results': [r.to_dict() for r in results]} diff --git a/core/dbt/task/runnable.py b/core/dbt/task/runnable.py index a8c65ad669b..deb4faf7857 100644 --- a/core/dbt/task/runnable.py +++ b/core/dbt/task/runnable.py @@ -4,6 +4,7 @@ from abc import abstractmethod from datetime import datetime from multiprocessing.dummy import Pool as ThreadPool +from typing import Union, List, Optional from dbt import rpc from dbt.task.base import ConfiguredTask @@ -40,10 +41,16 @@ def __init__(self, args, config): self.manifest = None self.linker = None - def _runtime_initialize(self): + def load_manifest(self): self.manifest = load_manifest(self.config) + + def compile_manifest(self): self.linker = compile_manifest(self.config, self.manifest) + def _runtime_initialize(self): + self.load_manifest() + self.compile_manifest() + class GraphRunnableTask(ManifestTask): def __init__(self, args, config): @@ -344,6 +351,17 @@ def handle_request(self, **kwargs): 'from_kwargs not implemented' ) + @staticmethod + def _listify( + value: Optional[Union[str, List[str]]] + ) -> Optional[List[str]]: + if value is None: + return None + elif isinstance(value, str): + return [value] + else: + return value + def decode_sql(self, sql): """Base64 decode a string. This should only be used for sql in calls. diff --git a/core/dbt/task/seed.py b/core/dbt/task/seed.py index 1101da51283..9ce24ae0d7b 100644 --- a/core/dbt/task/seed.py +++ b/core/dbt/task/seed.py @@ -1,8 +1,11 @@ import random +from typing import List, Dict, Any + from dbt.logger import GLOBAL_LOGGER as logger from dbt.node_runners import SeedRunner from dbt.node_types import NodeType from dbt.task.run import RunTask +from dbt.task.runnable import RemoteCallable import dbt.ui.printer @@ -44,3 +47,21 @@ def show_tables(self, results): for result in results: if result.error is None: self.show_table(result) + + +class RemoteSeedProjectTask(SeedTask, RemoteCallable): + METHOD_NAME = 'seed_project' + + def __init__(self, args, config, manifest): + super().__init__(args, config) + self.manifest = manifest.deepcopy(config=config) + + def load_manifest(self): + # we started out with a manifest! + pass + + def handle_request(self, show: bool = False) -> Dict[str, List[Any]]: + self.args.show = show + + results = self.run() + return {'results': [r.to_dict() for r in results]} diff --git a/core/dbt/task/test.py b/core/dbt/task/test.py index f20b9897924..e2290f0bd22 100644 --- a/core/dbt/task/test.py +++ b/core/dbt/task/test.py @@ -1,6 +1,9 @@ +from typing import Union, List, Dict, Any + from dbt.node_runners import TestRunner from dbt.node_types import NodeType from dbt.task.run import RunTask +from dbt.task.runnable import RemoteCallable class TestTask(RunTask): @@ -12,11 +15,8 @@ class TestTask(RunTask): def raise_on_first_error(self): return False - def before_run(self, adapter, selected_uids): + def safe_run_hooks(self, adapter, hook_type, extra_context): # Don't execute on-run-* hooks for tests - self.populate_adapter_cache(adapter) - - def after_run(self, adapter, results): pass def build_query(self): @@ -41,3 +41,30 @@ def build_query(self): def get_runner_type(self): return TestRunner + + +class RemoteTestProjectTask(TestTask, RemoteCallable): + METHOD_NAME = 'test_project' + + def __init__(self, args, config, manifest): + super().__init__(args, config) + self.manifest = manifest.deepcopy(config=config) + + def load_manifest(self): + # we started out with a manifest! + pass + + def handle_request( + self, + models: Union[None, str, List[str]] = None, + exclude: Union[None, str, List[str]] = None, + data: bool = False, + schema: bool = False, + ) -> Dict[str, List[Any]]: + self.args.models = self._listify(models) + self.args.exclude = self._listify(exclude) + self.args.data = data + self.args.schema = schema + + results = self.run() + return {'results': [r.to_dict() for r in results]} diff --git a/test/integration/042_sources_test/seed.sql b/test/integration/042_sources_test/seed.sql new file mode 100644 index 00000000000..40110b99088 --- /dev/null +++ b/test/integration/042_sources_test/seed.sql @@ -0,0 +1,113 @@ +create table {schema}.seed_expected ( + favorite_color TEXT, + id INTEGER, + first_name TEXT, + email TEXT, + ip_address TEXT, + updated_at TIMESTAMP WITHOUT TIME ZONE +); + + +INSERT INTO {schema}.seed_expected + ("favorite_color","id","first_name","email","ip_address","updated_at") +VALUES + ('blue',1,'Larry','lking0@miitbeian.gov.cn','''69.135.206.194''','2008-09-12 19:08:31'), + ('blue',2,'Larry','lperkins1@toplist.cz','''64.210.133.162''','1978-05-09 04:15:14'), + ('blue',3,'Anna','amontgomery2@miitbeian.gov.cn','''168.104.64.114''','2011-10-16 04:07:57'), + ('blue',4,'Sandra','sgeorge3@livejournal.com','''229.235.252.98''','1973-07-19 10:52:43'), + ('blue',5,'Fred','fwoods4@google.cn','''78.229.170.124''','2012-09-30 16:38:29'), + ('blue',6,'Stephen','shanson5@livejournal.com','''182.227.157.105''','1995-11-07 21:40:50'), + ('blue',7,'William','wmartinez6@upenn.edu','''135.139.249.50''','1982-09-05 03:11:59'), + ('blue',8,'Jessica','jlong7@hao123.com','''203.62.178.210''','1991-10-16 11:03:15'), + ('blue',9,'Douglas','dwhite8@tamu.edu','''178.187.247.1''','1979-10-01 09:49:48'), + ('blue',10,'Lisa','lcoleman9@nydailynews.com','''168.234.128.249''','2011-05-26 07:45:49'), + ('blue',11,'Ralph','rfieldsa@home.pl','''55.152.163.149''','1972-11-18 19:06:11'), + ('blue',12,'Louise','lnicholsb@samsung.com','''141.116.153.154''','2014-11-25 20:56:14'), + ('blue',13,'Clarence','cduncanc@sfgate.com','''81.171.31.133''','2011-11-17 07:02:36'), + ('blue',14,'Daniel','dfranklind@omniture.com','''8.204.211.37''','1980-09-13 00:09:04'), + ('blue',15,'Katherine','klanee@auda.org.au','''176.96.134.59''','1997-08-22 19:36:56'), + ('blue',16,'Billy','bwardf@wikia.com','''214.108.78.85''','2003-10-19 02:14:47'), + ('blue',17,'Annie','agarzag@ocn.ne.jp','''190.108.42.70''','1988-10-28 15:12:35'), + ('blue',18,'Shirley','scolemanh@fastcompany.com','''109.251.164.84''','1988-08-24 10:50:57'), + ('blue',19,'Roger','rfrazieri@scribd.com','''38.145.218.108''','1985-12-31 15:17:15'), + ('blue',20,'Lillian','lstanleyj@goodreads.com','''47.57.236.17''','1970-06-08 02:09:05'), + ('blue',21,'Aaron','arodriguezk@nps.gov','''205.245.118.221''','1985-10-11 23:07:49'), + ('blue',22,'Patrick','pparkerl@techcrunch.com','''19.8.100.182''','2006-03-29 12:53:56'), + ('blue',23,'Phillip','pmorenom@intel.com','''41.38.254.103''','2011-11-07 15:35:43'), + ('blue',24,'Henry','hgarcian@newsvine.com','''1.191.216.252''','2008-08-28 08:30:44'), + ('blue',25,'Irene','iturnero@opera.com','''50.17.60.190''','1994-04-01 07:15:02'), + ('blue',26,'Andrew','adunnp@pen.io','''123.52.253.176''','2000-11-01 06:03:25'), + ('blue',27,'David','dgutierrezq@wp.com','''238.23.203.42''','1988-01-25 07:29:18'), + ('blue',28,'Henry','hsanchezr@cyberchimps.com','''248.102.2.185''','1983-01-01 13:36:37'), + ('blue',29,'Evelyn','epetersons@gizmodo.com','''32.80.46.119''','1979-07-16 17:24:12'), + ('blue',30,'Tammy','tmitchellt@purevolume.com','''249.246.167.88''','2001-04-03 10:00:23'), + ('blue',31,'Jacqueline','jlittleu@domainmarket.com','''127.181.97.47''','1986-02-11 21:35:50'), + ('blue',32,'Earl','eortizv@opera.com','''166.47.248.240''','1996-07-06 08:16:27'), + ('blue',33,'Juan','jgordonw@sciencedirect.com','''71.77.2.200''','1987-01-31 03:46:44'), + ('blue',34,'Diane','dhowellx@nyu.edu','''140.94.133.12''','1994-06-11 02:30:05'), + ('blue',35,'Randy','rkennedyy@microsoft.com','''73.255.34.196''','2005-05-26 20:28:39'), + ('blue',36,'Janice','jriveraz@time.com','''22.214.227.32''','1990-02-09 04:16:52'), + ('blue',37,'Laura','lperry10@diigo.com','''159.148.145.73''','2015-03-17 05:59:25'), + ('blue',38,'Gary','gray11@statcounter.com','''40.193.124.56''','1970-01-27 10:04:51'), + ('blue',39,'Jesse','jmcdonald12@typepad.com','''31.7.86.103''','2009-03-14 08:14:29'), + ('blue',40,'Sandra','sgonzalez13@goodreads.com','''223.80.168.239''','1993-05-21 14:08:54'), + ('blue',41,'Scott','smoore14@archive.org','''38.238.46.83''','1980-08-30 11:16:56'), + ('blue',42,'Phillip','pevans15@cisco.com','''158.234.59.34''','2011-12-15 23:26:31'), + ('blue',43,'Steven','sriley16@google.ca','''90.247.57.68''','2011-10-29 19:03:28'), + ('blue',44,'Deborah','dbrown17@hexun.com','''179.125.143.240''','1995-04-10 14:36:07'), + ('blue',45,'Lori','lross18@ow.ly','''64.80.162.180''','1980-12-27 16:49:15'), + ('blue',46,'Sean','sjackson19@tumblr.com','''240.116.183.69''','1988-06-12 21:24:45'), + ('blue',47,'Terry','tbarnes1a@163.com','''118.38.213.137''','1997-09-22 16:43:19'), + ('blue',48,'Dorothy','dross1b@ebay.com','''116.81.76.49''','2005-02-28 13:33:24'), + ('blue',49,'Samuel','swashington1c@house.gov','''38.191.253.40''','1989-01-19 21:15:48'), + ('blue',50,'Ralph','rcarter1d@tinyurl.com','''104.84.60.174''','2007-08-11 10:21:49'), + ('green',51,'Wayne','whudson1e@princeton.edu','''90.61.24.102''','1983-07-03 16:58:12'), + ('green',52,'Rose','rjames1f@plala.or.jp','''240.83.81.10''','1995-06-08 11:46:23'), + ('green',53,'Louise','lcox1g@theglobeandmail.com','''105.11.82.145''','2016-09-19 14:45:51'), + ('green',54,'Kenneth','kjohnson1h@independent.co.uk','''139.5.45.94''','1976-08-17 11:26:19'), + ('green',55,'Donna','dbrown1i@amazon.co.uk','''19.45.169.45''','2006-05-27 16:51:40'), + ('green',56,'Johnny','jvasquez1j@trellian.com','''118.202.238.23''','1975-11-17 08:42:32'), + ('green',57,'Patrick','pramirez1k@tamu.edu','''231.25.153.198''','1997-08-06 11:51:09'), + ('green',58,'Helen','hlarson1l@prweb.com','''8.40.21.39''','1993-08-04 19:53:40'), + ('green',59,'Patricia','pspencer1m@gmpg.org','''212.198.40.15''','1977-08-03 16:37:27'), + ('green',60,'Joseph','jspencer1n@marriott.com','''13.15.63.238''','2005-07-23 20:22:06'), + ('green',61,'Phillip','pschmidt1o@blogtalkradio.com','''177.98.201.190''','1976-05-19 21:47:44'), + ('green',62,'Joan','jwebb1p@google.ru','''105.229.170.71''','1972-09-07 17:53:47'), + ('green',63,'Phyllis','pkennedy1q@imgur.com','''35.145.8.244''','2000-01-01 22:33:37'), + ('green',64,'Katherine','khunter1r@smh.com.au','''248.168.205.32''','1991-01-09 06:40:24'), + ('green',65,'Laura','lvasquez1s@wiley.com','''128.129.115.152''','1997-10-23 12:04:56'), + ('green',66,'Juan','jdunn1t@state.gov','''44.228.124.51''','2004-11-10 05:07:35'), + ('green',67,'Judith','jholmes1u@wiley.com','''40.227.179.115''','1977-08-02 17:01:45'), + ('green',68,'Beverly','bbaker1v@wufoo.com','''208.34.84.59''','2016-03-06 20:07:23'), + ('green',69,'Lawrence','lcarr1w@flickr.com','''59.158.212.223''','1988-09-13 06:07:21'), + ('green',70,'Gloria','gwilliams1x@mtv.com','''245.231.88.33''','1995-03-18 22:32:46'), + ('green',71,'Steven','ssims1y@cbslocal.com','''104.50.58.255''','2001-08-05 21:26:20'), + ('green',72,'Betty','bmills1z@arstechnica.com','''103.177.214.220''','1981-12-14 21:26:54'), + ('green',73,'Mildred','mfuller20@prnewswire.com','''151.158.8.130''','2000-04-19 10:13:55'), + ('green',74,'Donald','dday21@icq.com','''9.178.102.255''','1972-12-03 00:58:24'), + ('green',75,'Eric','ethomas22@addtoany.com','''85.2.241.227''','1992-11-01 05:59:30'), + ('green',76,'Joyce','jarmstrong23@sitemeter.com','''169.224.20.36''','1985-10-24 06:50:01'), + ('green',77,'Maria','mmartinez24@amazonaws.com','''143.189.167.135''','2005-10-05 05:17:42'), + ('green',78,'Harry','hburton25@youtube.com','''156.47.176.237''','1978-03-26 05:53:33'), + ('green',79,'Kevin','klawrence26@hao123.com','''79.136.183.83''','1994-10-12 04:38:52'), + ('green',80,'David','dhall27@prweb.com','''133.149.172.153''','1976-12-15 16:24:24'), + ('green',81,'Kathy','kperry28@twitter.com','''229.242.72.228''','1979-03-04 02:58:56'), + ('green',82,'Adam','aprice29@elegantthemes.com','''13.145.21.10''','1982-11-07 11:46:59'), + ('green',83,'Brandon','bgriffin2a@va.gov','''73.249.128.212''','2013-10-30 05:30:36'), + ('green',84,'Henry','hnguyen2b@discovery.com','''211.36.214.242''','1985-01-09 06:37:27'), + ('green',85,'Eric','esanchez2c@edublogs.org','''191.166.188.251''','2004-05-01 23:21:42'), + ('green',86,'Jason','jlee2d@jimdo.com','''193.92.16.182''','1973-01-08 09:05:39'), + ('green',87,'Diana','drichards2e@istockphoto.com','''19.130.175.245''','1994-10-05 22:50:49'), + ('green',88,'Andrea','awelch2f@abc.net.au','''94.155.233.96''','2002-04-26 08:41:44'), + ('green',89,'Louis','lwagner2g@miitbeian.gov.cn','''26.217.34.111''','2003-08-25 07:56:39'), + ('green',90,'Jane','jsims2h@seesaa.net','''43.4.220.135''','1987-03-20 20:39:04'), + ('green',91,'Larry','lgrant2i@si.edu','''97.126.79.34''','2000-09-07 20:26:19'), + ('green',92,'Louis','ldean2j@prnewswire.com','''37.148.40.127''','2011-09-16 20:12:14'), + ('green',93,'Jennifer','jcampbell2k@xing.com','''38.106.254.142''','1988-07-15 05:06:49'), + ('green',94,'Wayne','wcunningham2l@google.com.hk','''223.28.26.187''','2009-12-15 06:16:54'), + ('green',95,'Lori','lstevens2m@icq.com','''181.250.181.58''','1984-10-28 03:29:19'), + ('green',96,'Judy','jsimpson2n@marriott.com','''180.121.239.219''','1986-02-07 15:18:10'), + ('green',97,'Phillip','phoward2o@usa.gov','''255.247.0.175''','2002-12-26 08:44:45'), + ('green',98,'Gloria','gwalker2p@usa.gov','''156.140.7.128''','1997-10-04 07:58:58'), + ('green',99,'Paul','pjohnson2q@umn.edu','''183.59.198.197''','1991-11-14 12:33:55'), + ('green',100,'Frank','fgreene2r@blogspot.com','''150.143.68.121''','2010-06-12 23:55:39'); diff --git a/test/integration/042_sources_test/test_sources.py b/test/integration/042_sources_test/test_sources.py index c23dddaef63..516414ccbe9 100644 --- a/test/integration/042_sources_test/test_sources.py +++ b/test/integration/042_sources_test/test_sources.py @@ -401,9 +401,12 @@ def setUp(self): port=port ) self._server.start() + self.background_queries = [] def tearDown(self): self._server.terminate() + for query in self.background_queries: + query.terminate() super().tearDown() @property @@ -459,6 +462,7 @@ def background_query(self, _method, _sql=None, _test_request_id=1, if 'name' in kwargs: name += ' ' + kwargs['name'] bg_query = BackgroundQueryProcess(built, url, name=name) + self.background_queries.append(bg_query) bg_query.start() return bg_query @@ -913,3 +917,60 @@ def test_timeout_postgres(self): # on windows, process start is so slow that frequently we won't have collected any logs if os.name != 'nt': self.assertTrue(len(error_data['logs']) > 0) + + @use_profile('postgres') + def test_seed_project_postgres(self): + # testing "dbt seed" is tricky so we'll just jam some sql in there + self.run_sql_file("seed.sql") + result = self.query('seed_project', show=True).json() + dct = self.assertIsResult(result) + self.assertTablesEqual('source', 'seed_expected') + self.assertIn('results', dct) + results = dct['results'] + self.assertEqual(len(results), 4) + self.assertEqual( + set(r['node']['name'] for r in results), + {'expected_multi_source', 'other_source_table', 'other_table', 'source'} + ) + + @use_profile('postgres') + def test_compile_project_postgres(self): + self.run_dbt_with_vars(['seed']) + result = self.query('compile_project').json() + dct = self.assertIsResult(result) + self.assertIn('results', dct) + results = dct['results'] + self.assertEqual(len(results), 11) + compiled = set(r['node']['name'] for r in results) + self.assertTrue(compiled.issuperset( + {'descendant_model', 'multi_source_model', 'nonsource_descendant'} + )) + self.assertNotIn('ephemeral_model', compiled) + + @use_profile('postgres') + def test_run_project_postgres(self): + self.run_dbt_with_vars(['seed']) + result = self.query('run_project').json() + dct = self.assertIsResult(result) + self.assertIn('results', dct) + results = dct['results'] + self.assertEqual(len(results), 3) + self.assertEqual( + set(r['node']['name'] for r in results), + {'descendant_model', 'multi_source_model', 'nonsource_descendant'} + ) + self.assertTablesEqual('multi_source_model', 'expected_multi_source') + + @use_profile('postgres') + def test_test_project_postgres(self): + self.run_dbt_with_vars(['seed']) + result = self.query('run_project').json() + dct = self.assertIsResult(result) + result = self.query('test_project').json() + dct = self.assertIsResult(result) + self.assertIn('results', dct) + results = dct['results'] + self.assertEqual(len(results), 4) + for result in results: + self.assertEqual(result['status'], 0.0) + self.assertNotIn('fail', result)