-
Notifications
You must be signed in to change notification settings - Fork 1.6k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Only create schemas for selected nodes (#1239) #1258
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -205,9 +205,11 @@ def do_skip(self, cause=None): | |
self.skip_cause = cause | ||
|
||
@classmethod | ||
def get_model_schemas(cls, manifest): | ||
def get_model_schemas(cls, manifest, selected_uids): | ||
schemas = set() | ||
for node in manifest.nodes.values(): | ||
if node.unique_id not in selected_uids: | ||
continue | ||
if cls.is_refable(node) and not cls.is_ephemeral(node): | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. is this going to cause a problem with Sources? I imagine sources are refable (idk if you actually made that change in the corresponding PR), but we definitely wouldn't want to touch source schemas at all! Might be worth being more explicit here? I'd only care about schemas for:
No change required necessarily, but wanted to surface it because i saw it :) There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Sources are not |
||
schemas.add((node.database, node.schema)) | ||
|
||
|
@@ -218,7 +220,7 @@ def before_hooks(self, config, adapter, manifest): | |
pass | ||
|
||
@classmethod | ||
def before_run(self, config, adapter, manifest): | ||
def before_run(self, config, adapter, manifest, selected_uids): | ||
pass | ||
|
||
@classmethod | ||
|
@@ -339,8 +341,8 @@ def safe_run_hooks(cls, config, adapter, manifest, hook_type, | |
raise | ||
|
||
@classmethod | ||
def create_schemas(cls, config, adapter, manifest): | ||
required_schemas = cls.get_model_schemas(manifest) | ||
def create_schemas(cls, config, adapter, manifest, selected_uids): | ||
required_schemas = cls.get_model_schemas(manifest, selected_uids) | ||
|
||
# Snowflake needs to issue a "use {schema}" query, where schema | ||
# is the one defined in the profile. Create this schema if it | ||
|
@@ -364,10 +366,10 @@ def populate_adapter_cache(cls, config, adapter, manifest): | |
adapter.set_relations_cache(manifest) | ||
|
||
@classmethod | ||
def before_run(cls, config, adapter, manifest): | ||
def before_run(cls, config, adapter, manifest, selected_uids): | ||
cls.populate_adapter_cache(config, adapter, manifest) | ||
cls.safe_run_hooks(config, adapter, manifest, RunHookType.Start, {}) | ||
cls.create_schemas(config, adapter, manifest) | ||
cls.create_schemas(config, adapter, manifest, selected_uids) | ||
|
||
@classmethod | ||
def print_results_line(cls, results, execution_time): | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
{{ | ||
config(schema='_and_then') | ||
}} | ||
|
||
select * from {{ this.schema }}.seed |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
what happened here?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
the existing
check_schema_exists
was wrong and did not work.