Merge pull request #1375 from fishtown-analytics/feature/inline-macro…

…s-models

Support macros in execute/compile tasks (#1372)

core/dbt/clients/_jinja_blocks.py

@@ -0,0 +1,380 @@
+import re
+
+import dbt.exceptions
+
+
+def regex(pat):
+ return re.compile(pat, re.DOTALL | re.MULTILINE)
+
+
+class BlockData(object):
+ """raw plaintext data from the top level of the file."""
+ def __init__(self, contents):
+ self.block_type_name = '__dbt__data'
+ self.contents = contents
+ self.full_block = contents
+
+
+class BlockTag(object):
+ def __init__(self, block_type_name, block_name, contents=None, **kw):
+ self.block_type_name = block_type_name
+ self.block_name = block_name
+ self.contents = contents
+ self.full_block = None
+
+ def __str__(self):
+ return 'BlockTag({!r}, {!r})'.format(self.block_type_name,
+ self.block_name)
+
+ def __repr__(self):
+ return str(self)
+
+ @property
+ def end_block_type_name(self):
+ return 'end{}'.format(self.block_type_name)
+
+ def end_pat(self):
+ # we don't want to use string formatting here because jinja uses most
+ # of the string formatting operators in its syntax...
+ pattern = ''.join((
+ r'(?P<endblock>((?:\s*\{\%\-|\{\%)\s*',
+ self.end_block_type_name,
+ r'\s*(?:\-\%\}\s*|\%\})))',
+ ))
+ return regex(pattern)
+
+
+_NAME_PATTERN = r'[A-Za-z_][A-Za-z_0-9]*'
+
+COMMENT_START_PATTERN = regex(r'(?:(?P<comment_start>(\s*\{\#)))')
+COMMENT_END_PATTERN = regex(r'(.*?)(\s*\#\})')
+RAW_START_PATTERN = regex(
+ r'(?:\s*\{\%\-|\{\%)\s*(?P<raw_start>(raw))\s*(?:\-\%\}\s*|\%\})'
+)
+
+BLOCK_START_PATTERN = regex(''.join((
+ r'(?:\s*\{\%\-|\{\%)\s*',
+ r'(?P<block_type_name>({}))'.format(_NAME_PATTERN),
+ # some blocks have a 'block name'.
+ r'(?:\s+(?P<block_name>({})))?'.format(_NAME_PATTERN),
+)))
+
+TAG_CLOSE_PATTERN = regex(r'(?:\-\%\}\s*|\%\})')
+# if you do {% materialization foo, adapter="myadapter' %} and end up with
+# mismatched quotes this will still match, but jinja will fail somewhere
+# since the adapter= argument has to be an adapter name, and none have quotes
+# or anything else in them. So this should be fine.
+MATERIALIZATION_ARGS_PATTERN = regex(
+ r'\s*,\s*'
+ r'''(?P<adpater_arg>(adapter=(?:['"]{}['"])|default))'''
+ .format(_NAME_PATTERN)
+)
+# macros an stuff like macros get open parents, followed by a very complicated
+# argument spec! In fact, it's easiest to parse it in tiny little chunks
+# because we have to handle awful stuff like string parsing ;_;
+MACRO_ARGS_START_PATTERN = regex(r'\s*(?P<macro_start>\()\s*')
+MACRO_ARGS_END_PATTERN = regex(r'\s*(?P<macro_end>(\)))\s*')
+
+# macros can be like {% macro foo(bar) %} or {% macro foo(bar, baz) %} or
+# {% macro foo(bar, baz="quux") %} or ...
+# I think jinja disallows default values after required (like Python), but we
+# can ignore that and let jinja deal
+MACRO_ARG_PATTERN = regex(''.join((
+ r'\s*(?P<macro_arg_name>({}))\s*',
+ r'((?P<value>=)|(?P<more_args>,)?)\s*'.format(_NAME_PATTERN),
+)))
+
+# stolen from jinja's lexer. Note that we've consumed all prefix whitespace by
+# the time we want to use this.
+STRING_PATTERN = regex(
+ r"(?P<string>('([^'\\]*(?:\\.[^'\\]*)*)'|"
+ r'"([^"\\]*(?:\\.[^"\\]*)*)"))'
+)
+
+# any number of non-quote characters, followed by:
+# - quote: a quote mark indicating start of a string (you'll want to backtrack
+# the regex end on quotes and then match with the string pattern)
+# - a comma (so there will be another full argument)
+# - a closing parenthesis (you can now expect a closing tag)
+NON_STRING_MACRO_ARGS_PATTERN = regex(
+ # anything, followed by a quote, open/close paren, or comma
+ r'''(.*?)'''
+ r'''((?P<quote>(['"]))|(?P<open>(\())|(?P<close>(\)))|(?P<comma>(\,)))'''
+)
+
+
+class BlockIterator(object):
+ def __init__(self, data):
+ self.data = data
+ self.blocks = []
+ self._block_contents = None
+ self._parenthesis_stack = []
+ self.pos = 0
+
+ def advance(self, new_position):
+ blk = self.data[self.pos:new_position]
+
+ if self._block_contents is not None:
+ self._block_contents += blk
+
+ self.pos = new_position
+
+ def rewind(self, amount=1):
+ if self._block_contents is not None:
+ self._block_contents = self._block_contents[:-amount]
+
+ self.pos -= amount
+
+ def _search(self, pattern):
+ return pattern.search(self.data, self.pos)
+
+ def _match(self, pattern):
+ return pattern.match(self.data, self.pos)
+
+ def expect_comment_end(self):
+ """Expect a comment end and return the match object.
+ """
+ match = self._match(COMMENT_END_PATTERN)
+ if match is None:
+ dbt.exceptions.raise_compiler_error('unexpected EOF, expected #}')
+ self.advance(match.end())
+
+ def expect_raw_end(self):
+ end_pat = BlockTag('raw', None).end_pat()
+ match = self._search(end_pat)
+ if match is None:
+ dbt.exceptions.raise_compiler_error(
+ 'unexpected EOF, expected {% endraw %}'
+ )
+ self.advance(match.end())
+
+ def _first_match(self, *patterns, **kwargs):
+ matches = []
+ for pattern in patterns:
+ # default to 'search', but sometimes we want to 'match'.
+ if kwargs.get('method', 'search') == 'search':
+ match = self._search(pattern)
+ else:
+ match = self._match(pattern)
+ if match:
+ matches.append(match)
+ if not matches:
+ return None
+ # if there are multiple matches, pick the least greedy match
+ # TODO: do I need to account for m.start(), or is this ok?
+ return min(matches, key=lambda m: m.end())
+
+ def _expect_match(self, expected_name, *patterns, **kwargs):
+ match = self._first_match(*patterns, **kwargs)
+ if match is None:
+ msg = 'unexpected EOF, expected {}, got "{}"'.format(
+ expected_name, self.data[self.pos:]
+ )
+ dbt.exceptions.raise_compiler_error(msg)
+ return match
+
+ def handle_block(self, match, block_start=None):
+ """Handle a block. The current state of the parser should be after the
+ open block is completed:
+ {% blk foo %}my data {% endblk %}
+ ^ right here
+ """
+ # we have to handle comments inside blocks because you could do this:
+ # {% blk foo %}asdf {# {% endblk %} #} {%endblk%}
+ # they still end up in the data/raw_data of the block itself, but we
+ # have to know to ignore stuff until the end comment marker!
+ found = BlockTag(**match.groupdict())
+ # the full block started at the given match start, which may include
+ # prefixed whitespace! we'll strip it later
+ if block_start is None:
+ block_start = match.start()
+
+ self._block_contents = ''
+
+ # you can have as many comments in your block as you'd like!
+ while True:
+ match = self._expect_match(
+ '"{}"'.format(found.end_block_type_name),
+ found.end_pat(), COMMENT_START_PATTERN, RAW_START_PATTERN
+ )
+ groups = match.groupdict()
+ if groups.get('endblock') is not None:
+ break
+
+ self.advance(match.end())
+
+ if groups.get('comment_start') is not None:
+ self.expect_comment_end()
+ elif groups.get('raw_start') is not None:
+ self.expect_raw_end()
+ else:
+ raise dbt.exceptions.InternalException(
+ 'unhandled regex in handle_block, no match: {}'
+ .format(groups)
+ )
+
+ # we want to advance to just the end tag at first, to extract the
+ # contents
+ self.advance(match.start())
+ found.contents = self._block_contents
+ self._block_contents = None
+ # now advance to the end
+ self.advance(match.end())
+ found.full_block = self.data[block_start:self.pos]
+ return found
+
+ def handle_materialization(self, match):
+ self._expect_match('materialization args',
+ MATERIALIZATION_ARGS_PATTERN)
+ self._expect_match('%}', TAG_CLOSE_PATTERN)
+ # handle the block we started with!
+ self.blocks.append(self.handle_block(match))
+
+ def find_block(self):
+ open_block = (
+ r'(?:\s*\{\%\-|\{\%)\s*'
+ r'(?P<block_type_name>([A-Za-z_][A-Za-z_0-9]*))'
+ # some blocks have a 'block name'.
+ r'(?:\s+(?P<block_name>([A-Za-z_][A-Za-z_0-9]*)))?'
+ )
+
+ match = self._first_match(regex(open_block), COMMENT_START_PATTERN)
+ if match is None:
+ return False
+
+ raw_toplevel = self.data[self.pos:match.start()]
+ if len(raw_toplevel) > 0:
+ self.blocks.append(BlockData(raw_toplevel))
+
+ matchgroups = match.groupdict()
+
+ # comments are easy
+ if matchgroups.get('comment_start') is not None:
+ start = match.start()
+ self.expect_comment_end()
+ self.blocks.append(BlockData(self.data[start:self.pos]))
+ return True
+
+ if matchgroups.get('block_type_name') == 'raw':
+ start = match.start()
+ self.expect_raw_end()
+ self.blocks.append(BlockData(self.data[start:self.pos]))
+ return True
+
+ if matchgroups.get('block_type_name') == 'materialization':
+ self.advance(match.end())
+ self.handle_materialization(match)
+ return True
+
+ # we're somewhere like this {% block_type_name block_type
+ # we've either got arguments, a close of tag (%}), or bad input.
+ # we've handled materializations already (they're weird!)
+ # thankfully, comments aren't allowed *inside* a block def...
+ block_end_match = self._expect_match('%} or (...)',
+ TAG_CLOSE_PATTERN,
+ MACRO_ARGS_START_PATTERN)
+ self.advance(block_end_match.end())
+ if block_end_match.groupdict().get('macro_start') is not None:
+ # we've hit our first parenthesis!
+ self._parenthesis_stack = [True]
+ self._process_macro_args()
+ self.advance(self._expect_match('%}', TAG_CLOSE_PATTERN).end())
+
+ # tag close time!
+ self.blocks.append(self.handle_block(match))
+ return True
+
+ def _process_macro_default_arg(self):
+ """Handle the bit after an '=' in a macro default argument. This is
+ probably the trickiest thing. The goal here is to accept all strings
+ jinja would accept and always handle block start/end correctly: It's
+ fine to have false positives, jinja can fail later.
+
+ Return True if there are more arguments expected.
+ """
+ while self._parenthesis_stack:
+ match = self._expect_match(
+ 'macro argument',
+ # you could have a string
+ STRING_PATTERN,
+ # a quote, a comma, or a close parenthesis
+ NON_STRING_MACRO_ARGS_PATTERN,
+ # we want to "match", not "search"
+ method='match'
+ )
+ matchgroups = match.groupdict()
+ self.advance(match.end())
+ if matchgroups.get('string') is not None:
+ # we got a string value. There could be more data.
+ continue
+ elif matchgroups.get('quote') is not None:
+ # we got a bunch of data and then a string opening value.
+ # put the quote back on the menu
+ self.rewind()
+ # if we only got a single quote mark and didn't hit a string
+ # at all, this file has an unclosed quote. Fail accordingly.
+ if match.end() - match.start() == 1:
+ msg = (
+ 'Unclosed quotation mark at position {}. Context:\n{}'
+ .format(self.pos, self.data[self.pos-20:self.pos+20])
+ )
+ dbt.exceptions.raise_compiler_error(msg)
+ elif matchgroups.get('comma') is not None:
+ # small hack: if we hit a comma and there is one parenthesis
+ # left, return to look for a new name. otherwise we're still
+ # looking for the parameter close.
+ if len(self._parenthesis_stack) == 1:
+ return
+ elif matchgroups.get('close'):
+ self._parenthesis_stack.pop()
+ else:
+ raise dbt.exceptions.InternalException(
+ 'unhandled regex in _process_macro_default_arg(), no match'
+ ': {}'.format(matchgroups)
+ )
+
+ def _process_macro_args(self):
+ """Macro args are pretty tricky! Arg names themselves are simple, but
+ you can set arbitrary default values, including doing stuff like:
+ {% macro my_macro(arg="x" + ("}% {# {% endmacro %}" * 2)) %}
+
+ Which makes you a jerk, but is valid jinja.
+ """
+ # we are currently after the first parenthesis (+ any whitespace) after
+ # the macro args started. You can either have the close paren, or a
+ # name.
+ while self._parenthesis_stack:
+ match = self._expect_match('macro arguments',
+ MACRO_ARGS_END_PATTERN,
+ MACRO_ARG_PATTERN)
+ self.advance(match.end())
+ matchgroups = match.groupdict()
+ if matchgroups.get('macro_end') is not None:
+ self._parenthesis_stack.pop()
+ # we got an argument. let's see what it has
+ elif matchgroups.get('value') is not None:
+ # we have to process a single macro argument. This mutates
+ # the parenthesis stack! If it finds a comma, it will continue
+ # the loop.
+ self._process_macro_default_arg()
+ elif matchgroups.get('more_args') is not None:
+ continue
+ else:
+ raise dbt.exceptions.InternalException(
+ 'unhandled regex in _process_macro_args(), no match: {}'
+ .format(matchgroups)
+ )
+ # if there are more arguments or a macro arg end we'll catch them
+ # on the next loop around
+
+ def lex_for_blocks(self):
+ while self.data[self.pos:]:
+ found = self.find_block()
+ if not found:
+ break
+
+ raw_toplevel = self.data[self.pos:]
+ if len(raw_toplevel) > 0:
+ self.blocks.append(BlockData(raw_toplevel))
+
+ return self.blocks

core/dbt/clients/jinja.py

@@ -12,8 +12,7 @@
 import dbt.compat
 import dbt.exceptions
 
-from dbt.node_types import NodeType
-from dbt.utils import AttrDict
+from dbt.clients._jinja_blocks import BlockIterator
 
 from dbt.logger import GLOBAL_LOGGER as logger # noqa
 
@@ -281,3 +280,7 @@ def get_rendered(string, ctx, node=None,
 
 def undefined_error(msg):
  raise jinja2.exceptions.UndefinedError(msg)
+
+
+def extract_toplevel_blocks(data):
+ return BlockIterator(data).lex_for_blocks()