Ensure that every inline parser literal has its corresponding lexer rule

By following ANTLR's practice, every inline parser rule literal is converted
into a lexer rule. Either by finding an approrpiate existing lexer rule or
by artificially creating one and assigning its reference to the literal usage.

grammarinator/tool/processor.py

@@ -99,7 +99,12 @@ def __str__(self):
 
 class UnlexerRuleNode(RuleNode):
 
- def __init__(self, name):
+ _lit_cnt = 0
+
+ def __init__(self, name=None):
+ if not name:
+ name = f'T__{UnlexerRuleNode._lit_cnt}'
+ UnlexerRuleNode._lit_cnt += 1
  super().__init__(name, None, 'UnlexerRule')
  self.start_ranges = None
 
@@ -790,8 +795,16 @@ def build_expr(node, parent_id):
 
  if lexer_rule:
  rule.start_ranges.append((ord(src[0]), ord(src[0]) + 1))
-
- graph.add_edge(frm=parent_id, to=graph.add_node(LiteralNode(src=src)))
+ graph.add_edge(frm=parent_id, to=graph.add_node(LiteralNode(src=src)))
+ else:
+ # Ensure that every inline literal in parser rules has its lexer rule
+ # found or implicitly created.
+ lit_id = literal_lookup.get(src)
+ if not lit_id:
+ lit_id = graph.add_node(UnlexerRuleNode())
+ literal_lookup[src] = lit_id
+ graph.add_edge(frm=lit_id, to=graph.add_node(LiteralNode(src=src)))
+ graph.add_edge(frm=parent_id, to=lit_id)
 
  elif isinstance(node, ParserRuleContext) and node.getChildCount():
  for child in node.children:
@@ -802,6 +815,10 @@ def build_expr(node, parent_id):
 
  build_expr(node, rule.id)
 
+ # Save lexer rules with constant literals to enable resolving them in parser rules.
+ if lexer_rule and len(rule.out_edges) == 1 and isinstance(rule.out_edges[0].dst, LiteralNode):
+ literal_lookup[rule.out_edges[0].dst.src] = rule.id
+
  def build_prequel(node):
  assert isinstance(node, ANTLRv4Parser.GrammarSpecContext)
 
@@ -863,7 +880,8 @@ def build_rules(node):
  if duplicate_rules:
  raise ValueError(f'Rule redefinition(s): {", ".join(duplicate_rules)}')
 
- for rule_args in generator_rules:
+ # Ensure to process lexer rules first to lookup table from literal constants.
+ for rule_args in sorted(generator_rules, key=lambda r: int(isinstance(r[0], UnparserRuleNode))):
  build_rule(*rule_args)
 
  if default_rule:
@@ -883,6 +901,8 @@ def build_rules(node):
  dot_charset = Charset(Charset.dot[graph.dot])
  graph.charsets.append(dot_charset)
 
+ literal_lookup = {}
+
  for root in [lexer_root, parser_root]:
  if root:
  build_rules(root)

tests/grammars/CustomImplicitLiteralGenerator.py

@@ -0,0 +1,17 @@
+# Copyright (c) 2023 Renata Hodovan, Akos Kiss.
+#
+# Licensed under the BSD 3-Clause License
+# <LICENSE.rst or https://opensource.org/licenses/BSD-3-Clause>.
+# This file may not be copied, modified, or distributed except
+# according to those terms.
+
+# This custom unparser is used by ImplicitLiteral.g4
+
+from ImplicitLiteralGenerator import ImplicitLiteralGenerator
+
+
+class CustomImplicitLiteralGenerator(ImplicitLiteralGenerator):
+
+ def HELLO(self, parent=None):
+ super().HELLO(parent=parent)
+ self.hello_called = True

tests/grammars/ImplicitLiteral.g4

@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2023 Renata Hodovan, Akos Kiss.
+ *
+ * Licensed under the BSD 3-Clause License
+ * <LICENSE.rst or https://opensource.org/licenses/BSD-3-Clause>.
+ * This file may not be copied, modified, or distributed except
+ * according to those terms.
+ */
+
+/*
+ * Tests handling of implicit literals in parser rules.
+ */
+
+// TEST-PROCESS: {grammar}.g4 -o {tmpdir}
+// TEST-GENERATE: Custom{grammar}Generator.Custom{grammar}Generator -r start -j 1 -o {tmpdir}/{grammar}%d.txt
+
+
+grammar ImplicitLiteral;
+
+@members {
+def __init__(self, *, model=None, listeners=None, max_depth=inf):
+ super().__init__(model=model, listeners=listeners, max_depth=max_depth)
+ self.hello_called = False
+}
+
+
+start : 'hello' {assert self.hello_called, "Implicit lexer rule was not called."};
+HELLO : 'hello' ;