More updates to v1.0

lark/common.py

@@ -1,14 +1,14 @@
-from types import ModuleType
 from copy import deepcopy
+from types import ModuleType
+from typing import Callable, Collection, Dict, Optional, TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from .lark import PostLex
 
 from .utils import Serialize
 from .lexer import TerminalDef, Token
 
 ###{standalone
-from typing import Any, Callable, Collection, Dict, Optional, TYPE_CHECKING
-
-if TYPE_CHECKING:
- from .lark import PostLex
 
 _Callback = Callable[[Token], Token]
 

lark/exceptions.py

@@ -1,15 +1,14 @@
 from .utils import logger, NO_VALUE
-
-
-###{standalone
-
 from typing import Dict, Iterable, Callable, Union, TypeVar, Tuple, Any, List, Set, Optional, TYPE_CHECKING
+from collections.abc import Sequence
 
 if TYPE_CHECKING:
  from .lexer import Token
  from .parsers.lalr_interactive_parser import InteractiveParser
  from .tree import Tree
 
+###{standalone
+
 class LarkError(Exception):
  pass
 
@@ -18,7 +17,7 @@ class ConfigurationError(LarkError, ValueError):
  pass
 
 
-def assert_config(value, options, msg='Got %r, expected one of %s'):
+def assert_config(value, options: Sequence, msg='Got %r, expected one of %s'):
  if value not in options:
  raise ConfigurationError(msg % (value, options))
 

lark/grammar.py

@@ -1,10 +1,9 @@
+from typing import Optional, Tuple, ClassVar
 
 from .utils import Serialize
 
 ###{standalone
 
-from typing import Optional, Tuple, ClassVar
-
 class Symbol(Serialize):
  __slots__ = ('name',)
 

lark/grammars/python.lark

@@ -10,8 +10,8 @@ DEC_NUMBER: /0|[1-9][\d_]*/i
 HEX_NUMBER.2: /0x[\da-f]*/i
 OCT_NUMBER.2: /0o[0-7]*/i
 BIN_NUMBER.2 : /0b[0-1]*/i
-FLOAT_NUMBER.2: /((\d+\.[\d_]*|\.[\d_]+)(e[-+]?\d+)?|\d+(e[-+]?\d+))/i
-IMAG_NUMBER.2: /\d+j/i | FLOAT_NUMBER "j"i
+FLOAT_NUMBER.2: /((\d+\.[\d_]*|\.[\d_]+)([Ee][-+]?\d+)?|\d+([Ee][-+]?\d+))/
+IMAG_NUMBER.2: /\d+[Jj]/ | FLOAT_NUMBER /[Jj]/
 
 
 // Comma-separated list (with an optional trailing comma)

lark/indenter.py

@@ -1,19 +1,18 @@
 "Provides Indentation services for languages with indentation similar to Python"
 
 from abc import ABC, abstractmethod
+from typing import List, Iterator
 
 from .exceptions import LarkError
 from .lark import PostLex
 from .lexer import Token
 
 ###{standalone
-from typing import Tuple, List, Iterator, Optional
 
 class DedentError(LarkError):
  pass
 
 class Indenter(PostLex, ABC):
-
  paren_level: int
  indent_level: List[int]
 
@@ -75,31 +74,31 @@ def always_accept(self):
  @property
  @abstractmethod
  def NL_type(self) -> str:
- ...
+ return NotImplemented
 
  @property
  @abstractmethod
  def OPEN_PAREN_types(self) -> List[str]:
- ...
+ return NotImplemented
 
  @property
  @abstractmethod
  def CLOSE_PAREN_types(self) -> List[str]:
- ...
+ return NotImplemented
 
  @property
  @abstractmethod
  def INDENT_type(self) -> str:
- ...
+ return NotImplemented
 
  @property
  @abstractmethod
  def DEDENT_type(self) -> str:
- ...
+ return NotImplemented
 
  @property
  @abstractmethod
  def tab_len(self) -> int:
- ...
+ return NotImplemented
 
 ###}

lark/lark.py

@@ -1,7 +1,18 @@
 from abc import ABC, abstractmethod
 import sys, os, pickle, hashlib
 import tempfile
-
+from typing import (
+ TypeVar, Type, List, Dict, Iterator, Callable, Union, Optional,
+ Tuple, Iterable, IO, Any, TYPE_CHECKING
+)
+if TYPE_CHECKING:
+ from .parsers.lalr_interactive_parser import InteractiveParser
+ from .visitors import Transformer
+ if sys.version_info >= (3, 8):
+ from typing import Literal
+ else:
+ from typing_extensions import Literal
+
 from .exceptions import ConfigurationError, assert_config, UnexpectedInput
 from .utils import Serialize, SerializeMemoizer, FS, isascii, logger
 from .load_grammar import load_grammar, FromPackageLoader, Grammar, verify_used_files, PackageResource
@@ -21,18 +32,7 @@
 
 
 ###{standalone
-from typing import (
- TypeVar, Type, List, Dict, Iterator, Callable, Union, Optional,
- Tuple, Iterable, IO, Any, TYPE_CHECKING
-)
 
-if TYPE_CHECKING:
- from .parsers.lalr_interactive_parser import InteractiveParser
- from .visitors import Transformer
- if sys.version_info >= (3, 8):
- from typing import Literal
- else:
- from typing_extensions import Literal
 
 class PostLex(ABC):
  @abstractmethod

lark/lexer.py

@@ -3,21 +3,20 @@
 from abc import abstractmethod, ABC
 import re
 from contextlib import suppress
+from typing import (
+ TypeVar, Type, List, Dict, Iterator, Collection, Callable, Optional, FrozenSet, Any,
+ Pattern as REPattern, ClassVar, TYPE_CHECKING
+)
+from types import ModuleType
+if TYPE_CHECKING:
+ from .common import LexerConf
 
 from .utils import classify, get_regexp_width, Serialize
 from .exceptions import UnexpectedCharacters, LexError, UnexpectedToken
 
 ###{standalone
 from copy import copy
 
-from types import ModuleType
-from typing import (
- TypeVar, Type, Tuple, List, Dict, Iterator, Collection, Callable, Optional, FrozenSet, Any,
- Pattern as REPattern, ClassVar, TYPE_CHECKING
-)
-
-if TYPE_CHECKING:
- from .common import LexerConf
 
 class Pattern(Serialize, ABC):
 
@@ -218,7 +217,7 @@ def __eq__(self, other):
 
  return self.char_pos == other.char_pos and self.newline_char == other.newline_char
 
- def feed(self, token, test_newline=True):
+ def feed(self, token: Token, test_newline=True):
  """Consume a token and calculate the new line & column.
 
  As an optional optimization, set test_newline=False if token doesn't contain a newline.
@@ -320,7 +319,7 @@ def match(self, text, pos):
  return m.group(0), type_from_index[m.lastindex]
 
 
-def _regexp_has_newline(r):
+def _regexp_has_newline(r: str):
  r"""Expressions that may indicate newlines in a regexp:
  - newlines (\n)
  - escaped newline (\\n)
@@ -359,7 +358,7 @@ class Lexer(ABC):
  """
  @abstractmethod
  def lex(self, lexer_state: LexerState, parser_state: Any) -> Iterator[Token]:
- ...
+ return NotImplemented
 
  def make_lexer_state(self, text):
  line_ctr = LineCounter(b'\n' if isinstance(text, bytes) else '\n')

lark/parse_tree_builder.py

@@ -1,3 +1,5 @@
+from typing import List
+
 from .exceptions import GrammarError, ConfigurationError
 from .lexer import Token
 from .tree import Tree
@@ -151,7 +153,7 @@ def _should_expand(sym):
  return not sym.is_term and sym.name.startswith('_')
 
 
-def maybe_create_child_filter(expansion, keep_all_tokens, ambiguous, _empty_indices):
+def maybe_create_child_filter(expansion, keep_all_tokens, ambiguous, _empty_indices: List[bool]):
  # Prepare empty_indices as: How many Nones to insert at each index?
  if _empty_indices:
  assert _empty_indices.count(False) == len(expansion)

lark/parser_frontends.py

@@ -32,20 +32,13 @@ def __init__(self, parser_type, lexer_type):
  self.parser_type = parser_type
  self.lexer_type = lexer_type
 
- def __call__(self, lexer_conf, parser_conf, options):
- assert isinstance(lexer_conf, LexerConf)
- assert isinstance(parser_conf, ParserConf)
- parser_conf.parser_type = self.parser_type
- lexer_conf.lexer_type = self.lexer_type
- return ParsingFrontend(lexer_conf, parser_conf, options)
-
  def deserialize(self, data, memo, lexer_conf, callbacks, options):
  parser_conf = ParserConf.deserialize(data['parser_conf'], memo)
  parser = LALR_Parser.deserialize(data['parser'], memo, callbacks, options.debug)
  parser_conf.callbacks = callbacks
  return ParsingFrontend(lexer_conf, parser_conf, options, parser=parser)
 
-
+ # ... Continued later in the module
 
 
 class ParsingFrontend(Serialize):
@@ -237,3 +230,12 @@ def _transform(self, tree):
 
  def _apply_callback(self, tree):
  return self.callbacks[tree.rule](tree.children)
+
+
+class MakeParsingFrontend(MakeParsingFrontend):
+ def __call__(self, lexer_conf, parser_conf, options):
+ assert isinstance(lexer_conf, LexerConf)
+ assert isinstance(parser_conf, ParserConf)
+ parser_conf.parser_type = self.parser_type
+ lexer_conf.lexer_type = self.lexer_type
+ return ParsingFrontend(lexer_conf, parser_conf, options)

lark/tools/standalone.py

@@ -25,6 +25,13 @@
 #
 
 from abc import ABC, abstractmethod
+from collections.abc import Sequence
+from types import ModuleType
+from typing import (
+ TypeVar, Generic, Type, Tuple, List, Dict, Iterator, Collection, Callable, Optional, FrozenSet, Any,
+ Union, Iterable, IO, TYPE_CHECKING,
+ Pattern as REPattern, ClassVar, Set,
+)
 ###}
 
 import sys

lark/tree.py

@@ -1,3 +1,4 @@
+
 try:
  from future_builtins import filter # type: ignore
 except ImportError:
@@ -6,10 +7,7 @@
 import sys
 from copy import deepcopy
 
-
-###{standalone
-from collections import OrderedDict
-from typing import List, Callable, Iterator, Union, Optional, Any, TYPE_CHECKING
+from typing import List, Callable, Iterator, Union, Optional, TYPE_CHECKING
 
 if TYPE_CHECKING:
  from .lexer import TerminalDef
@@ -18,6 +16,9 @@
  else:
  from typing_extensions import Literal
 
+###{standalone
+from collections import OrderedDict
+
 class Meta:
 
  empty: bool

lark/visitors.py

@@ -1,3 +1,4 @@
+from typing import TypeVar, Tuple, List, Callable, Generic, Type, Union, Optional
 from abc import ABC
 from functools import wraps
 
@@ -8,7 +9,6 @@
 
 ###{standalone
 from inspect import getmembers, getmro
-from typing import TypeVar, Tuple, List, Callable, Generic, Type, Union, Optional
 
 _T = TypeVar('_T')
 _R = TypeVar('_R')

tests/test_tools.py

@@ -24,7 +24,7 @@ def _create_standalone(self, grammar, compress=False):
  standalone.gen_standalone(Lark(grammar, parser='lalr'), out=code_buf, compress=compress)
  code = code_buf.getvalue()
 
- context = {'__doc__': None}
+ context = {'__doc__': None, '__name__': 'test_standalone'}
  exec(code, context)
  return context