Merge pull request #117 from grumpyhome/unvendor-pythonparser

Unvendorize pythonparser (Fixes #108)

grumpy-tools-src/grumpy_tools/compiler/block.py

@@ -24,9 +24,9 @@
 
 from grumpy_tools.compiler import expr
 from grumpy_tools.compiler import util
-from grumpy_tools.vendor.pythonparser import algorithm
-from grumpy_tools.vendor.pythonparser import ast
-from grumpy_tools.vendor.pythonparser import source
+from pythonparser import algorithm
+from pythonparser import ast
+from pythonparser import source
 
 
 _non_word_re = re.compile('[^A-Za-z0-9_]')

grumpy-tools-src/grumpy_tools/compiler/block_test.py

@@ -24,7 +24,7 @@
 from grumpy_tools.compiler import block
 from grumpy_tools.compiler import imputil
 from grumpy_tools.compiler import util
-from grumpy_tools.vendor import pythonparser
+import pythonparser
 
 
 class PackageTest(unittest.TestCase):

grumpy-tools-src/grumpy_tools/compiler/expr_visitor.py

@@ -23,8 +23,8 @@
 
 from grumpy_tools.compiler import expr
 from grumpy_tools.compiler import util
-from grumpy_tools.vendor.pythonparser import algorithm
-from grumpy_tools.vendor.pythonparser import ast
+from pythonparser import algorithm
+from pythonparser import ast
 
 try:
  long # Python 2

grumpy-tools-src/grumpy_tools/compiler/expr_visitor_test.py

@@ -26,7 +26,7 @@
 from grumpy_tools.compiler import imputil
 from grumpy_tools.compiler import shard_test
 from grumpy_tools.compiler import stmt
-from grumpy_tools.vendor import pythonparser
+import pythonparser
 
 # Handles "Set self.maxDiff to None to see it" annoyance
 unittest.TestCase.maxDiff = None

grumpy-tools-src/grumpy_tools/compiler/imputil.py

@@ -33,10 +33,10 @@
 except ImportError:
  from backports.functools_lru_cache import lru_cache
 
-from grumpy_tools.compiler import util
-from grumpy_tools.vendor import pythonparser
-from grumpy_tools.vendor.pythonparser import algorithm
-from grumpy_tools.vendor.pythonparser import ast
+from grumpy_tools.compiler import util, parser
+import pythonparser
+from pythonparser import algorithm
+from pythonparser import ast
 
 try:
  xrange # Python 2
@@ -247,6 +247,7 @@ def visit_ImportFrom(self, node):
 
 
 def collect_imports(modname, script, gopath, package_dir=''):
+ parser.patch_pythonparser()
  with open(script) as py_file:
  py_contents = py_file.read()
  mod = pythonparser.parse(py_contents)

grumpy-tools-src/grumpy_tools/compiler/imputil_test.py

@@ -27,7 +27,10 @@
 
 from grumpy_tools.compiler import imputil
 from grumpy_tools.compiler import util
-from grumpy_tools.vendor import pythonparser
+from grumpy_tools.compiler.parser import patch_pythonparser
+import pythonparser
+
+patch_pythonparser()
 
 
 class ImportVisitorTest(unittest.TestCase):

grumpy-tools-src/grumpy_tools/compiler/parser.py

@@ -0,0 +1,79 @@
+# coding: utf-8
+"""
+Grumpy uses `pythonparser` as its AST parser. This module contains an augmented
+(extended) parser from it, letting us to accept special Grumpy-only syntax, like
+the `import '__go__/...'` syntax for importing Go code.
+"""
+import logging
+
+import pythonparser.parser
+from pythonparser.parser import Parser, Seq, Loc, Opt, Tok, List, Alt, Rule, action
+from pythonparser import ast
+
+logger = logging.getLogger(__name__)
+
+PYTHNOPARSER_PATCHED = False
+
+
+def patch_pythonparser():
+ global PYTHNOPARSER_PATCHED
+ if PYTHNOPARSER_PATCHED:
+ return False
+
+ logger.info('Monkeypatching pythonparser.parser.Parser with Grumpy extensions')
+ pythonparser.parser.Parser = GrumpyParser
+ PYTHNOPARSER_PATCHED = True
+ return True
+
+
+class GrumpyParser(Parser):
+ # From: https://github.com/google/grumpy/commit/9d80504e8d42c4a03ece9ed983b0ca160d170969#diff-c46e216e8423951b5f41dde139575b68R1038
+ @action(Rule("atom_5"))
+ def import_from_7(self, string):
+ return (None, 0), (string.loc, string.s)
+
+ # From: https://github.com/google/grumpy/commit/9d80504e8d42c4a03ece9ed983b0ca160d170969#diff-c46e216e8423951b5f41dde139575b68R1046
+ @action(Seq(Loc("from"), Alt(Parser.import_from_3, Parser.import_from_4, import_from_7),
+ Loc("import"), Alt(Parser.import_from_5,
+ Seq(Loc("("), Rule("import_as_names"), Loc(")")),
+ Parser.import_from_6)))
+ def import_from(self, from_loc, module_name, import_loc, names):
+ """
+ (2.6, 2.7)
+ import_from: ('from' ('.'* dotted_name | '.'+)
+ 'import' ('*' | '(' import_as_names ')' | import_as_names))
+ (3.0-)
+ # note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS
+ import_from: ('from' (('.' | '...')* dotted_name | ('.' | '...')+)
+ 'import' ('*' | '(' import_as_names ')' | import_as_names))
+ """
+ (dots_loc, dots_count), dotted_name_opt = module_name
+ module_loc = module = None
+ if dotted_name_opt:
+ module_loc, module = dotted_name_opt
+ lparen_loc, names, rparen_loc = names
+ loc = from_loc.join(names[-1].loc)
+ if rparen_loc:
+ loc = loc.join(rparen_loc)
+
+ if module == "__future__":
+ self.add_flags([x.name for x in names])
+
+ return ast.ImportFrom(names=names, module=module, level=dots_count,
+ keyword_loc=from_loc, dots_loc=dots_loc, module_loc=module_loc,
+ import_loc=import_loc, lparen_loc=lparen_loc, rparen_loc=rparen_loc,
+ loc=loc)
+
+ @action(Seq(Rule("atom_5"), Opt(Seq(Loc("as"), Tok("ident")))))
+ def str_as_name(self, string, as_name_opt):
+ asname_name = asname_loc = as_loc = None
+ loc = string.loc
+ if as_name_opt:
+ as_loc, asname = as_name_opt
+ asname_name = asname.value
+ asname_loc = asname.loc
+ loc = loc.join(asname.loc)
+ return ast.alias(name=string.s, asname=asname_name,
+ loc=loc, name_loc=string.loc, as_loc=as_loc, asname_loc=asname_loc)
+
+ dotted_as_names = List(Alt(Rule("dotted_as_name"), Rule("str_as_name")), ",", trailing=False)

grumpy-tools-src/grumpy_tools/compiler/stmt.py

@@ -27,8 +27,8 @@
 from grumpy_tools.compiler import expr_visitor
 from grumpy_tools.compiler import imputil
 from grumpy_tools.compiler import util
-from grumpy_tools.vendor.pythonparser import algorithm
-from grumpy_tools.vendor.pythonparser import ast
+from pythonparser import algorithm
+from pythonparser import ast
 
 
 _NATIVE_TYPE_PREFIX = 'type_'

grumpy-tools-src/grumpy_tools/compiler/stmt_test.py

@@ -28,8 +28,8 @@
 from grumpy_tools.compiler import shard_test
 from grumpy_tools.compiler import stmt
 from grumpy_tools.compiler import util
-from grumpy_tools.vendor import pythonparser
-from grumpy_tools.vendor.pythonparser import ast
+import pythonparser
+from pythonparser import ast
 
 import pytest
 

grumpy-tools-src/grumpy_tools/grumpc.py

@@ -33,14 +33,16 @@
 from .compiler import imputil
 from .compiler import stmt
 from .compiler import util
-from .vendor import pythonparser
+from .compiler.parser import patch_pythonparser
+import pythonparser
 from .pep_support.pep3147pycache import make_transpiled_module_folders, should_refresh, set_checksum, fixed_keyword
 from . import pydeps
 
 logger = logging.getLogger(__name__)
 
 
 def _parse_and_visit(stream, script, modname):
+ patch_pythonparser()
  gopath = os.environ['GOPATH']
 
  stream.seek(0)