pyparser.py 7.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233
  1. # mako/pyparser.py
  2. # Copyright (C) 2006-2016 the Mako authors and contributors <see AUTHORS file>
  3. #
  4. # This module is part of Mako and is released under
  5. # the MIT License: http://www.opensource.org/licenses/mit-license.php
  6. """Handles parsing of Python code.
  7. Parsing to AST is done via _ast on Python > 2.5, otherwise the compiler
  8. module is used.
  9. """
  10. from mako import exceptions, util, compat
  11. from mako.compat import arg_stringname
  12. import operator
  13. if compat.py3k:
  14. # words that cannot be assigned to (notably
  15. # smaller than the total keys in __builtins__)
  16. reserved = set(['True', 'False', 'None', 'print'])
  17. # the "id" attribute on a function node
  18. arg_id = operator.attrgetter('arg')
  19. else:
  20. # words that cannot be assigned to (notably
  21. # smaller than the total keys in __builtins__)
  22. reserved = set(['True', 'False', 'None'])
  23. # the "id" attribute on a function node
  24. arg_id = operator.attrgetter('id')
  25. import _ast
  26. util.restore__ast(_ast)
  27. from mako import _ast_util
  28. def parse(code, mode='exec', **exception_kwargs):
  29. """Parse an expression into AST"""
  30. try:
  31. return _ast_util.parse(code, '<unknown>', mode)
  32. except Exception:
  33. raise exceptions.SyntaxException(
  34. "(%s) %s (%r)" % (
  35. compat.exception_as().__class__.__name__,
  36. compat.exception_as(),
  37. code[0:50]
  38. ), **exception_kwargs)
  39. class FindIdentifiers(_ast_util.NodeVisitor):
  40. def __init__(self, listener, **exception_kwargs):
  41. self.in_function = False
  42. self.in_assign_targets = False
  43. self.local_ident_stack = set()
  44. self.listener = listener
  45. self.exception_kwargs = exception_kwargs
  46. def _add_declared(self, name):
  47. if not self.in_function:
  48. self.listener.declared_identifiers.add(name)
  49. else:
  50. self.local_ident_stack.add(name)
  51. def visit_ClassDef(self, node):
  52. self._add_declared(node.name)
  53. def visit_Assign(self, node):
  54. # flip around the visiting of Assign so the expression gets
  55. # evaluated first, in the case of a clause like "x=x+5" (x
  56. # is undeclared)
  57. self.visit(node.value)
  58. in_a = self.in_assign_targets
  59. self.in_assign_targets = True
  60. for n in node.targets:
  61. self.visit(n)
  62. self.in_assign_targets = in_a
  63. if compat.py3k:
  64. # ExceptHandler is in Python 2, but this block only works in
  65. # Python 3 (and is required there)
  66. def visit_ExceptHandler(self, node):
  67. if node.name is not None:
  68. self._add_declared(node.name)
  69. if node.type is not None:
  70. self.visit(node.type)
  71. for statement in node.body:
  72. self.visit(statement)
  73. def visit_Lambda(self, node, *args):
  74. self._visit_function(node, True)
  75. def visit_FunctionDef(self, node):
  76. self._add_declared(node.name)
  77. self._visit_function(node, False)
  78. def _expand_tuples(self, args):
  79. for arg in args:
  80. if isinstance(arg, _ast.Tuple):
  81. for n in arg.elts:
  82. yield n
  83. else:
  84. yield arg
  85. def _visit_function(self, node, islambda):
  86. # push function state onto stack. dont log any more
  87. # identifiers as "declared" until outside of the function,
  88. # but keep logging identifiers as "undeclared". track
  89. # argument names in each function header so they arent
  90. # counted as "undeclared"
  91. inf = self.in_function
  92. self.in_function = True
  93. local_ident_stack = self.local_ident_stack
  94. self.local_ident_stack = local_ident_stack.union([
  95. arg_id(arg) for arg in self._expand_tuples(node.args.args)
  96. ])
  97. if islambda:
  98. self.visit(node.body)
  99. else:
  100. for n in node.body:
  101. self.visit(n)
  102. self.in_function = inf
  103. self.local_ident_stack = local_ident_stack
  104. def visit_For(self, node):
  105. # flip around visit
  106. self.visit(node.iter)
  107. self.visit(node.target)
  108. for statement in node.body:
  109. self.visit(statement)
  110. for statement in node.orelse:
  111. self.visit(statement)
  112. def visit_Name(self, node):
  113. if isinstance(node.ctx, _ast.Store):
  114. # this is eqiuvalent to visit_AssName in
  115. # compiler
  116. self._add_declared(node.id)
  117. elif node.id not in reserved and node.id \
  118. not in self.listener.declared_identifiers and node.id \
  119. not in self.local_ident_stack:
  120. self.listener.undeclared_identifiers.add(node.id)
  121. def visit_Import(self, node):
  122. for name in node.names:
  123. if name.asname is not None:
  124. self._add_declared(name.asname)
  125. else:
  126. self._add_declared(name.name.split('.')[0])
  127. def visit_ImportFrom(self, node):
  128. for name in node.names:
  129. if name.asname is not None:
  130. self._add_declared(name.asname)
  131. else:
  132. if name.name == '*':
  133. raise exceptions.CompileException(
  134. "'import *' is not supported, since all identifier "
  135. "names must be explicitly declared. Please use the "
  136. "form 'from <modulename> import <name1>, <name2>, "
  137. "...' instead.", **self.exception_kwargs)
  138. self._add_declared(name.name)
  139. class FindTuple(_ast_util.NodeVisitor):
  140. def __init__(self, listener, code_factory, **exception_kwargs):
  141. self.listener = listener
  142. self.exception_kwargs = exception_kwargs
  143. self.code_factory = code_factory
  144. def visit_Tuple(self, node):
  145. for n in node.elts:
  146. p = self.code_factory(n, **self.exception_kwargs)
  147. self.listener.codeargs.append(p)
  148. self.listener.args.append(ExpressionGenerator(n).value())
  149. self.listener.declared_identifiers = \
  150. self.listener.declared_identifiers.union(
  151. p.declared_identifiers)
  152. self.listener.undeclared_identifiers = \
  153. self.listener.undeclared_identifiers.union(
  154. p.undeclared_identifiers)
  155. class ParseFunc(_ast_util.NodeVisitor):
  156. def __init__(self, listener, **exception_kwargs):
  157. self.listener = listener
  158. self.exception_kwargs = exception_kwargs
  159. def visit_FunctionDef(self, node):
  160. self.listener.funcname = node.name
  161. argnames = [arg_id(arg) for arg in node.args.args]
  162. if node.args.vararg:
  163. argnames.append(arg_stringname(node.args.vararg))
  164. if compat.py2k:
  165. # kw-only args don't exist in Python 2
  166. kwargnames = []
  167. else:
  168. kwargnames = [arg_id(arg) for arg in node.args.kwonlyargs]
  169. if node.args.kwarg:
  170. kwargnames.append(arg_stringname(node.args.kwarg))
  171. self.listener.argnames = argnames
  172. self.listener.defaults = node.args.defaults # ast
  173. self.listener.kwargnames = kwargnames
  174. if compat.py2k:
  175. self.listener.kwdefaults = []
  176. else:
  177. self.listener.kwdefaults = node.args.kw_defaults
  178. self.listener.varargs = node.args.vararg
  179. self.listener.kwargs = node.args.kwarg
  180. class ExpressionGenerator(object):
  181. def __init__(self, astnode):
  182. self.generator = _ast_util.SourceGenerator(' ' * 4)
  183. self.generator.visit(astnode)
  184. def value(self):
  185. return ''.join(self.generator.result)