parsetree.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616
  1. # mako/parsetree.py
  2. # Copyright (C) 2006-2016 the Mako authors and contributors <see AUTHORS file>
  3. #
  4. # This module is part of Mako and is released under
  5. # the MIT License: http://www.opensource.org/licenses/mit-license.php
  6. """defines the parse tree components for Mako templates."""
  7. from mako import exceptions, ast, util, filters, compat
  8. import re
  9. class Node(object):
  10. """base class for a Node in the parse tree."""
  11. def __init__(self, source, lineno, pos, filename):
  12. self.source = source
  13. self.lineno = lineno
  14. self.pos = pos
  15. self.filename = filename
  16. @property
  17. def exception_kwargs(self):
  18. return {'source': self.source, 'lineno': self.lineno,
  19. 'pos': self.pos, 'filename': self.filename}
  20. def get_children(self):
  21. return []
  22. def accept_visitor(self, visitor):
  23. def traverse(node):
  24. for n in node.get_children():
  25. n.accept_visitor(visitor)
  26. method = getattr(visitor, "visit" + self.__class__.__name__, traverse)
  27. method(self)
  28. class TemplateNode(Node):
  29. """a 'container' node that stores the overall collection of nodes."""
  30. def __init__(self, filename):
  31. super(TemplateNode, self).__init__('', 0, 0, filename)
  32. self.nodes = []
  33. self.page_attributes = {}
  34. def get_children(self):
  35. return self.nodes
  36. def __repr__(self):
  37. return "TemplateNode(%s, %r)" % (
  38. util.sorted_dict_repr(self.page_attributes),
  39. self.nodes)
  40. class ControlLine(Node):
  41. """defines a control line, a line-oriented python line or end tag.
  42. e.g.::
  43. % if foo:
  44. (markup)
  45. % endif
  46. """
  47. has_loop_context = False
  48. def __init__(self, keyword, isend, text, **kwargs):
  49. super(ControlLine, self).__init__(**kwargs)
  50. self.text = text
  51. self.keyword = keyword
  52. self.isend = isend
  53. self.is_primary = keyword in ['for', 'if', 'while', 'try', 'with']
  54. self.nodes = []
  55. if self.isend:
  56. self._declared_identifiers = []
  57. self._undeclared_identifiers = []
  58. else:
  59. code = ast.PythonFragment(text, **self.exception_kwargs)
  60. self._declared_identifiers = code.declared_identifiers
  61. self._undeclared_identifiers = code.undeclared_identifiers
  62. def get_children(self):
  63. return self.nodes
  64. def declared_identifiers(self):
  65. return self._declared_identifiers
  66. def undeclared_identifiers(self):
  67. return self._undeclared_identifiers
  68. def is_ternary(self, keyword):
  69. """return true if the given keyword is a ternary keyword
  70. for this ControlLine"""
  71. return keyword in {
  72. 'if': set(['else', 'elif']),
  73. 'try': set(['except', 'finally']),
  74. 'for': set(['else'])
  75. }.get(self.keyword, [])
  76. def __repr__(self):
  77. return "ControlLine(%r, %r, %r, %r)" % (
  78. self.keyword,
  79. self.text,
  80. self.isend,
  81. (self.lineno, self.pos)
  82. )
  83. class Text(Node):
  84. """defines plain text in the template."""
  85. def __init__(self, content, **kwargs):
  86. super(Text, self).__init__(**kwargs)
  87. self.content = content
  88. def __repr__(self):
  89. return "Text(%r, %r)" % (self.content, (self.lineno, self.pos))
  90. class Code(Node):
  91. """defines a Python code block, either inline or module level.
  92. e.g.::
  93. inline:
  94. <%
  95. x = 12
  96. %>
  97. module level:
  98. <%!
  99. import logger
  100. %>
  101. """
  102. def __init__(self, text, ismodule, **kwargs):
  103. super(Code, self).__init__(**kwargs)
  104. self.text = text
  105. self.ismodule = ismodule
  106. self.code = ast.PythonCode(text, **self.exception_kwargs)
  107. def declared_identifiers(self):
  108. return self.code.declared_identifiers
  109. def undeclared_identifiers(self):
  110. return self.code.undeclared_identifiers
  111. def __repr__(self):
  112. return "Code(%r, %r, %r)" % (
  113. self.text,
  114. self.ismodule,
  115. (self.lineno, self.pos)
  116. )
  117. class Comment(Node):
  118. """defines a comment line.
  119. # this is a comment
  120. """
  121. def __init__(self, text, **kwargs):
  122. super(Comment, self).__init__(**kwargs)
  123. self.text = text
  124. def __repr__(self):
  125. return "Comment(%r, %r)" % (self.text, (self.lineno, self.pos))
  126. class Expression(Node):
  127. """defines an inline expression.
  128. ${x+y}
  129. """
  130. def __init__(self, text, escapes, **kwargs):
  131. super(Expression, self).__init__(**kwargs)
  132. self.text = text
  133. self.escapes = escapes
  134. self.escapes_code = ast.ArgumentList(escapes, **self.exception_kwargs)
  135. self.code = ast.PythonCode(text, **self.exception_kwargs)
  136. def declared_identifiers(self):
  137. return []
  138. def undeclared_identifiers(self):
  139. # TODO: make the "filter" shortcut list configurable at parse/gen time
  140. return self.code.undeclared_identifiers.union(
  141. self.escapes_code.undeclared_identifiers.difference(
  142. set(filters.DEFAULT_ESCAPES.keys())
  143. )
  144. ).difference(self.code.declared_identifiers)
  145. def __repr__(self):
  146. return "Expression(%r, %r, %r)" % (
  147. self.text,
  148. self.escapes_code.args,
  149. (self.lineno, self.pos)
  150. )
  151. class _TagMeta(type):
  152. """metaclass to allow Tag to produce a subclass according to
  153. its keyword"""
  154. _classmap = {}
  155. def __init__(cls, clsname, bases, dict):
  156. if getattr(cls, '__keyword__', None) is not None:
  157. cls._classmap[cls.__keyword__] = cls
  158. super(_TagMeta, cls).__init__(clsname, bases, dict)
  159. def __call__(cls, keyword, attributes, **kwargs):
  160. if ":" in keyword:
  161. ns, defname = keyword.split(':')
  162. return type.__call__(CallNamespaceTag, ns, defname,
  163. attributes, **kwargs)
  164. try:
  165. cls = _TagMeta._classmap[keyword]
  166. except KeyError:
  167. raise exceptions.CompileException(
  168. "No such tag: '%s'" % keyword,
  169. source=kwargs['source'],
  170. lineno=kwargs['lineno'],
  171. pos=kwargs['pos'],
  172. filename=kwargs['filename']
  173. )
  174. return type.__call__(cls, keyword, attributes, **kwargs)
  175. class Tag(compat.with_metaclass(_TagMeta, Node)):
  176. """abstract base class for tags.
  177. <%sometag/>
  178. <%someothertag>
  179. stuff
  180. </%someothertag>
  181. """
  182. __keyword__ = None
  183. def __init__(self, keyword, attributes, expressions,
  184. nonexpressions, required, **kwargs):
  185. """construct a new Tag instance.
  186. this constructor not called directly, and is only called
  187. by subclasses.
  188. :param keyword: the tag keyword
  189. :param attributes: raw dictionary of attribute key/value pairs
  190. :param expressions: a set of identifiers that are legal attributes,
  191. which can also contain embedded expressions
  192. :param nonexpressions: a set of identifiers that are legal
  193. attributes, which cannot contain embedded expressions
  194. :param \**kwargs:
  195. other arguments passed to the Node superclass (lineno, pos)
  196. """
  197. super(Tag, self).__init__(**kwargs)
  198. self.keyword = keyword
  199. self.attributes = attributes
  200. self._parse_attributes(expressions, nonexpressions)
  201. missing = [r for r in required if r not in self.parsed_attributes]
  202. if len(missing):
  203. raise exceptions.CompileException(
  204. "Missing attribute(s): %s" %
  205. ",".join([repr(m) for m in missing]),
  206. **self.exception_kwargs)
  207. self.parent = None
  208. self.nodes = []
  209. def is_root(self):
  210. return self.parent is None
  211. def get_children(self):
  212. return self.nodes
  213. def _parse_attributes(self, expressions, nonexpressions):
  214. undeclared_identifiers = set()
  215. self.parsed_attributes = {}
  216. for key in self.attributes:
  217. if key in expressions:
  218. expr = []
  219. for x in re.compile(r'(\${.+?})',
  220. re.S).split(self.attributes[key]):
  221. m = re.compile(r'^\${(.+?)}$', re.S).match(x)
  222. if m:
  223. code = ast.PythonCode(m.group(1).rstrip(),
  224. **self.exception_kwargs)
  225. # we aren't discarding "declared_identifiers" here,
  226. # which we do so that list comprehension-declared
  227. # variables aren't counted. As yet can't find a
  228. # condition that requires it here.
  229. undeclared_identifiers = \
  230. undeclared_identifiers.union(
  231. code.undeclared_identifiers)
  232. expr.append('(%s)' % m.group(1))
  233. else:
  234. if x:
  235. expr.append(repr(x))
  236. self.parsed_attributes[key] = " + ".join(expr) or repr('')
  237. elif key in nonexpressions:
  238. if re.search(r'\${.+?}', self.attributes[key]):
  239. raise exceptions.CompileException(
  240. "Attibute '%s' in tag '%s' does not allow embedded "
  241. "expressions" % (key, self.keyword),
  242. **self.exception_kwargs)
  243. self.parsed_attributes[key] = repr(self.attributes[key])
  244. else:
  245. raise exceptions.CompileException(
  246. "Invalid attribute for tag '%s': '%s'" %
  247. (self.keyword, key),
  248. **self.exception_kwargs)
  249. self.expression_undeclared_identifiers = undeclared_identifiers
  250. def declared_identifiers(self):
  251. return []
  252. def undeclared_identifiers(self):
  253. return self.expression_undeclared_identifiers
  254. def __repr__(self):
  255. return "%s(%r, %s, %r, %r)" % (self.__class__.__name__,
  256. self.keyword,
  257. util.sorted_dict_repr(self.attributes),
  258. (self.lineno, self.pos),
  259. self.nodes
  260. )
  261. class IncludeTag(Tag):
  262. __keyword__ = 'include'
  263. def __init__(self, keyword, attributes, **kwargs):
  264. super(IncludeTag, self).__init__(
  265. keyword,
  266. attributes,
  267. ('file', 'import', 'args'),
  268. (), ('file',), **kwargs)
  269. self.page_args = ast.PythonCode(
  270. "__DUMMY(%s)" % attributes.get('args', ''),
  271. **self.exception_kwargs)
  272. def declared_identifiers(self):
  273. return []
  274. def undeclared_identifiers(self):
  275. identifiers = self.page_args.undeclared_identifiers.\
  276. difference(set(["__DUMMY"])).\
  277. difference(self.page_args.declared_identifiers)
  278. return identifiers.union(super(IncludeTag, self).
  279. undeclared_identifiers())
  280. class NamespaceTag(Tag):
  281. __keyword__ = 'namespace'
  282. def __init__(self, keyword, attributes, **kwargs):
  283. super(NamespaceTag, self).__init__(
  284. keyword, attributes,
  285. ('file',),
  286. ('name', 'inheritable',
  287. 'import', 'module'),
  288. (), **kwargs)
  289. self.name = attributes.get('name', '__anon_%s' % hex(abs(id(self))))
  290. if 'name' not in attributes and 'import' not in attributes:
  291. raise exceptions.CompileException(
  292. "'name' and/or 'import' attributes are required "
  293. "for <%namespace>",
  294. **self.exception_kwargs)
  295. if 'file' in attributes and 'module' in attributes:
  296. raise exceptions.CompileException(
  297. "<%namespace> may only have one of 'file' or 'module'",
  298. **self.exception_kwargs
  299. )
  300. def declared_identifiers(self):
  301. return []
  302. class TextTag(Tag):
  303. __keyword__ = 'text'
  304. def __init__(self, keyword, attributes, **kwargs):
  305. super(TextTag, self).__init__(
  306. keyword,
  307. attributes, (),
  308. ('filter'), (), **kwargs)
  309. self.filter_args = ast.ArgumentList(
  310. attributes.get('filter', ''),
  311. **self.exception_kwargs)
  312. def undeclared_identifiers(self):
  313. return self.filter_args.\
  314. undeclared_identifiers.\
  315. difference(filters.DEFAULT_ESCAPES.keys()).union(
  316. self.expression_undeclared_identifiers
  317. )
  318. class DefTag(Tag):
  319. __keyword__ = 'def'
  320. def __init__(self, keyword, attributes, **kwargs):
  321. expressions = ['buffered', 'cached'] + [
  322. c for c in attributes if c.startswith('cache_')]
  323. super(DefTag, self).__init__(
  324. keyword,
  325. attributes,
  326. expressions,
  327. ('name', 'filter', 'decorator'),
  328. ('name',),
  329. **kwargs)
  330. name = attributes['name']
  331. if re.match(r'^[\w_]+$', name):
  332. raise exceptions.CompileException(
  333. "Missing parenthesis in %def",
  334. **self.exception_kwargs)
  335. self.function_decl = ast.FunctionDecl("def " + name + ":pass",
  336. **self.exception_kwargs)
  337. self.name = self.function_decl.funcname
  338. self.decorator = attributes.get('decorator', '')
  339. self.filter_args = ast.ArgumentList(
  340. attributes.get('filter', ''),
  341. **self.exception_kwargs)
  342. is_anonymous = False
  343. is_block = False
  344. @property
  345. def funcname(self):
  346. return self.function_decl.funcname
  347. def get_argument_expressions(self, **kw):
  348. return self.function_decl.get_argument_expressions(**kw)
  349. def declared_identifiers(self):
  350. return self.function_decl.allargnames
  351. def undeclared_identifiers(self):
  352. res = []
  353. for c in self.function_decl.defaults:
  354. res += list(ast.PythonCode(c, **self.exception_kwargs).
  355. undeclared_identifiers)
  356. return set(res).union(
  357. self.filter_args.
  358. undeclared_identifiers.
  359. difference(filters.DEFAULT_ESCAPES.keys())
  360. ).union(
  361. self.expression_undeclared_identifiers
  362. ).difference(
  363. self.function_decl.allargnames
  364. )
  365. class BlockTag(Tag):
  366. __keyword__ = 'block'
  367. def __init__(self, keyword, attributes, **kwargs):
  368. expressions = ['buffered', 'cached', 'args'] + [
  369. c for c in attributes if c.startswith('cache_')]
  370. super(BlockTag, self).__init__(
  371. keyword,
  372. attributes,
  373. expressions,
  374. ('name', 'filter', 'decorator'),
  375. (),
  376. **kwargs)
  377. name = attributes.get('name')
  378. if name and not re.match(r'^[\w_]+$', name):
  379. raise exceptions.CompileException(
  380. "%block may not specify an argument signature",
  381. **self.exception_kwargs)
  382. if not name and attributes.get('args', None):
  383. raise exceptions.CompileException(
  384. "Only named %blocks may specify args",
  385. **self.exception_kwargs
  386. )
  387. self.body_decl = ast.FunctionArgs(attributes.get('args', ''),
  388. **self.exception_kwargs)
  389. self.name = name
  390. self.decorator = attributes.get('decorator', '')
  391. self.filter_args = ast.ArgumentList(
  392. attributes.get('filter', ''),
  393. **self.exception_kwargs)
  394. is_block = True
  395. @property
  396. def is_anonymous(self):
  397. return self.name is None
  398. @property
  399. def funcname(self):
  400. return self.name or "__M_anon_%d" % (self.lineno, )
  401. def get_argument_expressions(self, **kw):
  402. return self.body_decl.get_argument_expressions(**kw)
  403. def declared_identifiers(self):
  404. return self.body_decl.allargnames
  405. def undeclared_identifiers(self):
  406. return (self.filter_args.
  407. undeclared_identifiers.
  408. difference(filters.DEFAULT_ESCAPES.keys())
  409. ).union(self.expression_undeclared_identifiers)
  410. class CallTag(Tag):
  411. __keyword__ = 'call'
  412. def __init__(self, keyword, attributes, **kwargs):
  413. super(CallTag, self).__init__(keyword, attributes,
  414. ('args'), ('expr',), ('expr',), **kwargs)
  415. self.expression = attributes['expr']
  416. self.code = ast.PythonCode(self.expression, **self.exception_kwargs)
  417. self.body_decl = ast.FunctionArgs(attributes.get('args', ''),
  418. **self.exception_kwargs)
  419. def declared_identifiers(self):
  420. return self.code.declared_identifiers.union(self.body_decl.allargnames)
  421. def undeclared_identifiers(self):
  422. return self.code.undeclared_identifiers.\
  423. difference(self.code.declared_identifiers)
  424. class CallNamespaceTag(Tag):
  425. def __init__(self, namespace, defname, attributes, **kwargs):
  426. super(CallNamespaceTag, self).__init__(
  427. namespace + ":" + defname,
  428. attributes,
  429. tuple(attributes.keys()) + ('args', ),
  430. (),
  431. (),
  432. **kwargs)
  433. self.expression = "%s.%s(%s)" % (
  434. namespace,
  435. defname,
  436. ",".join(["%s=%s" % (k, v) for k, v in
  437. self.parsed_attributes.items()
  438. if k != 'args'])
  439. )
  440. self.code = ast.PythonCode(self.expression, **self.exception_kwargs)
  441. self.body_decl = ast.FunctionArgs(
  442. attributes.get('args', ''),
  443. **self.exception_kwargs)
  444. def declared_identifiers(self):
  445. return self.code.declared_identifiers.union(self.body_decl.allargnames)
  446. def undeclared_identifiers(self):
  447. return self.code.undeclared_identifiers.\
  448. difference(self.code.declared_identifiers)
  449. class InheritTag(Tag):
  450. __keyword__ = 'inherit'
  451. def __init__(self, keyword, attributes, **kwargs):
  452. super(InheritTag, self).__init__(
  453. keyword, attributes,
  454. ('file',), (), ('file',), **kwargs)
  455. class PageTag(Tag):
  456. __keyword__ = 'page'
  457. def __init__(self, keyword, attributes, **kwargs):
  458. expressions = \
  459. ['cached', 'args', 'expression_filter', 'enable_loop'] + \
  460. [c for c in attributes if c.startswith('cache_')]
  461. super(PageTag, self).__init__(
  462. keyword,
  463. attributes,
  464. expressions,
  465. (),
  466. (),
  467. **kwargs)
  468. self.body_decl = ast.FunctionArgs(attributes.get('args', ''),
  469. **self.exception_kwargs)
  470. self.filter_args = ast.ArgumentList(
  471. attributes.get('expression_filter', ''),
  472. **self.exception_kwargs)
  473. def declared_identifiers(self):
  474. return self.body_decl.allargnames