From c5eae67494631f904662235d9b6baf974482e466 Mon Sep 17 00:00:00 2001
From: Mike Bayer <mike_mp@zzzcomputing.com>
Date: Mon, 20 Nov 2006 08:44:22 +0000
Subject: [PATCH] various cleanup, more scoping tweaks, ast tweaks
---
lib/mako/ast.py | 25 +++++++++++++++++----
lib/mako/codegen.py | 51 ++++++++++++++++++++++++++-----------------
lib/mako/lexer.py | 3 +--
lib/mako/parsetree.py | 12 +++++-----
lib/mako/pygen.py | 1 +
lib/mako/runtime.py | 2 ++
test/ast.py | 15 ++++++++++++-
test/template.py | 24 ++++++++++++++++++++
8 files changed, 100 insertions(+), 33 deletions(-)
diff --git a/lib/mako/ast.py b/lib/mako/ast.py
index f8ffc28..8c53140 100644
--- a/lib/mako/ast.py
+++ b/lib/mako/ast.py
@@ -9,10 +9,16 @@ class PythonCode(object):
"""represents information about a string containing Python code"""
def __init__(self, code, lineno, pos):
self.code = code
+
+ # represents all identifiers which are assigned to at some point in the code
self.declared_identifiers = util.Set()
+
+ # represents all identifiers which are referenced before their assignment, if any
self.undeclared_identifiers = util.Set()
+
+ # note that an identifier can be in both the undeclared and declared lists.
- # note that using AST to parse instead of using code.co_varnames, code.co_names has several advantages:
+ # using AST to parse instead of using code.co_varnames, code.co_names has several advantages:
# - we can locate an identifier as "undeclared" even if its declared later in the same block of code
# - AST is less likely to break with version changes (for example, the behavior of co_names changed a little bit
# in python version 2.5)
@@ -23,8 +29,19 @@ class PythonCode(object):
class FindIdentifiers(object):
def visitAssName(s, node, *args):
- if node.name not in self.undeclared_identifiers:
- self.declared_identifiers.add(node.name)
+# if node.name not in self.undeclared_identifiers:
+ self.declared_identifiers.add(node.name)
+ def visitAssign(s, node, *args):
+ # flip around the visiting of Assign so the expression gets evaluated first,
+ # in the case of a clause like "x=x+5" (x is undeclared)
+ s.visit(node.expr, *args)
+ for n in node.nodes:
+ s.visit(n, *args)
+ def visitFor(s, node, *args):
+ # flip around visit
+ s.visit(node.list, *args)
+ s.visit(node.assign, *args)
+ s.visit(node.body, *args)
def visitTryExcept(s, node, *args):
for (decl, s2, s3) in node.handlers:
if decl is not None:
@@ -100,7 +117,7 @@ class FunctionDecl(object):
if not hasattr(self, 'funcname'):
raise exceptions.CompileException("Code '%s' is not a function declaration" % code, lineno, pos)
def get_argument_expressions(self, include_defaults=True):
- """return the argument declarations of this FunctionDecl as a printable list"""
+ """return the argument declarations of this FunctionDecl as a printable list."""
namedecls = []
defaults = [d for d in self.defaults]
kwargs = self.kwargs
diff --git a/lib/mako/codegen.py b/lib/mako/codegen.py
index d1fda48..7b97b96 100644
--- a/lib/mako/codegen.py
+++ b/lib/mako/codegen.py
@@ -35,7 +35,6 @@ class Compiler(object):
module_identifiers = module_identifiers.branch(n)
printer.writeline("# SOURCE LINE %d" % n.lineno, is_comment=True)
printer.write_indented_block(n.text)
-
main_identifiers = module_identifiers.branch(self.node)
module_identifiers.toplevelcomponents = module_identifiers.toplevelcomponents.union(main_identifiers.toplevelcomponents)
@@ -48,7 +47,6 @@ class Compiler(object):
_GenerateRenderMethod(printer, module_identifiers, node)
return buf.getvalue()
-
class _GenerateRenderMethod(object):
def __init__(self, printer, identifiers, node):
@@ -78,22 +76,32 @@ class _GenerateRenderMethod(object):
printer.write("\n\n")
def write_variable_declares(self, identifiers):
- """write variable declarations at the top of a function."""
+ """write variable declarations at the top of a function.
+
+ the variable declarations are generated based on the names that are referenced
+ in the function body before they are assigned. names that are re-assigned
+ from an enclosing scope are also declared as local variables so that the assignment
+ can proceed.
+
+ locally defined components (i.e. closures) are also generated, as well as 'stub' callables
+ referencing top-level components which are referenced in the function body."""
+
+ # collection of all components available to us in this scope
comp_idents = dict([(c.name, c) for c in identifiers.components])
# write explicit "context.get()" statements for variables that are already in the
# local namespace, that we are going to re-assign
- print "WVD", identifiers
to_write = identifiers.declared.intersection(identifiers.locally_declared)
# write "context.get()" for all variables we are going to need that arent in the namespace yet
to_write = to_write.union(identifiers.undeclared)
# write closure functions for closures that we define right here
- to_write = to_write.union(util.Set([c.name for c in identifiers.closurecomponents if c.parent is self.node]))
-
+ to_write = to_write.union(util.Set([c.name for c in identifiers.closurecomponents]))
+
+ # remove identifiers that are declared in the argument signature of the callable
to_write = to_write.difference(identifiers.argument_declared)
-
+
for ident in to_write:
if ident in comp_idents:
comp = comp_idents[ident]
@@ -110,6 +118,7 @@ class _GenerateRenderMethod(object):
self.last_source_line = node.lineno
def write_component_decl(self, node, identifiers):
+ """write a locally-available callable referencing a top-level component"""
funcname = node.function_decl.funcname
namedecls = node.function_decl.get_argument_expressions()
nameargs = node.function_decl.get_argument_expressions(include_defaults=False)
@@ -119,21 +128,20 @@ class _GenerateRenderMethod(object):
self.printer.writeline(None)
def write_inline_component(self, node, identifiers):
+ """write a locally-available component callable inside an enclosing component."""
namedecls = node.function_decl.get_argument_expressions()
self.printer.writeline("def %s(%s):" % (node.name, ",".join(namedecls)))
- print "INLINE NAME", node.name
+ #print "INLINE NAME", node.name
identifiers = identifiers.branch(node)
- print "IDENTIFIERES", identifiers
+
# if we assign to variables in this closure, then we have to nest inside
# of another callable so that the "context" variable is copied into the local scope
make_closure = len(identifiers.locally_declared) > 0
if make_closure:
self.printer.writeline("def %s(%s):" % (node.name, ",".join(['context'] + namedecls)))
- self.write_variable_declares(identifiers)
- else:
- self.write_variable_declares(identifiers)
+ self.write_variable_declares(identifiers)
for n in node.nodes:
n.accept_visitor(self)
@@ -172,7 +180,6 @@ class _GenerateRenderMethod(object):
self.printer.writeline("def make_namespace():")
export = []
identifiers = self.identifiers #.branch(node)
- print "NS IDENT", identifiers
class NSComponentVisitor(object):
def visitComponentTag(s, node):
self.write_inline_component(node, identifiers)
@@ -197,9 +204,10 @@ class _GenerateRenderMethod(object):
pass
class _Identifiers(object):
+ """tracks the status of identifier names as template code is rendered."""
def __init__(self, node=None, parent=None):
if parent is not None:
- # things that have already been declared in the current namespace (i.e. names we can just use)
+ # things that have already been declared in an enclosing namespace (i.e. names we can just use)
self.declared = util.Set(parent.declared).union([c.name for c in parent.closurecomponents]).union(parent.locally_declared)
# top level components that are available
@@ -208,12 +216,14 @@ class _Identifiers(object):
self.declared = util.Set()
self.toplevelcomponents = util.Set()
- # things within this level that are undeclared
+ # things within this level that are referenced before they are declared (e.g. assigned to)
self.undeclared = util.Set()
- # things that are declared locally
+ # things that are declared locally. some of these things could be in the "undeclared"
+ # list as well if they are referenced before declared
self.locally_declared = util.Set()
+ # things that are declared in the argument signature of the component callable
self.argument_declared = util.Set()
# closure components that are defined in this level
@@ -224,21 +234,21 @@ class _Identifiers(object):
node.accept_visitor(self)
def branch(self, node):
+ """create a new Identifiers for a new Node, with this Identifiers as the parent."""
return _Identifiers(node, self)
components = property(lambda s:s.toplevelcomponents.union(s.closurecomponents))
def __repr__(self):
return "Identifiers(%s, %s, %s, %s, %s)" % (repr(list(self.declared)), repr(list(self.locally_declared)), repr(list(self.undeclared)), repr([c.name for c in self.toplevelcomponents]), repr([c.name for c in self.closurecomponents]))
-
def check_declared(self, node):
- for ident in node.declared_identifiers():
- self.locally_declared.add(ident)
+ """update the state of this Identifiers with the undeclared and declared identifiers of the given node."""
for ident in node.undeclared_identifiers():
- print "UNDECL IDENT IN NODE", node, ident
if ident != 'context' and ident not in self.declared.union(self.locally_declared):
self.undeclared.add(ident)
+ for ident in node.declared_identifiers():
+ self.locally_declared.add(ident)
def visitExpression(self, node):
self.check_declared(node)
@@ -254,6 +264,7 @@ class _Identifiers(object):
self.closurecomponents.add(node)
for ident in node.declared_identifiers():
self.argument_declared.add(ident)
+ # visit components only one level deep
if node is self.node:
for n in node.nodes:
n.accept_visitor(self)
diff --git a/lib/mako/lexer.py b/lib/mako/lexer.py
index 2dfa5d1..2cae847 100644
--- a/lib/mako/lexer.py
+++ b/lib/mako/lexer.py
@@ -61,8 +61,7 @@ class Lexer(object):
self.control_line.append(node)
elif len(self.control_line) and not self.control_line[-1].is_ternary(node.keyword):
raise exceptions.SyntaxException("Keyword '%s' not a legal ternary for keyword '%s'" % (node.keyword, self.control_line[-1].keyword), self.matched_lineno, self.matched_charpos)
-
-
+
def parse(self):
length = len(self.text)
while (True):
diff --git a/lib/mako/parsetree.py b/lib/mako/parsetree.py
index 23936b9..849b6e9 100644
--- a/lib/mako/parsetree.py
+++ b/lib/mako/parsetree.py
@@ -204,12 +204,12 @@ class Tag(Node):
class IncludeTag(Tag):
__keyword__ = 'include'
def __init__(self, keyword, attributes, **kwargs):
- super(IncludeTag, self).__init__(keyword, attributes, util.Set(['file', 'import']), util.Set([]), util.Set(['file']), **kwargs)
+ super(IncludeTag, self).__init__(keyword, attributes, ('file', 'import'), (), ('file',), **kwargs)
class NamespaceTag(Tag):
__keyword__ = 'namespace'
def __init__(self, keyword, attributes, **kwargs):
- super(NamespaceTag, self).__init__(keyword, attributes, util.Set(['file']), util.Set(['name']), util.Set(['name']), **kwargs)
+ super(NamespaceTag, self).__init__(keyword, attributes, ('file',), ('name',), ('name',), **kwargs)
self.name = attributes['name']
def declared_identifiers(self):
return [self.name]
@@ -219,7 +219,7 @@ class NamespaceTag(Tag):
class ComponentTag(Tag):
__keyword__ = 'component'
def __init__(self, keyword, attributes, **kwargs):
- super(ComponentTag, self).__init__(keyword, attributes, util.Set([]), util.Set(['name']), util.Set(['name']), **kwargs)
+ super(ComponentTag, self).__init__(keyword, attributes, (), ('name',), ('name',), **kwargs)
name = attributes['name']
if re.match(r'^[\w_]+$',name):
name = name + "()"
@@ -236,15 +236,15 @@ class ComponentTag(Tag):
class CallTag(Tag):
__keyword__ = 'call'
def __init__(self, keyword, attributes, **kwargs):
- super(CallTag, self).__init__(keyword, attributes, util.Set([]), util.Set(['expr']), util.Set(['expr']), **kwargs)
+ super(CallTag, self).__init__(keyword, attributes, (), ('expr',), ('expr',), **kwargs)
class InheritTag(Tag):
__keyword__ = 'inherit'
def __init__(self, keyword, attributes, **kwargs):
- super(InheritTag, self).__init__(keyword, attributes, util.Set(['file']), util.Set([]), util.Set(['file']), **kwargs)
+ super(InheritTag, self).__init__(keyword, attributes, ('file',), (), ('file',), **kwargs)
class PageTag(Tag):
__keyword__ = 'page'
def __init__(self, keyword, attributes, **kwargs):
- super(PageTag, self).__init__(keyword, attributes, util.Set([]), util.Set([]), util.Set([]), **kwargs)
+ super(PageTag, self).__init__(keyword, attributes, (), (), (), **kwargs)
\ No newline at end of file
diff --git a/lib/mako/pygen.py b/lib/mako/pygen.py
index 1cd0d0d..8254396 100644
--- a/lib/mako/pygen.py
+++ b/lib/mako/pygen.py
@@ -197,6 +197,7 @@ class PythonPrinter(object):
def adjust_whitespace(text):
+ """remove the left-whitespace margin of a block of Python code."""
state = [False, False]
(backslashed, triplequoted) = (0, 1)
def in_multi_line(line):
diff --git a/lib/mako/runtime.py b/lib/mako/runtime.py
index 76cb4b4..f50632a 100644
--- a/lib/mako/runtime.py
+++ b/lib/mako/runtime.py
@@ -15,6 +15,8 @@ class Context(object):
def write(self, string):
self.buffer.write(string)
def update(self, **args):
+ """produce a copy of this Context, updating the argument dictionary
+ with the given keyword arguments."""
x = self.data.copy()
x.update(args)
c = Context(self.buffer, **x)
diff --git a/test/ast.py b/test/ast.py
index f418d61..5a7da59 100644
--- a/test/ast.py
+++ b/test/ast.py
@@ -24,7 +24,7 @@ print "hello world, ", a, b
print "Another expr", c
"""
parsed = ast.PythonCode(code, 0, 0)
- assert parsed.declared_identifiers == util.Set(['a','b','c', 'g', 'h', 'i', 'u', 'k', 'j', 'gh', 'lar'])
+ assert parsed.declared_identifiers == util.Set(['a','b','c', 'g', 'h', 'i', 'u', 'k', 'j', 'gh', 'lar', 'x'])
assert parsed.undeclared_identifiers == util.Set(['x', 'q', 'foo', 'gah', 'blah'])
parsed = ast.PythonCode("x + 5 * (y-z)", 0, 0)
@@ -45,6 +45,19 @@ for x in data:
assert parsed.undeclared_identifiers == util.Set(['get_data'])
assert parsed.declared_identifiers == util.Set(['result', 'data', 'x', 'hoho', 'foobar', 'foo', 'yaya'])
+ def test_locate_identifiers_3(self):
+ """test that combination assignment/expressions of the same identifier log the ident as 'undeclared'"""
+ code = """
+x = x + 5
+for y in range(1, y):
+ print "hi"
+[z for z in range(1, z)]
+(q for q in range (1, q))
+"""
+ parsed = ast.PythonCode(code, 0, 0)
+ print parsed.undeclared_identifiers
+ assert parsed.undeclared_identifiers == util.Set(['x', 'y', 'z', 'q'])
+
def test_no_global_imports(self):
code = """
from foo import *
diff --git a/test/template.py b/test/template.py
index 38e3b0e..8dd2f3e 100644
--- a/test/template.py
+++ b/test/template.py
@@ -99,6 +99,30 @@ im c
result = re.sub(r'[\s\n]+', ' ', result).strip()
assert result == "y is None y is 7"
+ def test_local_names_3(self):
+ """test in place assignment/undeclared variable combinations
+
+
+ i.e. things like 'x=x+1', x is declared and undeclared at the same time"""
+ template = Template("""
+ hi
+ <%component name="a">
+ y is ${y}
+
+ <%
+ x = x + y
+ a = 3
+ %>
+
+ x is ${x}
+ </%component>
+
+ ${a()}
+ """)
+ result = template.render(x=5, y=10)
+ result = re.sub(r'[\s\n]+', ' ', result).strip()
+ assert result == "hi y is 10 x is 15"
+
class NestedComponentTest(unittest.TestCase):
def test_nested_component(self):
template = """
--
GitLab