big overhaul to variable scoping in code generation

doc/build/genhtml.py

@@ -5,6 +5,7 @@ import cPickle as pickle
 sys.path = ['../../lib', './lib/'] + sys.path
 
 from mako.lookup import TemplateLookup
+from mako import exceptions
 
 import read_markdown, toc
 
@@ -39,7 +40,10 @@ def genfile(name, toc):
     outfile.write(lookup.get_template(infile).render(toc=toc, extension='html'))
     
 for filename in files:
-    genfile(filename, root)
+    try:
+        genfile(filename, root)
+    except:
+        print exceptions.text_error_template().render()
 
 
         

doc/build/templates/content_layout.html

 # defines the default layout for normal documentation pages (not including the index)
 <%inherit file="base.html"/>
 
+<%namespace file="nav.html" import="topnav"/>
 <%
     current = toc.get_by_file(self.template.module.filename)
 %>
 
 <A name="<% current.path %>"></a>
-${self.nav.topnav(item=current)}
-<div class="sectioncontent">
+
+${topnav(item=current)}
+
 ${next.body(toc=toc)}
-</div>

doc/build/templates/formatting.html

@@ -14,12 +14,18 @@
 <%namespace name="nav" file="nav.html"/>
 
 <%def name="section(toc, path, description=None)">
+    FORMATTING SECTION ${path}
     # Main section formatting element.
     <%
-        item = toc.get_by_path(path)
-        if item is None:
-            raise "path: " + path
+        import sys, traceback
+        try:
+            item = toc.get_by_path(path)
+        except:
+            context.write("EXCEPTION GETTING ITEM")
+            context.write(repr(sys.exc_info()[1].args))
+            context.write(repr(traceback.extract_tb(sys.exc_info()[2])))
     %>
+    CHECK1 ITEM ${item}
     <A name="${item.path}"></a>
 
     <div class="section" style="margin-left:${repr(item.depth * 10)}px;">
@@ -28,10 +34,12 @@
         re2 = re.compile(r"'''PYESC(.+?)PYESC'''", re.S)
         content = re2.sub(lambda m: m.group(1), content)
     %>
+    CHECK2
     % if item.depth > 1:
         <h3>${description or item.description}</h3>
     % endif
 
+    CHECK3
     ${content}
 
     % if item.depth > 1:
@@ -42,7 +50,7 @@
         <a href="#${ item.get_page_root().path }">back to section top</a>
         ${nav.pagenav(item=item)}
     % endif
-    
+    CHECK4
     </div>
 
 </%def>

lib/mako/ast.py

@@ -183,7 +183,13 @@ class FunctionDecl(object):
             else:
                 namedecls.insert(0, arg)
         return namedecls
+
+class FunctionArgs(FunctionDecl):
+    """the argument portion of a function declaration"""
+    def __init__(self, code, lineno, pos, filename):
+        super(FunctionArgs, self).__init__("def ANON(%s):pass" % code, lineno, pos, filename)
         
+            
 class ExpressionGenerator(object):
     """given an AST node, generates an equivalent literal Python expression."""
     def __init__(self, astnode):

lib/mako/codegen.py

@@ -30,7 +30,8 @@ class _GenerateRenderMethod(object):
         self.last_source_line = -1
         self.compiler = compiler
         self.node = node
-
+        self.identifier_stack = [None]
+        
         self.in_def = isinstance(node, parsetree.DefTag)
 
         if self.in_def:
@@ -58,7 +59,9 @@ class _GenerateRenderMethod(object):
         if defs is not None:
             for node in defs:
                 _GenerateRenderMethod(printer, compiler, node)
-            
+    
+    identifiers = property(lambda self:self.identifier_stack[-1])
+    
     def write_toplevel(self):
         """traverse a template structure for module-level directives and generate the
         start of module-level code."""
@@ -127,7 +130,8 @@ class _GenerateRenderMethod(object):
             self.printer.writeline("context.push_buffer()")
             self.printer.writeline("try:")
 
-        self.identifiers = self.compiler.identifiers.branch(self.node)
+        self.identifier_stack.append(self.compiler.identifiers.branch(self.node))
+
         if not self.in_def and len(self.identifiers.locally_assigned) > 0:
             self.printer.writeline("__locals = {}")
 
@@ -289,11 +293,14 @@ class _GenerateRenderMethod(object):
                 )
 
         identifiers = identifiers.branch(node, nested=nested)
-        self.write_variable_declares(identifiers)
 
+        self.write_variable_declares(identifiers)
+        
+        self.identifier_stack.append(identifiers)
         for n in node.nodes:
             n.accept_visitor(self)
-
+        self.identifier_stack.pop()
+        
         self.write_def_finish(node, buffered, filtered, cached)
         self.printer.writeline(None)
         if cached:
@@ -325,7 +332,7 @@ class _GenerateRenderMethod(object):
         self.printer.writeline("__%s = %s" % (name, name))
         cachekey = node_or_pagetag.parsed_attributes.get('cache_key', repr(name))
         self.printer.writeline("def %s(context, *args, **kwargs):" % name)
-        print "LIMIT", node_or_pagetag.undeclared_identifiers()
+
         self.write_variable_declares(identifiers, limit=node_or_pagetag.undeclared_identifiers())
         if buffered:
             self.printer.writelines(
@@ -403,18 +410,25 @@ class _GenerateRenderMethod(object):
         pass
 
     def visitCallTag(self, node):
-        self.printer.writeline("def ccall(context):")
+        self.printer.writeline("def ccall(caller):")
         export = ['body']
-        identifiers = self.identifiers.branch(node, includedefs=True, nested=True)
-        body_identifiers = identifiers.branch(node, includedefs=True, nested=False)
+        callable_identifiers = self.identifiers.branch(node, includedefs=True, nested=True)
+        body_identifiers = callable_identifiers.branch(node, includedefs=False, nested=False)
+        body_identifiers.add_declared('caller')
+        callable_identifiers.add_declared('caller')
+        
+        self.identifier_stack.append(body_identifiers)
         class DefVisitor(object):
             def visitDefTag(s, node):
-                self.write_inline_def(node, identifiers, nested=False)
+                self.write_inline_def(node, callable_identifiers, nested=False)
                 export.append(node.name)
         vis = DefVisitor()
         for n in node.nodes:
             n.accept_visitor(vis)
-        self.printer.writeline("def body(**kwargs):")
+        self.identifier_stack.pop()
+        
+        bodyargs = node.body_decl.get_argument_expressions()    
+        self.printer.writeline("def body(%s):" % ','.join(bodyargs))
         # TODO: figure out best way to specify buffering/nonbuffering (at call time would be better)
         buffered = False
         if buffered:
@@ -423,8 +437,11 @@ class _GenerateRenderMethod(object):
                 "try:"
             )
         self.write_variable_declares(body_identifiers)
+        self.identifier_stack.append(body_identifiers)
         for n in node.nodes:
             n.accept_visitor(self)
+        self.identifier_stack.pop()
+        
         self.write_def_finish(node, buffered, False, False)
         self.printer.writelines(
             None,
@@ -432,17 +449,9 @@ class _GenerateRenderMethod(object):
             None
         )
 
-        self.printer.writeline(
-        # preserve local instance of current caller in local scope
-        "__cl = context.locals_({'caller':context.caller_stack[-1]})",
-        )
-
         self.printer.writelines(
             # push on global "caller" to be picked up by the next ccall
-            "context.caller_stack.append(runtime.Namespace('caller', __cl, callables=ccall(__cl)))",
-            # TODO: clean this up - insure proper caller is set
-            "context._data['caller'] = runtime._StackFacade(context.caller_stack)",
-            #"context.write('GOING TO CALL %s WITH CONTEXT ID '+ repr(id(context)) + ' CALLER ' + repr(context.get('caller')))" % node.attributes['expr'],
+            "context.caller_stack.append(runtime.Namespace('caller', context, callables=ccall(context.caller_stack[-1])))",
             "try:")
         self.write_source_comment(node)
         self.printer.writelines(
@@ -458,7 +467,7 @@ class _Identifiers(object):
     def __init__(self, node=None, parent=None, includedefs=True, includenode=True, nested=False):
         if parent is not None:
             # things that have already been declared in an enclosing namespace (i.e. names we can just use)
-            self.declared = util.Set(parent.declared).union([c.name for c in parent.closuredefs]).union(parent.locally_declared)
+            self.declared = util.Set(parent.declared).union([c.name for c in parent.closuredefs]).union(parent.locally_declared).union(parent.argument_declared)
             
             # if these identifiers correspond to a "nested" scope, it means whatever the 
             # parent identifiers had as undeclared will have been declared by that parent, 
@@ -506,7 +515,7 @@ class _Identifiers(object):
     defs = property(lambda s:s.topleveldefs.union(s.closuredefs))
     
     def __repr__(self):
-        return "Identifiers(%s, %s, %s, %s, %s)" % (repr(list(self.declared)), repr(list(self.locally_declared)), repr(list(self.undeclared)), repr([c.name for c in self.topleveldefs]), repr([c.name for c in self.closuredefs]))
+        return "Identifiers(declared=%s, locally_declared=%s, undeclared=%s, topleveldefs=%s, closuredefs=%s, argumenetdeclared=%s)" % (repr(list(self.declared)), repr(list(self.locally_declared)), repr(list(self.undeclared)), repr([c.name for c in self.topleveldefs]), repr([c.name for c in self.closuredefs]), repr(self.argument_declared))
         
     def check_declared(self, node):
         """update the state of this Identifiers with the undeclared and declared identifiers of the given node."""
@@ -515,7 +524,12 @@ class _Identifiers(object):
                 self.undeclared.add(ident)
         for ident in node.declared_identifiers():
             self.locally_declared.add(ident)
-                
+    
+    def add_declared(self, ident):
+        self.declared.add(ident)
+        if ident in self.undeclared:
+            self.undeclared.remove(ident)
+                        
     def visitExpression(self, node):
         self.check_declared(node)
     def visitControlLine(self, node):
@@ -534,10 +548,10 @@ class _Identifiers(object):
         for ident in node.undeclared_identifiers():
             if ident != 'context' and ident not in self.declared.union(self.locally_declared):
                 self.undeclared.add(ident)
-        for ident in node.declared_identifiers():
-            self.argument_declared.add(ident)
         # visit defs only one level deep
         if node is self.node:
+            for ident in node.declared_identifiers():
+                self.argument_declared.add(ident)
             for n in node.nodes:
                 n.accept_visitor(self)
     def visitIncludeTag(self, node):
@@ -545,7 +559,16 @@ class _Identifiers(object):
     def visitPageTag(self, node):
         self.check_declared(node)            
     def visitCallTag(self, node):
-        self.check_declared(node)
         if node is self.node:
+            for ident in node.undeclared_identifiers():
+                if ident != 'context' and ident not in self.declared.union(self.locally_declared):
+                    self.undeclared.add(ident)
+            for ident in node.declared_identifiers():
+                self.argument_declared.add(ident)
             for n in node.nodes:
-                n.accept_visitor(self)
\ No newline at end of file
+                n.accept_visitor(self)
+        else:
+            for ident in node.undeclared_identifiers():
+                if ident != 'context' and ident not in self.declared.union(self.locally_declared):
+                    self.undeclared.add(ident)
+                
\ No newline at end of file

lib/mako/parsetree.py

@@ -258,10 +258,11 @@ class DefTag(Tag):
 class CallTag(Tag):
     __keyword__ = 'call'
     def __init__(self, keyword, attributes, **kwargs):
-        super(CallTag, self).__init__(keyword, attributes, (), ('expr',), ('expr',), **kwargs)
+        super(CallTag, self).__init__(keyword, attributes, ('args'), ('expr',), ('expr',), **kwargs)
         self.code = ast.PythonCode(attributes['expr'], self.lineno, self.pos, self.filename)
+        self.body_decl = ast.FunctionArgs(attributes.get('args', ''), self.lineno, self.pos, self.filename)
     def declared_identifiers(self):
-        return self.code.declared_identifiers
+        return self.code.declared_identifiers.union(self.body_decl.argnames)
     def undeclared_identifiers(self):
         return self.code.undeclared_identifiers
 

lib/mako/runtime.py

@@ -7,7 +7,9 @@
 """provides the Context class, the runtime namespace for templates."""
 from mako import exceptions, util
 import inspect, sys
-        
+
+x = 0
+
 class Context(object):
     """provides runtime namespace, output buffer, and various callstacks for templates."""
     def __init__(self, buffer, **data):
@@ -26,8 +28,6 @@ class Context(object):
     def keys(self):
         return self._data.keys()
     def __getitem__(self, key):
-        if key == 'caller':
-            return _StackFacade(self.caller_stack)
         return self._data[key]
     def _put(self, key, value):
         self._data[key] = value
@@ -49,9 +49,14 @@ class Context(object):
         c._with_template = self._with_template
         c.namespaces = self.namespaces
         c.caller_stack = self.caller_stack
-        if not c._data.has_key('caller'):
-            raise "WTF"
         return c
+    def localize_caller_stack(self):
+        global x
+        x += 1
+        if x > 20:
+            raise "HI"
+        print "LOCALIZE!"
+        return self
     def locals_(self, d):
         """create a new Context with a copy of this Context's current state, updated with the given dictionary."""
         c = self._copy()

test/cache.py

@@ -39,9 +39,12 @@ class CacheTest(unittest.TestCase):
         %>
         callcount: ${callcount}
 """)
-        print t.render()
-        print t.render()
-        print t.render()
+        t.render()
+        t.render()
+        assert result_lines(t.render()) == [
+            "this is foo",
+            "callcount: [1]"
+        ]
         
 
         

test/call.py

@@ -9,12 +9,10 @@ class CallTest(unittest.TestCase):
             hi im foo ${caller.body(y=5)}
         </%def>
         
-        <%call expr="foo()">
+        <%call expr="foo()" args="y, **kwargs">
             this is the body, y is ${y}
         </%call>
 """)
-        print t.code
-        print t.render()
         assert result_lines(t.render()) == ['hi im foo', 'this is the body, y is 5']
 
 
@@ -43,10 +41,11 @@ class CallTest(unittest.TestCase):
         ${bar()}
 
 """)
-        print t.code
+
         assert result_lines(t.render()) == ['foo calling comp1:', 'this is comp1, 5', 'foo calling body:', 'this is the body,', 'this is comp1, 6', 'this is bar']
 
-    def test_multi_call(self):
+    def test_chained_call(self):
+        """test %calls that are chained through their targets"""
         t = Template("""
             <%def name="a">
                 this is a. 
@@ -56,7 +55,8 @@ class CallTest(unittest.TestCase):
             </%def>
             <%def name="b">
                 this is b.  heres  my body: ${caller.body()}
-                whats in the body's caller's body ? ${caller.context['caller'].body()}
+                whats in the body's caller's body ?
+                ${context.caller_stack[-2].body()}
             </%def>
             
             <%call expr="a()">
@@ -64,7 +64,6 @@ class CallTest(unittest.TestCase):
             </%call>
             
 """)
-        print t.render()
         assert result_lines(t.render()) == [
             'this is a.',
             'this is b. heres my body:',
@@ -74,7 +73,66 @@ class CallTest(unittest.TestCase):
             'heres the main templ call'
         ]
 
-    def test_multi_call_in_nested(self):
+    def test_nested_call(self):
+        """test %calls that are nested inside each other"""
+        t = Template("""
+            <%def name="foo">
+                ${caller.body(x=10)}
+            </%def>
+
+            x is ${x}
+            <%def name="bar">
+                bar: ${caller.body()}
+            </%def>
+
+            <%call expr="foo()" args="x">
+                this is foo body: ${x}
+
+                <%call expr="bar()">
+                    this is bar body: ${x}
+                </%call>
+            </%call>
+""")
+        assert result_lines(t.render(x=5)) == [
+            "x is 5",
+            "this is foo body: 10",
+            "bar:",
+            "this is bar body: 10"
+        ]
+        
+    def test_nested_call_2(self):
+        t = Template("""
+            x is ${x}
+            <%def name="foo">
+                ${caller.foosub(x=10)}
+            </%def>
+
+            <%def name="bar">
+                bar: ${caller.barsub()}
+            </%def>
+
+            <%call expr="foo()">
+                <%def name="foosub(x)">
+                this is foo body: ${x}
+                
+                <%call expr="bar()">
+                    <%def name="barsub">
+                    this is bar body: ${x}
+                    </%def>
+                </%call>
+                
+                </%def>
+
+            </%call>
+""")
+        assert result_lines(t.render(x=5)) == [
+            "x is 5",
+            "this is foo body: 10",
+            "bar:",
+            "this is bar body: 10"
+        ]
+        
+    def test_chained_call_in_nested(self):
         t = Template("""
             <%def name="embedded">
             <%def name="a">
@@ -85,7 +143,7 @@ class CallTest(unittest.TestCase):
             </%def>
             <%def name="b">
                 this is b.  heres  my body: ${caller.body()}
-                whats in the body's caller's body ? ${caller.context['caller'].body()}
+                whats in the body's caller's body ? ${context.caller_stack[-2].body()}
             </%def>
 
             <%call expr="a()">
@@ -94,7 +152,6 @@ class CallTest(unittest.TestCase):
             </%def>
             ${embedded()}
 """)
-        print t.render()
         assert result_lines(t.render()) == [
             'this is a.',
             'this is b. heres my body:',
@@ -122,43 +179,6 @@ class CallTest(unittest.TestCase):
 """)
         assert result_lines(t.render()) == ['this is a', 'this is b', 'this is c:', "this is the body in b's call"]
 
-    def test_ccall_args(self):
-        t = Template("""
-            <%def name="foo">
-                foo context id: ${id(context)}
-                foo cstack: ${repr(context.caller_stack)}
-                foo, ccaller is ${context.get('caller')}
-                foo, context data is ${repr(context._data)}
-                ${caller.body(x=10)}
-            </%def>
-            
-            <%def name="bar">
-                bar context id: ${id(context)}
-                bar cstack: ${repr(context.caller_stack)}
-                bar, cs is ${context.caller_stack[-1]}
-                bar, caller is ${caller}
-                bar, ccaller is ${context.get('caller')}
-                bar, body is ${context.caller_stack[-1].body()}
-                bar, context data is ${repr(context._data)}
-            </%def>
-            
-            x is: ${x}
-
-            main context id: ${id(context)}
-            main cstack: ${repr(context.caller_stack)}
-            
-            <%call expr="foo()">
-                this is foo body: ${x}
-                
-                foocall context id: ${id(context)}
-                foocall cstack: ${repr(context.caller_stack)}
-                <%call expr="bar()">
-                    this is bar body: ${x}
-                </%call>
-            </%call>
-""")
-        print t.code
-        print t.render(x=5)
         
     def test_call_in_nested_2(self):
         t = Template("""
@@ -212,7 +232,11 @@ class SelfCacheTest(unittest.TestCase):
         ${foo()}
         ${foo()}
 """)
-        print t.render()
+        assert result_lines(t.render()) == [
+            "this is foo",
+            "cached:",
+            "this is foo"
+        ]
         
 if __name__ == '__main__':
     unittest.main()

test/def.py

 from mako.template import Template
 from mako import lookup
 import unittest
-from util import flatten_result
+from util import flatten_result, result_lines
 
 class DefTest(unittest.TestCase):
     def test_def_noargs(self):
@@ -251,6 +251,24 @@ class ScopeTest(unittest.TestCase):
     """)
         assert flatten_result(t.render()) == "main/a: a/y: 10 a/b: b/c: c/y: 10 b/y: 19 main/y: 7"
 
+    def test_scope_eleven(self):
+        t = Template("""
+            x is ${x}
+            <%def name="a(x)">
+                this is a, ${b()}
+                <%def name="b">
+                    this is b, x is ${x}
+                </%def>
+            </%def>
+            
+            ${a(x=5)}
+""")
+        assert result_lines(t.render(x=10)) == [
+            "x is 10",
+            "this is a,", 
+            "this is b, x is 5"
+        ]
+
     def test_unbound_scope(self):
         t = Template("""
             <%
@@ -329,8 +347,8 @@ class NestedDefTest(unittest.TestCase):
             </%def>
             ${a()}
 """)
-        print t.code
-        print t.render(x=10)
+        
+        assert flatten_result(t.render(x=10)) == "x is 10 this is a, x is 10 this is b: 10"
         
     def test_nested_with_args(self):
         t = Template("""

test/lexer.py

@@ -14,7 +14,6 @@ class LexerTest(unittest.TestCase):
         and some more text.
 """
         node = Lexer(template).parse()
-        print repr(node)
         assert repr(node) == r"""TemplateNode({}, [Text('\n<b>Hello world</b>\n        ', (1, 1)), DefTag('def', {'name': 'foo'}, (3, 9), ["Text('\\n                this is a def.\\n        ', (3, 26))"]), Text('\n        \n        and some more text.\n', (5, 16))])"""
 
     def test_unclosed_tag(self):

test/namespace.py

@@ -64,7 +64,6 @@ class NamespaceTest(unittest.TestCase):
         </%def>
 """)
 
-        print collection.get_template('main.html').render(x="context x")
         assert flatten_result(collection.get_template('main.html').render(x="context x")) == "this is main. def1: x is context x def2: x is there"
         
     def test_overload(self):
@@ -279,7 +278,7 @@ class NamespaceTest(unittest.TestCase):
             ${b()}
             ${x}
         """)
-        print collection.get_template('index.html').code
+
         assert result_lines(collection.get_template("index.html").render(bar="this is bar", x="this is x")) == [
             "this is foo",
             "this is bar",
@@ -343,9 +342,16 @@ class NamespaceTest(unittest.TestCase):
                 </%def>
             </%call>
         """)
-        print collection.get_template("index.html").code
-        print collection.get_template("functions.html").code
-        print collection.get_template("index.html").render()
+        #print collection.get_template("index.html").code
+        #print collection.get_template("functions.html").code
+        assert result_lines(collection.get_template("index.html").render()) == [
+            "this is bar.",
+            "this is index embedded",
+            "foo is",
+            "this is foo",
+            "this is lala",
+            "this is foo"
+        ]
 
 if __name__ == '__main__':
     unittest.main()

test/template.py

@@ -57,17 +57,6 @@ class GlobalsTest(unittest.TestCase):
 """)
         assert t.render().strip() == "y is hi"
 
-class FormatExceptionTest(unittest.TestCase):
-    def test_html(self):
-        t = Template("""
-        
-            hi there.
-            <%
-                raise "hello"
-            %>
-        """, format_exceptions=True)
-        res = t.render()
-        print res
     
             
 if __name__ == '__main__':