added missing files from [ticket:45] checkin

lib/mako/ext/babelplugin.py

+"""gettext message extraction via Babel: http://babel.edgewall.org/"""
+from StringIO import StringIO
+
+from babel.messages.extract import extract_python
+
+from mako import lexer, parsetree
+
+def extract(fileobj, keywords, comment_tags, options):
+    """Extract messages from Mako templates.
+
+    :param fileobj: the file-like object the messages should be extracted from
+    :param keywords: a list of keywords (i.e. function names) that should be
+                     recognized as translation functions
+    :param comment_tags: a list of translator tags to search for and include
+                         in the results
+    :param options: a dictionary of additional options (optional)
+    :return: an iterator over ``(lineno, funcname, message, comments)`` tuples
+    :rtype: ``iterator``
+    """
+    encoding = options.get('input_encoding', options.get('encoding', None))
+
+    template_node = lexer.Lexer(fileobj.read(),
+                                input_encoding=encoding).parse()
+    for extracted in extract_nodes(template_node.get_children(),
+                                        keywords, comment_tags, options):
+        yield extracted
+
+def extract_nodes(nodes, keywords, comment_tags, options):
+    """Extract messages from Mako's lexer node objects
+
+    :param nodes: an iterable of Mako parsetree.Node objects to extract from
+    :param keywords: a list of keywords (i.e. function names) that should be
+                     recognized as translation functions
+    :param comment_tags: a list of translator tags to search for and include
+                         in the results
+    :param options: a dictionary of additional options (optional)
+    :return: an iterator over ``(lineno, funcname, message, comments)`` tuples
+    :rtype: ``iterator``
+    """
+    translator_comments = []
+    in_translator_comments = False
+
+    for node in nodes:
+        child_nodes = None
+        if in_translator_comments and isinstance(node, parsetree.Text) and \
+                not node.content.strip():
+            # Ignore whitespace within translator comments
+            continue
+
+        if isinstance(node, parsetree.Comment):
+            value = node.text.strip()
+            if in_translator_comments:
+                translator_comments.extend(_split_comment(node.lineno, value))
+                continue
+            for comment_tag in comment_tags:
+                if value.startswith(comment_tag):
+                    in_translator_comments = True
+                    comment = value[len(comment_tag):].strip()
+                    translator_comments.extend(_split_comment(node.lineno,
+                                                              comment))
+            continue
+
+        if isinstance(node, parsetree.DefTag):
+            code = node.function_decl.code
+            child_nodes = node.nodes
+        elif isinstance(node, parsetree.CallTag):
+            code = node.code.code
+            child_nodes = node.nodes
+        elif isinstance(node, parsetree.PageTag):
+            code = node.body_decl.code
+        elif isinstance(node, parsetree.ControlLine):
+            if node.isend:
+                translator_comments = []
+                in_translator_comments = False
+                continue
+            code = node.text
+        elif isinstance(node, parsetree.Code):
+            # <% and <%! blocks would provide their own translator comments
+            translator_comments = []
+            in_translator_comments = False
+
+            code = node.code.code
+        elif isinstance(node, parsetree.Expression):
+            code = node.code.code
+        else:
+            translator_comments = []
+            in_translator_comments = False
+            continue
+
+        # Comments don't apply unless they immediately preceed the message
+        if translator_comments and \
+                translator_comments[-1][0] < node.lineno - 1:
+            translator_comments = []
+        else:
+            translator_comments = \
+                [comment[1] for comment in translator_comments]
+
+        if isinstance(code, unicode):
+            code = code.encode('ascii', 'backslashreplace')
+        code = StringIO(code)
+        for lineno, funcname, messages, python_translator_comments \
+                in extract_python(code, keywords, comment_tags, options):
+            yield (node.lineno + (lineno - 1), funcname, messages,
+                   translator_comments + python_translator_comments)
+
+        translator_comments = []
+        in_translator_comments = False
+
+        if child_nodes:
+            for extracted in extract_nodes(child_nodes, keywords, comment_tags,
+                                           options):
+                yield extracted
+
+
+def _split_comment(lineno, comment):
+    """Return the multiline comment at lineno split into a list of comment line
+    numbers and the accompanying comment line"""
+    comments = []
+    comment_index = 0
+    for comment_line in comment.splitlines():
+        comments.append((lineno + comment_index, comment_line))
+        comment_index += 1
+    return comments

test/babelplugin.py

+import unittest
+
+try:
+    import babel
+
+    import os
+    from mako.ext.babelplugin import extract
+
+    class ExtractMakoTestCase(unittest.TestCase):
+        def test_extract(self):
+            mako_tmpl = open(os.path.join(os.path.dirname(__file__),
+                                          'templates', 'gettext.mako'))
+            messages = list(extract(mako_tmpl, {'_': None, 'gettext': None,
+                                                'ungettext': (1, 2)},
+                                    ['TRANSLATOR:'], {}))
+            expected = \
+                [(1, u'_', 'Page arg 1', []),
+                 (1, u'_', 'Page arg 2', []),
+                 (10, u'gettext', 'Begin', []),
+                 (14, u'_', 'Hi there!', [u'Hi there!']),
+                 (19, u'_', 'Hello', []),
+                 (22, u'_', 'Welcome', []),
+                 (25, u'_', 'Yo', []),
+                 (36, u'_', 'The', [u'Ensure so and', u'so, thanks']),
+                 (36, u'ungettext', ('bunny', 'bunnies', ''), []),
+                 (41, u'_', 'Goodbye', [u'Good bye']),
+                 (44, u'_', 'Babel', []),
+                 (45, u'ungettext', ('hella', 'hellas'), []),
+                 (62, u'_', 'Goodbye, really!', [u'HTML comment']),
+                 (65, u'_', 'P.S. byebye', []),
+                 (71, u'_', 'Top', [])]
+            self.assertEqual(expected, messages)
+
+except ImportError:
+    import warnings
+    warnings.warn('babel not installed: skipping babelplugin test',
+                  RuntimeWarning, 1)
+
+if __name__ == '__main__':
+    unittest.main()

test/templates/gettext.mako

+<%page args="x, y=_('Page arg 1'), z=_('Page arg 2')"/>
+<%!
+import random
+def gettext(message): return message
+_ = gettext
+def ungettext(s, p, c):
+    if c == 1:
+        return s
+    return p
+top = gettext('Begin')
+%>
+<%
+   # TRANSLATOR: Hi there!
+   hithere = _('Hi there!')
+
+   # TRANSLATOR: you should not be seeing this in the .po
+   rows = [[v for v in range(0,10)] for row in range(0,10)]
+
+   hello = _('Hello')
+%>
+<div id="header">
+  ${_('Welcome')}
+</div>
+<table>
+    % for row in (hithere, hello, _('Yo')):
+        ${makerow(row)}
+    % endfor
+    ${makerow(count=2)}
+</table>
+
+
+<div id="main">
+
+## TRANSLATOR: Ensure so and
+## so, thanks
+  ${_('The')} fuzzy ${ungettext('bunny', 'bunnies', random.randint(1, 2))}
+</div>
+
+<div id="footer">
+  ## TRANSLATOR: Good bye
+  ${_('Goodbye')}
+</div>
+   
+<%def name="makerow(row=_('Babel'), count=1)">
+    <!-- ${ungettext('hella', 'hellas', count)} -->
+    % for i in range(count):
+      <tr>
+      % for name in row:
+          <td>${name}</td>\
+      % endfor
+      </tr>
+    % endfor
+</%def>
+
+<%def name="comment()">
+  <!-- ${caller.body()} -->
+</%def>
+
+<%call expr="comment">
+  P.S.
+  ## TRANSLATOR: HTML comment
+  ${_('Goodbye, really!')}
+</%call>
+
+<!-- ${_('P.S. byebye')} -->
+
+<div id="end">
+  <a href="#top">
+    ## TRANSLATOR: you won't see this either
+    
+    ${_('Top')}
+  </a>
+</div>