From 004aca569a2eebc903044f95e93f36516a44a658 Mon Sep 17 00:00:00 2001
From: Mike Bayer <mike_mp@zzzcomputing.com>
Date: Tue, 2 Mar 2010 23:05:41 +0000
Subject: [PATCH] - ensure lru threading test doesn't run - Source code
escaping has been simplified. In particular, module source files are now
generated with the Python "magic encoding comment", and source code is
passed through mostly unescaped, except for that code which is
regenerated from parsed Python source. This fixes usage of unicode in
<%namespace:defname> tags. [ticket:99]
---
CHANGES | 11 +++++++++--
mako/__init__.py | 2 +-
mako/codegen.py | 12 ++++++------
mako/lexer.py | 16 ++++++----------
mako/pyparser.py | 4 ++++
mako/runtime.py | 5 ++++-
mako/template.py | 8 ++++++--
mako/util.py | 3 ++-
setup.py | 2 +-
test/test_lru.py | 7 +++++--
test/test_template.py | 23 +++++++++++++++++++++++
11 files changed, 67 insertions(+), 26 deletions(-)
diff --git a/CHANGES b/CHANGES
index 66f47ed..f048aed 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,9 +1,16 @@
0.3
-- Python 2.3 support is dropped (who can resist
- @decorators) [ticket:123]
+- Python 2.3 support is dropped. [ticket:123]
- Unit tests now run with nose. [ticket:127]
+- Source code escaping has been simplified.
+ In particular, module source files are now
+ generated with the Python "magic encoding
+ comment", and source code is passed through
+ mostly unescaped, except for that code which
+ is regenerated from parsed Python source.
+ This fixes usage of unicode in
+ <%namespace:defname> tags. [ticket:99]
0.2.6
diff --git a/mako/__init__.py b/mako/__init__.py
index 6a30418..2b7e0cf 100644
--- a/mako/__init__.py
+++ b/mako/__init__.py
@@ -5,5 +5,5 @@
# the MIT License: http://www.opensource.org/licenses/mit-license.php
-__version__ = '0.2.5'
+__version__ = '0.3.0'
diff --git a/mako/codegen.py b/mako/codegen.py
index b3074f0..19fe18e 100644
--- a/mako/codegen.py
+++ b/mako/codegen.py
@@ -20,12 +20,12 @@ def compile(node,
buffer_filters=None,
imports=None,
source_encoding=None,
- generate_unicode=True):
+ generate_magic_comment=True):
"""Generate module source code given a parsetree node,
uri, and optional source filename"""
- buf = util.FastEncodingBuffer(unicode=generate_unicode)
+ buf = util.FastEncodingBuffer()
printer = PythonPrinter(buf)
_GenerateRenderMethod(printer,
@@ -35,7 +35,7 @@ def compile(node,
buffer_filters,
imports,
source_encoding,
- generate_unicode),
+ generate_magic_comment),
node)
return buf.getvalue()
@@ -47,14 +47,14 @@ class _CompileContext(object):
buffer_filters,
imports,
source_encoding,
- generate_unicode):
+ generate_magic_comment):
self.uri = uri
self.filename = filename
self.default_filters = default_filters
self.buffer_filters = buffer_filters
self.imports = imports
self.source_encoding = source_encoding
- self.generate_unicode = generate_unicode
+ self.generate_magic_comment = generate_magic_comment
class _GenerateRenderMethod(object):
"""A template visitor object which generates the
@@ -146,7 +146,7 @@ class _GenerateRenderMethod(object):
module_identifiers.declared = module_ident
# module-level names, python code
- if not self.compiler.generate_unicode and \
+ if self.compiler.generate_magic_comment and \
self.compiler.source_encoding:
self.printer.writeline("# -*- encoding:%s -*-" %
self.compiler.source_encoding)
diff --git a/mako/lexer.py b/mako/lexer.py
index 65be795..caf295b 100644
--- a/mako/lexer.py
+++ b/mako/lexer.py
@@ -128,15 +128,10 @@ class Lexer(object):
(node.keyword, self.control_line[-1].keyword),
**self.exception_kwargs)
- def escape_code(self, text):
- if not self.disable_unicode and self.encoding:
- return text.encode('ascii', 'backslashreplace')
- else:
- return text
-
def parse(self):
for preproc in self.preprocessor:
self.text = preproc(self.text)
+
if not isinstance(self.text, unicode) and self.text.startswith(codecs.BOM_UTF8):
self.text = self.text[len(codecs.BOM_UTF8):]
parsed_encoding = 'utf-8'
@@ -149,6 +144,7 @@ class Lexer(object):
0, 0, self.filename)
else:
parsed_encoding = self.match_encoding()
+
if parsed_encoding:
self.encoding = parsed_encoding
@@ -242,7 +238,7 @@ class Lexer(object):
key, val1, val2 = att
text = val1 or val2
text = text.replace('\r\n', '\n')
- attributes[key] = self.escape_code(text)
+ attributes[key] = text
self.append_node(parsetree.Tag, keyword, attributes)
if isend:
self.tag.pop()
@@ -326,7 +322,7 @@ class Lexer(object):
text = adjust_whitespace(text) + "\n"
self.append_node(
parsetree.Code,
- self.escape_code(text),
+ text,
match.group(1)=='!', lineno=line, pos=pos)
return True
else:
@@ -344,7 +340,7 @@ class Lexer(object):
text = text.replace('\r\n', '\n')
self.append_node(
parsetree.Expression,
- self.escape_code(text), escapes.strip(),
+ text, escapes.strip(),
lineno=line, pos=pos)
return True
else:
@@ -376,7 +372,7 @@ class Lexer(object):
"Keyword '%s' doesn't match keyword '%s'" %
(text, self.control_line[-1].keyword),
**self.exception_kwargs)
- self.append_node(parsetree.ControlLine, keyword, isend, self.escape_code(text))
+ self.append_node(parsetree.ControlLine, keyword, isend, text)
else:
self.append_node(parsetree.Comment, text)
return True
diff --git a/mako/pyparser.py b/mako/pyparser.py
index c79692c..34b2a6a 100644
--- a/mako/pyparser.py
+++ b/mako/pyparser.py
@@ -28,10 +28,14 @@ except ImportError:
def parse(code, mode='exec', **exception_kwargs):
"""Parse an expression into AST"""
+
+
try:
if _ast:
return _ast_util.parse(code, '<unknown>', mode)
else:
+ if isinstance(code, unicode):
+ code = code.encode('ascii', 'backslashreplace')
return compiler_parse(code, mode)
except Exception, e:
raise exceptions.SyntaxException("(%s) %s (%s)" % (e.__class__.__name__, str(e), repr(code[0:50])), **exception_kwargs)
diff --git a/mako/runtime.py b/mako/runtime.py
index a475b71..583b79e 100644
--- a/mako/runtime.py
+++ b/mako/runtime.py
@@ -361,7 +361,10 @@ def _render(template, callable_, args, data, as_unicode=False):
if as_unicode:
buf = util.FastEncodingBuffer(unicode=True)
elif template.output_encoding:
- buf = util.FastEncodingBuffer(unicode=as_unicode, encoding=template.output_encoding, errors=template.encoding_errors)
+ buf = util.FastEncodingBuffer(
+ unicode=as_unicode,
+ encoding=template.output_encoding,
+ errors=template.encoding_errors)
else:
buf = util.StringIO()
context = Context(buf, **data)
diff --git a/mako/template.py b/mako/template.py
index 4bf01e0..87f6898 100644
--- a/mako/template.py
+++ b/mako/template.py
@@ -352,7 +352,7 @@ def _compile_text(template, text, filename):
buffer_filters=template.buffer_filters,
imports=template.imports,
source_encoding=lexer.encoding,
- generate_unicode=not template.disable_unicode)
+ generate_magic_comment=template.disable_unicode)
cid = identifier
if isinstance(cid, unicode):
@@ -378,8 +378,12 @@ def _compile_module_file(template, text, filename, outputpath):
buffer_filters=template.buffer_filters,
imports=template.imports,
source_encoding=lexer.encoding,
- generate_unicode=not template.disable_unicode)
+ generate_magic_comment=True)
(dest, name) = tempfile.mkstemp()
+
+ if isinstance(source, unicode):
+ source = source.encode(lexer.encoding or 'ascii')
+
os.write(dest, source)
os.close(dest)
shutil.move(name, outputpath)
diff --git a/mako/util.py b/mako/util.py
index 472535d..6119fa0 100644
--- a/mako/util.py
+++ b/mako/util.py
@@ -72,7 +72,8 @@ class SetLikeDict(dict):
return x
class FastEncodingBuffer(object):
- """a very rudimentary buffer that is faster than StringIO, but doesnt crash on unicode data like cStringIO."""
+ """a very rudimentary buffer that is faster than StringIO,
+ but doesnt crash on unicode data like cStringIO."""
def __init__(self, encoding=None, errors='strict', unicode=False):
self.data = []
diff --git a/setup.py b/setup.py
index fe03b70..caed701 100644
--- a/setup.py
+++ b/setup.py
@@ -25,7 +25,7 @@ SVN version:
""",
classifiers=[
- 'Development Status :: 4 - Beta',
+ "Development Status :: 5 - Production/Stable",
'Environment :: Web Environment',
'Intended Audience :: Developers',
'Programming Language :: Python',
diff --git a/test/test_lru.py b/test/test_lru.py
index d75b47a..e2b5c15 100644
--- a/test/test_lru.py
+++ b/test/test_lru.py
@@ -41,7 +41,7 @@ class LRUTest(unittest.TestCase):
for id in (25, 24, 23, 14, 12, 19, 18, 17, 16, 15):
self.assert_(l.has_key(id))
- def disabled_test_threaded(self):
+ def _disabled_test_threaded(self):
size = 100
threshold = .5
all_elems = 2000
@@ -76,6 +76,7 @@ class LRUTest(unittest.TestCase):
cache[id] = e
time.sleep(random.random() / 1000)
+
for x in range(0,20):
thread.start_new_thread(request_elem, ())
@@ -100,7 +101,9 @@ class LRUTest(unittest.TestCase):
total_avg = average_regets_in_range(0, 2000)
# hotzone should be way above the others
- print "total fetches", total[0], "hotzone", hotzone_avg, "control", control_avg, "total", total_avg
+ print "total fetches", total[0], "hotzone", \
+ hotzone_avg, "control", \
+ control_avg, "total", total_avg
assert hotzone_avg > total_avg * 5 > control_avg * 5
diff --git a/test/test_template.py b/test/test_template.py
index 8b6b7d2..f06ab5b 100644
--- a/test/test_template.py
+++ b/test/test_template.py
@@ -142,6 +142,29 @@ class EncodingTest(TemplateTest):
filters=lambda s:s.strip(),
)
+ def test_unicode_literal_in_tag(self):
+ self._do_file_test(
+ "unicode_arguments.html",
+ [
+ u'x is: drôle de petit voix m’a réveillé',
+ u'x is: drôle de petit voix m’a réveillé',
+ u'x is: drôle de petit voix m’a réveillé',
+ u'x is: drôle de petit voix m’a réveillé',
+ ],
+ filters=result_lines
+ )
+
+ self._do_memory_test(
+ file(self._file_path("unicode_arguments.html")).read(),
+ [
+ u'x is: drôle de petit voix m’a réveillé',
+ u'x is: drôle de petit voix m’a réveillé',
+ u'x is: drôle de petit voix m’a réveillé',
+ u'x is: drôle de petit voix m’a réveillé',
+ ],
+ filters=result_lines
+ )
+
def test_unicode_literal_in_def(self):
self._do_memory_test(
u"""## -*- coding: utf-8 -*-
--
GitLab