- added the 'encoding_errors' parameter to Template/TemplateLookup

  for specifying the error handler associated with encoding to
  'output_encoding' [ticket:40]
- the Template returned by html_error_template now defaults to
  output_encoding=sys.getdefaultencoding(),
  encoding_errors='htmlentityreplace' [ticket:37]

CHANGES

-0.1.6
+0.1.6 
 - fix to module_directory path generation when the path is "./"
   [ticket:34]
 - TGPlugin passes options to string-based templates [ticket:35]
@@ -10,6 +10,12 @@
   stack is properly handled for the def.
 - fix to RichTraceback and exception reporting to get template
   source code as a unicode object #37
+- added the 'encoding_errors' parameter to Template/TemplateLookup
+  for specifying the error handler associated with encoding to
+  'output_encoding' [ticket:40]
+- the Template returned by html_error_template now defaults to
+  output_encoding=sys.getdefaultencoding(),
+  encoding_errors='htmlentityreplace' [ticket:37]
     
 0.1.5
 - AST expression generation - added in just about everything 

doc/build/content/unicode.txt

@@ -71,13 +71,13 @@ The `default_filters` argument can be used to entirely customize the filtering p
 
 Now that we have a template which produces a pure unicode output stream, all the hard work is done.  We can take the output and do anything with it.
 
-As stated in the "Usage" chapter, both `Template` and `TemplateLookup` accept an `output_encoding` parameter which can be used to encode the output in any Python supported codec:
+As stated in the "Usage" chapter, both `Template` and `TemplateLookup` accept `output_encoding` and `encoding_errors` parameters which can be used to encode the output in any Python supported codec:
 
     {python}
     from mako.template import Template
     from mako.lookup import TemplateLookup
     
-    mylookup = TemplateLookup(directories=['/docs'], output_encoding='utf-8')
+    mylookup = TemplateLookup(directories=['/docs'], output_encoding='utf-8', encoding_errors='replace')
     
     mytemplate = mylookup.get_template("foo.txt")
     print mytemplate.render()
@@ -90,7 +90,7 @@ And `render_unicode()` will return the template output as a Python `unicode` obj
 The above method disgards the output encoding keyword argument; you can encode yourself by saying:
 
     {python}
-    print mytemplate.render_unicode().encode('utf-8')
+    print mytemplate.render_unicode().encode('utf-8', 'replace')
     
 #### Buffer Selection
 

doc/build/content/usage.txt

@@ -99,13 +99,13 @@ Another important flag on `TemplateLookup` is `filesystem_checks`.  This default
 
 ### Using Unicode and Encoding
 
-Both `Template` and `TemplateLookup` accept an `output_encoding` parameter which can be used to encode the output in any Python supported codec:
+Both `Template` and `TemplateLookup` accept `output_encoding` and `encoding_errors` parameters which can be used to encode the output in any Python supported codec:
 
     {python}
     from mako.template import Template
     from mako.lookup import TemplateLookup
     
-    mylookup = TemplateLookup(directories=['/docs'], output_encoding='utf-8')
+    mylookup = TemplateLookup(directories=['/docs'], output_encoding='utf-8', encoding_errors='replace')
     
     mytemplate = mylookup.get_template("foo.txt")
     print mytemplate.render()
@@ -118,7 +118,7 @@ Additionally, the `render_unicode()` method exists which will return the templat
 The above method disgards the output encoding keyword argument; you can encode yourself by saying:
 
     {python}
-    print mytemplate.render_unicode().encode('utf-8')
+    print mytemplate.render_unicode().encode('utf-8', 'replace')
     
 Note that Mako's ability to return data in any encoding and/or `unicode` implies that the underlying output stream of the template is a Python unicode object.  This behavior is described fully in [unicode](rel:unicode).
 

lib/mako/exceptions.py

@@ -223,4 +223,4 @@ def html_error_template():
 
 </body>
 </html>
-""")
+""", output_encoding=sys.getdefaultencoding(), encoding_errors='htmlentityreplace')

lib/mako/lookup.py

@@ -37,7 +37,7 @@ class TemplateCollection(object):
         return uri
         
 class TemplateLookup(TemplateCollection):
-    def __init__(self, directories=None, module_directory=None, filesystem_checks=True, collection_size=-1, format_exceptions=False, error_handler=None, output_encoding=None, cache_type=None, cache_dir=None, cache_url=None, modulename_callable=None, default_filters=['unicode'], buffer_filters=[], imports=None, input_encoding=None, preprocessor=None):
+    def __init__(self, directories=None, module_directory=None, filesystem_checks=True, collection_size=-1, format_exceptions=False, error_handler=None, output_encoding=None, encoding_errors='strict', cache_type=None, cache_dir=None, cache_url=None, modulename_callable=None, default_filters=['unicode'], buffer_filters=[], imports=None, input_encoding=None, preprocessor=None):
         if isinstance(directories, basestring):
             directories = [directories]        
         self.directories = [posixpath.normpath(d) for d in directories or []]
@@ -45,7 +45,7 @@ class TemplateLookup(TemplateCollection):
         self.modulename_callable = modulename_callable
         self.filesystem_checks = filesystem_checks
         self.collection_size = collection_size
-        self.template_args = {'format_exceptions':format_exceptions, 'error_handler':error_handler, 'output_encoding':output_encoding, 'input_encoding':input_encoding, 'module_directory':module_directory, 'cache_type':cache_type, 'cache_dir':cache_dir or module_directory, 'cache_url':cache_url, 'default_filters':default_filters, 'buffer_filters':buffer_filters,  'imports':imports, 'preprocessor':preprocessor}
+        self.template_args = {'format_exceptions':format_exceptions, 'error_handler':error_handler, 'output_encoding':output_encoding, 'encoding_errors':encoding_errors, 'input_encoding':input_encoding, 'module_directory':module_directory, 'cache_type':cache_type, 'cache_dir':cache_dir or module_directory, 'cache_url':cache_url, 'default_filters':default_filters, 'buffer_filters':buffer_filters,  'imports':imports, 'preprocessor':preprocessor}
         if collection_size == -1:
             self.__collection = {}
             self._uri_cache = {}
@@ -131,4 +131,4 @@ class TemplateLookup(TemplateCollection):
         self.__collection[uri] = Template(text, lookup=self, uri=uri, **self.template_args)
     def put_template(self, uri, template):
         self.__collection[uri] = template
-            
\ No newline at end of file
+            

lib/mako/runtime.py

@@ -279,7 +279,7 @@ def _render(template, callable_, args, data, as_unicode=False):
     if as_unicode:
         buf = util.FastEncodingBuffer()
     elif template.output_encoding:
-        buf = util.FastEncodingBuffer(template.output_encoding)
+        buf = util.FastEncodingBuffer(template.output_encoding, template.encoding_errors)
     else:
         buf = util.StringIO()
     context = Context(buf, **data)

lib/mako/template.py

@@ -16,7 +16,7 @@ import imp, time, weakref, tempfile, shutil,  os, stat, sys, re
     
 class Template(object):
     """a compiled template"""
-    def __init__(self, text=None, filename=None, uri=None, format_exceptions=False, error_handler=None, lookup=None, output_encoding=None, module_directory=None, cache_type=None, cache_dir=None, cache_url=None, module_filename=None, input_encoding=None, default_filters=['unicode'], buffer_filters=[], imports=None, preprocessor=None):
+    def __init__(self, text=None, filename=None, uri=None, format_exceptions=False, error_handler=None, lookup=None, output_encoding=None, encoding_errors='strict', module_directory=None, cache_type=None, cache_dir=None, cache_url=None, module_filename=None, input_encoding=None, default_filters=['unicode'], buffer_filters=[], imports=None, preprocessor=None):
         """construct a new Template instance using either literal template text, or a previously loaded template module
         
         text - textual template source, or None if a module is to be provided
@@ -91,6 +91,7 @@ class Template(object):
         self.error_handler = error_handler
         self.lookup = lookup
         self.output_encoding = output_encoding
+        self.encoding_errors = encoding_errors
         self.cache_type = cache_type
         self.cache_dir = cache_dir
         self.cache_url = cache_url
@@ -134,6 +135,7 @@ class DefTemplate(Template):
         self.input_encoding = parent.input_encoding
         self.imports = parent.imports
         self.output_encoding = parent.output_encoding
+        self.encoding_errors = parent.encoding_errors
         self.format_exceptions = parent.format_exceptions
         self.error_handler = parent.error_handler
         self.lookup = parent.lookup

lib/mako/util.py

@@ -47,14 +47,15 @@ class SetLikeDict(dict):
          
 class FastEncodingBuffer(object):
     """a very rudimentary buffer that is faster than StringIO, but doesnt crash on unicode data like cStringIO."""
-    def __init__(self, encoding=None):
+    def __init__(self, encoding=None, errors='strict'):
         self.data = []
         self.encoding = encoding
+        self.errors = errors
     def write(self, text):
         self.data.append(text)
     def getvalue(self):
         if self.encoding:
-            return u''.join(self.data).encode(self.encoding)
+            return u''.join(self.data).encode(self.encoding, self.errors)
         else:
             return u''.join(self.data)
 

test/template.py

@@ -129,6 +129,11 @@ class EncodingTest(unittest.TestCase):
         val = u"""Alors vous imaginez ma surprise, au lever du jour, quand une drôle de petit voix m’a réveillé. Elle disait: « S’il vous plaît… dessine-moi un mouton! »"""
         template = Template(val, output_encoding='utf-8')
         assert template.render() == val.encode('utf-8')
+
+    def test_encoding_errors(self):
+        val = u"""KGB (transliteration of "КГБ") is the Russian-language abbreviation for Committee for State Security, (Russian: Комит́ет Госуд́арственной Безоп́асности (help·info); Komitet Gosudarstvennoy Bezopasnosti)"""
+        template = Template(val, output_encoding='iso-8859-1', encoding_errors='replace')
+        assert template.render() == val.encode('iso-8859-1', 'replace')
     
     def test_read_unicode(self):
         lookup = TemplateLookup(directories=['./test_htdocs'], filesystem_checks=True, output_encoding='utf-8')