[PATCH 1/2] Add Template.__unicode__() to return unicode() objects, while Template.__str__() returns encoded str() objects

[tyler-53]

Per my discussion in #cheetah on IRC with mikeb@ regarding the following issue:
        https://bugzilla.redhat.com/show_bug.cgi?id=529332

This, in addition to recent patches to cheetah/DummyTransaction.py should alleviate
migration issues for users still passing a mishmash of unicode()/str() objects into
Templates.

Additionally, __str__() should return a str() object, whereas __unicode__() should
return a unicode() object.
---
 cheetah/Template.py      |   25 ++++++++++++++++++++++++-
 cheetah/Tests/Unicode.py |   16 ++++++++++++++++
 2 files changed, 40 insertions(+), 1 deletions(-)

diff --git a/cheetah/Template.py b/cheetah/Template.py
index a8889d2..ec92208 100644
--- a/cheetah/Template.py
+++ b/cheetah/Template.py
@@ -994,22 +994,45 @@ class Template(Servlet):
             mainMethName = getattr(concreteTemplateClass,mainMethNameAttr, None)
             if mainMethName:
                 def __str__(self):
+                    rc = getattr(self, mainMethName)()
+                    if isinstance(rc, unicode):
+                        return rc.encode('utf-8')
+                    return rc
+                def __unicode__(self):
                     return getattr(self, mainMethName)()
             elif (hasattr(concreteTemplateClass, 'respond')
                   and concreteTemplateClass.respond!=Servlet.respond):
                 def __str__(self):
+                    rc = self.respond()
+                    if isinstance(rc, unicode):
+                        return rc.encode('utf-8')
+                    return rc
+                def __unicode__(self):
                     return self.respond()
             else:
                 def __str__(self):
+                    rc = None
+                    if hasattr(self, mainMethNameAttr):
+                        rc = getattr(self,mainMethNameAttr)()
+                    elif hasattr(self, 'respond'):
+                        rc = self.respond()
+                    else:
+                        rc = super(self.__class__, self).__str__()
+                    if isinstance(rc, unicode):
+                        return rc.encode('utf-8')
+                    return rc
+                def __unicode__(self):
                     if hasattr(self, mainMethNameAttr):
                         return getattr(self,mainMethNameAttr)()
                     elif hasattr(self, 'respond'):
                         return self.respond()
                     else:
-                        return super(self.__class__, self).__str__()
+                        return super(self.__class__, self).__unicode__()
                     
             __str__ = new.instancemethod(__str__, None, concreteTemplateClass)
+            __unicode__ = new.instancemethod(__unicode__, None, concreteTemplateClass)
             setattr(concreteTemplateClass, '__str__', __str__)
+            setattr(concreteTemplateClass, '__unicode__', __unicode__)
                 
     _addCheetahPlumbingCodeToClass = classmethod(_addCheetahPlumbingCodeToClass)
 
diff --git a/cheetah/Tests/Unicode.py b/cheetah/Tests/Unicode.py
index d627503..12c00ac 100644
--- a/cheetah/Tests/Unicode.py
+++ b/cheetah/Tests/Unicode.py
@@ -150,6 +150,22 @@ $someUnicodeString"""
         a = unicode(template).encode("utf-8")
         self.assertEquals("Bébé", a)
 
+class EncodeUnicodeCompatTest(unittest.TestCase):
+    """
+        Taken initially from Red Hat's bugzilla #529332
+        https://bugzilla.redhat.com/show_bug.cgi?id=529332
+    """
+    def runTest(self):
+        t = Template("""Foo ${var}""", filter='EncodeUnicode')
+        t.var = u"Text with some non-ascii characters: åäö"
+        
+        rc = t.respond()
+        assert isinstance(rc, unicode), ('Template.respond() should return unicode', rc)
+        
+        rc = str(t)
+        assert isinstance(rc, str), ('Template.__str__() should return a UTF-8 encoded string', rc)
+
+
 class Unicode_in_SearchList_Test(CommandLineTest):
     def test_BasicASCII(self):
         source = '''This is $adjective'''
--
1.6.5



------------------------------------------------------------------------------
Come build with us! The BlackBerry(R) Developer Conference in SF, CA
is the only developer event you need to attend this year. Jumpstart your
developing skills, take BlackBerry mobile applications to market and stay
ahead of the curve. Join us from November 9 - 12, 2009. Register now!
http://p.sf.net/sfu/devconference
_______________________________________________
Cheetahtemplate-discuss mailing list
Cheetahtemplate-discuss@...
https://lists.sourceforge.net/lists/listinfo/cheetahtemplate-discuss

[tyler-53]

Since v2.2.0, the handling of unicode objects has been
gracefully handled by Cheetah, such that the encoding operation
here is unnecessary (and likely an additional performance hindrance)
---
 cheetah/Filters.py |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/cheetah/Filters.py b/cheetah/Filters.py
index dd65f28..452afc5 100644
--- a/cheetah/Filters.py
+++ b/cheetah/Filters.py
@@ -58,7 +58,7 @@ class EncodeUnicode(Filter):
         >>> print t
         """
         if isinstance(val, unicode):
-            return val.encode(encoding)
+            return val
         if val is None:
             return ''
         return str(val)
--
1.6.5


------------------------------------------------------------------------------
Come build with us! The BlackBerry(R) Developer Conference in SF, CA
is the only developer event you need to attend this year. Jumpstart your
developing skills, take BlackBerry mobile applications to market and stay
ahead of the curve. Join us from November 9 - 12, 2009. Register now!
http://p.sf.net/sfu/devconference
_______________________________________________
Cheetahtemplate-discuss mailing list
Cheetahtemplate-discuss@...
https://lists.sourceforge.net/lists/listinfo/cheetahtemplate-discuss

[Mike Bonnet]

On 10/16/2009 06:27 PM, R. Tyler Ballance wrote:
> Per my discussion in #cheetah on IRC with mikeb@ regarding the following issue:
> https://bugzilla.redhat.com/show_bug.cgi?id=529332
>
> This, in addition to recent patches to cheetah/DummyTransaction.py should alleviate
> migration issues for users still passing a mishmash of unicode()/str() objects into
> Templates.
>
> Additionally, __str__() should return a str() object, whereas __unicode__() should
> return a unicode() object.

This fixes the bug, thanks!  I'll see about releasing an updated Cheetah
2.2.2 package with this patch applied.

> ---
>   cheetah/Template.py      |   25 ++++++++++++++++++++++++-
>   cheetah/Tests/Unicode.py |   16 ++++++++++++++++
>   2 files changed, 40 insertions(+), 1 deletions(-)
>
> diff --git a/cheetah/Template.py b/cheetah/Template.py
> index a8889d2..ec92208 100644
> --- a/cheetah/Template.py
> +++ b/cheetah/Template.py
> @@ -994,22 +994,45 @@ class Template(Servlet):
>               mainMethName = getattr(concreteTemplateClass,mainMethNameAttr, None)
>               if mainMethName:
>                   def __str__(self):
> +                    rc = getattr(self, mainMethName)()
> +                    if isinstance(rc, unicode):
> +                        return rc.encode('utf-8')
> +                    return rc
> +                def __unicode__(self):
>                       return getattr(self, mainMethName)()
>               elif (hasattr(concreteTemplateClass, 'respond')
>                     and concreteTemplateClass.respond!=Servlet.respond):
>                   def __str__(self):
> +                    rc = self.respond()
> +                    if isinstance(rc, unicode):
> +                        return rc.encode('utf-8')
> +                    return rc
> +                def __unicode__(self):
>                       return self.respond()
>               else:
>                   def __str__(self):
> +                    rc = None
> +                    if hasattr(self, mainMethNameAttr):
> +                        rc = getattr(self,mainMethNameAttr)()
> +                    elif hasattr(self, 'respond'):
> +                        rc = self.respond()
> +                    else:
> +                        rc = super(self.__class__, self).__str__()
> +                    if isinstance(rc, unicode):
> +                        return rc.encode('utf-8')
> +                    return rc
> +                def __unicode__(self):
>                       if hasattr(self, mainMethNameAttr):
>                           return getattr(self,mainMethNameAttr)()
>                       elif hasattr(self, 'respond'):
>                           return self.respond()
>                       else:
> -                        return super(self.__class__, self).__str__()
> +                        return super(self.__class__, self).__unicode__()
>
>               __str__ = new.instancemethod(__str__, None, concreteTemplateClass)
> +            __unicode__ = new.instancemethod(__unicode__, None, concreteTemplateClass)
>               setattr(concreteTemplateClass, '__str__', __str__)
> +            setattr(concreteTemplateClass, '__unicode__', __unicode__)
>
>       _addCheetahPlumbingCodeToClass = classmethod(_addCheetahPlumbingCodeToClass)
>
> diff --git a/cheetah/Tests/Unicode.py b/cheetah/Tests/Unicode.py
> index d627503..12c00ac 100644
> --- a/cheetah/Tests/Unicode.py
> +++ b/cheetah/Tests/Unicode.py
> @@ -150,6 +150,22 @@ $someUnicodeString"""
>           a = unicode(template).encode("utf-8")
>           self.assertEquals("Bébé", a)
>
> +class EncodeUnicodeCompatTest(unittest.TestCase):
> +    """
> +        Taken initially from Red Hat's bugzilla #529332
> +        https://bugzilla.redhat.com/show_bug.cgi?id=529332
> +    """
> +    def runTest(self):
> +        t = Template("""Foo ${var}""", filter='EncodeUnicode')
> +        t.var = u"Text with some non-ascii characters: åäö"
> +
> +        rc = t.respond()
> +        assert isinstance(rc, unicode), ('Template.respond() should return unicode', rc)
> +
> +        rc = str(t)
> +        assert isinstance(rc, str), ('Template.__str__() should return a UTF-8 encoded string', rc)
> +
> +
>   class Unicode_in_SearchList_Test(CommandLineTest):
>       def test_BasicASCII(self):
>           source = '''This is $adjective'''
>
>
>
> ------------------------------------------------------------------------------
> Come build with us! The BlackBerry(R) Developer Conference in SF, CA
> is the only developer event you need to attend this year. Jumpstart your
> developing skills, take BlackBerry mobile applications to market and stay
> ahead of the curve. Join us from November 9 - 12, 2009. Register now!
> http://p.sf.net/sfu/devconference
>
>
>
> _______________________________________________
> Cheetahtemplate-discuss mailing list
> Cheetahtemplate-discuss@...
> https://lists.sourceforge.net/lists/listinfo/cheetahtemplate-discuss

------------------------------------------------------------------------------
Come build with us! The BlackBerry(R) Developer Conference in SF, CA
is the only developer event you need to attend this year. Jumpstart your
developing skills, take BlackBerry mobile applications to market and stay
ahead of the curve. Join us from November 9 - 12, 2009. Register now!
http://p.sf.net/sfu/devconference
_______________________________________________
Cheetahtemplate-discuss mailing list
Cheetahtemplate-discuss@...
https://lists.sourceforge.net/lists/listinfo/cheetahtemplate-discuss

[Mike Bonnet]

On 10/16/2009 06:27 PM, R. Tyler Ballance wrote:

> Since v2.2.0, the handling of unicode objects has been
> gracefully handled by Cheetah, such that the encoding operation
> here is unnecessary (and likely an additional performance hindrance)
> ---
>   cheetah/Filters.py |    2 +-
>   1 files changed, 1 insertions(+), 1 deletions(-)
>
> diff --git a/cheetah/Filters.py b/cheetah/Filters.py
> index dd65f28..452afc5 100644
> --- a/cheetah/Filters.py
> +++ b/cheetah/Filters.py
> @@ -58,7 +58,7 @@ class EncodeUnicode(Filter):
>           >>>  print t
>           """
>           if isinstance(val, unicode):
> -            return val.encode(encoding)
> +            return val
>           if val is None:
>               return ''
>           return str(val)

I think this is definitely the right thing to do now that all strings
are unicode objects internally.

This is probably a good time to take a look at the filters in light of
the recent changes in encoding handling.

The base Filter object allows you to specify an encoding that it'll use
to encode unicode objects to strs.  Given that every substitution
variable needs to be converted to unicode before it can be concatenated
with the rest of the template (which is a unicode object), encoding them
as strs simply causes them to need to be decoded again in
DummyTransaction, an unnecessary round-trip.

The base Filter also converts non-basestring objects to strs by default,
when it would be more efficient to convert them directly to unicode
objects, and fall back to strs in the failure case.

Both of these changes can be made without changing the API, I'll send a
patch shortly.  However, the API should probably be changed to reflect
that it is no longer useful to encode substitution variables to another
encoding, everything is unicode.  Is this an API change worth making for
2.4.0?

------------------------------------------------------------------------------
Come build with us! The BlackBerry(R) Developer Conference in SF, CA
is the only developer event you need to attend this year. Jumpstart your
developing skills, take BlackBerry mobile applications to market and stay
ahead of the curve. Join us from November 9 - 12, 2009. Register now!
http://p.sf.net/sfu/devconference
_______________________________________________
Cheetahtemplate-discuss mailing list
Cheetahtemplate-discuss@...
https://lists.sourceforge.net/lists/listinfo/cheetahtemplate-discuss