Changeset 36

Timestamp:

01/29/06 17:34:38 (7 years ago)

Author:

xi

Message:

Implement new YAML<->Python string conversion (fix #42).

Location:

trunk

Files:

• lib/syck/dumpers.py (modified) (2 diffs)
• lib/syck/loaders.py (modified) (2 diffs)
• setup.cfg (modified) (1 diff)
• tests/test_pickle.py (modified) (2 diffs)
• tests/test_syck.py (modified) (1 diff)
• tests/test_unicode.py (added)

trunk/lib/syck/dumpers.py

@@ -109 +109 @@
 
     def represent_str(self, object):
-        return _syck.Scalar(str(object), tag="tag:yaml.org,2002:str")
+        try:
+            return _syck.Scalar(object.encode('ascii'), tag="tag:yaml.org,2002:str")
+        except UnicodeDecodeError:
+            try:
+                return _syck.Scalar(unicode(object, 'utf-8').encode('utf-8'),
+                        tag="tag:python.yaml.org,2002:str")
+            except UnicodeDecodeError:
+                return _syck.Scalar(object.encode('base64'),
+                        tag="tag:yaml.org,2002:binary")
+
+    def represent_unicode(self, object):
+        try:
+            return _syck.Scalar(object.encode('ascii'), tag="tag:python.yaml.org,2002:unicode")
+        except UnicodeEncodeError:
+            return _syck.Scalar(object.encode('utf-8'), tag="tag:yaml.org,2002:str")
 
     def represent_list(self, object):
@@ -139 +153 @@
     def represent_long(self, object):
         return _syck.Scalar(repr(object), tag="tag:python.yaml.org,2002:long")
-
-    def represent_unicode(self, object):
-        return _syck.Scalar(object.encode('utf-8'), tag="tag:python.yaml.org,2002:unicode")
 
     def represent_tuple(self, object):

trunk/lib/syck/loaders.py

@@ -25 +25 @@
 import _syck
 
-import sys, re
+import sys, re, warnings
 
 __all__ = ['GenericLoader', 'Loader',
-    'parse', 'load', 'parse_documents', 'load_documents']
+    'parse', 'load', 'parse_documents', 'load_documents',
+    'NotUnicodeInputWarning']
+
+class NotUnicodeInputWarning(UserWarning):
+    pass
 
 class GenericLoader(_syck.Parser):
@@ -180 +184 @@
         return False
 
+    def construct_str(self, node):
+        try:
+            value = unicode(node.value, 'utf-8')
+        except UnicodeDecodeError:
+            warnings.warn("scalar value is not utf-8", NotUnicodeInputWarning)
+            return node.value
+        try:
+            return value.encode('ascii')
+        except UnicodeEncodeError:
+            return value
+
     def construct_numeric_base60(self, num_type, node):
         digits = [num_type(part) for part in node.value.split(':')]

trunk/setup.cfg

@@ -5 +5 @@
 
 # List of directories to search for 'syck.h' (separated by ':').
-include_dirs=/usr/local/include:../../include
+#include_dirs=/usr/local/include:../../include
 
 # List of directories to search for 'libsyck.a' (separated by ':').
-library_dirs=/usr/local/lib:../../lib
+#library_dirs=/usr/local/lib:../../lib
 

trunk/tests/test_pickle.py

@@ -348 +348 @@
         nodes = syck.parse(NODES)
         output = syck.dump(nodes)
-        print output
+        #print output
         nodes2 = syck.load(output)
         output2 = syck.emit(nodes2)
@@ -358 +358 @@
         nodes = syck.parse(BUGGY_NODES)
         output = syck.dump(nodes)
-        print output
+        #print output
         nodes2 = syck.load(output)
         output2 = syck.emit(nodes2)

trunk/tests/test_syck.py

@@ -9 +9 @@
 from test_pickle import *
 from test_threads import *
+from test_unicode import *
 
 def main(module='__main__'):