Index: /branches/pyyaml3000/tests/test_appliance.py
===================================================================
--- /branches/pyyaml3000/tests/test_appliance.py	(revision 43)
+++ /branches/pyyaml3000/tests/test_appliance.py	(revision 44)
@@ -33,5 +33,5 @@
             if hasattr(self, attribute):
                 args.append(repr(getattr(self, attribute)))
-        return "%s(%s)" % (self.__class__.__name__, ''.join(args))
+        return "%s(%s)" % (self.__class__.__name__, ', '.join(args))
 
 class AliasNode(Node):
Index: /branches/pyyaml3000/tests/test_structure.py
===================================================================
--- /branches/pyyaml3000/tests/test_structure.py	(revision 44)
+++ /branches/pyyaml3000/tests/test_structure.py	(revision 44)
@@ -0,0 +1,81 @@
+
+import test_appliance
+
+from yaml.parser import *
+
+class TestStructure(test_appliance.TestAppliance):
+
+    def _testStructure(self, test_name, data_filename, structure_filename):
+        node1 = None
+        node2 = eval(file(structure_filename, 'rb').read())
+        try:
+            parser = Parser(data_filename, file(data_filename, 'rb').read())
+            node1 = parser.parse()
+            node1 = [self._convert(n) for n in node1]
+            if len(node1) == 1:
+                node1 = node1[0]
+            self.failUnlessEqual(node1, node2)
+        except:
+            print
+            print "DATA:"
+            print file(data_filename, 'rb').read()
+            print "NODE1:", node1
+            print "NODE2:", node2
+            raise
+
+    def _convert(self, node):
+        if isinstance(node, ScalarNode):
+            return True
+        elif isinstance(node, SequenceNode):
+            sequence = []
+            for item in node.value:
+                sequence.append(self._convert(item))
+            return sequence
+        elif isinstance(node, MappingNode):
+            mapping = []
+            for key, value in node.value:
+                mapping.append((self._convert(key), self._convert(value)))
+            return mapping
+        elif isinstance(node, AliasNode):
+            return '*'
+        else:
+            return node
+
+TestStructure.add_tests('testStructure', '.data', '.structure')
+
+class TestParser(test_appliance.TestAppliance):
+
+    def _testParser(self, test_name, data_filename, canonical_filename):
+        documents1 = None
+        documents2 = None
+        try:
+            parser = Parser(data_filename, file(data_filename, 'rb').read())
+            documents1 = parser.parse()
+            canonical = test_appliance.CanonicalParser(canonical_filename, file(canonical_filename, 'rb').read())
+            documents2 = canonical.parse()
+            self._compare(documents1, documents2)
+        except:
+            print
+            print "DATA1:"
+            print file(data_filename, 'rb').read()
+            print "DATA2:"
+            print file(canonical_filename, 'rb').read()
+            print "DOCUMENTS1:", documents1
+            print "DOCUMENTS2:", documents2
+            raise
+
+    def _compare(self, value1, value2):
+        if value1 is None and hasattr(value2, 'tag') and value2.tag == 'tag:yaml.org,2002:null':
+            return
+        self.failUnlessEqual(type(value1), type(value2))
+        if isinstance(value1, list) or isinstance(value1, tuple):
+            self.failUnlessEqual(len(value1), len(value2))
+            for item1, item2 in zip(value1, value2):
+                self._compare(item1, item2)
+        else:
+            self.failUnlessEqual(value1.__class__.__name__, value2.__class__.__name__)
+            if isinstance(value1, SequenceNode) or isinstance(value1, MappingNode):
+                self._compare(value1.value, value2.value)
+
+TestParser.add_tests('testParser', '.data', '.canonical')
+
Index: /branches/pyyaml3000/tests/test_yaml.py
===================================================================
--- /branches/pyyaml3000/tests/test_yaml.py	(revision 43)
+++ /branches/pyyaml3000/tests/test_yaml.py	(revision 44)
@@ -5,4 +5,5 @@
 from test_canonical import *
 from test_tokens import *
+from test_structure import *
 
 def main(module='__main__'):
Index: /branches/pyyaml3000/tests/data/spec-02-01.structure
===================================================================
--- /branches/pyyaml3000/tests/data/spec-02-01.structure	(revision 44)
+++ /branches/pyyaml3000/tests/data/spec-02-01.structure	(revision 44)
@@ -0,0 +1,1 @@
+[True, True, True]
Index: /branches/pyyaml3000/tests/data/spec-02-22.structure
===================================================================
--- /branches/pyyaml3000/tests/data/spec-02-22.structure	(revision 44)
+++ /branches/pyyaml3000/tests/data/spec-02-22.structure	(revision 44)
@@ -0,0 +1,1 @@
+[(True, True), (True, True), (True, True), (True, True)]
Index: /branches/pyyaml3000/tests/data/spec-02-09.structure
===================================================================
--- /branches/pyyaml3000/tests/data/spec-02-09.structure	(revision 44)
+++ /branches/pyyaml3000/tests/data/spec-02-09.structure	(revision 44)
@@ -0,0 +1,1 @@
+[(True, [True, True]), (True, [True, True])]
Index: /branches/pyyaml3000/tests/data/spec-02-10.structure
===================================================================
--- /branches/pyyaml3000/tests/data/spec-02-10.structure	(revision 44)
+++ /branches/pyyaml3000/tests/data/spec-02-10.structure	(revision 44)
@@ -0,0 +1,1 @@
+[(True, [True, True]), (True, ['*', True])]
Index: /branches/pyyaml3000/tests/data/spec-02-18.structure
===================================================================
--- /branches/pyyaml3000/tests/data/spec-02-18.structure	(revision 44)
+++ /branches/pyyaml3000/tests/data/spec-02-18.structure	(revision 44)
@@ -0,0 +1,1 @@
+[(True, True), (True, True)]
Index: /branches/pyyaml3000/tests/data/spec-09-12.data
===================================================================
--- /branches/pyyaml3000/tests/data/spec-09-12.data	(revision 38)
+++ /branches/pyyaml3000/tests/data/spec-09-12.data	(revision 44)
@@ -4,5 +4,5 @@
 - -123
 # Inside flow collection:
-- [ ::std::vector,
+- [ '::std::vector',
   "Up, up and away!",
   -123 ]
Index: /branches/pyyaml3000/tests/data/spec-09-25.canonical
===================================================================
--- /branches/pyyaml3000/tests/data/spec-09-25.canonical	(revision 38)
+++ /branches/pyyaml3000/tests/data/spec-09-25.canonical	(revision 44)
@@ -1,6 +1,4 @@
 %YAML 1.1
 ---
-!!seq [
-  !!str "literal\n\
-        \ttext\n"
-]
+!!str "literal\n\
+      \ttext\n"
Index: /branches/pyyaml3000/tests/data/spec-02-06.structure
===================================================================
--- /branches/pyyaml3000/tests/data/spec-02-06.structure	(revision 44)
+++ /branches/pyyaml3000/tests/data/spec-02-06.structure	(revision 44)
@@ -0,0 +1,4 @@
+[
+    (True, [(True, True), (True, True)]),
+    (True, [(True, True), (True, True)]),
+]
Index: /branches/pyyaml3000/tests/data/spec-02-27.structure
===================================================================
--- /branches/pyyaml3000/tests/data/spec-02-27.structure	(revision 44)
+++ /branches/pyyaml3000/tests/data/spec-02-27.structure	(revision 44)
@@ -0,0 +1,17 @@
+[
+(True, True),
+(True, True),
+(True, [
+    (True, True),
+    (True, True),
+    (True, [(True, True), (True, True), (True, True), (True, True)]),
+    ]),
+(True, '*'),
+(True, [
+        [(True, True), (True, True), (True, True), (True, True)],
+        [(True, True), (True, True), (True, True), (True, True)],
+    ]),
+(True, True),
+(True, True),
+(True, True),
+]
Index: /branches/pyyaml3000/tests/data/spec-02-15.structure
===================================================================
--- /branches/pyyaml3000/tests/data/spec-02-15.structure	(revision 44)
+++ /branches/pyyaml3000/tests/data/spec-02-15.structure	(revision 44)
@@ -0,0 +1,1 @@
+True
Index: /branches/pyyaml3000/tests/data/spec-05-03.canonical
===================================================================
--- /branches/pyyaml3000/tests/data/spec-05-03.canonical	(revision 38)
+++ /branches/pyyaml3000/tests/data/spec-05-03.canonical	(revision 44)
@@ -9,5 +9,6 @@
   : !!map {
     ? !!str "sky" : !!str "blue",
-    ? !!str "sea" : !!str "green",
+#    ? !!str "sea" : !!str "green",
+    ? !!map { ? !!str "sea" : !!str "green" } : !!null "",
   }
 }
Index: /branches/pyyaml3000/tests/data/spec-02-03.structure
===================================================================
--- /branches/pyyaml3000/tests/data/spec-02-03.structure	(revision 44)
+++ /branches/pyyaml3000/tests/data/spec-02-03.structure	(revision 44)
@@ -0,0 +1,1 @@
+[(True, [True, True, True]), (True, [True, True, True])]
Index: /branches/pyyaml3000/tests/data/spec-02-24.structure
===================================================================
--- /branches/pyyaml3000/tests/data/spec-02-24.structure	(revision 44)
+++ /branches/pyyaml3000/tests/data/spec-02-24.structure	(revision 44)
@@ -0,0 +1,5 @@
+[
+[(True, [(True, True), (True, True)]), (True, True)],
+[(True, '*'), (True, [(True, True), (True, True)])],
+[(True, '*'), (True, True), (True, True)],
+]
Index: /branches/pyyaml3000/tests/data/spec-10-05.canonical
===================================================================
--- /branches/pyyaml3000/tests/data/spec-10-05.canonical	(revision 42)
+++ /branches/pyyaml3000/tests/data/spec-10-05.canonical	(revision 44)
@@ -2,5 +2,5 @@
 ---
 !!seq [
-  !!str "",
+  !!null "",
   !!str "block node\n",
   !!seq [
Index: /branches/pyyaml3000/tests/data/spec-09-31.canonical
===================================================================
--- /branches/pyyaml3000/tests/data/spec-09-31.canonical	(revision 38)
+++ /branches/pyyaml3000/tests/data/spec-09-31.canonical	(revision 44)
@@ -1,9 +1,7 @@
 %YAML 1.1
 ---
-!!seq [
-  !!str "folded line\n\
-        next line\n\
-        \  * bullet\n\
-        \  * list\n\
-        last line\n"
-]
+!!str "folded line\n\
+      next line\n\
+      \  * bullet\n\
+      \  * list\n\
+      last line\n"
Index: /branches/pyyaml3000/tests/data/spec-02-12.structure
===================================================================
--- /branches/pyyaml3000/tests/data/spec-02-12.structure	(revision 44)
+++ /branches/pyyaml3000/tests/data/spec-02-12.structure	(revision 44)
@@ -0,0 +1,5 @@
+[
+[(True, True), (True, True)],
+[(True, True), (True, True)],
+[(True, True), (True, True)],
+]
Index: /branches/pyyaml3000/tests/data/spec-10-07.data
===================================================================
--- /branches/pyyaml3000/tests/data/spec-10-07.data	(revision 38)
+++ /branches/pyyaml3000/tests/data/spec-10-07.data	(revision 44)
@@ -1,7 +1,8 @@
 {
-? : value # Empty key
+#? : value # Empty key
+? ~ : value, # Empty key
 ? explicit
  key: value,
-simple key : value
+simple key : value,
 [ collection, simple, key ]: value
 }
Index: /branches/pyyaml3000/tests/data/spec-10-14.canonical
===================================================================
--- /branches/pyyaml3000/tests/data/spec-10-14.canonical	(revision 38)
+++ /branches/pyyaml3000/tests/data/spec-10-14.canonical	(revision 44)
@@ -3,5 +3,5 @@
 !!map {
   ? !!str "plain key"
-  : !!str "",
+  : !!null "",
   ? !!str "quoted key\n"
   : !!seq [
Index: /branches/pyyaml3000/tests/data/spec-08-13.canonical
===================================================================
--- /branches/pyyaml3000/tests/data/spec-08-13.canonical	(revision 42)
+++ /branches/pyyaml3000/tests/data/spec-08-13.canonical	(revision 44)
@@ -3,6 +3,8 @@
 !!map {
   ? !!str "foo"
-  : !!str "",
-  ? !!str ""
+#  : !!str "",
+#  ? !!str ""
+  : !!null "",
+  ? !!null ""
   : !!str "bar",
 }
Index: /branches/pyyaml3000/tests/data/spec-02-21.structure
===================================================================
--- /branches/pyyaml3000/tests/data/spec-02-21.structure	(revision 44)
+++ /branches/pyyaml3000/tests/data/spec-02-21.structure	(revision 44)
@@ -0,0 +1,1 @@
+[(True, True), (True, True), (True, True), (True, True)]
Index: /branches/pyyaml3000/tests/data/spec-02-08.structure
===================================================================
--- /branches/pyyaml3000/tests/data/spec-02-08.structure	(revision 44)
+++ /branches/pyyaml3000/tests/data/spec-02-08.structure	(revision 44)
@@ -0,0 +1,4 @@
+[
+[(True, True), (True, True), (True, True)],
+[(True, True), (True, True), (True, True)],
+]
Index: /branches/pyyaml3000/tests/data/spec-02-17.structure
===================================================================
--- /branches/pyyaml3000/tests/data/spec-02-17.structure	(revision 44)
+++ /branches/pyyaml3000/tests/data/spec-02-17.structure	(revision 44)
@@ -0,0 +1,1 @@
+[(True, True), (True, True), (True, True), (True, True), (True, True), (True, True)]
Index: /branches/pyyaml3000/tests/data/spec-10-11.canonical
===================================================================
--- /branches/pyyaml3000/tests/data/spec-10-11.canonical	(revision 38)
+++ /branches/pyyaml3000/tests/data/spec-10-11.canonical	(revision 44)
@@ -8,9 +8,9 @@
   !!map {
     ? !!str "explicit key2"
-    : !!str "",
+    : !!null "",
   },
   !!map {
     ? !!str "explicit key3"
-    : !!str "",
+    : !!null "",
   },
   !!map {
@@ -20,5 +20,5 @@
   !!map {
     ? !!str "simple key2"
-    : !!str "",
+    : !!null "",
   },
 ]
Index: /branches/pyyaml3000/tests/data/spec-02-05.structure
===================================================================
--- /branches/pyyaml3000/tests/data/spec-02-05.structure	(revision 44)
+++ /branches/pyyaml3000/tests/data/spec-02-05.structure	(revision 44)
@@ -0,0 +1,5 @@
+[
+    [True, True, True],
+    [True, True, True],
+    [True, True, True],
+]
Index: /branches/pyyaml3000/tests/data/spec-02-26.structure
===================================================================
--- /branches/pyyaml3000/tests/data/spec-02-26.structure	(revision 44)
+++ /branches/pyyaml3000/tests/data/spec-02-26.structure	(revision 44)
@@ -0,0 +1,5 @@
+[
+[(True, True)],
+[(True, True)],
+[(True, True)],
+]
Index: /branches/pyyaml3000/tests/data/spec-08-15.data
===================================================================
--- /branches/pyyaml3000/tests/data/spec-08-15.data	(revision 38)
+++ /branches/pyyaml3000/tests/data/spec-08-15.data	(revision 44)
@@ -1,6 +1,5 @@
-seq:
 - # Empty plain scalar
 - ? foo
   :
   ?
-  : bar,
+  : bar
Index: /branches/pyyaml3000/tests/data/spec-10-07.canonical
===================================================================
--- /branches/pyyaml3000/tests/data/spec-10-07.canonical	(revision 38)
+++ /branches/pyyaml3000/tests/data/spec-10-07.canonical	(revision 44)
@@ -2,5 +2,5 @@
 ---
 !!map {
-  ? !!str ""
+  ? !!null ""
   : !!str "value",
   ? !!str "explicit key"
Index: /branches/pyyaml3000/tests/data/spec-09-33.canonical
===================================================================
--- /branches/pyyaml3000/tests/data/spec-09-33.canonical	(revision 38)
+++ /branches/pyyaml3000/tests/data/spec-09-33.canonical	(revision 44)
@@ -1,9 +1,7 @@
 %YAML 1.1
 ---
-!!seq [
-  !!str "folded line\n\
-        next line\n\
-        \  * bullet\n\
-        \  * list\n\
-        last line\n"
-]
+!!str "folded line\n\
+      next line\n\
+      \  * bullet\n\
+      \  * list\n\
+      last line\n"
Index: /branches/pyyaml3000/tests/data/spec-02-14.structure
===================================================================
--- /branches/pyyaml3000/tests/data/spec-02-14.structure	(revision 44)
+++ /branches/pyyaml3000/tests/data/spec-02-14.structure	(revision 44)
@@ -0,0 +1,1 @@
+True
Index: /branches/pyyaml3000/tests/data/spec-08-12.data
===================================================================
--- /branches/pyyaml3000/tests/data/spec-08-12.data	(revision 38)
+++ /branches/pyyaml3000/tests/data/spec-08-12.data	(revision 44)
@@ -4,4 +4,5 @@
   !!str 'Tagged',
   *anchor, # Alias node
-  !!str,   # Empty plain scalar
+#  !!str,   # Empty plain scalar
+  '',   # Empty plain scalar
 ]
Index: /branches/pyyaml3000/tests/data/spec-02-02.structure
===================================================================
--- /branches/pyyaml3000/tests/data/spec-02-02.structure	(revision 44)
+++ /branches/pyyaml3000/tests/data/spec-02-02.structure	(revision 44)
@@ -0,0 +1,1 @@
+[(True, True), (True, True), (True, True)]
Index: /branches/pyyaml3000/tests/data/spec-08-15.canonical
===================================================================
--- /branches/pyyaml3000/tests/data/spec-08-15.canonical	(revision 42)
+++ /branches/pyyaml3000/tests/data/spec-08-15.canonical	(revision 44)
@@ -2,9 +2,9 @@
 ---
 !!seq [
-  !!str "",
+  !!null "",
   !!map {
     ? !!str "foo"
-    : !!str "",
-    ? !!str ""
+    : !!null "",
+    ? !!null ""
     : !!str "bar",
   }
Index: /branches/pyyaml3000/tests/data/spec-02-23.structure
===================================================================
--- /branches/pyyaml3000/tests/data/spec-02-23.structure	(revision 44)
+++ /branches/pyyaml3000/tests/data/spec-02-23.structure	(revision 44)
@@ -0,0 +1,1 @@
+[(True, True), (True, True), (True, True)]
Index: /branches/pyyaml3000/tests/data/spec-09-29.canonical
===================================================================
--- /branches/pyyaml3000/tests/data/spec-09-29.canonical	(revision 38)
+++ /branches/pyyaml3000/tests/data/spec-09-29.canonical	(revision 44)
@@ -1,6 +1,4 @@
 %YAML 1.1
 ---
-!!seq [
-  !!str "folded text\n\
-        \tlines\n"
-]
+!!str "folded text\n\
+      \tlines\n"
Index: /branches/pyyaml3000/tests/data/spec-07-10.canonical
===================================================================
--- /branches/pyyaml3000/tests/data/spec-07-10.canonical	(revision 38)
+++ /branches/pyyaml3000/tests/data/spec-07-10.canonical	(revision 44)
@@ -12,3 +12,4 @@
 }
 ---
-!!str ""
+#!!str ""
+!!null ""
Index: /branches/pyyaml3000/tests/data/spec-09-30.canonical
===================================================================
--- /branches/pyyaml3000/tests/data/spec-09-30.canonical	(revision 38)
+++ /branches/pyyaml3000/tests/data/spec-09-30.canonical	(revision 44)
@@ -1,9 +1,7 @@
 %YAML 1.1
 ---
-!!seq [
-  !!str "folded line\n\
-        next line\n\
-        \  * bullet\n\
-        \  * list\n\
-        last line\n"
-]
+!!str "folded line\n\
+      next line\n\
+      \  * bullet\n\
+      \  * list\n\
+      last line\n"
Index: /branches/pyyaml3000/tests/data/spec-02-11.structure
===================================================================
--- /branches/pyyaml3000/tests/data/spec-02-11.structure	(revision 44)
+++ /branches/pyyaml3000/tests/data/spec-02-11.structure	(revision 44)
@@ -0,0 +1,4 @@
+[
+([True, True], [True]),
+([True, True], [True, True, True]),
+]
Index: /branches/pyyaml3000/tests/data/spec-02-19.structure
===================================================================
--- /branches/pyyaml3000/tests/data/spec-02-19.structure	(revision 44)
+++ /branches/pyyaml3000/tests/data/spec-02-19.structure	(revision 44)
@@ -0,0 +1,1 @@
+[(True, True), (True, True), (True, True), (True, True), (True, True)]
Index: /branches/pyyaml3000/tests/data/spec-10-13.canonical
===================================================================
--- /branches/pyyaml3000/tests/data/spec-10-13.canonical	(revision 38)
+++ /branches/pyyaml3000/tests/data/spec-10-13.canonical	(revision 44)
@@ -3,5 +3,5 @@
 !!map {
   ? !!str "explicit key"
-  : !!str "",
+  : !!null "",
   ? !!str "block key\n"
   : !!seq [
Index: /branches/pyyaml3000/tests/data/spec-02-20.structure
===================================================================
--- /branches/pyyaml3000/tests/data/spec-02-20.structure	(revision 44)
+++ /branches/pyyaml3000/tests/data/spec-02-20.structure	(revision 44)
@@ -0,0 +1,1 @@
+[(True, True), (True, True), (True, True), (True, True), (True, True), (True, True)]
Index: /branches/pyyaml3000/tests/data/spec-02-07.structure
===================================================================
--- /branches/pyyaml3000/tests/data/spec-02-07.structure	(revision 44)
+++ /branches/pyyaml3000/tests/data/spec-02-07.structure	(revision 44)
@@ -0,0 +1,4 @@
+[
+[True, True, True],
+[True, True],
+]
Index: /branches/pyyaml3000/tests/data/spec-02-28.structure
===================================================================
--- /branches/pyyaml3000/tests/data/spec-02-28.structure	(revision 44)
+++ /branches/pyyaml3000/tests/data/spec-02-28.structure	(revision 44)
@@ -0,0 +1,10 @@
+[
+[(True, True), (True, True), (True, True)],
+[(True, True), (True, True), (True, True)],
+[(True, True), (True, True), (True, True),
+(True, [
+    [(True, True), (True, True), (True, True)],
+    [(True, True), (True, True), (True, True)],
+    ]),
+]
+]
Index: /branches/pyyaml3000/tests/data/spec-10-09.canonical
===================================================================
--- /branches/pyyaml3000/tests/data/spec-10-09.canonical	(revision 38)
+++ /branches/pyyaml3000/tests/data/spec-10-09.canonical	(revision 44)
@@ -5,4 +5,4 @@
   : !!str "value",
   ? !!str "empty"
-  : !!str "",
+  : !!null "",
 }
Index: /branches/pyyaml3000/tests/data/spec-02-16.structure
===================================================================
--- /branches/pyyaml3000/tests/data/spec-02-16.structure	(revision 44)
+++ /branches/pyyaml3000/tests/data/spec-02-16.structure	(revision 44)
@@ -0,0 +1,1 @@
+[(True, True), (True, True), (True, True)]
Index: /branches/pyyaml3000/tests/data/spec-10-10.canonical
===================================================================
--- /branches/pyyaml3000/tests/data/spec-10-10.canonical	(revision 38)
+++ /branches/pyyaml3000/tests/data/spec-10-10.canonical	(revision 44)
@@ -5,12 +5,12 @@
   : !!str "explicit value",
   ? !!str "explicit key2"
-  : !!str "",
+  : !!null "",
   ? !!str "explicit key3"
-  : !!str "",
+  : !!null "",
   ? !!str "simple key1"
   : !!str "explicit value",
   ? !!str "simple key2"
-  : !!str "",
+  : !!null "",
   ? !!str "simple key3"
-  : !!str "",
+  : !!null "",
 }
Index: /branches/pyyaml3000/tests/data/spec-02-04.structure
===================================================================
--- /branches/pyyaml3000/tests/data/spec-02-04.structure	(revision 44)
+++ /branches/pyyaml3000/tests/data/spec-02-04.structure	(revision 44)
@@ -0,0 +1,4 @@
+[
+    [(True, True), (True, True), (True, True)],
+    [(True, True), (True, True), (True, True)],
+]
Index: /branches/pyyaml3000/tests/data/spec-02-25.structure
===================================================================
--- /branches/pyyaml3000/tests/data/spec-02-25.structure	(revision 44)
+++ /branches/pyyaml3000/tests/data/spec-02-25.structure	(revision 44)
@@ -0,0 +1,1 @@
+[(True, None), (True, None), (True, None)]
Index: /branches/pyyaml3000/tests/data/spec-08-01.data
===================================================================
--- /branches/pyyaml3000/tests/data/spec-08-01.data	(revision 38)
+++ /branches/pyyaml3000/tests/data/spec-08-01.data	(revision 44)
@@ -1,4 +1,2 @@
-!!str
- &a1
-  "foo" : !!str bar
+!!str &a1 "foo" : !!str bar
 &a2 baz : *a1
Index: /branches/pyyaml3000/tests/data/spec-09-32.canonical
===================================================================
--- /branches/pyyaml3000/tests/data/spec-09-32.canonical	(revision 38)
+++ /branches/pyyaml3000/tests/data/spec-09-32.canonical	(revision 44)
@@ -1,9 +1,7 @@
 %YAML 1.1
 ---
-!!seq [
-  !!str "folded line\n\
-        next line\n\
-        \  * bullet\n\
-        \  * list\n\
-        last line\n"
-]
+!!str "folded line\n\
+      next line\n\
+      \  * bullet\n\
+      \  * list\n\
+      last line\n"
Index: /branches/pyyaml3000/tests/data/spec-02-13.structure
===================================================================
--- /branches/pyyaml3000/tests/data/spec-02-13.structure	(revision 44)
+++ /branches/pyyaml3000/tests/data/spec-02-13.structure	(revision 44)
@@ -0,0 +1,1 @@
+True
Index: /branches/pyyaml3000/tests/test_tokens.py
===================================================================
--- /branches/pyyaml3000/tests/test_tokens.py	(revision 43)
+++ /branches/pyyaml3000/tests/test_tokens.py	(revision 44)
@@ -26,23 +26,23 @@
 
     replaces = {
-        YAMLDirective: '%',
-        TagDirective: '%',
-        ReservedDirective: '%',
-        DocumentStart: '---',
-        DocumentEnd: '...',
-        Alias: '*',
-        Anchor: '&',
-        Tag: '!',
-        Scalar: '_',
-        BlockSequenceStart: '[[',
-        BlockMappingStart: '{{',
-        BlockEnd: ']}',
-        FlowSequenceStart: '[',
-        FlowSequenceEnd: ']',
-        FlowMappingStart: '{',
-        FlowMappingEnd: '}',
-        Entry: ',',
-        Key: '?',
-        Value: ':',
+        YAMLDirectiveToken: '%',
+        TagDirectiveToken: '%',
+        ReservedDirectiveToken: '%',
+        DocumentStartToken: '---',
+        DocumentEndToken: '...',
+        AliasToken: '*',
+        AnchorToken: '&',
+        TagToken: '!',
+        ScalarToken: '_',
+        BlockSequenceStartToken: '[[',
+        BlockMappingStartToken: '{{',
+        BlockEndToken: ']}',
+        FlowSequenceStartToken: '[',
+        FlowSequenceEndToken: ']',
+        FlowMappingStartToken: '{',
+        FlowMappingEndToken: '}',
+        EntryToken: ',',
+        KeyToken: '?',
+        ValueToken: ':',
     }
 
@@ -53,5 +53,5 @@
             scanner = Scanner(data_filename, file(data_filename, 'rb').read())
             tokens1 = []
-            while not isinstance(scanner.peek_token(), End):
+            while not isinstance(scanner.peek_token(), EndToken):
                 tokens1.append(scanner.get_token())
             tokens1 = [self.replaces[t.__class__] for t in tokens1]
@@ -75,5 +75,5 @@
                 scanner = Scanner(filename, file(filename, 'rb').read())
                 tokens = []
-                while not isinstance(scanner.peek_token(), End):
+                while not isinstance(scanner.peek_token(), EndToken):
                     tokens.append(scanner.get_token().__class__.__name__)
             except:
Index: /branches/pyyaml3000/lib/yaml/stream.py
===================================================================
--- /branches/pyyaml3000/lib/yaml/stream.py	(revision 43)
+++ /branches/pyyaml3000/lib/yaml/stream.py	(revision 44)
@@ -19,5 +19,5 @@
             if self.index >= len(self.data):
                 break
-            if self.data[self.index] in u'\r\n':
+            if self.data[self.index] in u'\r\n\x85\u2028\u2029':
                 self.line += 1
                 self.column = 0
Index: /branches/pyyaml3000/lib/yaml/parser.py
===================================================================
--- /branches/pyyaml3000/lib/yaml/parser.py	(revision 43)
+++ /branches/pyyaml3000/lib/yaml/parser.py	(revision 44)
@@ -40,126 +40,162 @@
 # flow_mapping_entry: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START KEY }
 
+from scanner import *
+
+class Error(Exception):
+    pass
+
+class Node:
+    def __repr__(self):
+        args = []
+        for attribute in ['anchor', 'tag', 'value']:
+            if hasattr(self, attribute):
+                args.append(repr(getattr(self, attribute)))
+        return "%s(%s)" % (self.__class__.__name__, ', '.join(args))
+
+class AliasNode(Node):
+    def __init__(self, anchor):
+        self.anchor = anchor
+
+class ScalarNode(Node):
+    def __init__(self, anchor, tag, value):
+        self.anchor = anchor
+        self.tag = tag
+        self.value = value
+
+class SequenceNode(Node):
+    def __init__(self, anchor, tag, value):
+        self.anchor = anchor
+        self.tag = tag
+        self.value = value
+
+class MappingNode(Node):
+    def __init__(self, anchor, tag, value):
+        self.anchor = anchor
+        self.tag = tag
+        self.value = value
+
 class Parser:
 
-    def parse(self, source, data):
-        scanner = Scanner()
-        self.tokens = scanner.scan(source, data)
-        self.tokens.append('END')
-        documents = self.parse_stream()
-        if len(documents) == 1:
-            return documents[0]
-        return documents
+    def __init__(self, source, data):
+        self.scanner = Scanner(source, data)
+
+    def is_token(self, *choices):
+        token = self.scanner.peek_token()
+        for choice in choices:
+            if isinstance(token, choices):
+                return True
+        return False
+
+    def get_token(self):
+        return self.scanner.get_token()
+
+    def parse(self):
+        return self.parse_stream()
 
     def parse_stream(self):
         documents = []
-        if self.tokens[0] not in ['DIRECTIVE', 'DOCUMENT_START', 'END']:
+        if not self.is_token(DirectiveToken, DocumentStartToken, EndToken):
             documents.append(self.parse_block_node())
-        while self.tokens[0] != 'END':
-            while self.tokens[0] == 'DIRECTIVE':
-                self.tokens.pop(0)
-            if self.tokens[0] != 'DOCUMENT_START':
-                self.error('DOCUMENT_START is expected')
-            self.tokens.pop(0)
-            if self.tokens[0] in ['DIRECTIVE', 'DOCUMENT_START', 'DOCUMENT_END', 'END']:
+        while not self.is_token(EndToken):
+            while self.is_token(DirectiveToken):
+                self.get_token()
+            if not self.is_token(DocumentStartToken):
+                self.fail('DOCUMENT-START is expected')
+            self.get_token()
+            if self.is_token(DirectiveToken,
+                    DocumentStartToken, DocumentEndToken, EndToken):
                 documents.append(None)
             else:
                 documents.append(self.parse_block_node())
-            while self.tokens[0] == 'DOCUMENT_END':
-                self.tokens.pop(0)
-        if self.tokens[0] != 'END':
-            self.error("END is expected")
-        return tuple(documents)
+            while self.is_token(DocumentEndToken):
+                self.get_token()
+        if not self.is_token(EndToken):
+            self.fail("END is expected")
+        return documents
 
     def parse_block_node(self):
-        if self.tokens[0] == 'ALIAS':
-            self.tokens.pop(0)
-            return '*'
-        if self.tokens[0] == 'TAG':
-            self.tokens.pop(0)
-            if self.tokens[0] == 'ANCHOR':
-                self.tokens.pop(0)
-        elif self.tokens[0] == 'ANCHOR':
-            self.tokens.pop(0)
-            if self.tokens[0] == 'TAG':
-                self.tokens.pop(0)
-        return self.parse_block_content()
+        return self.parse_node(block=True)
 
     def parse_flow_node(self):
-        if self.tokens[0] == 'ALIAS':
-            self.tokens.pop(0)
-            return '*'
-        if self.tokens[0] == 'TAG':
-            self.tokens.pop(0)
-            if self.tokens[0] == 'ANCHOR':
-                self.tokens.pop(0)
-        elif self.tokens[0] == 'ANCHOR':
-            self.tokens.pop(0)
-            if self.tokens[0] == 'TAG':
-                self.tokens.pop(0)
-        return self.parse_flow_content()
+        return self.parse_node()
 
     def parse_block_node_or_indentless_sequence(self):
-        if self.tokens[0] == 'ALIAS':
-            self.tokens.pop(0)
-            return '*'
-        if self.tokens[0] == 'TAG':
-            self.tokens.pop(0)
-            if self.tokens[0] == 'ANCHOR':
-                self.tokens.pop(0)
-        elif self.tokens[0] == 'ANCHOR':
-            self.tokens.pop(0)
-            if self.tokens[0] == 'TAG':
-                self.tokens.pop(0)
-        if self.tokens[0] == 'ENTRY':
-            return self.parse_indentless_sequence(self)
-        return self.parse_block_content()
+        return self.parse_node(block=True, indentless_sequence=True)
+
+    def parse_node(self, block=False, indentless_sequence=False):
+        if self.is_token(AliasToken):
+            token = self.get_token()
+            return AliasNode(token.value)
+        anchor = None
+        tag = None
+        if self.is_token(AnchorToken):
+            anchor = self.get_token().value
+            if self.is_token(TagToken):
+                tag = self.get_token().value
+        elif self.is_token(TagToken):
+            tag = self.get_token().value
+            if self.is_token(AnchorToken):
+                anchor = self.get_token().value
+        if indentless_sequence and self.is_token(EntryToken):
+            NodeClass = SequenceNode
+            value = self.parse_indentless_sequence()
+        else:
+            if self.is_token(ScalarToken):
+                NodeClass = ScalarNode
+            elif self.is_token(BlockSequenceStartToken, FlowSequenceStartToken):
+                NodeClass = SequenceNode
+            elif self.is_token(BlockMappingStartToken, FlowMappingStartToken):
+                NodeClass = MappingNode
+            if block:
+                value = self.parse_block_content()
+            else:
+                value = self.parse_flow_content()
+        return NodeClass(anchor, tag, value)
 
     def parse_block_content(self):
-        if self.tokens[0] == 'SCALAR':
-            self.tokens.pop(0)
-            return True
-        elif self.tokens[0] == 'BLOCK_SEQ_START':
+        if self.is_token(ScalarToken):
+            return self.get_token().value
+        elif self.is_token(BlockSequenceStartToken):
             return self.parse_block_sequence()
-        elif self.tokens[0] == 'BLOCK_MAP_START':
+        elif self.is_token(BlockMappingStartToken):
             return self.parse_block_mapping()
-        elif self.tokens[0] == 'FLOW_SEQ_START':
+        elif self.is_token(FlowSequenceStartToken):
             return self.parse_flow_sequence()
-        elif self.tokens[0] == 'FLOW_MAP_START':
+        elif self.is_token(FlowMappingStartToken):
             return self.parse_flow_mapping()
         else:
-            self.error('block content is expected')
+            self.fail('block content is expected')
 
     def parse_flow_content(self):
-        if self.tokens[0] == 'SCALAR':
-            self.tokens.pop(0)
-            return True
-        elif self.tokens[0] == 'FLOW_SEQ_START':
+        if self.is_token(ScalarToken):
+            return self.get_token().value
+        elif self.is_token(FlowSequenceStartToken):
             return self.parse_flow_sequence()
-        elif self.tokens[0] == 'FLOW_MAP_START':
+        elif self.is_token(FlowMappingStartToken):
             return self.parse_flow_mapping()
         else:
-            self.error('flow content is expected')
+            self.fail('flow content is expected')
 
     def parse_block_sequence(self):
         sequence = []
-        if self.tokens[0] != 'BLOCK_SEQ_START':
-            self.error('BLOCK_SEQ_START is expected')
-        self.tokens.pop(0)
-        while self.tokens[0] == 'ENTRY':
-            self.tokens.pop(0)
-            if self.tokens[0] not in ['ENTRY', 'BLOCK_END']:
+        if not self.is_token(BlockSequenceStartToken):
+            self.fail('BLOCK-SEQUENCE-START is expected')
+        self.get_token()
+        while self.is_token(EntryToken):
+            self.get_token()
+            if not self.is_token(EntryToken, BlockEndToken):
                 sequence.append(self.parse_block_node())
             else:
                 sequence.append(None)
-        if self.tokens[0] != 'BLOCK_END':
-            self.error('BLOCK_END is expected')
-        self.tokens.pop(0)
+        if not self.is_token(BlockEndToken):
+            self.fail('BLOCK-END is expected')
+        self.get_token()
         return sequence
 
     def parse_indentless_sequence(self):
         sequence = []
-        while self.tokens[0] == 'ENTRY':
-            self.tokens.pop(0)
-            if self.tokens[0] not in ['ENTRY']:
+        while self.is_token(EntryToken):
+            self.get_token()
+            if not self.is_token(EntryToken):
                 sequence.append(self.parse_block_node())
             else:
@@ -169,82 +205,83 @@
     def parse_block_mapping(self):
         mapping = []
-        if self.tokens[0] != 'BLOCK_MAP_START':
-            self.error('BLOCK_MAP_START is expected')
-        self.tokens.pop(0)
-        while self.tokens[0] in ['KEY', 'VALUE']:
+        if not self.is_token(BlockMappingStartToken):
+            self.fail('BLOCK-MAPPING-START is expected')
+        self.get_token()
+        while self.is_token(KeyToken, ValueToken):
             key = None
             value = None
-            if self.tokens[0] == 'KEY':
-                self.tokens.pop(0)
-                if self.tokens[0] not in ['KEY', 'VALUE', 'BLOCK_END']:
+            if self.is_token(KeyToken):
+                self.get_token()
+                if not self.is_token(KeyToken, ValueToken, BlockEndToken):
                     key = self.parse_block_node_or_indentless_sequence()
-            if self.tokens[0] == 'VALUE':
-                self.tokens.pop(0)
-                if self.tokens[0] not in ['KEY', 'VALUE', 'BLOCK_END']:
+            if self.is_token(ValueToken):
+                self.get_token()
+                if not self.is_token(KeyToken, ValueToken, BlockEndToken):
                     value = self.parse_block_node_or_indentless_sequence()
             mapping.append((key, value))
-        if self.tokens[0] != 'BLOCK_END':
-            self.error('BLOCK_END is expected')
-        self.tokens.pop(0)
+        if not self.is_token(BlockEndToken):
+            self.fail('BLOCK-END is expected')
+        self.get_token()
         return mapping
 
     def parse_flow_sequence(self):
         sequence = []
-        if self.tokens[0] != 'FLOW_SEQ_START':
-            self.error('FLOW_SEQ_START is expected')
-        self.tokens.pop(0)
-        while self.tokens[0] != 'FLOW_SEQ_END':
-            if self.tokens[0] == 'KEY':
-                self.tokens.pop(0)
+        if not self.is_token(FlowSequenceStartToken):
+            self.fail('FLOW-SEQUENCE-START is expected')
+        self.get_token()
+        while not self.is_token(FlowSequenceEndToken):
+            if self.is_token(KeyToken):
+                self.get_token()
                 key = None
                 value = None
-                if self.tokens[0] != 'VALUE':
+                if not self.is_token(ValueToken):
                     key = self.parse_flow_node()
-                if self.tokens[0] == 'VALUE':
-                    self.tokens.pop(0)
-                    if self.tokens[0] not in ['ENTRY', 'FLOW_SEQ_END']:
+                if self.is_token(ValueToken):
+                    self.get_token()
+                    if not self.is_token(EntryToken, FlowSequenceEndToken):
                         value = self.parse_flow_node()
-                sequence.append([(key, value)])
+                node = MappingNode(None, None, [(key, value)])
+                sequence.append(node)
             else:
                 sequence.append(self.parse_flow_node())
-            if self.tokens[0] not in ['ENTRY', 'FLOW_SEQ_END']:
-                self.error("ENTRY or FLOW_SEQ_END is expected")
-            if self.tokens[0] == 'ENTRY':
-                self.tokens.pop(0)
-        if self.tokens[0] != 'FLOW_SEQ_END':
-            self.error('FLOW_SEQ_END is expected')
-        self.tokens.pop(0)
+            if not self.is_token(EntryToken, FlowSequenceEndToken):
+                self.fail("ENTRY or FLOW-SEQUENCE-END are expected")
+            if self.is_token(EntryToken):
+                self.get_token()
+        if not self.is_token(FlowSequenceEndToken):
+            self.fail('FLOW-SEQUENCE-END is expected')
+        self.get_token()
         return sequence
 
     def parse_flow_mapping(self):
         mapping = []
-        if self.tokens[0] != 'FLOW_MAP_START':
-            self.error('FLOW_MAP_START is expected')
-        self.tokens.pop(0)
-        while self.tokens[0] != 'FLOW_MAP_END':
-            if self.tokens[0] == 'KEY':
-                self.tokens.pop(0)
+        if not self.is_token(FlowMappingStartToken):
+            self.fail('FLOW-MAPPING-START is expected')
+        self.get_token()
+        while not self.is_token(FlowMappingEndToken):
+            if self.is_token(KeyToken):
+                self.get_token()
                 key = None
                 value = None
-                if self.tokens[0] != 'VALUE':
+                if not self.is_token(ValueToken):
                     key = self.parse_flow_node()
-                if self.tokens[0] == 'VALUE':
-                    self.tokens.pop(0)
-                    if self.tokens[0] not in ['ENTRY', 'FLOW_MAP_END']:
+                if self.is_token(ValueToken):
+                    self.get_token()
+                    if not self.is_token(EntryToken, FlowMappingEndToken):
                         value = self.parse_flow_node()
                 mapping.append((key, value))
             else:
                 mapping.append((self.parse_flow_node(), None))
-            if self.tokens[0] not in ['ENTRY', 'FLOW_MAP_END']:
-                self.error("ENTRY or FLOW_MAP_END is expected")
-            if self.tokens[0] == 'ENTRY':
-                self.tokens.pop(0)
-        if self.tokens[0] != 'FLOW_MAP_END':
-            self.error('FLOW_MAP_END is expected')
-        self.tokens.pop(0)
+            if not self.is_token(EntryToken, FlowMappingEndToken):
+                self.fail("ENTRY or FLOW-MAPPING-END are expected")
+            if self.is_token(EntryToken):
+                self.get_token()
+        if not self.is_token(FlowMappingEndToken):
+            self.fail('FLOW-MAPPING-END is expected')
+        self.get_token()
         return mapping
 
-    def error(self, message):
-        raise Error(message+': '+str(self.tokens))
-
-
+    def fail(self, message):
+        marker = self.scanner.peek_token().start_marker
+        raise Error(message+':\n'+marker.get_snippet())
+
Index: /branches/pyyaml3000/lib/yaml/scanner.py
===================================================================
--- /branches/pyyaml3000/lib/yaml/scanner.py	(revision 43)
+++ /branches/pyyaml3000/lib/yaml/scanner.py	(revision 44)
@@ -23,5 +23,8 @@
         self.end_marker = end_marker
 
-class YAMLDirective(Token):
+class DirectiveToken(Token):
+    pass
+
+class YAMLDirectiveToken(DirectiveToken):
     def __init__(self, major_version, minor_version, start_marker, end_marker):
         self.major_version = major_version
@@ -30,8 +33,8 @@
         self.end_marker = end_marker
 
-class TagDirective(Token):
-    pass
-
-class ReservedDirective(Token):
+class TagDirectiveToken(DirectiveToken):
+    pass
+
+class ReservedDirectiveToken(DirectiveToken):
     def __init__(self, name, start_marker, end_marker):
         self.name = name
@@ -39,44 +42,44 @@
         self.end_marker = end_marker
 
-class DocumentStart(Token):
-    pass
-
-class DocumentEnd(Token):
-    pass
-
-class End(Token):
-    pass
-
-class BlockSequenceStart(Token):
-    pass
-
-class BlockMappingStart(Token):
-    pass
-
-class BlockEnd(Token):
-    pass
-
-class FlowSequenceStart(Token):
-    pass
-
-class FlowMappingStart(Token):
-    pass
-
-class FlowSequenceEnd(Token):
-    pass
-
-class FlowMappingEnd(Token):
-    pass
-
-class Key(Token):
-    pass
-
-class Value(Token):
-    pass
-
-class Entry(Token):
-    pass
-
-class Alias(Token):
+class DocumentStartToken(Token):
+    pass
+
+class DocumentEndToken(Token):
+    pass
+
+class EndToken(Token):
+    pass
+
+class BlockSequenceStartToken(Token):
+    pass
+
+class BlockMappingStartToken(Token):
+    pass
+
+class BlockEndToken(Token):
+    pass
+
+class FlowSequenceStartToken(Token):
+    pass
+
+class FlowMappingStartToken(Token):
+    pass
+
+class FlowSequenceEndToken(Token):
+    pass
+
+class FlowMappingEndToken(Token):
+    pass
+
+class KeyToken(Token):
+    pass
+
+class ValueToken(Token):
+    pass
+
+class EntryToken(Token):
+    pass
+
+class AliasToken(Token):
     def __init__(self, value, start_marker, end_marker):
         self.value = value
@@ -84,5 +87,5 @@
         self.end_marker = end_marker
 
-class Anchor(Token):
+class AnchorToken(Token):
     def __init__(self, value, start_marker, end_marker):
         self.value = value
@@ -90,5 +93,5 @@
         self.end_marker = end_marker
 
-class Tag(Token):
+class TagToken(Token):
     def __init__(self, value, start_marker, end_marker):
         self.value = value
@@ -96,5 +99,5 @@
         self.end_marker = end_marker
 
-class Scalar(Token):
+class ScalarToken(Token):
     def __init__(self, value, plain, start_marker, end_marker):
         self.value = value
@@ -380,5 +383,5 @@
             marker = self.stream.get_marker()
             self.indent = self.indents.pop()
-            self.tokens.append(BlockEnd(marker, marker))
+            self.tokens.append(BlockEndToken(marker, marker))
 
     def add_indent(self, column):
@@ -405,5 +408,5 @@
         
         # Add END.
-        self.tokens.append(End(marker, marker))
+        self.tokens.append(EndToken(marker, marker))
 
         # The stream is ended.
@@ -423,8 +426,8 @@
 
     def fetch_document_start(self):
-        self.fetch_document_indicator(DocumentStart)
+        self.fetch_document_indicator(DocumentStartToken)
 
     def fetch_document_end(self):
-        self.fetch_document_indicator(DocumentEnd)
+        self.fetch_document_indicator(DocumentEndToken)
 
     def fetch_document_indicator(self, TokenClass):
@@ -445,16 +448,16 @@
 
     def fetch_flow_sequence_start(self):
-        self.fetch_flow_collection_start(FlowSequenceStart)
+        self.fetch_flow_collection_start(FlowSequenceStartToken)
 
     def fetch_flow_mapping_start(self):
-        self.fetch_flow_collection_start(FlowMappingStart)
+        self.fetch_flow_collection_start(FlowMappingStartToken)
 
     def fetch_flow_collection_start(self, TokenClass):
+
+        # '[' and '{' may start a simple key.
+        self.save_possible_simple_key()
 
         # Increase the flow level.
         self.flow_level += 1
-
-        # '[' and '{' may start a simple key.
-        self.save_possible_simple_key()
 
         # Simple keys are allowed after '[' and '{'.
@@ -468,8 +471,8 @@
 
     def fetch_flow_sequence_end(self):
-        self.fetch_flow_collection_end(FlowSequenceEnd)
+        self.fetch_flow_collection_end(FlowSequenceEndToken)
 
     def fetch_flow_mapping_end(self):
-        self.fetch_flow_collection_end(FlowMappingEnd)
+        self.fetch_flow_collection_end(FlowMappingEndToken)
 
     def fetch_flow_collection_end(self, TokenClass):
@@ -502,5 +505,5 @@
             if self.add_indent(self.stream.column):
                 marker = self.stream.get_marker()
-                self.tokens.append(BlockSequenceStart(marker, marker))
+                self.tokens.append(BlockSequenceStartToken(marker, marker))
 
         # Simple keys are allowed after '-' and ','.
@@ -514,5 +517,5 @@
         self.stream.read()
         end_marker = self.stream.get_marker()
-        self.tokens.append(Entry(start_marker, end_marker))
+        self.tokens.append(EntryToken(start_marker, end_marker))
 
     def fetch_key(self):
@@ -528,5 +531,5 @@
             if self.add_indent(self.stream.column):
                 marker = self.stream.get_marker()
-                self.tokens.append(BlockMappingStart(marker, marker))
+                self.tokens.append(BlockMappingStartToken(marker, marker))
 
         # Simple keys are allowed after '?' in the block context.
@@ -540,5 +543,5 @@
         self.stream.read()
         end_marker = self.stream.get_marker()
-        self.tokens.append(Key(start_marker, end_marker))
+        self.tokens.append(KeyToken(start_marker, end_marker))
 
     def fetch_value(self):
@@ -551,5 +554,5 @@
             del self.possible_simple_keys[self.flow_level]
             self.tokens.insert(key.token_number-self.tokens_taken,
-                    Key(key.marker, key.marker))
+                    KeyToken(key.marker, key.marker))
 
             # If this key starts a new block mapping, we need to add
@@ -558,5 +561,5 @@
                 if self.add_indent(key.column):
                     self.tokens.insert(key.token_number-self.tokens_taken,
-                            BlockMappingStart(key.marker, key.marker))
+                            BlockMappingStartToken(key.marker, key.marker))
 
             # There cannot be two simple keys one after another.
@@ -576,5 +579,5 @@
         self.stream.read()
         end_marker = self.stream.get_marker()
-        self.tokens.append(Value(start_marker, end_marker))
+        self.tokens.append(ValueToken(start_marker, end_marker))
 
     def fetch_alias(self):
@@ -587,5 +590,5 @@
 
         # Scan and add ALIAS.
-        self.scan_anchor(Alias)
+        self.scan_anchor(AliasToken)
 
     def fetch_anchor(self):
@@ -598,5 +601,5 @@
 
         # Scan and add ANCHOR.
-        self.scan_anchor(Anchor)
+        self.scan_anchor(AnchorToken)
 
     def fetch_tag(self):
@@ -739,9 +742,9 @@
         marker = self.stream.get_marker()
         if self.stream.peek(5) == u'%YAML ':
-            self.tokens.append(YAMLDirective(1, 1, marker, marker))
+            self.tokens.append(YAMLDirectiveToken(1, 1, marker, marker))
         elif self.stream.peek(4) == u'%TAG ':
-            self.tokens.append(TagDirective(marker, marker))
+            self.tokens.append(TagDirectiveToken(marker, marker))
         else:
-            self.tokens.append(ReservedDirective('', marker, marker))
+            self.tokens.append(ReservedDirectiveToken('', marker, marker))
         while self.stream.peek() not in u'\0\r\n':
             self.stream.read()
@@ -760,5 +763,5 @@
             self.stream.read()
         end_marker = self.stream.get_marker()
-        self.tokens.append(Tag('', start_marker, end_marker))
+        self.tokens.append(TagToken('', start_marker, end_marker))
 
     def scan_block_scalar(self, folded):
@@ -768,5 +771,5 @@
             indent = 1
         while True:
-            while self.stream.peek() and self.stream.peek() and self.stream.peek() not in u'\0\r\n':
+            while self.stream.peek() and self.stream.peek() and self.stream.peek() not in u'\0\r\n\x85\u2028\u2029':
                 self.stream.read()
             if self.stream.peek() != u'\0':
@@ -776,7 +779,7 @@
                 self.stream.read()
                 count += 1
-            if count < indent and self.stream.peek() not in u'#\r\n':
+            if count < indent and self.stream.peek() not in u'#\r\n\x85\u2028\u2029':
                 break
-        self.tokens.append(Scalar('', False, start_marker, start_marker))
+        self.tokens.append(ScalarToken('', False, start_marker, start_marker))
 
     def scan_flow_scalar(self, double):
@@ -791,5 +794,5 @@
                 self.stream.read(1)
         self.stream.read(1)
-        self.tokens.append(Scalar('', False, marker, marker))
+        self.tokens.append(ScalarToken('', False, marker, marker))
 
     def scan_plain(self):
@@ -823,5 +826,5 @@
                 break
             space = True
-        self.tokens.append(Scalar('', True, marker, marker))
+        self.tokens.append(ScalarToken('', True, marker, marker))
 
     def invalid_token(self):