Index: branches/pyyaml3000/tests/test_tokens.py
===================================================================
--- branches/pyyaml3000/tests/test_tokens.py	(revision 39)
+++ branches/pyyaml3000/tests/test_tokens.py	(revision 43)
@@ -2,5 +2,5 @@
 import test_appliance
 
-from yaml.scanner import Scanner
+from yaml.scanner import *
 
 class TestTokens(test_appliance.TestAppliance):
@@ -26,21 +26,23 @@
 
     replaces = {
-        'DIRECTIVE': '%',
-        'DOCUMENT_START': '---',
-        'DOCUMENT_END': '...',
-        'ALIAS': '*',
-        'ANCHOR': '&',
-        'TAG': '!',
-        'SCALAR': '_',
-        'BLOCK_SEQ_START': '[[',
-        'BLOCK_MAP_START': '{{',
-        'BLOCK_END': ']}',
-        'FLOW_SEQ_START': '[',
-        'FLOW_SEQ_END': ']',
-        'FLOW_MAP_START': '{',
-        'FLOW_MAP_END': '}',
-        'ENTRY': ',',
-        'KEY': '?',
-        'VALUE': ':',
+        YAMLDirective: '%',
+        TagDirective: '%',
+        ReservedDirective: '%',
+        DocumentStart: '---',
+        DocumentEnd: '...',
+        Alias: '*',
+        Anchor: '&',
+        Tag: '!',
+        Scalar: '_',
+        BlockSequenceStart: '[[',
+        BlockMappingStart: '{{',
+        BlockEnd: ']}',
+        FlowSequenceStart: '[',
+        FlowSequenceEnd: ']',
+        FlowMappingStart: '{',
+        FlowMappingEnd: '}',
+        Entry: ',',
+        Key: '?',
+        Value: ':',
     }
 
@@ -49,10 +51,14 @@
         tokens2 = file(tokens_filename, 'rb').read().split()
         try:
-            scanner = Scanner()
-            tokens1 = scanner.scan(data_filename, file(data_filename, 'rb').read())
-            tokens1 = [self.replaces[t] for t in tokens1]
+            scanner = Scanner(data_filename, file(data_filename, 'rb').read())
+            tokens1 = []
+            while not isinstance(scanner.peek_token(), End):
+                tokens1.append(scanner.get_token())
+            tokens1 = [self.replaces[t.__class__] for t in tokens1]
             self.failUnlessEqual(tokens1, tokens2)
         except:
             print
+            print "DATA:"
+            print file(data_filename, 'rb').read()
             print "TOKENS1:", tokens1
             print "TOKENS2:", tokens2
@@ -61,2 +67,21 @@
 TestTokens.add_tests('testTokens', '.data', '.tokens')
 
+class TestScanner(test_appliance.TestAppliance):
+
+    def _testScanner(self, test_name, data_filename, canonical_filename):
+        for filename in [canonical_filename, data_filename]:
+            tokens = None
+            try:
+                scanner = Scanner(filename, file(filename, 'rb').read())
+                tokens = []
+                while not isinstance(scanner.peek_token(), End):
+                    tokens.append(scanner.get_token().__class__.__name__)
+            except:
+                print
+                print "DATA:"
+                print file(data_filename, 'rb').read()
+                print "TOKENS:", tokens
+                raise
+
+TestScanner.add_tests('testScanner', '.data', '.canonical')
+