source: pyyaml/trunk/lib/yaml/scanner.py @ 149

Revision 149, 51.3 KB checked in by xi, 9 years ago (diff)

Raise an error for colons in the flow context.

Line 
1
2# Scanner produces tokens of the following types:
3# STREAM-START
4# STREAM-END
5# DIRECTIVE(name, value)
6# DOCUMENT-START
7# DOCUMENT-END
8# BLOCK-SEQUENCE-START
9# BLOCK-MAPPING-START
10# BLOCK-END
11# FLOW-SEQUENCE-START
12# FLOW-MAPPING-START
13# FLOW-SEQUENCE-END
14# FLOW-MAPPING-END
15# BLOCK-ENTRY
16# FLOW-ENTRY
17# KEY
18# VALUE
19# ALIAS(value)
20# ANCHOR(value)
21# TAG(value)
22# SCALAR(value, plain)
23#
24# Read comments in the Scanner code for more details.
25#
26
27__all__ = ['Scanner', 'ScannerError']
28
29from error import MarkedYAMLError
30from tokens import *
31
32class ScannerError(MarkedYAMLError):
33    pass
34
35class SimpleKey:
36    # See below simple keys treatment.
37
38    def __init__(self, token_number, required, index, line, column, mark):
39        self.token_number = token_number
40        self.required = required
41        self.index = index
42        self.line = line
43        self.column = column
44        self.mark = mark
45
46class Scanner:
47
48    def __init__(self):
49        """Initialize the scanner."""
50        # It is assumed that Scanner and Reader will have a common descendant.
51        # Reader do the dirty work of checking for BOM and converting the
52        # input data to Unicode. It also adds NUL to the end.
53        #
54        # Reader supports the following methods
55        #   self.peek(i=0)       # peek the next i-th character
56        #   self.prefix(l=1)     # peek the next l characters
57        #   self.forward(l=1)    # read the next l characters and move the pointer.
58
59        # Had we reached the end of the stream?
60        self.done = False
61
62        # The number of unclosed '{' and '['. `flow_level == 0` means block
63        # context.
64        self.flow_level = 0
65
66        # List of processed tokens that are not yet emitted.
67        self.tokens = []
68
69        # Add the STREAM-START token.
70        self.fetch_stream_start()
71
72        # Number of tokens that were emitted through the `get_token` method.
73        self.tokens_taken = 0
74
75        # The current indentation level.
76        self.indent = -1
77
78        # Past indentation levels.
79        self.indents = []
80
81        # Variables related to simple keys treatment.
82
83        # A simple key is a key that is not denoted by the '?' indicator.
84        # Example of simple keys:
85        #   ---
86        #   block simple key: value
87        #   ? not a simple key:
88        #   : { flow simple key: value }
89        # We emit the KEY token before all keys, so when we find a potential
90        # simple key, we try to locate the corresponding ':' indicator.
91        # Simple keys should be limited to a single line and 1024 characters.
92
93        # Can a simple key start at the current position? A simple key may
94        # start:
95        # - at the beginning of the line, not counting indentation spaces
96        #       (in block context),
97        # - after '{', '[', ',' (in the flow context),
98        # - after '?', ':', '-' (in the block context).
99        # In the block context, this flag also signifies if a block collection
100        # may start at the current position.
101        self.allow_simple_key = True
102
103        # Keep track of possible simple keys. This is a dictionary. The key
104        # is `flow_level`; there can be no more that one possible simple key
105        # for each level. The value is a SimpleKey record:
106        #   (token_number, required, index, line, column, mark)
107        # A simple key may start with ALIAS, ANCHOR, TAG, SCALAR(flow),
108        # '[', or '{' tokens.
109        self.possible_simple_keys = {}
110
111    # Public methods.
112
113    def check_token(self, *choices):
114        # Check if the next token is one of the given types.
115        while self.need_more_tokens():
116            self.fetch_more_tokens()
117        if self.tokens:
118            if not choices:
119                return True
120            for choice in choices:
121                if isinstance(self.tokens[0], choice):
122                    return True
123        return False
124
125    def peek_token(self):
126        # Return the next token, but do not delete if from the queue.
127        while self.need_more_tokens():
128            self.fetch_more_tokens()
129        if self.tokens:
130            return self.tokens[0]
131
132    def get_token(self):
133        # Return the next token.
134        while self.need_more_tokens():
135            self.fetch_more_tokens()
136        if self.tokens:
137            self.tokens_taken += 1
138            return self.tokens.pop(0)
139
140    def __iter__(self):
141        # Iterator protocol.
142        while self.need_more_tokens():
143            self.fetch_more_tokens()
144        while self.tokens:
145            self.tokens_taken += 1
146            yield self.tokens.pop(0)
147            while self.need_more_tokens():
148                self.fetch_more_tokens()
149
150    # Private methods.
151
152    def need_more_tokens(self):
153        if self.done:
154            return False
155        if not self.tokens:
156            return True
157        # The current token may be a potential simple key, so we
158        # need to look further.
159        self.stale_possible_simple_keys()
160        if self.next_possible_simple_key() == self.tokens_taken:
161            return True
162
163    def fetch_more_tokens(self):
164
165        # Eat whitespaces and comments until we reach the next token.
166        self.scan_to_next_token()
167
168        # Remove obsolete possible simple keys.
169        self.stale_possible_simple_keys()
170
171        # Compare the current indentation and column. It may add some tokens
172        # and decrease the current indentation level.
173        self.unwind_indent(self.column)
174
175        # Peek the next character.
176        ch = self.peek()
177
178        # Is it the end of stream?
179        if ch == u'\0':
180            return self.fetch_stream_end()
181
182        # Is it a directive?
183        if ch == u'%' and self.check_directive():
184            return self.fetch_directive()
185
186        # Is it the document start?
187        if ch == u'-' and self.check_document_start():
188            return self.fetch_document_start()
189
190        # Is it the document end?
191        if ch == u'.' and self.check_document_end():
192            return self.fetch_document_end()
193
194        # TODO: support for BOM within a stream.
195        #if ch == u'\uFEFF':
196        #    return self.fetch_bom()    <-- issue BOMToken
197
198        # Note: the order of the following checks is NOT significant.
199
200        # Is it the flow sequence start indicator?
201        if ch == u'[':
202            return self.fetch_flow_sequence_start()
203
204        # Is it the flow mapping start indicator?
205        if ch == u'{':
206            return self.fetch_flow_mapping_start()
207
208        # Is it the flow sequence end indicator?
209        if ch == u']':
210            return self.fetch_flow_sequence_end()
211
212        # Is it the flow mapping end indicator?
213        if ch == u'}':
214            return self.fetch_flow_mapping_end()
215
216        # Is it the flow entry indicator?
217        if ch in u',':
218            return self.fetch_flow_entry()
219
220        # Is it the block entry indicator?
221        if ch in u'-' and self.check_block_entry():
222            return self.fetch_block_entry()
223
224        # Is it the key indicator?
225        if ch == u'?' and self.check_key():
226            return self.fetch_key()
227
228        # Is it the value indicator?
229        if ch == u':' and self.check_value():
230            return self.fetch_value()
231
232        # Is it an alias?
233        if ch == u'*':
234            return self.fetch_alias()
235
236        # Is it an anchor?
237        if ch == u'&':
238            return self.fetch_anchor()
239
240        # Is it a tag?
241        if ch == u'!':
242            return self.fetch_tag()
243
244        # Is it a literal scalar?
245        if ch == u'|' and not self.flow_level:
246            return self.fetch_literal()
247
248        # Is it a folded scalar?
249        if ch == u'>' and not self.flow_level:
250            return self.fetch_folded()
251
252        # Is it a single quoted scalar?
253        if ch == u'\'':
254            return self.fetch_single()
255
256        # Is it a double quoted scalar?
257        if ch == u'\"':
258            return self.fetch_double()
259
260        # It must be a plain scalar then.
261        if self.check_plain():
262            return self.fetch_plain()
263
264        # No? It's an error. Let's produce a nice error message.
265        raise ScannerError("while scanning for the next token", None,
266                "found character %r that cannot start any token"
267                % ch.encode('utf-8'), self.get_mark())
268
269    # Simple keys treatment.
270
271    def next_possible_simple_key(self):
272        # Return the number of the nearest possible simple key. Actually we
273        # don't need to loop through the whole dictionary. We may replace it
274        # with the following code:
275        #   if not self.possible_simple_keys:
276        #       return None
277        #   return self.possible_simple_keys[
278        #           min(self.possible_simple_keys.keys())].token_number
279        min_token_number = None
280        for level in self.possible_simple_keys:
281            key = self.possible_simple_keys[level]
282            if min_token_number is None or key.token_number < min_token_number:
283                min_token_number = key.token_number
284        return min_token_number
285
286    def stale_possible_simple_keys(self):
287        # Remove entries that are no longer possible simple keys. According to
288        # the YAML specification, simple keys
289        # - should be limited to a single line,
290        # - should be no longer than 1024 characters.
291        # Disabling this procedure will allow simple keys of any length and
292        # height (may cause problems if indentation is broken though).
293        for level in self.possible_simple_keys.keys():
294            key = self.possible_simple_keys[level]
295            if key.line != self.line  \
296                    or self.index-key.index > 1024:
297                if key.required:
298                    raise ScannerError("while scanning a simple key", key.mark,
299                            "could not found expected ':'", self.get_mark())
300                del self.possible_simple_keys[level]
301
302    def save_possible_simple_key(self):
303        # The next token may start a simple key. We check if it's possible
304        # and save its position. This function is called for
305        #   ALIAS, ANCHOR, TAG, SCALAR(flow), '[', and '{'.
306
307        # Check if a simple key is required at the current position.
308        required = not self.flow_level and self.indent == self.column
309
310        # A simple key is required only if it is the first token in the current
311        # line. Therefore it is always allowed.
312        assert self.allow_simple_key or not required
313
314        # The next token might be a simple key. Let's save it's number and
315        # position.
316        if self.allow_simple_key:
317            self.remove_possible_simple_key()
318            token_number = self.tokens_taken+len(self.tokens)
319            key = SimpleKey(token_number, required,
320                    self.index, self.line, self.column, self.get_mark())
321            self.possible_simple_keys[self.flow_level] = key
322
323    def remove_possible_simple_key(self):
324        # Remove the saved possible key position at the current flow level.
325        if self.flow_level in self.possible_simple_keys:
326            key = self.possible_simple_keys[self.flow_level]
327           
328            # I don't think it's possible, but I could be wrong.
329            assert not key.required
330            #if key.required:
331            #    raise ScannerError("while scanning a simple key", key.mark,
332            #            "could not found expected ':'", self.get_mark())
333
334    # Indentation functions.
335
336    def unwind_indent(self, column):
337
338        ## In flow context, tokens should respect indentation.
339        ## Actually the condition should be `self.indent >= column` according to
340        ## the spec. But this condition will prohibit intuitively correct
341        ## constructions such as
342        ## key : {
343        ## }
344        #if self.flow_level and self.indent > column:
345        #    raise ScannerError(None, None,
346        #            "invalid intendation or unclosed '[' or '{'",
347        #            self.get_mark())
348
349        # In the flow context, indentation is ignored. We make the scanner less
350        # restrictive then specification requires.
351        if self.flow_level:
352            return
353
354        # In block context, we may need to issue the BLOCK-END tokens.
355        while self.indent > column:
356            mark = self.get_mark()
357            self.indent = self.indents.pop()
358            self.tokens.append(BlockEndToken(mark, mark))
359
360    def add_indent(self, column):
361        # Check if we need to increase indentation.
362        if self.indent < column:
363            self.indents.append(self.indent)
364            self.indent = column
365            return True
366        return False
367
368    # Fetchers.
369
370    def fetch_stream_start(self):
371        # We always add STREAM-START as the first token and STREAM-END as the
372        # last token.
373
374        # Read the token.
375        mark = self.get_mark()
376       
377        # Add STREAM-START.
378        self.tokens.append(StreamStartToken(mark, mark,
379            encoding=self.encoding))
380       
381
382    def fetch_stream_end(self):
383
384        # Set the current intendation to -1.
385        self.unwind_indent(-1)
386
387        # Reset everything (not really needed).
388        self.allow_simple_key = False
389        self.possible_simple_keys = {}
390
391        # Read the token.
392        mark = self.get_mark()
393       
394        # Add STREAM-END.
395        self.tokens.append(StreamEndToken(mark, mark))
396
397        # The steam is finished.
398        self.done = True
399
400    def fetch_directive(self):
401       
402        # Set the current intendation to -1.
403        self.unwind_indent(-1)
404
405        # Reset simple keys.
406        self.remove_possible_simple_key()
407        self.allow_simple_key = False
408
409        # Scan and add DIRECTIVE.
410        self.tokens.append(self.scan_directive())
411
412    def fetch_document_start(self):
413        self.fetch_document_indicator(DocumentStartToken)
414
415    def fetch_document_end(self):
416        self.fetch_document_indicator(DocumentEndToken)
417
418    def fetch_document_indicator(self, TokenClass):
419
420        # Set the current intendation to -1.
421        self.unwind_indent(-1)
422
423        # Reset simple keys. Note that there could not be a block collection
424        # after '---'.
425        self.remove_possible_simple_key()
426        self.allow_simple_key = False
427
428        # Add DOCUMENT-START or DOCUMENT-END.
429        start_mark = self.get_mark()
430        self.forward(3)
431        end_mark = self.get_mark()
432        self.tokens.append(TokenClass(start_mark, end_mark))
433
434    def fetch_flow_sequence_start(self):
435        self.fetch_flow_collection_start(FlowSequenceStartToken)
436
437    def fetch_flow_mapping_start(self):
438        self.fetch_flow_collection_start(FlowMappingStartToken)
439
440    def fetch_flow_collection_start(self, TokenClass):
441
442        # '[' and '{' may start a simple key.
443        self.save_possible_simple_key()
444
445        # Increase the flow level.
446        self.flow_level += 1
447
448        # Simple keys are allowed after '[' and '{'.
449        self.allow_simple_key = True
450
451        # Add FLOW-SEQUENCE-START or FLOW-MAPPING-START.
452        start_mark = self.get_mark()
453        self.forward()
454        end_mark = self.get_mark()
455        self.tokens.append(TokenClass(start_mark, end_mark))
456
457    def fetch_flow_sequence_end(self):
458        self.fetch_flow_collection_end(FlowSequenceEndToken)
459
460    def fetch_flow_mapping_end(self):
461        self.fetch_flow_collection_end(FlowMappingEndToken)
462
463    def fetch_flow_collection_end(self, TokenClass):
464
465        # Reset possible simple key on the current level.
466        self.remove_possible_simple_key()
467
468        # Decrease the flow level.
469        self.flow_level -= 1
470
471        # No simple keys after ']' or '}'.
472        self.allow_simple_key = False
473
474        # Add FLOW-SEQUENCE-END or FLOW-MAPPING-END.
475        start_mark = self.get_mark()
476        self.forward()
477        end_mark = self.get_mark()
478        self.tokens.append(TokenClass(start_mark, end_mark))
479
480    def fetch_flow_entry(self):
481
482        # Simple keys are allowed after ','.
483        self.allow_simple_key = True
484
485        # Reset possible simple key on the current level.
486        self.remove_possible_simple_key()
487
488        # Add FLOW-ENTRY.
489        start_mark = self.get_mark()
490        self.forward()
491        end_mark = self.get_mark()
492        self.tokens.append(FlowEntryToken(start_mark, end_mark))
493
494    def fetch_block_entry(self):
495
496        # Block context needs additional checks.
497        if not self.flow_level:
498
499            # Are we allowed to start a new entry?
500            if not self.allow_simple_key:
501                raise ScannerError(None, None,
502                        "sequence entries are not allowed here",
503                        self.get_mark())
504
505            # We may need to add BLOCK-SEQUENCE-START.
506            if self.add_indent(self.column):
507                mark = self.get_mark()
508                self.tokens.append(BlockSequenceStartToken(mark, mark))
509
510        # It's an error for the block entry to occur in the flow context,
511        # but we let the parser detect this.
512        else:
513            pass
514
515        # Simple keys are allowed after '-'.
516        self.allow_simple_key = True
517
518        # Reset possible simple key on the current level.
519        self.remove_possible_simple_key()
520
521        # Add BLOCK-ENTRY.
522        start_mark = self.get_mark()
523        self.forward()
524        end_mark = self.get_mark()
525        self.tokens.append(BlockEntryToken(start_mark, end_mark))
526
527    def fetch_key(self):
528       
529        # Block context needs additional checks.
530        if not self.flow_level:
531
532            # Are we allowed to start a key (not nessesary a simple)?
533            if not self.allow_simple_key:
534                raise ScannerError(None, None,
535                        "mapping keys are not allowed here",
536                        self.get_mark())
537
538            # We may need to add BLOCK-MAPPING-START.
539            if self.add_indent(self.column):
540                mark = self.get_mark()
541                self.tokens.append(BlockMappingStartToken(mark, mark))
542
543        # Simple keys are allowed after '?' in the block context.
544        self.allow_simple_key = not self.flow_level
545
546        # Reset possible simple key on the current level.
547        self.remove_possible_simple_key()
548
549        # Add KEY.
550        start_mark = self.get_mark()
551        self.forward()
552        end_mark = self.get_mark()
553        self.tokens.append(KeyToken(start_mark, end_mark))
554
555    def fetch_value(self):
556
557        # Do we determine a simple key?
558        if self.flow_level in self.possible_simple_keys:
559
560            # Add KEY.
561            key = self.possible_simple_keys[self.flow_level]
562            del self.possible_simple_keys[self.flow_level]
563            self.tokens.insert(key.token_number-self.tokens_taken,
564                    KeyToken(key.mark, key.mark))
565
566            # If this key starts a new block mapping, we need to add
567            # BLOCK-MAPPING-START.
568            if not self.flow_level:
569                if self.add_indent(key.column):
570                    self.tokens.insert(key.token_number-self.tokens_taken,
571                            BlockMappingStartToken(key.mark, key.mark))
572
573            # There cannot be two simple keys one after another.
574            self.allow_simple_key = False
575
576        # It must be a part of a complex key.
577        else:
578           
579            # Block context needs additional checks.
580            # (Do we really need them? They will be catched by the parser
581            # anyway.)
582            if not self.flow_level:
583
584                # We are allowed to start a complex value if and only if
585                # we can start a simple key.
586                if not self.allow_simple_key:
587                    raise ScannerError(None, None,
588                            "mapping values are not allowed here",
589                            self.get_mark())
590
591            # Simple keys are allowed after ':' in the block context.
592            self.allow_simple_key = not self.flow_level
593
594            # Reset possible simple key on the current level.
595            self.remove_possible_simple_key()
596
597        # Add VALUE.
598        start_mark = self.get_mark()
599        self.forward()
600        end_mark = self.get_mark()
601        self.tokens.append(ValueToken(start_mark, end_mark))
602
603    def fetch_alias(self):
604
605        # ALIAS could be a simple key.
606        self.save_possible_simple_key()
607
608        # No simple keys after ALIAS.
609        self.allow_simple_key = False
610
611        # Scan and add ALIAS.
612        self.tokens.append(self.scan_anchor(AliasToken))
613
614    def fetch_anchor(self):
615
616        # ANCHOR could start a simple key.
617        self.save_possible_simple_key()
618
619        # No simple keys after ANCHOR.
620        self.allow_simple_key = False
621
622        # Scan and add ANCHOR.
623        self.tokens.append(self.scan_anchor(AnchorToken))
624
625    def fetch_tag(self):
626
627        # TAG could start a simple key.
628        self.save_possible_simple_key()
629
630        # No simple keys after TAG.
631        self.allow_simple_key = False
632
633        # Scan and add TAG.
634        self.tokens.append(self.scan_tag())
635
636    def fetch_literal(self):
637        self.fetch_block_scalar(style='|')
638
639    def fetch_folded(self):
640        self.fetch_block_scalar(style='>')
641
642    def fetch_block_scalar(self, style):
643
644        # A simple key may follow a block scalar.
645        self.allow_simple_key = True
646
647        # Reset possible simple key on the current level.
648        self.remove_possible_simple_key()
649
650        # Scan and add SCALAR.
651        self.tokens.append(self.scan_block_scalar(style))
652
653    def fetch_single(self):
654        self.fetch_flow_scalar(style='\'')
655
656    def fetch_double(self):
657        self.fetch_flow_scalar(style='"')
658
659    def fetch_flow_scalar(self, style):
660
661        # A flow scalar could be a simple key.
662        self.save_possible_simple_key()
663
664        # No simple keys after flow scalars.
665        self.allow_simple_key = False
666
667        # Scan and add SCALAR.
668        self.tokens.append(self.scan_flow_scalar(style))
669
670    def fetch_plain(self):
671
672        # A plain scalar could be a simple key.
673        self.save_possible_simple_key()
674
675        # No simple keys after plain scalars. But note that `scan_plain` will
676        # change this flag if the scan is finished at the beginning of the
677        # line.
678        self.allow_simple_key = False
679
680        # Scan and add SCALAR. May change `allow_simple_key`.
681        self.tokens.append(self.scan_plain())
682
683    # Checkers.
684
685    def check_directive(self):
686
687        # DIRECTIVE:        ^ '%' ...
688        # The '%' indicator is already checked.
689        if self.column == 0:
690            return True
691
692    def check_document_start(self):
693
694        # DOCUMENT-START:   ^ '---' (' '|'\n')
695        if self.column == 0:
696            if self.prefix(3) == u'---'  \
697                    and self.peek(3) in u'\0 \t\r\n\x85\u2028\u2029':
698                return True
699
700    def check_document_end(self):
701
702        # DOCUMENT-END:     ^ '...' (' '|'\n')
703        if self.column == 0:
704            if self.prefix(3) == u'...'  \
705                    and self.peek(3) in u'\0 \t\r\n\x85\u2028\u2029':
706                return True
707
708    def check_block_entry(self):
709
710        # BLOCK-ENTRY:      '-' (' '|'\n')
711        return self.peek(1) in u'\0 \t\r\n\x85\u2028\u2029'
712
713    def check_key(self):
714
715        # KEY(flow context):    '?'
716        if self.flow_level:
717            return True
718
719        # KEY(block context):   '?' (' '|'\n')
720        else:
721            return self.peek(1) in u'\0 \t\r\n\x85\u2028\u2029'
722
723    def check_value(self):
724
725        # VALUE(flow context):  ':'
726        if self.flow_level:
727            return True
728
729        # VALUE(block context): ':' (' '|'\n')
730        else:
731            return self.peek(1) in u'\0 \t\r\n\x85\u2028\u2029'
732
733    def check_plain(self):
734
735        # A plain scalar may start with any non-space character except:
736        #   '-', '?', ':', ',', '[', ']', '{', '}',
737        #   '#', '&', '*', '!', '|', '>', '\'', '\"',
738        #   '%', '@', '`'.
739        #
740        # It may also start with
741        #   '-', '?', ':'
742        # if it is followed by a non-space character.
743        #
744        # Note that we limit the last rule to the block context (except the
745        # '-' character) because we want the flow context to be space
746        # independent.
747        ch = self.peek()
748        return ch not in u'\0 \t\r\n\x85\u2028\u2029-?:,[]{}#&*!|>\'\"%@`'  \
749                or (self.peek(1) not in u'\0 \t\r\n\x85\u2028\u2029'
750                        and (ch == u'-' or (not self.flow_level and ch in u'?:')))
751
752    # Scanners.
753
754    def scan_to_next_token(self):
755        # We ignore spaces, line breaks and comments.
756        # If we find a line break in the block context, we set the flag
757        # `allow_simple_key` on.
758        # The byte order mark is stripped if it's the first character in the
759        # stream. We do not yet support BOM inside the stream as the
760        # specification requires. Any such mark will be considered as a part
761        # of the document.
762        #
763        # TODO: We need to make tab handling rules more sane. A good rule is
764        #   Tabs cannot precede tokens
765        #   BLOCK-SEQUENCE-START, BLOCK-MAPPING-START, BLOCK-END,
766        #   KEY(block), VALUE(block), BLOCK-ENTRY
767        # So the checking code is
768        #   if <TAB>:
769        #       self.allow_simple_keys = False
770        # We also need to add the check for `allow_simple_keys == True` to
771        # `unwind_indent` before issuing BLOCK-END.
772        # Scanners for block, flow, and plain scalars need to be modified.
773
774        if self.index == 0 and self.peek() == u'\uFEFF':
775            self.forward()
776        found = False
777        while not found:
778            while self.peek() == u' ':
779                self.forward()
780            if self.peek() == u'#':
781                while self.peek() not in u'\0\r\n\x85\u2028\u2029':
782                    self.forward()
783            if self.scan_line_break():
784                if not self.flow_level:
785                    self.allow_simple_key = True
786            else:
787                found = True
788
789    def scan_directive(self):
790        # See the specification for details.
791        start_mark = self.get_mark()
792        self.forward()
793        name = self.scan_directive_name(start_mark)
794        value = None
795        if name == u'YAML':
796            value = self.scan_yaml_directive_value(start_mark)
797            end_mark = self.get_mark()
798        elif name == u'TAG':
799            value = self.scan_tag_directive_value(start_mark)
800            end_mark = self.get_mark()
801        else:
802            end_mark = self.get_mark()
803            while self.peek() not in u'\0\r\n\x85\u2028\u2029':
804                self.forward()
805        self.scan_directive_ignored_line(start_mark)
806        return DirectiveToken(name, value, start_mark, end_mark)
807
808    def scan_directive_name(self, start_mark):
809        # See the specification for details.
810        length = 0
811        ch = self.peek(length)
812        while u'0' <= ch <= u'9' or u'A' <= ch <= 'Z' or u'a' <= ch <= 'z'  \
813                or ch in u'-_':
814            length += 1
815            ch = self.peek(length)
816        if not length:
817            raise ScannerError("while scanning a directive", start_mark,
818                    "expected alphabetic or numeric character, but found %r"
819                    % ch.encode('utf-8'), self.get_mark())
820        value = self.prefix(length)
821        self.forward(length)
822        ch = self.peek()
823        if ch not in u'\0 \r\n\x85\u2028\u2029':
824            raise ScannerError("while scanning a directive", start_mark,
825                    "expected alphabetic or numeric character, but found %r"
826                    % ch.encode('utf-8'), self.get_mark())
827        return value
828
829    def scan_yaml_directive_value(self, start_mark):
830        # See the specification for details.
831        while self.peek() == u' ':
832            self.forward()
833        major = self.scan_yaml_directive_number(start_mark)
834        if self.peek() != '.':
835            raise ScannerError("while scanning a directive", start_mark,
836                    "expected a digit or '.', but found %r"
837                    % self.peek().encode('utf-8'),
838                    self.get_mark())
839        self.forward()
840        minor = self.scan_yaml_directive_number(start_mark)
841        if self.peek() not in u'\0 \r\n\x85\u2028\u2029':
842            raise ScannerError("while scanning a directive", start_mark,
843                    "expected a digit or ' ', but found %r"
844                    % self.peek().encode('utf-8'),
845                    self.get_mark())
846        return (major, minor)
847
848    def scan_yaml_directive_number(self, start_mark):
849        # See the specification for details.
850        ch = self.peek()
851        if not (u'0' <= ch <= '9'):
852            raise ScannerError("while scanning a directive", start_mark,
853                    "expected a digit, but found %r" % ch.encode('utf-8'),
854                    self.get_mark())
855        length = 0
856        while u'0' <= self.peek(length) <= u'9':
857            length += 1
858        value = int(self.prefix(length))
859        self.forward(length)
860        return value
861
862    def scan_tag_directive_value(self, start_mark):
863        # See the specification for details.
864        while self.peek() == u' ':
865            self.forward()
866        handle = self.scan_tag_directive_handle(start_mark)
867        while self.peek() == u' ':
868            self.forward()
869        prefix = self.scan_tag_directive_prefix(start_mark)
870        return (handle, prefix)
871
872    def scan_tag_directive_handle(self, start_mark):
873        # See the specification for details.
874        value = self.scan_tag_handle('directive', start_mark)
875        ch = self.peek()
876        if ch != u' ':
877            raise ScannerError("while scanning a directive", start_mark,
878                    "expected ' ', but found %r" % ch.encode('utf-8'),
879                    self.get_mark())
880        return value
881
882    def scan_tag_directive_prefix(self, start_mark):
883        # See the specification for details.
884        value = self.scan_tag_uri('directive', start_mark)
885        ch = self.peek()
886        if ch not in u'\0 \r\n\x85\u2028\u2029':
887            raise ScannerError("while scanning a directive", start_mark,
888                    "expected ' ', but found %r" % ch.encode('utf-8'),
889                    self.get_mark())
890        return value
891
892    def scan_directive_ignored_line(self, start_mark):
893        # See the specification for details.
894        while self.peek() == u' ':
895            self.forward()
896        if self.peek() == u'#':
897            while self.peek() not in u'\0\r\n\x85\u2028\u2029':
898                self.forward()
899        ch = self.peek()
900        if ch not in u'\0\r\n\x85\u2028\u2029':
901            raise ScannerError("while scanning a directive", start_mark,
902                    "expected a comment or a line break, but found %r"
903                        % ch.encode('utf-8'), self.get_mark())
904        self.scan_line_break()
905
906    def scan_anchor(self, TokenClass):
907        # The specification does not restrict characters for anchors and
908        # aliases. This may lead to problems, for instance, the document:
909        #   [ *alias, value ]
910        # can be interpteted in two ways, as
911        #   [ "value" ]
912        # and
913        #   [ *alias , "value" ]
914        # Therefore we restrict aliases to numbers and ASCII letters.
915        start_mark = self.get_mark()
916        indicator = self.peek()
917        if indicator == '*':
918            name = 'alias'
919        else:
920            name = 'anchor'
921        self.forward()
922        length = 0
923        ch = self.peek(length)
924        while u'0' <= ch <= u'9' or u'A' <= ch <= 'Z' or u'a' <= ch <= 'z'  \
925                or ch in u'-_':
926            length += 1
927            ch = self.peek(length)
928        if not length:
929            raise ScannerError("while scanning an %s" % name, start_mark,
930                    "expected alphabetic or numeric character, but found %r"
931                    % ch.encode('utf-8'), self.get_mark())
932        value = self.prefix(length)
933        self.forward(length)
934        ch = self.peek()
935        if ch not in u'\0 \t\r\n\x85\u2028\u2029?:,]}%@`':
936            raise ScannerError("while scanning an %s" % name, start_mark,
937                    "expected alphabetic or numeric character, but found %r"
938                    % ch.encode('utf-8'), self.get_mark())
939        end_mark = self.get_mark()
940        return TokenClass(value, start_mark, end_mark)
941
942    def scan_tag(self):
943        # See the specification for details.
944        start_mark = self.get_mark()
945        ch = self.peek(1)
946        if ch == u'<':
947            handle = None
948            self.forward(2)
949            suffix = self.scan_tag_uri('tag', start_mark)
950            if self.peek() != u'>':
951                raise ScannerError("while parsing a tag", start_mark,
952                        "expected '>', but found %r" % self.peek().encode('utf-8'),
953                        self.get_mark())
954            self.forward()
955        elif ch in u'\0 \t\r\n\x85\u2028\u2029':
956            handle = None
957            suffix = u'!'
958            self.forward()
959        else:
960            length = 1
961            use_handle = False
962            while ch not in u'\0 \r\n\x85\u2028\u2029':
963                if ch == u'!':
964                    use_handle = True
965                    break
966                length += 1
967                ch = self.peek(length)
968            handle = u'!'
969            if use_handle:
970                handle = self.scan_tag_handle('tag', start_mark)
971            else:
972                handle = u'!'
973                self.forward()
974            suffix = self.scan_tag_uri('tag', start_mark)
975        ch = self.peek()
976        if ch not in u'\0 \r\n\x85\u2028\u2029':
977            raise ScannerError("while scanning a tag", start_mark,
978                    "expected ' ', but found %r" % ch.encode('utf-8'),
979                    self.get_mark())
980        value = (handle, suffix)
981        end_mark = self.get_mark()
982        return TagToken(value, start_mark, end_mark)
983
984    def scan_block_scalar(self, style):
985        # See the specification for details.
986
987        if style == '>':
988            folded = True
989        else:
990            folded = False
991
992        chunks = []
993        start_mark = self.get_mark()
994
995        # Scan the header.
996        self.forward()
997        chomping, increment = self.scan_block_scalar_indicators(start_mark)
998        self.scan_block_scalar_ignored_line(start_mark)
999
1000        # Determine the indentation level and go to the first non-empty line.
1001        min_indent = self.indent+1
1002        if min_indent < 1:
1003            min_indent = 1
1004        if increment is None:
1005            breaks, max_indent, end_mark = self.scan_block_scalar_indentation()
1006            indent = max(min_indent, max_indent)
1007        else:
1008            indent = min_indent+increment-1
1009            breaks, end_mark = self.scan_block_scalar_breaks(indent)
1010        line_break = u''
1011
1012        # Scan the inner part of the block scalar.
1013        while self.column == indent and self.peek() != u'\0':
1014            chunks.extend(breaks)
1015            leading_non_space = self.peek() not in u' \t'
1016            length = 0
1017            while self.peek(length) not in u'\0\r\n\x85\u2028\u2029':
1018                length += 1
1019            chunks.append(self.prefix(length))
1020            self.forward(length)
1021            line_break = self.scan_line_break()
1022            breaks, end_mark = self.scan_block_scalar_breaks(indent)
1023            if self.column == indent and self.peek() != u'\0':
1024
1025                # Unfortunately, folding rules are ambiguous.
1026                #
1027                # This is the folding according to the specification:
1028               
1029                if folded and line_break == u'\n'   \
1030                        and leading_non_space and self.peek() not in u' \t':
1031                    if not breaks:
1032                        chunks.append(u' ')
1033                else:
1034                    chunks.append(line_break)
1035               
1036                # This is Clark Evans's interpretation (also in the spec
1037                # examples):
1038                #
1039                #if folded and line_break == u'\n':
1040                #    if not breaks:
1041                #        if self.peek() not in ' \t':
1042                #            chunks.append(u' ')
1043                #        else:
1044                #            chunks.append(line_break)
1045                #else:
1046                #    chunks.append(line_break)
1047            else:
1048                break
1049
1050        # Chomp the tail.
1051        if chomping is not False:
1052            chunks.append(line_break)
1053        if chomping is True:
1054            chunks.extend(breaks)
1055
1056        # We are done.
1057        return ScalarToken(u''.join(chunks), False, start_mark, end_mark,
1058                style)
1059
1060    def scan_block_scalar_indicators(self, start_mark):
1061        # See the specification for details.
1062        chomping = None
1063        increment = None
1064        ch = self.peek()
1065        if ch in u'+-':
1066            if ch == '+':
1067                chomping = True
1068            else:
1069                chomping = False
1070            self.forward()
1071            ch = self.peek()
1072            if ch in u'0123456789':
1073                increment = int(ch)
1074                if increment == 0:
1075                    raise ScannerError("while scanning a block scalar", start_mark,
1076                            "expected indentation indicator in the range 1-9, but found 0",
1077                            self.get_mark())
1078                self.forward()
1079        elif ch in u'0123456789':
1080            increment = int(ch)
1081            if increment == 0:
1082                raise ScannerError("while scanning a block scalar", start_mark,
1083                        "expected indentation indicator in the range 1-9, but found 0",
1084                        self.get_mark())
1085            self.forward()
1086            ch = self.peek()
1087            if ch in u'+-':
1088                if ch == '+':
1089                    chomping = True
1090                else:
1091                    chomping = False
1092                self.forward()
1093        ch = self.peek()
1094        if ch not in u'\0 \r\n\x85\u2028\u2029':
1095            raise ScannerError("while scanning a block scalar", start_mark,
1096                    "expected chomping or indentation indicators, but found %r"
1097                        % ch.encode('utf-8'), self.get_mark())
1098        return chomping, increment
1099
1100    def scan_block_scalar_ignored_line(self, start_mark):
1101        # See the specification for details.
1102        while self.peek() == u' ':
1103            self.forward()
1104        if self.peek() == u'#':
1105            while self.peek() not in u'\0\r\n\x85\u2028\u2029':
1106                self.forward()
1107        ch = self.peek()
1108        if ch not in u'\0\r\n\x85\u2028\u2029':
1109            raise ScannerError("while scanning a block scalar", start_mark,
1110                    "expected a comment or a line break, but found %r"
1111                        % ch.encode('utf-8'), self.get_mark())
1112        self.scan_line_break()
1113
1114    def scan_block_scalar_indentation(self):
1115        # See the specification for details.
1116        chunks = []
1117        max_indent = 0
1118        end_mark = self.get_mark()
1119        while self.peek() in u' \r\n\x85\u2028\u2029':
1120            if self.peek() != u' ':
1121                chunks.append(self.scan_line_break())
1122                end_mark = self.get_mark()
1123            else:
1124                self.forward()
1125                if self.column > max_indent:
1126                    max_indent = self.column
1127        return chunks, max_indent, end_mark
1128
1129    def scan_block_scalar_breaks(self, indent):
1130        # See the specification for details.
1131        chunks = []
1132        end_mark = self.get_mark()
1133        while self.column < indent and self.peek() == u' ':
1134            self.forward()
1135        while self.peek() in u'\r\n\x85\u2028\u2029':
1136            chunks.append(self.scan_line_break())
1137            end_mark = self.get_mark()
1138            while self.column < indent and self.peek() == u' ':
1139                self.forward()
1140        return chunks, end_mark
1141
1142    def scan_flow_scalar(self, style):
1143        # See the specification for details.
1144        # Note that we loose indentation rules for quoted scalars. Quoted
1145        # scalars don't need to adhere indentation because " and ' clearly
1146        # mark the beginning and the end of them. Therefore we are less
1147        # restrictive then the specification requires. We only need to check
1148        # that document separators are not included in scalars.
1149        if style == '"':
1150            double = True
1151        else:
1152            double = False
1153        chunks = []
1154        start_mark = self.get_mark()
1155        quote = self.peek()
1156        self.forward()
1157        chunks.extend(self.scan_flow_scalar_non_spaces(double, start_mark))
1158        while self.peek() != quote:
1159            chunks.extend(self.scan_flow_scalar_spaces(double, start_mark))
1160            chunks.extend(self.scan_flow_scalar_non_spaces(double, start_mark))
1161        self.forward()
1162        end_mark = self.get_mark()
1163        return ScalarToken(u''.join(chunks), False, start_mark, end_mark,
1164                style)
1165
1166    ESCAPE_REPLACEMENTS = {
1167        u'0':   u'\0',
1168        u'a':   u'\x07',
1169        u'b':   u'\x08',
1170        u't':   u'\x09',
1171        u'\t':  u'\x09',
1172        u'n':   u'\x0A',
1173        u'v':   u'\x0B',
1174        u'f':   u'\x0C',
1175        u'r':   u'\x0D',
1176        u'e':   u'\x1B',
1177        u' ':   u'\x20',
1178        u'\"':  u'\"',
1179        u'\\':  u'\\',
1180        u'N':   u'\x85',
1181        u'_':   u'\xA0',
1182        u'L':   u'\u2028',
1183        u'P':   u'\u2029',
1184    }
1185
1186    ESCAPE_CODES = {
1187        u'x':   2,
1188        u'u':   4,
1189        u'U':   8,
1190    }
1191
1192    def scan_flow_scalar_non_spaces(self, double, start_mark):
1193        # See the specification for details.
1194        chunks = []
1195        while True:
1196            length = 0
1197            while self.peek(length) not in u'\'\"\\\0 \t\r\n\x85\u2028\u2029':
1198                length += 1
1199            if length:
1200                chunks.append(self.prefix(length))
1201                self.forward(length)
1202            ch = self.peek()
1203            if not double and ch == u'\'' and self.peek(1) == u'\'':
1204                chunks.append(u'\'')
1205                self.forward(2)
1206            elif (double and ch == u'\'') or (not double and ch in u'\"\\'):
1207                chunks.append(ch)
1208                self.forward()
1209            elif double and ch == u'\\':
1210                self.forward()
1211                ch = self.peek()
1212                if ch in self.ESCAPE_REPLACEMENTS:
1213                    chunks.append(self.ESCAPE_REPLACEMENTS[ch])
1214                    self.forward()
1215                elif ch in self.ESCAPE_CODES:
1216                    length = self.ESCAPE_CODES[ch]
1217                    self.forward()
1218                    for k in range(length):
1219                        if self.peek(k) not in u'0123456789ABCDEFabcdef':
1220                            raise ScannerError("while scanning a double-quoted scalar", start_mark,
1221                                    "expected escape sequence of %d hexdecimal numbers, but found %r" %
1222                                        (length, self.peek(k).encode('utf-8')), self.get_mark())
1223                    code = int(self.prefix(length), 16)
1224                    chunks.append(unichr(code))
1225                    self.forward(length)
1226                elif ch in u'\r\n\x85\u2028\u2029':
1227                    self.scan_line_break()
1228                    chunks.extend(self.scan_flow_scalar_breaks(double, start_mark))
1229                else:
1230                    raise ScannerError("while scanning a double-quoted scalar", start_mark,
1231                            "found unknown escape character %r" % ch.encode('utf-8'), self.get_mark())
1232            else:
1233                return chunks
1234
1235    def scan_flow_scalar_spaces(self, double, start_mark):
1236        # See the specification for details.
1237        chunks = []
1238        length = 0
1239        while self.peek(length) in u' \t':
1240            length += 1
1241        whitespaces = self.prefix(length)
1242        self.forward(length)
1243        ch = self.peek()
1244        if ch == u'\0':
1245            raise ScannerError("while scanning a quoted scalar", start_mark,
1246                    "found unexpected end of stream", self.get_mark())
1247        elif ch in u'\r\n\x85\u2028\u2029':
1248            line_break = self.scan_line_break()
1249            breaks = self.scan_flow_scalar_breaks(double, start_mark)
1250            if line_break != u'\n':
1251                chunks.append(line_break)
1252            elif not breaks:
1253                chunks.append(u' ')
1254            chunks.extend(breaks)
1255        else:
1256            chunks.append(whitespaces)
1257        return chunks
1258
1259    def scan_flow_scalar_breaks(self, double, start_mark):
1260        # See the specification for details.
1261        chunks = []
1262        while True:
1263            # Instead of checking indentation, we check for document
1264            # separators.
1265            prefix = self.prefix(3)
1266            if (prefix == u'---' or prefix == u'...')   \
1267                    and self.peek(3) in u'\0 \t\r\n\x85\u2028\u2029':
1268                raise ScannerError("while scanning a quoted scalar", start_mark,
1269                        "found unexpected document separator", self.get_mark())
1270            while self.peek() in u' \t':
1271                self.forward()
1272            if self.peek() in u'\r\n\x85\u2028\u2029':
1273                chunks.append(self.scan_line_break())
1274            else:
1275                return chunks
1276
1277    def scan_plain(self):
1278        # See the specification for details.
1279        # We add an additional restriction for the flow context:
1280        #   plain scalars in the flow context cannot contain ',', ':' and '?'.
1281        # We also keep track of the `allow_simple_key` flag here.
1282        # Indentation rules are loosed for the flow context.
1283        chunks = []
1284        start_mark = self.get_mark()
1285        end_mark = start_mark
1286        indent = self.indent+1
1287        # We allow zero indentation for scalars, but then we need to check for
1288        # document separators at the beginning of the line.
1289        #if indent == 0:
1290        #    indent = 1
1291        spaces = []
1292        while True:
1293            length = 0
1294            if self.peek() == u'#':
1295                break
1296            while True:
1297                ch = self.peek(length)
1298                if ch in u'\0 \t\r\n\x85\u2028\u2029'   \
1299                        or (not self.flow_level and ch == u':' and
1300                                self.peek(length+1) in u'\0 \t\r\n\x28\u2028\u2029') \
1301                        or (self.flow_level and ch in u',:?[]{}'):
1302                    break
1303                length += 1
1304            # It's not clear what we should do with ':' in the flow context.
1305            if (self.flow_level and ch == u':'
1306                    and self.peek(length+1) not in u'\0 \t\r\n\x28\u2028\u2029,[]{}'):
1307                self.forward(length)
1308                raise ScannerError("while scanning a plain scalar", start_mark,
1309                    "found unexpected ':'", self.get_mark(),
1310                    "Please check http://pyyaml.org/wiki/YAMLColonInFlowContext for details.")
1311            if length == 0:
1312                break
1313            self.allow_simple_key = False
1314            chunks.extend(spaces)
1315            chunks.append(self.prefix(length))
1316            self.forward(length)
1317            end_mark = self.get_mark()
1318            spaces = self.scan_plain_spaces(indent, start_mark)
1319            if not spaces or self.peek() == u'#' \
1320                    or (not self.flow_level and self.column < indent):
1321                break
1322        return ScalarToken(u''.join(chunks), True, start_mark, end_mark)
1323
1324    def scan_plain_spaces(self, indent, start_mark):
1325        # See the specification for details.
1326        # The specification is really confusing about tabs in plain scalars.
1327        # We just forbid them completely. Do not use tabs in YAML!
1328        chunks = []
1329        length = 0
1330        while self.peek(length) in u' ':
1331            length += 1
1332        whitespaces = self.prefix(length)
1333        self.forward(length)
1334        ch = self.peek()
1335        if ch in u'\r\n\x85\u2028\u2029':
1336            line_break = self.scan_line_break()
1337            self.allow_simple_key = True
1338            prefix = self.prefix(3)
1339            if (prefix == u'---' or prefix == u'...')   \
1340                    and self.peek(3) in u'\0 \t\r\n\x85\u2028\u2029':
1341                return
1342            breaks = []
1343            while self.peek() in u' \r\n\x85\u2028\u2029':
1344                if self.peek() == ' ':
1345                    self.forward()
1346                else:
1347                    breaks.append(self.scan_line_break())
1348                    prefix = self.prefix(3)
1349                    if (prefix == u'---' or prefix == u'...')   \
1350                            and self.peek(3) in u'\0 \t\r\n\x85\u2028\u2029':
1351                        return
1352            if line_break != u'\n':
1353                chunks.append(line_break)
1354            elif not breaks:
1355                chunks.append(u' ')
1356            chunks.extend(breaks)
1357        elif whitespaces:
1358            chunks.append(whitespaces)
1359        return chunks
1360
1361    def scan_tag_handle(self, name, start_mark):
1362        # See the specification for details.
1363        # For some strange reasons, the specification does not allow '_' in
1364        # tag handles. I have allowed it anyway.
1365        ch = self.peek()
1366        if ch != u'!':
1367            raise ScannerError("while scanning a %s" % name, start_mark,
1368                    "expected '!', but found %r" % ch.encode('utf-8'),
1369                    self.get_mark())
1370        length = 1
1371        ch = self.peek(length)
1372        if ch != u' ':
1373            while u'0' <= ch <= u'9' or u'A' <= ch <= 'Z' or u'a' <= ch <= 'z'  \
1374                    or ch in u'-_':
1375                length += 1
1376                ch = self.peek(length)
1377            if ch != u'!':
1378                self.forward(length)
1379                raise ScannerError("while scanning a %s" % name, start_mark,
1380                        "expected '!', but found %r" % ch.encode('utf-8'),
1381                        self.get_mark())
1382            length += 1
1383        value = self.prefix(length)
1384        self.forward(length)
1385        return value
1386
1387    def scan_tag_uri(self, name, start_mark):
1388        # See the specification for details.
1389        # Note: we do not check if URI is well-formed.
1390        chunks = []
1391        length = 0
1392        ch = self.peek(length)
1393        while u'0' <= ch <= u'9' or u'A' <= ch <= 'Z' or u'a' <= ch <= 'z'  \
1394                or ch in u'-;/?:@&=+$,_.!~*\'()[]%':
1395            if ch == u'%':
1396                chunks.append(self.prefix(length))
1397                self.forward(length)
1398                length = 0
1399                chunks.append(self.scan_uri_escapes(name, start_mark))
1400            else:
1401                length += 1
1402            ch = self.peek(length)
1403        if length:
1404            chunks.append(self.prefix(length))
1405            self.forward(length)
1406            length = 0
1407        if not chunks:
1408            raise ScannerError("while parsing a %s" % name, start_mark,
1409                    "expected URI, but found %r" % ch.encode('utf-8'),
1410                    self.get_mark())
1411        return u''.join(chunks)
1412
1413    def scan_uri_escapes(self, name, start_mark):
1414        # See the specification for details.
1415        bytes = []
1416        mark = self.get_mark()
1417        while self.peek() == u'%':
1418            self.forward()
1419            for k in range(2):
1420                if self.peek(k) not in u'0123456789ABCDEFabcdef':
1421                    raise ScannerError("while scanning a %s" % name, start_mark,
1422                            "expected URI escape sequence of 2 hexdecimal numbers, but found %r" %
1423                                (self.peek(k).encode('utf-8')), self.get_mark())
1424            bytes.append(chr(int(self.prefix(2), 16)))
1425            self.forward(2)
1426        try:
1427            value = unicode(''.join(bytes), 'utf-8')
1428        except UnicodeDecodeError, exc:
1429            raise ScannerError("while scanning a %s" % name, start_mark, str(exc), mark)
1430        return value
1431
1432    def scan_line_break(self):
1433        # Transforms:
1434        #   '\r\n'      :   '\n'
1435        #   '\r'        :   '\n'
1436        #   '\n'        :   '\n'
1437        #   '\x85'      :   '\n'
1438        #   '\u2028'    :   '\u2028'
1439        #   '\u2029     :   '\u2029'
1440        #   default     :   ''
1441        ch = self.peek()
1442        if ch in u'\r\n\x85':
1443            if self.prefix(2) == u'\r\n':
1444                self.forward(2)
1445            else:
1446                self.forward()
1447            return u'\n'
1448        elif ch in u'\u2028\u2029':
1449            self.forward()
1450            return ch
1451        return u''
1452
1453#try:
1454#    import psyco
1455#    psyco.bind(Scanner)
1456#except ImportError:
1457#    pass
1458
Note: See TracBrowser for help on using the repository browser.