source: branches/pyyaml3000/lib/yaml/scanner.py @ 47

Revision 47, 27.5 KB checked in by xi, 7 years ago (diff)

Working on the scanner.
Line 
1
2# Tokens:
3# YAML-DIRECTIVE(major_version, minor_version), TAG-DIRECTIVE(handle, prefix)
4# RESERVED-DIRECTIVE(name)
5# DOCUMENT-START, DOCUMENT-END
6# BLOCK-SEQUENCE-START, BLOCK-MAPPING-START, BLOCK-END
7# FLOW-SEQUENCE-START, FLOW-MAPPING-START, FLOW-SEQUENCE-END, FLOW-MAPPING-END
8# ENTRY, KEY, VALUE
9# ALIAS(name), ANCHOR(name), TAG(value), SCALAR(value, plain)
10
11__all__ = ['Scanner', 'ScannerError']
12
13from error import YAMLError
14from tokens import *
15
16class ScannerError(YAMLError):
17    # TODO:
18    # ScannerError: while reading a quoted string
19    #         in '...', line 5, column 10:
20    # key: "valu\?e"
21    #      ^
22    # got unknown quote character '?'
23    #         in '...', line 5, column 15:
24    # key: "valu\?e"
25    #            ^
26    def __init__(self, context=None, context_marker=None,
27            problem=None, problem_marker=None, description=None):
28        self.context = context
29        self.context_marker = context_marker
30        self.problem = problem
31        self.problem_marker = problem_marker
32        self.description = description
33
34    def __str__(self):
35        lines = []
36        for (place, marker) in [(self.context, self.context_marker),
37                                (self.problem, self.problem_marker)]:
38            if place is not None:
39                lines.append(place)
40                if marker is not None:
41                    lines.append(str(marker))
42        if self.description is not None:
43            lines.append(self.description)
44        return '\n'.join(lines)
45
46class SimpleKey:
47    def __init__(self, token_number, required, index, line, column, marker):
48        self.token_number = token_number
49        self.required = required
50        self.index = index
51        self.line = line
52        self.column = column
53        self.marker = marker
54
55class Scanner:
56
57
58    def __init__(self, reader):
59        """Initialize the scanner."""
60        # The input stream. The Reader class do the dirty work of checking for
61        # BOM and converting the input data to Unicode. It also adds NUL to
62        # the end.
63        #
64        # Reader supports the following methods
65        #   self.reader.peek(k=1)   # peek the next k characters
66        #   self.reader.forward(k=1)   # read the next k characters and move the
67        #                           # pointer
68        self.reader = reader
69
70        # Had we reached the end of the stream?
71        self.done = False
72
73        # The number of unclosed '{' and '['. `flow_level == 0` means block
74        # context.
75        self.flow_level = 0
76
77        # List of processed tokens that are not yet emitted.
78        self.tokens = []
79
80        # Number of tokens that were emitted through the `get_token` method.
81        self.tokens_taken = 0
82
83        # The current indentation level.
84        self.indent = -1
85
86        # Past indentation levels.
87        self.indents = []
88
89        # Variables related to simple keys treatment.
90
91        # A simple key is a key that is not denoted by the '?' indicator.
92        # Example of simple keys:
93        #   ---
94        #   block simple key: value
95        #   ? not a simple key:
96        #   : { flow simple key: value }
97        # We emit the KEY token before all keys, so when we find a potential
98        # simple key, we try to locate the corresponding ':' indicator.
99        # Simple keys should be limited to a single line and 1024 characters.
100
101        # Can a simple key start at the current position? A simple key may
102        # start:
103        # - at the beginning of the line, not counting indentation spaces
104        #       (in block context),
105        # - after '{', '[', ',' (in the flow context),
106        # - after '?', ':', '-' (in the block context).
107        # In the block context, this flag also signify if a block collection
108        # may start at the current position.
109        self.allow_simple_key = True
110
111        # Keep track of possible simple keys. This is a dictionary. The key
112        # is `flow_level`; there can be no more that one possible simple key
113        # for each level. The value is a SimpleKey record:
114        #   (token_number, required, index, line, column, marker)
115        # A simple key may start with ALIAS, ANCHOR, TAG, SCALAR(flow),
116        # '[', or '{' tokens.
117        self.possible_simple_keys = {}
118
119    # Two public methods.
120
121    def peek_token(self):
122        """Get the current token."""
123        while self.need_more_tokens():
124            self.fetch_more_tokens()
125        if self.tokens:
126            return self.tokens[0]
127
128    def get_token(self):
129        "Get the current token and remove it from the list of pending tokens."""
130        while self.need_more_tokens():
131            self.fetch_more_tokens()
132        if self.tokens:
133            self.tokens_taken += 1
134            return self.tokens.pop(0)
135
136    # Private methods.
137
138    def need_more_tokens(self):
139        if self.done:
140            return False
141        if not self.tokens:
142            return True
143        # The current token may be a potential simple key, so we
144        # need to look further.
145        self.stale_possible_simple_keys()
146        if self.next_possible_simple_key() == self.tokens_taken:
147            return True
148
149    def fetch_more_tokens(self):
150
151        # Eat whitespaces and comments until we reach the next token.
152        self.scan_to_next_token()
153
154        # Remove obsolete possible simple keys.
155        self.stale_possible_simple_keys()
156
157        # Compare the current indentation and column. It may add some tokens
158        # and decrease the current indentation level.
159        self.unwind_indent(self.reader.column)
160
161        # Peek the next character.
162        ch = self.reader.peek()
163
164        # Is it the end of reader?
165        if ch == u'\0':
166            return self.fetch_end()
167
168        # Is it a directive?
169        if ch == u'%' and self.check_directive():
170            return self.fetch_directive()
171
172        # Is it the document start?
173        if ch == u'-' and self.check_document_start():
174            return self.fetch_document_start()
175
176        # Is it the document end?
177        if ch == u'.' and self.check_document_end():
178            return self.fetch_document_end()
179
180        # Note: the order of the following checks is NOT significant.
181
182        # Is it the flow sequence start indicator?
183        if ch == u'[':
184            return self.fetch_flow_sequence_start()
185
186        # Is it the flow mapping start indicator?
187        if ch == u'{':
188            return self.fetch_flow_mapping_start()
189
190        # Is it the flow sequence end indicator?
191        if ch == u']':
192            return self.fetch_flow_sequence_end()
193
194        # Is it the flow mapping end indicator?
195        if ch == u'}':
196            return self.fetch_flow_mapping_end()
197
198        # Is it the entry indicator?
199        if ch in u'-,' and self.check_entry():
200            return self.fetch_entry()
201
202        # Is it the key indicator?
203        if ch == u'?' and self.check_key():
204            return self.fetch_key()
205
206        # Is it the value indicator?
207        if ch == u':' and self.check_value():
208            return self.fetch_value()
209
210        # Is it an alias?
211        if ch == u'*':
212            return self.fetch_alias()
213
214        # Is it an anchor?
215        if ch == u'&':
216            return self.fetch_anchor()
217
218        # Is it a tag?
219        if ch == u'!':
220            return self.fetch_tag()
221
222        # Is it a literal scalar?
223        if ch == u'|' and not self.flow_level:
224            return self.fetch_literal()
225
226        # Is it a folded scalar?
227        if ch == u'>' and not self.flow_level:
228            return self.fetch_folded()
229
230        # Is it a single quoted scalar?
231        if ch == u'\'':
232            return self.fetch_single()
233
234        # Is it a double quoted scalar?
235        if ch == u'\"':
236            return self.fetch_double()
237
238        # It must be a plain scalar then.
239        if self.check_plain():
240            return self.fetch_plain()
241
242        # No? It's an error. Let's produce a nice error message.
243        self.invalid_token()
244
245    # Simple keys treatment.
246
247    def next_possible_simple_key(self):
248        # Return the number of the nearest possible simple key. Actually we
249        # don't need to loop through the whole dictionary. We may replace it
250        # with the following code:
251        #   if not self.possible_simple_keys:
252        #       return None
253        #   return self.possible_simple_keys[
254        #           min(self.possible_simple_keys.keys())].token_number
255        min_token_number = None
256        for level in self.possible_simple_keys:
257            key = self.possible_simple_keys[level]
258            if min_token_number is None or key.token_number < min_token_number:
259                min_token_number = key.token_number
260        return min_token_number
261
262    def stale_possible_simple_keys(self):
263        # Remove entries that are no longer possible simple keys. According to
264        # the YAML specification, simple keys
265        # - should be limited to a single line,
266        # - should be no longer than 1024 characters.
267        # Disabling this procedure will allow simple keys of any length and
268        # height (may cause problems if indentation is broken though).
269        for level in self.possible_simple_keys.keys():
270            key = self.possible_simple_keys[level]
271            if key.line != self.reader.line  \
272                    or self.reader.index-key.index > 1024:
273                if key.required:
274                    raise ScannerError("while scanning a simple key", key.marker,
275                            "could not found expected ':'", self.reader.get_marker())
276                del self.possible_simple_keys[level]
277
278    def save_possible_simple_key(self):
279        # The next token may start a simple key. We check if it's possible
280        # and save its position. This function is called for
281        #   ALIAS, ANCHOR, TAG, SCALAR(flow), '[', and '{'.
282
283        # Check if a simple key is required at the current position.
284        required = not self.flow_level and self.indent == self.reader.column
285
286        # A simple key is required only if it is the first token in the current
287        # line. Therefore it is always allowed.
288        assert self.allow_simple_key or not required
289
290        # The next token might be a simple key. Let's save it's number and
291        # position.
292        if self.allow_simple_key:
293            self.remove_possible_simple_key()
294            token_number = self.tokens_taken+len(self.tokens)
295            index = self.reader.index
296            line = self.reader.line
297            column = self.reader.column
298            marker = self.reader.get_marker()
299            key = SimpleKey(token_number, required,
300                    index, line, column, marker)
301            self.possible_simple_keys[self.flow_level] = key
302
303    def remove_possible_simple_key(self):
304        # Remove the saved possible key position at the current flow level.
305        if self.flow_level in self.possible_simple_keys:
306            key = self.possible_simple_keys[self.flow_level]
307           
308            # I don't think it's possible, but I could be wrong.
309            assert not key.required
310            #if key.required:
311            #    raise ScannerError("while scanning a simple key", key.marker,
312            #            "could not found expected ':'", self.reader.get_marker())
313
314    # Indentation functions.
315
316    def unwind_indent(self, column):
317
318        # In flow context, tokens should respect indentation.
319        # Actually the condition should be `self.indent >= column` according to
320        # the spec. But this condition will prohibit intuitively correct
321        # constructions such as
322        # key : {
323        # }
324        if self.flow_level and self.indent > column:
325            raise ScannerError(None, None,
326                    "invalid intendation or unclosed '[' or '{'",
327                    self.reader.get_marker())
328
329        # In block context, we may need to issue the BLOCK-END tokens.
330        while self.indent > column:
331            marker = self.reader.get_marker()
332            self.indent = self.indents.pop()
333            self.tokens.append(BlockEndToken(marker, marker))
334
335    def add_indent(self, column):
336        # Check if we need to increase indentation.
337        if self.indent < column:
338            self.indents.append(self.indent)
339            self.indent = column
340            return True
341        return False
342
343    # Fetchers.
344
345    def fetch_end(self):
346
347        # Set the current intendation to -1.
348        self.unwind_indent(-1)
349
350        # Reset everything (not really needed).
351        self.allow_simple_key = False
352        self.possible_simple_keys = {}
353
354        # Read the token.
355        marker = self.reader.get_marker()
356       
357        # Add END.
358        self.tokens.append(StreamEndToken(marker, marker))
359
360        # The reader is ended.
361        self.done = True
362
363    def fetch_directive(self):
364       
365        # Set the current intendation to -1.
366        self.unwind_indent(-1)
367
368        # Reset simple keys.
369        self.remove_possible_simple_key()
370        self.allow_simple_key = False
371
372        # Scan and add DIRECTIVE.
373        self.tokens.append(self.scan_directive())
374
375    def fetch_document_start(self):
376        self.fetch_document_indicator(DocumentStartToken)
377
378    def fetch_document_end(self):
379        self.fetch_document_indicator(DocumentEndToken)
380
381    def fetch_document_indicator(self, TokenClass):
382
383        # Set the current intendation to -1.
384        self.unwind_indent(-1)
385
386        # Reset simple keys. Note that there could not be a block collection
387        # after '---'.
388        self.remove_possible_simple_key()
389        self.allow_simple_key = False
390
391        # Add DOCUMENT-START or DOCUMENT-END.
392        start_marker = self.reader.get_marker()
393        self.reader.forward(3)
394        end_marker = self.reader.get_marker()
395        self.tokens.append(TokenClass(start_marker, end_marker))
396
397    def fetch_flow_sequence_start(self):
398        self.fetch_flow_collection_start(FlowSequenceStartToken)
399
400    def fetch_flow_mapping_start(self):
401        self.fetch_flow_collection_start(FlowMappingStartToken)
402
403    def fetch_flow_collection_start(self, TokenClass):
404
405        # '[' and '{' may start a simple key.
406        self.save_possible_simple_key()
407
408        # Increase the flow level.
409        self.flow_level += 1
410
411        # Simple keys are allowed after '[' and '{'.
412        self.allow_simple_key = True
413
414        # Add FLOW-SEQUENCE-START or FLOW-MAPPING-START.
415        start_marker = self.reader.get_marker()
416        self.reader.forward()
417        end_marker = self.reader.get_marker()
418        self.tokens.append(TokenClass(start_marker, end_marker))
419
420    def fetch_flow_sequence_end(self):
421        self.fetch_flow_collection_end(FlowSequenceEndToken)
422
423    def fetch_flow_mapping_end(self):
424        self.fetch_flow_collection_end(FlowMappingEndToken)
425
426    def fetch_flow_collection_end(self, TokenClass):
427
428        # Reset possible simple key on the current level.
429        self.remove_possible_simple_key()
430
431        # Decrease the flow level.
432        self.flow_level -= 1
433
434        # No simple keys after ']' or '}'.
435        self.allow_simple_key = False
436
437        # Add FLOW-SEQUENCE-END or FLOW-MAPPING-END.
438        start_marker = self.reader.get_marker()
439        self.reader.forward()
440        end_marker = self.reader.get_marker()
441        self.tokens.append(TokenClass(start_marker, end_marker))
442
443    def fetch_entry(self):
444
445        # Block context needs additional checks.
446        if not self.flow_level:
447
448            # Are we allowed to start a new entry?
449            if not self.allow_simple_key:
450                raise ScannerError(None, None,
451                        "sequence entries are not allowed here",
452                        self.reader.get_marker())
453
454            # We may need to add BLOCK-SEQUENCE-START.
455            if self.add_indent(self.reader.column):
456                marker = self.reader.get_marker()
457                self.tokens.append(BlockSequenceStartToken(marker, marker))
458
459        # Simple keys are allowed after '-' and ','.
460        self.allow_simple_key = True
461
462        # Reset possible simple key on the current level.
463        self.remove_possible_simple_key()
464
465        # Add ENTRY.
466        start_marker = self.reader.get_marker()
467        self.reader.forward()
468        end_marker = self.reader.get_marker()
469        self.tokens.append(EntryToken(start_marker, end_marker))
470
471    def fetch_key(self):
472       
473        # Block context needs additional checks.
474        if not self.flow_level:
475
476            # Are we allowed to start a key (not nessesary a simple)?
477            if not self.allow_simple_key:
478                raise ScannerError(None, None,
479                        "mapping keys are not allowed here",
480                        self.reader.get_marker())
481
482            # We may need to add BLOCK-MAPPING-START.
483            if self.add_indent(self.reader.column):
484                marker = self.reader.get_marker()
485                self.tokens.append(BlockMappingStartToken(marker, marker))
486
487        # Simple keys are allowed after '?' in the block context.
488        self.allow_simple_key = not self.flow_level
489
490        # Reset possible simple key on the current level.
491        self.remove_possible_simple_key()
492
493        # Add KEY.
494        start_marker = self.reader.get_marker()
495        self.reader.forward()
496        end_marker = self.reader.get_marker()
497        self.tokens.append(KeyToken(start_marker, end_marker))
498
499    def fetch_value(self):
500
501        # Do we determine a simple key?
502        if self.flow_level in self.possible_simple_keys:
503
504            # Add KEY.
505            key = self.possible_simple_keys[self.flow_level]
506            del self.possible_simple_keys[self.flow_level]
507            self.tokens.insert(key.token_number-self.tokens_taken,
508                    KeyToken(key.marker, key.marker))
509
510            # If this key starts a new block mapping, we need to add
511            # BLOCK-MAPPING-START.
512            if not self.flow_level:
513                if self.add_indent(key.column):
514                    self.tokens.insert(key.token_number-self.tokens_taken,
515                            BlockMappingStartToken(key.marker, key.marker))
516
517            # There cannot be two simple keys one after another.
518            self.allow_simple_key = False
519
520        # It must be a part of a complex key.
521        else:
522           
523            # Block context needs additional checks.
524            # (Do we really need them? They will be catched by the parser
525            # anyway.)
526            if not self.flow_level:
527
528                # We are allowed to start a complex value if and only if
529                # we can start a simple key.
530                if not self.allow_simple_key:
531                    raise ScannerError(None, None,
532                            "mapping values are not allowed here",
533                            self.reader.get_marker())
534
535            # Simple keys are allowed after ':' in the block context.
536            self.allow_simple_key = not self.flow_level
537
538            # Reset possible simple key on the current level.
539            self.remove_possible_simple_key()
540
541        # Add VALUE.
542        start_marker = self.reader.get_marker()
543        self.reader.forward()
544        end_marker = self.reader.get_marker()
545        self.tokens.append(ValueToken(start_marker, end_marker))
546
547    def fetch_alias(self):
548
549        # ALIAS could be a simple key.
550        self.save_possible_simple_key()
551
552        # No simple keys after ALIAS.
553        self.allow_simple_key = False
554
555        # Scan and add ALIAS.
556        self.tokens.append(self.scan_anchor(AliasToken))
557
558    def fetch_anchor(self):
559
560        # ANCHOR could start a simple key.
561        self.save_possible_simple_key()
562
563        # No simple keys after ANCHOR.
564        self.allow_simple_key = False
565
566        # Scan and add ANCHOR.
567        self.tokens.append(self.scan_anchor(AnchorToken))
568
569    def fetch_tag(self):
570
571        # TAG could start a simple key.
572        self.save_possible_simple_key()
573
574        # No simple keys after TAG.
575        self.allow_simple_key = False
576
577        # Scan and add TAG.
578        self.tokens.append(self.scan_tag())
579
580    def fetch_literal(self):
581        self.fetch_block_scalar(folded=False)
582
583    def fetch_folded(self):
584        self.fetch_block_scalar(folded=True)
585
586    def fetch_block_scalar(self, folded):
587
588        # A simple key may follow a block scalar.
589        self.allow_simple_key = True
590
591        # Reset possible simple key on the current level.
592        self.remove_possible_simple_key()
593
594        # Scan and add SCALAR.
595        self.tokens.append(self.scan_block_scalar(folded))
596
597    def fetch_single(self):
598        self.fetch_flow_scalar(double=False)
599
600    def fetch_double(self):
601        self.fetch_flow_scalar(double=True)
602
603    def fetch_flow_scalar(self, double):
604
605        # A flow scalar could be a simple key.
606        self.save_possible_simple_key()
607
608        # No simple keys after flow scalars.
609        self.allow_simple_key = False
610
611        # Scan and add SCALAR.
612        self.tokens.append(self.scan_flow_scalar(double))
613
614    def fetch_plain(self):
615
616        # A plain scalar could be a simple key.
617        self.save_possible_simple_key()
618
619        # No simple keys after plain scalars. But note that `scan_plain` will
620        # change this flag if the scan is finished at the beginning of the
621        # line.
622        self.allow_simple_key = False
623
624        # Scan and add SCALAR. May change `allow_simple_key`.
625        self.tokens.append(self.scan_plain())
626
627    # Checkers.
628
629    def check_directive(self):
630
631        # DIRECTIVE:        ^ '%' ...
632        # The '%' indicator is already checked.
633        if self.reader.column == 0:
634            return True
635
636    def check_document_start(self):
637
638        # DOCUMENT-START:   ^ '---' (' '|'\n')
639        if self.reader.column == 0:
640            prefix = self.reader.peek(4)
641            if prefix[:3] == u'---' and prefix[3] in u'\0 \t\r\n\x85\u2028\u2029':
642                return True
643
644    def check_document_end(self):
645
646        # DOCUMENT-END:     ^ '...' (' '|'\n')
647        if self.reader.column == 0:
648            prefix = self.reader.peek(4)
649            if prefix[:3] == u'...' and prefix[3] in u'\0 \t\r\n\x85\u2028\u2029':
650                return True
651
652    def check_entry(self):
653
654        # ENTRY(flow context):      ','
655        if self.flow_level:
656            return self.reader.peek() == u','
657
658        # ENTRY(block context):     '-' (' '|'\n')
659        else:
660            prefix = self.reader.peek(2)
661            return prefix[0] == u'-' and prefix[1] in u'\0 \t\r\n\x85\u2028\u2029'
662
663    def check_key(self):
664
665        # KEY(flow context):    '?'
666        if self.flow_level:
667            return True
668
669        # KEY(block context):   '?' (' '|'\n')
670        else:
671            prefix = self.reader.peek(2)
672            return prefix[1] in u'\0 \t\r\n\x85\u2028\u2029'
673
674    def check_value(self):
675
676        # VALUE(flow context):  ':'
677        if self.flow_level:
678            return True
679
680        # VALUE(block context): ':' (' '|'\n')
681        else:
682            prefix = self.reader.peek(2)
683            return prefix[1] in u'\0 \t\r\n\x85\u2028\u2029'
684
685    def check_plain(self):
686        return True
687
688    # Scanners.
689
690    def scan_to_next_token(self):
691        # We ignore spaces, line breaks and comments.
692        # If we find a line break in the block context, we set the flag
693        # `allow_simple_key` on.
694        found = False
695        while not found:
696            while self.reader.peek() == u' ':
697                self.reader.forward()
698            if self.reader.peek() == u'#':
699                while self.reader.peek() not in u'\0\r\n\x85\u2028\u2029':
700                    self.reader.forward()
701            if self.scan_line_break():
702                if not self.flow_level:
703                    self.allow_simple_key = True
704            else:
705                found = True
706
707    def scan_directive(self):
708        marker = self.reader.get_marker()
709        if self.reader.peek(5) == u'%YAML ':
710            token = YAMLDirectiveToken(1, 1, marker, marker)
711        elif self.reader.peek(4) == u'%TAG ':
712            token = TagDirectiveToken(marker, marker)
713        else:
714            token = ReservedDirectiveToken('', marker, marker)
715        while self.reader.peek() not in u'\0\r\n':
716            self.reader.forward()
717        self.reader.forward()
718        return token
719
720    def scan_anchor(self, TokenClass):
721        start_marker = self.reader.get_marker()
722        while self.reader.peek() not in u'\0 \t\r\n,:':
723            self.reader.forward()
724        end_marker = self.reader.get_marker()
725        return TokenClass('', start_marker, end_marker)
726
727    def scan_tag(self):
728        start_marker = self.reader.get_marker()
729        while self.reader.peek() not in u'\0 \t\r\n':
730            self.reader.forward()
731        end_marker = self.reader.get_marker()
732        return TagToken('', start_marker, end_marker)
733
734    def scan_block_scalar(self, folded):
735        start_marker = self.reader.get_marker()
736        indent = self.indent+1
737        if indent < 1:
738            indent = 1
739        while True:
740            while self.reader.peek() and self.reader.peek() and self.reader.peek() not in u'\0\r\n\x85\u2028\u2029':
741                self.reader.forward()
742            if self.reader.peek() != u'\0':
743                self.reader.forward()
744            count = 0
745            while count < indent and self.reader.peek() == u' ':
746                self.reader.forward()
747                count += 1
748            if count < indent and self.reader.peek() not in u'#\r\n\x85\u2028\u2029':
749                break
750        return ScalarToken('', False, start_marker, start_marker)
751
752    def scan_flow_scalar(self, double):
753        marker = self.reader.get_marker()
754        quote = self.reader.peek()
755        self.reader.forward()
756        while self.reader.peek() != quote:
757            if double and self.reader.peek() == u'\\':
758                self.reader.forward(2)
759            elif not double and self.reader.peek(3)[1:] == u'\'\'':
760                self.reader.forward(3)
761            else:
762                self.reader.forward(1)
763        self.reader.forward(1)
764        return ScalarToken('', False, marker, marker)
765
766    def scan_plain(self):
767        indent = self.indent+1
768        if indent < 1:
769            indent = 1
770        space = False
771        marker = self.reader.get_marker()
772        while True:
773            while self.reader.peek() == u' ':
774                self.reader.forward()
775                space = True
776            while self.reader.peek() not in u'\0\r\n?:,[]{}#'   \
777                    or (not space and self.reader.peek() == '#')    \
778                    or (not self.flow_level and self.reader.peek() in '?,[]{}') \
779                    or (not self.flow_level and self.reader.peek() == ':' and self.reader.peek(2)[1] not in u' \0\r\n'):
780                space = self.reader.peek() not in u' \t'
781                self.reader.forward()
782                self.allow_simple_key = False
783            if self.reader.peek() not in u'\r\n':
784                break
785            while self.reader.peek() in u'\r\n':
786                self.reader.forward()
787                if not self.flow_level:
788                    self.allow_simple_key = True
789            count = 0
790            while self.reader.peek() == u' ' and count < indent:
791                self.reader.forward()
792                count += 1
793            if count < indent:
794                break
795            space = True
796        return ScalarToken('', True, marker, marker)
797
798    def scan_line_break(self):
799        # Transforms:
800        #   '\r\n'      :   '\n'
801        #   '\r'        :   '\n'
802        #   '\n'        :   '\n'
803        #   '\x85'      :   '\n'
804        #   '\u2028'    :   '\u2028'
805        #   '\u2029     :   '\u2029'
806        #   default     :   ''
807        ch = self.reader.peek()
808        if ch in u'\r\n\x85':
809            if self.reader.peek(2) == u'\r\n':
810                self.forward(2)
811            else:
812                self.reader.forward()
813            return u'\n'
814        elif ch in u'\u2028\u2029':
815            self.reader.forward()
816            return ch
817        return u''
818
819    def invalid_token(self):
820        self.fail("invalid token")
821
822#try:
823#    import psyco
824#    psyco.bind(Scanner)
825#except ImportError:
826#    pass
827
Note: See TracBrowser for help on using the repository browser.