source: branches/pyyaml3000/lib/yaml/scanner.py @ 46

Revision 46, 24.7 KB checked in by xi, 8 years ago (diff)

Some renaming.

RevLine 
[39]1
[43]2# Tokens:
3# YAML-DIRECTIVE(major_version, minor_version), TAG-DIRECTIVE(handle, prefix)
4# RESERVED-DIRECTIVE(name)
5# DOCUMENT-START, DOCUMENT-END
6# BLOCK-SEQUENCE-START, BLOCK-MAPPING-START, BLOCK-END
7# FLOW-SEQUENCE-START, FLOW-MAPPING-START, FLOW-SEQUENCE-END, FLOW-MAPPING-END
8# ENTRY, KEY, VALUE
9# ALIAS(name), ANCHOR(name), TAG(value), SCALAR(value, plain)
10
[46]11__all__ = ['Scanner', 'ScannerError']
[43]12
[46]13from error import YAMLError
14from tokens import *
[39]15
[46]16class ScannerError(YAMLError):
17    # TODO:
18    # ScannerError: while reading a quoted string
19    #         in '...', line 5, column 10:
20    # key: "valu\?e"
21    #      ^
22    # got unknown quote character '?'
23    #         in '...', line 5, column 15:
24    # key: "valu\?e"
25    #            ^
[43]26    pass
27
28class SimpleKey:
29    def __init__(self, token_number, required, index, line, column, marker):
30        self.token_number = token_number
31        self.required = required
32        self.index = index
33        self.line = line
34        self.column = column
35        self.marker = marker
36
[39]37class Scanner:
38
[46]39
40    def __init__(self, reader):
[39]41        """Initialize the scanner."""
[46]42        # The input stream. The Reader class do the dirty work of checking for
[43]43        # BOM and converting the input data to Unicode. It also adds NUL to
44        # the end.
[39]45        #
[46]46        # Reader supports the following methods
47        #   self.reader.peek(k=1)   # peek the next k characters
48        #   self.reader.forward(k=1)   # read the next k characters and move the
[39]49        #                           # pointer
[46]50        self.reader = reader
[39]51
52        # Had we reached the end of the stream?
53        self.done = False
54
55        # The number of unclosed '{' and '['. `flow_level == 0` means block
56        # context.
57        self.flow_level = 0
58
59        # List of processed tokens that are not yet emitted.
60        self.tokens = []
61
62        # Number of tokens that were emitted through the `get_token` method.
63        self.tokens_taken = 0
64
65        # The current indentation level.
66        self.indent = -1
67
68        # Past indentation levels.
69        self.indents = []
70
[43]71        # Variables related to simple keys treatment.
[39]72
73        # A simple key is a key that is not denoted by the '?' indicator.
74        # Example of simple keys:
75        #   ---
76        #   block simple key: value
77        #   ? not a simple key:
78        #   : { flow simple key: value }
79        # We emit the KEY token before all keys, so when we find a potential
80        # simple key, we try to locate the corresponding ':' indicator.
81        # Simple keys should be limited to a single line and 1024 characters.
82
[43]83        # Can a simple key start at the current position? A simple key may
84        # start:
85        # - at the beginning of the line, not counting indentation spaces
86        #       (in block context),
87        # - after '{', '[', ',' (in the flow context),
88        # - after '?', ':', '-' (in the block context).
89        # In the block context, this flag also signify if a block collection
90        # may start at the current position.
91        self.allow_simple_key = True
[39]92
93        # Keep track of possible simple keys. This is a dictionary. The key
94        # is `flow_level`; there can be no more that one possible simple key
[43]95        # for each level. The value is a SimpleKey record:
96        #   (token_number, required, index, line, column, marker)
97        # A simple key may start with ALIAS, ANCHOR, TAG, SCALAR(flow),
98        # '[', or '{' tokens.
[39]99        self.possible_simple_keys = {}
100
[43]101    # Two public methods.
[39]102
103    def peek_token(self):
104        """Get the current token."""
[43]105        while self.need_more_tokens():
[39]106            self.fetch_more_tokens()
107        if self.tokens:
108            return self.tokens[0]
109
110    def get_token(self):
[43]111        "Get the current token and remove it from the list of pending tokens."""
[39]112        while self.need_more_tokens():
113            self.fetch_more_tokens()
114        if self.tokens:
115            self.tokens_taken += 1
116            return self.tokens.pop(0)
117
[43]118    # Private methods.
[39]119
120    def need_more_tokens(self):
121        if self.done:
122            return False
123        if not self.tokens:
124            return True
125        # The current token may be a potential simple key, so we
126        # need to look further.
[43]127        self.stale_possible_simple_keys()
[39]128        if self.next_possible_simple_key() == self.tokens_taken:
129            return True
130
131    def fetch_more_tokens(self):
132
133        # Eat whitespaces and comments until we reach the next token.
[43]134        self.scan_to_next_token()
[39]135
[43]136        # Remove obsolete possible simple keys.
137        self.stale_possible_simple_keys()
138
[39]139        # Compare the current indentation and column. It may add some tokens
[43]140        # and decrease the current indentation level.
[46]141        self.unwind_indent(self.reader.column)
[39]142
[43]143        #print
[46]144        #print self.reader.get_marker().get_snippet()
[43]145
[39]146        # Peek the next character.
[46]147        ch = self.reader.peek()
[39]148
[46]149        # Is it the end of reader?
[43]150        if ch == u'\0':
[39]151            return self.fetch_end()
152
153        # Is it a directive?
154        if ch == u'%' and self.check_directive():
155            return self.fetch_directive()
156
157        # Is it the document start?
158        if ch == u'-' and self.check_document_start():
159            return self.fetch_document_start()
160
161        # Is it the document end?
162        if ch == u'.' and self.check_document_end():
163            return self.fetch_document_end()
164
165        # Note: the order of the following checks is NOT significant.
166
167        # Is it the flow sequence start indicator?
168        if ch == u'[':
169            return self.fetch_flow_sequence_start()
170
171        # Is it the flow mapping start indicator?
172        if ch == u'{':
173            return self.fetch_flow_mapping_start()
174
175        # Is it the flow sequence end indicator?
176        if ch == u']':
177            return self.fetch_flow_sequence_end()
178
179        # Is it the flow mapping end indicator?
180        if ch == u'}':
181            return self.fetch_flow_mapping_end()
182
[43]183        # Is it the entry indicator?
184        if ch in u'-,' and self.check_entry():
185            return self.fetch_entry()
186
[39]187        # Is it the key indicator?
188        if ch == u'?' and self.check_key():
189            return self.fetch_key()
190
191        # Is it the value indicator?
192        if ch == u':' and self.check_value():
193            return self.fetch_value()
194
195        # Is it an alias?
196        if ch == u'*':
197            return self.fetch_alias()
198
199        # Is it an anchor?
200        if ch == u'&':
201            return self.fetch_anchor()
202
[43]203        # Is it a tag?
[39]204        if ch == u'!':
205            return self.fetch_tag()
206
[43]207        # Is it a literal scalar?
208        if ch == u'|' and not self.flow_level:
[39]209            return self.fetch_literal()
210
211        # Is it a folded scalar?
[43]212        if ch == u'>' and not self.flow_level:
[39]213            return self.fetch_folded()
214
215        # Is it a single quoted scalar?
216        if ch == u'\'':
217            return self.fetch_single()
218
219        # Is it a double quoted scalar?
220        if ch == u'\"':
221            return self.fetch_double()
222
[43]223        # It must be a plain scalar then.
[39]224        if self.check_plain():
225            return self.fetch_plain()
226
[43]227        # No? It's an error. Let's produce a nice error message.
[39]228        self.invalid_token()
229
[43]230    # Simple keys treatment.
231
232    def next_possible_simple_key(self):
233        # Return the number of the nearest possible simple key. Actually we
234        # don't need to loop through the whole dictionary. We may replace it
235        # with the following code:
236        #   if not self.possible_simple_keys:
237        #       return None
238        #   return self.possible_simple_keys[
239        #           min(self.possible_simple_keys.keys())].token_number
240        min_token_number = None
241        for level in self.possible_simple_keys:
242            key = self.possible_simple_keys[level]
243            if min_token_number is None or key.token_number < min_token_number:
244                min_token_number = key.token_number
245        return min_token_number
246
247    def stale_possible_simple_keys(self):
248        # Remove entries that are no longer possible simple keys. According to
249        # the YAML specification, simple keys
250        # - should be limited to a single line,
251        # - should be no longer than 1024 characters.
252        # Disabling this procedure will allow simple keys of any length and
253        # height (may cause problems if indentation is broken though).
254        for level in self.possible_simple_keys.keys():
255            key = self.possible_simple_keys[level]
[46]256            if key.line != self.reader.line  \
257                    or self.reader.index-key.index > 1024:
[43]258                if key.required:
259                    self.fail("simple key is required")
260                del self.possible_simple_keys[level]
261
262    def save_possible_simple_key(self):
263        # The next token may start a simple key. We check if it's possible
264        # and save its position. This function is called for
265        #   ALIAS, ANCHOR, TAG, SCALAR(flow), '[', and '{'.
266
267        # Check if a simple key is required at the current position.
[46]268        required = not self.flow_level and self.indent == self.reader.column
[43]269
270        # The next token might be a simple key. Let's save it's number and
271        # position.
272        if self.allow_simple_key:
273            self.remove_possible_simple_key()
274            token_number = self.tokens_taken+len(self.tokens)
[46]275            index = self.reader.index
276            line = self.reader.line
277            column = self.reader.column
278            marker = self.reader.get_marker()
[43]279            key = SimpleKey(token_number, required,
280                    index, line, column, marker)
281            self.possible_simple_keys[self.flow_level] = key
282
283        # A simple key is required at the current position.
284        elif required:
285            self.fail("simple key is required")
286
287    def remove_possible_simple_key(self):
288        # Remove the saved possible key position at the current flow level.
289        if self.flow_level in self.possible_simple_keys:
290            key = self.possible_simple_keys[self.flow_level]
291            if key.required:
292                self.fail("simple key is required")
293
294    # Indentation functions.
295
296    def unwind_indent(self, column):
297
298        # In flow context, tokens should respect indentation.
299        if self.flow_level and self.indent > column:
300            self.fail("invalid intendation in the flow context")
301
302        # In block context, we may need to issue the BLOCK-END tokens.
303        while self.indent > column:
[46]304            marker = self.reader.get_marker()
[43]305            self.indent = self.indents.pop()
[44]306            self.tokens.append(BlockEndToken(marker, marker))
[43]307
308    def add_indent(self, column):
309        # Check if we need to increase indentation.
310        if self.indent < column:
311            self.indents.append(self.indent)
312            self.indent = column
313            return True
314        return False
315
316    # Fetchers.
317
[39]318    def fetch_end(self):
319
320        # Set the current intendation to -1.
[43]321        self.unwind_indent(-1)
[39]322
323        # Reset everything (not really needed).
[43]324        self.allow_simple_key = False
[39]325        self.possible_simple_keys = {}
326
[43]327        # Read the token.
[46]328        marker = self.reader.get_marker()
[43]329       
330        # Add END.
[44]331        self.tokens.append(EndToken(marker, marker))
[39]332
[46]333        # The reader is ended.
[39]334        self.done = True
335
[43]336    def fetch_directive(self):
337       
338        # Set the current intendation to -1.
339        self.unwind_indent(-1)
[39]340
[43]341        # Reset simple keys.
342        self.remove_possible_simple_key()
343        self.allow_simple_key = False
[39]344
[43]345        # Scan and add DIRECTIVE.
346        self.scan_directive()
[39]347
348    def fetch_document_start(self):
[44]349        self.fetch_document_indicator(DocumentStartToken)
[39]350
[43]351    def fetch_document_end(self):
[44]352        self.fetch_document_indicator(DocumentEndToken)
[43]353
354    def fetch_document_indicator(self, TokenClass):
355
[39]356        # Set the current intendation to -1.
[43]357        self.unwind_indent(-1)
[39]358
[43]359        # Reset simple keys. Note that there could not be a block collection
360        # after '---'.
361        self.remove_possible_simple_key()
362        self.allow_simple_key = False
[39]363
[43]364        # Add DOCUMENT-START or DOCUMENT-END.
[46]365        start_marker = self.reader.get_marker()
366        self.reader.forward(3)
367        end_marker = self.reader.get_marker()
[43]368        self.tokens.append(TokenClass(start_marker, end_marker))
[39]369
[43]370    def fetch_flow_sequence_start(self):
[44]371        self.fetch_flow_collection_start(FlowSequenceStartToken)
[39]372
[43]373    def fetch_flow_mapping_start(self):
[44]374        self.fetch_flow_collection_start(FlowMappingStartToken)
[43]375
376    def fetch_flow_collection_start(self, TokenClass):
377
[44]378        # '[' and '{' may start a simple key.
379        self.save_possible_simple_key()
380
[43]381        # Increase the flow level.
382        self.flow_level += 1
383
384        # Simple keys are allowed after '[' and '{'.
385        self.allow_simple_key = True
386
387        # Add FLOW-SEQUENCE-START or FLOW-MAPPING-START.
[46]388        start_marker = self.reader.get_marker()
389        self.reader.forward()
390        end_marker = self.reader.get_marker()
[43]391        self.tokens.append(TokenClass(start_marker, end_marker))
[39]392
[43]393    def fetch_flow_sequence_end(self):
[44]394        self.fetch_flow_collection_end(FlowSequenceEndToken)
[39]395
[43]396    def fetch_flow_mapping_end(self):
[44]397        self.fetch_flow_collection_end(FlowMappingEndToken)
[43]398
399    def fetch_flow_collection_end(self, TokenClass):
400
401        # Reset possible simple key on the current level.
402        self.remove_possible_simple_key()
403
404        # Decrease the flow level.
405        self.flow_level -= 1
406
407        # No simple keys after ']' or '}'.
408        self.allow_simple_key = False
409
410        # Add FLOW-SEQUENCE-END or FLOW-MAPPING-END.
[46]411        start_marker = self.reader.get_marker()
412        self.reader.forward()
413        end_marker = self.reader.get_marker()
[43]414        self.tokens.append(TokenClass(start_marker, end_marker))
[39]415
[43]416    def fetch_entry(self):
[39]417
[43]418        # Block context needs additional checks.
419        if not self.flow_level:
[39]420
[43]421            # Are we allowed to start a new entry?
422            if not self.allow_simple_key:
423                self.fail("Cannot start a new entry here")
[39]424
[43]425            # We may need to add BLOCK-SEQUENCE-START.
[46]426            if self.add_indent(self.reader.column):
427                marker = self.reader.get_marker()
[44]428                self.tokens.append(BlockSequenceStartToken(marker, marker))
[39]429
[43]430        # Simple keys are allowed after '-' and ','.
431        self.allow_simple_key = True
[39]432
[43]433        # Reset possible simple key on the current level.
434        self.remove_possible_simple_key()
[39]435
[43]436        # Add ENTRY.
[46]437        start_marker = self.reader.get_marker()
438        self.reader.forward()
439        end_marker = self.reader.get_marker()
[44]440        self.tokens.append(EntryToken(start_marker, end_marker))
[39]441
[43]442    def fetch_key(self):
443       
444        # Block context needs additional checks.
445        if not self.flow_level:
[39]446
[43]447            # Are we allowed to start a key (not nessesary a simple)?
448            if not self.allow_simple_key:
449                self.fail("Cannot start a new key here")
450
451            # We may need to add BLOCK-MAPPING-START.
[46]452            if self.add_indent(self.reader.column):
453                marker = self.reader.get_marker()
[44]454                self.tokens.append(BlockMappingStartToken(marker, marker))
[43]455
456        # Simple keys are allowed after '?' in the block context.
457        self.allow_simple_key = not self.flow_level
458
459        # Reset possible simple key on the current level.
460        self.remove_possible_simple_key()
461
462        # Add KEY.
[46]463        start_marker = self.reader.get_marker()
464        self.reader.forward()
465        end_marker = self.reader.get_marker()
[44]466        self.tokens.append(KeyToken(start_marker, end_marker))
[39]467
[43]468    def fetch_value(self):
[39]469
[43]470        # Do we determine a simple key?
471        if self.flow_level in self.possible_simple_keys:
[39]472
[43]473            # Add KEY.
474            key = self.possible_simple_keys[self.flow_level]
475            del self.possible_simple_keys[self.flow_level]
476            self.tokens.insert(key.token_number-self.tokens_taken,
[44]477                    KeyToken(key.marker, key.marker))
[39]478
[43]479            # If this key starts a new block mapping, we need to add
480            # BLOCK-MAPPING-START.
481            if not self.flow_level:
482                if self.add_indent(key.column):
483                    self.tokens.insert(key.token_number-self.tokens_taken,
[44]484                            BlockMappingStartToken(key.marker, key.marker))
[37]485
[43]486            # There cannot be two simple keys one after another.
487            self.allow_simple_key = False
[37]488
[43]489        # It must be a part of a complex key.
490        else:
491           
492            # Simple keys are allowed after ':' in the block context.
493            self.allow_simple_key = not self.flow_level
[37]494
[43]495            # Reset possible simple key on the current level.
496            self.remove_possible_simple_key()
[37]497
[43]498        # Add VALUE.
[46]499        start_marker = self.reader.get_marker()
500        self.reader.forward()
501        end_marker = self.reader.get_marker()
[44]502        self.tokens.append(ValueToken(start_marker, end_marker))
[37]503
[43]504    def fetch_alias(self):
[37]505
[43]506        # ALIAS could be a simple key.
507        self.save_possible_simple_key()
[37]508
[43]509        # No simple keys after ALIAS.
510        self.allow_simple_key = False
[37]511
[43]512        # Scan and add ALIAS.
[44]513        self.scan_anchor(AliasToken)
[37]514
[43]515    def fetch_anchor(self):
[37]516
[43]517        # ANCHOR could start a simple key.
518        self.save_possible_simple_key()
[37]519
[43]520        # No simple keys after ANCHOR.
521        self.allow_simple_key = False
[37]522
[43]523        # Scan and add ANCHOR.
[44]524        self.scan_anchor(AnchorToken)
[37]525
[43]526    def fetch_tag(self):
[37]527
[43]528        # TAG could start a simple key.
529        self.save_possible_simple_key()
[37]530
[43]531        # No simple keys after TAG.
532        self.allow_simple_key = False
[37]533
[43]534        # Scan and add TAG.
535        self.scan_tag()
[37]536
[43]537    def fetch_literal(self):
538        self.fetch_block_scalar(folded=False)
[37]539
[43]540    def fetch_folded(self):
541        self.fetch_block_scalar(folded=True)
[37]542
[43]543    def fetch_block_scalar(self, folded):
[37]544
[43]545        # A simple key may follow a block scalar.
546        self.allow_simple_key = True
[37]547
[43]548        # Reset possible simple key on the current level.
549        self.remove_possible_simple_key()
[37]550
[43]551        # Scan and add SCALAR.
552        self.scan_block_scalar(folded)
[37]553
[43]554    def fetch_single(self):
555        self.fetch_flow_scalar(double=False)
[37]556
[43]557    def fetch_double(self):
558        self.fetch_flow_scalar(double=True)
[37]559
[43]560    def fetch_flow_scalar(self, double):
[37]561
[43]562        # A flow scalar could be a simple key.
563        self.save_possible_simple_key()
[37]564
[43]565        # No simple keys after flow scalars.
566        self.allow_simple_key = False
[37]567
[43]568        # Scan and add SCALAR.
569        self.scan_flow_scalar(double)
[37]570
[43]571    def fetch_plain(self):
[37]572
[43]573        # A plain scalar could be a simple key.
574        self.save_possible_simple_key()
[37]575
[43]576        # No simple keys after plain scalars. But note that `scan_plain` will
577        # change this flag if the scan is finished at the beginning of the
578        # line.
579        self.allow_simple_key = False
[37]580
[43]581        # Scan and add SCALAR. May change `allow_simple_key`.
582        self.scan_plain()
[37]583
[43]584    # Checkers.
[37]585
[43]586    def check_directive(self):
[37]587
[43]588        # DIRECTIVE:        ^ '%' ...
589        # The '%' indicator is already checked.
[46]590        if self.reader.column == 0:
[43]591            return True
[37]592
[43]593    def check_document_start(self):
[37]594
[43]595        # DOCUMENT-START:   ^ '---' (' '|'\n')
[46]596        if self.reader.column == 0:
597            prefix = self.reader.peek(4)
[43]598            if prefix[:3] == u'---' and prefix[3] in u'\0 \t\r\n\x85\u2028\u2029':
599                return True
[37]600
[43]601    def check_document_end(self):
[37]602
[43]603        # DOCUMENT-END:     ^ '...' (' '|'\n')
[46]604        if self.reader.column == 0:
605            prefix = self.reader.peek(4)
[43]606            if prefix[:3] == u'...' and prefix[3] in u'\0 \t\r\n\x85\u2028\u2029':
607                return True
[37]608
[43]609    def check_entry(self):
610
611        # ENTRY(flow context):      ','
612        if self.flow_level:
[46]613            return self.reader.peek() == u','
[43]614
615        # ENTRY(block context):     '-' (' '|'\n')
616        else:
[46]617            prefix = self.reader.peek(2)
[43]618            return prefix[0] == u'-' and prefix[1] in u'\0 \t\r\n\x85\u2028\u2029'
619
620    def check_key(self):
621
622        # KEY(flow context):    '?'
623        if self.flow_level:
[37]624            return True
[43]625
626        # KEY(block context):   '?' (' '|'\n')
[37]627        else:
[46]628            prefix = self.reader.peek(2)
[43]629            return prefix[1] in u'\0 \t\r\n\x85\u2028\u2029'
[37]630
[43]631    def check_value(self):
632
633        # VALUE(flow context):  ':'
634        if self.flow_level:
[37]635            return True
[43]636
637        # VALUE(block context): ':' (' '|'\n')
[37]638        else:
[46]639            prefix = self.reader.peek(2)
[43]640            return prefix[1] in u'\0 \t\r\n\x85\u2028\u2029'
[37]641
[43]642    def check_plain(self):
643        return True
[37]644
[43]645    # Scanners.
646
647    def scan_to_next_token(self):
648        found = False
649        while not found:
[46]650            while self.reader.peek() == u' ':
651                self.reader.forward()
652            if self.reader.peek() == u'#':
653                while self.reader.peek() not in u'\r\n':
654                    self.reader.forward()
655            if self.reader.peek() in u'\r\n':
656                self.reader.forward()
[43]657                if not self.flow_level:
658                    self.allow_simple_key = True
[37]659            else:
[43]660                found = True
[37]661
[43]662    def scan_directive(self):
[46]663        marker = self.reader.get_marker()
664        if self.reader.peek(5) == u'%YAML ':
[44]665            self.tokens.append(YAMLDirectiveToken(1, 1, marker, marker))
[46]666        elif self.reader.peek(4) == u'%TAG ':
[44]667            self.tokens.append(TagDirectiveToken(marker, marker))
[43]668        else:
[44]669            self.tokens.append(ReservedDirectiveToken('', marker, marker))
[46]670        while self.reader.peek() not in u'\0\r\n':
671            self.reader.forward()
672        self.reader.forward()
[37]673
[43]674    def scan_anchor(self, TokenClass):
[46]675        start_marker = self.reader.get_marker()
676        while self.reader.peek() not in u'\0 \t\r\n,:':
677            self.reader.forward()
678        end_marker = self.reader.get_marker()
[43]679        self.tokens.append(TokenClass('', start_marker, end_marker))
[37]680
[43]681    def scan_tag(self):
[46]682        start_marker = self.reader.get_marker()
683        while self.reader.peek() not in u'\0 \t\r\n':
684            self.reader.forward()
685        end_marker = self.reader.get_marker()
[44]686        self.tokens.append(TagToken('', start_marker, end_marker))
[43]687
688    def scan_block_scalar(self, folded):
[46]689        start_marker = self.reader.get_marker()
[43]690        indent = self.indent+1
691        if indent < 1:
692            indent = 1
693        while True:
[46]694            while self.reader.peek() and self.reader.peek() and self.reader.peek() not in u'\0\r\n\x85\u2028\u2029':
695                self.reader.forward()
696            if self.reader.peek() != u'\0':
697                self.reader.forward()
[43]698            count = 0
[46]699            while count < indent and self.reader.peek() == u' ':
700                self.reader.forward()
[43]701                count += 1
[46]702            if count < indent and self.reader.peek() not in u'#\r\n\x85\u2028\u2029':
[43]703                break
[44]704        self.tokens.append(ScalarToken('', False, start_marker, start_marker))
[43]705
706    def scan_flow_scalar(self, double):
[46]707        marker = self.reader.get_marker()
708        quote = self.reader.peek()
709        self.reader.forward()
710        while self.reader.peek() != quote:
711            if double and self.reader.peek() == u'\\':
712                self.reader.forward(2)
713            elif not double and self.reader.peek(3)[1:] == u'\'\'':
714                self.reader.forward(3)
[37]715            else:
[46]716                self.reader.forward(1)
717        self.reader.forward(1)
[44]718        self.tokens.append(ScalarToken('', False, marker, marker))
[37]719
[43]720    def scan_plain(self):
721        indent = self.indent+1
722        if indent < 1:
723            indent = 1
724        space = False
[46]725        marker = self.reader.get_marker()
[43]726        while True:
[46]727            while self.reader.peek() == u' ':
728                self.reader.forward()
[43]729                space = True
[46]730            while self.reader.peek() not in u'\0\r\n?:,[]{}#'   \
731                    or (not space and self.reader.peek() == '#')    \
732                    or (not self.flow_level and self.reader.peek() in '?,[]{}') \
733                    or (not self.flow_level and self.reader.peek() == ':' and self.reader.peek(2)[1] not in u' \0\r\n'):
734                space = self.reader.peek() not in u' \t'
735                self.reader.forward()
[43]736                self.allow_simple_key = False
[46]737            if self.reader.peek() not in u'\r\n':
[43]738                break
[46]739            while self.reader.peek() in u'\r\n':
740                self.reader.forward()
[43]741                if not self.flow_level:
742                    self.allow_simple_key = True
743            count = 0
[46]744            while self.reader.peek() == u' ' and count < indent:
745                self.reader.forward()
[43]746                count += 1
747            if count < indent:
748                break
749            space = True
[44]750        self.tokens.append(ScalarToken('', True, marker, marker))
[37]751
[43]752    def invalid_token(self):
753        self.fail("invalid token")
754
755    def fail(self, message):
756        raise ScannerError(message)
757
[45]758#try:
759#    import psyco
760#    psyco.bind(Scanner)
761#except ImportError:
762#    pass
763
Note: See TracBrowser for help on using the repository browser.