source: branches/pyyaml3000/lib/yaml/scanner.py @ 43

Revision 43, 26.0 KB checked in by xi, 9 years ago (diff)

Scanner is mostly completed. Need to write alias, tag, and scalar scanners.

RevLine 
[39]1
[43]2# Tokens:
3# YAML-DIRECTIVE(major_version, minor_version), TAG-DIRECTIVE(handle, prefix)
4# RESERVED-DIRECTIVE(name)
5# DOCUMENT-START, DOCUMENT-END
6# BLOCK-SEQUENCE-START, BLOCK-MAPPING-START, BLOCK-END
7# FLOW-SEQUENCE-START, FLOW-MAPPING-START, FLOW-SEQUENCE-END, FLOW-MAPPING-END
8# ENTRY, KEY, VALUE
9# ALIAS(name), ANCHOR(name), TAG(value), SCALAR(value, plain)
10
11
[39]12from marker import Marker
[43]13#from error import YAMLError
[39]14from stream import Stream
15
[43]16#class ScannerError(YAMLError):
17class ScannerError(Exception):
18    pass
19
20class Token:
21    def __init__(self, start_marker, end_marker):
22        self.start_marker = start_marker
23        self.end_marker = end_marker
24
25class YAMLDirective(Token):
26    def __init__(self, major_version, minor_version, start_marker, end_marker):
27        self.major_version = major_version
28        self.minor_version = minor_version
29        self.start_marker = start_marker
30        self.end_marker = end_marker
31
32class TagDirective(Token):
33    pass
34
35class ReservedDirective(Token):
36    def __init__(self, name, start_marker, end_marker):
37        self.name = name
38        self.start_marker = start_marker
39        self.end_marker = end_marker
40
41class DocumentStart(Token):
42    pass
43
44class DocumentEnd(Token):
45    pass
46
47class End(Token):
48    pass
49
50class BlockSequenceStart(Token):
51    pass
52
53class BlockMappingStart(Token):
54    pass
55
56class BlockEnd(Token):
57    pass
58
59class FlowSequenceStart(Token):
60    pass
61
62class FlowMappingStart(Token):
63    pass
64
65class FlowSequenceEnd(Token):
66    pass
67
68class FlowMappingEnd(Token):
69    pass
70
71class Key(Token):
72    pass
73
74class Value(Token):
75    pass
76
77class Entry(Token):
78    pass
79
80class Alias(Token):
81    def __init__(self, value, start_marker, end_marker):
82        self.value = value
83        self.start_marker = start_marker
84        self.end_marker = end_marker
85
86class Anchor(Token):
87    def __init__(self, value, start_marker, end_marker):
88        self.value = value
89        self.start_marker = start_marker
90        self.end_marker = end_marker
91
92class Tag(Token):
93    def __init__(self, value, start_marker, end_marker):
94        self.value = value
95        self.start_marker = start_marker
96        self.end_marker = end_marker
97
98class Scalar(Token):
99    def __init__(self, value, plain, start_marker, end_marker):
100        self.value = value
101        self.plain = plain
102        self.start_marker = start_marker
103        self.end_marker = end_marker
104
105class SimpleKey:
106    def __init__(self, token_number, required, index, line, column, marker):
107        self.token_number = token_number
108        self.required = required
109        self.index = index
110        self.line = line
111        self.column = column
112        self.marker = marker
113
[39]114class Scanner:
115
116    def __init__(self, source, data):
117        """Initialize the scanner."""
118        # The input stream. The Stream class do the dirty work of checking for
[43]119        # BOM and converting the input data to Unicode. It also adds NUL to
120        # the end.
[39]121        #
122        # Stream supports the following methods
123        #   self.stream.peek(k=1)   # peek the next k characters
124        #   self.stream.read(k=1)   # read the next k characters and move the
125        #                           # pointer
126        self.stream = Stream(source, data)
127
128        # Had we reached the end of the stream?
129        self.done = False
130
131        # The number of unclosed '{' and '['. `flow_level == 0` means block
132        # context.
133        self.flow_level = 0
134
135        # List of processed tokens that are not yet emitted.
136        self.tokens = []
137
138        # Number of tokens that were emitted through the `get_token` method.
139        self.tokens_taken = 0
140
141        # The current indentation level.
142        self.indent = -1
143
144        # Past indentation levels.
145        self.indents = []
146
[43]147        # Variables related to simple keys treatment.
[39]148
149        # A simple key is a key that is not denoted by the '?' indicator.
150        # Example of simple keys:
151        #   ---
152        #   block simple key: value
153        #   ? not a simple key:
154        #   : { flow simple key: value }
155        # We emit the KEY token before all keys, so when we find a potential
156        # simple key, we try to locate the corresponding ':' indicator.
157        # Simple keys should be limited to a single line and 1024 characters.
158
[43]159        # Can a simple key start at the current position? A simple key may
160        # start:
161        # - at the beginning of the line, not counting indentation spaces
162        #       (in block context),
163        # - after '{', '[', ',' (in the flow context),
164        # - after '?', ':', '-' (in the block context).
165        # In the block context, this flag also signify if a block collection
166        # may start at the current position.
167        self.allow_simple_key = True
[39]168
169        # Keep track of possible simple keys. This is a dictionary. The key
170        # is `flow_level`; there can be no more that one possible simple key
[43]171        # for each level. The value is a SimpleKey record:
172        #   (token_number, required, index, line, column, marker)
173        # A simple key may start with ALIAS, ANCHOR, TAG, SCALAR(flow),
174        # '[', or '{' tokens.
[39]175        self.possible_simple_keys = {}
176
[43]177    # Two public methods.
[39]178
179    def peek_token(self):
180        """Get the current token."""
[43]181        while self.need_more_tokens():
[39]182            self.fetch_more_tokens()
183        if self.tokens:
184            return self.tokens[0]
185
186    def get_token(self):
[43]187        "Get the current token and remove it from the list of pending tokens."""
[39]188        while self.need_more_tokens():
189            self.fetch_more_tokens()
190        if self.tokens:
191            self.tokens_taken += 1
192            return self.tokens.pop(0)
193
[43]194    # Private methods.
[39]195
196    def need_more_tokens(self):
197        if self.done:
198            return False
199        if not self.tokens:
200            return True
201        # The current token may be a potential simple key, so we
202        # need to look further.
[43]203        self.stale_possible_simple_keys()
[39]204        if self.next_possible_simple_key() == self.tokens_taken:
205            return True
206
207    def fetch_more_tokens(self):
208
209        # Eat whitespaces and comments until we reach the next token.
[43]210        self.scan_to_next_token()
[39]211
[43]212        # Remove obsolete possible simple keys.
213        self.stale_possible_simple_keys()
214
[39]215        # Compare the current indentation and column. It may add some tokens
[43]216        # and decrease the current indentation level.
[39]217        self.unwind_indent(self.stream.column)
218
[43]219        #print
220        #print self.stream.get_marker().get_snippet()
221
[39]222        # Peek the next character.
223        ch = self.stream.peek()
224
225        # Is it the end of stream?
[43]226        if ch == u'\0':
[39]227            return self.fetch_end()
228
229        # Is it a directive?
230        if ch == u'%' and self.check_directive():
231            return self.fetch_directive()
232
233        # Is it the document start?
234        if ch == u'-' and self.check_document_start():
235            return self.fetch_document_start()
236
237        # Is it the document end?
238        if ch == u'.' and self.check_document_end():
239            return self.fetch_document_end()
240
241        # Note: the order of the following checks is NOT significant.
242
243        # Is it the flow sequence start indicator?
244        if ch == u'[':
245            return self.fetch_flow_sequence_start()
246
247        # Is it the flow mapping start indicator?
248        if ch == u'{':
249            return self.fetch_flow_mapping_start()
250
251        # Is it the flow sequence end indicator?
252        if ch == u']':
253            return self.fetch_flow_sequence_end()
254
255        # Is it the flow mapping end indicator?
256        if ch == u'}':
257            return self.fetch_flow_mapping_end()
258
[43]259        # Is it the entry indicator?
260        if ch in u'-,' and self.check_entry():
261            return self.fetch_entry()
262
[39]263        # Is it the key indicator?
264        if ch == u'?' and self.check_key():
265            return self.fetch_key()
266
267        # Is it the value indicator?
268        if ch == u':' and self.check_value():
269            return self.fetch_value()
270
271        # Is it an alias?
272        if ch == u'*':
273            return self.fetch_alias()
274
275        # Is it an anchor?
276        if ch == u'&':
277            return self.fetch_anchor()
278
[43]279        # Is it a tag?
[39]280        if ch == u'!':
281            return self.fetch_tag()
282
[43]283        # Is it a literal scalar?
284        if ch == u'|' and not self.flow_level:
[39]285            return self.fetch_literal()
286
287        # Is it a folded scalar?
[43]288        if ch == u'>' and not self.flow_level:
[39]289            return self.fetch_folded()
290
291        # Is it a single quoted scalar?
292        if ch == u'\'':
293            return self.fetch_single()
294
295        # Is it a double quoted scalar?
296        if ch == u'\"':
297            return self.fetch_double()
298
[43]299        # It must be a plain scalar then.
[39]300        if self.check_plain():
301            return self.fetch_plain()
302
[43]303        # No? It's an error. Let's produce a nice error message.
[39]304        self.invalid_token()
305
[43]306    # Simple keys treatment.
307
308    def next_possible_simple_key(self):
309        # Return the number of the nearest possible simple key. Actually we
310        # don't need to loop through the whole dictionary. We may replace it
311        # with the following code:
312        #   if not self.possible_simple_keys:
313        #       return None
314        #   return self.possible_simple_keys[
315        #           min(self.possible_simple_keys.keys())].token_number
316        min_token_number = None
317        for level in self.possible_simple_keys:
318            key = self.possible_simple_keys[level]
319            if min_token_number is None or key.token_number < min_token_number:
320                min_token_number = key.token_number
321        return min_token_number
322
323    def stale_possible_simple_keys(self):
324        # Remove entries that are no longer possible simple keys. According to
325        # the YAML specification, simple keys
326        # - should be limited to a single line,
327        # - should be no longer than 1024 characters.
328        # Disabling this procedure will allow simple keys of any length and
329        # height (may cause problems if indentation is broken though).
330        for level in self.possible_simple_keys.keys():
331            key = self.possible_simple_keys[level]
332            if key.line != self.stream.line  \
333                    or self.stream.index-key.index > 1024:
334                if key.required:
335                    self.fail("simple key is required")
336                del self.possible_simple_keys[level]
337
338    def save_possible_simple_key(self):
339        # The next token may start a simple key. We check if it's possible
340        # and save its position. This function is called for
341        #   ALIAS, ANCHOR, TAG, SCALAR(flow), '[', and '{'.
342
343        # Check if a simple key is required at the current position.
344        required = not self.flow_level and self.indent == self.stream.column
345
346        # The next token might be a simple key. Let's save it's number and
347        # position.
348        if self.allow_simple_key:
349            self.remove_possible_simple_key()
350            token_number = self.tokens_taken+len(self.tokens)
351            index = self.stream.index
352            line = self.stream.line
353            column = self.stream.column
354            marker = self.stream.get_marker()
355            key = SimpleKey(token_number, required,
356                    index, line, column, marker)
357            self.possible_simple_keys[self.flow_level] = key
358
359        # A simple key is required at the current position.
360        elif required:
361            self.fail("simple key is required")
362
363    def remove_possible_simple_key(self):
364        # Remove the saved possible key position at the current flow level.
365        if self.flow_level in self.possible_simple_keys:
366            key = self.possible_simple_keys[self.flow_level]
367            if key.required:
368                self.fail("simple key is required")
369
370    # Indentation functions.
371
372    def unwind_indent(self, column):
373
374        # In flow context, tokens should respect indentation.
375        if self.flow_level and self.indent > column:
376            self.fail("invalid intendation in the flow context")
377
378        # In block context, we may need to issue the BLOCK-END tokens.
379        while self.indent > column:
380            marker = self.stream.get_marker()
381            self.indent = self.indents.pop()
382            self.tokens.append(BlockEnd(marker, marker))
383
384    def add_indent(self, column):
385        # Check if we need to increase indentation.
386        if self.indent < column:
387            self.indents.append(self.indent)
388            self.indent = column
389            return True
390        return False
391
392    # Fetchers.
393
[39]394    def fetch_end(self):
395
396        # Set the current intendation to -1.
[43]397        self.unwind_indent(-1)
[39]398
399        # Reset everything (not really needed).
[43]400        self.allow_simple_key = False
[39]401        self.possible_simple_keys = {}
402
[43]403        # Read the token.
[39]404        marker = self.stream.get_marker()
[43]405       
406        # Add END.
407        self.tokens.append(End(marker, marker))
[39]408
409        # The stream is ended.
410        self.done = True
411
[43]412    def fetch_directive(self):
413       
414        # Set the current intendation to -1.
415        self.unwind_indent(-1)
[39]416
[43]417        # Reset simple keys.
418        self.remove_possible_simple_key()
419        self.allow_simple_key = False
[39]420
[43]421        # Scan and add DIRECTIVE.
422        self.scan_directive()
[39]423
424    def fetch_document_start(self):
[43]425        self.fetch_document_indicator(DocumentStart)
[39]426
[43]427    def fetch_document_end(self):
428        self.fetch_document_indicator(DocumentEnd)
429
430    def fetch_document_indicator(self, TokenClass):
431
[39]432        # Set the current intendation to -1.
[43]433        self.unwind_indent(-1)
[39]434
[43]435        # Reset simple keys. Note that there could not be a block collection
436        # after '---'.
437        self.remove_possible_simple_key()
438        self.allow_simple_key = False
[39]439
[43]440        # Add DOCUMENT-START or DOCUMENT-END.
441        start_marker = self.stream.get_marker()
442        self.stream.read(3)
443        end_marker = self.stream.get_marker()
444        self.tokens.append(TokenClass(start_marker, end_marker))
[39]445
[43]446    def fetch_flow_sequence_start(self):
447        self.fetch_flow_collection_start(FlowSequenceStart)
[39]448
[43]449    def fetch_flow_mapping_start(self):
450        self.fetch_flow_collection_start(FlowMappingStart)
451
452    def fetch_flow_collection_start(self, TokenClass):
453
454        # Increase the flow level.
455        self.flow_level += 1
456
457        # '[' and '{' may start a simple key.
458        self.save_possible_simple_key()
459
460        # Simple keys are allowed after '[' and '{'.
461        self.allow_simple_key = True
462
463        # Add FLOW-SEQUENCE-START or FLOW-MAPPING-START.
[39]464        start_marker = self.stream.get_marker()
[43]465        self.stream.read()
466        end_marker = self.stream.get_marker()
467        self.tokens.append(TokenClass(start_marker, end_marker))
[39]468
[43]469    def fetch_flow_sequence_end(self):
470        self.fetch_flow_collection_end(FlowSequenceEnd)
[39]471
[43]472    def fetch_flow_mapping_end(self):
473        self.fetch_flow_collection_end(FlowMappingEnd)
474
475    def fetch_flow_collection_end(self, TokenClass):
476
477        # Reset possible simple key on the current level.
478        self.remove_possible_simple_key()
479
480        # Decrease the flow level.
481        self.flow_level -= 1
482
483        # No simple keys after ']' or '}'.
484        self.allow_simple_key = False
485
486        # Add FLOW-SEQUENCE-END or FLOW-MAPPING-END.
487        start_marker = self.stream.get_marker()
488        self.stream.read()
[39]489        end_marker = self.stream.get_marker()
[43]490        self.tokens.append(TokenClass(start_marker, end_marker))
[39]491
[43]492    def fetch_entry(self):
[39]493
[43]494        # Block context needs additional checks.
495        if not self.flow_level:
[39]496
[43]497            # Are we allowed to start a new entry?
498            if not self.allow_simple_key:
499                self.fail("Cannot start a new entry here")
[39]500
[43]501            # We may need to add BLOCK-SEQUENCE-START.
502            if self.add_indent(self.stream.column):
503                marker = self.stream.get_marker()
504                self.tokens.append(BlockSequenceStart(marker, marker))
[39]505
[43]506        # Simple keys are allowed after '-' and ','.
507        self.allow_simple_key = True
[39]508
[43]509        # Reset possible simple key on the current level.
510        self.remove_possible_simple_key()
[39]511
[43]512        # Add ENTRY.
[39]513        start_marker = self.stream.get_marker()
[43]514        self.stream.read()
515        end_marker = self.stream.get_marker()
516        self.tokens.append(Entry(start_marker, end_marker))
[39]517
[43]518    def fetch_key(self):
519       
520        # Block context needs additional checks.
521        if not self.flow_level:
[39]522
[43]523            # Are we allowed to start a key (not nessesary a simple)?
524            if not self.allow_simple_key:
525                self.fail("Cannot start a new key here")
526
527            # We may need to add BLOCK-MAPPING-START.
528            if self.add_indent(self.stream.column):
529                marker = self.stream.get_marker()
530                self.tokens.append(BlockMappingStart(marker, marker))
531
532        # Simple keys are allowed after '?' in the block context.
533        self.allow_simple_key = not self.flow_level
534
535        # Reset possible simple key on the current level.
536        self.remove_possible_simple_key()
537
538        # Add KEY.
539        start_marker = self.stream.get_marker()
540        self.stream.read()
[39]541        end_marker = self.stream.get_marker()
[43]542        self.tokens.append(Key(start_marker, end_marker))
[39]543
[43]544    def fetch_value(self):
[39]545
[43]546        # Do we determine a simple key?
547        if self.flow_level in self.possible_simple_keys:
[39]548
[43]549            # Add KEY.
550            key = self.possible_simple_keys[self.flow_level]
551            del self.possible_simple_keys[self.flow_level]
552            self.tokens.insert(key.token_number-self.tokens_taken,
553                    Key(key.marker, key.marker))
[39]554
[43]555            # If this key starts a new block mapping, we need to add
556            # BLOCK-MAPPING-START.
557            if not self.flow_level:
558                if self.add_indent(key.column):
559                    self.tokens.insert(key.token_number-self.tokens_taken,
560                            BlockMappingStart(key.marker, key.marker))
[37]561
[43]562            # There cannot be two simple keys one after another.
563            self.allow_simple_key = False
[37]564
[43]565        # It must be a part of a complex key.
566        else:
567           
568            # Simple keys are allowed after ':' in the block context.
569            self.allow_simple_key = not self.flow_level
[37]570
[43]571            # Reset possible simple key on the current level.
572            self.remove_possible_simple_key()
[37]573
[43]574        # Add VALUE.
575        start_marker = self.stream.get_marker()
576        self.stream.read()
577        end_marker = self.stream.get_marker()
578        self.tokens.append(Value(start_marker, end_marker))
[37]579
[43]580    def fetch_alias(self):
[37]581
[43]582        # ALIAS could be a simple key.
583        self.save_possible_simple_key()
[37]584
[43]585        # No simple keys after ALIAS.
586        self.allow_simple_key = False
[37]587
[43]588        # Scan and add ALIAS.
589        self.scan_anchor(Alias)
[37]590
[43]591    def fetch_anchor(self):
[37]592
[43]593        # ANCHOR could start a simple key.
594        self.save_possible_simple_key()
[37]595
[43]596        # No simple keys after ANCHOR.
597        self.allow_simple_key = False
[37]598
[43]599        # Scan and add ANCHOR.
600        self.scan_anchor(Anchor)
[37]601
[43]602    def fetch_tag(self):
[37]603
[43]604        # TAG could start a simple key.
605        self.save_possible_simple_key()
[37]606
[43]607        # No simple keys after TAG.
608        self.allow_simple_key = False
[37]609
[43]610        # Scan and add TAG.
611        self.scan_tag()
[37]612
[43]613    def fetch_literal(self):
614        self.fetch_block_scalar(folded=False)
[37]615
[43]616    def fetch_folded(self):
617        self.fetch_block_scalar(folded=True)
[37]618
[43]619    def fetch_block_scalar(self, folded):
[37]620
[43]621        # A simple key may follow a block scalar.
622        self.allow_simple_key = True
[37]623
[43]624        # Reset possible simple key on the current level.
625        self.remove_possible_simple_key()
[37]626
[43]627        # Scan and add SCALAR.
628        self.scan_block_scalar(folded)
[37]629
[43]630    def fetch_single(self):
631        self.fetch_flow_scalar(double=False)
[37]632
[43]633    def fetch_double(self):
634        self.fetch_flow_scalar(double=True)
[37]635
[43]636    def fetch_flow_scalar(self, double):
[37]637
[43]638        # A flow scalar could be a simple key.
639        self.save_possible_simple_key()
[37]640
[43]641        # No simple keys after flow scalars.
642        self.allow_simple_key = False
[37]643
[43]644        # Scan and add SCALAR.
645        self.scan_flow_scalar(double)
[37]646
[43]647    def fetch_plain(self):
[37]648
[43]649        # A plain scalar could be a simple key.
650        self.save_possible_simple_key()
[37]651
[43]652        # No simple keys after plain scalars. But note that `scan_plain` will
653        # change this flag if the scan is finished at the beginning of the
654        # line.
655        self.allow_simple_key = False
[37]656
[43]657        # Scan and add SCALAR. May change `allow_simple_key`.
658        self.scan_plain()
[37]659
[43]660    # Checkers.
[37]661
[43]662    def check_directive(self):
[37]663
[43]664        # DIRECTIVE:        ^ '%' ...
665        # The '%' indicator is already checked.
666        if self.stream.column == 0:
667            return True
[37]668
[43]669    def check_document_start(self):
[37]670
[43]671        # DOCUMENT-START:   ^ '---' (' '|'\n')
672        if self.stream.column == 0:
673            prefix = self.stream.peek(4)
674            if prefix[:3] == u'---' and prefix[3] in u'\0 \t\r\n\x85\u2028\u2029':
675                return True
[37]676
[43]677    def check_document_end(self):
[37]678
[43]679        # DOCUMENT-END:     ^ '...' (' '|'\n')
680        if self.stream.column == 0:
681            prefix = self.stream.peek(4)
682            if prefix[:3] == u'...' and prefix[3] in u'\0 \t\r\n\x85\u2028\u2029':
683                return True
[37]684
[43]685    def check_entry(self):
686
687        # ENTRY(flow context):      ','
688        if self.flow_level:
689            return self.stream.peek() == u','
690
691        # ENTRY(block context):     '-' (' '|'\n')
692        else:
693            prefix = self.stream.peek(2)
694            return prefix[0] == u'-' and prefix[1] in u'\0 \t\r\n\x85\u2028\u2029'
695
696    def check_key(self):
697
698        # KEY(flow context):    '?'
699        if self.flow_level:
[37]700            return True
[43]701
702        # KEY(block context):   '?' (' '|'\n')
[37]703        else:
[43]704            prefix = self.stream.peek(2)
705            return prefix[1] in u'\0 \t\r\n\x85\u2028\u2029'
[37]706
[43]707    def check_value(self):
708
709        # VALUE(flow context):  ':'
710        if self.flow_level:
[37]711            return True
[43]712
713        # VALUE(block context): ':' (' '|'\n')
[37]714        else:
[43]715            prefix = self.stream.peek(2)
716            return prefix[1] in u'\0 \t\r\n\x85\u2028\u2029'
[37]717
[43]718    def check_plain(self):
719        return True
[37]720
[43]721    # Scanners.
722
723    def scan_to_next_token(self):
724        found = False
725        while not found:
726            while self.stream.peek() == u' ':
727                self.stream.read()
728            if self.stream.peek() == u'#':
729                while self.stream.peek() not in u'\r\n':
730                    self.stream.read()
731            if self.stream.peek() in u'\r\n':
732                self.stream.read()
733                if not self.flow_level:
734                    self.allow_simple_key = True
[37]735            else:
[43]736                found = True
[37]737
[43]738    def scan_directive(self):
739        marker = self.stream.get_marker()
740        if self.stream.peek(5) == u'%YAML ':
741            self.tokens.append(YAMLDirective(1, 1, marker, marker))
742        elif self.stream.peek(4) == u'%TAG ':
743            self.tokens.append(TagDirective(marker, marker))
744        else:
745            self.tokens.append(ReservedDirective('', marker, marker))
746        while self.stream.peek() not in u'\0\r\n':
747            self.stream.read()
748        self.stream.read()
[37]749
[43]750    def scan_anchor(self, TokenClass):
751        start_marker = self.stream.get_marker()
752        while self.stream.peek() not in u'\0 \t\r\n,:':
753            self.stream.read()
754        end_marker = self.stream.get_marker()
755        self.tokens.append(TokenClass('', start_marker, end_marker))
[37]756
[43]757    def scan_tag(self):
758        start_marker = self.stream.get_marker()
759        while self.stream.peek() not in u'\0 \t\r\n':
760            self.stream.read()
761        end_marker = self.stream.get_marker()
762        self.tokens.append(Tag('', start_marker, end_marker))
763
764    def scan_block_scalar(self, folded):
765        start_marker = self.stream.get_marker()
766        indent = self.indent+1
767        if indent < 1:
768            indent = 1
769        while True:
770            while self.stream.peek() and self.stream.peek() and self.stream.peek() not in u'\0\r\n':
771                self.stream.read()
772            if self.stream.peek() != u'\0':
773                self.stream.read()
774            count = 0
775            while count < indent and self.stream.peek() == u' ':
776                self.stream.read()
777                count += 1
778            if count < indent and self.stream.peek() not in u'#\r\n':
779                break
780        self.tokens.append(Scalar('', False, start_marker, start_marker))
781
782    def scan_flow_scalar(self, double):
783        marker = self.stream.get_marker()
784        quote = self.stream.read()
785        while self.stream.peek() != quote:
786            if double and self.stream.peek() == u'\\':
787                self.stream.read(2)
788            elif not double and self.stream.peek(3)[1:] == u'\'\'':
789                self.stream.read(3)
[37]790            else:
[43]791                self.stream.read(1)
792        self.stream.read(1)
793        self.tokens.append(Scalar('', False, marker, marker))
[37]794
[43]795    def scan_plain(self):
796        indent = self.indent+1
797        if indent < 1:
798            indent = 1
799        space = False
800        marker = self.stream.get_marker()
801        while True:
802            while self.stream.peek() == u' ':
803                self.stream.read()
804                space = True
805            while self.stream.peek() not in u'\0\r\n?:,[]{}#'   \
806                    or (not space and self.stream.peek() == '#')    \
807                    or (not self.flow_level and self.stream.peek() in '?,[]{}') \
808                    or (not self.flow_level and self.stream.peek() == ':' and self.stream.peek(2)[1] not in u' \0\r\n'):
809                space = self.stream.peek() not in u' \t'
810                self.stream.read()
811                self.allow_simple_key = False
812            if self.stream.peek() not in u'\r\n':
813                break
814            while self.stream.peek() in u'\r\n':
815                self.stream.read()
816                if not self.flow_level:
817                    self.allow_simple_key = True
818            count = 0
819            while self.stream.peek() == u' ' and count < indent:
820                self.stream.read()
821                count += 1
822            if count < indent:
823                break
824            space = True
825        self.tokens.append(Scalar('', True, marker, marker))
[37]826
[43]827    def invalid_token(self):
828        self.fail("invalid token")
829
830    def fail(self, message):
831        raise ScannerError(message)
832
Note: See TracBrowser for help on using the repository browser.