source: pyyaml/trunk/lib/yaml/scanner.py @ 130

Revision 130, 52.7 KB checked in by xi, 9 years ago (diff)

Add style information to events generated by Parser.

RevLine 
[39]1
[55]2# Scanner produces tokens of the following types:
[118]3# STREAM-START
4# STREAM-END
[55]5# DIRECTIVE(name, value)
6# DOCUMENT-START
7# DOCUMENT-END
8# BLOCK-SEQUENCE-START
9# BLOCK-MAPPING-START
10# BLOCK-END
11# FLOW-SEQUENCE-START
12# FLOW-MAPPING-START
13# FLOW-SEQUENCE-END
14# FLOW-MAPPING-END
15# BLOCK-ENTRY
16# FLOW-ENTRY
17# KEY
18# VALUE
19# ALIAS(value)
20# ANCHOR(value)
21# TAG(value)
22# SCALAR(value, plain)
[57]23#
24# Read comments in the Scanner code for more details.
25#
[43]26
[46]27__all__ = ['Scanner', 'ScannerError']
[43]28
[52]29from error import MarkedYAMLError
[46]30from tokens import *
[39]31
[52]32class ScannerError(MarkedYAMLError):
33    pass
[51]34
[43]35class SimpleKey:
[51]36    # See below simple keys treatment.
37
[116]38    def __init__(self, token_number, required, index, line, column, mark):
[43]39        self.token_number = token_number
40        self.required = required
41        self.index = index
42        self.line = line
43        self.column = column
[116]44        self.mark = mark
[43]45
[39]46class Scanner:
47
[46]48
49    def __init__(self, reader):
[39]50        """Initialize the scanner."""
[46]51        # The input stream. The Reader class do the dirty work of checking for
[43]52        # BOM and converting the input data to Unicode. It also adds NUL to
53        # the end.
[39]54        #
[46]55        # Reader supports the following methods
[48]56        #   self.reader.peek(i=0)       # peek the next i-th character
57        #   self.reader.prefix(l=1)     # peek the next l characters
58        #   self.reader.forward(l=1)    # read the next l characters
59                                        # and move the pointer
[46]60        self.reader = reader
[39]61
62        # Had we reached the end of the stream?
63        self.done = False
64
65        # The number of unclosed '{' and '['. `flow_level == 0` means block
66        # context.
67        self.flow_level = 0
68
69        # List of processed tokens that are not yet emitted.
70        self.tokens = []
71
[118]72        # Add the STREAM-START token.
73        self.fetch_stream_start()
74
[39]75        # Number of tokens that were emitted through the `get_token` method.
76        self.tokens_taken = 0
77
78        # The current indentation level.
79        self.indent = -1
80
81        # Past indentation levels.
82        self.indents = []
83
[43]84        # Variables related to simple keys treatment.
[39]85
86        # A simple key is a key that is not denoted by the '?' indicator.
87        # Example of simple keys:
88        #   ---
89        #   block simple key: value
90        #   ? not a simple key:
91        #   : { flow simple key: value }
92        # We emit the KEY token before all keys, so when we find a potential
93        # simple key, we try to locate the corresponding ':' indicator.
94        # Simple keys should be limited to a single line and 1024 characters.
95
[43]96        # Can a simple key start at the current position? A simple key may
97        # start:
98        # - at the beginning of the line, not counting indentation spaces
99        #       (in block context),
100        # - after '{', '[', ',' (in the flow context),
101        # - after '?', ':', '-' (in the block context).
[60]102        # In the block context, this flag also signifies if a block collection
[43]103        # may start at the current position.
104        self.allow_simple_key = True
[39]105
106        # Keep track of possible simple keys. This is a dictionary. The key
107        # is `flow_level`; there can be no more that one possible simple key
[43]108        # for each level. The value is a SimpleKey record:
[116]109        #   (token_number, required, index, line, column, mark)
[43]110        # A simple key may start with ALIAS, ANCHOR, TAG, SCALAR(flow),
111        # '[', or '{' tokens.
[39]112        self.possible_simple_keys = {}
113
[51]114    # Public methods.
[39]115
[51]116    def check(self, *choices):
117        # Check if the next token is one of the given types.
[43]118        while self.need_more_tokens():
[39]119            self.fetch_more_tokens()
120        if self.tokens:
[51]121            for choice in choices:
122                if isinstance(self.tokens[0], choice):
123                    return True
124        return False
125
126    def peek(self):
127        # Return the next token, but do not delete if from the queue.
128        while self.need_more_tokens():
129            self.fetch_more_tokens()
130        if self.tokens:
[39]131            return self.tokens[0]
132
[51]133    def get(self):
134        # Return the next token.
[39]135        while self.need_more_tokens():
136            self.fetch_more_tokens()
137        if self.tokens:
138            self.tokens_taken += 1
139            return self.tokens.pop(0)
140
[51]141    def __iter__(self):
142        # Iterator protocol.
143        while self.need_more_tokens():
144            self.fetch_more_tokens()
145        while self.tokens:
146            self.tokens_taken += 1
147            yield self.tokens.pop(0)
148            while self.need_more_tokens():
149                self.fetch_more_tokens()
150
[43]151    # Private methods.
[39]152
153    def need_more_tokens(self):
154        if self.done:
155            return False
156        if not self.tokens:
157            return True
158        # The current token may be a potential simple key, so we
159        # need to look further.
[43]160        self.stale_possible_simple_keys()
[39]161        if self.next_possible_simple_key() == self.tokens_taken:
162            return True
163
164    def fetch_more_tokens(self):
165
166        # Eat whitespaces and comments until we reach the next token.
[43]167        self.scan_to_next_token()
[39]168
[43]169        # Remove obsolete possible simple keys.
170        self.stale_possible_simple_keys()
171
[39]172        # Compare the current indentation and column. It may add some tokens
[43]173        # and decrease the current indentation level.
[46]174        self.unwind_indent(self.reader.column)
[39]175
176        # Peek the next character.
[46]177        ch = self.reader.peek()
[39]178
[48]179        # Is it the end of stream?
[43]180        if ch == u'\0':
[48]181            return self.fetch_stream_end()
[39]182
183        # Is it a directive?
184        if ch == u'%' and self.check_directive():
185            return self.fetch_directive()
186
187        # Is it the document start?
188        if ch == u'-' and self.check_document_start():
189            return self.fetch_document_start()
190
191        # Is it the document end?
192        if ch == u'.' and self.check_document_end():
193            return self.fetch_document_end()
194
[52]195        # TODO: support for BOM within a stream.
196        #if ch == u'\uFEFF':
197        #    return self.fetch_bom()    <-- issue BOMToken
198
[39]199        # Note: the order of the following checks is NOT significant.
200
201        # Is it the flow sequence start indicator?
202        if ch == u'[':
203            return self.fetch_flow_sequence_start()
204
205        # Is it the flow mapping start indicator?
206        if ch == u'{':
207            return self.fetch_flow_mapping_start()
208
209        # Is it the flow sequence end indicator?
210        if ch == u']':
211            return self.fetch_flow_sequence_end()
212
213        # Is it the flow mapping end indicator?
214        if ch == u'}':
215            return self.fetch_flow_mapping_end()
216
[51]217        # Is it the flow entry indicator?
218        if ch in u',':
219            return self.fetch_flow_entry()
[43]220
[51]221        # Is it the block entry indicator?
222        if ch in u'-' and self.check_block_entry():
223            return self.fetch_block_entry()
224
[39]225        # Is it the key indicator?
226        if ch == u'?' and self.check_key():
227            return self.fetch_key()
228
229        # Is it the value indicator?
230        if ch == u':' and self.check_value():
231            return self.fetch_value()
232
233        # Is it an alias?
234        if ch == u'*':
235            return self.fetch_alias()
236
237        # Is it an anchor?
238        if ch == u'&':
239            return self.fetch_anchor()
240
[43]241        # Is it a tag?
[39]242        if ch == u'!':
243            return self.fetch_tag()
244
[43]245        # Is it a literal scalar?
246        if ch == u'|' and not self.flow_level:
[39]247            return self.fetch_literal()
248
249        # Is it a folded scalar?
[43]250        if ch == u'>' and not self.flow_level:
[39]251            return self.fetch_folded()
252
253        # Is it a single quoted scalar?
254        if ch == u'\'':
255            return self.fetch_single()
256
257        # Is it a double quoted scalar?
258        if ch == u'\"':
259            return self.fetch_double()
260
[43]261        # It must be a plain scalar then.
[39]262        if self.check_plain():
263            return self.fetch_plain()
264
[43]265        # No? It's an error. Let's produce a nice error message.
[48]266        raise ScannerError("while scanning for the next token", None,
267                "found character %r that cannot start any token"
[116]268                % ch.encode('utf-8'), self.reader.get_mark())
[39]269
[43]270    # Simple keys treatment.
271
272    def next_possible_simple_key(self):
273        # Return the number of the nearest possible simple key. Actually we
274        # don't need to loop through the whole dictionary. We may replace it
275        # with the following code:
276        #   if not self.possible_simple_keys:
277        #       return None
278        #   return self.possible_simple_keys[
279        #           min(self.possible_simple_keys.keys())].token_number
280        min_token_number = None
281        for level in self.possible_simple_keys:
282            key = self.possible_simple_keys[level]
283            if min_token_number is None or key.token_number < min_token_number:
284                min_token_number = key.token_number
285        return min_token_number
286
287    def stale_possible_simple_keys(self):
288        # Remove entries that are no longer possible simple keys. According to
289        # the YAML specification, simple keys
290        # - should be limited to a single line,
291        # - should be no longer than 1024 characters.
292        # Disabling this procedure will allow simple keys of any length and
293        # height (may cause problems if indentation is broken though).
294        for level in self.possible_simple_keys.keys():
295            key = self.possible_simple_keys[level]
[46]296            if key.line != self.reader.line  \
297                    or self.reader.index-key.index > 1024:
[43]298                if key.required:
[116]299                    raise ScannerError("while scanning a simple key", key.mark,
300                            "could not found expected ':'", self.reader.get_mark())
[43]301                del self.possible_simple_keys[level]
302
303    def save_possible_simple_key(self):
304        # The next token may start a simple key. We check if it's possible
305        # and save its position. This function is called for
306        #   ALIAS, ANCHOR, TAG, SCALAR(flow), '[', and '{'.
307
308        # Check if a simple key is required at the current position.
[46]309        required = not self.flow_level and self.indent == self.reader.column
[43]310
[47]311        # A simple key is required only if it is the first token in the current
312        # line. Therefore it is always allowed.
313        assert self.allow_simple_key or not required
314
[43]315        # The next token might be a simple key. Let's save it's number and
316        # position.
317        if self.allow_simple_key:
318            self.remove_possible_simple_key()
319            token_number = self.tokens_taken+len(self.tokens)
[46]320            index = self.reader.index
321            line = self.reader.line
322            column = self.reader.column
[116]323            mark = self.reader.get_mark()
[43]324            key = SimpleKey(token_number, required,
[116]325                    index, line, column, mark)
[43]326            self.possible_simple_keys[self.flow_level] = key
327
328    def remove_possible_simple_key(self):
329        # Remove the saved possible key position at the current flow level.
330        if self.flow_level in self.possible_simple_keys:
331            key = self.possible_simple_keys[self.flow_level]
[47]332           
333            # I don't think it's possible, but I could be wrong.
334            assert not key.required
335            #if key.required:
[116]336            #    raise ScannerError("while scanning a simple key", key.mark,
337            #            "could not found expected ':'", self.reader.get_mark())
[43]338
339    # Indentation functions.
340
341    def unwind_indent(self, column):
342
[117]343        ## In flow context, tokens should respect indentation.
344        ## Actually the condition should be `self.indent >= column` according to
345        ## the spec. But this condition will prohibit intuitively correct
346        ## constructions such as
347        ## key : {
348        ## }
349        #if self.flow_level and self.indent > column:
350        #    raise ScannerError(None, None,
351        #            "invalid intendation or unclosed '[' or '{'",
352        #            self.reader.get_mark())
[43]353
[117]354        # In the flow context, indentation is ignored. We make the scanner less
355        # restrictive then specification requires.
356        if self.flow_level:
357            return
358
[43]359        # In block context, we may need to issue the BLOCK-END tokens.
360        while self.indent > column:
[116]361            mark = self.reader.get_mark()
[43]362            self.indent = self.indents.pop()
[116]363            self.tokens.append(BlockEndToken(mark, mark))
[43]364
365    def add_indent(self, column):
366        # Check if we need to increase indentation.
367        if self.indent < column:
368            self.indents.append(self.indent)
369            self.indent = column
370            return True
371        return False
372
373    # Fetchers.
374
[118]375    def fetch_stream_start(self):
376        # We always add STREAM-START as the first token and STREAM-END as the
377        # last token.
378
379        # Read the token.
380        mark = self.reader.get_mark()
381       
[130]382        # Add STREAM-START.
383        self.tokens.append(StreamStartToken(mark, mark,
384            encoding=self.reader.encoding))
[118]385       
386
[48]387    def fetch_stream_end(self):
[39]388
389        # Set the current intendation to -1.
[43]390        self.unwind_indent(-1)
[39]391
392        # Reset everything (not really needed).
[43]393        self.allow_simple_key = False
[39]394        self.possible_simple_keys = {}
395
[43]396        # Read the token.
[116]397        mark = self.reader.get_mark()
[43]398       
[118]399        # Add STREAM-END.
[116]400        self.tokens.append(StreamEndToken(mark, mark))
[39]401
[46]402        # The reader is ended.
[39]403        self.done = True
404
[43]405    def fetch_directive(self):
406       
407        # Set the current intendation to -1.
408        self.unwind_indent(-1)
[39]409
[43]410        # Reset simple keys.
411        self.remove_possible_simple_key()
412        self.allow_simple_key = False
[39]413
[43]414        # Scan and add DIRECTIVE.
[47]415        self.tokens.append(self.scan_directive())
[39]416
417    def fetch_document_start(self):
[44]418        self.fetch_document_indicator(DocumentStartToken)
[39]419
[43]420    def fetch_document_end(self):
[44]421        self.fetch_document_indicator(DocumentEndToken)
[43]422
423    def fetch_document_indicator(self, TokenClass):
424
[39]425        # Set the current intendation to -1.
[43]426        self.unwind_indent(-1)
[39]427
[43]428        # Reset simple keys. Note that there could not be a block collection
429        # after '---'.
430        self.remove_possible_simple_key()
431        self.allow_simple_key = False
[39]432
[43]433        # Add DOCUMENT-START or DOCUMENT-END.
[116]434        start_mark = self.reader.get_mark()
[46]435        self.reader.forward(3)
[116]436        end_mark = self.reader.get_mark()
437        self.tokens.append(TokenClass(start_mark, end_mark))
[39]438
[43]439    def fetch_flow_sequence_start(self):
[44]440        self.fetch_flow_collection_start(FlowSequenceStartToken)
[39]441
[43]442    def fetch_flow_mapping_start(self):
[44]443        self.fetch_flow_collection_start(FlowMappingStartToken)
[43]444
445    def fetch_flow_collection_start(self, TokenClass):
446
[44]447        # '[' and '{' may start a simple key.
448        self.save_possible_simple_key()
449
[43]450        # Increase the flow level.
451        self.flow_level += 1
452
453        # Simple keys are allowed after '[' and '{'.
454        self.allow_simple_key = True
455
456        # Add FLOW-SEQUENCE-START or FLOW-MAPPING-START.
[116]457        start_mark = self.reader.get_mark()
[46]458        self.reader.forward()
[116]459        end_mark = self.reader.get_mark()
460        self.tokens.append(TokenClass(start_mark, end_mark))
[39]461
[43]462    def fetch_flow_sequence_end(self):
[44]463        self.fetch_flow_collection_end(FlowSequenceEndToken)
[39]464
[43]465    def fetch_flow_mapping_end(self):
[44]466        self.fetch_flow_collection_end(FlowMappingEndToken)
[43]467
468    def fetch_flow_collection_end(self, TokenClass):
469
470        # Reset possible simple key on the current level.
471        self.remove_possible_simple_key()
472
473        # Decrease the flow level.
474        self.flow_level -= 1
475
476        # No simple keys after ']' or '}'.
477        self.allow_simple_key = False
478
479        # Add FLOW-SEQUENCE-END or FLOW-MAPPING-END.
[116]480        start_mark = self.reader.get_mark()
[46]481        self.reader.forward()
[116]482        end_mark = self.reader.get_mark()
483        self.tokens.append(TokenClass(start_mark, end_mark))
[39]484
[51]485    def fetch_flow_entry(self):
[39]486
[51]487        # Simple keys are allowed after ','.
488        self.allow_simple_key = True
489
490        # Reset possible simple key on the current level.
491        self.remove_possible_simple_key()
492
493        # Add FLOW-ENTRY.
[116]494        start_mark = self.reader.get_mark()
[51]495        self.reader.forward()
[116]496        end_mark = self.reader.get_mark()
497        self.tokens.append(FlowEntryToken(start_mark, end_mark))
[51]498
499    def fetch_block_entry(self):
500
[43]501        # Block context needs additional checks.
502        if not self.flow_level:
[39]503
[43]504            # Are we allowed to start a new entry?
505            if not self.allow_simple_key:
[47]506                raise ScannerError(None, None,
507                        "sequence entries are not allowed here",
[116]508                        self.reader.get_mark())
[39]509
[43]510            # We may need to add BLOCK-SEQUENCE-START.
[46]511            if self.add_indent(self.reader.column):
[116]512                mark = self.reader.get_mark()
513                self.tokens.append(BlockSequenceStartToken(mark, mark))
[39]514
[51]515        # It's an error for the block entry to occur in the flow context,
516        # but we let the parser detect this.
517        else:
518            pass
519
520        # Simple keys are allowed after '-'.
[43]521        self.allow_simple_key = True
[39]522
[43]523        # Reset possible simple key on the current level.
524        self.remove_possible_simple_key()
[39]525
[51]526        # Add BLOCK-ENTRY.
[116]527        start_mark = self.reader.get_mark()
[46]528        self.reader.forward()
[116]529        end_mark = self.reader.get_mark()
530        self.tokens.append(BlockEntryToken(start_mark, end_mark))
[39]531
[43]532    def fetch_key(self):
533       
534        # Block context needs additional checks.
535        if not self.flow_level:
[39]536
[43]537            # Are we allowed to start a key (not nessesary a simple)?
538            if not self.allow_simple_key:
[47]539                raise ScannerError(None, None,
540                        "mapping keys are not allowed here",
[116]541                        self.reader.get_mark())
[43]542
543            # We may need to add BLOCK-MAPPING-START.
[46]544            if self.add_indent(self.reader.column):
[116]545                mark = self.reader.get_mark()
546                self.tokens.append(BlockMappingStartToken(mark, mark))
[43]547
548        # Simple keys are allowed after '?' in the block context.
549        self.allow_simple_key = not self.flow_level
550
551        # Reset possible simple key on the current level.
552        self.remove_possible_simple_key()
553
554        # Add KEY.
[116]555        start_mark = self.reader.get_mark()
[46]556        self.reader.forward()
[116]557        end_mark = self.reader.get_mark()
558        self.tokens.append(KeyToken(start_mark, end_mark))
[39]559
[43]560    def fetch_value(self):
[39]561
[43]562        # Do we determine a simple key?
563        if self.flow_level in self.possible_simple_keys:
[39]564
[43]565            # Add KEY.
566            key = self.possible_simple_keys[self.flow_level]
567            del self.possible_simple_keys[self.flow_level]
568            self.tokens.insert(key.token_number-self.tokens_taken,
[116]569                    KeyToken(key.mark, key.mark))
[39]570
[43]571            # If this key starts a new block mapping, we need to add
572            # BLOCK-MAPPING-START.
573            if not self.flow_level:
574                if self.add_indent(key.column):
575                    self.tokens.insert(key.token_number-self.tokens_taken,
[116]576                            BlockMappingStartToken(key.mark, key.mark))
[37]577
[43]578            # There cannot be two simple keys one after another.
579            self.allow_simple_key = False
[37]580
[43]581        # It must be a part of a complex key.
582        else:
583           
[47]584            # Block context needs additional checks.
585            # (Do we really need them? They will be catched by the parser
586            # anyway.)
587            if not self.flow_level:
588
589                # We are allowed to start a complex value if and only if
590                # we can start a simple key.
591                if not self.allow_simple_key:
592                    raise ScannerError(None, None,
593                            "mapping values are not allowed here",
[116]594                            self.reader.get_mark())
[47]595
[43]596            # Simple keys are allowed after ':' in the block context.
597            self.allow_simple_key = not self.flow_level
[37]598
[43]599            # Reset possible simple key on the current level.
600            self.remove_possible_simple_key()
[37]601
[43]602        # Add VALUE.
[116]603        start_mark = self.reader.get_mark()
[46]604        self.reader.forward()
[116]605        end_mark = self.reader.get_mark()
606        self.tokens.append(ValueToken(start_mark, end_mark))
[37]607
[43]608    def fetch_alias(self):
[37]609
[43]610        # ALIAS could be a simple key.
611        self.save_possible_simple_key()
[37]612
[43]613        # No simple keys after ALIAS.
614        self.allow_simple_key = False
[37]615
[43]616        # Scan and add ALIAS.
[47]617        self.tokens.append(self.scan_anchor(AliasToken))
[37]618
[43]619    def fetch_anchor(self):
[37]620
[43]621        # ANCHOR could start a simple key.
622        self.save_possible_simple_key()
[37]623
[43]624        # No simple keys after ANCHOR.
625        self.allow_simple_key = False
[37]626
[43]627        # Scan and add ANCHOR.
[47]628        self.tokens.append(self.scan_anchor(AnchorToken))
[37]629
[43]630    def fetch_tag(self):
[37]631
[43]632        # TAG could start a simple key.
633        self.save_possible_simple_key()
[37]634
[43]635        # No simple keys after TAG.
636        self.allow_simple_key = False
[37]637
[43]638        # Scan and add TAG.
[47]639        self.tokens.append(self.scan_tag())
[37]640
[43]641    def fetch_literal(self):
[130]642        self.fetch_block_scalar(style='|')
[37]643
[43]644    def fetch_folded(self):
[130]645        self.fetch_block_scalar(style='>')
[37]646
[130]647    def fetch_block_scalar(self, style):
[37]648
[43]649        # A simple key may follow a block scalar.
650        self.allow_simple_key = True
[37]651
[43]652        # Reset possible simple key on the current level.
653        self.remove_possible_simple_key()
[37]654
[43]655        # Scan and add SCALAR.
[130]656        self.tokens.append(self.scan_block_scalar(style))
[37]657
[43]658    def fetch_single(self):
[130]659        self.fetch_flow_scalar(style='\'')
[37]660
[43]661    def fetch_double(self):
[130]662        self.fetch_flow_scalar(style='"')
[37]663
[130]664    def fetch_flow_scalar(self, style):
[37]665
[43]666        # A flow scalar could be a simple key.
667        self.save_possible_simple_key()
[37]668
[43]669        # No simple keys after flow scalars.
670        self.allow_simple_key = False
[37]671
[43]672        # Scan and add SCALAR.
[130]673        self.tokens.append(self.scan_flow_scalar(style))
[37]674
[43]675    def fetch_plain(self):
[37]676
[43]677        # A plain scalar could be a simple key.
678        self.save_possible_simple_key()
[37]679
[43]680        # No simple keys after plain scalars. But note that `scan_plain` will
681        # change this flag if the scan is finished at the beginning of the
682        # line.
683        self.allow_simple_key = False
[37]684
[43]685        # Scan and add SCALAR. May change `allow_simple_key`.
[47]686        self.tokens.append(self.scan_plain())
[37]687
[43]688    # Checkers.
[37]689
[43]690    def check_directive(self):
[37]691
[43]692        # DIRECTIVE:        ^ '%' ...
693        # The '%' indicator is already checked.
[46]694        if self.reader.column == 0:
[43]695            return True
[37]696
[43]697    def check_document_start(self):
[37]698
[43]699        # DOCUMENT-START:   ^ '---' (' '|'\n')
[46]700        if self.reader.column == 0:
[48]701            if self.reader.prefix(3) == u'---'  \
702                    and self.reader.peek(3) in u'\0 \t\r\n\x85\u2028\u2029':
[43]703                return True
[37]704
[43]705    def check_document_end(self):
[37]706
[43]707        # DOCUMENT-END:     ^ '...' (' '|'\n')
[46]708        if self.reader.column == 0:
709            prefix = self.reader.peek(4)
[48]710            if self.reader.prefix(3) == u'...'  \
711                    and self.reader.peek(3) in u'\0 \t\r\n\x85\u2028\u2029':
[43]712                return True
[37]713
[51]714    def check_block_entry(self):
[43]715
[51]716        # BLOCK-ENTRY:      '-' (' '|'\n')
717        return self.reader.peek(1) in u'\0 \t\r\n\x85\u2028\u2029'
[43]718
719    def check_key(self):
720
721        # KEY(flow context):    '?'
722        if self.flow_level:
[37]723            return True
[43]724
725        # KEY(block context):   '?' (' '|'\n')
[37]726        else:
[48]727            return self.reader.peek(1) in u'\0 \t\r\n\x85\u2028\u2029'
[37]728
[43]729    def check_value(self):
730
731        # VALUE(flow context):  ':'
732        if self.flow_level:
[37]733            return True
[43]734
735        # VALUE(block context): ':' (' '|'\n')
[37]736        else:
[48]737            return self.reader.peek(1) in u'\0 \t\r\n\x85\u2028\u2029'
[37]738
[43]739    def check_plain(self):
[37]740
[48]741        # A plain scalar may start with any non-space character except:
742        #   '-', '?', ':', ',', '[', ']', '{', '}',
743        #   '#', '&', '*', '!', '|', '>', '\'', '\"',
744        #   '%', '@', '`'.
745        #
746        # It may also start with
747        #   '-', '?', ':'
748        # if it is followed by a non-space character.
749        #
750        # Note that we limit the last rule to the block context (except the
751        # '-' character) because we want the flow context to be space
752        # independent.
753        ch = self.reader.peek()
754        return ch not in u'\0 \t\r\n\x85\u2028\u2029-?:,[]{}#&*!|>\'\"%@`'  \
755                or (self.reader.peek(1) not in u'\0 \t\r\n\x85\u2028\u2029'
756                        and (ch == '-' or (not self.flow_level and ch in u'?:')))
757
[43]758    # Scanners.
759
760    def scan_to_next_token(self):
[47]761        # We ignore spaces, line breaks and comments.
762        # If we find a line break in the block context, we set the flag
763        # `allow_simple_key` on.
[51]764        # The byte order mark is stripped if it's the first character in the
765        # stream. We do not yet support BOM inside the stream as the
766        # specification requires. Any such mark will be considered as a part
767        # of the document.
[52]768        #
769        # TODO: We need to make tab handling rules more sane. A good rule is
770        #   Tabs cannot precede tokens
771        #   BLOCK-SEQUENCE-START, BLOCK-MAPPING-START, BLOCK-END,
772        #   KEY(block), VALUE(block), BLOCK-ENTRY
773        # So the checking code is
774        #   if <TAB>:
775        #       self.allow_simple_keys = False
776        # We also need to add the check for `allow_simple_keys == True` to
777        # `unwind_indent` before issuing BLOCK-END.
778        # Scanners for block, flow, and plain scalars need to be modified.
779
[51]780        if self.reader.index == 0 and self.reader.peek() == u'\uFEFF':
781            self.reader.forward()
[43]782        found = False
783        while not found:
[46]784            while self.reader.peek() == u' ':
785                self.reader.forward()
786            if self.reader.peek() == u'#':
[47]787                while self.reader.peek() not in u'\0\r\n\x85\u2028\u2029':
[46]788                    self.reader.forward()
[47]789            if self.scan_line_break():
[43]790                if not self.flow_level:
791                    self.allow_simple_key = True
[37]792            else:
[43]793                found = True
[37]794
[43]795    def scan_directive(self):
[48]796        # See the specification for details.
[116]797        start_mark = self.reader.get_mark()
[48]798        self.reader.forward()
[116]799        name = self.scan_directive_name(start_mark)
[48]800        value = None
801        if name == u'YAML':
[116]802            value = self.scan_yaml_directive_value(start_mark)
803            end_mark = self.reader.get_mark()
[48]804        elif name == u'TAG':
[116]805            value = self.scan_tag_directive_value(start_mark)
806            end_mark = self.reader.get_mark()
[43]807        else:
[116]808            end_mark = self.reader.get_mark()
[48]809            while self.reader.peek() not in u'\0\r\n\x85\u2028\u2029':
810                self.reader.forward()
[116]811        self.scan_directive_ignored_line(start_mark)
812        return DirectiveToken(name, value, start_mark, end_mark)
[48]813
[116]814    def scan_directive_name(self, start_mark):
[48]815        # See the specification for details.
816        length = 0
817        ch = self.reader.peek(length)
818        while u'0' <= ch <= u'9' or u'A' <= ch <= 'Z' or u'a' <= ch <= 'z'  \
819                or ch in u'-_':
820            length += 1
821            ch = self.reader.peek(length)
822        if not length:
[116]823            raise ScannerError("while scanning a directive", start_mark,
[52]824                    "expected alphabetic or numeric character, but found %r"
[116]825                    % ch.encode('utf-8'), self.reader.get_mark())
[48]826        value = self.reader.prefix(length)
827        self.reader.forward(length)
828        ch = self.reader.peek()
829        if ch not in u'\0 \r\n\x85\u2028\u2029':
[116]830            raise ScannerError("while scanning a directive", start_mark,
[48]831                    "expected alphabetic or numeric character, but found %r"
[116]832                    % ch.encode('utf-8'), self.reader.get_mark())
[48]833        return value
834
[116]835    def scan_yaml_directive_value(self, start_mark):
[48]836        # See the specification for details.
837        while self.reader.peek() == u' ':
[46]838            self.reader.forward()
[116]839        major = self.scan_yaml_directive_number(start_mark)
[48]840        if self.reader.peek() != '.':
[116]841            raise ScannerError("while scanning a directive", start_mark,
[52]842                    "expected a digit or '.', but found %r"
843                    % self.reader.peek().encode('utf-8'),
[116]844                    self.reader.get_mark())
[46]845        self.reader.forward()
[116]846        minor = self.scan_yaml_directive_number(start_mark)
[48]847        if self.reader.peek() not in u'\0 \r\n\x85\u2028\u2029':
[116]848            raise ScannerError("while scanning a directive", start_mark,
[52]849                    "expected a digit or ' ', but found %r"
850                    % self.reader.peek().encode('utf-8'),
[116]851                    self.reader.get_mark())
[48]852        return (major, minor)
[37]853
[116]854    def scan_yaml_directive_number(self, start_mark):
[48]855        # See the specification for details.
856        ch = self.reader.peek()
857        if not (u'0' <= ch <= '9'):
[116]858            raise ScannerError("while scanning a directive", start_mark,
[48]859                    "expected a digit, but found %r" % ch.encode('utf-8'),
[116]860                    self.reader.get_mark())
[48]861        length = 0
862        while u'0' <= self.reader.peek(length) <= u'9':
863            length += 1
864        value = int(self.reader.prefix(length))
865        self.reader.forward(length)
866        return value
867
[116]868    def scan_tag_directive_value(self, start_mark):
[48]869        # See the specification for details.
870        while self.reader.peek() == u' ':
871            self.reader.forward()
[116]872        handle = self.scan_tag_directive_handle(start_mark)
[48]873        while self.reader.peek() == u' ':
874            self.reader.forward()
[116]875        prefix = self.scan_tag_directive_prefix(start_mark)
[48]876        return (handle, prefix)
877
[116]878    def scan_tag_directive_handle(self, start_mark):
[48]879        # See the specification for details.
[116]880        value = self.scan_tag_handle('directive', start_mark)
[52]881        ch = self.reader.peek()
882        if ch != u' ':
[116]883            raise ScannerError("while scanning a directive", start_mark,
[48]884                    "expected ' ', but found %r" % ch.encode('utf-8'),
[116]885                    self.reader.get_mark())
[48]886        return value
887
[116]888    def scan_tag_directive_prefix(self, start_mark):
[48]889        # See the specification for details.
[116]890        value = self.scan_tag_uri('directive', start_mark)
[48]891        ch = self.reader.peek()
892        if ch not in u'\0 \r\n\x85\u2028\u2029':
[116]893            raise ScannerError("while scanning a directive", start_mark,
[48]894                    "expected ' ', but found %r" % ch.encode('utf-8'),
[116]895                    self.reader.get_mark())
[48]896        return value
897
[116]898    def scan_directive_ignored_line(self, start_mark):
[48]899        # See the specification for details.
900        while self.reader.peek() == u' ':
901            self.reader.forward()
902        if self.reader.peek() == u'#':
903            while self.reader.peek() not in u'\0\r\n\x85\u2028\u2029':
904                self.reader.forward()
905        ch = self.reader.peek()
906        if ch not in u'\0\r\n\x85\u2028\u2029':
[116]907            raise ScannerError("while scanning a directive", start_mark,
[48]908                    "expected a comment or a line break, but found %r"
[116]909                        % ch.encode('utf-8'), self.reader.get_mark())
[48]910        self.scan_line_break()
911
[43]912    def scan_anchor(self, TokenClass):
[48]913        # The specification does not restrict characters for anchors and
914        # aliases. This may lead to problems, for instance, the document:
915        #   [ *alias, value ]
916        # can be interpteted in two ways, as
917        #   [ "value" ]
918        # and
919        #   [ *alias , "value" ]
920        # Therefore we restrict aliases to numbers and ASCII letters.
[116]921        start_mark = self.reader.get_mark()
[48]922        indicator = self.reader.peek()
923        if indicator == '*':
924            name = 'alias'
925        else:
926            name = 'anchor'
927        self.reader.forward()
928        length = 0
929        ch = self.reader.peek(length)
930        while u'0' <= ch <= u'9' or u'A' <= ch <= 'Z' or u'a' <= ch <= 'z'  \
931                or ch in u'-_':
932            length += 1
933            ch = self.reader.peek(length)
934        if not length:
[116]935            raise ScannerError("while scanning an %s" % name, start_mark,
[52]936                    "expected alphabetic or numeric character, but found %r"
[116]937                    % ch.encode('utf-8'), self.reader.get_mark())
[48]938        value = self.reader.prefix(length)
939        self.reader.forward(length)
940        ch = self.reader.peek()
941        if ch not in u'\0 \t\r\n\x85\u2028\u2029?:,]}%@`':
[116]942            raise ScannerError("while scanning an %s" % name, start_mark,
[48]943                    "expected alphabetic or numeric character, but found %r"
[116]944                    % ch.encode('utf-8'), self.reader.get_mark())
945        end_mark = self.reader.get_mark()
946        return TokenClass(value, start_mark, end_mark)
[37]947
[43]948    def scan_tag(self):
[48]949        # See the specification for details.
[116]950        start_mark = self.reader.get_mark()
[48]951        ch = self.reader.peek(1)
952        if ch == u'<':
953            handle = None
954            self.reader.forward(2)
[116]955            suffix = self.scan_tag_uri('tag', start_mark)
[48]956            if self.reader.peek() != u'>':
[116]957                raise ScannerError("while parsing a tag", start_mark,
[52]958                        "expected '>', but found %r" % self.reader.peek().encode('utf-8'),
[116]959                        self.reader.get_mark())
[46]960            self.reader.forward()
[48]961        elif ch in u'\0 \t\r\n\x85\u2028\u2029':
962            handle = None
963            suffix = u'!'
964            self.reader.forward()
965        else:
966            length = 1
967            use_handle = False
968            while ch not in u'\0 \r\n\x85\u2028\u2029':
969                if ch == u'!':
970                    use_handle = True
971                    break
972                length += 1
973                ch = self.reader.peek(length)
974            handle = u'!'
975            if use_handle:
[116]976                handle = self.scan_tag_handle('tag', start_mark)
[48]977            else:
978                handle = u'!'
979                self.reader.forward()
[116]980            suffix = self.scan_tag_uri('tag', start_mark)
[48]981        ch = self.reader.peek()
982        if ch not in u'\0 \r\n\x85\u2028\u2029':
[116]983            raise ScannerError("while scanning a tag", start_mark,
[48]984                    "expected ' ', but found %r" % ch.encode('utf-8'),
[116]985                    self.reader.get_mark())
[48]986        value = (handle, suffix)
[116]987        end_mark = self.reader.get_mark()
988        return TagToken(value, start_mark, end_mark)
[43]989
[130]990    def scan_block_scalar(self, style):
[48]991        # See the specification for details.
992
[130]993        if style == '>':
994            folded = True
995        else:
996            folded = False
997
[48]998        chunks = []
[116]999        start_mark = self.reader.get_mark()
[48]1000
1001        # Scan the header.
1002        self.reader.forward()
[116]1003        chomping, increment = self.scan_block_scalar_indicators(start_mark)
1004        self.scan_block_scalar_ignored_line(start_mark)
[48]1005
1006        # Determine the indentation level and go to the first non-empty line.
1007        min_indent = self.indent+1
1008        if min_indent < 1:
1009            min_indent = 1
1010        if increment is None:
[116]1011            breaks, max_indent, end_mark = self.scan_block_scalar_indentation()
[48]1012            indent = max(min_indent, max_indent)
1013        else:
1014            indent = min_indent+increment-1
[116]1015            breaks, end_mark = self.scan_block_scalar_breaks(indent)
[48]1016        line_break = u''
1017
1018        # Scan the inner part of the block scalar.
1019        while self.reader.column == indent and self.reader.peek() != u'\0':
1020            chunks.extend(breaks)
1021            leading_non_space = self.reader.peek() not in u' \t'
1022            length = 0
1023            while self.reader.peek(length) not in u'\0\r\n\x85\u2028\u2029':
1024                length += 1
1025            chunks.append(self.reader.prefix(length))
1026            self.reader.forward(length)
1027            line_break = self.scan_line_break()
[116]1028            breaks, end_mark = self.scan_block_scalar_breaks(indent)
[48]1029            if self.reader.column == indent and self.reader.peek() != u'\0':
[130]1030
[48]1031                # Unfortunately, folding rules are ambiguous.
1032                #
1033                # This is the folding according to the specification:
[51]1034               
1035                if folded and line_break == u'\n'   \
1036                        and leading_non_space and self.reader.peek() not in u' \t':
1037                    if not breaks:
1038                        chunks.append(u' ')
1039                else:
1040                    chunks.append(line_break)
1041               
1042                # This is Clark Evans's interpretation (also in the spec
1043                # examples):
[48]1044                #
[51]1045                #if folded and line_break == u'\n':
[48]1046                #    if not breaks:
[51]1047                #        if self.reader.peek() not in ' \t':
1048                #            chunks.append(u' ')
1049                #        else:
1050                #            chunks.append(line_break)
[48]1051                #else:
1052                #    chunks.append(line_break)
1053            else:
1054                break
1055
1056        # Chomp the tail.
1057        if chomping is not False:
1058            chunks.append(line_break)
1059        if chomping is True:
1060            chunks.extend(breaks)
1061
1062        # We are done.
[130]1063        return ScalarToken(u''.join(chunks), False, start_mark, end_mark,
1064                style)
[48]1065
[116]1066    def scan_block_scalar_indicators(self, start_mark):
[48]1067        # See the specification for details.
1068        chomping = None
1069        increment = None
1070        ch = self.reader.peek()
1071        if ch in u'+-':
1072            if ch == '+':
1073                chomping = True
1074            else:
1075                chomping = False
1076            self.reader.forward()
1077            ch = self.reader.peek()
1078            if ch in u'0123456789':
1079                increment = int(ch)
1080                if increment == 0:
[116]1081                    raise ScannerError("while scanning a block scalar", start_mark,
[48]1082                            "expected indentation indicator in the range 1-9, but found 0",
[116]1083                            self.reader.get_mark())
[46]1084                self.reader.forward()
[48]1085        elif ch in u'0123456789':
1086            increment = int(ch)
1087            if increment == 0:
[116]1088                raise ScannerError("while scanning a block scalar", start_mark,
[48]1089                        "expected indentation indicator in the range 1-9, but found 0",
[116]1090                        self.reader.get_mark())
[48]1091            self.reader.forward()
1092            ch = self.reader.peek()
1093            if ch in u'+-':
1094                if ch == '+':
1095                    chomping = True
1096                else:
1097                    chomping = False
[46]1098                self.reader.forward()
[48]1099        ch = self.reader.peek()
1100        if ch not in u'\0 \r\n\x85\u2028\u2029':
[116]1101            raise ScannerError("while scanning a block scalar", start_mark,
[48]1102                    "expected chomping or indentation indicators, but found %r"
[116]1103                        % ch.encode('utf-8'), self.reader.get_mark())
[48]1104        return chomping, increment
1105
[116]1106    def scan_block_scalar_ignored_line(self, start_mark):
[48]1107        # See the specification for details.
1108        while self.reader.peek() == u' ':
1109            self.reader.forward()
1110        if self.reader.peek() == u'#':
1111            while self.reader.peek() not in u'\0\r\n\x85\u2028\u2029':
[46]1112                self.reader.forward()
[48]1113        ch = self.reader.peek()
1114        if ch not in u'\0\r\n\x85\u2028\u2029':
[116]1115            raise ScannerError("while scanning a block scalar", start_mark,
[48]1116                    "expected a comment or a line break, but found %r"
[116]1117                        % ch.encode('utf-8'), self.reader.get_mark())
[48]1118        self.scan_line_break()
[43]1119
[48]1120    def scan_block_scalar_indentation(self):
1121        # See the specification for details.
1122        chunks = []
1123        max_indent = 0
[116]1124        end_mark = self.reader.get_mark()
[48]1125        while self.reader.peek() in u' \r\n\x85\u2028\u2029':
1126            if self.reader.peek() != u' ':
1127                chunks.append(self.scan_line_break())
[116]1128                end_mark = self.reader.get_mark()
[48]1129            else:
1130                self.reader.forward()
1131                if self.reader.column > max_indent:
1132                    max_indent = self.reader.column
[116]1133        return chunks, max_indent, end_mark
[48]1134
1135    def scan_block_scalar_breaks(self, indent):
1136        # See the specification for details.
1137        chunks = []
[116]1138        end_mark = self.reader.get_mark()
[48]1139        while self.reader.column < indent and self.reader.peek() == u' ':
1140            self.reader.forward()
1141        while self.reader.peek() in u'\r\n\x85\u2028\u2029':
1142            chunks.append(self.scan_line_break())
[116]1143            end_mark = self.reader.get_mark()
[48]1144            while self.reader.column < indent and self.reader.peek() == u' ':
1145                self.reader.forward()
[116]1146        return chunks, end_mark
[48]1147
[130]1148    def scan_flow_scalar(self, style):
[48]1149        # See the specification for details.
[117]1150        # Note that we loose indentation rules for quoted scalars. Quoted
1151        # scalars don't need to adhere indentation because " and ' clearly
1152        # mark the beginning and the end of them. Therefore we are less
1153        # restrictive then the specification requires. We only need to check
1154        # that document separators are not included in scalars.
[130]1155        if style == '"':
1156            double = True
1157        else:
1158            double = False
[48]1159        chunks = []
[116]1160        start_mark = self.reader.get_mark()
[46]1161        quote = self.reader.peek()
1162        self.reader.forward()
[117]1163        chunks.extend(self.scan_flow_scalar_non_spaces(double, start_mark))
[46]1164        while self.reader.peek() != quote:
[117]1165            chunks.extend(self.scan_flow_scalar_spaces(double, start_mark))
1166            chunks.extend(self.scan_flow_scalar_non_spaces(double, start_mark))
[48]1167        self.reader.forward()
[116]1168        end_mark = self.reader.get_mark()
[130]1169        return ScalarToken(u''.join(chunks), False, start_mark, end_mark,
1170                style)
[48]1171
1172    ESCAPE_REPLACEMENTS = {
1173        u'0':   u'\0',
1174        u'a':   u'\x07',
1175        u'b':   u'\x08',
1176        u't':   u'\x09',
1177        u'\t':  u'\x09',
1178        u'n':   u'\x0A',
1179        u'v':   u'\x0B',
1180        u'f':   u'\x0C',
1181        u'r':   u'\x0D',
1182        u'e':   u'\x1B',
1183        u' ':   u'\x20',
1184        u'\"':  u'\"',
1185        u'\\':  u'\\',
1186        u'N':   u'\x85',
1187        u'_':   u'\xA0',
1188        u'L':   u'\u2028',
1189        u'P':   u'\u2029',
1190    }
1191
1192    ESCAPE_CODES = {
1193        u'x':   2,
1194        u'u':   4,
1195        u'U':   8,
1196    }
1197
[117]1198    def scan_flow_scalar_non_spaces(self, double, start_mark):
[48]1199        # See the specification for details.
1200        chunks = []
1201        while True:
1202            length = 0
1203            while self.reader.peek(length) not in u'\'\"\\\0 \t\r\n\x85\u2028\u2029':
1204                length += 1
1205            if length:
1206                chunks.append(self.reader.prefix(length))
1207                self.reader.forward(length)
1208            ch = self.reader.peek()
1209            if not double and ch == u'\'' and self.reader.peek(1) == u'\'':
1210                chunks.append(u'\'')
[46]1211                self.reader.forward(2)
[48]1212            elif (double and ch == u'\'') or (not double and ch in u'\"\\'):
1213                chunks.append(ch)
1214                self.reader.forward()
1215            elif double and ch == u'\\':
1216                self.reader.forward()
1217                ch = self.reader.peek()
1218                if ch in self.ESCAPE_REPLACEMENTS:
1219                    chunks.append(self.ESCAPE_REPLACEMENTS[ch])
1220                    self.reader.forward()
1221                elif ch in self.ESCAPE_CODES:
1222                    length = self.ESCAPE_CODES[ch]
1223                    self.reader.forward()
1224                    for k in range(length):
1225                        if self.reader.peek(k) not in u'0123456789ABCDEFabcdef':
[116]1226                            raise ScannerError("while scanning a double-quoted scalar", start_mark,
[48]1227                                    "expected escape sequence of %d hexdecimal numbers, but found %r" %
[116]1228                                        (length, self.reader.peek(k).encode('utf-8')), self.reader.get_mark())
[48]1229                    code = int(self.reader.prefix(length), 16)
1230                    chunks.append(unichr(code))
1231                    self.reader.forward(length)
1232                elif ch in u'\r\n\x85\u2028\u2029':
1233                    self.scan_line_break()
[117]1234                    chunks.extend(self.scan_flow_scalar_breaks(double, start_mark))
[48]1235                else:
[116]1236                    raise ScannerError("while scanning a double-quoted scalar", start_mark,
1237                            "found unknown escape character %r" % ch.encode('utf-8'), self.reader.get_mark())
[37]1238            else:
[48]1239                return chunks
[37]1240
[117]1241    def scan_flow_scalar_spaces(self, double, start_mark):
[48]1242        # See the specification for details.
1243        chunks = []
1244        length = 0
1245        while self.reader.peek(length) in u' \t':
1246            length += 1
1247        whitespaces = self.reader.prefix(length)
1248        self.reader.forward(length)
1249        ch = self.reader.peek()
1250        if ch == u'\0':
[116]1251            raise ScannerError("while scanning a quoted scalar", start_mark,
1252                    "found unexpected end of stream", self.reader.get_mark())
[48]1253        elif ch in u'\r\n\x85\u2028\u2029':
1254            line_break = self.scan_line_break()
[117]1255            breaks = self.scan_flow_scalar_breaks(double, start_mark)
[48]1256            if line_break != u'\n':
1257                chunks.append(line_break)
1258            elif not breaks:
1259                chunks.append(u' ')
1260            chunks.extend(breaks)
1261        else:
1262            chunks.append(whitespaces)
1263        return chunks
1264
[117]1265    def scan_flow_scalar_breaks(self, double, start_mark):
[48]1266        # See the specification for details.
1267        chunks = []
1268        while True:
[117]1269            # Instead of checking indentation, we check for document
1270            # separators.
1271            prefix = self.reader.prefix(3)
1272            if (prefix == u'---' or prefix == u'...')   \
1273                    and self.reader.peek(3) in u'\0 \t\r\n\x85\u2028\u2029':
[116]1274                raise ScannerError("while scanning a quoted scalar", start_mark,
[117]1275                        "found unexpected document separator", self.reader.get_mark())
[48]1276            while self.reader.peek() in u' \t':
1277                self.reader.forward()
1278            if self.reader.peek() in u'\r\n\x85\u2028\u2029':
1279                chunks.append(self.scan_line_break())
1280            else:
1281                return chunks
1282
[43]1283    def scan_plain(self):
[48]1284        # See the specification for details.
1285        # We add an additional restriction for the flow context:
[117]1286        #   plain scalars in the flow context cannot contain ',', ':' and '?'.
[48]1287        # We also keep track of the `allow_simple_key` flag here.
[117]1288        # Indentation rules are loosed for the flow context.
[48]1289        chunks = []
[116]1290        start_mark = self.reader.get_mark()
1291        end_mark = start_mark
[43]1292        indent = self.indent+1
[117]1293        # We allow zero indentation for scalars, but then we need to check for
1294        # document separators at the beginning of the line.
1295        #if indent == 0:
1296        #    indent = 1
[48]1297        spaces = []
[43]1298        while True:
[48]1299            length = 0
1300            if self.reader.peek() == u'#':
[43]1301                break
[48]1302            while True:
1303                ch = self.reader.peek(length)
1304                if ch in u'\0 \t\r\n\x85\u2028\u2029'   \
1305                        or (not self.flow_level and ch == u':' and
1306                                self.reader.peek(length+1) in u'\0 \t\r\n\x28\u2028\u2029') \
1307                        or (self.flow_level and ch in u',:?[]{}'):
1308                    break
1309                length += 1
1310            if length == 0:
[43]1311                break
[48]1312            self.allow_simple_key = False
1313            chunks.extend(spaces)
1314            chunks.append(self.reader.prefix(length))
1315            self.reader.forward(length)
[116]1316            end_mark = self.reader.get_mark()
[117]1317            spaces = self.scan_plain_spaces(indent, start_mark)
[48]1318            if not spaces or self.reader.peek() == u'#' \
[117]1319                    or (not self.flow_level and self.reader.column < indent):
[48]1320                break
[116]1321        return ScalarToken(u''.join(chunks), True, start_mark, end_mark)
[37]1322
[117]1323    def scan_plain_spaces(self, indent, start_mark):
[48]1324        # See the specification for details.
1325        # The specification is really confusing about tabs in plain scalars.
1326        # We just forbid them completely. Do not use tabs in YAML!
1327        chunks = []
1328        length = 0
1329        while self.reader.peek(length) in u' ':
1330            length += 1
1331        whitespaces = self.reader.prefix(length)
1332        self.reader.forward(length)
1333        ch = self.reader.peek()
1334        if ch in u'\r\n\x85\u2028\u2029':
1335            line_break = self.scan_line_break()
1336            self.allow_simple_key = True
[117]1337            prefix = self.reader.prefix(3)
1338            if (prefix == u'---' or prefix == u'...')   \
1339                    and self.reader.peek(3) in u'\0 \t\r\n\x85\u2028\u2029':
1340                return
[48]1341            breaks = []
1342            while self.reader.peek() in u' \r\n\x85\u2028\u2029':
1343                if self.reader.peek() == ' ':
1344                    self.reader.forward()
1345                else:
1346                    breaks.append(self.scan_line_break())
[117]1347                    prefix = self.reader.prefix(3)
1348                    if (prefix == u'---' or prefix == u'...')   \
1349                            and self.reader.peek(3) in u'\0 \t\r\n\x85\u2028\u2029':
1350                        return
[48]1351            if line_break != u'\n':
1352                chunks.append(line_break)
1353            elif not breaks:
1354                chunks.append(u' ')
1355            chunks.extend(breaks)
1356        elif whitespaces:
1357            chunks.append(whitespaces)
1358        return chunks
1359
[116]1360    def scan_tag_handle(self, name, start_mark):
[48]1361        # See the specification for details.
1362        # For some strange reasons, the specification does not allow '_' in
1363        # tag handles. I have allowed it anyway.
[52]1364        ch = self.reader.peek()
1365        if ch != u'!':
[116]1366            raise ScannerError("while scanning a %s" % name, start_mark,
[48]1367                    "expected '!', but found %r" % ch.encode('utf-8'),
[116]1368                    self.reader.get_mark())
[48]1369        length = 1
1370        ch = self.reader.peek(length)
1371        if ch != u' ':
1372            while u'0' <= ch <= u'9' or u'A' <= ch <= 'Z' or u'a' <= ch <= 'z'  \
1373                    or ch in u'-_':
1374                length += 1
1375                ch = self.reader.peek(length)
1376            if ch != u'!':
1377                self.reader.forward(length)
[116]1378                raise ScannerError("while scanning a %s" % name, start_mark,
[48]1379                        "expected '!', but found %r" % ch.encode('utf-8'),
[116]1380                        self.reader.get_mark())
[48]1381            length += 1
1382        value = self.reader.prefix(length)
1383        self.reader.forward(length)
1384        return value
1385
[116]1386    def scan_tag_uri(self, name, start_mark):
[48]1387        # See the specification for details.
1388        # Note: we do not check if URI is well-formed.
1389        chunks = []
1390        length = 0
1391        ch = self.reader.peek(length)
1392        while u'0' <= ch <= u'9' or u'A' <= ch <= 'Z' or u'a' <= ch <= 'z'  \
1393                or ch in u'-;/?:@&=+$,_.!~*\'()[]%':
1394            if ch == u'%':
1395                chunks.append(self.reader.prefix(length))
1396                self.reader.forward(length)
1397                length = 0
[116]1398                chunks.append(self.scan_uri_escapes(name, start_mark))
[48]1399            else:
1400                length += 1
1401            ch = self.reader.peek(length)
1402        if length:
1403            chunks.append(self.reader.prefix(length))
1404            self.reader.forward(length)
1405            length = 0
1406        if not chunks:
[116]1407            raise ScannerError("while parsing a %s" % name, start_mark,
[48]1408                    "expected URI, but found %r" % ch.encode('utf-8'),
[116]1409                    self.reader.get_mark())
[48]1410        return u''.join(chunks)
1411
[116]1412    def scan_uri_escapes(self, name, start_mark):
[48]1413        # See the specification for details.
1414        bytes = []
[116]1415        mark = self.reader.get_mark()
[48]1416        while self.reader.peek() == u'%':
1417            self.reader.forward()
1418            for k in range(2):
1419                if self.reader.peek(k) not in u'0123456789ABCDEFabcdef':
[116]1420                    raise ScannerError("while scanning a %s" % name, start_mark,
[48]1421                            "expected URI escape sequence of 2 hexdecimal numbers, but found %r" %
[116]1422                                (self.reader.peek(k).encode('utf-8')), self.reader.get_mark())
[48]1423            bytes.append(chr(int(self.reader.prefix(2), 16)))
1424            self.reader.forward(2)
1425        try:
1426            value = unicode(''.join(bytes), 'utf-8')
1427        except UnicodeDecodeError, exc:
[116]1428            raise ScannerError("while scanning a %s" % name, start_mark, str(exc), mark)
[48]1429        return value
1430
[47]1431    def scan_line_break(self):
1432        # Transforms:
1433        #   '\r\n'      :   '\n'
1434        #   '\r'        :   '\n'
1435        #   '\n'        :   '\n'
1436        #   '\x85'      :   '\n'
1437        #   '\u2028'    :   '\u2028'
1438        #   '\u2029     :   '\u2029'
1439        #   default     :   ''
1440        ch = self.reader.peek()
1441        if ch in u'\r\n\x85':
[48]1442            if self.reader.prefix(2) == u'\r\n':
[60]1443                self.reader.forward(2)
[47]1444            else:
1445                self.reader.forward()
1446            return u'\n'
1447        elif ch in u'\u2028\u2029':
1448            self.reader.forward()
1449            return ch
1450        return u''
1451
[45]1452#try:
1453#    import psyco
1454#    psyco.bind(Scanner)
1455#except ImportError:
1456#    pass
1457
Note: See TracBrowser for help on using the repository browser.