source: pyyaml/branches/working-on-emitter/lib/yaml/scanner.py @ 121

Revision 121, 52.5 KB checked in by xi, 8 years ago (diff)

Add a branch for working on Emitter.

RevLine 
[39]1
[55]2# Scanner produces tokens of the following types:
[118]3# STREAM-START
4# STREAM-END
[55]5# DIRECTIVE(name, value)
6# DOCUMENT-START
7# DOCUMENT-END
8# BLOCK-SEQUENCE-START
9# BLOCK-MAPPING-START
10# BLOCK-END
11# FLOW-SEQUENCE-START
12# FLOW-MAPPING-START
13# FLOW-SEQUENCE-END
14# FLOW-MAPPING-END
15# BLOCK-ENTRY
16# FLOW-ENTRY
17# KEY
18# VALUE
19# ALIAS(value)
20# ANCHOR(value)
21# TAG(value)
22# SCALAR(value, plain)
[57]23#
24# Read comments in the Scanner code for more details.
25#
[43]26
[46]27__all__ = ['Scanner', 'ScannerError']
[43]28
[52]29from error import MarkedYAMLError
[46]30from tokens import *
[39]31
[52]32class ScannerError(MarkedYAMLError):
33    pass
[51]34
[43]35class SimpleKey:
[51]36    # See below simple keys treatment.
37
[116]38    def __init__(self, token_number, required, index, line, column, mark):
[43]39        self.token_number = token_number
40        self.required = required
41        self.index = index
42        self.line = line
43        self.column = column
[116]44        self.mark = mark
[43]45
[39]46class Scanner:
47
[46]48
49    def __init__(self, reader):
[39]50        """Initialize the scanner."""
[46]51        # The input stream. The Reader class do the dirty work of checking for
[43]52        # BOM and converting the input data to Unicode. It also adds NUL to
53        # the end.
[39]54        #
[46]55        # Reader supports the following methods
[48]56        #   self.reader.peek(i=0)       # peek the next i-th character
57        #   self.reader.prefix(l=1)     # peek the next l characters
58        #   self.reader.forward(l=1)    # read the next l characters
59                                        # and move the pointer
[46]60        self.reader = reader
[39]61
62        # Had we reached the end of the stream?
63        self.done = False
64
65        # The number of unclosed '{' and '['. `flow_level == 0` means block
66        # context.
67        self.flow_level = 0
68
69        # List of processed tokens that are not yet emitted.
70        self.tokens = []
71
[118]72        # Add the STREAM-START token.
73        self.fetch_stream_start()
74
[39]75        # Number of tokens that were emitted through the `get_token` method.
76        self.tokens_taken = 0
77
78        # The current indentation level.
79        self.indent = -1
80
81        # Past indentation levels.
82        self.indents = []
83
[43]84        # Variables related to simple keys treatment.
[39]85
86        # A simple key is a key that is not denoted by the '?' indicator.
87        # Example of simple keys:
88        #   ---
89        #   block simple key: value
90        #   ? not a simple key:
91        #   : { flow simple key: value }
92        # We emit the KEY token before all keys, so when we find a potential
93        # simple key, we try to locate the corresponding ':' indicator.
94        # Simple keys should be limited to a single line and 1024 characters.
95
[43]96        # Can a simple key start at the current position? A simple key may
97        # start:
98        # - at the beginning of the line, not counting indentation spaces
99        #       (in block context),
100        # - after '{', '[', ',' (in the flow context),
101        # - after '?', ':', '-' (in the block context).
[60]102        # In the block context, this flag also signifies if a block collection
[43]103        # may start at the current position.
104        self.allow_simple_key = True
[39]105
106        # Keep track of possible simple keys. This is a dictionary. The key
107        # is `flow_level`; there can be no more that one possible simple key
[43]108        # for each level. The value is a SimpleKey record:
[116]109        #   (token_number, required, index, line, column, mark)
[43]110        # A simple key may start with ALIAS, ANCHOR, TAG, SCALAR(flow),
111        # '[', or '{' tokens.
[39]112        self.possible_simple_keys = {}
113
[51]114    # Public methods.
[39]115
[51]116    def check(self, *choices):
117        # Check if the next token is one of the given types.
[43]118        while self.need_more_tokens():
[39]119            self.fetch_more_tokens()
120        if self.tokens:
[51]121            for choice in choices:
122                if isinstance(self.tokens[0], choice):
123                    return True
124        return False
125
126    def peek(self):
127        # Return the next token, but do not delete if from the queue.
128        while self.need_more_tokens():
129            self.fetch_more_tokens()
130        if self.tokens:
[39]131            return self.tokens[0]
132
[51]133    def get(self):
134        # Return the next token.
[39]135        while self.need_more_tokens():
136            self.fetch_more_tokens()
137        if self.tokens:
138            self.tokens_taken += 1
139            return self.tokens.pop(0)
140
[51]141    def __iter__(self):
142        # Iterator protocol.
143        while self.need_more_tokens():
144            self.fetch_more_tokens()
145        while self.tokens:
146            self.tokens_taken += 1
147            yield self.tokens.pop(0)
148            while self.need_more_tokens():
149                self.fetch_more_tokens()
150
[43]151    # Private methods.
[39]152
153    def need_more_tokens(self):
154        if self.done:
155            return False
156        if not self.tokens:
157            return True
158        # The current token may be a potential simple key, so we
159        # need to look further.
[43]160        self.stale_possible_simple_keys()
[39]161        if self.next_possible_simple_key() == self.tokens_taken:
162            return True
163
164    def fetch_more_tokens(self):
165
166        # Eat whitespaces and comments until we reach the next token.
[43]167        self.scan_to_next_token()
[39]168
[43]169        # Remove obsolete possible simple keys.
170        self.stale_possible_simple_keys()
171
[39]172        # Compare the current indentation and column. It may add some tokens
[43]173        # and decrease the current indentation level.
[46]174        self.unwind_indent(self.reader.column)
[39]175
176        # Peek the next character.
[46]177        ch = self.reader.peek()
[39]178
[48]179        # Is it the end of stream?
[43]180        if ch == u'\0':
[48]181            return self.fetch_stream_end()
[39]182
183        # Is it a directive?
184        if ch == u'%' and self.check_directive():
185            return self.fetch_directive()
186
187        # Is it the document start?
188        if ch == u'-' and self.check_document_start():
189            return self.fetch_document_start()
190
191        # Is it the document end?
192        if ch == u'.' and self.check_document_end():
193            return self.fetch_document_end()
194
[52]195        # TODO: support for BOM within a stream.
196        #if ch == u'\uFEFF':
197        #    return self.fetch_bom()    <-- issue BOMToken
198
[39]199        # Note: the order of the following checks is NOT significant.
200
201        # Is it the flow sequence start indicator?
202        if ch == u'[':
203            return self.fetch_flow_sequence_start()
204
205        # Is it the flow mapping start indicator?
206        if ch == u'{':
207            return self.fetch_flow_mapping_start()
208
209        # Is it the flow sequence end indicator?
210        if ch == u']':
211            return self.fetch_flow_sequence_end()
212
213        # Is it the flow mapping end indicator?
214        if ch == u'}':
215            return self.fetch_flow_mapping_end()
216
[51]217        # Is it the flow entry indicator?
218        if ch in u',':
219            return self.fetch_flow_entry()
[43]220
[51]221        # Is it the block entry indicator?
222        if ch in u'-' and self.check_block_entry():
223            return self.fetch_block_entry()
224
[39]225        # Is it the key indicator?
226        if ch == u'?' and self.check_key():
227            return self.fetch_key()
228
229        # Is it the value indicator?
230        if ch == u':' and self.check_value():
231            return self.fetch_value()
232
233        # Is it an alias?
234        if ch == u'*':
235            return self.fetch_alias()
236
237        # Is it an anchor?
238        if ch == u'&':
239            return self.fetch_anchor()
240
[43]241        # Is it a tag?
[39]242        if ch == u'!':
243            return self.fetch_tag()
244
[43]245        # Is it a literal scalar?
246        if ch == u'|' and not self.flow_level:
[39]247            return self.fetch_literal()
248
249        # Is it a folded scalar?
[43]250        if ch == u'>' and not self.flow_level:
[39]251            return self.fetch_folded()
252
253        # Is it a single quoted scalar?
254        if ch == u'\'':
255            return self.fetch_single()
256
257        # Is it a double quoted scalar?
258        if ch == u'\"':
259            return self.fetch_double()
260
[43]261        # It must be a plain scalar then.
[39]262        if self.check_plain():
263            return self.fetch_plain()
264
[43]265        # No? It's an error. Let's produce a nice error message.
[48]266        raise ScannerError("while scanning for the next token", None,
267                "found character %r that cannot start any token"
[116]268                % ch.encode('utf-8'), self.reader.get_mark())
[39]269
[43]270    # Simple keys treatment.
271
272    def next_possible_simple_key(self):
273        # Return the number of the nearest possible simple key. Actually we
274        # don't need to loop through the whole dictionary. We may replace it
275        # with the following code:
276        #   if not self.possible_simple_keys:
277        #       return None
278        #   return self.possible_simple_keys[
279        #           min(self.possible_simple_keys.keys())].token_number
280        min_token_number = None
281        for level in self.possible_simple_keys:
282            key = self.possible_simple_keys[level]
283            if min_token_number is None or key.token_number < min_token_number:
284                min_token_number = key.token_number
285        return min_token_number
286
287    def stale_possible_simple_keys(self):
288        # Remove entries that are no longer possible simple keys. According to
289        # the YAML specification, simple keys
290        # - should be limited to a single line,
291        # - should be no longer than 1024 characters.
292        # Disabling this procedure will allow simple keys of any length and
293        # height (may cause problems if indentation is broken though).
294        for level in self.possible_simple_keys.keys():
295            key = self.possible_simple_keys[level]
[46]296            if key.line != self.reader.line  \
297                    or self.reader.index-key.index > 1024:
[43]298                if key.required:
[116]299                    raise ScannerError("while scanning a simple key", key.mark,
300                            "could not found expected ':'", self.reader.get_mark())
[43]301                del self.possible_simple_keys[level]
302
303    def save_possible_simple_key(self):
304        # The next token may start a simple key. We check if it's possible
305        # and save its position. This function is called for
306        #   ALIAS, ANCHOR, TAG, SCALAR(flow), '[', and '{'.
307
308        # Check if a simple key is required at the current position.
[46]309        required = not self.flow_level and self.indent == self.reader.column
[43]310
[47]311        # A simple key is required only if it is the first token in the current
312        # line. Therefore it is always allowed.
313        assert self.allow_simple_key or not required
314
[43]315        # The next token might be a simple key. Let's save it's number and
316        # position.
317        if self.allow_simple_key:
318            self.remove_possible_simple_key()
319            token_number = self.tokens_taken+len(self.tokens)
[46]320            index = self.reader.index
321            line = self.reader.line
322            column = self.reader.column
[116]323            mark = self.reader.get_mark()
[43]324            key = SimpleKey(token_number, required,
[116]325                    index, line, column, mark)
[43]326            self.possible_simple_keys[self.flow_level] = key
327
328    def remove_possible_simple_key(self):
329        # Remove the saved possible key position at the current flow level.
330        if self.flow_level in self.possible_simple_keys:
331            key = self.possible_simple_keys[self.flow_level]
[47]332           
333            # I don't think it's possible, but I could be wrong.
334            assert not key.required
335            #if key.required:
[116]336            #    raise ScannerError("while scanning a simple key", key.mark,
337            #            "could not found expected ':'", self.reader.get_mark())
[43]338
339    # Indentation functions.
340
341    def unwind_indent(self, column):
342
[117]343        ## In flow context, tokens should respect indentation.
344        ## Actually the condition should be `self.indent >= column` according to
345        ## the spec. But this condition will prohibit intuitively correct
346        ## constructions such as
347        ## key : {
348        ## }
349        #if self.flow_level and self.indent > column:
350        #    raise ScannerError(None, None,
351        #            "invalid intendation or unclosed '[' or '{'",
352        #            self.reader.get_mark())
[43]353
[117]354        # In the flow context, indentation is ignored. We make the scanner less
355        # restrictive then specification requires.
356        if self.flow_level:
357            return
358
[43]359        # In block context, we may need to issue the BLOCK-END tokens.
360        while self.indent > column:
[116]361            mark = self.reader.get_mark()
[43]362            self.indent = self.indents.pop()
[116]363            self.tokens.append(BlockEndToken(mark, mark))
[43]364
365    def add_indent(self, column):
366        # Check if we need to increase indentation.
367        if self.indent < column:
368            self.indents.append(self.indent)
369            self.indent = column
370            return True
371        return False
372
373    # Fetchers.
374
[118]375    def fetch_stream_start(self):
376        # We always add STREAM-START as the first token and STREAM-END as the
377        # last token.
378
379        # Read the token.
380        mark = self.reader.get_mark()
381       
382        # Add STREAM-END.
383        self.tokens.append(StreamStartToken(mark, mark))
384       
385
[48]386    def fetch_stream_end(self):
[39]387
388        # Set the current intendation to -1.
[43]389        self.unwind_indent(-1)
[39]390
391        # Reset everything (not really needed).
[43]392        self.allow_simple_key = False
[39]393        self.possible_simple_keys = {}
394
[43]395        # Read the token.
[116]396        mark = self.reader.get_mark()
[43]397       
[118]398        # Add STREAM-END.
[116]399        self.tokens.append(StreamEndToken(mark, mark))
[39]400
[46]401        # The reader is ended.
[39]402        self.done = True
403
[43]404    def fetch_directive(self):
405       
406        # Set the current intendation to -1.
407        self.unwind_indent(-1)
[39]408
[43]409        # Reset simple keys.
410        self.remove_possible_simple_key()
411        self.allow_simple_key = False
[39]412
[43]413        # Scan and add DIRECTIVE.
[47]414        self.tokens.append(self.scan_directive())
[39]415
416    def fetch_document_start(self):
[44]417        self.fetch_document_indicator(DocumentStartToken)
[39]418
[43]419    def fetch_document_end(self):
[44]420        self.fetch_document_indicator(DocumentEndToken)
[43]421
422    def fetch_document_indicator(self, TokenClass):
423
[39]424        # Set the current intendation to -1.
[43]425        self.unwind_indent(-1)
[39]426
[43]427        # Reset simple keys. Note that there could not be a block collection
428        # after '---'.
429        self.remove_possible_simple_key()
430        self.allow_simple_key = False
[39]431
[43]432        # Add DOCUMENT-START or DOCUMENT-END.
[116]433        start_mark = self.reader.get_mark()
[46]434        self.reader.forward(3)
[116]435        end_mark = self.reader.get_mark()
436        self.tokens.append(TokenClass(start_mark, end_mark))
[39]437
[43]438    def fetch_flow_sequence_start(self):
[44]439        self.fetch_flow_collection_start(FlowSequenceStartToken)
[39]440
[43]441    def fetch_flow_mapping_start(self):
[44]442        self.fetch_flow_collection_start(FlowMappingStartToken)
[43]443
444    def fetch_flow_collection_start(self, TokenClass):
445
[44]446        # '[' and '{' may start a simple key.
447        self.save_possible_simple_key()
448
[43]449        # Increase the flow level.
450        self.flow_level += 1
451
452        # Simple keys are allowed after '[' and '{'.
453        self.allow_simple_key = True
454
455        # Add FLOW-SEQUENCE-START or FLOW-MAPPING-START.
[116]456        start_mark = self.reader.get_mark()
[46]457        self.reader.forward()
[116]458        end_mark = self.reader.get_mark()
459        self.tokens.append(TokenClass(start_mark, end_mark))
[39]460
[43]461    def fetch_flow_sequence_end(self):
[44]462        self.fetch_flow_collection_end(FlowSequenceEndToken)
[39]463
[43]464    def fetch_flow_mapping_end(self):
[44]465        self.fetch_flow_collection_end(FlowMappingEndToken)
[43]466
467    def fetch_flow_collection_end(self, TokenClass):
468
469        # Reset possible simple key on the current level.
470        self.remove_possible_simple_key()
471
472        # Decrease the flow level.
473        self.flow_level -= 1
474
475        # No simple keys after ']' or '}'.
476        self.allow_simple_key = False
477
478        # Add FLOW-SEQUENCE-END or FLOW-MAPPING-END.
[116]479        start_mark = self.reader.get_mark()
[46]480        self.reader.forward()
[116]481        end_mark = self.reader.get_mark()
482        self.tokens.append(TokenClass(start_mark, end_mark))
[39]483
[51]484    def fetch_flow_entry(self):
[39]485
[51]486        # Simple keys are allowed after ','.
487        self.allow_simple_key = True
488
489        # Reset possible simple key on the current level.
490        self.remove_possible_simple_key()
491
492        # Add FLOW-ENTRY.
[116]493        start_mark = self.reader.get_mark()
[51]494        self.reader.forward()
[116]495        end_mark = self.reader.get_mark()
496        self.tokens.append(FlowEntryToken(start_mark, end_mark))
[51]497
498    def fetch_block_entry(self):
499
[43]500        # Block context needs additional checks.
501        if not self.flow_level:
[39]502
[43]503            # Are we allowed to start a new entry?
504            if not self.allow_simple_key:
[47]505                raise ScannerError(None, None,
506                        "sequence entries are not allowed here",
[116]507                        self.reader.get_mark())
[39]508
[43]509            # We may need to add BLOCK-SEQUENCE-START.
[46]510            if self.add_indent(self.reader.column):
[116]511                mark = self.reader.get_mark()
512                self.tokens.append(BlockSequenceStartToken(mark, mark))
[39]513
[51]514        # It's an error for the block entry to occur in the flow context,
515        # but we let the parser detect this.
516        else:
517            pass
518
519        # Simple keys are allowed after '-'.
[43]520        self.allow_simple_key = True
[39]521
[43]522        # Reset possible simple key on the current level.
523        self.remove_possible_simple_key()
[39]524
[51]525        # Add BLOCK-ENTRY.
[116]526        start_mark = self.reader.get_mark()
[46]527        self.reader.forward()
[116]528        end_mark = self.reader.get_mark()
529        self.tokens.append(BlockEntryToken(start_mark, end_mark))
[39]530
[43]531    def fetch_key(self):
532       
533        # Block context needs additional checks.
534        if not self.flow_level:
[39]535
[43]536            # Are we allowed to start a key (not nessesary a simple)?
537            if not self.allow_simple_key:
[47]538                raise ScannerError(None, None,
539                        "mapping keys are not allowed here",
[116]540                        self.reader.get_mark())
[43]541
542            # We may need to add BLOCK-MAPPING-START.
[46]543            if self.add_indent(self.reader.column):
[116]544                mark = self.reader.get_mark()
545                self.tokens.append(BlockMappingStartToken(mark, mark))
[43]546
547        # Simple keys are allowed after '?' in the block context.
548        self.allow_simple_key = not self.flow_level
549
550        # Reset possible simple key on the current level.
551        self.remove_possible_simple_key()
552
553        # Add KEY.
[116]554        start_mark = self.reader.get_mark()
[46]555        self.reader.forward()
[116]556        end_mark = self.reader.get_mark()
557        self.tokens.append(KeyToken(start_mark, end_mark))
[39]558
[43]559    def fetch_value(self):
[39]560
[43]561        # Do we determine a simple key?
562        if self.flow_level in self.possible_simple_keys:
[39]563
[43]564            # Add KEY.
565            key = self.possible_simple_keys[self.flow_level]
566            del self.possible_simple_keys[self.flow_level]
567            self.tokens.insert(key.token_number-self.tokens_taken,
[116]568                    KeyToken(key.mark, key.mark))
[39]569
[43]570            # If this key starts a new block mapping, we need to add
571            # BLOCK-MAPPING-START.
572            if not self.flow_level:
573                if self.add_indent(key.column):
574                    self.tokens.insert(key.token_number-self.tokens_taken,
[116]575                            BlockMappingStartToken(key.mark, key.mark))
[37]576
[43]577            # There cannot be two simple keys one after another.
578            self.allow_simple_key = False
[37]579
[43]580        # It must be a part of a complex key.
581        else:
582           
[47]583            # Block context needs additional checks.
584            # (Do we really need them? They will be catched by the parser
585            # anyway.)
586            if not self.flow_level:
587
588                # We are allowed to start a complex value if and only if
589                # we can start a simple key.
590                if not self.allow_simple_key:
591                    raise ScannerError(None, None,
592                            "mapping values are not allowed here",
[116]593                            self.reader.get_mark())
[47]594
[43]595            # Simple keys are allowed after ':' in the block context.
596            self.allow_simple_key = not self.flow_level
[37]597
[43]598            # Reset possible simple key on the current level.
599            self.remove_possible_simple_key()
[37]600
[43]601        # Add VALUE.
[116]602        start_mark = self.reader.get_mark()
[46]603        self.reader.forward()
[116]604        end_mark = self.reader.get_mark()
605        self.tokens.append(ValueToken(start_mark, end_mark))
[37]606
[43]607    def fetch_alias(self):
[37]608
[43]609        # ALIAS could be a simple key.
610        self.save_possible_simple_key()
[37]611
[43]612        # No simple keys after ALIAS.
613        self.allow_simple_key = False
[37]614
[43]615        # Scan and add ALIAS.
[47]616        self.tokens.append(self.scan_anchor(AliasToken))
[37]617
[43]618    def fetch_anchor(self):
[37]619
[43]620        # ANCHOR could start a simple key.
621        self.save_possible_simple_key()
[37]622
[43]623        # No simple keys after ANCHOR.
624        self.allow_simple_key = False
[37]625
[43]626        # Scan and add ANCHOR.
[47]627        self.tokens.append(self.scan_anchor(AnchorToken))
[37]628
[43]629    def fetch_tag(self):
[37]630
[43]631        # TAG could start a simple key.
632        self.save_possible_simple_key()
[37]633
[43]634        # No simple keys after TAG.
635        self.allow_simple_key = False
[37]636
[43]637        # Scan and add TAG.
[47]638        self.tokens.append(self.scan_tag())
[37]639
[43]640    def fetch_literal(self):
641        self.fetch_block_scalar(folded=False)
[37]642
[43]643    def fetch_folded(self):
644        self.fetch_block_scalar(folded=True)
[37]645
[43]646    def fetch_block_scalar(self, folded):
[37]647
[43]648        # A simple key may follow a block scalar.
649        self.allow_simple_key = True
[37]650
[43]651        # Reset possible simple key on the current level.
652        self.remove_possible_simple_key()
[37]653
[43]654        # Scan and add SCALAR.
[47]655        self.tokens.append(self.scan_block_scalar(folded))
[37]656
[43]657    def fetch_single(self):
658        self.fetch_flow_scalar(double=False)
[37]659
[43]660    def fetch_double(self):
661        self.fetch_flow_scalar(double=True)
[37]662
[43]663    def fetch_flow_scalar(self, double):
[37]664
[43]665        # A flow scalar could be a simple key.
666        self.save_possible_simple_key()
[37]667
[43]668        # No simple keys after flow scalars.
669        self.allow_simple_key = False
[37]670
[43]671        # Scan and add SCALAR.
[47]672        self.tokens.append(self.scan_flow_scalar(double))
[37]673
[43]674    def fetch_plain(self):
[37]675
[43]676        # A plain scalar could be a simple key.
677        self.save_possible_simple_key()
[37]678
[43]679        # No simple keys after plain scalars. But note that `scan_plain` will
680        # change this flag if the scan is finished at the beginning of the
681        # line.
682        self.allow_simple_key = False
[37]683
[43]684        # Scan and add SCALAR. May change `allow_simple_key`.
[47]685        self.tokens.append(self.scan_plain())
[37]686
[43]687    # Checkers.
[37]688
[43]689    def check_directive(self):
[37]690
[43]691        # DIRECTIVE:        ^ '%' ...
692        # The '%' indicator is already checked.
[46]693        if self.reader.column == 0:
[43]694            return True
[37]695
[43]696    def check_document_start(self):
[37]697
[43]698        # DOCUMENT-START:   ^ '---' (' '|'\n')
[46]699        if self.reader.column == 0:
[48]700            if self.reader.prefix(3) == u'---'  \
701                    and self.reader.peek(3) in u'\0 \t\r\n\x85\u2028\u2029':
[43]702                return True
[37]703
[43]704    def check_document_end(self):
[37]705
[43]706        # DOCUMENT-END:     ^ '...' (' '|'\n')
[46]707        if self.reader.column == 0:
708            prefix = self.reader.peek(4)
[48]709            if self.reader.prefix(3) == u'...'  \
710                    and self.reader.peek(3) in u'\0 \t\r\n\x85\u2028\u2029':
[43]711                return True
[37]712
[51]713    def check_block_entry(self):
[43]714
[51]715        # BLOCK-ENTRY:      '-' (' '|'\n')
716        return self.reader.peek(1) in u'\0 \t\r\n\x85\u2028\u2029'
[43]717
718    def check_key(self):
719
720        # KEY(flow context):    '?'
721        if self.flow_level:
[37]722            return True
[43]723
724        # KEY(block context):   '?' (' '|'\n')
[37]725        else:
[48]726            return self.reader.peek(1) in u'\0 \t\r\n\x85\u2028\u2029'
[37]727
[43]728    def check_value(self):
729
730        # VALUE(flow context):  ':'
731        if self.flow_level:
[37]732            return True
[43]733
734        # VALUE(block context): ':' (' '|'\n')
[37]735        else:
[48]736            return self.reader.peek(1) in u'\0 \t\r\n\x85\u2028\u2029'
[37]737
[43]738    def check_plain(self):
[37]739
[48]740        # A plain scalar may start with any non-space character except:
741        #   '-', '?', ':', ',', '[', ']', '{', '}',
742        #   '#', '&', '*', '!', '|', '>', '\'', '\"',
743        #   '%', '@', '`'.
744        #
745        # It may also start with
746        #   '-', '?', ':'
747        # if it is followed by a non-space character.
748        #
749        # Note that we limit the last rule to the block context (except the
750        # '-' character) because we want the flow context to be space
751        # independent.
752        ch = self.reader.peek()
753        return ch not in u'\0 \t\r\n\x85\u2028\u2029-?:,[]{}#&*!|>\'\"%@`'  \
754                or (self.reader.peek(1) not in u'\0 \t\r\n\x85\u2028\u2029'
755                        and (ch == '-' or (not self.flow_level and ch in u'?:')))
756
[43]757    # Scanners.
758
759    def scan_to_next_token(self):
[47]760        # We ignore spaces, line breaks and comments.
761        # If we find a line break in the block context, we set the flag
762        # `allow_simple_key` on.
[51]763        # The byte order mark is stripped if it's the first character in the
764        # stream. We do not yet support BOM inside the stream as the
765        # specification requires. Any such mark will be considered as a part
766        # of the document.
[52]767        #
768        # TODO: We need to make tab handling rules more sane. A good rule is
769        #   Tabs cannot precede tokens
770        #   BLOCK-SEQUENCE-START, BLOCK-MAPPING-START, BLOCK-END,
771        #   KEY(block), VALUE(block), BLOCK-ENTRY
772        # So the checking code is
773        #   if <TAB>:
774        #       self.allow_simple_keys = False
775        # We also need to add the check for `allow_simple_keys == True` to
776        # `unwind_indent` before issuing BLOCK-END.
777        # Scanners for block, flow, and plain scalars need to be modified.
778
[51]779        if self.reader.index == 0 and self.reader.peek() == u'\uFEFF':
780            self.reader.forward()
[43]781        found = False
782        while not found:
[46]783            while self.reader.peek() == u' ':
784                self.reader.forward()
785            if self.reader.peek() == u'#':
[47]786                while self.reader.peek() not in u'\0\r\n\x85\u2028\u2029':
[46]787                    self.reader.forward()
[47]788            if self.scan_line_break():
[43]789                if not self.flow_level:
790                    self.allow_simple_key = True
[37]791            else:
[43]792                found = True
[37]793
[43]794    def scan_directive(self):
[48]795        # See the specification for details.
[116]796        start_mark = self.reader.get_mark()
[48]797        self.reader.forward()
[116]798        name = self.scan_directive_name(start_mark)
[48]799        value = None
800        if name == u'YAML':
[116]801            value = self.scan_yaml_directive_value(start_mark)
802            end_mark = self.reader.get_mark()
[48]803        elif name == u'TAG':
[116]804            value = self.scan_tag_directive_value(start_mark)
805            end_mark = self.reader.get_mark()
[43]806        else:
[116]807            end_mark = self.reader.get_mark()
[48]808            while self.reader.peek() not in u'\0\r\n\x85\u2028\u2029':
809                self.reader.forward()
[116]810        self.scan_directive_ignored_line(start_mark)
811        return DirectiveToken(name, value, start_mark, end_mark)
[48]812
[116]813    def scan_directive_name(self, start_mark):
[48]814        # See the specification for details.
815        length = 0
816        ch = self.reader.peek(length)
817        while u'0' <= ch <= u'9' or u'A' <= ch <= 'Z' or u'a' <= ch <= 'z'  \
818                or ch in u'-_':
819            length += 1
820            ch = self.reader.peek(length)
821        if not length:
[116]822            raise ScannerError("while scanning a directive", start_mark,
[52]823                    "expected alphabetic or numeric character, but found %r"
[116]824                    % ch.encode('utf-8'), self.reader.get_mark())
[48]825        value = self.reader.prefix(length)
826        self.reader.forward(length)
827        ch = self.reader.peek()
828        if ch not in u'\0 \r\n\x85\u2028\u2029':
[116]829            raise ScannerError("while scanning a directive", start_mark,
[48]830                    "expected alphabetic or numeric character, but found %r"
[116]831                    % ch.encode('utf-8'), self.reader.get_mark())
[48]832        return value
833
[116]834    def scan_yaml_directive_value(self, start_mark):
[48]835        # See the specification for details.
836        while self.reader.peek() == u' ':
[46]837            self.reader.forward()
[116]838        major = self.scan_yaml_directive_number(start_mark)
[48]839        if self.reader.peek() != '.':
[116]840            raise ScannerError("while scanning a directive", start_mark,
[52]841                    "expected a digit or '.', but found %r"
842                    % self.reader.peek().encode('utf-8'),
[116]843                    self.reader.get_mark())
[46]844        self.reader.forward()
[116]845        minor = self.scan_yaml_directive_number(start_mark)
[48]846        if self.reader.peek() not in u'\0 \r\n\x85\u2028\u2029':
[116]847            raise ScannerError("while scanning a directive", start_mark,
[52]848                    "expected a digit or ' ', but found %r"
849                    % self.reader.peek().encode('utf-8'),
[116]850                    self.reader.get_mark())
[48]851        return (major, minor)
[37]852
[116]853    def scan_yaml_directive_number(self, start_mark):
[48]854        # See the specification for details.
855        ch = self.reader.peek()
856        if not (u'0' <= ch <= '9'):
[116]857            raise ScannerError("while scanning a directive", start_mark,
[48]858                    "expected a digit, but found %r" % ch.encode('utf-8'),
[116]859                    self.reader.get_mark())
[48]860        length = 0
861        while u'0' <= self.reader.peek(length) <= u'9':
862            length += 1
863        value = int(self.reader.prefix(length))
864        self.reader.forward(length)
865        return value
866
[116]867    def scan_tag_directive_value(self, start_mark):
[48]868        # See the specification for details.
869        while self.reader.peek() == u' ':
870            self.reader.forward()
[116]871        handle = self.scan_tag_directive_handle(start_mark)
[48]872        while self.reader.peek() == u' ':
873            self.reader.forward()
[116]874        prefix = self.scan_tag_directive_prefix(start_mark)
[48]875        return (handle, prefix)
876
[116]877    def scan_tag_directive_handle(self, start_mark):
[48]878        # See the specification for details.
[116]879        value = self.scan_tag_handle('directive', start_mark)
[52]880        ch = self.reader.peek()
881        if ch != u' ':
[116]882            raise ScannerError("while scanning a directive", start_mark,
[48]883                    "expected ' ', but found %r" % ch.encode('utf-8'),
[116]884                    self.reader.get_mark())
[48]885        return value
886
[116]887    def scan_tag_directive_prefix(self, start_mark):
[48]888        # See the specification for details.
[116]889        value = self.scan_tag_uri('directive', start_mark)
[48]890        ch = self.reader.peek()
891        if ch not in u'\0 \r\n\x85\u2028\u2029':
[116]892            raise ScannerError("while scanning a directive", start_mark,
[48]893                    "expected ' ', but found %r" % ch.encode('utf-8'),
[116]894                    self.reader.get_mark())
[48]895        return value
896
[116]897    def scan_directive_ignored_line(self, start_mark):
[48]898        # See the specification for details.
899        while self.reader.peek() == u' ':
900            self.reader.forward()
901        if self.reader.peek() == u'#':
902            while self.reader.peek() not in u'\0\r\n\x85\u2028\u2029':
903                self.reader.forward()
904        ch = self.reader.peek()
905        if ch not in u'\0\r\n\x85\u2028\u2029':
[116]906            raise ScannerError("while scanning a directive", start_mark,
[48]907                    "expected a comment or a line break, but found %r"
[116]908                        % ch.encode('utf-8'), self.reader.get_mark())
[48]909        self.scan_line_break()
910
[43]911    def scan_anchor(self, TokenClass):
[48]912        # The specification does not restrict characters for anchors and
913        # aliases. This may lead to problems, for instance, the document:
914        #   [ *alias, value ]
915        # can be interpteted in two ways, as
916        #   [ "value" ]
917        # and
918        #   [ *alias , "value" ]
919        # Therefore we restrict aliases to numbers and ASCII letters.
[116]920        start_mark = self.reader.get_mark()
[48]921        indicator = self.reader.peek()
922        if indicator == '*':
923            name = 'alias'
924        else:
925            name = 'anchor'
926        self.reader.forward()
927        length = 0
928        ch = self.reader.peek(length)
929        while u'0' <= ch <= u'9' or u'A' <= ch <= 'Z' or u'a' <= ch <= 'z'  \
930                or ch in u'-_':
931            length += 1
932            ch = self.reader.peek(length)
933        if not length:
[116]934            raise ScannerError("while scanning an %s" % name, start_mark,
[52]935                    "expected alphabetic or numeric character, but found %r"
[116]936                    % ch.encode('utf-8'), self.reader.get_mark())
[48]937        value = self.reader.prefix(length)
938        self.reader.forward(length)
939        ch = self.reader.peek()
940        if ch not in u'\0 \t\r\n\x85\u2028\u2029?:,]}%@`':
[116]941            raise ScannerError("while scanning an %s" % name, start_mark,
[48]942                    "expected alphabetic or numeric character, but found %r"
[116]943                    % ch.encode('utf-8'), self.reader.get_mark())
944        end_mark = self.reader.get_mark()
945        return TokenClass(value, start_mark, end_mark)
[37]946
[43]947    def scan_tag(self):
[48]948        # See the specification for details.
[116]949        start_mark = self.reader.get_mark()
[48]950        ch = self.reader.peek(1)
951        if ch == u'<':
952            handle = None
953            self.reader.forward(2)
[116]954            suffix = self.scan_tag_uri('tag', start_mark)
[48]955            if self.reader.peek() != u'>':
[116]956                raise ScannerError("while parsing a tag", start_mark,
[52]957                        "expected '>', but found %r" % self.reader.peek().encode('utf-8'),
[116]958                        self.reader.get_mark())
[46]959            self.reader.forward()
[48]960        elif ch in u'\0 \t\r\n\x85\u2028\u2029':
961            handle = None
962            suffix = u'!'
963            self.reader.forward()
964        else:
965            length = 1
966            use_handle = False
967            while ch not in u'\0 \r\n\x85\u2028\u2029':
968                if ch == u'!':
969                    use_handle = True
970                    break
971                length += 1
972                ch = self.reader.peek(length)
973            handle = u'!'
974            if use_handle:
[116]975                handle = self.scan_tag_handle('tag', start_mark)
[48]976            else:
977                handle = u'!'
978                self.reader.forward()
[116]979            suffix = self.scan_tag_uri('tag', start_mark)
[48]980        ch = self.reader.peek()
981        if ch not in u'\0 \r\n\x85\u2028\u2029':
[116]982            raise ScannerError("while scanning a tag", start_mark,
[48]983                    "expected ' ', but found %r" % ch.encode('utf-8'),
[116]984                    self.reader.get_mark())
[48]985        value = (handle, suffix)
[116]986        end_mark = self.reader.get_mark()
987        return TagToken(value, start_mark, end_mark)
[43]988
989    def scan_block_scalar(self, folded):
[48]990        # See the specification for details.
991
992        chunks = []
[116]993        start_mark = self.reader.get_mark()
[48]994
995        # Scan the header.
996        self.reader.forward()
[116]997        chomping, increment = self.scan_block_scalar_indicators(start_mark)
998        self.scan_block_scalar_ignored_line(start_mark)
[48]999
1000        # Determine the indentation level and go to the first non-empty line.
1001        min_indent = self.indent+1
1002        if min_indent < 1:
1003            min_indent = 1
1004        if increment is None:
[116]1005            breaks, max_indent, end_mark = self.scan_block_scalar_indentation()
[48]1006            indent = max(min_indent, max_indent)
1007        else:
1008            indent = min_indent+increment-1
[116]1009            breaks, end_mark = self.scan_block_scalar_breaks(indent)
[48]1010        line_break = u''
1011
1012        # Scan the inner part of the block scalar.
1013        while self.reader.column == indent and self.reader.peek() != u'\0':
1014            chunks.extend(breaks)
1015            leading_non_space = self.reader.peek() not in u' \t'
1016            length = 0
1017            while self.reader.peek(length) not in u'\0\r\n\x85\u2028\u2029':
1018                length += 1
1019            chunks.append(self.reader.prefix(length))
1020            self.reader.forward(length)
1021            line_break = self.scan_line_break()
[116]1022            breaks, end_mark = self.scan_block_scalar_breaks(indent)
[48]1023            if self.reader.column == indent and self.reader.peek() != u'\0':
1024                # Unfortunately, folding rules are ambiguous.
1025                #
1026                # This is the folding according to the specification:
[51]1027               
1028                if folded and line_break == u'\n'   \
1029                        and leading_non_space and self.reader.peek() not in u' \t':
1030                    if not breaks:
1031                        chunks.append(u' ')
1032                else:
1033                    chunks.append(line_break)
1034               
1035                # This is Clark Evans's interpretation (also in the spec
1036                # examples):
[48]1037                #
[51]1038                #if folded and line_break == u'\n':
[48]1039                #    if not breaks:
[51]1040                #        if self.reader.peek() not in ' \t':
1041                #            chunks.append(u' ')
1042                #        else:
1043                #            chunks.append(line_break)
[48]1044                #else:
1045                #    chunks.append(line_break)
1046            else:
1047                break
1048
1049        # Chomp the tail.
1050        if chomping is not False:
1051            chunks.append(line_break)
1052        if chomping is True:
1053            chunks.extend(breaks)
1054
1055        # We are done.
[116]1056        return ScalarToken(u''.join(chunks), False, start_mark, end_mark)
[48]1057
[116]1058    def scan_block_scalar_indicators(self, start_mark):
[48]1059        # See the specification for details.
1060        chomping = None
1061        increment = None
1062        ch = self.reader.peek()
1063        if ch in u'+-':
1064            if ch == '+':
1065                chomping = True
1066            else:
1067                chomping = False
1068            self.reader.forward()
1069            ch = self.reader.peek()
1070            if ch in u'0123456789':
1071                increment = int(ch)
1072                if increment == 0:
[116]1073                    raise ScannerError("while scanning a block scalar", start_mark,
[48]1074                            "expected indentation indicator in the range 1-9, but found 0",
[116]1075                            self.reader.get_mark())
[46]1076                self.reader.forward()
[48]1077        elif ch in u'0123456789':
1078            increment = int(ch)
1079            if increment == 0:
[116]1080                raise ScannerError("while scanning a block scalar", start_mark,
[48]1081                        "expected indentation indicator in the range 1-9, but found 0",
[116]1082                        self.reader.get_mark())
[48]1083            self.reader.forward()
1084            ch = self.reader.peek()
1085            if ch in u'+-':
1086                if ch == '+':
1087                    chomping = True
1088                else:
1089                    chomping = False
[46]1090                self.reader.forward()
[48]1091        ch = self.reader.peek()
1092        if ch not in u'\0 \r\n\x85\u2028\u2029':
[116]1093            raise ScannerError("while scanning a block scalar", start_mark,
[48]1094                    "expected chomping or indentation indicators, but found %r"
[116]1095                        % ch.encode('utf-8'), self.reader.get_mark())
[48]1096        return chomping, increment
1097
[116]1098    def scan_block_scalar_ignored_line(self, start_mark):
[48]1099        # See the specification for details.
1100        while self.reader.peek() == u' ':
1101            self.reader.forward()
1102        if self.reader.peek() == u'#':
1103            while self.reader.peek() not in u'\0\r\n\x85\u2028\u2029':
[46]1104                self.reader.forward()
[48]1105        ch = self.reader.peek()
1106        if ch not in u'\0\r\n\x85\u2028\u2029':
[116]1107            raise ScannerError("while scanning a block scalar", start_mark,
[48]1108                    "expected a comment or a line break, but found %r"
[116]1109                        % ch.encode('utf-8'), self.reader.get_mark())
[48]1110        self.scan_line_break()
[43]1111
[48]1112    def scan_block_scalar_indentation(self):
1113        # See the specification for details.
1114        chunks = []
1115        max_indent = 0
[116]1116        end_mark = self.reader.get_mark()
[48]1117        while self.reader.peek() in u' \r\n\x85\u2028\u2029':
1118            if self.reader.peek() != u' ':
1119                chunks.append(self.scan_line_break())
[116]1120                end_mark = self.reader.get_mark()
[48]1121            else:
1122                self.reader.forward()
1123                if self.reader.column > max_indent:
1124                    max_indent = self.reader.column
[116]1125        return chunks, max_indent, end_mark
[48]1126
1127    def scan_block_scalar_breaks(self, indent):
1128        # See the specification for details.
1129        chunks = []
[116]1130        end_mark = self.reader.get_mark()
[48]1131        while self.reader.column < indent and self.reader.peek() == u' ':
1132            self.reader.forward()
1133        while self.reader.peek() in u'\r\n\x85\u2028\u2029':
1134            chunks.append(self.scan_line_break())
[116]1135            end_mark = self.reader.get_mark()
[48]1136            while self.reader.column < indent and self.reader.peek() == u' ':
1137                self.reader.forward()
[116]1138        return chunks, end_mark
[48]1139
[43]1140    def scan_flow_scalar(self, double):
[48]1141        # See the specification for details.
[117]1142        # Note that we loose indentation rules for quoted scalars. Quoted
1143        # scalars don't need to adhere indentation because " and ' clearly
1144        # mark the beginning and the end of them. Therefore we are less
1145        # restrictive then the specification requires. We only need to check
1146        # that document separators are not included in scalars.
[48]1147        chunks = []
[116]1148        start_mark = self.reader.get_mark()
[46]1149        quote = self.reader.peek()
1150        self.reader.forward()
[117]1151        chunks.extend(self.scan_flow_scalar_non_spaces(double, start_mark))
[46]1152        while self.reader.peek() != quote:
[117]1153            chunks.extend(self.scan_flow_scalar_spaces(double, start_mark))
1154            chunks.extend(self.scan_flow_scalar_non_spaces(double, start_mark))
[48]1155        self.reader.forward()
[116]1156        end_mark = self.reader.get_mark()
1157        return ScalarToken(u''.join(chunks), False, start_mark, end_mark)
[48]1158
1159    ESCAPE_REPLACEMENTS = {
1160        u'0':   u'\0',
1161        u'a':   u'\x07',
1162        u'b':   u'\x08',
1163        u't':   u'\x09',
1164        u'\t':  u'\x09',
1165        u'n':   u'\x0A',
1166        u'v':   u'\x0B',
1167        u'f':   u'\x0C',
1168        u'r':   u'\x0D',
1169        u'e':   u'\x1B',
1170        u' ':   u'\x20',
1171        u'\"':  u'\"',
1172        u'\\':  u'\\',
1173        u'N':   u'\x85',
1174        u'_':   u'\xA0',
1175        u'L':   u'\u2028',
1176        u'P':   u'\u2029',
1177    }
1178
1179    ESCAPE_CODES = {
1180        u'x':   2,
1181        u'u':   4,
1182        u'U':   8,
1183    }
1184
[117]1185    def scan_flow_scalar_non_spaces(self, double, start_mark):
[48]1186        # See the specification for details.
1187        chunks = []
1188        while True:
1189            length = 0
1190            while self.reader.peek(length) not in u'\'\"\\\0 \t\r\n\x85\u2028\u2029':
1191                length += 1
1192            if length:
1193                chunks.append(self.reader.prefix(length))
1194                self.reader.forward(length)
1195            ch = self.reader.peek()
1196            if not double and ch == u'\'' and self.reader.peek(1) == u'\'':
1197                chunks.append(u'\'')
[46]1198                self.reader.forward(2)
[48]1199            elif (double and ch == u'\'') or (not double and ch in u'\"\\'):
1200                chunks.append(ch)
1201                self.reader.forward()
1202            elif double and ch == u'\\':
1203                self.reader.forward()
1204                ch = self.reader.peek()
1205                if ch in self.ESCAPE_REPLACEMENTS:
1206                    chunks.append(self.ESCAPE_REPLACEMENTS[ch])
1207                    self.reader.forward()
1208                elif ch in self.ESCAPE_CODES:
1209                    length = self.ESCAPE_CODES[ch]
1210                    self.reader.forward()
1211                    for k in range(length):
1212                        if self.reader.peek(k) not in u'0123456789ABCDEFabcdef':
[116]1213                            raise ScannerError("while scanning a double-quoted scalar", start_mark,
[48]1214                                    "expected escape sequence of %d hexdecimal numbers, but found %r" %
[116]1215                                        (length, self.reader.peek(k).encode('utf-8')), self.reader.get_mark())
[48]1216                    code = int(self.reader.prefix(length), 16)
1217                    chunks.append(unichr(code))
1218                    self.reader.forward(length)
1219                elif ch in u'\r\n\x85\u2028\u2029':
1220                    self.scan_line_break()
[117]1221                    chunks.extend(self.scan_flow_scalar_breaks(double, start_mark))
[48]1222                else:
[116]1223                    raise ScannerError("while scanning a double-quoted scalar", start_mark,
1224                            "found unknown escape character %r" % ch.encode('utf-8'), self.reader.get_mark())
[37]1225            else:
[48]1226                return chunks
[37]1227
[117]1228    def scan_flow_scalar_spaces(self, double, start_mark):
[48]1229        # See the specification for details.
1230        chunks = []
1231        length = 0
1232        while self.reader.peek(length) in u' \t':
1233            length += 1
1234        whitespaces = self.reader.prefix(length)
1235        self.reader.forward(length)
1236        ch = self.reader.peek()
1237        if ch == u'\0':
[116]1238            raise ScannerError("while scanning a quoted scalar", start_mark,
1239                    "found unexpected end of stream", self.reader.get_mark())
[48]1240        elif ch in u'\r\n\x85\u2028\u2029':
1241            line_break = self.scan_line_break()
[117]1242            breaks = self.scan_flow_scalar_breaks(double, start_mark)
[48]1243            if line_break != u'\n':
1244                chunks.append(line_break)
1245            elif not breaks:
1246                chunks.append(u' ')
1247            chunks.extend(breaks)
1248        else:
1249            chunks.append(whitespaces)
1250        return chunks
1251
[117]1252    def scan_flow_scalar_breaks(self, double, start_mark):
[48]1253        # See the specification for details.
1254        chunks = []
1255        while True:
[117]1256            # Instead of checking indentation, we check for document
1257            # separators.
1258            prefix = self.reader.prefix(3)
1259            if (prefix == u'---' or prefix == u'...')   \
1260                    and self.reader.peek(3) in u'\0 \t\r\n\x85\u2028\u2029':
[116]1261                raise ScannerError("while scanning a quoted scalar", start_mark,
[117]1262                        "found unexpected document separator", self.reader.get_mark())
[48]1263            while self.reader.peek() in u' \t':
1264                self.reader.forward()
1265            if self.reader.peek() in u'\r\n\x85\u2028\u2029':
1266                chunks.append(self.scan_line_break())
1267            else:
1268                return chunks
1269
[43]1270    def scan_plain(self):
[48]1271        # See the specification for details.
1272        # We add an additional restriction for the flow context:
[117]1273        #   plain scalars in the flow context cannot contain ',', ':' and '?'.
[48]1274        # We also keep track of the `allow_simple_key` flag here.
[117]1275        # Indentation rules are loosed for the flow context.
[48]1276        chunks = []
[116]1277        start_mark = self.reader.get_mark()
1278        end_mark = start_mark
[43]1279        indent = self.indent+1
[117]1280        # We allow zero indentation for scalars, but then we need to check for
1281        # document separators at the beginning of the line.
1282        #if indent == 0:
1283        #    indent = 1
[48]1284        spaces = []
[43]1285        while True:
[48]1286            length = 0
1287            if self.reader.peek() == u'#':
[43]1288                break
[48]1289            while True:
1290                ch = self.reader.peek(length)
1291                if ch in u'\0 \t\r\n\x85\u2028\u2029'   \
1292                        or (not self.flow_level and ch == u':' and
1293                                self.reader.peek(length+1) in u'\0 \t\r\n\x28\u2028\u2029') \
1294                        or (self.flow_level and ch in u',:?[]{}'):
1295                    break
1296                length += 1
1297            if length == 0:
[43]1298                break
[48]1299            self.allow_simple_key = False
1300            chunks.extend(spaces)
1301            chunks.append(self.reader.prefix(length))
1302            self.reader.forward(length)
[116]1303            end_mark = self.reader.get_mark()
[117]1304            spaces = self.scan_plain_spaces(indent, start_mark)
[48]1305            if not spaces or self.reader.peek() == u'#' \
[117]1306                    or (not self.flow_level and self.reader.column < indent):
[48]1307                break
[116]1308        return ScalarToken(u''.join(chunks), True, start_mark, end_mark)
[37]1309
[117]1310    def scan_plain_spaces(self, indent, start_mark):
[48]1311        # See the specification for details.
1312        # The specification is really confusing about tabs in plain scalars.
1313        # We just forbid them completely. Do not use tabs in YAML!
1314        chunks = []
1315        length = 0
1316        while self.reader.peek(length) in u' ':
1317            length += 1
1318        whitespaces = self.reader.prefix(length)
1319        self.reader.forward(length)
1320        ch = self.reader.peek()
1321        if ch in u'\r\n\x85\u2028\u2029':
1322            line_break = self.scan_line_break()
1323            self.allow_simple_key = True
[117]1324            prefix = self.reader.prefix(3)
1325            if (prefix == u'---' or prefix == u'...')   \
1326                    and self.reader.peek(3) in u'\0 \t\r\n\x85\u2028\u2029':
1327                return
[48]1328            breaks = []
1329            while self.reader.peek() in u' \r\n\x85\u2028\u2029':
1330                if self.reader.peek() == ' ':
1331                    self.reader.forward()
1332                else:
1333                    breaks.append(self.scan_line_break())
[117]1334                    prefix = self.reader.prefix(3)
1335                    if (prefix == u'---' or prefix == u'...')   \
1336                            and self.reader.peek(3) in u'\0 \t\r\n\x85\u2028\u2029':
1337                        return
[48]1338            if line_break != u'\n':
1339                chunks.append(line_break)
1340            elif not breaks:
1341                chunks.append(u' ')
1342            chunks.extend(breaks)
1343        elif whitespaces:
1344            chunks.append(whitespaces)
1345        return chunks
1346
[116]1347    def scan_tag_handle(self, name, start_mark):
[48]1348        # See the specification for details.
1349        # For some strange reasons, the specification does not allow '_' in
1350        # tag handles. I have allowed it anyway.
[52]1351        ch = self.reader.peek()
1352        if ch != u'!':
[116]1353            raise ScannerError("while scanning a %s" % name, start_mark,
[48]1354                    "expected '!', but found %r" % ch.encode('utf-8'),
[116]1355                    self.reader.get_mark())
[48]1356        length = 1
1357        ch = self.reader.peek(length)
1358        if ch != u' ':
1359            while u'0' <= ch <= u'9' or u'A' <= ch <= 'Z' or u'a' <= ch <= 'z'  \
1360                    or ch in u'-_':
1361                length += 1
1362                ch = self.reader.peek(length)
1363            if ch != u'!':
1364                self.reader.forward(length)
[116]1365                raise ScannerError("while scanning a %s" % name, start_mark,
[48]1366                        "expected '!', but found %r" % ch.encode('utf-8'),
[116]1367                        self.reader.get_mark())
[48]1368            length += 1
1369        value = self.reader.prefix(length)
1370        self.reader.forward(length)
1371        return value
1372
[116]1373    def scan_tag_uri(self, name, start_mark):
[48]1374        # See the specification for details.
1375        # Note: we do not check if URI is well-formed.
1376        chunks = []
1377        length = 0
1378        ch = self.reader.peek(length)
1379        while u'0' <= ch <= u'9' or u'A' <= ch <= 'Z' or u'a' <= ch <= 'z'  \
1380                or ch in u'-;/?:@&=+$,_.!~*\'()[]%':
1381            if ch == u'%':
1382                chunks.append(self.reader.prefix(length))
1383                self.reader.forward(length)
1384                length = 0
[116]1385                chunks.append(self.scan_uri_escapes(name, start_mark))
[48]1386            else:
1387                length += 1
1388            ch = self.reader.peek(length)
1389        if length:
1390            chunks.append(self.reader.prefix(length))
1391            self.reader.forward(length)
1392            length = 0
1393        if not chunks:
[116]1394            raise ScannerError("while parsing a %s" % name, start_mark,
[48]1395                    "expected URI, but found %r" % ch.encode('utf-8'),
[116]1396                    self.reader.get_mark())
[48]1397        return u''.join(chunks)
1398
[116]1399    def scan_uri_escapes(self, name, start_mark):
[48]1400        # See the specification for details.
1401        bytes = []
[116]1402        mark = self.reader.get_mark()
[48]1403        while self.reader.peek() == u'%':
1404            self.reader.forward()
1405            for k in range(2):
1406                if self.reader.peek(k) not in u'0123456789ABCDEFabcdef':
[116]1407                    raise ScannerError("while scanning a %s" % name, start_mark,
[48]1408                            "expected URI escape sequence of 2 hexdecimal numbers, but found %r" %
[116]1409                                (self.reader.peek(k).encode('utf-8')), self.reader.get_mark())
[48]1410            bytes.append(chr(int(self.reader.prefix(2), 16)))
1411            self.reader.forward(2)
1412        try:
1413            value = unicode(''.join(bytes), 'utf-8')
1414        except UnicodeDecodeError, exc:
[116]1415            raise ScannerError("while scanning a %s" % name, start_mark, str(exc), mark)
[48]1416        return value
1417
[47]1418    def scan_line_break(self):
1419        # Transforms:
1420        #   '\r\n'      :   '\n'
1421        #   '\r'        :   '\n'
1422        #   '\n'        :   '\n'
1423        #   '\x85'      :   '\n'
1424        #   '\u2028'    :   '\u2028'
1425        #   '\u2029     :   '\u2029'
1426        #   default     :   ''
1427        ch = self.reader.peek()
1428        if ch in u'\r\n\x85':
[48]1429            if self.reader.prefix(2) == u'\r\n':
[60]1430                self.reader.forward(2)
[47]1431            else:
1432                self.reader.forward()
1433            return u'\n'
1434        elif ch in u'\u2028\u2029':
1435            self.reader.forward()
1436            return ch
1437        return u''
1438
[45]1439#try:
1440#    import psyco
1441#    psyco.bind(Scanner)
1442#except ImportError:
1443#    pass
1444
Note: See TracBrowser for help on using the repository browser.