Context Navigation

source: branches/pyyaml3000/lib/yaml/scanner.py @ 55

Revision 55, 51.5 KB checked in by xi, 7 years ago (diff)
Working on Constructor.

Rev	Line
[39]	1
[55]	2	# Scanner produces tokens of the following types:
	3	# DIRECTIVE(name, value)
	4	# DOCUMENT-START
	5	# DOCUMENT-END
	6	# STREAM-END
	7	# BLOCK-SEQUENCE-START
	8	# BLOCK-MAPPING-START
	9	# BLOCK-END
	10	# FLOW-SEQUENCE-START
	11	# FLOW-MAPPING-START
	12	# FLOW-SEQUENCE-END
	13	# FLOW-MAPPING-END
	14	# BLOCK-ENTRY
	15	# FLOW-ENTRY
	16	# KEY
	17	# VALUE
	18	# ALIAS(value)
	19	# ANCHOR(value)
	20	# TAG(value)
	21	# SCALAR(value, plain)
[43]	22
[46]	23	__all__ = ['Scanner', 'ScannerError']
[43]	24
[52]	25	from error import MarkedYAMLError
[46]	26	from tokens import *
[39]	27
[52]	28	class ScannerError(MarkedYAMLError):
	29	pass
[51]	30
[43]	31	class SimpleKey:
[51]	32	# See below simple keys treatment.
	33
[43]	34	def __init__(self, token_number, required, index, line, column, marker):
	35	self.token_number = token_number
	36	self.required = required
	37	self.index = index
	38	self.line = line
	39	self.column = column
	40	self.marker = marker
	41
[39]	42	class Scanner:
	43
[46]	44
	45	def __init__(self, reader):
[39]	46	"""Initialize the scanner."""
[46]	47	# The input stream. The Reader class do the dirty work of checking for
[43]	48	# BOM and converting the input data to Unicode. It also adds NUL to
	49	# the end.
[39]	50	#
[46]	51	# Reader supports the following methods
[48]	52	# self.reader.peek(i=0) # peek the next i-th character
	53	# self.reader.prefix(l=1) # peek the next l characters
	54	# self.reader.forward(l=1) # read the next l characters
	55	# and move the pointer
[46]	56	self.reader = reader
[39]	57
	58	# Had we reached the end of the stream?
	59	self.done = False
	60
	61	# The number of unclosed '{' and '['. `flow_level == 0` means block
	62	# context.
	63	self.flow_level = 0
	64
	65	# List of processed tokens that are not yet emitted.
	66	self.tokens = []
	67
	68	# Number of tokens that were emitted through the `get_token` method.
	69	self.tokens_taken = 0
	70
	71	# The current indentation level.
	72	self.indent = -1
	73
	74	# Past indentation levels.
	75	self.indents = []
	76
[43]	77	# Variables related to simple keys treatment.
[39]	78
	79	# A simple key is a key that is not denoted by the '?' indicator.
	80	# Example of simple keys:
	81	# ---
	82	# block simple key: value
	83	# ? not a simple key:
	84	# : { flow simple key: value }
	85	# We emit the KEY token before all keys, so when we find a potential
	86	# simple key, we try to locate the corresponding ':' indicator.
	87	# Simple keys should be limited to a single line and 1024 characters.
	88
[43]	89	# Can a simple key start at the current position? A simple key may
	90	# start:
	91	# - at the beginning of the line, not counting indentation spaces
	92	# (in block context),
	93	# - after '{', '[', ',' (in the flow context),
	94	# - after '?', ':', '-' (in the block context).
	95	# In the block context, this flag also signify if a block collection
	96	# may start at the current position.
	97	self.allow_simple_key = True
[39]	98
	99	# Keep track of possible simple keys. This is a dictionary. The key
	100	# is `flow_level`; there can be no more that one possible simple key
[43]	101	# for each level. The value is a SimpleKey record:
	102	# (token_number, required, index, line, column, marker)
	103	# A simple key may start with ALIAS, ANCHOR, TAG, SCALAR(flow),
	104	# '[', or '{' tokens.
[39]	105	self.possible_simple_keys = {}
	106
[51]	107	# Public methods.
[39]	108
[51]	109	def check(self, *choices):
	110	# Check if the next token is one of the given types.
[43]	111	while self.need_more_tokens():
[39]	112	self.fetch_more_tokens()
	113	if self.tokens:
[51]	114	for choice in choices:
	115	if isinstance(self.tokens[0], choice):
	116	return True
	117	return False
	118
	119	def peek(self):
	120	# Return the next token, but do not delete if from the queue.
	121	while self.need_more_tokens():
	122	self.fetch_more_tokens()
	123	if self.tokens:
[39]	124	return self.tokens[0]
	125
[51]	126	def get(self):
	127	# Return the next token.
[39]	128	while self.need_more_tokens():
	129	self.fetch_more_tokens()
	130	if self.tokens:
	131	self.tokens_taken += 1
	132	return self.tokens.pop(0)
	133
[51]	134	def __iter__(self):
	135	# Iterator protocol.
	136	while self.need_more_tokens():
	137	self.fetch_more_tokens()
	138	while self.tokens:
	139	self.tokens_taken += 1
	140	yield self.tokens.pop(0)
	141	while self.need_more_tokens():
	142	self.fetch_more_tokens()
	143
[43]	144	# Private methods.
[39]	145
	146	def need_more_tokens(self):
	147	if self.done:
	148	return False
	149	if not self.tokens:
	150	return True
	151	# The current token may be a potential simple key, so we
	152	# need to look further.
[43]	153	self.stale_possible_simple_keys()
[39]	154	if self.next_possible_simple_key() == self.tokens_taken:
	155	return True
	156
	157	def fetch_more_tokens(self):
	158
	159	# Eat whitespaces and comments until we reach the next token.
[43]	160	self.scan_to_next_token()
[39]	161
[43]	162	# Remove obsolete possible simple keys.
	163	self.stale_possible_simple_keys()
	164
[39]	165	# Compare the current indentation and column. It may add some tokens
[43]	166	# and decrease the current indentation level.
[46]	167	self.unwind_indent(self.reader.column)
[39]	168
	169	# Peek the next character.
[46]	170	ch = self.reader.peek()
[39]	171
[48]	172	# Is it the end of stream?
[43]	173	if ch == u'\0':
[48]	174	return self.fetch_stream_end()
[39]	175
	176	# Is it a directive?
	177	if ch == u'%' and self.check_directive():
	178	return self.fetch_directive()
	179
	180	# Is it the document start?
	181	if ch == u'-' and self.check_document_start():
	182	return self.fetch_document_start()
	183
	184	# Is it the document end?
	185	if ch == u'.' and self.check_document_end():
	186	return self.fetch_document_end()
	187
[52]	188	# TODO: support for BOM within a stream.
	189	#if ch == u'\uFEFF':
	190	# return self.fetch_bom() <-- issue BOMToken
	191
[39]	192	# Note: the order of the following checks is NOT significant.
	193
	194	# Is it the flow sequence start indicator?
	195	if ch == u'[':
	196	return self.fetch_flow_sequence_start()
	197
	198	# Is it the flow mapping start indicator?
	199	if ch == u'{':
	200	return self.fetch_flow_mapping_start()
	201
	202	# Is it the flow sequence end indicator?
	203	if ch == u']':
	204	return self.fetch_flow_sequence_end()
	205
	206	# Is it the flow mapping end indicator?
	207	if ch == u'}':
	208	return self.fetch_flow_mapping_end()
	209
[51]	210	# Is it the flow entry indicator?
	211	if ch in u',':
	212	return self.fetch_flow_entry()
[43]	213
[51]	214	# Is it the block entry indicator?
	215	if ch in u'-' and self.check_block_entry():
	216	return self.fetch_block_entry()
	217
[39]	218	# Is it the key indicator?
	219	if ch == u'?' and self.check_key():
	220	return self.fetch_key()
	221
	222	# Is it the value indicator?
	223	if ch == u':' and self.check_value():
	224	return self.fetch_value()
	225
	226	# Is it an alias?
	227	if ch == u'*':
	228	return self.fetch_alias()
	229
	230	# Is it an anchor?
	231	if ch == u'&':
	232	return self.fetch_anchor()
	233
[43]	234	# Is it a tag?
[39]	235	if ch == u'!':
	236	return self.fetch_tag()
	237
[43]	238	# Is it a literal scalar?
	239	if ch == u'\|' and not self.flow_level:
[39]	240	return self.fetch_literal()
	241
	242	# Is it a folded scalar?
[43]	243	if ch == u'>' and not self.flow_level:
[39]	244	return self.fetch_folded()
	245
	246	# Is it a single quoted scalar?
	247	if ch == u'\'':
	248	return self.fetch_single()
	249
	250	# Is it a double quoted scalar?
	251	if ch == u'\"':
	252	return self.fetch_double()
	253
[43]	254	# It must be a plain scalar then.
[39]	255	if self.check_plain():
	256	return self.fetch_plain()
	257
[43]	258	# No? It's an error. Let's produce a nice error message.
[48]	259	raise ScannerError("while scanning for the next token", None,
	260	"found character %r that cannot start any token"
	261	% ch.encode('utf-8'), self.reader.get_marker())
[39]	262
[43]	263	# Simple keys treatment.
	264
	265	def next_possible_simple_key(self):
	266	# Return the number of the nearest possible simple key. Actually we
	267	# don't need to loop through the whole dictionary. We may replace it
	268	# with the following code:
	269	# if not self.possible_simple_keys:
	270	# return None
	271	# return self.possible_simple_keys[
	272	# min(self.possible_simple_keys.keys())].token_number
	273	min_token_number = None
	274	for level in self.possible_simple_keys:
	275	key = self.possible_simple_keys[level]
	276	if min_token_number is None or key.token_number < min_token_number:
	277	min_token_number = key.token_number
	278	return min_token_number
	279
	280	def stale_possible_simple_keys(self):
	281	# Remove entries that are no longer possible simple keys. According to
	282	# the YAML specification, simple keys
	283	# - should be limited to a single line,
	284	# - should be no longer than 1024 characters.
	285	# Disabling this procedure will allow simple keys of any length and
	286	# height (may cause problems if indentation is broken though).
	287	for level in self.possible_simple_keys.keys():
	288	key = self.possible_simple_keys[level]
[46]	289	if key.line != self.reader.line \
	290	or self.reader.index-key.index > 1024:
[43]	291	if key.required:
[47]	292	raise ScannerError("while scanning a simple key", key.marker,
	293	"could not found expected ':'", self.reader.get_marker())
[43]	294	del self.possible_simple_keys[level]
	295
	296	def save_possible_simple_key(self):
	297	# The next token may start a simple key. We check if it's possible
	298	# and save its position. This function is called for
	299	# ALIAS, ANCHOR, TAG, SCALAR(flow), '[', and '{'.
	300
	301	# Check if a simple key is required at the current position.
[46]	302	required = not self.flow_level and self.indent == self.reader.column
[43]	303
[47]	304	# A simple key is required only if it is the first token in the current
	305	# line. Therefore it is always allowed.
	306	assert self.allow_simple_key or not required
	307
[43]	308	# The next token might be a simple key. Let's save it's number and
	309	# position.
	310	if self.allow_simple_key:
	311	self.remove_possible_simple_key()
	312	token_number = self.tokens_taken+len(self.tokens)
[46]	313	index = self.reader.index
	314	line = self.reader.line
	315	column = self.reader.column
	316	marker = self.reader.get_marker()
[43]	317	key = SimpleKey(token_number, required,
	318	index, line, column, marker)
	319	self.possible_simple_keys[self.flow_level] = key
	320
	321	def remove_possible_simple_key(self):
	322	# Remove the saved possible key position at the current flow level.
	323	if self.flow_level in self.possible_simple_keys:
	324	key = self.possible_simple_keys[self.flow_level]
[47]	325
	326	# I don't think it's possible, but I could be wrong.
	327	assert not key.required
	328	#if key.required:
	329	# raise ScannerError("while scanning a simple key", key.marker,
	330	# "could not found expected ':'", self.reader.get_marker())
[43]	331
	332	# Indentation functions.
	333
	334	def unwind_indent(self, column):
	335
	336	# In flow context, tokens should respect indentation.
[47]	337	# Actually the condition should be `self.indent >= column` according to
	338	# the spec. But this condition will prohibit intuitively correct
	339	# constructions such as
	340	# key : {
	341	# }
[43]	342	if self.flow_level and self.indent > column:
[47]	343	raise ScannerError(None, None,
	344	"invalid intendation or unclosed '[' or '{'",
	345	self.reader.get_marker())
[43]	346
	347	# In block context, we may need to issue the BLOCK-END tokens.
	348	while self.indent > column:
[46]	349	marker = self.reader.get_marker()
[43]	350	self.indent = self.indents.pop()
[44]	351	self.tokens.append(BlockEndToken(marker, marker))
[43]	352
	353	def add_indent(self, column):
	354	# Check if we need to increase indentation.
	355	if self.indent < column:
	356	self.indents.append(self.indent)
	357	self.indent = column
	358	return True
	359	return False
	360
	361	# Fetchers.
	362
[48]	363	def fetch_stream_end(self):
[39]	364
	365	# Set the current intendation to -1.
[43]	366	self.unwind_indent(-1)
[39]	367
	368	# Reset everything (not really needed).
[43]	369	self.allow_simple_key = False
[39]	370	self.possible_simple_keys = {}
	371
[43]	372	# Read the token.
[46]	373	marker = self.reader.get_marker()
[43]	374
	375	# Add END.
[47]	376	self.tokens.append(StreamEndToken(marker, marker))
[39]	377
[46]	378	# The reader is ended.
[39]	379	self.done = True
	380
[43]	381	def fetch_directive(self):
	382
	383	# Set the current intendation to -1.
	384	self.unwind_indent(-1)
[39]	385
[43]	386	# Reset simple keys.
	387	self.remove_possible_simple_key()
	388	self.allow_simple_key = False
[39]	389
[43]	390	# Scan and add DIRECTIVE.
[47]	391	self.tokens.append(self.scan_directive())
[39]	392
	393	def fetch_document_start(self):
[44]	394	self.fetch_document_indicator(DocumentStartToken)
[39]	395
[43]	396	def fetch_document_end(self):
[44]	397	self.fetch_document_indicator(DocumentEndToken)
[43]	398
	399	def fetch_document_indicator(self, TokenClass):
	400
[39]	401	# Set the current intendation to -1.
[43]	402	self.unwind_indent(-1)
[39]	403
[43]	404	# Reset simple keys. Note that there could not be a block collection
	405	# after '---'.
	406	self.remove_possible_simple_key()
	407	self.allow_simple_key = False
[39]	408
[43]	409	# Add DOCUMENT-START or DOCUMENT-END.
[46]	410	start_marker = self.reader.get_marker()
	411	self.reader.forward(3)
	412	end_marker = self.reader.get_marker()
[43]	413	self.tokens.append(TokenClass(start_marker, end_marker))
[39]	414
[43]	415	def fetch_flow_sequence_start(self):
[44]	416	self.fetch_flow_collection_start(FlowSequenceStartToken)
[39]	417
[43]	418	def fetch_flow_mapping_start(self):
[44]	419	self.fetch_flow_collection_start(FlowMappingStartToken)
[43]	420
	421	def fetch_flow_collection_start(self, TokenClass):
	422
[44]	423	# '[' and '{' may start a simple key.
	424	self.save_possible_simple_key()
	425
[43]	426	# Increase the flow level.
	427	self.flow_level += 1
	428
	429	# Simple keys are allowed after '[' and '{'.
	430	self.allow_simple_key = True
	431
	432	# Add FLOW-SEQUENCE-START or FLOW-MAPPING-START.
[46]	433	start_marker = self.reader.get_marker()
	434	self.reader.forward()
	435	end_marker = self.reader.get_marker()
[43]	436	self.tokens.append(TokenClass(start_marker, end_marker))
[39]	437
[43]	438	def fetch_flow_sequence_end(self):
[44]	439	self.fetch_flow_collection_end(FlowSequenceEndToken)
[39]	440
[43]	441	def fetch_flow_mapping_end(self):
[44]	442	self.fetch_flow_collection_end(FlowMappingEndToken)
[43]	443
	444	def fetch_flow_collection_end(self, TokenClass):
	445
	446	# Reset possible simple key on the current level.
	447	self.remove_possible_simple_key()
	448
	449	# Decrease the flow level.
	450	self.flow_level -= 1
	451
	452	# No simple keys after ']' or '}'.
	453	self.allow_simple_key = False
	454
	455	# Add FLOW-SEQUENCE-END or FLOW-MAPPING-END.
[46]	456	start_marker = self.reader.get_marker()
	457	self.reader.forward()
	458	end_marker = self.reader.get_marker()
[43]	459	self.tokens.append(TokenClass(start_marker, end_marker))
[39]	460
[51]	461	def fetch_flow_entry(self):
[39]	462
[51]	463	# Simple keys are allowed after ','.
	464	self.allow_simple_key = True
	465
	466	# Reset possible simple key on the current level.
	467	self.remove_possible_simple_key()
	468
	469	# Add FLOW-ENTRY.
	470	start_marker = self.reader.get_marker()
	471	self.reader.forward()
	472	end_marker = self.reader.get_marker()
	473	self.tokens.append(FlowEntryToken(start_marker, end_marker))
	474
	475	def fetch_block_entry(self):
	476
[43]	477	# Block context needs additional checks.
	478	if not self.flow_level:
[39]	479
[43]	480	# Are we allowed to start a new entry?
	481	if not self.allow_simple_key:
[47]	482	raise ScannerError(None, None,
	483	"sequence entries are not allowed here",
	484	self.reader.get_marker())
[39]	485
[43]	486	# We may need to add BLOCK-SEQUENCE-START.
[46]	487	if self.add_indent(self.reader.column):
	488	marker = self.reader.get_marker()
[44]	489	self.tokens.append(BlockSequenceStartToken(marker, marker))
[39]	490
[51]	491	# It's an error for the block entry to occur in the flow context,
	492	# but we let the parser detect this.
	493	else:
	494	pass
	495
	496	# Simple keys are allowed after '-'.
[43]	497	self.allow_simple_key = True
[39]	498
[43]	499	# Reset possible simple key on the current level.
	500	self.remove_possible_simple_key()
[39]	501
[51]	502	# Add BLOCK-ENTRY.
[46]	503	start_marker = self.reader.get_marker()
	504	self.reader.forward()
	505	end_marker = self.reader.get_marker()
[51]	506	self.tokens.append(BlockEntryToken(start_marker, end_marker))
[39]	507
[43]	508	def fetch_key(self):
	509
	510	# Block context needs additional checks.
	511	if not self.flow_level:
[39]	512
[43]	513	# Are we allowed to start a key (not nessesary a simple)?
	514	if not self.allow_simple_key:
[47]	515	raise ScannerError(None, None,
	516	"mapping keys are not allowed here",
	517	self.reader.get_marker())
[43]	518
	519	# We may need to add BLOCK-MAPPING-START.
[46]	520	if self.add_indent(self.reader.column):
	521	marker = self.reader.get_marker()
[44]	522	self.tokens.append(BlockMappingStartToken(marker, marker))
[43]	523
	524	# Simple keys are allowed after '?' in the block context.
	525	self.allow_simple_key = not self.flow_level
	526
	527	# Reset possible simple key on the current level.
	528	self.remove_possible_simple_key()
	529
	530	# Add KEY.
[46]	531	start_marker = self.reader.get_marker()
	532	self.reader.forward()
	533	end_marker = self.reader.get_marker()
[44]	534	self.tokens.append(KeyToken(start_marker, end_marker))
[39]	535
[43]	536	def fetch_value(self):
[39]	537
[43]	538	# Do we determine a simple key?
	539	if self.flow_level in self.possible_simple_keys:
[39]	540
[43]	541	# Add KEY.
	542	key = self.possible_simple_keys[self.flow_level]
	543	del self.possible_simple_keys[self.flow_level]
	544	self.tokens.insert(key.token_number-self.tokens_taken,
[44]	545	KeyToken(key.marker, key.marker))
[39]	546
[43]	547	# If this key starts a new block mapping, we need to add
	548	# BLOCK-MAPPING-START.
	549	if not self.flow_level:
	550	if self.add_indent(key.column):
	551	self.tokens.insert(key.token_number-self.tokens_taken,
[44]	552	BlockMappingStartToken(key.marker, key.marker))
[37]	553
[43]	554	# There cannot be two simple keys one after another.
	555	self.allow_simple_key = False
[37]	556
[43]	557	# It must be a part of a complex key.
	558	else:
	559
[47]	560	# Block context needs additional checks.
	561	# (Do we really need them? They will be catched by the parser
	562	# anyway.)
	563	if not self.flow_level:
	564
	565	# We are allowed to start a complex value if and only if
	566	# we can start a simple key.
	567	if not self.allow_simple_key:
	568	raise ScannerError(None, None,
	569	"mapping values are not allowed here",
	570	self.reader.get_marker())
	571
[43]	572	# Simple keys are allowed after ':' in the block context.
	573	self.allow_simple_key = not self.flow_level
[37]	574
[43]	575	# Reset possible simple key on the current level.
	576	self.remove_possible_simple_key()
[37]	577
[43]	578	# Add VALUE.
[46]	579	start_marker = self.reader.get_marker()
	580	self.reader.forward()
	581	end_marker = self.reader.get_marker()
[44]	582	self.tokens.append(ValueToken(start_marker, end_marker))
[37]	583
[43]	584	def fetch_alias(self):
[37]	585
[43]	586	# ALIAS could be a simple key.
	587	self.save_possible_simple_key()
[37]	588
[43]	589	# No simple keys after ALIAS.
	590	self.allow_simple_key = False
[37]	591
[43]	592	# Scan and add ALIAS.
[47]	593	self.tokens.append(self.scan_anchor(AliasToken))
[37]	594
[43]	595	def fetch_anchor(self):
[37]	596
[43]	597	# ANCHOR could start a simple key.
	598	self.save_possible_simple_key()
[37]	599
[43]	600	# No simple keys after ANCHOR.
	601	self.allow_simple_key = False
[37]	602
[43]	603	# Scan and add ANCHOR.
[47]	604	self.tokens.append(self.scan_anchor(AnchorToken))
[37]	605
[43]	606	def fetch_tag(self):
[37]	607
[43]	608	# TAG could start a simple key.
	609	self.save_possible_simple_key()
[37]	610
[43]	611	# No simple keys after TAG.
	612	self.allow_simple_key = False
[37]	613
[43]	614	# Scan and add TAG.
[47]	615	self.tokens.append(self.scan_tag())
[37]	616
[43]	617	def fetch_literal(self):
	618	self.fetch_block_scalar(folded=False)
[37]	619
[43]	620	def fetch_folded(self):
	621	self.fetch_block_scalar(folded=True)
[37]	622
[43]	623	def fetch_block_scalar(self, folded):
[37]	624
[43]	625	# A simple key may follow a block scalar.
	626	self.allow_simple_key = True
[37]	627
[43]	628	# Reset possible simple key on the current level.
	629	self.remove_possible_simple_key()
[37]	630
[43]	631	# Scan and add SCALAR.
[47]	632	self.tokens.append(self.scan_block_scalar(folded))
[37]	633
[43]	634	def fetch_single(self):
	635	self.fetch_flow_scalar(double=False)
[37]	636
[43]	637	def fetch_double(self):
	638	self.fetch_flow_scalar(double=True)
[37]	639
[43]	640	def fetch_flow_scalar(self, double):
[37]	641
[43]	642	# A flow scalar could be a simple key.
	643	self.save_possible_simple_key()
[37]	644
[43]	645	# No simple keys after flow scalars.
	646	self.allow_simple_key = False
[37]	647
[43]	648	# Scan and add SCALAR.
[47]	649	self.tokens.append(self.scan_flow_scalar(double))
[37]	650
[43]	651	def fetch_plain(self):
[37]	652
[43]	653	# A plain scalar could be a simple key.
	654	self.save_possible_simple_key()
[37]	655
[43]	656	# No simple keys after plain scalars. But note that `scan_plain` will
	657	# change this flag if the scan is finished at the beginning of the
	658	# line.
	659	self.allow_simple_key = False
[37]	660
[43]	661	# Scan and add SCALAR. May change `allow_simple_key`.
[47]	662	self.tokens.append(self.scan_plain())
[37]	663
[43]	664	# Checkers.
[37]	665
[43]	666	def check_directive(self):
[37]	667
[43]	668	# DIRECTIVE: ^ '%' ...
	669	# The '%' indicator is already checked.
[46]	670	if self.reader.column == 0:
[43]	671	return True
[37]	672
[43]	673	def check_document_start(self):
[37]	674
[43]	675	# DOCUMENT-START: ^ '---' (' '\|'\n')
[46]	676	if self.reader.column == 0:
[48]	677	if self.reader.prefix(3) == u'---' \
	678	and self.reader.peek(3) in u'\0 \t\r\n\x85\u2028\u2029':
[43]	679	return True
[37]	680
[43]	681	def check_document_end(self):
[37]	682
[43]	683	# DOCUMENT-END: ^ '...' (' '\|'\n')
[46]	684	if self.reader.column == 0:
	685	prefix = self.reader.peek(4)
[48]	686	if self.reader.prefix(3) == u'...' \
	687	and self.reader.peek(3) in u'\0 \t\r\n\x85\u2028\u2029':
[43]	688	return True
[37]	689
[51]	690	def check_block_entry(self):
[43]	691
[51]	692	# BLOCK-ENTRY: '-' (' '\|'\n')
	693	return self.reader.peek(1) in u'\0 \t\r\n\x85\u2028\u2029'
[43]	694
	695	def check_key(self):
	696
	697	# KEY(flow context): '?'
	698	if self.flow_level:
[37]	699	return True
[43]	700
	701	# KEY(block context): '?' (' '\|'\n')
[37]	702	else:
[48]	703	return self.reader.peek(1) in u'\0 \t\r\n\x85\u2028\u2029'
[37]	704
[43]	705	def check_value(self):
	706
	707	# VALUE(flow context): ':'
	708	if self.flow_level:
[37]	709	return True
[43]	710
	711	# VALUE(block context): ':' (' '\|'\n')
[37]	712	else:
[48]	713	return self.reader.peek(1) in u'\0 \t\r\n\x85\u2028\u2029'
[37]	714
[43]	715	def check_plain(self):
[37]	716
[48]	717	# A plain scalar may start with any non-space character except:
	718	# '-', '?', ':', ',', '[', ']', '{', '}',
	719	# '#', '&', '*', '!', '\|', '>', '\'', '\"',
	720	# '%', '@', '`'.
	721	#
	722	# It may also start with
	723	# '-', '?', ':'
	724	# if it is followed by a non-space character.
	725	#
	726	# Note that we limit the last rule to the block context (except the
	727	# '-' character) because we want the flow context to be space
	728	# independent.
	729	ch = self.reader.peek()
	730	return ch not in u'\0 \t\r\n\x85\u2028\u2029-?:,[]{}#&*!\|>\'\"%@`' \
	731	or (self.reader.peek(1) not in u'\0 \t\r\n\x85\u2028\u2029'
	732	and (ch == '-' or (not self.flow_level and ch in u'?:')))
	733
[43]	734	# Scanners.
	735
	736	def scan_to_next_token(self):
[47]	737	# We ignore spaces, line breaks and comments.
	738	# If we find a line break in the block context, we set the flag
	739	# `allow_simple_key` on.
[51]	740	# The byte order mark is stripped if it's the first character in the
	741	# stream. We do not yet support BOM inside the stream as the
	742	# specification requires. Any such mark will be considered as a part
	743	# of the document.
[52]	744	#
	745	# TODO: We need to make tab handling rules more sane. A good rule is
	746	# Tabs cannot precede tokens
	747	# BLOCK-SEQUENCE-START, BLOCK-MAPPING-START, BLOCK-END,
	748	# KEY(block), VALUE(block), BLOCK-ENTRY
	749	# So the checking code is
	750	# if <TAB>:
	751	# self.allow_simple_keys = False
	752	# We also need to add the check for `allow_simple_keys == True` to
	753	# `unwind_indent` before issuing BLOCK-END.
	754	# Scanners for block, flow, and plain scalars need to be modified.
	755
[51]	756	if self.reader.index == 0 and self.reader.peek() == u'\uFEFF':
	757	self.reader.forward()
[43]	758	found = False
	759	while not found:
[46]	760	while self.reader.peek() == u' ':
	761	self.reader.forward()
	762	if self.reader.peek() == u'#':
[47]	763	while self.reader.peek() not in u'\0\r\n\x85\u2028\u2029':
[46]	764	self.reader.forward()
[47]	765	if self.scan_line_break():
[43]	766	if not self.flow_level:
	767	self.allow_simple_key = True
[37]	768	else:
[43]	769	found = True
[37]	770
[43]	771	def scan_directive(self):
[48]	772	# See the specification for details.
	773	start_marker = self.reader.get_marker()
	774	self.reader.forward()
	775	name = self.scan_directive_name(start_marker)
	776	value = None
	777	if name == u'YAML':
	778	value = self.scan_yaml_directive_value(start_marker)
	779	end_marker = self.reader.get_marker()
	780	elif name == u'TAG':
	781	value = self.scan_tag_directive_value(start_marker)
	782	end_marker = self.reader.get_marker()
[43]	783	else:
[48]	784	end_marker = self.reader.get_marker()
	785	while self.reader.peek() not in u'\0\r\n\x85\u2028\u2029':
	786	self.reader.forward()
	787	self.scan_directive_ignored_line(start_marker)
	788	return DirectiveToken(name, value, start_marker, end_marker)
	789
	790	def scan_directive_name(self, start_marker):
	791	# See the specification for details.
	792	length = 0
	793	ch = self.reader.peek(length)
	794	while u'0' <= ch <= u'9' or u'A' <= ch <= 'Z' or u'a' <= ch <= 'z' \
	795	or ch in u'-_':
	796	length += 1
	797	ch = self.reader.peek(length)
	798	if not length:
	799	raise ScannerError("while scanning a directive", start_marker,
[52]	800	"expected alphabetic or numeric character, but found %r"
	801	% ch.encode('utf-8'), self.reader.get_marker())
[48]	802	value = self.reader.prefix(length)
	803	self.reader.forward(length)
	804	ch = self.reader.peek()
	805	if ch not in u'\0 \r\n\x85\u2028\u2029':
[52]	806	raise ScannerError("while scanning a directive", start_marker,
[48]	807	"expected alphabetic or numeric character, but found %r"
	808	% ch.encode('utf-8'), self.reader.get_marker())
	809	return value
	810
	811	def scan_yaml_directive_value(self, start_marker):
	812	# See the specification for details.
	813	while self.reader.peek() == u' ':
[46]	814	self.reader.forward()
[48]	815	major = self.scan_yaml_directive_number(start_marker)
	816	if self.reader.peek() != '.':
	817	raise ScannerError("while scanning a directive", start_marker,
[52]	818	"expected a digit or '.', but found %r"
	819	% self.reader.peek().encode('utf-8'),
[48]	820	self.reader.get_marker())
[46]	821	self.reader.forward()
[48]	822	minor = self.scan_yaml_directive_number(start_marker)
	823	if self.reader.peek() not in u'\0 \r\n\x85\u2028\u2029':
	824	raise ScannerError("while scanning a directive", start_marker,
[52]	825	"expected a digit or ' ', but found %r"
	826	% self.reader.peek().encode('utf-8'),
[48]	827	self.reader.get_marker())
	828	return (major, minor)
[37]	829
[48]	830	def scan_yaml_directive_number(self, start_marker):
	831	# See the specification for details.
	832	ch = self.reader.peek()
	833	if not (u'0' <= ch <= '9'):
	834	raise ScannerError("while scanning a directive", start_marker,
	835	"expected a digit, but found %r" % ch.encode('utf-8'),
	836	self.reader.get_marker())
	837	length = 0
	838	while u'0' <= self.reader.peek(length) <= u'9':
	839	length += 1
	840	value = int(self.reader.prefix(length))
	841	self.reader.forward(length)
	842	return value
	843
	844	def scan_tag_directive_value(self, start_marker):
	845	# See the specification for details.
	846	while self.reader.peek() == u' ':
	847	self.reader.forward()
	848	handle = self.scan_tag_directive_handle(start_marker)
	849	while self.reader.peek() == u' ':
	850	self.reader.forward()
	851	prefix = self.scan_tag_directive_prefix(start_marker)
	852	return (handle, prefix)
	853
	854	def scan_tag_directive_handle(self, start_marker):
	855	# See the specification for details.
	856	value = self.scan_tag_handle('directive', start_marker)
[52]	857	ch = self.reader.peek()
	858	if ch != u' ':
[48]	859	raise ScannerError("while scanning a directive", start_marker,
	860	"expected ' ', but found %r" % ch.encode('utf-8'),
	861	self.reader.get_marker())
	862	return value
	863
	864	def scan_tag_directive_prefix(self, start_marker):
	865	# See the specification for details.
	866	value = self.scan_tag_uri('directive', start_marker)
	867	ch = self.reader.peek()
	868	if ch not in u'\0 \r\n\x85\u2028\u2029':
	869	raise ScannerError("while scanning a directive", start_marker,
	870	"expected ' ', but found %r" % ch.encode('utf-8'),
	871	self.reader.get_marker())
	872	return value
	873
	874	def scan_directive_ignored_line(self, start_marker):
	875	# See the specification for details.
	876	while self.reader.peek() == u' ':
	877	self.reader.forward()
	878	if self.reader.peek() == u'#':
	879	while self.reader.peek() not in u'\0\r\n\x85\u2028\u2029':
	880	self.reader.forward()
	881	ch = self.reader.peek()
	882	if ch not in u'\0\r\n\x85\u2028\u2029':
	883	raise ScannerError("while scanning a directive", start_marker,
	884	"expected a comment or a line break, but found %r"
	885	% ch.encode('utf-8'), self.reader.get_marker())
	886	self.scan_line_break()
	887
[43]	888	def scan_anchor(self, TokenClass):
[48]	889	# The specification does not restrict characters for anchors and
	890	# aliases. This may lead to problems, for instance, the document:
	891	# [ *alias, value ]
	892	# can be interpteted in two ways, as
	893	# [ "value" ]
	894	# and
	895	# [ *alias , "value" ]
	896	# Therefore we restrict aliases to numbers and ASCII letters.
[46]	897	start_marker = self.reader.get_marker()
[48]	898	indicator = self.reader.peek()
	899	if indicator == '*':
	900	name = 'alias'
	901	else:
	902	name = 'anchor'
	903	self.reader.forward()
	904	length = 0
	905	ch = self.reader.peek(length)
	906	while u'0' <= ch <= u'9' or u'A' <= ch <= 'Z' or u'a' <= ch <= 'z' \
	907	or ch in u'-_':
	908	length += 1
	909	ch = self.reader.peek(length)
	910	if not length:
	911	raise ScannerError("while scanning an %s" % name, start_marker,
[52]	912	"expected alphabetic or numeric character, but found %r"
	913	% ch.encode('utf-8'), self.reader.get_marker())
[48]	914	value = self.reader.prefix(length)
	915	self.reader.forward(length)
	916	ch = self.reader.peek()
	917	if ch not in u'\0 \t\r\n\x85\u2028\u2029?:,]}%@`':
	918	raise ScannerError("while scanning an %s" % name, start_marker,
	919	"expected alphabetic or numeric character, but found %r"
	920	% ch.encode('utf-8'), self.reader.get_marker())
[46]	921	end_marker = self.reader.get_marker()
[48]	922	return TokenClass(value, start_marker, end_marker)
[37]	923
[43]	924	def scan_tag(self):
[48]	925	# See the specification for details.
[46]	926	start_marker = self.reader.get_marker()
[48]	927	ch = self.reader.peek(1)
	928	if ch == u'<':
	929	handle = None
	930	self.reader.forward(2)
	931	suffix = self.scan_tag_uri('tag', start_marker)
	932	if self.reader.peek() != u'>':
[52]	933	raise ScannerError("while parsing a tag", start_marker,
	934	"expected '>', but found %r" % self.reader.peek().encode('utf-8'),
[48]	935	self.reader.get_marker())
[46]	936	self.reader.forward()
[48]	937	elif ch in u'\0 \t\r\n\x85\u2028\u2029':
	938	handle = None
	939	suffix = u'!'
	940	self.reader.forward()
	941	else:
	942	length = 1
	943	use_handle = False
	944	while ch not in u'\0 \r\n\x85\u2028\u2029':
	945	if ch == u'!':
	946	use_handle = True
	947	break
	948	length += 1
	949	ch = self.reader.peek(length)
	950	handle = u'!'
	951	if use_handle:
	952	handle = self.scan_tag_handle('tag', start_marker)
	953	else:
	954	handle = u'!'
	955	self.reader.forward()
	956	suffix = self.scan_tag_uri('tag', start_marker)
	957	ch = self.reader.peek()
	958	if ch not in u'\0 \r\n\x85\u2028\u2029':
	959	raise ScannerError("while scanning a tag", start_marker,
	960	"expected ' ', but found %r" % ch.encode('utf-8'),
	961	self.reader.get_marker())
	962	value = (handle, suffix)
[46]	963	end_marker = self.reader.get_marker()
[48]	964	return TagToken(value, start_marker, end_marker)
[43]	965
	966	def scan_block_scalar(self, folded):
[48]	967	# See the specification for details.
	968
	969	chunks = []
[46]	970	start_marker = self.reader.get_marker()
[48]	971
	972	# Scan the header.
	973	self.reader.forward()
	974	chomping, increment = self.scan_block_scalar_indicators(start_marker)
	975	self.scan_block_scalar_ignored_line(start_marker)
	976
	977	# Determine the indentation level and go to the first non-empty line.
	978	min_indent = self.indent+1
	979	if min_indent < 1:
	980	min_indent = 1
	981	if increment is None:
	982	breaks, max_indent, end_marker = self.scan_block_scalar_indentation()
	983	indent = max(min_indent, max_indent)
	984	else:
	985	indent = min_indent+increment-1
	986	breaks, end_marker = self.scan_block_scalar_breaks(indent)
	987	line_break = u''
	988
	989	# Scan the inner part of the block scalar.
	990	while self.reader.column == indent and self.reader.peek() != u'\0':
	991	chunks.extend(breaks)
	992	leading_non_space = self.reader.peek() not in u' \t'
	993	length = 0
	994	while self.reader.peek(length) not in u'\0\r\n\x85\u2028\u2029':
	995	length += 1
	996	chunks.append(self.reader.prefix(length))
	997	self.reader.forward(length)
	998	line_break = self.scan_line_break()
	999	breaks, end_marker = self.scan_block_scalar_breaks(indent)
	1000	if self.reader.column == indent and self.reader.peek() != u'\0':
	1001	# Unfortunately, folding rules are ambiguous.
	1002	#
	1003	# This is the folding according to the specification:
[51]	1004
	1005	if folded and line_break == u'\n' \
	1006	and leading_non_space and self.reader.peek() not in u' \t':
	1007	if not breaks:
	1008	chunks.append(u' ')
	1009	else:
	1010	chunks.append(line_break)
	1011
	1012	# This is Clark Evans's interpretation (also in the spec
	1013	# examples):
[48]	1014	#
[51]	1015	#if folded and line_break == u'\n':
[48]	1016	# if not breaks:
[51]	1017	# if self.reader.peek() not in ' \t':
	1018	# chunks.append(u' ')
	1019	# else:
	1020	# chunks.append(line_break)
[48]	1021	#else:
	1022	# chunks.append(line_break)
	1023	else:
	1024	break
	1025
	1026	# Chomp the tail.
	1027	if chomping is not False:
	1028	chunks.append(line_break)
	1029	if chomping is True:
	1030	chunks.extend(breaks)
	1031
	1032	# We are done.
	1033	return ScalarToken(u''.join(chunks), False, start_marker, end_marker)
	1034
	1035	def scan_block_scalar_indicators(self, start_marker):
	1036	# See the specification for details.
	1037	chomping = None
	1038	increment = None
	1039	ch = self.reader.peek()
	1040	if ch in u'+-':
	1041	if ch == '+':
	1042	chomping = True
	1043	else:
	1044	chomping = False
	1045	self.reader.forward()
	1046	ch = self.reader.peek()
	1047	if ch in u'0123456789':
	1048	increment = int(ch)
	1049	if increment == 0:
	1050	raise ScannerError("while scanning a block scalar", start_marker,
	1051	"expected indentation indicator in the range 1-9, but found 0",
	1052	self.reader.get_marker())
[46]	1053	self.reader.forward()
[48]	1054	elif ch in u'0123456789':
	1055	increment = int(ch)
	1056	if increment == 0:
	1057	raise ScannerError("while scanning a block scalar", start_marker,
	1058	"expected indentation indicator in the range 1-9, but found 0",
	1059	self.reader.get_marker())
	1060	self.reader.forward()
	1061	ch = self.reader.peek()
	1062	if ch in u'+-':
	1063	if ch == '+':
	1064	chomping = True
	1065	else:
	1066	chomping = False
[46]	1067	self.reader.forward()
[48]	1068	ch = self.reader.peek()
	1069	if ch not in u'\0 \r\n\x85\u2028\u2029':
	1070	raise ScannerError("while scanning a block scalar", start_marker,
	1071	"expected chomping or indentation indicators, but found %r"
	1072	% ch.encode('utf-8'), self.reader.get_marker())
	1073	return chomping, increment
	1074
	1075	def scan_block_scalar_ignored_line(self, start_marker):
	1076	# See the specification for details.
	1077	while self.reader.peek() == u' ':
	1078	self.reader.forward()
	1079	if self.reader.peek() == u'#':
	1080	while self.reader.peek() not in u'\0\r\n\x85\u2028\u2029':
[46]	1081	self.reader.forward()
[48]	1082	ch = self.reader.peek()
	1083	if ch not in u'\0\r\n\x85\u2028\u2029':
	1084	raise ScannerError("while scanning a block scalar", start_marker,
	1085	"expected a comment or a line break, but found %r"
	1086	% ch.encode('utf-8'), self.reader.get_marker())
	1087	self.scan_line_break()
[43]	1088
[48]	1089	def scan_block_scalar_indentation(self):
	1090	# See the specification for details.
	1091	chunks = []
	1092	max_indent = 0
	1093	end_marker = self.reader.get_marker()
	1094	while self.reader.peek() in u' \r\n\x85\u2028\u2029':
	1095	if self.reader.peek() != u' ':
	1096	chunks.append(self.scan_line_break())
	1097	end_marker = self.reader.get_marker()
	1098	else:
	1099	self.reader.forward()
	1100	if self.reader.column > max_indent:
	1101	max_indent = self.reader.column
	1102	return chunks, max_indent, end_marker
	1103
	1104	def scan_block_scalar_breaks(self, indent):
	1105	# See the specification for details.
	1106	chunks = []
	1107	end_marker = self.reader.get_marker()
	1108	while self.reader.column < indent and self.reader.peek() == u' ':
	1109	self.reader.forward()
	1110	while self.reader.peek() in u'\r\n\x85\u2028\u2029':
	1111	chunks.append(self.scan_line_break())
	1112	end_marker = self.reader.get_marker()
	1113	while self.reader.column < indent and self.reader.peek() == u' ':
	1114	self.reader.forward()
	1115	return chunks, end_marker
	1116
[43]	1117	def scan_flow_scalar(self, double):
[48]	1118	# See the specification for details.
	1119	chunks = []
	1120	start_marker = self.reader.get_marker()
	1121	indent = self.indent+1
	1122	if indent == 0:
	1123	indent = 1
[46]	1124	quote = self.reader.peek()
	1125	self.reader.forward()
[48]	1126	chunks.extend(self.scan_flow_scalar_non_spaces(double, indent, start_marker))
[46]	1127	while self.reader.peek() != quote:
[48]	1128	chunks.extend(self.scan_flow_scalar_spaces(double, indent, start_marker))
	1129	chunks.extend(self.scan_flow_scalar_non_spaces(double, indent, start_marker))
	1130	self.reader.forward()
	1131	end_marker = self.reader.get_marker()
	1132	return ScalarToken(u''.join(chunks), False, start_marker, end_marker)
	1133
	1134	ESCAPE_REPLACEMENTS = {
	1135	u'0': u'\0',
	1136	u'a': u'\x07',
	1137	u'b': u'\x08',
	1138	u't': u'\x09',
	1139	u'\t': u'\x09',
	1140	u'n': u'\x0A',
	1141	u'v': u'\x0B',
	1142	u'f': u'\x0C',
	1143	u'r': u'\x0D',
	1144	u'e': u'\x1B',
	1145	u' ': u'\x20',
	1146	u'\"': u'\"',
	1147	u'\\': u'\\',
	1148	u'N': u'\x85',
	1149	u'_': u'\xA0',
	1150	u'L': u'\u2028',
	1151	u'P': u'\u2029',
	1152	}
	1153
	1154	ESCAPE_CODES = {
	1155	u'x': 2,
	1156	u'u': 4,
	1157	u'U': 8,
	1158	}
	1159
	1160	def scan_flow_scalar_non_spaces(self, double, indent, start_marker):
	1161	# See the specification for details.
	1162	chunks = []
	1163	while True:
	1164	length = 0
	1165	while self.reader.peek(length) not in u'\'\"\\\0 \t\r\n\x85\u2028\u2029':
	1166	length += 1
	1167	if length:
	1168	chunks.append(self.reader.prefix(length))
	1169	self.reader.forward(length)
	1170	ch = self.reader.peek()
	1171	if not double and ch == u'\'' and self.reader.peek(1) == u'\'':
	1172	chunks.append(u'\'')
[46]	1173	self.reader.forward(2)
[48]	1174	elif (double and ch == u'\'') or (not double and ch in u'\"\\'):
	1175	chunks.append(ch)
	1176	self.reader.forward()
	1177	elif double and ch == u'\\':
	1178	self.reader.forward()
	1179	ch = self.reader.peek()
	1180	if ch in self.ESCAPE_REPLACEMENTS:
	1181	chunks.append(self.ESCAPE_REPLACEMENTS[ch])
	1182	self.reader.forward()
	1183	elif ch in self.ESCAPE_CODES:
	1184	length = self.ESCAPE_CODES[ch]
	1185	self.reader.forward()
	1186	for k in range(length):
	1187	if self.reader.peek(k) not in u'0123456789ABCDEFabcdef':
	1188	raise ScannerError("while scanning a double-quoted scalar", start_marker,
	1189	"expected escape sequence of %d hexdecimal numbers, but found %r" %
	1190	(length, self.reader.peek(k).encode('utf-8')), self.reader.get_marker())
	1191	code = int(self.reader.prefix(length), 16)
	1192	chunks.append(unichr(code))
	1193	self.reader.forward(length)
	1194	elif ch in u'\r\n\x85\u2028\u2029':
	1195	self.scan_line_break()
	1196	chunks.extend(self.scan_flow_scalar_breaks(double, indent, start_marker))
	1197	else:
	1198	raise ScannerError("while scanning a double-quoted scalar", start_marker,
	1199	"found unknown escape character %r" % ch.encode('utf-8'), self.reader.get_marker())
[37]	1200	else:
[48]	1201	return chunks
[37]	1202
[48]	1203	def scan_flow_scalar_spaces(self, double, indent, start_marker):
	1204	# See the specification for details.
	1205	chunks = []
	1206	length = 0
	1207	while self.reader.peek(length) in u' \t':
	1208	length += 1
	1209	whitespaces = self.reader.prefix(length)
	1210	self.reader.forward(length)
	1211	ch = self.reader.peek()
	1212	if ch == u'\0':
	1213	raise ScannerError("while scanning a quoted scalar", start_marker,
	1214	"found unexpected end of stream", self.reader.get_marker())
	1215	elif ch in u'\r\n\x85\u2028\u2029':
	1216	line_break = self.scan_line_break()
	1217	breaks = self.scan_flow_scalar_breaks(double, indent, start_marker)
	1218	if line_break != u'\n':
	1219	chunks.append(line_break)
	1220	elif not breaks:
	1221	chunks.append(u' ')
	1222	chunks.extend(breaks)
	1223	else:
	1224	chunks.append(whitespaces)
	1225	return chunks
	1226
	1227	def scan_flow_scalar_breaks(self, double, indent, start_marker):
	1228	# See the specification for details.
	1229	chunks = []
	1230	while True:
	1231	while self.reader.column < indent and self.reader.peek() == u' ':
	1232	self.reader.forward()
	1233	if self.reader.column < indent \
	1234	and self.reader.peek() not in u'\0\r\n\x85\u2028\u2029':
	1235	s = 's'
	1236	if indent == 1:
	1237	s = ''
	1238	raise ScannerError("while scanning a quoted scalar", start_marker,
	1239	"expected %d space%s indentation, but found %r"
	1240	% (indent, s, self.reader.peek().encode('utf-8')),
	1241	self.reader.get_marker())
	1242	while self.reader.peek() in u' \t':
	1243	self.reader.forward()
	1244	if self.reader.peek() in u'\r\n\x85\u2028\u2029':
	1245	chunks.append(self.scan_line_break())
	1246	else:
	1247	return chunks
	1248
[43]	1249	def scan_plain(self):
[48]	1250	# See the specification for details.
	1251	# We add an additional restriction for the flow context:
	1252	# plain scalars in the flow context cannot contain ':' and '?'.
	1253	# We also keep track of the `allow_simple_key` flag here.
	1254	chunks = []
	1255	start_marker = self.reader.get_marker()
	1256	end_marker = start_marker
[43]	1257	indent = self.indent+1
[48]	1258	if indent == 0:
[43]	1259	indent = 1
[48]	1260	spaces = []
[43]	1261	while True:
[48]	1262	length = 0
	1263	if self.reader.peek() == u'#':
[43]	1264	break
[48]	1265	while True:
	1266	ch = self.reader.peek(length)
	1267	if ch in u'\0 \t\r\n\x85\u2028\u2029' \
	1268	or (not self.flow_level and ch == u':' and
	1269	self.reader.peek(length+1) in u'\0 \t\r\n\x28\u2028\u2029') \
	1270	or (self.flow_level and ch in u',:?[]{}'):
	1271	break
	1272	length += 1
	1273	if length == 0:
[43]	1274	break
[48]	1275	self.allow_simple_key = False
	1276	chunks.extend(spaces)
	1277	chunks.append(self.reader.prefix(length))
	1278	self.reader.forward(length)
	1279	end_marker = self.reader.get_marker()
	1280	spaces = self.scan_plain_spaces(indent)
	1281	if not spaces or self.reader.peek() == u'#' \
	1282	or self.reader.column < indent:
	1283	break
	1284	return ScalarToken(u''.join(chunks), True, start_marker, end_marker)
[37]	1285
[48]	1286	def scan_plain_spaces(self, indent):
	1287	# See the specification for details.
	1288	# The specification is really confusing about tabs in plain scalars.
	1289	# We just forbid them completely. Do not use tabs in YAML!
	1290	chunks = []
	1291	length = 0
	1292	while self.reader.peek(length) in u' ':
	1293	length += 1
	1294	whitespaces = self.reader.prefix(length)
	1295	self.reader.forward(length)
	1296	ch = self.reader.peek()
	1297	if ch in u'\r\n\x85\u2028\u2029':
	1298	line_break = self.scan_line_break()
	1299	self.allow_simple_key = True
	1300	breaks = []
	1301	while self.reader.peek() in u' \r\n\x85\u2028\u2029':
	1302	if self.reader.peek() == ' ':
	1303	self.reader.forward()
	1304	else:
	1305	breaks.append(self.scan_line_break())
	1306	if line_break != u'\n':
	1307	chunks.append(line_break)
	1308	elif not breaks:
	1309	chunks.append(u' ')
	1310	chunks.extend(breaks)
	1311	elif whitespaces:
	1312	chunks.append(whitespaces)
	1313	return chunks
	1314
	1315	def scan_tag_handle(self, name, start_marker):
	1316	# See the specification for details.
	1317	# For some strange reasons, the specification does not allow '_' in
	1318	# tag handles. I have allowed it anyway.
[52]	1319	ch = self.reader.peek()
	1320	if ch != u'!':
[48]	1321	raise ScannerError("while scanning a %s" % name, start_marker,
	1322	"expected '!', but found %r" % ch.encode('utf-8'),
	1323	self.reader.get_marker())
	1324	length = 1
	1325	ch = self.reader.peek(length)
	1326	if ch != u' ':
	1327	while u'0' <= ch <= u'9' or u'A' <= ch <= 'Z' or u'a' <= ch <= 'z' \
	1328	or ch in u'-_':
	1329	length += 1
	1330	ch = self.reader.peek(length)
	1331	if ch != u'!':
	1332	self.reader.forward(length)
	1333	raise ScannerError("while scanning a %s" % name, start_marker,
	1334	"expected '!', but found %r" % ch.encode('utf-8'),
	1335	self.reader.get_marker())
	1336	length += 1
	1337	value = self.reader.prefix(length)
	1338	self.reader.forward(length)
	1339	return value
	1340
	1341	def scan_tag_uri(self, name, start_marker):
	1342	# See the specification for details.
	1343	# Note: we do not check if URI is well-formed.
	1344	chunks = []
	1345	length = 0
	1346	ch = self.reader.peek(length)
	1347	while u'0' <= ch <= u'9' or u'A' <= ch <= 'Z' or u'a' <= ch <= 'z' \
	1348	or ch in u'-;/?:@&=+$,_.!~*\'()[]%':
	1349	if ch == u'%':
	1350	chunks.append(self.reader.prefix(length))
	1351	self.reader.forward(length)
	1352	length = 0
	1353	chunks.append(self.scan_uri_escapes(name, start_marker))
	1354	else:
	1355	length += 1
	1356	ch = self.reader.peek(length)
	1357	if length:
	1358	chunks.append(self.reader.prefix(length))
	1359	self.reader.forward(length)
	1360	length = 0
	1361	if not chunks:
	1362	raise ScannerError("while parsing a %s" % name, start_marker,
	1363	"expected URI, but found %r" % ch.encode('utf-8'),
	1364	self.reader.get_marker())
	1365	return u''.join(chunks)
	1366
	1367	def scan_uri_escapes(self, name, start_marker):
	1368	# See the specification for details.
	1369	bytes = []
	1370	marker = self.reader.get_marker()
	1371	while self.reader.peek() == u'%':
	1372	self.reader.forward()
	1373	for k in range(2):
	1374	if self.reader.peek(k) not in u'0123456789ABCDEFabcdef':
	1375	raise ScannerError("while scanning a %s" % name, start_marker,
	1376	"expected URI escape sequence of 2 hexdecimal numbers, but found %r" %
	1377	(self.reader.peek(k).encode('utf-8')), self.reader.get_marker())
	1378	bytes.append(chr(int(self.reader.prefix(2), 16)))
	1379	self.reader.forward(2)
	1380	try:
	1381	value = unicode(''.join(bytes), 'utf-8')
	1382	except UnicodeDecodeError, exc:
	1383	raise ScannerError("while scanning a %s" % name, start_marker, str(exc), marker)
	1384	return value
	1385
[47]	1386	def scan_line_break(self):
	1387	# Transforms:
	1388	# '\r\n' : '\n'
	1389	# '\r' : '\n'
	1390	# '\n' : '\n'
	1391	# '\x85' : '\n'
	1392	# '\u2028' : '\u2028'
	1393	# '\u2029 : '\u2029'
	1394	# default : ''
	1395	ch = self.reader.peek()
	1396	if ch in u'\r\n\x85':
[48]	1397	if self.reader.prefix(2) == u'\r\n':
[47]	1398	self.forward(2)
	1399	else:
	1400	self.reader.forward()
	1401	return u'\n'
	1402	elif ch in u'\u2028\u2029':
	1403	self.reader.forward()
	1404	return ch
	1405	return u''
	1406
[45]	1407	#try:
	1408	# import psyco
	1409	# psyco.bind(Scanner)
	1410	#except ImportError:
	1411	# pass
	1412

Note: See TracBrowser for help on using the repository browser.

Download in other formats: