Context Navigation

source: branches/pyyaml3000/lib/yaml/scanner.py @ 52

Revision 52, 51.5 KB checked in by xi, 7 years ago (diff)
Cleanup error messages.

Rev	Line
[39]	1
[43]	2	# Tokens:
	3	# YAML-DIRECTIVE(major_version, minor_version), TAG-DIRECTIVE(handle, prefix)
	4	# RESERVED-DIRECTIVE(name)
	5	# DOCUMENT-START, DOCUMENT-END
	6	# BLOCK-SEQUENCE-START, BLOCK-MAPPING-START, BLOCK-END
	7	# FLOW-SEQUENCE-START, FLOW-MAPPING-START, FLOW-SEQUENCE-END, FLOW-MAPPING-END
	8	# ENTRY, KEY, VALUE
	9	# ALIAS(name), ANCHOR(name), TAG(value), SCALAR(value, plain)
	10
[46]	11	__all__ = ['Scanner', 'ScannerError']
[43]	12
[52]	13	from error import MarkedYAMLError
[46]	14	from tokens import *
[39]	15
[52]	16	class ScannerError(MarkedYAMLError):
	17	pass
[51]	18
[43]	19	class SimpleKey:
[51]	20	# See below simple keys treatment.
	21
[43]	22	def __init__(self, token_number, required, index, line, column, marker):
	23	self.token_number = token_number
	24	self.required = required
	25	self.index = index
	26	self.line = line
	27	self.column = column
	28	self.marker = marker
	29
[39]	30	class Scanner:
	31
[46]	32
	33	def __init__(self, reader):
[39]	34	"""Initialize the scanner."""
[46]	35	# The input stream. The Reader class do the dirty work of checking for
[43]	36	# BOM and converting the input data to Unicode. It also adds NUL to
	37	# the end.
[39]	38	#
[46]	39	# Reader supports the following methods
[48]	40	# self.reader.peek(i=0) # peek the next i-th character
	41	# self.reader.prefix(l=1) # peek the next l characters
	42	# self.reader.forward(l=1) # read the next l characters
	43	# and move the pointer
[46]	44	self.reader = reader
[39]	45
	46	# Had we reached the end of the stream?
	47	self.done = False
	48
	49	# The number of unclosed '{' and '['. `flow_level == 0` means block
	50	# context.
	51	self.flow_level = 0
	52
	53	# List of processed tokens that are not yet emitted.
	54	self.tokens = []
	55
	56	# Number of tokens that were emitted through the `get_token` method.
	57	self.tokens_taken = 0
	58
	59	# The current indentation level.
	60	self.indent = -1
	61
	62	# Past indentation levels.
	63	self.indents = []
	64
[43]	65	# Variables related to simple keys treatment.
[39]	66
	67	# A simple key is a key that is not denoted by the '?' indicator.
	68	# Example of simple keys:
	69	# ---
	70	# block simple key: value
	71	# ? not a simple key:
	72	# : { flow simple key: value }
	73	# We emit the KEY token before all keys, so when we find a potential
	74	# simple key, we try to locate the corresponding ':' indicator.
	75	# Simple keys should be limited to a single line and 1024 characters.
	76
[43]	77	# Can a simple key start at the current position? A simple key may
	78	# start:
	79	# - at the beginning of the line, not counting indentation spaces
	80	# (in block context),
	81	# - after '{', '[', ',' (in the flow context),
	82	# - after '?', ':', '-' (in the block context).
	83	# In the block context, this flag also signify if a block collection
	84	# may start at the current position.
	85	self.allow_simple_key = True
[39]	86
	87	# Keep track of possible simple keys. This is a dictionary. The key
	88	# is `flow_level`; there can be no more that one possible simple key
[43]	89	# for each level. The value is a SimpleKey record:
	90	# (token_number, required, index, line, column, marker)
	91	# A simple key may start with ALIAS, ANCHOR, TAG, SCALAR(flow),
	92	# '[', or '{' tokens.
[39]	93	self.possible_simple_keys = {}
	94
[51]	95	# Public methods.
[39]	96
[51]	97	def check(self, *choices):
	98	# Check if the next token is one of the given types.
[43]	99	while self.need_more_tokens():
[39]	100	self.fetch_more_tokens()
	101	if self.tokens:
[51]	102	for choice in choices:
	103	if isinstance(self.tokens[0], choice):
	104	return True
	105	return False
	106
	107	def peek(self):
	108	# Return the next token, but do not delete if from the queue.
	109	while self.need_more_tokens():
	110	self.fetch_more_tokens()
	111	if self.tokens:
[39]	112	return self.tokens[0]
	113
[51]	114	def get(self):
	115	# Return the next token.
[39]	116	while self.need_more_tokens():
	117	self.fetch_more_tokens()
	118	if self.tokens:
	119	self.tokens_taken += 1
	120	return self.tokens.pop(0)
	121
[51]	122	def __iter__(self):
	123	# Iterator protocol.
	124	while self.need_more_tokens():
	125	self.fetch_more_tokens()
	126	while self.tokens:
	127	self.tokens_taken += 1
	128	yield self.tokens.pop(0)
	129	while self.need_more_tokens():
	130	self.fetch_more_tokens()
	131
[43]	132	# Private methods.
[39]	133
	134	def need_more_tokens(self):
	135	if self.done:
	136	return False
	137	if not self.tokens:
	138	return True
	139	# The current token may be a potential simple key, so we
	140	# need to look further.
[43]	141	self.stale_possible_simple_keys()
[39]	142	if self.next_possible_simple_key() == self.tokens_taken:
	143	return True
	144
	145	def fetch_more_tokens(self):
	146
	147	# Eat whitespaces and comments until we reach the next token.
[43]	148	self.scan_to_next_token()
[39]	149
[43]	150	# Remove obsolete possible simple keys.
	151	self.stale_possible_simple_keys()
	152
[39]	153	# Compare the current indentation and column. It may add some tokens
[43]	154	# and decrease the current indentation level.
[46]	155	self.unwind_indent(self.reader.column)
[39]	156
	157	# Peek the next character.
[46]	158	ch = self.reader.peek()
[39]	159
[48]	160	# Is it the end of stream?
[43]	161	if ch == u'\0':
[48]	162	return self.fetch_stream_end()
[39]	163
	164	# Is it a directive?
	165	if ch == u'%' and self.check_directive():
	166	return self.fetch_directive()
	167
	168	# Is it the document start?
	169	if ch == u'-' and self.check_document_start():
	170	return self.fetch_document_start()
	171
	172	# Is it the document end?
	173	if ch == u'.' and self.check_document_end():
	174	return self.fetch_document_end()
	175
[52]	176	# TODO: support for BOM within a stream.
	177	#if ch == u'\uFEFF':
	178	# return self.fetch_bom() <-- issue BOMToken
	179
[39]	180	# Note: the order of the following checks is NOT significant.
	181
	182	# Is it the flow sequence start indicator?
	183	if ch == u'[':
	184	return self.fetch_flow_sequence_start()
	185
	186	# Is it the flow mapping start indicator?
	187	if ch == u'{':
	188	return self.fetch_flow_mapping_start()
	189
	190	# Is it the flow sequence end indicator?
	191	if ch == u']':
	192	return self.fetch_flow_sequence_end()
	193
	194	# Is it the flow mapping end indicator?
	195	if ch == u'}':
	196	return self.fetch_flow_mapping_end()
	197
[51]	198	# Is it the flow entry indicator?
	199	if ch in u',':
	200	return self.fetch_flow_entry()
[43]	201
[51]	202	# Is it the block entry indicator?
	203	if ch in u'-' and self.check_block_entry():
	204	return self.fetch_block_entry()
	205
[39]	206	# Is it the key indicator?
	207	if ch == u'?' and self.check_key():
	208	return self.fetch_key()
	209
	210	# Is it the value indicator?
	211	if ch == u':' and self.check_value():
	212	return self.fetch_value()
	213
	214	# Is it an alias?
	215	if ch == u'*':
	216	return self.fetch_alias()
	217
	218	# Is it an anchor?
	219	if ch == u'&':
	220	return self.fetch_anchor()
	221
[43]	222	# Is it a tag?
[39]	223	if ch == u'!':
	224	return self.fetch_tag()
	225
[43]	226	# Is it a literal scalar?
	227	if ch == u'\|' and not self.flow_level:
[39]	228	return self.fetch_literal()
	229
	230	# Is it a folded scalar?
[43]	231	if ch == u'>' and not self.flow_level:
[39]	232	return self.fetch_folded()
	233
	234	# Is it a single quoted scalar?
	235	if ch == u'\'':
	236	return self.fetch_single()
	237
	238	# Is it a double quoted scalar?
	239	if ch == u'\"':
	240	return self.fetch_double()
	241
[43]	242	# It must be a plain scalar then.
[39]	243	if self.check_plain():
	244	return self.fetch_plain()
	245
[43]	246	# No? It's an error. Let's produce a nice error message.
[48]	247	raise ScannerError("while scanning for the next token", None,
	248	"found character %r that cannot start any token"
	249	% ch.encode('utf-8'), self.reader.get_marker())
[39]	250
[43]	251	# Simple keys treatment.
	252
	253	def next_possible_simple_key(self):
	254	# Return the number of the nearest possible simple key. Actually we
	255	# don't need to loop through the whole dictionary. We may replace it
	256	# with the following code:
	257	# if not self.possible_simple_keys:
	258	# return None
	259	# return self.possible_simple_keys[
	260	# min(self.possible_simple_keys.keys())].token_number
	261	min_token_number = None
	262	for level in self.possible_simple_keys:
	263	key = self.possible_simple_keys[level]
	264	if min_token_number is None or key.token_number < min_token_number:
	265	min_token_number = key.token_number
	266	return min_token_number
	267
	268	def stale_possible_simple_keys(self):
	269	# Remove entries that are no longer possible simple keys. According to
	270	# the YAML specification, simple keys
	271	# - should be limited to a single line,
	272	# - should be no longer than 1024 characters.
	273	# Disabling this procedure will allow simple keys of any length and
	274	# height (may cause problems if indentation is broken though).
	275	for level in self.possible_simple_keys.keys():
	276	key = self.possible_simple_keys[level]
[46]	277	if key.line != self.reader.line \
	278	or self.reader.index-key.index > 1024:
[43]	279	if key.required:
[47]	280	raise ScannerError("while scanning a simple key", key.marker,
	281	"could not found expected ':'", self.reader.get_marker())
[43]	282	del self.possible_simple_keys[level]
	283
	284	def save_possible_simple_key(self):
	285	# The next token may start a simple key. We check if it's possible
	286	# and save its position. This function is called for
	287	# ALIAS, ANCHOR, TAG, SCALAR(flow), '[', and '{'.
	288
	289	# Check if a simple key is required at the current position.
[46]	290	required = not self.flow_level and self.indent == self.reader.column
[43]	291
[47]	292	# A simple key is required only if it is the first token in the current
	293	# line. Therefore it is always allowed.
	294	assert self.allow_simple_key or not required
	295
[43]	296	# The next token might be a simple key. Let's save it's number and
	297	# position.
	298	if self.allow_simple_key:
	299	self.remove_possible_simple_key()
	300	token_number = self.tokens_taken+len(self.tokens)
[46]	301	index = self.reader.index
	302	line = self.reader.line
	303	column = self.reader.column
	304	marker = self.reader.get_marker()
[43]	305	key = SimpleKey(token_number, required,
	306	index, line, column, marker)
	307	self.possible_simple_keys[self.flow_level] = key
	308
	309	def remove_possible_simple_key(self):
	310	# Remove the saved possible key position at the current flow level.
	311	if self.flow_level in self.possible_simple_keys:
	312	key = self.possible_simple_keys[self.flow_level]
[47]	313
	314	# I don't think it's possible, but I could be wrong.
	315	assert not key.required
	316	#if key.required:
	317	# raise ScannerError("while scanning a simple key", key.marker,
	318	# "could not found expected ':'", self.reader.get_marker())
[43]	319
	320	# Indentation functions.
	321
	322	def unwind_indent(self, column):
	323
	324	# In flow context, tokens should respect indentation.
[47]	325	# Actually the condition should be `self.indent >= column` according to
	326	# the spec. But this condition will prohibit intuitively correct
	327	# constructions such as
	328	# key : {
	329	# }
[43]	330	if self.flow_level and self.indent > column:
[47]	331	raise ScannerError(None, None,
	332	"invalid intendation or unclosed '[' or '{'",
	333	self.reader.get_marker())
[43]	334
	335	# In block context, we may need to issue the BLOCK-END tokens.
	336	while self.indent > column:
[46]	337	marker = self.reader.get_marker()
[43]	338	self.indent = self.indents.pop()
[44]	339	self.tokens.append(BlockEndToken(marker, marker))
[43]	340
	341	def add_indent(self, column):
	342	# Check if we need to increase indentation.
	343	if self.indent < column:
	344	self.indents.append(self.indent)
	345	self.indent = column
	346	return True
	347	return False
	348
	349	# Fetchers.
	350
[48]	351	def fetch_stream_end(self):
[39]	352
	353	# Set the current intendation to -1.
[43]	354	self.unwind_indent(-1)
[39]	355
	356	# Reset everything (not really needed).
[43]	357	self.allow_simple_key = False
[39]	358	self.possible_simple_keys = {}
	359
[43]	360	# Read the token.
[46]	361	marker = self.reader.get_marker()
[43]	362
	363	# Add END.
[47]	364	self.tokens.append(StreamEndToken(marker, marker))
[39]	365
[46]	366	# The reader is ended.
[39]	367	self.done = True
	368
[43]	369	def fetch_directive(self):
	370
	371	# Set the current intendation to -1.
	372	self.unwind_indent(-1)
[39]	373
[43]	374	# Reset simple keys.
	375	self.remove_possible_simple_key()
	376	self.allow_simple_key = False
[39]	377
[43]	378	# Scan and add DIRECTIVE.
[47]	379	self.tokens.append(self.scan_directive())
[39]	380
	381	def fetch_document_start(self):
[44]	382	self.fetch_document_indicator(DocumentStartToken)
[39]	383
[43]	384	def fetch_document_end(self):
[44]	385	self.fetch_document_indicator(DocumentEndToken)
[43]	386
	387	def fetch_document_indicator(self, TokenClass):
	388
[39]	389	# Set the current intendation to -1.
[43]	390	self.unwind_indent(-1)
[39]	391
[43]	392	# Reset simple keys. Note that there could not be a block collection
	393	# after '---'.
	394	self.remove_possible_simple_key()
	395	self.allow_simple_key = False
[39]	396
[43]	397	# Add DOCUMENT-START or DOCUMENT-END.
[46]	398	start_marker = self.reader.get_marker()
	399	self.reader.forward(3)
	400	end_marker = self.reader.get_marker()
[43]	401	self.tokens.append(TokenClass(start_marker, end_marker))
[39]	402
[43]	403	def fetch_flow_sequence_start(self):
[44]	404	self.fetch_flow_collection_start(FlowSequenceStartToken)
[39]	405
[43]	406	def fetch_flow_mapping_start(self):
[44]	407	self.fetch_flow_collection_start(FlowMappingStartToken)
[43]	408
	409	def fetch_flow_collection_start(self, TokenClass):
	410
[44]	411	# '[' and '{' may start a simple key.
	412	self.save_possible_simple_key()
	413
[43]	414	# Increase the flow level.
	415	self.flow_level += 1
	416
	417	# Simple keys are allowed after '[' and '{'.
	418	self.allow_simple_key = True
	419
	420	# Add FLOW-SEQUENCE-START or FLOW-MAPPING-START.
[46]	421	start_marker = self.reader.get_marker()
	422	self.reader.forward()
	423	end_marker = self.reader.get_marker()
[43]	424	self.tokens.append(TokenClass(start_marker, end_marker))
[39]	425
[43]	426	def fetch_flow_sequence_end(self):
[44]	427	self.fetch_flow_collection_end(FlowSequenceEndToken)
[39]	428
[43]	429	def fetch_flow_mapping_end(self):
[44]	430	self.fetch_flow_collection_end(FlowMappingEndToken)
[43]	431
	432	def fetch_flow_collection_end(self, TokenClass):
	433
	434	# Reset possible simple key on the current level.
	435	self.remove_possible_simple_key()
	436
	437	# Decrease the flow level.
	438	self.flow_level -= 1
	439
	440	# No simple keys after ']' or '}'.
	441	self.allow_simple_key = False
	442
	443	# Add FLOW-SEQUENCE-END or FLOW-MAPPING-END.
[46]	444	start_marker = self.reader.get_marker()
	445	self.reader.forward()
	446	end_marker = self.reader.get_marker()
[43]	447	self.tokens.append(TokenClass(start_marker, end_marker))
[39]	448
[51]	449	def fetch_flow_entry(self):
[39]	450
[51]	451	# Simple keys are allowed after ','.
	452	self.allow_simple_key = True
	453
	454	# Reset possible simple key on the current level.
	455	self.remove_possible_simple_key()
	456
	457	# Add FLOW-ENTRY.
	458	start_marker = self.reader.get_marker()
	459	self.reader.forward()
	460	end_marker = self.reader.get_marker()
	461	self.tokens.append(FlowEntryToken(start_marker, end_marker))
	462
	463	def fetch_block_entry(self):
	464
[43]	465	# Block context needs additional checks.
	466	if not self.flow_level:
[39]	467
[43]	468	# Are we allowed to start a new entry?
	469	if not self.allow_simple_key:
[47]	470	raise ScannerError(None, None,
	471	"sequence entries are not allowed here",
	472	self.reader.get_marker())
[39]	473
[43]	474	# We may need to add BLOCK-SEQUENCE-START.
[46]	475	if self.add_indent(self.reader.column):
	476	marker = self.reader.get_marker()
[44]	477	self.tokens.append(BlockSequenceStartToken(marker, marker))
[39]	478
[51]	479	# It's an error for the block entry to occur in the flow context,
	480	# but we let the parser detect this.
	481	else:
	482	pass
	483
	484	# Simple keys are allowed after '-'.
[43]	485	self.allow_simple_key = True
[39]	486
[43]	487	# Reset possible simple key on the current level.
	488	self.remove_possible_simple_key()
[39]	489
[51]	490	# Add BLOCK-ENTRY.
[46]	491	start_marker = self.reader.get_marker()
	492	self.reader.forward()
	493	end_marker = self.reader.get_marker()
[51]	494	self.tokens.append(BlockEntryToken(start_marker, end_marker))
[39]	495
[43]	496	def fetch_key(self):
	497
	498	# Block context needs additional checks.
	499	if not self.flow_level:
[39]	500
[43]	501	# Are we allowed to start a key (not nessesary a simple)?
	502	if not self.allow_simple_key:
[47]	503	raise ScannerError(None, None,
	504	"mapping keys are not allowed here",
	505	self.reader.get_marker())
[43]	506
	507	# We may need to add BLOCK-MAPPING-START.
[46]	508	if self.add_indent(self.reader.column):
	509	marker = self.reader.get_marker()
[44]	510	self.tokens.append(BlockMappingStartToken(marker, marker))
[43]	511
	512	# Simple keys are allowed after '?' in the block context.
	513	self.allow_simple_key = not self.flow_level
	514
	515	# Reset possible simple key on the current level.
	516	self.remove_possible_simple_key()
	517
	518	# Add KEY.
[46]	519	start_marker = self.reader.get_marker()
	520	self.reader.forward()
	521	end_marker = self.reader.get_marker()
[44]	522	self.tokens.append(KeyToken(start_marker, end_marker))
[39]	523
[43]	524	def fetch_value(self):
[39]	525
[43]	526	# Do we determine a simple key?
	527	if self.flow_level in self.possible_simple_keys:
[39]	528
[43]	529	# Add KEY.
	530	key = self.possible_simple_keys[self.flow_level]
	531	del self.possible_simple_keys[self.flow_level]
	532	self.tokens.insert(key.token_number-self.tokens_taken,
[44]	533	KeyToken(key.marker, key.marker))
[39]	534
[43]	535	# If this key starts a new block mapping, we need to add
	536	# BLOCK-MAPPING-START.
	537	if not self.flow_level:
	538	if self.add_indent(key.column):
	539	self.tokens.insert(key.token_number-self.tokens_taken,
[44]	540	BlockMappingStartToken(key.marker, key.marker))
[37]	541
[43]	542	# There cannot be two simple keys one after another.
	543	self.allow_simple_key = False
[37]	544
[43]	545	# It must be a part of a complex key.
	546	else:
	547
[47]	548	# Block context needs additional checks.
	549	# (Do we really need them? They will be catched by the parser
	550	# anyway.)
	551	if not self.flow_level:
	552
	553	# We are allowed to start a complex value if and only if
	554	# we can start a simple key.
	555	if not self.allow_simple_key:
	556	raise ScannerError(None, None,
	557	"mapping values are not allowed here",
	558	self.reader.get_marker())
	559
[43]	560	# Simple keys are allowed after ':' in the block context.
	561	self.allow_simple_key = not self.flow_level
[37]	562
[43]	563	# Reset possible simple key on the current level.
	564	self.remove_possible_simple_key()
[37]	565
[43]	566	# Add VALUE.
[46]	567	start_marker = self.reader.get_marker()
	568	self.reader.forward()
	569	end_marker = self.reader.get_marker()
[44]	570	self.tokens.append(ValueToken(start_marker, end_marker))
[37]	571
[43]	572	def fetch_alias(self):
[37]	573
[43]	574	# ALIAS could be a simple key.
	575	self.save_possible_simple_key()
[37]	576
[43]	577	# No simple keys after ALIAS.
	578	self.allow_simple_key = False
[37]	579
[43]	580	# Scan and add ALIAS.
[47]	581	self.tokens.append(self.scan_anchor(AliasToken))
[37]	582
[43]	583	def fetch_anchor(self):
[37]	584
[43]	585	# ANCHOR could start a simple key.
	586	self.save_possible_simple_key()
[37]	587
[43]	588	# No simple keys after ANCHOR.
	589	self.allow_simple_key = False
[37]	590
[43]	591	# Scan and add ANCHOR.
[47]	592	self.tokens.append(self.scan_anchor(AnchorToken))
[37]	593
[43]	594	def fetch_tag(self):
[37]	595
[43]	596	# TAG could start a simple key.
	597	self.save_possible_simple_key()
[37]	598
[43]	599	# No simple keys after TAG.
	600	self.allow_simple_key = False
[37]	601
[43]	602	# Scan and add TAG.
[47]	603	self.tokens.append(self.scan_tag())
[37]	604
[43]	605	def fetch_literal(self):
	606	self.fetch_block_scalar(folded=False)
[37]	607
[43]	608	def fetch_folded(self):
	609	self.fetch_block_scalar(folded=True)
[37]	610
[43]	611	def fetch_block_scalar(self, folded):
[37]	612
[43]	613	# A simple key may follow a block scalar.
	614	self.allow_simple_key = True
[37]	615
[43]	616	# Reset possible simple key on the current level.
	617	self.remove_possible_simple_key()
[37]	618
[43]	619	# Scan and add SCALAR.
[47]	620	self.tokens.append(self.scan_block_scalar(folded))
[37]	621
[43]	622	def fetch_single(self):
	623	self.fetch_flow_scalar(double=False)
[37]	624
[43]	625	def fetch_double(self):
	626	self.fetch_flow_scalar(double=True)
[37]	627
[43]	628	def fetch_flow_scalar(self, double):
[37]	629
[43]	630	# A flow scalar could be a simple key.
	631	self.save_possible_simple_key()
[37]	632
[43]	633	# No simple keys after flow scalars.
	634	self.allow_simple_key = False
[37]	635
[43]	636	# Scan and add SCALAR.
[47]	637	self.tokens.append(self.scan_flow_scalar(double))
[37]	638
[43]	639	def fetch_plain(self):
[37]	640
[43]	641	# A plain scalar could be a simple key.
	642	self.save_possible_simple_key()
[37]	643
[43]	644	# No simple keys after plain scalars. But note that `scan_plain` will
	645	# change this flag if the scan is finished at the beginning of the
	646	# line.
	647	self.allow_simple_key = False
[37]	648
[43]	649	# Scan and add SCALAR. May change `allow_simple_key`.
[47]	650	self.tokens.append(self.scan_plain())
[37]	651
[43]	652	# Checkers.
[37]	653
[43]	654	def check_directive(self):
[37]	655
[43]	656	# DIRECTIVE: ^ '%' ...
	657	# The '%' indicator is already checked.
[46]	658	if self.reader.column == 0:
[43]	659	return True
[37]	660
[43]	661	def check_document_start(self):
[37]	662
[43]	663	# DOCUMENT-START: ^ '---' (' '\|'\n')
[46]	664	if self.reader.column == 0:
[48]	665	if self.reader.prefix(3) == u'---' \
	666	and self.reader.peek(3) in u'\0 \t\r\n\x85\u2028\u2029':
[43]	667	return True
[37]	668
[43]	669	def check_document_end(self):
[37]	670
[43]	671	# DOCUMENT-END: ^ '...' (' '\|'\n')
[46]	672	if self.reader.column == 0:
	673	prefix = self.reader.peek(4)
[48]	674	if self.reader.prefix(3) == u'...' \
	675	and self.reader.peek(3) in u'\0 \t\r\n\x85\u2028\u2029':
[43]	676	return True
[37]	677
[51]	678	def check_block_entry(self):
[43]	679
[51]	680	# BLOCK-ENTRY: '-' (' '\|'\n')
	681	return self.reader.peek(1) in u'\0 \t\r\n\x85\u2028\u2029'
[43]	682
	683	def check_key(self):
	684
	685	# KEY(flow context): '?'
	686	if self.flow_level:
[37]	687	return True
[43]	688
	689	# KEY(block context): '?' (' '\|'\n')
[37]	690	else:
[48]	691	return self.reader.peek(1) in u'\0 \t\r\n\x85\u2028\u2029'
[37]	692
[43]	693	def check_value(self):
	694
	695	# VALUE(flow context): ':'
	696	if self.flow_level:
[37]	697	return True
[43]	698
	699	# VALUE(block context): ':' (' '\|'\n')
[37]	700	else:
[48]	701	return self.reader.peek(1) in u'\0 \t\r\n\x85\u2028\u2029'
[37]	702
[43]	703	def check_plain(self):
[37]	704
[48]	705	# A plain scalar may start with any non-space character except:
	706	# '-', '?', ':', ',', '[', ']', '{', '}',
	707	# '#', '&', '*', '!', '\|', '>', '\'', '\"',
	708	# '%', '@', '`'.
	709	#
	710	# It may also start with
	711	# '-', '?', ':'
	712	# if it is followed by a non-space character.
	713	#
	714	# Note that we limit the last rule to the block context (except the
	715	# '-' character) because we want the flow context to be space
	716	# independent.
	717	ch = self.reader.peek()
	718	return ch not in u'\0 \t\r\n\x85\u2028\u2029-?:,[]{}#&*!\|>\'\"%@`' \
	719	or (self.reader.peek(1) not in u'\0 \t\r\n\x85\u2028\u2029'
	720	and (ch == '-' or (not self.flow_level and ch in u'?:')))
	721
[43]	722	# Scanners.
	723
	724	def scan_to_next_token(self):
[47]	725	# We ignore spaces, line breaks and comments.
	726	# If we find a line break in the block context, we set the flag
	727	# `allow_simple_key` on.
[51]	728	# The byte order mark is stripped if it's the first character in the
	729	# stream. We do not yet support BOM inside the stream as the
	730	# specification requires. Any such mark will be considered as a part
	731	# of the document.
[52]	732	#
	733	# TODO: We need to make tab handling rules more sane. A good rule is
	734	# Tabs cannot precede tokens
	735	# BLOCK-SEQUENCE-START, BLOCK-MAPPING-START, BLOCK-END,
	736	# KEY(block), VALUE(block), BLOCK-ENTRY
	737	# So the checking code is
	738	# if <TAB>:
	739	# self.allow_simple_keys = False
	740	# We also need to add the check for `allow_simple_keys == True` to
	741	# `unwind_indent` before issuing BLOCK-END.
	742	# Scanners for block, flow, and plain scalars need to be modified.
	743
[51]	744	if self.reader.index == 0 and self.reader.peek() == u'\uFEFF':
	745	self.reader.forward()
[43]	746	found = False
	747	while not found:
[46]	748	while self.reader.peek() == u' ':
	749	self.reader.forward()
	750	if self.reader.peek() == u'#':
[47]	751	while self.reader.peek() not in u'\0\r\n\x85\u2028\u2029':
[46]	752	self.reader.forward()
[47]	753	if self.scan_line_break():
[43]	754	if not self.flow_level:
	755	self.allow_simple_key = True
[37]	756	else:
[43]	757	found = True
[37]	758
[43]	759	def scan_directive(self):
[48]	760	# See the specification for details.
	761	start_marker = self.reader.get_marker()
	762	self.reader.forward()
	763	name = self.scan_directive_name(start_marker)
	764	value = None
	765	if name == u'YAML':
	766	value = self.scan_yaml_directive_value(start_marker)
	767	end_marker = self.reader.get_marker()
	768	elif name == u'TAG':
	769	value = self.scan_tag_directive_value(start_marker)
	770	end_marker = self.reader.get_marker()
[43]	771	else:
[48]	772	end_marker = self.reader.get_marker()
	773	while self.reader.peek() not in u'\0\r\n\x85\u2028\u2029':
	774	self.reader.forward()
	775	self.scan_directive_ignored_line(start_marker)
	776	return DirectiveToken(name, value, start_marker, end_marker)
	777
	778	def scan_directive_name(self, start_marker):
	779	# See the specification for details.
	780	length = 0
	781	ch = self.reader.peek(length)
	782	while u'0' <= ch <= u'9' or u'A' <= ch <= 'Z' or u'a' <= ch <= 'z' \
	783	or ch in u'-_':
	784	length += 1
	785	ch = self.reader.peek(length)
	786	if not length:
	787	raise ScannerError("while scanning a directive", start_marker,
[52]	788	"expected alphabetic or numeric character, but found %r"
	789	% ch.encode('utf-8'), self.reader.get_marker())
[48]	790	value = self.reader.prefix(length)
	791	self.reader.forward(length)
	792	ch = self.reader.peek()
	793	if ch not in u'\0 \r\n\x85\u2028\u2029':
[52]	794	raise ScannerError("while scanning a directive", start_marker,
[48]	795	"expected alphabetic or numeric character, but found %r"
	796	% ch.encode('utf-8'), self.reader.get_marker())
	797	return value
	798
	799	def scan_yaml_directive_value(self, start_marker):
	800	# See the specification for details.
	801	while self.reader.peek() == u' ':
[46]	802	self.reader.forward()
[48]	803	major = self.scan_yaml_directive_number(start_marker)
	804	if self.reader.peek() != '.':
	805	raise ScannerError("while scanning a directive", start_marker,
[52]	806	"expected a digit or '.', but found %r"
	807	% self.reader.peek().encode('utf-8'),
[48]	808	self.reader.get_marker())
[46]	809	self.reader.forward()
[48]	810	minor = self.scan_yaml_directive_number(start_marker)
	811	if self.reader.peek() not in u'\0 \r\n\x85\u2028\u2029':
	812	raise ScannerError("while scanning a directive", start_marker,
[52]	813	"expected a digit or ' ', but found %r"
	814	% self.reader.peek().encode('utf-8'),
[48]	815	self.reader.get_marker())
	816	return (major, minor)
[37]	817
[48]	818	def scan_yaml_directive_number(self, start_marker):
	819	# See the specification for details.
	820	ch = self.reader.peek()
	821	if not (u'0' <= ch <= '9'):
	822	raise ScannerError("while scanning a directive", start_marker,
	823	"expected a digit, but found %r" % ch.encode('utf-8'),
	824	self.reader.get_marker())
	825	length = 0
	826	while u'0' <= self.reader.peek(length) <= u'9':
	827	length += 1
	828	value = int(self.reader.prefix(length))
	829	self.reader.forward(length)
	830	return value
	831
	832	def scan_tag_directive_value(self, start_marker):
	833	# See the specification for details.
	834	while self.reader.peek() == u' ':
	835	self.reader.forward()
	836	handle = self.scan_tag_directive_handle(start_marker)
	837	while self.reader.peek() == u' ':
	838	self.reader.forward()
	839	prefix = self.scan_tag_directive_prefix(start_marker)
	840	return (handle, prefix)
	841
	842	def scan_tag_directive_handle(self, start_marker):
	843	# See the specification for details.
	844	value = self.scan_tag_handle('directive', start_marker)
[52]	845	ch = self.reader.peek()
	846	if ch != u' ':
[48]	847	raise ScannerError("while scanning a directive", start_marker,
	848	"expected ' ', but found %r" % ch.encode('utf-8'),
	849	self.reader.get_marker())
	850	return value
	851
	852	def scan_tag_directive_prefix(self, start_marker):
	853	# See the specification for details.
	854	value = self.scan_tag_uri('directive', start_marker)
	855	ch = self.reader.peek()
	856	if ch not in u'\0 \r\n\x85\u2028\u2029':
	857	raise ScannerError("while scanning a directive", start_marker,
	858	"expected ' ', but found %r" % ch.encode('utf-8'),
	859	self.reader.get_marker())
	860	return value
	861
	862	def scan_directive_ignored_line(self, start_marker):
	863	# See the specification for details.
	864	while self.reader.peek() == u' ':
	865	self.reader.forward()
	866	if self.reader.peek() == u'#':
	867	while self.reader.peek() not in u'\0\r\n\x85\u2028\u2029':
	868	self.reader.forward()
	869	ch = self.reader.peek()
	870	if ch not in u'\0\r\n\x85\u2028\u2029':
	871	raise ScannerError("while scanning a directive", start_marker,
	872	"expected a comment or a line break, but found %r"
	873	% ch.encode('utf-8'), self.reader.get_marker())
	874	self.scan_line_break()
	875
[43]	876	def scan_anchor(self, TokenClass):
[48]	877	# The specification does not restrict characters for anchors and
	878	# aliases. This may lead to problems, for instance, the document:
	879	# [ *alias, value ]
	880	# can be interpteted in two ways, as
	881	# [ "value" ]
	882	# and
	883	# [ *alias , "value" ]
	884	# Therefore we restrict aliases to numbers and ASCII letters.
[46]	885	start_marker = self.reader.get_marker()
[48]	886	indicator = self.reader.peek()
	887	if indicator == '*':
	888	name = 'alias'
	889	else:
	890	name = 'anchor'
	891	self.reader.forward()
	892	length = 0
	893	ch = self.reader.peek(length)
	894	while u'0' <= ch <= u'9' or u'A' <= ch <= 'Z' or u'a' <= ch <= 'z' \
	895	or ch in u'-_':
	896	length += 1
	897	ch = self.reader.peek(length)
	898	if not length:
	899	raise ScannerError("while scanning an %s" % name, start_marker,
[52]	900	"expected alphabetic or numeric character, but found %r"
	901	% ch.encode('utf-8'), self.reader.get_marker())
[48]	902	value = self.reader.prefix(length)
	903	self.reader.forward(length)
	904	ch = self.reader.peek()
	905	if ch not in u'\0 \t\r\n\x85\u2028\u2029?:,]}%@`':
	906	raise ScannerError("while scanning an %s" % name, start_marker,
	907	"expected alphabetic or numeric character, but found %r"
	908	% ch.encode('utf-8'), self.reader.get_marker())
[46]	909	end_marker = self.reader.get_marker()
[48]	910	return TokenClass(value, start_marker, end_marker)
[37]	911
[43]	912	def scan_tag(self):
[48]	913	# See the specification for details.
[46]	914	start_marker = self.reader.get_marker()
[48]	915	ch = self.reader.peek(1)
	916	if ch == u'<':
	917	handle = None
	918	self.reader.forward(2)
	919	suffix = self.scan_tag_uri('tag', start_marker)
	920	if self.reader.peek() != u'>':
[52]	921	raise ScannerError("while parsing a tag", start_marker,
	922	"expected '>', but found %r" % self.reader.peek().encode('utf-8'),
[48]	923	self.reader.get_marker())
[46]	924	self.reader.forward()
[48]	925	elif ch in u'\0 \t\r\n\x85\u2028\u2029':
	926	handle = None
	927	suffix = u'!'
	928	self.reader.forward()
	929	else:
	930	length = 1
	931	use_handle = False
	932	while ch not in u'\0 \r\n\x85\u2028\u2029':
	933	if ch == u'!':
	934	use_handle = True
	935	break
	936	length += 1
	937	ch = self.reader.peek(length)
	938	handle = u'!'
	939	if use_handle:
	940	handle = self.scan_tag_handle('tag', start_marker)
	941	else:
	942	handle = u'!'
	943	self.reader.forward()
	944	suffix = self.scan_tag_uri('tag', start_marker)
	945	ch = self.reader.peek()
	946	if ch not in u'\0 \r\n\x85\u2028\u2029':
	947	raise ScannerError("while scanning a tag", start_marker,
	948	"expected ' ', but found %r" % ch.encode('utf-8'),
	949	self.reader.get_marker())
	950	value = (handle, suffix)
[46]	951	end_marker = self.reader.get_marker()
[48]	952	return TagToken(value, start_marker, end_marker)
[43]	953
	954	def scan_block_scalar(self, folded):
[48]	955	# See the specification for details.
	956
	957	chunks = []
[46]	958	start_marker = self.reader.get_marker()
[48]	959
	960	# Scan the header.
	961	self.reader.forward()
	962	chomping, increment = self.scan_block_scalar_indicators(start_marker)
	963	self.scan_block_scalar_ignored_line(start_marker)
	964
	965	# Determine the indentation level and go to the first non-empty line.
	966	min_indent = self.indent+1
	967	if min_indent < 1:
	968	min_indent = 1
	969	if increment is None:
	970	breaks, max_indent, end_marker = self.scan_block_scalar_indentation()
	971	indent = max(min_indent, max_indent)
	972	else:
	973	indent = min_indent+increment-1
	974	breaks, end_marker = self.scan_block_scalar_breaks(indent)
	975	line_break = u''
	976
	977	# Scan the inner part of the block scalar.
	978	while self.reader.column == indent and self.reader.peek() != u'\0':
	979	chunks.extend(breaks)
	980	leading_non_space = self.reader.peek() not in u' \t'
	981	length = 0
	982	while self.reader.peek(length) not in u'\0\r\n\x85\u2028\u2029':
	983	length += 1
	984	chunks.append(self.reader.prefix(length))
	985	self.reader.forward(length)
	986	line_break = self.scan_line_break()
	987	breaks, end_marker = self.scan_block_scalar_breaks(indent)
	988	if self.reader.column == indent and self.reader.peek() != u'\0':
	989	# Unfortunately, folding rules are ambiguous.
	990	#
	991	# This is the folding according to the specification:
[51]	992
	993	if folded and line_break == u'\n' \
	994	and leading_non_space and self.reader.peek() not in u' \t':
	995	if not breaks:
	996	chunks.append(u' ')
	997	else:
	998	chunks.append(line_break)
	999
	1000	# This is Clark Evans's interpretation (also in the spec
	1001	# examples):
[48]	1002	#
[51]	1003	#if folded and line_break == u'\n':
[48]	1004	# if not breaks:
[51]	1005	# if self.reader.peek() not in ' \t':
	1006	# chunks.append(u' ')
	1007	# else:
	1008	# chunks.append(line_break)
[48]	1009	#else:
	1010	# chunks.append(line_break)
	1011	else:
	1012	break
	1013
	1014	# Chomp the tail.
	1015	if chomping is not False:
	1016	chunks.append(line_break)
	1017	if chomping is True:
	1018	chunks.extend(breaks)
	1019
	1020	# We are done.
	1021	return ScalarToken(u''.join(chunks), False, start_marker, end_marker)
	1022
	1023	def scan_block_scalar_indicators(self, start_marker):
	1024	# See the specification for details.
	1025	chomping = None
	1026	increment = None
	1027	ch = self.reader.peek()
	1028	if ch in u'+-':
	1029	if ch == '+':
	1030	chomping = True
	1031	else:
	1032	chomping = False
	1033	self.reader.forward()
	1034	ch = self.reader.peek()
	1035	if ch in u'0123456789':
	1036	increment = int(ch)
	1037	if increment == 0:
	1038	raise ScannerError("while scanning a block scalar", start_marker,
	1039	"expected indentation indicator in the range 1-9, but found 0",
	1040	self.reader.get_marker())
[46]	1041	self.reader.forward()
[48]	1042	elif ch in u'0123456789':
	1043	increment = int(ch)
	1044	if increment == 0:
	1045	raise ScannerError("while scanning a block scalar", start_marker,
	1046	"expected indentation indicator in the range 1-9, but found 0",
	1047	self.reader.get_marker())
	1048	self.reader.forward()
	1049	ch = self.reader.peek()
	1050	if ch in u'+-':
	1051	if ch == '+':
	1052	chomping = True
	1053	else:
	1054	chomping = False
[46]	1055	self.reader.forward()
[48]	1056	ch = self.reader.peek()
	1057	if ch not in u'\0 \r\n\x85\u2028\u2029':
	1058	raise ScannerError("while scanning a block scalar", start_marker,
	1059	"expected chomping or indentation indicators, but found %r"
	1060	% ch.encode('utf-8'), self.reader.get_marker())
	1061	return chomping, increment
	1062
	1063	def scan_block_scalar_ignored_line(self, start_marker):
	1064	# See the specification for details.
	1065	while self.reader.peek() == u' ':
	1066	self.reader.forward()
	1067	if self.reader.peek() == u'#':
	1068	while self.reader.peek() not in u'\0\r\n\x85\u2028\u2029':
[46]	1069	self.reader.forward()
[48]	1070	ch = self.reader.peek()
	1071	if ch not in u'\0\r\n\x85\u2028\u2029':
	1072	raise ScannerError("while scanning a block scalar", start_marker,
	1073	"expected a comment or a line break, but found %r"
	1074	% ch.encode('utf-8'), self.reader.get_marker())
	1075	self.scan_line_break()
[43]	1076
[48]	1077	def scan_block_scalar_indentation(self):
	1078	# See the specification for details.
	1079	chunks = []
	1080	max_indent = 0
	1081	end_marker = self.reader.get_marker()
	1082	while self.reader.peek() in u' \r\n\x85\u2028\u2029':
	1083	if self.reader.peek() != u' ':
	1084	chunks.append(self.scan_line_break())
	1085	end_marker = self.reader.get_marker()
	1086	else:
	1087	self.reader.forward()
	1088	if self.reader.column > max_indent:
	1089	max_indent = self.reader.column
	1090	return chunks, max_indent, end_marker
	1091
	1092	def scan_block_scalar_breaks(self, indent):
	1093	# See the specification for details.
	1094	chunks = []
	1095	end_marker = self.reader.get_marker()
	1096	while self.reader.column < indent and self.reader.peek() == u' ':
	1097	self.reader.forward()
	1098	while self.reader.peek() in u'\r\n\x85\u2028\u2029':
	1099	chunks.append(self.scan_line_break())
	1100	end_marker = self.reader.get_marker()
	1101	while self.reader.column < indent and self.reader.peek() == u' ':
	1102	self.reader.forward()
	1103	return chunks, end_marker
	1104
[43]	1105	def scan_flow_scalar(self, double):
[48]	1106	# See the specification for details.
	1107	chunks = []
	1108	start_marker = self.reader.get_marker()
	1109	indent = self.indent+1
	1110	if indent == 0:
	1111	indent = 1
[46]	1112	quote = self.reader.peek()
	1113	self.reader.forward()
[48]	1114	chunks.extend(self.scan_flow_scalar_non_spaces(double, indent, start_marker))
[46]	1115	while self.reader.peek() != quote:
[48]	1116	chunks.extend(self.scan_flow_scalar_spaces(double, indent, start_marker))
	1117	chunks.extend(self.scan_flow_scalar_non_spaces(double, indent, start_marker))
	1118	self.reader.forward()
	1119	end_marker = self.reader.get_marker()
	1120	return ScalarToken(u''.join(chunks), False, start_marker, end_marker)
	1121
	1122	ESCAPE_REPLACEMENTS = {
	1123	u'0': u'\0',
	1124	u'a': u'\x07',
	1125	u'b': u'\x08',
	1126	u't': u'\x09',
	1127	u'\t': u'\x09',
	1128	u'n': u'\x0A',
	1129	u'v': u'\x0B',
	1130	u'f': u'\x0C',
	1131	u'r': u'\x0D',
	1132	u'e': u'\x1B',
	1133	u' ': u'\x20',
	1134	u'\"': u'\"',
	1135	u'\\': u'\\',
	1136	u'N': u'\x85',
	1137	u'_': u'\xA0',
	1138	u'L': u'\u2028',
	1139	u'P': u'\u2029',
	1140	}
	1141
	1142	ESCAPE_CODES = {
	1143	u'x': 2,
	1144	u'u': 4,
	1145	u'U': 8,
	1146	}
	1147
	1148	def scan_flow_scalar_non_spaces(self, double, indent, start_marker):
	1149	# See the specification for details.
	1150	chunks = []
	1151	while True:
	1152	length = 0
	1153	while self.reader.peek(length) not in u'\'\"\\\0 \t\r\n\x85\u2028\u2029':
	1154	length += 1
	1155	if length:
	1156	chunks.append(self.reader.prefix(length))
	1157	self.reader.forward(length)
	1158	ch = self.reader.peek()
	1159	if not double and ch == u'\'' and self.reader.peek(1) == u'\'':
	1160	chunks.append(u'\'')
[46]	1161	self.reader.forward(2)
[48]	1162	elif (double and ch == u'\'') or (not double and ch in u'\"\\'):
	1163	chunks.append(ch)
	1164	self.reader.forward()
	1165	elif double and ch == u'\\':
	1166	self.reader.forward()
	1167	ch = self.reader.peek()
	1168	if ch in self.ESCAPE_REPLACEMENTS:
	1169	chunks.append(self.ESCAPE_REPLACEMENTS[ch])
	1170	self.reader.forward()
	1171	elif ch in self.ESCAPE_CODES:
	1172	length = self.ESCAPE_CODES[ch]
	1173	self.reader.forward()
	1174	for k in range(length):
	1175	if self.reader.peek(k) not in u'0123456789ABCDEFabcdef':
	1176	raise ScannerError("while scanning a double-quoted scalar", start_marker,
	1177	"expected escape sequence of %d hexdecimal numbers, but found %r" %
	1178	(length, self.reader.peek(k).encode('utf-8')), self.reader.get_marker())
	1179	code = int(self.reader.prefix(length), 16)
	1180	chunks.append(unichr(code))
	1181	self.reader.forward(length)
	1182	elif ch in u'\r\n\x85\u2028\u2029':
	1183	self.scan_line_break()
	1184	chunks.extend(self.scan_flow_scalar_breaks(double, indent, start_marker))
	1185	else:
	1186	raise ScannerError("while scanning a double-quoted scalar", start_marker,
	1187	"found unknown escape character %r" % ch.encode('utf-8'), self.reader.get_marker())
[37]	1188	else:
[48]	1189	return chunks
[37]	1190
[48]	1191	def scan_flow_scalar_spaces(self, double, indent, start_marker):
	1192	# See the specification for details.
	1193	chunks = []
	1194	length = 0
	1195	while self.reader.peek(length) in u' \t':
	1196	length += 1
	1197	whitespaces = self.reader.prefix(length)
	1198	self.reader.forward(length)
	1199	ch = self.reader.peek()
	1200	if ch == u'\0':
	1201	raise ScannerError("while scanning a quoted scalar", start_marker,
	1202	"found unexpected end of stream", self.reader.get_marker())
	1203	elif ch in u'\r\n\x85\u2028\u2029':
	1204	line_break = self.scan_line_break()
	1205	breaks = self.scan_flow_scalar_breaks(double, indent, start_marker)
	1206	if line_break != u'\n':
	1207	chunks.append(line_break)
	1208	elif not breaks:
	1209	chunks.append(u' ')
	1210	chunks.extend(breaks)
	1211	else:
	1212	chunks.append(whitespaces)
	1213	return chunks
	1214
	1215	def scan_flow_scalar_breaks(self, double, indent, start_marker):
	1216	# See the specification for details.
	1217	chunks = []
	1218	while True:
	1219	while self.reader.column < indent and self.reader.peek() == u' ':
	1220	self.reader.forward()
	1221	if self.reader.column < indent \
	1222	and self.reader.peek() not in u'\0\r\n\x85\u2028\u2029':
	1223	s = 's'
	1224	if indent == 1:
	1225	s = ''
	1226	raise ScannerError("while scanning a quoted scalar", start_marker,
	1227	"expected %d space%s indentation, but found %r"
	1228	% (indent, s, self.reader.peek().encode('utf-8')),
	1229	self.reader.get_marker())
	1230	while self.reader.peek() in u' \t':
	1231	self.reader.forward()
	1232	if self.reader.peek() in u'\r\n\x85\u2028\u2029':
	1233	chunks.append(self.scan_line_break())
	1234	else:
	1235	return chunks
	1236
[43]	1237	def scan_plain(self):
[48]	1238	# See the specification for details.
	1239	# We add an additional restriction for the flow context:
	1240	# plain scalars in the flow context cannot contain ':' and '?'.
	1241	# We also keep track of the `allow_simple_key` flag here.
	1242	chunks = []
	1243	start_marker = self.reader.get_marker()
	1244	end_marker = start_marker
[43]	1245	indent = self.indent+1
[48]	1246	if indent == 0:
[43]	1247	indent = 1
[48]	1248	spaces = []
[43]	1249	while True:
[48]	1250	length = 0
	1251	if self.reader.peek() == u'#':
[43]	1252	break
[48]	1253	while True:
	1254	ch = self.reader.peek(length)
	1255	if ch in u'\0 \t\r\n\x85\u2028\u2029' \
	1256	or (not self.flow_level and ch == u':' and
	1257	self.reader.peek(length+1) in u'\0 \t\r\n\x28\u2028\u2029') \
	1258	or (self.flow_level and ch in u',:?[]{}'):
	1259	break
	1260	length += 1
	1261	if length == 0:
[43]	1262	break
[48]	1263	self.allow_simple_key = False
	1264	chunks.extend(spaces)
	1265	chunks.append(self.reader.prefix(length))
	1266	self.reader.forward(length)
	1267	end_marker = self.reader.get_marker()
	1268	spaces = self.scan_plain_spaces(indent)
	1269	if not spaces or self.reader.peek() == u'#' \
	1270	or self.reader.column < indent:
	1271	break
	1272	return ScalarToken(u''.join(chunks), True, start_marker, end_marker)
[37]	1273
[48]	1274	def scan_plain_spaces(self, indent):
	1275	# See the specification for details.
	1276	# The specification is really confusing about tabs in plain scalars.
	1277	# We just forbid them completely. Do not use tabs in YAML!
	1278	chunks = []
	1279	length = 0
	1280	while self.reader.peek(length) in u' ':
	1281	length += 1
	1282	whitespaces = self.reader.prefix(length)
	1283	self.reader.forward(length)
	1284	ch = self.reader.peek()
	1285	if ch in u'\r\n\x85\u2028\u2029':
	1286	line_break = self.scan_line_break()
	1287	self.allow_simple_key = True
	1288	breaks = []
	1289	while self.reader.peek() in u' \r\n\x85\u2028\u2029':
	1290	if self.reader.peek() == ' ':
	1291	self.reader.forward()
	1292	else:
	1293	breaks.append(self.scan_line_break())
	1294	if line_break != u'\n':
	1295	chunks.append(line_break)
	1296	elif not breaks:
	1297	chunks.append(u' ')
	1298	chunks.extend(breaks)
	1299	elif whitespaces:
	1300	chunks.append(whitespaces)
	1301	return chunks
	1302
	1303	def scan_tag_handle(self, name, start_marker):
	1304	# See the specification for details.
	1305	# For some strange reasons, the specification does not allow '_' in
	1306	# tag handles. I have allowed it anyway.
[52]	1307	ch = self.reader.peek()
	1308	if ch != u'!':
[48]	1309	raise ScannerError("while scanning a %s" % name, start_marker,
	1310	"expected '!', but found %r" % ch.encode('utf-8'),
	1311	self.reader.get_marker())
	1312	length = 1
	1313	ch = self.reader.peek(length)
	1314	if ch != u' ':
	1315	while u'0' <= ch <= u'9' or u'A' <= ch <= 'Z' or u'a' <= ch <= 'z' \
	1316	or ch in u'-_':
	1317	length += 1
	1318	ch = self.reader.peek(length)
	1319	if ch != u'!':
	1320	self.reader.forward(length)
	1321	raise ScannerError("while scanning a %s" % name, start_marker,
	1322	"expected '!', but found %r" % ch.encode('utf-8'),
	1323	self.reader.get_marker())
	1324	length += 1
	1325	value = self.reader.prefix(length)
	1326	self.reader.forward(length)
	1327	return value
	1328
	1329	def scan_tag_uri(self, name, start_marker):
	1330	# See the specification for details.
	1331	# Note: we do not check if URI is well-formed.
	1332	chunks = []
	1333	length = 0
	1334	ch = self.reader.peek(length)
	1335	while u'0' <= ch <= u'9' or u'A' <= ch <= 'Z' or u'a' <= ch <= 'z' \
	1336	or ch in u'-;/?:@&=+$,_.!~*\'()[]%':
	1337	if ch == u'%':
	1338	chunks.append(self.reader.prefix(length))
	1339	self.reader.forward(length)
	1340	length = 0
	1341	chunks.append(self.scan_uri_escapes(name, start_marker))
	1342	else:
	1343	length += 1
	1344	ch = self.reader.peek(length)
	1345	if length:
	1346	chunks.append(self.reader.prefix(length))
	1347	self.reader.forward(length)
	1348	length = 0
	1349	if not chunks:
	1350	raise ScannerError("while parsing a %s" % name, start_marker,
	1351	"expected URI, but found %r" % ch.encode('utf-8'),
	1352	self.reader.get_marker())
	1353	return u''.join(chunks)
	1354
	1355	def scan_uri_escapes(self, name, start_marker):
	1356	# See the specification for details.
	1357	bytes = []
	1358	marker = self.reader.get_marker()
	1359	while self.reader.peek() == u'%':
	1360	self.reader.forward()
	1361	for k in range(2):
	1362	if self.reader.peek(k) not in u'0123456789ABCDEFabcdef':
	1363	raise ScannerError("while scanning a %s" % name, start_marker,
	1364	"expected URI escape sequence of 2 hexdecimal numbers, but found %r" %
	1365	(self.reader.peek(k).encode('utf-8')), self.reader.get_marker())
	1366	bytes.append(chr(int(self.reader.prefix(2), 16)))
	1367	self.reader.forward(2)
	1368	try:
	1369	value = unicode(''.join(bytes), 'utf-8')
	1370	except UnicodeDecodeError, exc:
	1371	raise ScannerError("while scanning a %s" % name, start_marker, str(exc), marker)
	1372	return value
	1373
[47]	1374	def scan_line_break(self):
	1375	# Transforms:
	1376	# '\r\n' : '\n'
	1377	# '\r' : '\n'
	1378	# '\n' : '\n'
	1379	# '\x85' : '\n'
	1380	# '\u2028' : '\u2028'
	1381	# '\u2029 : '\u2029'
	1382	# default : ''
	1383	ch = self.reader.peek()
	1384	if ch in u'\r\n\x85':
[48]	1385	if self.reader.prefix(2) == u'\r\n':
[47]	1386	self.forward(2)
	1387	else:
	1388	self.reader.forward()
	1389	return u'\n'
	1390	elif ch in u'\u2028\u2029':
	1391	self.reader.forward()
	1392	return ch
	1393	return u''
	1394
[45]	1395	#try:
	1396	# import psyco
	1397	# psyco.bind(Scanner)
	1398	#except ImportError:
	1399	# pass
	1400

Note: See TracBrowser for help on using the repository browser.

Download in other formats: