Index: branches/pyyaml3000/lib/yaml/scanner.py =================================================================== --- branches/pyyaml3000/lib/yaml/scanner.py (revision 43) +++ branches/pyyaml3000/lib/yaml/scanner.py (revision 44) @@ -23,5 +23,8 @@ self.end_marker = end_marker -class YAMLDirective(Token): +class DirectiveToken(Token): + pass + +class YAMLDirectiveToken(DirectiveToken): def __init__(self, major_version, minor_version, start_marker, end_marker): self.major_version = major_version @@ -30,8 +33,8 @@ self.end_marker = end_marker -class TagDirective(Token): - pass - -class ReservedDirective(Token): +class TagDirectiveToken(DirectiveToken): + pass + +class ReservedDirectiveToken(DirectiveToken): def __init__(self, name, start_marker, end_marker): self.name = name @@ -39,44 +42,44 @@ self.end_marker = end_marker -class DocumentStart(Token): - pass - -class DocumentEnd(Token): - pass - -class End(Token): - pass - -class BlockSequenceStart(Token): - pass - -class BlockMappingStart(Token): - pass - -class BlockEnd(Token): - pass - -class FlowSequenceStart(Token): - pass - -class FlowMappingStart(Token): - pass - -class FlowSequenceEnd(Token): - pass - -class FlowMappingEnd(Token): - pass - -class Key(Token): - pass - -class Value(Token): - pass - -class Entry(Token): - pass - -class Alias(Token): +class DocumentStartToken(Token): + pass + +class DocumentEndToken(Token): + pass + +class EndToken(Token): + pass + +class BlockSequenceStartToken(Token): + pass + +class BlockMappingStartToken(Token): + pass + +class BlockEndToken(Token): + pass + +class FlowSequenceStartToken(Token): + pass + +class FlowMappingStartToken(Token): + pass + +class FlowSequenceEndToken(Token): + pass + +class FlowMappingEndToken(Token): + pass + +class KeyToken(Token): + pass + +class ValueToken(Token): + pass + +class EntryToken(Token): + pass + +class AliasToken(Token): def __init__(self, value, start_marker, end_marker): self.value = value @@ -84,5 +87,5 @@ self.end_marker = end_marker -class Anchor(Token): +class AnchorToken(Token): def __init__(self, value, start_marker, end_marker): self.value = value @@ -90,5 +93,5 @@ self.end_marker = end_marker -class Tag(Token): +class TagToken(Token): def __init__(self, value, start_marker, end_marker): self.value = value @@ -96,5 +99,5 @@ self.end_marker = end_marker -class Scalar(Token): +class ScalarToken(Token): def __init__(self, value, plain, start_marker, end_marker): self.value = value @@ -380,5 +383,5 @@ marker = self.stream.get_marker() self.indent = self.indents.pop() - self.tokens.append(BlockEnd(marker, marker)) + self.tokens.append(BlockEndToken(marker, marker)) def add_indent(self, column): @@ -405,5 +408,5 @@ # Add END. - self.tokens.append(End(marker, marker)) + self.tokens.append(EndToken(marker, marker)) # The stream is ended. @@ -423,8 +426,8 @@ def fetch_document_start(self): - self.fetch_document_indicator(DocumentStart) + self.fetch_document_indicator(DocumentStartToken) def fetch_document_end(self): - self.fetch_document_indicator(DocumentEnd) + self.fetch_document_indicator(DocumentEndToken) def fetch_document_indicator(self, TokenClass): @@ -445,16 +448,16 @@ def fetch_flow_sequence_start(self): - self.fetch_flow_collection_start(FlowSequenceStart) + self.fetch_flow_collection_start(FlowSequenceStartToken) def fetch_flow_mapping_start(self): - self.fetch_flow_collection_start(FlowMappingStart) + self.fetch_flow_collection_start(FlowMappingStartToken) def fetch_flow_collection_start(self, TokenClass): + + # '[' and '{' may start a simple key. + self.save_possible_simple_key() # Increase the flow level. self.flow_level += 1 - - # '[' and '{' may start a simple key. - self.save_possible_simple_key() # Simple keys are allowed after '[' and '{'. @@ -468,8 +471,8 @@ def fetch_flow_sequence_end(self): - self.fetch_flow_collection_end(FlowSequenceEnd) + self.fetch_flow_collection_end(FlowSequenceEndToken) def fetch_flow_mapping_end(self): - self.fetch_flow_collection_end(FlowMappingEnd) + self.fetch_flow_collection_end(FlowMappingEndToken) def fetch_flow_collection_end(self, TokenClass): @@ -502,5 +505,5 @@ if self.add_indent(self.stream.column): marker = self.stream.get_marker() - self.tokens.append(BlockSequenceStart(marker, marker)) + self.tokens.append(BlockSequenceStartToken(marker, marker)) # Simple keys are allowed after '-' and ','. @@ -514,5 +517,5 @@ self.stream.read() end_marker = self.stream.get_marker() - self.tokens.append(Entry(start_marker, end_marker)) + self.tokens.append(EntryToken(start_marker, end_marker)) def fetch_key(self): @@ -528,5 +531,5 @@ if self.add_indent(self.stream.column): marker = self.stream.get_marker() - self.tokens.append(BlockMappingStart(marker, marker)) + self.tokens.append(BlockMappingStartToken(marker, marker)) # Simple keys are allowed after '?' in the block context. @@ -540,5 +543,5 @@ self.stream.read() end_marker = self.stream.get_marker() - self.tokens.append(Key(start_marker, end_marker)) + self.tokens.append(KeyToken(start_marker, end_marker)) def fetch_value(self): @@ -551,5 +554,5 @@ del self.possible_simple_keys[self.flow_level] self.tokens.insert(key.token_number-self.tokens_taken, - Key(key.marker, key.marker)) + KeyToken(key.marker, key.marker)) # If this key starts a new block mapping, we need to add @@ -558,5 +561,5 @@ if self.add_indent(key.column): self.tokens.insert(key.token_number-self.tokens_taken, - BlockMappingStart(key.marker, key.marker)) + BlockMappingStartToken(key.marker, key.marker)) # There cannot be two simple keys one after another. @@ -576,5 +579,5 @@ self.stream.read() end_marker = self.stream.get_marker() - self.tokens.append(Value(start_marker, end_marker)) + self.tokens.append(ValueToken(start_marker, end_marker)) def fetch_alias(self): @@ -587,5 +590,5 @@ # Scan and add ALIAS. - self.scan_anchor(Alias) + self.scan_anchor(AliasToken) def fetch_anchor(self): @@ -598,5 +601,5 @@ # Scan and add ANCHOR. - self.scan_anchor(Anchor) + self.scan_anchor(AnchorToken) def fetch_tag(self): @@ -739,9 +742,9 @@ marker = self.stream.get_marker() if self.stream.peek(5) == u'%YAML ': - self.tokens.append(YAMLDirective(1, 1, marker, marker)) + self.tokens.append(YAMLDirectiveToken(1, 1, marker, marker)) elif self.stream.peek(4) == u'%TAG ': - self.tokens.append(TagDirective(marker, marker)) + self.tokens.append(TagDirectiveToken(marker, marker)) else: - self.tokens.append(ReservedDirective('', marker, marker)) + self.tokens.append(ReservedDirectiveToken('', marker, marker)) while self.stream.peek() not in u'\0\r\n': self.stream.read() @@ -760,5 +763,5 @@ self.stream.read() end_marker = self.stream.get_marker() - self.tokens.append(Tag('', start_marker, end_marker)) + self.tokens.append(TagToken('', start_marker, end_marker)) def scan_block_scalar(self, folded): @@ -768,5 +771,5 @@ indent = 1 while True: - while self.stream.peek() and self.stream.peek() and self.stream.peek() not in u'\0\r\n': + while self.stream.peek() and self.stream.peek() and self.stream.peek() not in u'\0\r\n\x85\u2028\u2029': self.stream.read() if self.stream.peek() != u'\0': @@ -776,7 +779,7 @@ self.stream.read() count += 1 - if count < indent and self.stream.peek() not in u'#\r\n': + if count < indent and self.stream.peek() not in u'#\r\n\x85\u2028\u2029': break - self.tokens.append(Scalar('', False, start_marker, start_marker)) + self.tokens.append(ScalarToken('', False, start_marker, start_marker)) def scan_flow_scalar(self, double): @@ -791,5 +794,5 @@ self.stream.read(1) self.stream.read(1) - self.tokens.append(Scalar('', False, marker, marker)) + self.tokens.append(ScalarToken('', False, marker, marker)) def scan_plain(self): @@ -823,5 +826,5 @@ break space = True - self.tokens.append(Scalar('', True, marker, marker)) + self.tokens.append(ScalarToken('', True, marker, marker)) def invalid_token(self):