Index: /pyyaml/trunk/examples/yaml-hl/yaml_hl_html.cfg =================================================================== --- /pyyaml/trunk/examples/yaml-hl/yaml_hl_html.cfg (revision 119) +++ /pyyaml/trunk/examples/yaml-hl/yaml_hl_html.cfg (revision 130) @@ -27,7 +27,8 @@ document-start: { start: } document-end: { end: } - sequence: { start: } - mapping: { start: } - collection-end: { end: } + sequence-start: { start: } + sequence-end: { end: } + mapping-start: { start: } + mapping-end: { end: } scalar: { start: , end: } Index: /pyyaml/trunk/examples/yaml-hl/yaml_hl_ascii.cfg =================================================================== --- /pyyaml/trunk/examples/yaml-hl/yaml_hl_ascii.cfg (revision 119) +++ /pyyaml/trunk/examples/yaml-hl/yaml_hl_ascii.cfg (revision 130) @@ -28,7 +28,8 @@ document-start: document-end: - sequence: - mapping: - collection-end: + sequence-start: + sequence-end: + mapping-start: + mapping-end: scalar: Index: /pyyaml/trunk/tests/test_appliance.py =================================================================== --- /pyyaml/trunk/tests/test_appliance.py (revision 118) +++ /pyyaml/trunk/tests/test_appliance.py (revision 130) @@ -244,8 +244,8 @@ self.events.append(ScalarEvent(anchor, tag, self.get_value(), None, None)) elif self.test_token(FlowSequenceStartToken): - self.events.append(SequenceEvent(anchor, tag, None, None)) + self.events.append(SequenceStartEvent(anchor, tag, None, None)) self.parse_sequence() elif self.test_token(FlowMappingStartToken): - self.events.append(MappingEvent(anchor, tag, None, None)) + self.events.append(MappingStartEvent(anchor, tag, None, None)) self.parse_mapping() else: @@ -262,5 +262,5 @@ self.parse_node() self.consume_token(FlowSequenceEndToken) - self.events.append(CollectionEndEvent(None, None)) + self.events.append(SequenceEndEvent(None, None)) # mapping: MAPPING-START (map_entry (ENTRY map_entry)*)? ENTRY? MAPPING-END @@ -274,5 +274,5 @@ self.parse_map_entry() self.consume_token(FlowMappingEndToken) - self.events.append(CollectionEndEvent(None, None)) + self.events.append(MappingEndEvent(None, None)) # map_entry: KEY node VALUE node Index: /pyyaml/trunk/tests/test_structure.py =================================================================== --- /pyyaml/trunk/tests/test_structure.py (revision 118) +++ /pyyaml/trunk/tests/test_structure.py (revision 130) @@ -36,15 +36,15 @@ else: return None - elif parser.check(SequenceEvent): + elif parser.check(SequenceStartEvent): parser.get() sequence = [] - while not parser.check(CollectionEndEvent): + while not parser.check(SequenceEndEvent): sequence.append(self._convert(parser)) parser.get() return sequence - elif parser.check(MappingEvent): + elif parser.check(MappingStartEvent): parser.get() mapping = [] - while not parser.check(CollectionEndEvent): + while not parser.check(MappingEndEvent): key = self._convert(parser) value = self._convert(parser) @@ -93,5 +93,5 @@ #self.failUnlessEqual(event1.tag, event2.tag) self.failUnlessEqual(event1.value, event2.value) - if isinstance(event1, CollectionEvent): + if isinstance(event1, CollectionStartEvent): #self.failUnlessEqual(event1.anchor, event2.anchor) #self.failUnlessEqual(event1.tag, event2.tag) @@ -214,5 +214,5 @@ self.failUnlessEqual(event1.tag, event2.tag) self.failUnlessEqual(event1.value, event2.value) - if isinstance(event1, CollectionEvent): + if isinstance(event1, CollectionStartEvent): self.failUnlessEqual(event1.anchor, event2.anchor) self.failUnlessEqual(event1.tag, event2.tag) Index: /pyyaml/trunk/tests/test_yaml.py =================================================================== --- /pyyaml/trunk/tests/test_yaml.py (revision 116) +++ /pyyaml/trunk/tests/test_yaml.py (revision 130) @@ -10,4 +10,5 @@ from test_detector import * from test_constructor import * +#from test_emitter import * from test_syck import * Index: /pyyaml/trunk/tests/data/documents.events =================================================================== --- /pyyaml/trunk/tests/data/documents.events (revision 130) +++ /pyyaml/trunk/tests/data/documents.events (revision 130) @@ -0,0 +1,11 @@ +- !StreamStart +- !DocumentStart +- !Scalar { implicit: true } +- !DocumentEnd +- !DocumentStart +- !Scalar { implicit: true } +- !DocumentEnd +- !DocumentStart +- !Scalar { implicit: true } +- !DocumentEnd +- !StreamEnd Index: /pyyaml/trunk/tests/data/sequences.events =================================================================== --- /pyyaml/trunk/tests/data/sequences.events (revision 130) +++ /pyyaml/trunk/tests/data/sequences.events (revision 130) @@ -0,0 +1,81 @@ +- !StreamStart + +- !DocumentStart +- !SequenceStart +- !SequenceEnd +- !DocumentEnd + +- !DocumentStart +- !SequenceStart { tag: '!mytag' } +- !SequenceEnd +- !DocumentEnd + +- !DocumentStart +- !SequenceStart +- !SequenceStart +- !SequenceEnd +- !SequenceStart { tag: '!mytag' } +- !SequenceEnd +- !SequenceStart +- !Scalar +- !Scalar { value: 'data' } +- !Scalar { tag: '!mytag', value: 'data' } +- !SequenceEnd +- !SequenceStart +- !SequenceStart +- !SequenceStart +- !Scalar +- !SequenceEnd +- !SequenceEnd +- !SequenceEnd +- !SequenceStart +- !SequenceStart { tag: '!mytag' } +- !SequenceStart +- !Scalar { value: 'data' } +- !SequenceEnd +- !SequenceEnd +- !SequenceEnd +- !SequenceEnd +- !DocumentEnd + +- !DocumentStart +- !SequenceStart +- !MappingStart +- !Scalar { value: 'key1' } +- !SequenceStart +- !Scalar { value: 'data1' } +- !Scalar { value: 'data2' } +- !SequenceEnd +- !Scalar { value: 'key2' } +- !SequenceStart { tag: '!mytag1' } +- !Scalar { value: 'data3' } +- !SequenceStart +- !Scalar { value: 'data4' } +- !Scalar { value: 'data5' } +- !SequenceEnd +- !SequenceStart { tag: '!mytag2' } +- !Scalar { value: 'data6' } +- !Scalar { value: 'data7' } +- !SequenceEnd +- !SequenceEnd +- !MappingEnd +- !SequenceEnd +- !DocumentEnd + +- !DocumentStart +- !SequenceStart +- !SequenceStart { flow: true } +- !SequenceStart +- !SequenceEnd +- !Scalar +- !Scalar { value: 'data' } +- !Scalar { tag: '!mytag', value: 'data' } +- !SequenceStart { tag: '!mytag' } +- !Scalar { value: 'data' } +- !Scalar { value: 'data' } +- !SequenceEnd +- !SequenceEnd +- !SequenceEnd +- !DocumentEnd + +- !StreamEnd Index: /pyyaml/trunk/tests/data/mappings.events =================================================================== --- /pyyaml/trunk/tests/data/mappings.events (revision 130) +++ /pyyaml/trunk/tests/data/mappings.events (revision 130) @@ -0,0 +1,44 @@ +- !StreamStart + +- !DocumentStart +- !MappingStart +- !Scalar { implicit: true, value: 'key' } +- !Scalar { implicit: true, value: 'value' } +- !Scalar { implicit: true, value: 'empty mapping' } +- !MappingStart +- !MappingEnd +- !Scalar { implicit: true, value: 'empty mapping with tag' } +- !MappingStart { tag: '!mytag' } +- !MappingEnd +- !Scalar { implicit: true, value: 'block mapping' } +- !MappingStart +- !MappingStart +- !Scalar { implicit: true, value: 'complex' } +- !Scalar { implicit: true, value: 'key' } +- !Scalar { implicit: true, value: 'complex' } +- !Scalar { implicit: true, value: 'key' } +- !MappingEnd +- !MappingStart +- !Scalar { implicit: true, value: 'complex' } +- !Scalar { implicit: true, value: 'key' } +- !MappingEnd +- !MappingEnd +- !Scalar { implicit: true, value: 'flow mapping' } +- !MappingStart { flow: true } +- !Scalar { implicit: true, value: 'key' } +- !Scalar { implicit: true, value: 'value' } +- !MappingStart +- !Scalar { implicit: true, value: 'complex' } +- !Scalar { implicit: true, value: 'key' } +- !Scalar { implicit: true, value: 'complex' } +- !Scalar { implicit: true, value: 'key' } +- !MappingEnd +- !MappingStart +- !Scalar { implicit: true, value: 'complex' } +- !Scalar { implicit: true, value: 'key' } +- !MappingEnd +- !MappingEnd +- !MappingEnd +- !DocumentEnd + +- !StreamEnd Index: /pyyaml/trunk/tests/data/spec-08-12.canonical =================================================================== --- /pyyaml/trunk/tests/data/spec-08-12.canonical (revision 38) +++ /pyyaml/trunk/tests/data/spec-08-12.canonical (revision 130) @@ -7,3 +7,4 @@ *A, !!str "", + !!str "", ] Index: /pyyaml/trunk/tests/data/scalars.events =================================================================== --- /pyyaml/trunk/tests/data/scalars.events (revision 130) +++ /pyyaml/trunk/tests/data/scalars.events (revision 130) @@ -0,0 +1,24 @@ +- !StreamStart + +- !DocumentStart +- !MappingStart +- !Scalar { implicit: true, value: 'empty scalar' } +- !Scalar { implicit: true, value: '' } +- !Scalar { implicit: true, value: 'implicit scalar' } +- !Scalar { implicit: true, value: 'data' } +- !Scalar { implicit: true, value: 'quoted scalar' } +- !Scalar { value: 'data', style: '"' } +- !Scalar { implicit: true, value: 'block scalar' } +- !Scalar { value: 'data', style: '|' } +- !Scalar { implicit: true, value: 'empty scalar with tag' } +- !Scalar { implicit: true, tag: '!mytag', value: '' } +- !Scalar { implicit: true, value: 'implicit scalar with tag' } +- !Scalar { implicit: true, tag: '!mytag', value: 'data' } +- !Scalar { implicit: true, value: 'quoted scalar with tag' } +- !Scalar { value: 'data', style: '"', tag: '!mytag' } +- !Scalar { implicit: true, value: 'block scalar with tag' } +- !Scalar { value: 'data', style: '|', tag: '!mytag' } +- !MappingEnd +- !DocumentEnd + +- !StreamEnd Index: /pyyaml/trunk/tests/data/spec-08-12.data =================================================================== --- /pyyaml/trunk/tests/data/spec-08-12.data (revision 44) +++ /pyyaml/trunk/tests/data/spec-08-12.data (revision 130) @@ -4,5 +4,5 @@ !!str 'Tagged', *anchor, # Alias node -# !!str, # Empty plain scalar + !!str , # Empty plain scalar '', # Empty plain scalar ] Index: /pyyaml/trunk/tests/data/aliases.events =================================================================== --- /pyyaml/trunk/tests/data/aliases.events (revision 130) +++ /pyyaml/trunk/tests/data/aliases.events (revision 130) @@ -0,0 +1,8 @@ +- !StreamStart +- !DocumentStart +- !SequenceStart +- !Scalar { anchor: 'myanchor', tag: '!mytag', value: 'data' } +- !Alias { anchor: 'myanchor' } +- !SequenceEnd +- !DocumentEnd +- !StreamEnd Index: /pyyaml/trunk/tests/test_emitter.py =================================================================== --- /pyyaml/trunk/tests/test_emitter.py (revision 130) +++ /pyyaml/trunk/tests/test_emitter.py (revision 130) @@ -0,0 +1,73 @@ + +import test_appliance, sys, StringIO + +from yaml import * +import yaml + +class TestEmitterOnCanonical(test_appliance.TestAppliance): + + def _testEmitterOnCanonical(self, test_name, canonical_filename): + events = list(iter(Parser(Scanner(Reader(file(canonical_filename, 'rb')))))) + #writer = sys.stdout + writer = StringIO.StringIO() + emitter = Emitter(writer) + #print "-"*30 + #print "ORIGINAL DATA:" + #print file(canonical_filename, 'rb').read() + for event in events: + emitter.emit(event) + data = writer.getvalue() + new_events = list(parse(data)) + self.failUnlessEqual(len(events), len(new_events)) + for event, new_event in zip(events, new_events): + self.failUnlessEqual(event.__class__, new_event.__class__) + +TestEmitterOnCanonical.add_tests('testEmitterOnCanonical', '.canonical') + +class EventsConstructor(Constructor): + + def construct_event(self, node): + if isinstance(node, ScalarNode): + mapping = {} + else: + mapping = self.construct_mapping(node) + class_name = str(node.tag[1:])+'Event' + if class_name in ['AliasEvent', 'ScalarEvent', 'SequenceStartEvent', 'MappingStartEvent']: + mapping.setdefault('anchor', None) + if class_name in ['ScalarEvent', 'SequenceStartEvent', 'MappingStartEvent']: + mapping.setdefault('tag', None) + if class_name == 'ScalarEvent': + mapping.setdefault('value', '') + value = getattr(yaml, class_name)(**mapping) + return value + +EventsConstructor.add_constructor(None, EventsConstructor.construct_event) + +class TestEmitter(test_appliance.TestAppliance): + + def _testEmitter(self, test_name, events_filename): + events = load_document(file(events_filename, 'rb'), Constructor=EventsConstructor) + self._dump(events_filename, events) + writer = StringIO.StringIO() + emitter = Emitter(writer) + for event in events: + emitter.emit(event) + data = writer.getvalue() + new_events = list(parse(data)) + self.failUnlessEqual(len(events), len(new_events)) + for event, new_event in zip(events, new_events): + self.failUnlessEqual(event.__class__, new_event.__class__) + + def _dump(self, events_filename, events): + writer = sys.stdout + emitter = Emitter(writer) + print "="*30 + print "EVENTS:" + print file(events_filename, 'rb').read() + print '-'*30 + print "OUTPUT:" + for event in events: + emitter.emit(event) + +TestEmitter.add_tests('testEmitter', '.events') + Index: /pyyaml/trunk/lib/yaml/tokens.py =================================================================== --- /pyyaml/trunk/lib/yaml/tokens.py (revision 118) +++ /pyyaml/trunk/lib/yaml/tokens.py (revision 130) @@ -31,4 +31,9 @@ class StreamStartToken(Token): id = '' + def __init__(self, start_mark=None, end_mark=None, + encoding=None): + self.start_mark = start_mark + self.end_mark = end_mark + self.encoding = encoding class StreamEndToken(Token): @@ -91,8 +96,9 @@ class ScalarToken(Token): id = '' - def __init__(self, value, plain, start_mark, end_mark): + def __init__(self, value, implicit, start_mark, end_mark, style=None): self.value = value - self.plain = plain + self.implicit = implicit self.start_mark = start_mark self.end_mark = end_mark + self.style = style Index: /pyyaml/trunk/lib/yaml/events.py =================================================================== --- /pyyaml/trunk/lib/yaml/events.py (revision 118) +++ /pyyaml/trunk/lib/yaml/events.py (revision 130) @@ -1,11 +1,12 @@ + +# Abstract classes. class Event: - def __init__(self, start_mark, end_mark): + def __init__(self, start_mark=None, end_mark=None): self.start_mark = start_mark self.end_mark = end_mark def __repr__(self): - attributes = [key for key in self.__dict__ - if not key.endswith('_mark')] - attributes.sort() + attributes = [key for key in ['anchor', 'tag', 'value'] + if hasattr(self, key)] arguments = ', '.join(['%s=%r' % (key, getattr(self, key)) for key in attributes]) @@ -13,8 +14,51 @@ class NodeEvent(Event): - def __init__(self, anchor, start_mark, end_mark): + def __init__(self, anchor, start_mark=None, end_mark=None): self.anchor = anchor self.start_mark = start_mark self.end_mark = end_mark + +class CollectionStartEvent(NodeEvent): + def __init__(self, anchor, tag, start_mark=None, end_mark=None, + flow_style=None): + self.anchor = anchor + self.tag = tag + self.start_mark = start_mark + self.end_mark = end_mark + self.flow_style = flow_style + +class CollectionEndEvent(Event): + pass + +# Implementations. + +class StreamStartEvent(Event): + def __init__(self, start_mark=None, end_mark=None, + encoding=None, canonical=None, indent=None, width=None): + self.start_mark = start_mark + self.end_mark = end_mark + self.encoding = encoding + self.canonical = canonical + self.indent = indent + self.width = width + +class StreamEndEvent(Event): + pass + +class DocumentStartEvent(Event): + def __init__(self, start_mark=None, end_mark=None, + implicit=None, version=None, tags=None): + self.start_mark = start_mark + self.end_mark = end_mark + self.implicit = implicit + self.version = version + self.tags = tags + +class DocumentEndEvent(Event): + def __init__(self, start_mark=None, end_mark=None, + implicit=None): + self.start_mark = start_mark + self.end_mark = end_mark + self.implicit = implicit class AliasEvent(NodeEvent): @@ -22,5 +66,6 @@ class ScalarEvent(NodeEvent): - def __init__(self, anchor, tag, value, start_mark, end_mark): + def __init__(self, anchor, tag, value, start_mark=None, end_mark=None, + implicit=None, style=None): self.anchor = anchor self.tag = tag @@ -28,31 +73,17 @@ self.start_mark = start_mark self.end_mark = end_mark + self.implicit = implicit + self.style = style -class CollectionEvent(NodeEvent): - def __init__(self, anchor, tag, start_mark, end_mark): - self.anchor = anchor - self.tag = tag - self.start_mark = start_mark - self.end_mark = end_mark - -class SequenceEvent(CollectionEvent): +class SequenceStartEvent(CollectionStartEvent): pass -class MappingEvent(CollectionEvent): +class SequenceEndEvent(CollectionEndEvent): pass -class CollectionEndEvent(Event): +class MappingStartEvent(CollectionStartEvent): pass -class DocumentStartEvent(Event): +class MappingEndEvent(CollectionEndEvent): pass -class DocumentEndEvent(Event): - pass - -class StreamStartEvent(Event): - pass - -class StreamEndEvent(Event): - pass - Index: /pyyaml/trunk/lib/yaml/composer.py =================================================================== --- /pyyaml/trunk/lib/yaml/composer.py (revision 118) +++ /pyyaml/trunk/lib/yaml/composer.py (revision 130) @@ -72,7 +72,7 @@ if self.parser.check(ScalarEvent): node = self.compose_scalar_node() - elif self.parser.check(SequenceEvent): + elif self.parser.check(SequenceStartEvent): node = self.compose_sequence_node() - elif self.parser.check(MappingEvent): + elif self.parser.check(MappingStartEvent): node = self.compose_mapping_node() if anchor is not None: @@ -88,5 +88,5 @@ start_event = self.parser.get() value = [] - while not self.parser.check(CollectionEndEvent): + while not self.parser.check(SequenceEndEvent): value.append(self.compose_node()) end_event = self.parser.get() @@ -97,5 +97,5 @@ start_event = self.parser.get() value = {} - while not self.parser.check(CollectionEndEvent): + while not self.parser.check(MappingEndEvent): key_event = self.parser.peek() item_key = self.compose_node() Index: /pyyaml/trunk/lib/yaml/scanner.py =================================================================== --- /pyyaml/trunk/lib/yaml/scanner.py (revision 118) +++ /pyyaml/trunk/lib/yaml/scanner.py (revision 130) @@ -380,6 +380,7 @@ mark = self.reader.get_mark() - # Add STREAM-END. - self.tokens.append(StreamStartToken(mark, mark)) + # Add STREAM-START. + self.tokens.append(StreamStartToken(mark, mark, + encoding=self.reader.encoding)) @@ -639,10 +640,10 @@ def fetch_literal(self): - self.fetch_block_scalar(folded=False) + self.fetch_block_scalar(style='|') def fetch_folded(self): - self.fetch_block_scalar(folded=True) - - def fetch_block_scalar(self, folded): + self.fetch_block_scalar(style='>') + + def fetch_block_scalar(self, style): # A simple key may follow a block scalar. @@ -653,13 +654,13 @@ # Scan and add SCALAR. - self.tokens.append(self.scan_block_scalar(folded)) + self.tokens.append(self.scan_block_scalar(style)) def fetch_single(self): - self.fetch_flow_scalar(double=False) + self.fetch_flow_scalar(style='\'') def fetch_double(self): - self.fetch_flow_scalar(double=True) - - def fetch_flow_scalar(self, double): + self.fetch_flow_scalar(style='"') + + def fetch_flow_scalar(self, style): # A flow scalar could be a simple key. @@ -670,5 +671,5 @@ # Scan and add SCALAR. - self.tokens.append(self.scan_flow_scalar(double)) + self.tokens.append(self.scan_flow_scalar(style)) def fetch_plain(self): @@ -987,6 +988,11 @@ return TagToken(value, start_mark, end_mark) - def scan_block_scalar(self, folded): - # See the specification for details. + def scan_block_scalar(self, style): + # See the specification for details. + + if style == '>': + folded = True + else: + folded = False chunks = [] @@ -1022,4 +1028,5 @@ breaks, end_mark = self.scan_block_scalar_breaks(indent) if self.reader.column == indent and self.reader.peek() != u'\0': + # Unfortunately, folding rules are ambiguous. # @@ -1054,5 +1061,6 @@ # We are done. - return ScalarToken(u''.join(chunks), False, start_mark, end_mark) + return ScalarToken(u''.join(chunks), False, start_mark, end_mark, + style) def scan_block_scalar_indicators(self, start_mark): @@ -1138,5 +1146,5 @@ return chunks, end_mark - def scan_flow_scalar(self, double): + def scan_flow_scalar(self, style): # See the specification for details. # Note that we loose indentation rules for quoted scalars. Quoted @@ -1145,4 +1153,8 @@ # restrictive then the specification requires. We only need to check # that document separators are not included in scalars. + if style == '"': + double = True + else: + double = False chunks = [] start_mark = self.reader.get_mark() @@ -1155,5 +1167,6 @@ self.reader.forward() end_mark = self.reader.get_mark() - return ScalarToken(u''.join(chunks), False, start_mark, end_mark) + return ScalarToken(u''.join(chunks), False, start_mark, end_mark, + style) ESCAPE_REPLACEMENTS = { Index: /pyyaml/trunk/lib/yaml/parser.py =================================================================== --- /pyyaml/trunk/lib/yaml/parser.py (revision 118) +++ /pyyaml/trunk/lib/yaml/parser.py (revision 130) @@ -131,5 +131,6 @@ # Parse start of stream. token = self.scanner.get() - yield StreamStartEvent(token.start_mark, token.end_mark) + yield StreamStartEvent(token.start_mark, token.end_mark, + encoding=token.encoding) # Parse implicit document. @@ -139,13 +140,17 @@ token = self.scanner.peek() start_mark = end_mark = token.start_mark - yield DocumentStartEvent(start_mark, end_mark) + yield DocumentStartEvent(start_mark, end_mark, + implicit=True) for event in self.parse_block_node(): yield event token = self.scanner.peek() start_mark = end_mark = token.start_mark + implicit = True while self.scanner.check(DocumentEndToken): token = self.scanner.get() end_mark = token.end_mark - yield DocumentEndEvent(start_mark, end_mark) + implicit = True + yield DocumentEndEvent(start_mark, end_mark, + implicit=implicit) # Parse explicit documents. @@ -153,5 +158,5 @@ token = self.scanner.peek() start_mark = token.start_mark - self.process_directives() + version, tags = self.process_directives() if not self.scanner.check(DocumentStartToken): raise ParserError(None, None, @@ -161,5 +166,6 @@ token = self.scanner.get() end_mark = token.end_mark - yield DocumentStartEvent(start_mark, end_mark) + yield DocumentStartEvent(start_mark, end_mark, + implicit=False, version=version, tags=tags) if self.scanner.check(DirectiveToken, DocumentStartToken, DocumentEndToken, StreamEndToken): @@ -170,8 +176,11 @@ token = self.scanner.peek() start_mark = end_mark = token.start_mark + implicit=True while self.scanner.check(DocumentEndToken): token = self.scanner.get() end_mark = token.end_mark - yield DocumentEndEvent(start_mark, end_mark) + implicit=False + yield DocumentEndEvent(start_mark, end_mark, + implicit=implicit) # Parse end of stream. @@ -202,7 +211,12 @@ token.start_mark) self.tag_handles[handle] = prefix + if self.tag_handles: + value = self.yaml_version, self.tag_handles.copy() + else: + value = self.yaml_version, None for key in self.DEFAULT_TAGS: if key not in self.tag_handles: self.tag_handles[key] = self.DEFAULT_TAGS[key] + return value def parse_block_node(self): @@ -233,17 +247,20 @@ if self.scanner.check(AnchorToken): token = self.scanner.get() - start_mark = end_mark = token.start_mark + start_mark = token.start_mark + end_mark = token.end_mark anchor = token.value if self.scanner.check(TagToken): token = self.scanner.get() - end_mark = tag_mark = token.start_mark + tag_mark = token.start_mark + end_mark = token.end_mark tag = token.value elif self.scanner.check(TagToken): token = self.scanner.get() - start_mark = end_mark = tag_mark = token.start_mark + start_mark = tag_mark = token.start_mark + end_mark = token.end_mark tag = token.value if self.scanner.check(AnchorToken): token = self.scanner.get() - end_mark = token.start_mark + end_mark = token.end_mark anchor = token.value if tag is not None: @@ -259,13 +276,13 @@ if tag is None: if not (self.scanner.check(ScalarToken) and - self.scanner.peek().plain): + self.scanner.peek().implicit): tag = u'!' if start_mark is None: - start_mark = self.scanner.peek().start_mark + start_mark = end_mark = self.scanner.peek().start_mark event = None collection_events = None if indentless_sequence and self.scanner.check(BlockEntryToken): end_mark = self.scanner.peek().end_mark - event = SequenceEvent(anchor, tag, start_mark, end_mark) + event = SequenceStartEvent(anchor, tag, start_mark, end_mark) collection_events = self.parse_indentless_sequence() else: @@ -274,21 +291,31 @@ end_mark = token.end_mark event = ScalarEvent(anchor, tag, token.value, - start_mark, end_mark) + start_mark, end_mark, + implicit=token.implicit, style=token.style) elif self.scanner.check(FlowSequenceStartToken): end_mark = self.scanner.peek().end_mark - event = SequenceEvent(anchor, tag, start_mark, end_mark) + event = SequenceStartEvent(anchor, tag, start_mark, end_mark, + flow_style=True) collection_events = self.parse_flow_sequence() elif self.scanner.check(FlowMappingStartToken): end_mark = self.scanner.peek().end_mark - event = MappingEvent(anchor, tag, start_mark, end_mark) + event = MappingStartEvent(anchor, tag, start_mark, end_mark, + flow_style=True) collection_events = self.parse_flow_mapping() elif block and self.scanner.check(BlockSequenceStartToken): end_mark = self.scanner.peek().start_mark - event = SequenceEvent(anchor, tag, start_mark, end_mark) + event = SequenceStartEvent(anchor, tag, start_mark, end_mark, + flow_style=False) collection_events = self.parse_block_sequence() elif block and self.scanner.check(BlockMappingStartToken): end_mark = self.scanner.peek().start_mark - event = MappingEvent(anchor, tag, start_mark, end_mark) + event = MappingStartEvent(anchor, tag, start_mark, end_mark, + flow_style=False) collection_events = self.parse_block_mapping() + elif anchor is not None or tag is not None: + # Empty scalars are allowed even if a tag or an anchor is + # specified. + event = ScalarEvent(anchor, tag, u'', start_mark, end_mark, + implicit=True) else: if block: @@ -321,5 +348,5 @@ "expected , but found %r" % token.id, token.start_mark) token = self.scanner.get() - yield CollectionEndEvent(token.start_mark, token.end_mark) + yield SequenceEndEvent(token.start_mark, token.end_mark) def parse_indentless_sequence(self): @@ -334,5 +361,5 @@ yield self.process_empty_scalar(token.end_mark) token = self.scanner.peek() - yield CollectionEndEvent(token.start_mark, token.start_mark) + yield SequenceEndEvent(token.start_mark, token.start_mark) def parse_block_mapping(self): @@ -366,5 +393,5 @@ "expected , but found %r" % token.id, token.start_mark) token = self.scanner.get() - yield CollectionEndEvent(token.start_mark, token.end_mark) + yield MappingEndEvent(token.start_mark, token.end_mark) def parse_flow_sequence(self): @@ -384,6 +411,7 @@ if self.scanner.check(KeyToken): token = self.scanner.get() - yield MappingEvent(None, u'!', - token.start_mark, token.end_mark) + yield MappingStartEvent(None, u'!', + token.start_mark, token.end_mark, + flow_style=True) if not self.scanner.check(ValueToken, FlowEntryToken, FlowSequenceEndToken): @@ -403,5 +431,5 @@ yield self.process_empty_scalar(token.start_mark) token = self.scanner.peek() - yield CollectionEndEvent(token.start_mark, token.start_mark) + yield MappingEndEvent(token.start_mark, token.start_mark) else: for event in self.parse_flow_node(): @@ -414,5 +442,5 @@ self.scanner.get() token = self.scanner.get() - yield CollectionEndEvent(token.start_mark, token.end_mark) + yield SequenceEndEvent(token.start_mark, token.end_mark) def parse_flow_mapping(self): @@ -458,7 +486,8 @@ "expected '}', but found %r" % token.id, token.start_mark) token = self.scanner.get() - yield CollectionEndEvent(token.start_mark, token.end_mark) + yield MappingEndEvent(token.start_mark, token.end_mark) def process_empty_scalar(self, mark): - return ScalarEvent(None, None, u'', mark, mark) - + return ScalarEvent(None, None, u'', mark, mark, + implicit=True) + Index: /pyyaml/trunk/lib/yaml/reader.py =================================================================== --- /pyyaml/trunk/lib/yaml/reader.py (revision 119) +++ /pyyaml/trunk/lib/yaml/reader.py (revision 130) @@ -101,4 +101,5 @@ self.raw_buffer = None self.raw_decode = None + self.encoding = None self.index = 0 self.line = 0 @@ -157,8 +158,11 @@ if self.raw_buffer.startswith(codecs.BOM_UTF16_LE): self.raw_decode = utf_16_le_decode + self.encoding = 'utf-16-le' elif self.raw_buffer.startswith(codecs.BOM_UTF16_BE): self.raw_decode = utf_16_be_decode + self.encoding = 'utf-16-be' else: self.raw_decode = utf_8_decode + self.encoding = 'utf-8' self.update(1)