Index: /pyyaml/branches/working-on-emitter/tests/data/spec-08-12.data =================================================================== --- /pyyaml/branches/working-on-emitter/tests/data/spec-08-12.data (revision 44) +++ /pyyaml/branches/working-on-emitter/tests/data/spec-08-12.data (revision 127) @@ -4,5 +4,5 @@ !!str 'Tagged', *anchor, # Alias node -# !!str, # Empty plain scalar + !!str , # Empty plain scalar '', # Empty plain scalar ] Index: /pyyaml/branches/working-on-emitter/tests/data/spec-08-12.canonical =================================================================== --- /pyyaml/branches/working-on-emitter/tests/data/spec-08-12.canonical (revision 38) +++ /pyyaml/branches/working-on-emitter/tests/data/spec-08-12.canonical (revision 127) @@ -7,3 +7,4 @@ *A, !!str "", + !!str "", ] Index: /pyyaml/branches/working-on-emitter/tests/test_emitter.py =================================================================== --- /pyyaml/branches/working-on-emitter/tests/test_emitter.py (revision 122) +++ /pyyaml/branches/working-on-emitter/tests/test_emitter.py (revision 127) @@ -1,4 +1,4 @@ -import test_appliance, sys +import test_appliance, sys, StringIO from yaml import * @@ -8,9 +8,10 @@ def _testEmitterOnCanonical(self, test_name, canonical_filename): events = list(iter(Parser(Scanner(Reader(file(canonical_filename, 'rb')))))) - writer = sys.stdout + #writer = sys.stdout + writer = StringIO.StringIO() emitter = Emitter(writer) - print "-"*30 - print "ORIGINAL DATA:" - print file(canonical_filename, 'rb').read() + #print "-"*30 + #print "ORIGINAL DATA:" + #print file(canonical_filename, 'rb').read() for event in events: emitter.emit(event) Index: /pyyaml/branches/working-on-emitter/lib/yaml/reader.py =================================================================== --- /pyyaml/branches/working-on-emitter/lib/yaml/reader.py (revision 121) +++ /pyyaml/branches/working-on-emitter/lib/yaml/reader.py (revision 127) @@ -101,4 +101,5 @@ self.raw_buffer = None self.raw_decode = None + self.encoding = None self.index = 0 self.line = 0 @@ -157,8 +158,11 @@ if self.raw_buffer.startswith(codecs.BOM_UTF16_LE): self.raw_decode = utf_16_le_decode + self.encoding = 'utf-16-le' elif self.raw_buffer.startswith(codecs.BOM_UTF16_BE): self.raw_decode = utf_16_be_decode + self.encoding = 'utf-16-be' else: self.raw_decode = utf_8_decode + self.encoding = 'utf-8' self.update(1) Index: /pyyaml/branches/working-on-emitter/lib/yaml/tokens.py =================================================================== --- /pyyaml/branches/working-on-emitter/lib/yaml/tokens.py (revision 122) +++ /pyyaml/branches/working-on-emitter/lib/yaml/tokens.py (revision 127) @@ -25,4 +25,9 @@ class DocumentStartToken(Token): id = '' + def __init__(self, name, value, start_mark=None, end_mark=None): + self.name = name + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark class DocumentEndToken(Token): @@ -31,4 +36,9 @@ class StreamStartToken(Token): id = '' + def __init__(self, start_mark=None, end_mark=None, + encoding=None): + self.start_mark = start_mark + self.end_mark = end_mark + self.encoding = encoding class StreamEndToken(Token): @@ -37,7 +47,17 @@ class BlockSequenceStartToken(Token): id = '' + def __init__(self, start_mark=None, end_mark=None, + inline=None): + self.start_mark = start_mark + self.end_mark = end_mark + self.inline = inline class BlockMappingStartToken(Token): id = '' + def __init__(self, start_mark=None, end_mark=None, + inline=None): + self.start_mark = start_mark + self.end_mark = end_mark + self.inline = inline class BlockEndToken(Token): @@ -91,8 +111,10 @@ class ScalarToken(Token): id = '' - def __init__(self, value, plain, start_mark=None, end_mark=None): + def __init__(self, value, plain, start_mark=None, end_mark=None, + style=None): self.value = value self.plain = plain self.start_mark = start_mark self.end_mark = end_mark + self.style = style Index: /pyyaml/branches/working-on-emitter/lib/yaml/events.py =================================================================== --- /pyyaml/branches/working-on-emitter/lib/yaml/events.py (revision 122) +++ /pyyaml/branches/working-on-emitter/lib/yaml/events.py (revision 127) @@ -21,5 +21,6 @@ class ScalarEvent(NodeEvent): - def __init__(self, anchor, tag, value, start_mark=None, end_mark=None): + def __init__(self, anchor, tag, value, start_mark=None, end_mark=None, + implicit=None, style=None): self.anchor = anchor self.tag = tag @@ -27,11 +28,16 @@ self.start_mark = start_mark self.end_mark = end_mark + self.implicit = implicit + self.style = style class CollectionEvent(NodeEvent): - def __init__(self, anchor, tag, start_mark=None, end_mark=None): + def __init__(self, anchor, tag, start_mark=None, end_mark=None, + flow=None, compact=None): self.anchor = anchor self.tag = tag self.start_mark = start_mark self.end_mark = end_mark + self.flow = flow + self.compact = compact class SequenceEvent(CollectionEvent): @@ -45,5 +51,14 @@ class DocumentStartEvent(Event): - pass + def __init__(self, start_mark=None, end_mark=None, + indent=None, implicit=None, version=None, tags=None, + canonical=None): + self.start_mark = start_mark + self.end_mark = end_mark + self.indent = indent + self.implicit = implicit + self.version = version + self.tags = tags + self.canonical = canonical class DocumentEndEvent(Event): @@ -51,5 +66,9 @@ class StreamStartEvent(Event): - pass + def __init__(self, start_mark=None, end_mark=None, + encoding=None): + self.start_mark = start_mark + self.end_mark = end_mark + self.encoding = encoding class StreamEndEvent(Event): Index: /pyyaml/branches/working-on-emitter/lib/yaml/emitter.py =================================================================== --- /pyyaml/branches/working-on-emitter/lib/yaml/emitter.py (revision 122) +++ /pyyaml/branches/working-on-emitter/lib/yaml/emitter.py (revision 127) @@ -176,5 +176,5 @@ def write_indent(self): - self.writer.write("\n"+" "*(self.level*4)) + self.writer.write("\n"+" "*(self.level*2)) self.soft_space = False Index: /pyyaml/branches/working-on-emitter/lib/yaml/scanner.py =================================================================== --- /pyyaml/branches/working-on-emitter/lib/yaml/scanner.py (revision 121) +++ /pyyaml/branches/working-on-emitter/lib/yaml/scanner.py (revision 127) @@ -36,5 +36,6 @@ # See below simple keys treatment. - def __init__(self, token_number, required, index, line, column, mark): + def __init__(self, token_number, required, index, line, column, mark=None, + inline=None): self.token_number = token_number self.required = required @@ -43,4 +44,5 @@ self.column = column self.mark = mark + self.inline = inline class Scanner: @@ -81,4 +83,8 @@ # Past indentation levels. self.indents = [] + + # Used for providing style information to the parser. + self.current_line = self.previous_line = self.reader.line + self.current_column = self.previus_column = self.reader.column # Variables related to simple keys treatment. @@ -322,6 +328,7 @@ column = self.reader.column mark = self.reader.get_mark() + inline = (self.current_line == self.previous_line) key = SimpleKey(token_number, required, - index, line, column, mark) + index, line, column, mark, inline) self.possible_simple_keys[self.flow_level] = key @@ -381,6 +388,6 @@ # Add STREAM-END. - self.tokens.append(StreamStartToken(mark, mark)) - + self.tokens.append(StreamStartToken(mark, mark, + encoding=self.reader.encoding)) def fetch_stream_end(self): @@ -510,5 +517,6 @@ if self.add_indent(self.reader.column): mark = self.reader.get_mark() - self.tokens.append(BlockSequenceStartToken(mark, mark)) + inline = (self.current_line == self.previous_line) + self.tokens.append(BlockSequenceStartToken(mark, mark, inline)) # It's an error for the block entry to occur in the flow context, @@ -543,5 +551,6 @@ if self.add_indent(self.reader.column): mark = self.reader.get_mark() - self.tokens.append(BlockMappingStartToken(mark, mark)) + inline = (self.current_line == self.previous_line) + self.tokens.append(BlockMappingStartToken(mark, mark, inline)) # Simple keys are allowed after '?' in the block context. @@ -573,5 +582,6 @@ if self.add_indent(key.column): self.tokens.insert(key.token_number-self.tokens_taken, - BlockMappingStartToken(key.mark, key.mark)) + BlockMappingStartToken(key.mark, key.mark, + key.inline)) # There cannot be two simple keys one after another. @@ -791,4 +801,9 @@ else: found = True + + self.previous_line = self.current_line + self.previous_column = self.current_column + self.current_line = self.reader.line + self.current_column = self.reader.column def scan_directive(self): @@ -1054,5 +1069,10 @@ # We are done. - return ScalarToken(u''.join(chunks), False, start_mark, end_mark) + if folded: + style = '>' + else: + style = '|' + return ScalarToken(u''.join(chunks), False, start_mark, end_mark, + style) def scan_block_scalar_indicators(self, start_mark): @@ -1155,5 +1175,10 @@ self.reader.forward() end_mark = self.reader.get_mark() - return ScalarToken(u''.join(chunks), False, start_mark, end_mark) + if double: + style = '"' + else: + style = '\'' + return ScalarToken(u''.join(chunks), False, start_mark, end_mark, + style) ESCAPE_REPLACEMENTS = { @@ -1306,5 +1331,5 @@ or (not self.flow_level and self.reader.column < indent): break - return ScalarToken(u''.join(chunks), True, start_mark, end_mark) + return ScalarToken(u''.join(chunks), True, start_mark, end_mark, '') def scan_plain_spaces(self, indent, start_mark): Index: /pyyaml/branches/working-on-emitter/lib/yaml/parser.py =================================================================== --- /pyyaml/branches/working-on-emitter/lib/yaml/parser.py (revision 121) +++ /pyyaml/branches/working-on-emitter/lib/yaml/parser.py (revision 127) @@ -131,5 +131,6 @@ # Parse start of stream. token = self.scanner.get() - yield StreamStartEvent(token.start_mark, token.end_mark) + yield StreamStartEvent(token.start_mark, token.end_mark, + encoding=token.encoding) # Parse implicit document. @@ -139,5 +140,5 @@ token = self.scanner.peek() start_mark = end_mark = token.start_mark - yield DocumentStartEvent(start_mark, end_mark) + yield DocumentStartEvent(start_mark, end_mark, implicit=True) for event in self.parse_block_node(): yield event @@ -153,5 +154,5 @@ token = self.scanner.peek() start_mark = token.start_mark - self.process_directives() + version, tags = self.process_directives() if not self.scanner.check(DocumentStartToken): raise ParserError(None, None, @@ -161,5 +162,6 @@ token = self.scanner.get() end_mark = token.end_mark - yield DocumentStartEvent(start_mark, end_mark) + yield DocumentStartEvent(start_mark, end_mark, + implicit=False, version=version, tags=tags) if self.scanner.check(DirectiveToken, DocumentStartToken, DocumentEndToken, StreamEndToken): @@ -202,7 +204,12 @@ token.start_mark) self.tag_handles[handle] = prefix + version_value = self.yaml_version + tags_value = None + if self.tag_handles: + tags_value = self.tag_handles.copy() for key in self.DEFAULT_TAGS: if key not in self.tag_handles: self.tag_handles[key] = self.DEFAULT_TAGS[key] + return version_value, tags_value def parse_block_node(self): @@ -233,17 +240,20 @@ if self.scanner.check(AnchorToken): token = self.scanner.get() - start_mark = end_mark = token.start_mark + start_mark = token.start_mark + end_mark = token.end_mark anchor = token.value if self.scanner.check(TagToken): token = self.scanner.get() - end_mark = tag_mark = token.start_mark + tag_mark = token.start_mark + end_mark = token.end_mark tag = token.value elif self.scanner.check(TagToken): token = self.scanner.get() - start_mark = end_mark = tag_mark = token.start_mark + start_mark = tag_mark = token.start_mark + end_mark = token.end_mark tag = token.value if self.scanner.check(AnchorToken): token = self.scanner.get() - end_mark = token.start_mark + end_mark = token.end_mark anchor = token.value if tag is not None: @@ -262,10 +272,11 @@ tag = u'!' if start_mark is None: - start_mark = self.scanner.peek().start_mark + start_mark = end_mark = self.scanner.peek().start_mark event = None collection_events = None if indentless_sequence and self.scanner.check(BlockEntryToken): end_mark = self.scanner.peek().end_mark - event = SequenceEvent(anchor, tag, start_mark, end_mark) + event = SequenceEvent(anchor, tag, start_mark, end_mark, + flow=False, compact=False) collection_events = self.parse_indentless_sequence() else: @@ -274,21 +285,33 @@ end_mark = token.end_mark event = ScalarEvent(anchor, tag, token.value, - start_mark, end_mark) + start_mark, end_mark, + implicit=(tag is None), style=token.style) elif self.scanner.check(FlowSequenceStartToken): end_mark = self.scanner.peek().end_mark - event = SequenceEvent(anchor, tag, start_mark, end_mark) + event = SequenceEvent(anchor, tag, start_mark, end_mark, + flow=True) collection_events = self.parse_flow_sequence() elif self.scanner.check(FlowMappingStartToken): end_mark = self.scanner.peek().end_mark - event = MappingEvent(anchor, tag, start_mark, end_mark) + event = MappingEvent(anchor, tag, start_mark, end_mark, + flow=True) collection_events = self.parse_flow_mapping() elif block and self.scanner.check(BlockSequenceStartToken): end_mark = self.scanner.peek().start_mark - event = SequenceEvent(anchor, tag, start_mark, end_mark) + compact = self.scanner.peek().inline + event = SequenceEvent(anchor, tag, start_mark, end_mark, + flow=False, compact=compact) collection_events = self.parse_block_sequence() elif block and self.scanner.check(BlockMappingStartToken): end_mark = self.scanner.peek().start_mark - event = MappingEvent(anchor, tag, start_mark, end_mark) + compact = self.scanner.peek().inline + event = MappingEvent(anchor, tag, start_mark, end_mark, + flow=False, compact=compact) collection_events = self.parse_block_mapping() + elif anchor is not None or tag is not None: + # Empty scalars are allowed even if a tag or an anchor is + # specified. + event = ScalarEvent(anchor, tag, u'', start_mark, end_mark, + implicit=False, style='') else: if block: @@ -385,5 +408,6 @@ token = self.scanner.get() yield MappingEvent(None, u'!', - token.start_mark, token.end_mark) + token.start_mark, token.end_mark, + flow=True, compact=True) if not self.scanner.check(ValueToken, FlowEntryToken, FlowSequenceEndToken): @@ -461,4 +485,4 @@ def process_empty_scalar(self, mark): - return ScalarEvent(None, None, u'', mark, mark) - + return ScalarEvent(None, None, u'', mark, mark, implicit=True) +