Index: pyyaml/trunk/lib/yaml/emitter.py =================================================================== --- pyyaml/trunk/lib/yaml/emitter.py (revision 133) +++ pyyaml/trunk/lib/yaml/emitter.py (revision 136) @@ -12,4 +12,6 @@ from events import * +import re + class EmitterError(YAMLError): pass @@ -18,5 +20,6 @@ def __init__(self, scalar, empty, multiline, allow_flow_plain, allow_block_plain, - allow_single_quoted, allow_double_quoted, allow_block): + allow_single_quoted, allow_double_quoted, + allow_block): self.scalar = scalar self.empty = empty @@ -35,10 +38,11 @@ } - def __init__(self, writer): - - # The writer should have the methods `write` and possibly `flush`. - self.writer = writer - - # Encoding is provided by STREAM-START. + def __init__(self, stream, canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None): + + # The stream should have the methods `write` and possibly `flush`. + self.stream = stream + + # Encoding can be overriden by STREAM-START. self.encoding = None @@ -76,16 +80,26 @@ # Formatting details. - self.canonical = False - self.allow_unicode = False + self.canonical = canonical + self.allow_unicode = allow_unicode + self.best_indent = 2 + if indent and 1 < indent < 10: + self.best_indent = indent + self.best_width = 80 + if width and width > self.best_indent*2: + self.best_width = width self.best_line_break = u'\n' - self.best_indent = 2 - self.best_width = 80 + if line_break in [u'\r', u'\n', u'\r\n']: + self.best_line_break = line_break + + # Tag prefixes. self.tag_prefixes = None - # Analyses cache. - self.anchor_text = None - self.tag_text = None - self.scalar_analysis = None - self.scalar_style = None + # Prepared anchor and tag. + self.prepared_anchor = None + self.prepared_tag = None + + # Scalar analysis and style. + self.analysis = None + self.style = None def emit(self, event): @@ -140,13 +154,6 @@ def expect_stream_start(self): if isinstance(self.event, StreamStartEvent): - self.encoding = self.event.encoding - self.canonical = self.event.canonical - self.allow_unicode = self.event.allow_unicode - if self.event.indent and self.event.indent > 1: - self.best_indent = self.event.indent - if self.event.width and self.event.width > self.best_indent: - self.best_width = self.event.width - if self.event.line_break in [u'\r', u'\n', u'\r\n']: - self.best_line_break = self.event.line_break + if self.event.encoding: + self.encoding = self.event.encoding self.write_stream_start() self.state = self.expect_first_document_start @@ -166,5 +173,5 @@ if isinstance(self.event, DocumentStartEvent): if self.event.version: - version_text = self.analyze_version(self.event.version) + version_text = self.prepare_version(self.event.version) self.write_version_directive(version_text) self.tag_prefixes = self.DEFAULT_TAG_PREFIXES.copy() @@ -175,6 +182,6 @@ prefix = self.event.tags[handle] self.tag_prefixes[prefix] = handle - handle_text = self.analyze_tag_handle(handle) - prefix_text = self.analyze_tag_prefix(prefix) + handle_text = self.prepare_tag_handle(handle) + prefix_text = self.prepare_tag_prefix(prefix) self.write_tag_directive(handle_text, prefix_text) implicit = (first and not self.event.explicit and not self.canonical @@ -200,4 +207,5 @@ self.write_indicator(u'...', True) self.write_indent() + self.flush_stream() self.state = self.expect_document_start else: @@ -419,18 +427,19 @@ length = 0 if isinstance(self.event, NodeEvent) and self.event.anchor is not None: - if self.anchor_text is None: - self.anchor_text = self.analyze_anchor(self.event.anchor) - length += len(self.anchor_text) + if self.prepared_anchor is None: + self.prepared_anchor = self.prepare_anchor(self.event.anchor) + length += len(self.prepared_anchor) if isinstance(self.event, (ScalarEvent, CollectionStartEvent)) \ and self.event.tag is not None: - if self.tag_text is None: - self.tag_text = self.analyze_tag(self.event.tag) - length += len(self.tag_text) + if self.prepared_tag is None: + self.prepared_tag = self.prepare_tag(self.event.tag) + length += len(self.prepared_tag) if isinstance(self.event, ScalarEvent): - if self.scalar_analysis is None: - self.scalar_analysis = self.analyze_scalar(self.event.value) - length += len(self.scalar_analysis.scalar) + if self.analysis is None: + self.analysis = self.analyze_scalar(self.event.value) + length += len(self.analysis.scalar) return (length < 128 and (isinstance(self.event, AliasEvent) - or (isinstance(self.event, ScalarEvent) and not self.scalar_analysis.multiline) + or (isinstance(self.event, ScalarEvent) + and not self.analysis.empty and not self.analysis.multiline) or self.check_empty_sequence() or self.check_empty_mapping())) @@ -439,66 +448,77 @@ def process_anchor(self, indicator): if self.event.anchor is None: + self.prepared_anchor = None return - if self.anchor_text is None: - self.anchor_text = self.analyze_anchor(self.event.anchor) - if self.anchor_text: - self.write_indicator(indicator+self.anchor_text, True) - self.anchor_text = None + if self.prepared_anchor is None: + self.prepared_anchor = self.prepare_anchor(self.event.anchor) + if self.prepared_anchor: + self.write_indicator(indicator+self.prepared_anchor, True) + self.prepared_anchor = None def process_tag(self): - if self.event.tag is None: + tag = self.event.tag + if isinstance(self.event, ScalarEvent): + if self.style is None: + self.style = self.choose_scalar_style() + if self.style == '': + self.prepared_tag = None + return + if self.event.implicit and not tag: + tag = u'!' + self.prepared_tag = None + if not tag: + self.prepared_tag = None return - if isinstance(self.event, ScalarEvent) and self.best_scalar_style() == '': - return - if self.tag_text is None: - self.tag_text = self.analyze_tag(self.event.tag) - if self.tag_text: - self.write_indicator(self.tag_text, True) - self.tag_text = None - - def best_scalar_style(self): - if self.scalar_analysis is None: - self.scalar_analysis = self.analyze_scalar(self.event.value) - if self.canonical: + if self.prepared_tag is None: + self.prepared_tag = self.prepare_tag(tag) + if self.prepared_tag: + self.write_indicator(self.prepared_tag, True) + self.prepared_tag = None + + def choose_scalar_style(self): + if self.analysis is None: + self.analysis = self.analyze_scalar(self.event.value) + if self.event.style == '"' or self.canonical: return '"' - if (self.event.implicit and not self.event.style - and ((self.flow_level and self.scalar_analysis.allow_flow_plain) - or (not self.flow_level and self.scalar_analysis.allow_block_plain)) - and (len(self.scalar_analysis.scalar) > 0 - or (not self.flow_level and not self.simple_key_context))): - return '' - elif self.event.style == '\'' and self.scalar_analysis.allow_single_quoted: - return '\'' - elif self.event.style in ['|', '>'] and not self.flow_level and self.scalar_analysis.allow_block: - return self.event.style - else: - return '"' - return style + if not self.event.style and self.event.implicit: + if (not (self.simple_key_context and + (self.analysis.empty or self.analysis.multiline)) + and (self.flow_level and self.analysis.allow_flow_plain + or (not self.flow_level and self.analysis.allow_block_plain))): + return '' + if self.event.style and self.event.style in '|>': + if not self.flow_level and self.analysis.allow_block: + return self.event.style + if not self.event.style or self.event.style == '\'': + if (self.analysis.allow_single_quoted and + not (self.simple_key_context and self.analysis.multiline)): + return '\'' + return '"' def process_scalar(self): - if self.scalar_analysis is None: - self.scalar_analysis = self.analyze_scalar(self.event.value) - style = self.best_scalar_style() - if self.scalar_analysis.multiline and not self.simple_key_context \ - and style not in ['|', '>']: - self.write_indent() - if style == '"': - self.write_double_quoted(self.scalar_analysis.scalar, - split=(not self.simple_key_context)) - elif style == '\'': - self.write_single_quoted(self.scalar_analysis.scalar, - split=(not self.simple_key_context)) - elif style == '>': - self.write_folded(self.scalar_analysis.scalar) - elif style == '|': - self.write_literal(self.scalar_analysis.scalar) - else: - self.write_plain(self.scalar_analysis.scalar, - split=(not self.simple_key_context)) - self.scalar_analysis = None + if self.analysis is None: + self.analysis = self.analyze_scalar(self.event.value) + if self.style is None: + self.style = self.choose_scalar_style() + split = (not self.simple_key_context) + #if self.analysis.multiline and split \ + # and (not self.style or self.style in '\'\"'): + # self.write_indent() + if self.style == '"': + self.write_double_quoted(self.analysis.scalar, split) + elif self.style == '\'': + self.write_single_quoted(self.analysis.scalar, split) + elif self.style == '>': + self.write_folded(self.analysis.scalar) + elif self.style == '|': + self.write_literal(self.analysis.scalar) + else: + self.write_plain(self.analysis.scalar, split) + self.analysis = None + self.style = None # Analyzers. - def analyze_version(self, version): + def prepare_version(self, version): major, minor = version if major != 1: @@ -506,5 +526,5 @@ return u'%d.%d' % (major, minor) - def analyze_tag_handle(self, handle): + def prepare_tag_handle(self, handle): if not handle: raise EmitterError("tag handle must not be empty") @@ -519,5 +539,5 @@ return handle - def analyze_tag_prefix(self, prefix): + def prepare_tag_prefix(self, prefix): if not prefix: raise EmitterError("tag prefix must not be empty") @@ -542,7 +562,9 @@ return u''.join(chunks) - def analyze_tag(self, tag): + def prepare_tag(self, tag): if not tag: raise EmitterError("tag must not be empty") + if tag == u'!': + return tag handle = None suffix = tag @@ -575,5 +597,5 @@ return u'!<%s>' % suffix_text - def analyze_anchor(self, anchor): + def prepare_anchor(self, anchor): if not anchor: raise EmitterError("anchor must not be empty") @@ -585,5 +607,7 @@ return anchor - def analyze_scalar(self, scalar): # It begs for refactoring. + def analyze_scalar(self, scalar): + + # Empty scalar is a special case. if not scalar: return ScalarAnalysis(scalar=scalar, empty=True, multiline=False, @@ -591,135 +615,211 @@ allow_single_quoted=True, allow_double_quoted=True, allow_block=False) - contains_block_indicator = False - contains_flow_indicator = False - contains_line_breaks = False - contains_unicode_characters = False - contains_special_characters = False - contains_inline_spaces = False # non-space space+ non-space - contains_inline_breaks = False # non-space break+ non-space - contains_leading_spaces = False # ^ space+ (non-space | $) - contains_leading_breaks = False # ^ break+ (non-space | $) - contains_trailing_spaces = False # non-space space+ $ - contains_trailing_breaks = False # non-space break+ $ - contains_inline_breaks_spaces = False # non-space break+ space+ non-space - contains_mixed_breaks_spaces = False # anything else + + # Indicators and special characters. + block_indicators = False + flow_indicators = False + line_breaks = False + special_characters = False + + # Whitespaces. + inline_spaces = False # non-space space+ non-space + inline_breaks = False # non-space break+ non-space + leading_spaces = False # ^ space+ (non-space | $) + leading_breaks = False # ^ break+ (non-space | $) + trailing_spaces = False # (^ | non-space) space+ $ + trailing_breaks = False # (^ | non-space) break+ $ + inline_breaks_spaces = False # non-space break+ space+ non-space + mixed_breaks_spaces = False # anything else + + # Check document indicators. if scalar.startswith(u'---') or scalar.startswith(u'...'): - contains_block_indicator = True - contains_flow_indicator = True - first = True - last = (len(scalar) == 1) - preceeded_by_space = False - followed_by_space = (len(scalar) > 1 and + block_indicators = True + flow_indicators = True + + # First character or preceded by a whitespace. + preceeded_by_space = True + + # Last character or followed by a whitespace. + followed_by_space = (len(scalar) == 1 or scalar[1] in u'\0 \t\r\n\x85\u2028\u2029') - spaces = breaks = mixed = leading = False + + # The current series of whitespaces contain plain spaces. + spaces = False + + # The current series of whitespaces contain line breaks. + breaks = False + + # The current series of whitespaces contain a space followed by a + # break. + mixed = False + + # The current series of whitespaces start at the beginning of the + # scalar. + leading = False + index = 0 while index < len(scalar): ch = scalar[index] - if first: + + # Check for indicators. + + if index == 0: + # Leading indicators are special characters. if ch in u'#,[]{}#&*!|>\'\"%@`': - contains_flow_indicator = True - contains_block_indicator = True + flow_indicators = True + block_indicators = True if ch in u'?:': - contains_flow_indicator = True - if followed_by_space or last: - contains_block_indicator = True - if ch == u'-' and (followed_by_space or last): - contains_flow_indicator = True - contains_block_indicator = True + flow_indicators = True + if followed_by_space: + block_indicators = True + if ch == u'-' and followed_by_space: + flow_indicators = True + block_indicators = True else: + # Some indicators cannot appear within a scalar as well. if ch in u',?[]{}': - contains_flow_indicator = True + flow_indicators = True if ch == u':': - contains_flow_indicator = True - if followed_by_space or last: - contains_block_indicator = True - if ch == u'#' and (preceeded_by_space or first): - contains_flow_indicator = True - contains_block_indicator = True + flow_indicators = True + if followed_by_space: + block_indicators = True + if ch == u'#' and preceeded_by_space: + flow_indicators = True + block_indicators = True + + # Check for line breaks, special, and unicode characters. + if ch in u'\n\x85\u2028\u2029': - contains_line_breaks = True + line_breaks = True if not (ch == u'\n' or u'\x20' <= ch <= u'\x7E'): - if ch < u'\x80': - contains_special_characters = True + if ch < u'\x80' or ch == u'\uFEFF': # '\uFEFF' is BOM. + special_characters = True else: - contains_unicode_characters = True - if ch == u' ': - if not spaces and not breaks: - leading = first - spaces = True - elif ch in u'\n\x85\u2028\u2029': - if not spaces and not breaks: - leading = first - breaks = True - if spaces: - mixed = True - if ch not in u' \n\x85\u2028\u2029': + unicode_characters = True + if not self.allow_unicode: + special_characters = True + + # Spaces, line breaks, and how they are mixed. State machine. + + # Start or continue series of whitespaces. + if ch in u' \n\x85\u2028\u2029': + if spaces and breaks: + if ch != u' ': # break+ (space+ break+) => mixed + mixed = True + elif spaces: + if ch != u' ': # (space+ break+) => mixed + breaks = True + mixed = True + elif breaks: + if ch == u' ': # break+ space+ + spaces = True + else: + leading = (index == 0) + if ch == u' ': # space+ + spaces = True + else: # break+ + breaks = True + + # Series of whitespaces ended with a non-space. + elif spaces or breaks: if leading: if spaces and breaks: - contains_mixed_breaks_spaces = True + mixed_breaks_spaces = True elif spaces: - contains_leading_spaces = True + leading_spaces = True elif breaks: - contains_leading_breaks = True + leading_breaks = True else: if mixed: - contains_mixed_break_spaces = True + mixed_breaks_spaces = True elif spaces and breaks: - contains_inline_breaks_spaces = True + inline_breaks_spaces = True elif spaces: - contains_inline_spaces = True + inline_spaces = True elif breaks: - contains_inline_breaks = True + inline_breaks = True spaces = breaks = mixed = leading = False - elif last: + + # Series of whitespaces reach the end. + if (spaces or breaks) and (index == len(scalar)-1): if spaces and breaks: - contains_mixed_break_spaces = True + mixed_breaks_spaces = True elif spaces: + trailing_spaces = True if leading: - contains_leading_spaces = True - else: - contains_trailing_spaces = True + leading_spaces = True elif breaks: + trailing_breaks = True if leading: - contains_leading_breaks = True - else: - contains_trailing_breaks = True + leading_breaks = True + spaces = breaks = mixed = leading = False + + # Prepare for the next character. index += 1 - first = False - last = (index+1 == len(scalar)) preceeded_by_space = (ch in u'\0 \t\r\n\x85\u2028\u2029') - followed_by_space = (index+1 < len(scalar) and + followed_by_space = (index+1 >= len(scalar) or scalar[index+1] in u'\0 \t\r\n\x85\u2028\u2029') - if contains_unicode_characters and not self.allow_unicode: - contains_special_characters = True - allow_flow_plain = not (contains_flow_indicator or contains_special_characters - or contains_leading_spaces or contains_leading_breaks - or contains_trailing_spaces or contains_trailing_breaks - or contains_inline_breaks_spaces or contains_mixed_breaks_spaces) - allow_block_plain = not (contains_block_indicator or contains_special_characters - or contains_leading_spaces or contains_leading_breaks - or contains_trailing_spaces or contains_trailing_breaks - or contains_inline_breaks_spaces or contains_mixed_breaks_spaces) - allow_single_quoted = not (contains_special_characters - or contains_inline_breaks_spaces or contains_mixed_breaks_spaces) + + # Let's decide what styles are allowed. + allow_flow_plain = True + allow_block_plain = True + allow_single_quoted = True allow_double_quoted = True - allow_block = not (contains_special_characters - or contains_leading_spaces or contains_leading_breaks - or contains_trailing_spaces or contains_mixed_breaks_spaces) - return ScalarAnalysis(scalar=scalar, empty=False, multiline=contains_line_breaks, - allow_flow_plain=allow_flow_plain, allow_block_plain=allow_block_plain, - allow_single_quoted=allow_single_quoted, allow_double_quoted=allow_double_quoted, + allow_block = True + + # Leading and trailing whitespace are bad for plain scalars. We also + # do not want to mess with leading whitespaces for block scalars. + if leading_spaces or leading_breaks or trailing_spaces: + allow_flow_plain = allow_block_plain = allow_block = False + + # Trailing breaks are fine for block scalars, but unacceptable for + # plain scalars. + if trailing_breaks: + allow_flow_plain = allow_block_plain = False + + # The combination of (space+ break+) is only acceptable for block + # scalars. + if inline_breaks_spaces: + allow_flow_plain = allow_block_plain = allow_single_quoted = False + + # Mixed spaces and breaks, as well as special character are only + # allowed for double quoted scalars. + if mixed_breaks_spaces or special_characters: + allow_flow_plain = allow_block_plain = \ + allow_single_quoted = allow_block = False + + # We don't emit multiline plain scalars. + if line_breaks: + allow_flow_plain = allow_block_plain = False + + # Flow indicators are forbidden for flow plain scalars. + if flow_indicators: + allow_flow_plain = False + + # Block indicators are forbidden for block plain scalars. + if block_indicators: + allow_block_plain = False + + return ScalarAnalysis(scalar=scalar, + empty=False, multiline=line_breaks, + allow_flow_plain=allow_flow_plain, + allow_block_plain=allow_block_plain, + allow_single_quoted=allow_single_quoted, + allow_double_quoted=allow_double_quoted, allow_block=allow_block) # Writers. + + def flush_stream(self): + if hasattr(self.stream, 'flush'): + self.stream.flush() def write_stream_start(self): # Write BOM if needed. if self.encoding and self.encoding.startswith('utf-16'): - self.writer.write(u'\xFF\xFE'.encode(self.encoding)) + self.stream.write(u'\xFF\xFE'.encode(self.encoding)) def write_stream_end(self): - if hasattr(self.writer, 'flush'): - self.writer.flush() + self.flush_stream() def write_indicator(self, indicator, need_whitespace, @@ -734,5 +834,5 @@ if self.encoding: data = data.encode(self.encoding) - self.writer.write(data) + self.stream.write(data) def write_indent(self): @@ -747,5 +847,5 @@ if self.encoding: data = data.encode(self.encoding) - self.writer.write(data) + self.stream.write(data) def write_line_break(self, data=None): @@ -758,5 +858,5 @@ if self.encoding: data = data.encode(self.encoding) - self.writer.write(data) + self.stream.write(data) def write_version_directive(self, version_text): @@ -764,5 +864,5 @@ if self.encoding: data = data.encode(self.encoding) - self.writer.write(data) + self.stream.write(data) self.write_line_break() @@ -771,8 +871,8 @@ if self.encoding: data = data.encode(self.encoding) - self.writer.write(data) + self.stream.write(data) self.write_line_break() - # Scalar writers. + # Scalar streams. def write_single_quoted(self, text, split=True): @@ -795,5 +895,5 @@ if self.encoding: data = data.encode(self.encoding) - self.writer.write(data) + self.stream.write(data) start = end elif breaks: @@ -815,5 +915,5 @@ if self.encoding: data = data.encode(self.encoding) - self.writer.write(data) + self.stream.write(data) start = end if ch == u'\'': @@ -822,5 +922,5 @@ if self.encoding: data = data.encode(self.encoding) - self.writer.write(data) + self.stream.write(data) start = end + 1 if ch is not None: @@ -864,5 +964,5 @@ if self.encoding: data = data.encode(self.encoding) - self.writer.write(data) + self.stream.write(data) start = end if ch is not None: @@ -878,5 +978,5 @@ if self.encoding: data = data.encode(self.encoding) - self.writer.write(data) + self.stream.write(data) start = end+1 if 0 < end < len(text)-1 and (ch == u' ' or start >= end) \ @@ -888,5 +988,5 @@ if self.encoding: data = data.encode(self.encoding) - self.writer.write(data) + self.stream.write(data) self.write_indent() self.whitespace = False @@ -897,5 +997,5 @@ if self.encoding: data = data.encode(self.encoding) - self.writer.write(data) + self.stream.write(data) end += 1 self.write_indicator(u'"', False) @@ -948,5 +1048,5 @@ if self.encoding: data = data.encode(self.encoding) - self.writer.write(data) + self.stream.write(data) start = end else: @@ -955,5 +1055,5 @@ if self.encoding: data = data.encode(self.encoding) - self.writer.write(data) + self.stream.write(data) if ch is None: self.write_line_break() @@ -989,5 +1089,5 @@ if self.encoding: data = data.encode(self.encoding) - self.writer.write(data) + self.stream.write(data) if ch is None: self.write_line_break() @@ -1005,5 +1105,5 @@ if self.encoding: data = data.encode(self.encoding) - self.writer.write(data) + self.stream.write(data) self.writespace = False self.indention = False @@ -1026,5 +1126,5 @@ if self.encoding: data = data.encode(self.encoding) - self.writer.write(data) + self.stream.write(data) start = end elif breaks: @@ -1047,5 +1147,5 @@ if self.encoding: data = data.encode(self.encoding) - self.writer.write(data) + self.stream.write(data) start = end if ch is not None: