Index: /branches/pyyaml3000/lib/yaml/scanner.py =================================================================== --- /branches/pyyaml3000/lib/yaml/scanner.py (revision 46) +++ /branches/pyyaml3000/lib/yaml/scanner.py (revision 47) @@ -24,5 +24,23 @@ # key: "valu\?e" # ^ - pass + def __init__(self, context=None, context_marker=None, + problem=None, problem_marker=None, description=None): + self.context = context + self.context_marker = context_marker + self.problem = problem + self.problem_marker = problem_marker + self.description = description + + def __str__(self): + lines = [] + for (place, marker) in [(self.context, self.context_marker), + (self.problem, self.problem_marker)]: + if place is not None: + lines.append(place) + if marker is not None: + lines.append(str(marker)) + if self.description is not None: + lines.append(self.description) + return '\n'.join(lines) class SimpleKey: @@ -140,7 +158,4 @@ # and decrease the current indentation level. self.unwind_indent(self.reader.column) - - #print - #print self.reader.get_marker().get_snippet() # Peek the next character. @@ -257,5 +272,6 @@ or self.reader.index-key.index > 1024: if key.required: - self.fail("simple key is required") + raise ScannerError("while scanning a simple key", key.marker, + "could not found expected ':'", self.reader.get_marker()) del self.possible_simple_keys[level] @@ -267,4 +283,8 @@ # Check if a simple key is required at the current position. required = not self.flow_level and self.indent == self.reader.column + + # A simple key is required only if it is the first token in the current + # line. Therefore it is always allowed. + assert self.allow_simple_key or not required # The next token might be a simple key. Let's save it's number and @@ -281,14 +301,14 @@ self.possible_simple_keys[self.flow_level] = key - # A simple key is required at the current position. - elif required: - self.fail("simple key is required") - def remove_possible_simple_key(self): # Remove the saved possible key position at the current flow level. if self.flow_level in self.possible_simple_keys: key = self.possible_simple_keys[self.flow_level] - if key.required: - self.fail("simple key is required") + + # I don't think it's possible, but I could be wrong. + assert not key.required + #if key.required: + # raise ScannerError("while scanning a simple key", key.marker, + # "could not found expected ':'", self.reader.get_marker()) # Indentation functions. @@ -297,6 +317,13 @@ # In flow context, tokens should respect indentation. + # Actually the condition should be `self.indent >= column` according to + # the spec. But this condition will prohibit intuitively correct + # constructions such as + # key : { + # } if self.flow_level and self.indent > column: - self.fail("invalid intendation in the flow context") + raise ScannerError(None, None, + "invalid intendation or unclosed '[' or '{'", + self.reader.get_marker()) # In block context, we may need to issue the BLOCK-END tokens. @@ -329,5 +356,5 @@ # Add END. - self.tokens.append(EndToken(marker, marker)) + self.tokens.append(StreamEndToken(marker, marker)) # The reader is ended. @@ -344,5 +371,5 @@ # Scan and add DIRECTIVE. - self.scan_directive() + self.tokens.append(self.scan_directive()) def fetch_document_start(self): @@ -421,5 +448,7 @@ # Are we allowed to start a new entry? if not self.allow_simple_key: - self.fail("Cannot start a new entry here") + raise ScannerError(None, None, + "sequence entries are not allowed here", + self.reader.get_marker()) # We may need to add BLOCK-SEQUENCE-START. @@ -447,5 +476,7 @@ # Are we allowed to start a key (not nessesary a simple)? if not self.allow_simple_key: - self.fail("Cannot start a new key here") + raise ScannerError(None, None, + "mapping keys are not allowed here", + self.reader.get_marker()) # We may need to add BLOCK-MAPPING-START. @@ -490,4 +521,16 @@ else: + # Block context needs additional checks. + # (Do we really need them? They will be catched by the parser + # anyway.) + if not self.flow_level: + + # We are allowed to start a complex value if and only if + # we can start a simple key. + if not self.allow_simple_key: + raise ScannerError(None, None, + "mapping values are not allowed here", + self.reader.get_marker()) + # Simple keys are allowed after ':' in the block context. self.allow_simple_key = not self.flow_level @@ -511,5 +554,5 @@ # Scan and add ALIAS. - self.scan_anchor(AliasToken) + self.tokens.append(self.scan_anchor(AliasToken)) def fetch_anchor(self): @@ -522,5 +565,5 @@ # Scan and add ANCHOR. - self.scan_anchor(AnchorToken) + self.tokens.append(self.scan_anchor(AnchorToken)) def fetch_tag(self): @@ -533,5 +576,5 @@ # Scan and add TAG. - self.scan_tag() + self.tokens.append(self.scan_tag()) def fetch_literal(self): @@ -550,5 +593,5 @@ # Scan and add SCALAR. - self.scan_block_scalar(folded) + self.tokens.append(self.scan_block_scalar(folded)) def fetch_single(self): @@ -567,5 +610,5 @@ # Scan and add SCALAR. - self.scan_flow_scalar(double) + self.tokens.append(self.scan_flow_scalar(double)) def fetch_plain(self): @@ -580,5 +623,5 @@ # Scan and add SCALAR. May change `allow_simple_key`. - self.scan_plain() + self.tokens.append(self.scan_plain()) # Checkers. @@ -646,4 +689,7 @@ def scan_to_next_token(self): + # We ignore spaces, line breaks and comments. + # If we find a line break in the block context, we set the flag + # `allow_simple_key` on. found = False while not found: @@ -651,8 +697,7 @@ self.reader.forward() if self.reader.peek() == u'#': - while self.reader.peek() not in u'\r\n': + while self.reader.peek() not in u'\0\r\n\x85\u2028\u2029': self.reader.forward() - if self.reader.peek() in u'\r\n': - self.reader.forward() + if self.scan_line_break(): if not self.flow_level: self.allow_simple_key = True @@ -663,12 +708,13 @@ marker = self.reader.get_marker() if self.reader.peek(5) == u'%YAML ': - self.tokens.append(YAMLDirectiveToken(1, 1, marker, marker)) + token = YAMLDirectiveToken(1, 1, marker, marker) elif self.reader.peek(4) == u'%TAG ': - self.tokens.append(TagDirectiveToken(marker, marker)) + token = TagDirectiveToken(marker, marker) else: - self.tokens.append(ReservedDirectiveToken('', marker, marker)) + token = ReservedDirectiveToken('', marker, marker) while self.reader.peek() not in u'\0\r\n': self.reader.forward() self.reader.forward() + return token def scan_anchor(self, TokenClass): @@ -677,5 +723,5 @@ self.reader.forward() end_marker = self.reader.get_marker() - self.tokens.append(TokenClass('', start_marker, end_marker)) + return TokenClass('', start_marker, end_marker) def scan_tag(self): @@ -684,5 +730,5 @@ self.reader.forward() end_marker = self.reader.get_marker() - self.tokens.append(TagToken('', start_marker, end_marker)) + return TagToken('', start_marker, end_marker) def scan_block_scalar(self, folded): @@ -702,5 +748,5 @@ if count < indent and self.reader.peek() not in u'#\r\n\x85\u2028\u2029': break - self.tokens.append(ScalarToken('', False, start_marker, start_marker)) + return ScalarToken('', False, start_marker, start_marker) def scan_flow_scalar(self, double): @@ -716,5 +762,5 @@ self.reader.forward(1) self.reader.forward(1) - self.tokens.append(ScalarToken('', False, marker, marker)) + return ScalarToken('', False, marker, marker) def scan_plain(self): @@ -748,11 +794,29 @@ break space = True - self.tokens.append(ScalarToken('', True, marker, marker)) + return ScalarToken('', True, marker, marker) + + def scan_line_break(self): + # Transforms: + # '\r\n' : '\n' + # '\r' : '\n' + # '\n' : '\n' + # '\x85' : '\n' + # '\u2028' : '\u2028' + # '\u2029 : '\u2029' + # default : '' + ch = self.reader.peek() + if ch in u'\r\n\x85': + if self.reader.peek(2) == u'\r\n': + self.forward(2) + else: + self.reader.forward() + return u'\n' + elif ch in u'\u2028\u2029': + self.reader.forward() + return ch + return u'' def invalid_token(self): self.fail("invalid token") - - def fail(self, message): - raise ScannerError(message) #try: Index: /branches/pyyaml3000/lib/yaml/parser.py =================================================================== --- /branches/pyyaml3000/lib/yaml/parser.py (revision 46) +++ /branches/pyyaml3000/lib/yaml/parser.py (revision 47) @@ -96,7 +96,7 @@ def parse_stream(self): documents = [] - if not self.is_token(DirectiveToken, DocumentStartToken, EndToken): + if not self.is_token(DirectiveToken, DocumentStartToken, StreamEndToken): documents.append(self.parse_block_node()) - while not self.is_token(EndToken): + while not self.is_token(StreamEndToken): while self.is_token(DirectiveToken): self.get_token() @@ -105,5 +105,5 @@ self.get_token() if self.is_token(DirectiveToken, - DocumentStartToken, DocumentEndToken, EndToken): + DocumentStartToken, DocumentEndToken, StreamEndToken): documents.append(None) else: @@ -111,6 +111,6 @@ while self.is_token(DocumentEndToken): self.get_token() - if not self.is_token(EndToken): - self.fail("END is expected") + if not self.is_token(StreamEndToken): + self.fail("STREAM-END is expected") return documents @@ -285,4 +285,4 @@ def fail(self, message): marker = self.scanner.peek_token().start_marker - raise Error(message+':\n'+marker.get_snippet()) - + raise ParserError(message+':\n'+marker.get_snippet()) + Index: /branches/pyyaml3000/lib/yaml/reader.py =================================================================== --- /branches/pyyaml3000/lib/yaml/reader.py (revision 46) +++ /branches/pyyaml3000/lib/yaml/reader.py (revision 47) @@ -66,5 +66,5 @@ self.pointer = pointer - def get_snippet(self, max_length=79): + def get_snippet(self, indent=4, max_length=75): if self.buffer is None: return None @@ -86,6 +86,14 @@ break snippet = self.buffer[start:end].encode('utf-8') - return head + snippet + tail + '\n' \ - + ' '*(self.pointer-start+len(head)) + '^' + '\n' + return ' '*indent + head + snippet + tail + '\n' \ + + ' '*(indent+self.pointer-start+len(head)) + '^' + + def __str__(self): + snippet = self.get_snippet() + where = " in \"%s\", line %d, column %d" \ + % (self.name, self.line+1, self.column+1) + if snippet is not None: + where += ":\n"+snippet + return where class ReaderError(YAMLError): @@ -101,10 +109,10 @@ if isinstance(self.character, str): return "'%s' codec can't decode byte #x%02x: %s\n" \ - "\tin '%s', position %d." \ + " in \"%s\", position %d" \ % (self.encoding, ord(self.character), self.reason, self.name, self.position) else: return "unacceptable character #x%04x: %s\n" \ - "\tin '%s', position %d." \ + " in \"%s\", position %d" \ % (ord(self.character), self.reason, self.name, self.position) Index: /branches/pyyaml3000/lib/yaml/tokens.py =================================================================== --- /branches/pyyaml3000/lib/yaml/tokens.py (revision 46) +++ /branches/pyyaml3000/lib/yaml/tokens.py (revision 47) @@ -6,7 +6,8 @@ class DirectiveToken(Token): - pass + code = '' class YAMLDirectiveToken(DirectiveToken): + code = '<%YAML directive>' def __init__(self, major_version, minor_version, start_marker, end_marker): self.major_version = major_version @@ -16,7 +17,8 @@ class TagDirectiveToken(DirectiveToken): - pass + code = '<%TAG directive>' class ReservedDirectiveToken(DirectiveToken): + code = '' def __init__(self, name, start_marker, end_marker): self.name = name @@ -25,43 +27,44 @@ class DocumentStartToken(Token): - pass + code = '' class DocumentEndToken(Token): - pass + code = '' -class EndToken(Token): - pass +class StreamEndToken(Token): + code = '' class BlockSequenceStartToken(Token): - pass + code = '' class BlockMappingStartToken(Token): - pass + code = '' class BlockEndToken(Token): - pass + code = '' class FlowSequenceStartToken(Token): - pass + code = '[' class FlowMappingStartToken(Token): - pass + code = '{' class FlowSequenceEndToken(Token): - pass + code = ']' class FlowMappingEndToken(Token): - pass + code = '}' class KeyToken(Token): - pass + code = '?' class ValueToken(Token): - pass + code = ':' class EntryToken(Token): - pass + code = '- or ,' class AliasToken(Token): + code = '' def __init__(self, value, start_marker, end_marker): self.value = value @@ -70,4 +73,5 @@ class AnchorToken(Token): + code = '' def __init__(self, value, start_marker, end_marker): self.value = value @@ -76,4 +80,5 @@ class TagToken(Token): + code = '' def __init__(self, value, start_marker, end_marker): self.value = value @@ -82,4 +87,5 @@ class ScalarToken(Token): + code = '' def __init__(self, value, plain, start_marker, end_marker): self.value = value Index: /branches/pyyaml3000/tests/test_appliance.py =================================================================== --- /branches/pyyaml3000/tests/test_appliance.py (revision 45) +++ /branches/pyyaml3000/tests/test_appliance.py (revision 47) @@ -70,5 +70,5 @@ return "%s(%s)" % (self.__class__.__name__, ''.join(args)) -class EndToken(Token): +class StreamEndToken(Token): pass @@ -133,5 +133,5 @@ ch = self.data[self.index] if ch == u'\0': - tokens.append(EndToken()) + tokens.append(StreamEndToken()) break elif ch == u'%': @@ -286,5 +286,5 @@ def parse_stream(self): documents = [] - while not self.test_token(EndToken): + while not self.test_token(StreamEndToken): if self.test_token(DirectiveToken, DocumentStartToken): documents.append(self.parse_document()) Index: /branches/pyyaml3000/tests/test_errors.py =================================================================== --- /branches/pyyaml3000/tests/test_errors.py (revision 47) +++ /branches/pyyaml3000/tests/test_errors.py (revision 47) @@ -0,0 +1,32 @@ + +import test_appliance + +from yaml.error import YAMLError +from yaml.reader import Reader +from yaml.scanner import Scanner + +class TestErrors(test_appliance.TestAppliance): + + def _testErrors(self, test_name, invalid_filename): + #self._load(invalid_filename) + self.failUnlessRaises(YAMLError, lambda: self._load(invalid_filename)) + + def _testStringErrors(self, test_name, invalid_filename): + #self._load_string(invalid_filename) + self.failUnlessRaises(YAMLError, lambda: self._load_string(invalid_filename)) + + def _load(self, filename): + reader = Reader(file(filename, 'rb')) + scanner = Scanner(reader) + while scanner.peek_token(): + scanner.get_token() + + def _load_string(self, filename): + reader = Reader(file(filename, 'rb').read()) + scanner = Scanner(reader) + while scanner.peek_token(): + scanner.get_token() + +TestErrors.add_tests('testErrors', '.error-message') +TestErrors.add_tests('testStringErrors', '.error-message') + Index: /branches/pyyaml3000/tests/test_marker.py =================================================================== --- /branches/pyyaml3000/tests/test_marker.py (revision 46) +++ /branches/pyyaml3000/tests/test_marker.py (revision 47) @@ -20,5 +20,5 @@ index += 1 marker = Marker(test_name, line, column, unicode(input), index) - snippet = marker.get_snippet() + snippet = marker.get_snippet(indent=2, max_length=79) #print "INPUT:" #print input @@ -26,7 +26,7 @@ #print snippet self.failUnless(isinstance(snippet, str)) - self.failUnlessEqual(snippet.count('\n'), 2) - data, pointer, dummy = snippet.split('\n') - self.failUnless(len(data) < 80) + self.failUnlessEqual(snippet.count('\n'), 1) + data, pointer = snippet.split('\n') + self.failUnless(len(data) < 82) self.failUnlessEqual(data[len(pointer)-1], '*') Index: /branches/pyyaml3000/tests/test_yaml.py =================================================================== --- /branches/pyyaml3000/tests/test_yaml.py (revision 46) +++ /branches/pyyaml3000/tests/test_yaml.py (revision 47) @@ -7,4 +7,5 @@ from test_tokens import * from test_structure import * +from test_errors import * def main(module='__main__'): Index: /branches/pyyaml3000/tests/data/forbidden-value.error-message =================================================================== --- /branches/pyyaml3000/tests/data/forbidden-value.error-message (revision 47) +++ /branches/pyyaml3000/tests/data/forbidden-value.error-message (revision 47) @@ -0,0 +1,1 @@ +test: key: value Index: /branches/pyyaml3000/tests/data/invalid-utf8-byte.error-message =================================================================== --- /branches/pyyaml3000/tests/data/invalid-utf8-byte.error-message (revision 47) +++ /branches/pyyaml3000/tests/data/invalid-utf8-byte.error-message (revision 47) @@ -0,0 +1,18 @@ +------------------------------------------------------------------------------------------------------------------------------- +------------------------------------------------------------------------------------------------------------------------------- +------------------------------------------------------------------------------------------------------------------------------- +------------------------------------------------------------------------------------------------------------------------------- +------------------------------------------------------------------------------------------------------------------------------- +------------------------------------------------------------------------------------------------------------------------------- +------------------------------------------------------------------------------------------------------------------------------- +------------------------------------------------------------------------------------------------------------------------------- +------------------------------------------------------------------------------------------------------------------------------- +------------------------------------------------------------------------------------------------------------------------------- +------------------------------------------------------------------------------------------------------------------------------- +------------------------------------------------------------------------------------------------------------------------------- +------------------------------------------------------------------------------------------------------------------------------- +------------------------------------------------------------------------------------------------------------------------------- +------------------------------------------------------------------------------------------------------------------------------- +------------------------------------------------------------------------------------------------------------------------------- +Invalid byte ('\xFF'): ΓΏ <-- +------------------------------------------------------------------------------------------------------------------------------- Index: /branches/pyyaml3000/tests/data/spec-10-07.data =================================================================== --- /branches/pyyaml3000/tests/data/spec-10-07.data (revision 44) +++ /branches/pyyaml3000/tests/data/spec-10-07.data (revision 47) @@ -1,5 +1,4 @@ { -#? : value # Empty key -? ~ : value, # Empty key +? : value, # Empty key ? explicit key: value, Index: /branches/pyyaml3000/tests/data/invalid-character.error-message =================================================================== --- /branches/pyyaml3000/tests/data/invalid-character.error-message (revision 47) +++ /branches/pyyaml3000/tests/data/invalid-character.error-message (revision 47) @@ -0,0 +1,18 @@ +------------------------------------------------------------------------------------------------------------------------------- +------------------------------------------------------------------------------------------------------------------------------- +------------------------------------------------------------------------------------------------------------------------------- +------------------------------------------------------------------------------------------------------------------------------- +------------------------------------------------------------------------------------------------------------------------------- +------------------------------------------------------------------------------------------------------------------------------- +------------------------------------------------------------------------------------------------------------------------------- +------------------------------------------------------------------------------------------------------------------------------- +------------------------------------------------------------------------------------------------------------------------------- +------------------------------------------------------------------------------------------------------------------------------- +------------------------------------------------------------------------------------------------------------------------------- +------------------------------------------------------------------------------------------------------------------------------- +------------------------------------------------------------------------------------------------------------------------------- +------------------------------------------------------------------------------------------------------------------------------- +------------------------------------------------------------------------------------------------------------------------------- +------------------------------------------------------------------------------------------------------------------------------- +Control character ('\x0'): <-- +------------------------------------------------------------------------------------------------------------------------------- Index: /branches/pyyaml3000/tests/data/forbidden-entry.error-message =================================================================== --- /branches/pyyaml3000/tests/data/forbidden-entry.error-message (revision 47) +++ /branches/pyyaml3000/tests/data/forbidden-entry.error-message (revision 47) @@ -0,0 +1,2 @@ +test: - foo + - bar Index: /branches/pyyaml3000/tests/data/invalid-simple-key.error-message =================================================================== --- /branches/pyyaml3000/tests/data/invalid-simple-key.error-message (revision 47) +++ /branches/pyyaml3000/tests/data/invalid-simple-key.error-message (revision 47) @@ -0,0 +1,3 @@ +key: value +invalid simple key +next key: next value Index: /branches/pyyaml3000/tests/data/forbidden-key.error-message =================================================================== --- /branches/pyyaml3000/tests/data/forbidden-key.error-message (revision 47) +++ /branches/pyyaml3000/tests/data/forbidden-key.error-message (revision 47) @@ -0,0 +1,2 @@ +test: ? foo + : bar Index: /branches/pyyaml3000/tests/data/unclosed-bracket.error-message =================================================================== --- /branches/pyyaml3000/tests/data/unclosed-bracket.error-message (revision 47) +++ /branches/pyyaml3000/tests/data/unclosed-bracket.error-message (revision 47) @@ -0,0 +1,5 @@ +test: + - [ foo: bar + - baz +"we could have detected the unclosed bracket on the above line, but this would forbid such syntax as": { +} Index: /branches/pyyaml3000/tests/test_tokens.py =================================================================== --- /branches/pyyaml3000/tests/test_tokens.py (revision 46) +++ /branches/pyyaml3000/tests/test_tokens.py (revision 47) @@ -55,5 +55,5 @@ scanner = Scanner(Reader(file(data_filename, 'rb'))) tokens1 = [] - while not isinstance(scanner.peek_token(), EndToken): + while not isinstance(scanner.peek_token(), StreamEndToken): tokens1.append(scanner.get_token()) tokens1 = [self.replaces[t.__class__] for t in tokens1] @@ -77,5 +77,5 @@ scanner = Scanner(Reader(file(filename, 'rb'))) tokens = [] - while not isinstance(scanner.peek_token(), EndToken): + while not isinstance(scanner.peek_token(), StreamEndToken): tokens.append(scanner.get_token().__class__.__name__) except: