diff -ur original_src/api.c src/api.c --- original_src/api.c 2008-12-27 15:18:21.000000000 -0800 +++ src/api.c 2009-08-18 18:18:41.000000000 -0700 @@ -611,51 +611,6 @@ } /* - * Check if a string is a valid UTF-8 sequence. - * - * Check 'reader.c' for more details on UTF-8 encoding. - */ - -static int -yaml_check_utf8(yaml_char_t *start, size_t length) -{ - yaml_char_t *end = start+length; - yaml_char_t *pointer = start; - - while (pointer < end) { - unsigned char octet; - unsigned int width; - unsigned int value; - size_t k; - - octet = pointer[0]; - width = (octet & 0x80) == 0x00 ? 1 : - (octet & 0xE0) == 0xC0 ? 2 : - (octet & 0xF0) == 0xE0 ? 3 : - (octet & 0xF8) == 0xF0 ? 4 : 0; - value = (octet & 0x80) == 0x00 ? octet & 0x7F : - (octet & 0xE0) == 0xC0 ? octet & 0x1F : - (octet & 0xF0) == 0xE0 ? octet & 0x0F : - (octet & 0xF8) == 0xF0 ? octet & 0x07 : 0; - if (!width) return 0; - if (pointer+width > end) return 0; - for (k = 1; k < width; k ++) { - octet = pointer[k]; - if ((octet & 0xC0) != 0x80) return 0; - value = (value << 6) + (octet & 0x3F); - } - if (!((width == 1) || - (width == 2 && value >= 0x80) || - (width == 3 && value >= 0x800) || - (width == 4 && value >= 0x10000))) return 0; - - pointer += width; - } - - return 1; -} - -/* * Create STREAM-START. */ @@ -731,12 +686,6 @@ tag_directive != tag_directives_end; tag_directive ++) { assert(tag_directive->handle); assert(tag_directive->prefix); - if (!yaml_check_utf8(tag_directive->handle, - strlen((char *)tag_directive->handle))) - goto error; - if (!yaml_check_utf8(tag_directive->prefix, - strlen((char *)tag_directive->prefix))) - goto error; value.handle = yaml_strdup(tag_directive->handle); value.prefix = yaml_strdup(tag_directive->prefix); if (!value.handle || !value.prefix) goto error; @@ -796,8 +745,6 @@ assert(event); /* Non-NULL event object is expected. */ assert(anchor); /* Non-NULL anchor is expected. */ - if (!yaml_check_utf8(anchor, strlen((char *)anchor))) return 0; - anchor_copy = yaml_strdup(anchor); if (!anchor_copy) return 0; @@ -827,13 +774,11 @@ assert(value); /* Non-NULL anchor is expected. */ if (anchor) { - if (!yaml_check_utf8(anchor, strlen((char *)anchor))) goto error; anchor_copy = yaml_strdup(anchor); if (!anchor_copy) goto error; } if (tag) { - if (!yaml_check_utf8(tag, strlen((char *)tag))) goto error; tag_copy = yaml_strdup(tag); if (!tag_copy) goto error; } @@ -842,7 +787,6 @@ length = strlen((char *)value); } - if (!yaml_check_utf8(value, length)) goto error; value_copy = yaml_malloc(length+1); if (!value_copy) goto error; memcpy(value_copy, value, length); @@ -877,13 +821,11 @@ assert(event); /* Non-NULL event object is expected. */ if (anchor) { - if (!yaml_check_utf8(anchor, strlen((char *)anchor))) goto error; anchor_copy = yaml_strdup(anchor); if (!anchor_copy) goto error; } if (tag) { - if (!yaml_check_utf8(tag, strlen((char *)tag))) goto error; tag_copy = yaml_strdup(tag); if (!tag_copy) goto error; } @@ -932,13 +874,11 @@ assert(event); /* Non-NULL event object is expected. */ if (anchor) { - if (!yaml_check_utf8(anchor, strlen((char *)anchor))) goto error; anchor_copy = yaml_strdup(anchor); if (!anchor_copy) goto error; } if (tag) { - if (!yaml_check_utf8(tag, strlen((char *)tag))) goto error; tag_copy = yaml_strdup(tag); if (!tag_copy) goto error; } @@ -1072,12 +1012,6 @@ tag_directive != tag_directives_end; tag_directive ++) { assert(tag_directive->handle); assert(tag_directive->prefix); - if (!yaml_check_utf8(tag_directive->handle, - strlen((char *)tag_directive->handle))) - goto error; - if (!yaml_check_utf8(tag_directive->prefix, - strlen((char *)tag_directive->prefix))) - goto error; value.handle = yaml_strdup(tag_directive->handle); value.prefix = yaml_strdup(tag_directive->prefix); if (!value.handle || !value.prefix) goto error; @@ -1210,7 +1144,6 @@ tag = (yaml_char_t *)YAML_DEFAULT_SCALAR_TAG; } - if (!yaml_check_utf8(tag, strlen((char *)tag))) goto error; tag_copy = yaml_strdup(tag); if (!tag_copy) goto error; @@ -1218,7 +1151,6 @@ length = strlen((char *)value); } - if (!yaml_check_utf8(value, length)) goto error; value_copy = yaml_malloc(length+1); if (!value_copy) goto error; memcpy(value_copy, value, length); @@ -1262,7 +1194,6 @@ tag = (yaml_char_t *)YAML_DEFAULT_SEQUENCE_TAG; } - if (!yaml_check_utf8(tag, strlen((char *)tag))) goto error; tag_copy = yaml_strdup(tag); if (!tag_copy) goto error; @@ -1307,7 +1238,6 @@ tag = (yaml_char_t *)YAML_DEFAULT_MAPPING_TAG; } - if (!yaml_check_utf8(tag, strlen((char *)tag))) goto error; tag_copy = yaml_strdup(tag); if (!tag_copy) goto error; diff -ur original_src/emitter.c src/emitter.c --- original_src/emitter.c 2008-12-27 15:18:21.000000000 -0800 +++ src/emitter.c 2009-08-18 17:50:47.000000000 -0700 @@ -37,7 +37,7 @@ 1)) /* - * Copy a character from a string into buffer. + * Copy a byte from a string into buffer. */ #define WRITE(emitter,string) \ @@ -46,6 +46,12 @@ emitter->column ++, \ 1)) +#define WRITEN(emitter,string,n) \ + (FLUSH(emitter) \ + && (COPYN(emitter->buffer,string,n), \ + emitter->column ++, \ + 1)) + /* * Copy a line break character from a string into buffer. */ @@ -56,7 +62,7 @@ (PUT_BREAK(emitter), \ string.pointer ++, \ 1) : \ - (COPY(emitter->buffer,string), \ + (COPYN(emitter->buffer,string,WIDTH(string)), \ emitter->column = 0, \ emitter->line ++, \ 1))) @@ -1458,6 +1464,57 @@ return 1; } +/** + Return the number of bytes in the next UTF-8 character. + + Returns 0 for any error, includinge incorrect bytes, not enough + bytes before the end pointer, and overlong encodings. If 0 is + returned, pointer[0] will always have the high bit set and will + thus never match any ASCII character. + + The encodings of surrogate halves are allowed! Otherwise it is not + possible to losslessly encode invalid UTF-16 into UTF-8. (There is + a vocal contingent trying to sabotage UTF-8 by declaring surrogate + half encodings invalid. Anybody such claim should be investigated + carefully: if that user's code does not also reject invalid UTF-16, + then they are being hypocrites and can be ignored.) +*/ +static unsigned int utf8_width(yaml_char_t* pointer, yaml_char_t* end) +{ + unsigned char octet = pointer[0]; + if (octet < 0x80) { + /* 1-byte character */ + return 1; + } else if (octet < 0xC2) { + /* continuation byte or overlong 2-byte encoding */ + return 0; + } else if (octet < 0xE0) { + /* 2-byte character */ + if (end-pointer < 2) return 0; + if ((pointer[1] & 0xC0) != 0x80) return 0; + return 2; + } else if (octet < 0xF0) { + /* 3-byte character */ + if (end-pointer < 3) return 0; + if (octet == 0xE0 && pointer[1] < 0xA0) return 0; /* overlong */ + if ((pointer[1] & 0xC0) != 0x80) return 0; + if ((pointer[2] & 0xC0) != 0x80) return 0; + return 3; + } else if (octet < 0xF5) { + /* 4-byte character */ + if (end-pointer < 4) return 0; + if (octet == 0xF0 && pointer[1] < 0x90) return 0; /* overlong */ + if (octet == 0xF4 && pointer[1] > 0x8F) return 0; /* > 0x10FFFF */ + if ((pointer[1] & 0xC0) != 0x80) return 0; + if ((pointer[2] & 0xC0) != 0x80) return 0; + if ((pointer[3] & 0xC0) != 0x80) return 0; + return 4; + } else { + /* can never appear in UTF-8 */ + return 0; + } +} + /* * Check if a scalar is valid. */ @@ -1481,7 +1538,6 @@ int space_break = 0; int preceeded_by_whitespace = 0; - int followed_by_whitespace = 0; int previous_space = 0; int previous_break = 0; @@ -1510,10 +1566,17 @@ } preceeded_by_whitespace = 1; - followed_by_whitespace = IS_BLANKZ_AT(string, WIDTH(string)); while (string.pointer != string.end) { + unsigned int width = utf8_width(string.pointer, string.end); + + if (!width) { + special_characters = 1; + string.pointer++; + continue; + } + if (string.start == string.pointer) { if (CHECK(string, '#') || CHECK(string, ',') @@ -1530,12 +1593,12 @@ if (CHECK(string, '?') || CHECK(string, ':')) { flow_indicators = 1; - if (followed_by_whitespace) { + if (IS_BLANKZ_AT(string, 1)) { block_indicators = 1; } } - if (CHECK(string, '-') && followed_by_whitespace) { + if (CHECK(string, '-') && IS_BLANKZ_AT(string, 1)) { flow_indicators = 1; block_indicators = 1; } @@ -1550,7 +1613,7 @@ if (CHECK(string, ':')) { flow_indicators = 1; - if (followed_by_whitespace) { + if (IS_BLANKZ_AT(string, 1)) { block_indicators = 1; } } @@ -1566,10 +1629,6 @@ special_characters = 1; } - if (IS_BREAK(string)) { - line_breaks = 1; - } - if (IS_SPACE(string)) { if (string.start == string.pointer) { @@ -1586,6 +1645,7 @@ } else if (IS_BREAK(string)) { + line_breaks = 1; if (string.start == string.pointer) { leading_break = 1; } @@ -1605,10 +1665,7 @@ } preceeded_by_whitespace = IS_BLANKZ(string); - MOVE(string); - if (string.pointer != string.end) { - followed_by_whitespace = IS_BLANKZ_AT(string, WIDTH(string)); - } + MOVEN(string, width); } emitter->scalar_data.multiline = line_breaks; @@ -1851,18 +1908,15 @@ if (!WRITE(emitter, string)) return 0; } else { - int width = WIDTH(string); - unsigned int value; - while (width --) { - value = *(string.pointer++); - if (!PUT(emitter, '%')) return 0; - if (!PUT(emitter, (value >> 4) - + ((value >> 4) < 10 ? '0' : 'A' - 10))) - return 0; - if (!PUT(emitter, (value & 0x0F) - + ((value & 0x0F) < 10 ? '0' : 'A' - 10))) - return 0; - } + /* %-encode all other bytes, including valid and invalid UTF-8 */ + unsigned char value = *(string.pointer++); + if (!PUT(emitter, '%')) return 0; + if (!PUT(emitter, (value >> 4) + + ((value >> 4) < 10 ? '0' : 'A' - 10))) + return 0; + if (!PUT(emitter, (value & 0x0F) + + ((value & 0x0F) < 10 ? '0' : 'A' - 10))) + return 0; } } @@ -1991,6 +2045,8 @@ return 1; } +static unsigned char utf8_mask[5] = {0xFF, 0x7F, 0x1F, 0x0F, 0x07}; + static int yaml_emitter_write_double_quoted_scalar(yaml_emitter_t *emitter, yaml_char_t *value, size_t length, int allow_breaks) @@ -2003,27 +2059,40 @@ while (string.pointer != string.end) { + unsigned int width = utf8_width(string.pointer, string.end); + + if (width == 0) { + /* + UTF-8 encoding error. This is a byte with the high bit + set. The parser has been altered to read \XNN as a raw + byte. I would prefer to use lowercase x but the old + writer produces that for legal UTF-8 encodings of + U+0080..U+00FF. It is not clear what other parsers will + do with this, though previous libyaml threw an error. + */ + unsigned int value = string.pointer[0]; + int digit; + if (!PUT(emitter, '\\')) return 0; + if (!PUT(emitter, 'X')) return 0; + digit = (value >> 4) & 0x0F; + if (!PUT(emitter, digit + (digit < 10 ? '0' : 'A'-10))) return 0; + digit = value & 0x0F; + if (!PUT(emitter, digit + (digit < 10 ? '0' : 'A'-10))) return 0; + MOVE(string); + spaces = 0; + continue; + } + if (!IS_PRINTABLE(string) || (!emitter->unicode && !IS_ASCII(string)) || IS_BOM(string) || IS_BREAK(string) || CHECK(string, '"') || CHECK(string, '\\')) { - unsigned char octet; - unsigned int width; unsigned int value; int k; - octet = string.pointer[0]; - width = (octet & 0x80) == 0x00 ? 1 : - (octet & 0xE0) == 0xC0 ? 2 : - (octet & 0xF0) == 0xE0 ? 3 : - (octet & 0xF8) == 0xF0 ? 4 : 0; - value = (octet & 0x80) == 0x00 ? octet & 0x7F : - (octet & 0xE0) == 0xC0 ? octet & 0x1F : - (octet & 0xF0) == 0xE0 ? octet & 0x0F : - (octet & 0xF8) == 0xF0 ? octet & 0x07 : 0; + value = string.pointer[0] & utf8_mask[width]; for (k = 1; k < (int)width; k ++) { - octet = string.pointer[k]; - value = (value << 6) + (octet & 0x3F); + value = (value << 6) + (string.pointer[k] & 0x3F); } string.pointer += width; @@ -2092,11 +2161,13 @@ break; default: - if (value <= 0xFF) { + // \xNN sequence is disabled so that it may be used in the + // future for invalid byte sequences + /*if (value <= 0xFF) { if (!PUT(emitter, 'x')) return 0; width = 2; } - else if (value <= 0xFFFF) { + else*/ if (value <= 0xFFFF) { if (!PUT(emitter, 'u')) return 0; width = 4; } @@ -2131,7 +2202,7 @@ } else { - if (!WRITE(emitter, string)) return 0; + if (!WRITEN(emitter, string, width)) return 0; spaces = 0; } } diff -ur original_src/reader.c src/reader.c --- original_src/reader.c 2008-12-27 15:18:21.000000000 -0800 +++ src/reader.c 2009-08-18 16:06:10.000000000 -0700 @@ -187,7 +187,6 @@ while (parser->raw_buffer.pointer != parser->raw_buffer.last) { unsigned int value = 0, value2 = 0; - int incomplete = 0; unsigned char octet; unsigned int width = 0; int low, high; @@ -199,96 +198,22 @@ switch (parser->encoding) { case YAML_UTF8_ENCODING: - - /* - * Decode a UTF-8 character. Check RFC 3629 - * (http://www.ietf.org/rfc/rfc3629.txt) for more details. - * - * The following table (taken from the RFC) is used for - * decoding. - * - * Char. number range | UTF-8 octet sequence - * (hexadecimal) | (binary) - * --------------------+------------------------------------ - * 0000 0000-0000 007F | 0xxxxxxx - * 0000 0080-0000 07FF | 110xxxxx 10xxxxxx - * 0000 0800-0000 FFFF | 1110xxxx 10xxxxxx 10xxxxxx - * 0001 0000-0010 FFFF | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx - * - * Additionally, the characters in the range 0xD800-0xDFFF - * are prohibited as they are reserved for use with UTF-16 - * surrogate pairs. - */ - - /* Determine the length of the UTF-8 sequence. */ - octet = parser->raw_buffer.pointer[0]; - width = (octet & 0x80) == 0x00 ? 1 : - (octet & 0xE0) == 0xC0 ? 2 : - (octet & 0xF0) == 0xE0 ? 3 : - (octet & 0xF8) == 0xF0 ? 4 : 0; - - /* Check if the leading octet is valid. */ - - if (!width) - return yaml_parser_set_reader_error(parser, - "Invalid leading UTF-8 octet", - parser->offset, octet); - - /* Check if the raw buffer contains an incomplete character. */ - - if (width > raw_unread) { - if (parser->eof) { - return yaml_parser_set_reader_error(parser, - "Incomplete UTF-8 octet sequence", - parser->offset, -1); - } - incomplete = 1; - break; - } - - /* Decode the leading octet. */ - - value = (octet & 0x80) == 0x00 ? octet & 0x7F : - (octet & 0xE0) == 0xC0 ? octet & 0x1F : - (octet & 0xF0) == 0xE0 ? octet & 0x0F : - (octet & 0xF8) == 0xF0 ? octet & 0x07 : 0; - - /* Check and decode the trailing octets. */ - - for (k = 1; k < width; k ++) - { - octet = parser->raw_buffer.pointer[k]; - - /* Check if the octet is valid. */ - - if ((octet & 0xC0) != 0x80) - return yaml_parser_set_reader_error(parser, - "Invalid trailing UTF-8 octet", - parser->offset+k, octet); - - /* Decode the octet. */ - - value = (value << 6) + (octet & 0x3F); - } - - /* Check the length of the sequence against the value. */ - - if (!((width == 1) || - (width == 2 && value >= 0x80) || - (width == 3 && value >= 0x800) || - (width == 4 && value >= 0x10000))) - return yaml_parser_set_reader_error(parser, - "Invalid length of a UTF-8 sequence", - parser->offset, -1); - - /* Check the range of the value. */ - - if ((value >= 0xD800 && value <= 0xDFFF) || value > 0x10FFFF) + /* + We can only disallow characters without the high + bit set. Characters with the high bit set are required + for invalid UTF-8 strings to be encoded, as we + cannot rely on any backslash sequences working. + */ + if (! (octet == 0x09 || octet == 0x0A || octet == 0x0D + || (octet >= 0x20 && octet != 0x7F) )) return yaml_parser_set_reader_error(parser, - "Invalid Unicode character", + "Control characters are not allowed", parser->offset, value); + parser->raw_buffer.pointer++; + parser->offset++; + *(parser->buffer.last++) = octet; break; case YAML_UTF16LE_ENCODING: @@ -331,7 +256,6 @@ "Incomplete UTF-16 character", parser->offset, -1); } - incomplete = 1; break; } @@ -339,109 +263,88 @@ value = parser->raw_buffer.pointer[low] + (parser->raw_buffer.pointer[high] << 8); - - /* Check for unexpected low surrogate area. */ - - if ((value & 0xFC00) == 0xDC00) - return yaml_parser_set_reader_error(parser, - "Unexpected low surrogate area", - parser->offset, value); + width = 2; /* Check for a high surrogate area. */ if ((value & 0xFC00) == 0xD800) { - width = 4; - /* Check for incomplete surrogate pair. */ if (raw_unread < 4) { - if (parser->eof) { - return yaml_parser_set_reader_error(parser, - "Incomplete UTF-16 surrogate pair", - parser->offset, -1); + if (parser->eof) { /* trailing high surrogate */ + width = 2; + } else { + break; /* Can't tell until we have more raw characters */ + } + } else { + + /* Get the next character. */ + + value2 = parser->raw_buffer.pointer[low+2] + + (parser->raw_buffer.pointer[high+2] << 8); + + /* Check for a low surrogate area. */ + if ((value2 & 0xFC00) == 0xDC00) { + width = 4; + /* Generate the value of the surrogate pair. */ + value = 0x10000 + ((value & 0x3FF) << 10) + (value2 & 0x3FF); } - incomplete = 1; - break; } + } - /* Get the next character. */ + /* Check if the raw buffer contains enough bytes to form a character. */ - value2 = parser->raw_buffer.pointer[low+2] - + (parser->raw_buffer.pointer[high+2] << 8); + /* + * Check if the character is in the allowed range: + * #x9 | #xA | #xD | [#x20-#x7E] (8 bit) + * | #x85 | [#xA0-#xD7FF] | [#xE000-#xFFFD] (16 bit) + * | [#x10000-#x10FFFF] (32 bit) + */ + /* Modified to allow all 16-bit values as \uNNNN may not + work for some parsers for these values. */ + if (! (value == 0x09 || value == 0x0A || value == 0x0D + || (value >= 0x20 && value <= 0x7E) + || (value == 0x85) || value >= 0xA0)) + return yaml_parser_set_reader_error(parser, + "Control characters are not allowed", + parser->offset, value); - /* Check for a low surrogate area. */ + /* Move the raw pointers. */ - if ((value2 & 0xFC00) != 0xDC00) - return yaml_parser_set_reader_error(parser, - "Expected low surrogate area", - parser->offset+2, value2); + parser->raw_buffer.pointer += width; + parser->offset += width; - /* Generate the value of the surrogate pair. */ + /* Finally put the character into the buffer. */ - value = 0x10000 + ((value & 0x3FF) << 10) + (value2 & 0x3FF); + /* 0000 0000-0000 007F -> 0xxxxxxx */ + if (value <= 0x7F) { + *(parser->buffer.last++) = value; } - + /* 0000 0080-0000 07FF -> 110xxxxx 10xxxxxx */ + else if (value <= 0x7FF) { + *(parser->buffer.last++) = 0xC0 + (value >> 6); + *(parser->buffer.last++) = 0x80 + (value & 0x3F); + } + /* 0000 0800-0000 FFFF -> 1110xxxx 10xxxxxx 10xxxxxx */ + else if (value <= 0xFFFF) { + *(parser->buffer.last++) = 0xE0 + (value >> 12); + *(parser->buffer.last++) = 0x80 + ((value >> 6) & 0x3F); + *(parser->buffer.last++) = 0x80 + (value & 0x3F); + } + /* 0001 0000-0010 FFFF -> 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */ else { - width = 2; + *(parser->buffer.last++) = 0xF0 + (value >> 18); + *(parser->buffer.last++) = 0x80 + ((value >> 12) & 0x3F); + *(parser->buffer.last++) = 0x80 + ((value >> 6) & 0x3F); + *(parser->buffer.last++) = 0x80 + (value & 0x3F); } - break; default: assert(1); /* Impossible. */ } - /* Check if the raw buffer contains enough bytes to form a character. */ - - if (incomplete) break; - - /* - * Check if the character is in the allowed range: - * #x9 | #xA | #xD | [#x20-#x7E] (8 bit) - * | #x85 | [#xA0-#xD7FF] | [#xE000-#xFFFD] (16 bit) - * | [#x10000-#x10FFFF] (32 bit) - */ - - if (! (value == 0x09 || value == 0x0A || value == 0x0D - || (value >= 0x20 && value <= 0x7E) - || (value == 0x85) || (value >= 0xA0 && value <= 0xD7FF) - || (value >= 0xE000 && value <= 0xFFFD) - || (value >= 0x10000 && value <= 0x10FFFF))) - return yaml_parser_set_reader_error(parser, - "Control characters are not allowed", - parser->offset, value); - - /* Move the raw pointers. */ - - parser->raw_buffer.pointer += width; - parser->offset += width; - - /* Finally put the character into the buffer. */ - - /* 0000 0000-0000 007F -> 0xxxxxxx */ - if (value <= 0x7F) { - *(parser->buffer.last++) = value; - } - /* 0000 0080-0000 07FF -> 110xxxxx 10xxxxxx */ - else if (value <= 0x7FF) { - *(parser->buffer.last++) = 0xC0 + (value >> 6); - *(parser->buffer.last++) = 0x80 + (value & 0x3F); - } - /* 0000 0800-0000 FFFF -> 1110xxxx 10xxxxxx 10xxxxxx */ - else if (value <= 0xFFFF) { - *(parser->buffer.last++) = 0xE0 + (value >> 12); - *(parser->buffer.last++) = 0x80 + ((value >> 6) & 0x3F); - *(parser->buffer.last++) = 0x80 + (value & 0x3F); - } - /* 0001 0000-0010 FFFF -> 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */ - else { - *(parser->buffer.last++) = 0xF0 + (value >> 18); - *(parser->buffer.last++) = 0x80 + ((value >> 12) & 0x3F); - *(parser->buffer.last++) = 0x80 + ((value >> 6) & 0x3F); - *(parser->buffer.last++) = 0x80 + (value & 0x3F); - } - parser->unread ++; } diff -ur original_src/scanner.c src/scanner.c --- original_src/scanner.c 2008-12-27 15:18:21.000000000 -0800 +++ src/scanner.c 2009-08-18 17:56:35.000000000 -0700 @@ -495,8 +495,13 @@ (parser->mark.index ++, \ parser->mark.column ++, \ parser->unread --, \ - parser->buffer.pointer += WIDTH(parser->buffer)) + parser->buffer.pointer ++) +#define SKIPN(parser,n) \ + (parser->mark.index ++, \ + parser->mark.column ++, \ + parser->unread --, \ + parser->buffer.pointer += (n)) #define SKIP_LINE(parser) \ (IS_CRLF(parser->buffer) ? \ (parser->mark.index += 2, \ @@ -523,6 +528,14 @@ parser->unread --, \ 1) : 0) +#define READN(parser,string,n) \ + (STRING_EXTEND(parser,string) ? \ + (COPYN(string,parser->buffer,n), \ + parser->mark.index ++, \ + parser->mark.column ++, \ + parser->unread --, \ + 1) : 0) + /* * Copy a line break character to a string buffer and advance pointers. */ @@ -1924,7 +1937,7 @@ if (!CACHE(parser, 1)) return 0; if (parser->mark.column == 0 && IS_BOM(parser->buffer)) - SKIP(parser); + SKIPN(parser,3); /* UTF-8 BOM is 3 bytes */ /* * Eat whitespaces. @@ -2657,63 +2670,30 @@ yaml_parser_scan_uri_escapes(yaml_parser_t *parser, int directive, yaml_mark_t start_mark, yaml_string_t *string) { - int width = 0; - - /* Decode the required number of characters. */ - - do { - - unsigned char octet = 0; - - /* Check for a URI-escaped octet. */ - - if (!CACHE(parser, 3)) return 0; - - if (!(CHECK(parser->buffer, '%') - && IS_HEX_AT(parser->buffer, 1) - && IS_HEX_AT(parser->buffer, 2))) { - return yaml_parser_set_scanner_error(parser, directive ? - "while parsing a %TAG directive" : "while parsing a tag", - start_mark, "did not find URI escaped octet"); - } - - /* Get the octet. */ + unsigned char octet = 0; - octet = (AS_HEX_AT(parser->buffer, 1) << 4) + AS_HEX_AT(parser->buffer, 2); + /* Check for a URI-escaped octet. */ - /* If it is the leading octet, determine the length of the UTF-8 sequence. */ + if (!CACHE(parser, 3)) return 0; - if (!width) - { - width = (octet & 0x80) == 0x00 ? 1 : - (octet & 0xE0) == 0xC0 ? 2 : - (octet & 0xF0) == 0xE0 ? 3 : - (octet & 0xF8) == 0xF0 ? 4 : 0; - if (!width) { - return yaml_parser_set_scanner_error(parser, directive ? - "while parsing a %TAG directive" : "while parsing a tag", - start_mark, "found an incorrect leading UTF-8 octet"); - } - } - else - { - /* Check if the trailing octet is correct. */ + if (!(CHECK(parser->buffer, '%') + && IS_HEX_AT(parser->buffer, 1) + && IS_HEX_AT(parser->buffer, 2))) { + return yaml_parser_set_scanner_error(parser, directive ? + "while parsing a %TAG directive" : "while parsing a tag", + start_mark, "did not find URI escaped octet"); + } - if ((octet & 0xC0) != 0x80) { - return yaml_parser_set_scanner_error(parser, directive ? - "while parsing a %TAG directive" : "while parsing a tag", - start_mark, "found an incorrect trailing UTF-8 octet"); - } - } + /* Get the octet. */ - /* Copy the octet and move the pointers. */ + octet = (AS_HEX_AT(parser->buffer, 1) << 4) + AS_HEX_AT(parser->buffer, 2); - *(string->pointer++) = octet; - SKIP(parser); - SKIP(parser); - SKIP(parser); + /* Copy the octet and move the pointers. */ - } while (--width); + *(string->pointer++) = octet; + SKIP(parser); + SKIP(parser); + SKIP(parser); return 1; } @@ -3102,6 +3082,7 @@ else if (!single && CHECK(parser->buffer, '\\')) { size_t code_length = 0; + int raw_code = 0; if (!STRING_EXTEND(parser, string)) goto error; @@ -3188,6 +3169,11 @@ code_length = 2; break; + case 'X': + code_length = 2; + raw_code = 1; + break; + case 'u': code_length = 4; break; @@ -3233,7 +3219,7 @@ goto error; } - if (value <= 0x7F) { + if (value <= 0x7F || raw_code) { *(string.pointer++) = value; } else if (value <= 0x7FF) { @@ -3262,7 +3248,7 @@ else { - /* It is a non-escaped non-blank character. */ + /* It is a non-escaped non-blank byte. */ if (!READ(parser, string)) goto error; } @@ -3474,7 +3460,7 @@ } } - /* Copy the character. */ + /* Copy the byte. */ if (!READ(parser, string)) goto error; diff -ur original_src/yaml_private.h src/yaml_private.h --- original_src/yaml_private.h 2008-12-27 15:18:21.000000000 -0800 +++ src/yaml_private.h 2009-08-18 17:57:05.000000000 -0700 @@ -25,7 +25,7 @@ yaml_strdup(const yaml_char_t *); /* - * Reader: Ensure that the buffer contains at least `length` characters. + * Reader: Ensure that the buffer contains at least `length` bytes. */ YAML_DECLARE(int) @@ -237,6 +237,7 @@ /* * Check if the character can be printed unescaped. + * Only correct if you know you are looking at valid UTF-8! */ #define IS_PRINTABLE_AT(string,offset) \ @@ -355,6 +356,7 @@ /* * Determine the width of the character. + * Only correct if you know you are looking at valid UTF-8! */ #define WIDTH_AT(string,offset) \ @@ -366,30 +368,23 @@ #define WIDTH(string) WIDTH_AT((string),0) /* - * Move the string pointer to the next character. + * Move the string pointer to the next byte. */ -#define MOVE(string) ((string).pointer += WIDTH((string))) +#define MOVE(string) ((string).pointer++) + +#define MOVEN(string,n) ((string).pointer += n) /* - * Copy a character and move the pointers of both strings. + * Copy a byte and move the pointers of both strings. */ #define COPY(string_a,string_b) \ - ((*(string_b).pointer & 0x80) == 0x00 ? \ - (*((string_a).pointer++) = *((string_b).pointer++)) : \ - (*(string_b).pointer & 0xE0) == 0xC0 ? \ - (*((string_a).pointer++) = *((string_b).pointer++), \ - *((string_a).pointer++) = *((string_b).pointer++)) : \ - (*(string_b).pointer & 0xF0) == 0xE0 ? \ - (*((string_a).pointer++) = *((string_b).pointer++), \ - *((string_a).pointer++) = *((string_b).pointer++), \ - *((string_a).pointer++) = *((string_b).pointer++)) : \ - (*(string_b).pointer & 0xF8) == 0xF0 ? \ - (*((string_a).pointer++) = *((string_b).pointer++), \ - *((string_a).pointer++) = *((string_b).pointer++), \ - *((string_a).pointer++) = *((string_b).pointer++), \ - *((string_a).pointer++) = *((string_b).pointer++)) : 0) + (*((string_a).pointer++) = *((string_b).pointer++)) + +#define COPYN(string_a,string_b,n) \ + (memcpy(string_a.pointer, string_b.pointer, n), \ + string_a.pointer += n, string_b.pointer += n) /* * Stack and queue management.