Context Navigation

Back to Ticket #138

Ticket #138: patch.2

File patch.2, 37.8 KB (added by spitzak@…, 4 years ago)
New patch that fixes handling of %nn in tags

Line
1	diff -ur original_src/api.c src/api.c
2	--- original_src/api.c 2008-12-27 15:18:21.000000000 -0800
3	+++ src/api.c 2009-08-18 18:18:41.000000000 -0700
4	@@ -611,51 +611,6 @@
5	}
6
7	/*
8	- * Check if a string is a valid UTF-8 sequence.
9	- *
10	- * Check 'reader.c' for more details on UTF-8 encoding.
11	- */
12	-
13	-static int
14	-yaml_check_utf8(yaml_char_t *start, size_t length)
15	-{
16	- yaml_char_t *end = start+length;
17	- yaml_char_t *pointer = start;
18	-
19	- while (pointer < end) {
20	- unsigned char octet;
21	- unsigned int width;
22	- unsigned int value;
23	- size_t k;
24	-
25	- octet = pointer[0];
26	- width = (octet & 0x80) == 0x00 ? 1 :
27	- (octet & 0xE0) == 0xC0 ? 2 :
28	- (octet & 0xF0) == 0xE0 ? 3 :
29	- (octet & 0xF8) == 0xF0 ? 4 : 0;
30	- value = (octet & 0x80) == 0x00 ? octet & 0x7F :
31	- (octet & 0xE0) == 0xC0 ? octet & 0x1F :
32	- (octet & 0xF0) == 0xE0 ? octet & 0x0F :
33	- (octet & 0xF8) == 0xF0 ? octet & 0x07 : 0;
34	- if (!width) return 0;
35	- if (pointer+width > end) return 0;
36	- for (k = 1; k < width; k ++) {
37	- octet = pointer[k];
38	- if ((octet & 0xC0) != 0x80) return 0;
39	- value = (value << 6) + (octet & 0x3F);
40	- }
41	- if (!((width == 1) \|\|
42	- (width == 2 && value >= 0x80) \|\|
43	- (width == 3 && value >= 0x800) \|\|
44	- (width == 4 && value >= 0x10000))) return 0;
45	-
46	- pointer += width;
47	- }
48	-
49	- return 1;
50	-}
51	-
52	-/*
53	* Create STREAM-START.
54	*/
55
56	@@ -731,12 +686,6 @@
57	tag_directive != tag_directives_end; tag_directive ++) {
58	assert(tag_directive->handle);
59	assert(tag_directive->prefix);
60	- if (!yaml_check_utf8(tag_directive->handle,
61	- strlen((char *)tag_directive->handle)))
62	- goto error;
63	- if (!yaml_check_utf8(tag_directive->prefix,
64	- strlen((char *)tag_directive->prefix)))
65	- goto error;
66	value.handle = yaml_strdup(tag_directive->handle);
67	value.prefix = yaml_strdup(tag_directive->prefix);
68	if (!value.handle \|\| !value.prefix) goto error;
69	@@ -796,8 +745,6 @@
70	assert(event); /* Non-NULL event object is expected. */
71	assert(anchor); /* Non-NULL anchor is expected. */
72
73	- if (!yaml_check_utf8(anchor, strlen((char *)anchor))) return 0;
74	-
75	anchor_copy = yaml_strdup(anchor);
76	if (!anchor_copy)
77	return 0;
78	@@ -827,13 +774,11 @@
79	assert(value); /* Non-NULL anchor is expected. */
80
81	if (anchor) {
82	- if (!yaml_check_utf8(anchor, strlen((char *)anchor))) goto error;
83	anchor_copy = yaml_strdup(anchor);
84	if (!anchor_copy) goto error;
85	}
86
87	if (tag) {
88	- if (!yaml_check_utf8(tag, strlen((char *)tag))) goto error;
89	tag_copy = yaml_strdup(tag);
90	if (!tag_copy) goto error;
91	}
92	@@ -842,7 +787,6 @@
93	length = strlen((char *)value);
94	}
95
96	- if (!yaml_check_utf8(value, length)) goto error;
97	value_copy = yaml_malloc(length+1);
98	if (!value_copy) goto error;
99	memcpy(value_copy, value, length);
100	@@ -877,13 +821,11 @@
101	assert(event); /* Non-NULL event object is expected. */
102
103	if (anchor) {
104	- if (!yaml_check_utf8(anchor, strlen((char *)anchor))) goto error;
105	anchor_copy = yaml_strdup(anchor);
106	if (!anchor_copy) goto error;
107	}
108
109	if (tag) {
110	- if (!yaml_check_utf8(tag, strlen((char *)tag))) goto error;
111	tag_copy = yaml_strdup(tag);
112	if (!tag_copy) goto error;
113	}
114	@@ -932,13 +874,11 @@
115	assert(event); /* Non-NULL event object is expected. */
116
117	if (anchor) {
118	- if (!yaml_check_utf8(anchor, strlen((char *)anchor))) goto error;
119	anchor_copy = yaml_strdup(anchor);
120	if (!anchor_copy) goto error;
121	}
122
123	if (tag) {
124	- if (!yaml_check_utf8(tag, strlen((char *)tag))) goto error;
125	tag_copy = yaml_strdup(tag);
126	if (!tag_copy) goto error;
127	}
128	@@ -1072,12 +1012,6 @@
129	tag_directive != tag_directives_end; tag_directive ++) {
130	assert(tag_directive->handle);
131	assert(tag_directive->prefix);
132	- if (!yaml_check_utf8(tag_directive->handle,
133	- strlen((char *)tag_directive->handle)))
134	- goto error;
135	- if (!yaml_check_utf8(tag_directive->prefix,
136	- strlen((char *)tag_directive->prefix)))
137	- goto error;
138	value.handle = yaml_strdup(tag_directive->handle);
139	value.prefix = yaml_strdup(tag_directive->prefix);
140	if (!value.handle \|\| !value.prefix) goto error;
141	@@ -1210,7 +1144,6 @@
142	tag = (yaml_char_t *)YAML_DEFAULT_SCALAR_TAG;
143	}
144
145	- if (!yaml_check_utf8(tag, strlen((char *)tag))) goto error;
146	tag_copy = yaml_strdup(tag);
147	if (!tag_copy) goto error;
148
149	@@ -1218,7 +1151,6 @@
150	length = strlen((char *)value);
151	}
152
153	- if (!yaml_check_utf8(value, length)) goto error;
154	value_copy = yaml_malloc(length+1);
155	if (!value_copy) goto error;
156	memcpy(value_copy, value, length);
157	@@ -1262,7 +1194,6 @@
158	tag = (yaml_char_t *)YAML_DEFAULT_SEQUENCE_TAG;
159	}
160
161	- if (!yaml_check_utf8(tag, strlen((char *)tag))) goto error;
162	tag_copy = yaml_strdup(tag);
163	if (!tag_copy) goto error;
164
165	@@ -1307,7 +1238,6 @@
166	tag = (yaml_char_t *)YAML_DEFAULT_MAPPING_TAG;
167	}
168
169	- if (!yaml_check_utf8(tag, strlen((char *)tag))) goto error;
170	tag_copy = yaml_strdup(tag);
171	if (!tag_copy) goto error;
172
173	diff -ur original_src/emitter.c src/emitter.c
174	--- original_src/emitter.c 2008-12-27 15:18:21.000000000 -0800
175	+++ src/emitter.c 2009-08-18 17:50:47.000000000 -0700
176	@@ -37,7 +37,7 @@
177	1))
178
179	/*
180	- * Copy a character from a string into buffer.
181	+ * Copy a byte from a string into buffer.
182	*/
183
184	#define WRITE(emitter,string) \
185	@@ -46,6 +46,12 @@
186	emitter->column ++, \
187	1))
188
189	+#define WRITEN(emitter,string,n) \
190	+ (FLUSH(emitter) \
191	+ && (COPYN(emitter->buffer,string,n), \
192	+ emitter->column ++, \
193	+ 1))
194	+
195	/*
196	* Copy a line break character from a string into buffer.
197	*/
198	@@ -56,7 +62,7 @@
199	(PUT_BREAK(emitter), \
200	string.pointer ++, \
201	1) : \
202	- (COPY(emitter->buffer,string), \
203	+ (COPYN(emitter->buffer,string,WIDTH(string)), \
204	emitter->column = 0, \
205	emitter->line ++, \
206	1)))
207	@@ -1458,6 +1464,57 @@
208	return 1;
209	}
210
211	+/**
212	+ Return the number of bytes in the next UTF-8 character.
213	+
214	+ Returns 0 for any error, includinge incorrect bytes, not enough
215	+ bytes before the end pointer, and overlong encodings. If 0 is
216	+ returned, pointer[0] will always have the high bit set and will
217	+ thus never match any ASCII character.
218	+
219	+ The encodings of surrogate halves are allowed! Otherwise it is not
220	+ possible to losslessly encode invalid UTF-16 into UTF-8. (There is
221	+ a vocal contingent trying to sabotage UTF-8 by declaring surrogate
222	+ half encodings invalid. Anybody such claim should be investigated
223	+ carefully: if that user's code does not also reject invalid UTF-16,
224	+ then they are being hypocrites and can be ignored.)
225	+*/
226	+static unsigned int utf8_width(yaml_char_t* pointer, yaml_char_t* end)
227	+{
228	+ unsigned char octet = pointer[0];
229	+ if (octet < 0x80) {
230	+ /* 1-byte character */
231	+ return 1;
232	+ } else if (octet < 0xC2) {
233	+ /* continuation byte or overlong 2-byte encoding */
234	+ return 0;
235	+ } else if (octet < 0xE0) {
236	+ /* 2-byte character */
237	+ if (end-pointer < 2) return 0;
238	+ if ((pointer[1] & 0xC0) != 0x80) return 0;
239	+ return 2;
240	+ } else if (octet < 0xF0) {
241	+ /* 3-byte character */
242	+ if (end-pointer < 3) return 0;
243	+ if (octet == 0xE0 && pointer[1] < 0xA0) return 0; /* overlong */
244	+ if ((pointer[1] & 0xC0) != 0x80) return 0;
245	+ if ((pointer[2] & 0xC0) != 0x80) return 0;
246	+ return 3;
247	+ } else if (octet < 0xF5) {
248	+ /* 4-byte character */
249	+ if (end-pointer < 4) return 0;
250	+ if (octet == 0xF0 && pointer[1] < 0x90) return 0; /* overlong */
251	+ if (octet == 0xF4 && pointer[1] > 0x8F) return 0; /* > 0x10FFFF */
252	+ if ((pointer[1] & 0xC0) != 0x80) return 0;
253	+ if ((pointer[2] & 0xC0) != 0x80) return 0;
254	+ if ((pointer[3] & 0xC0) != 0x80) return 0;
255	+ return 4;
256	+ } else {
257	+ /* can never appear in UTF-8 */
258	+ return 0;
259	+ }
260	+}
261	+
262	/*
263	* Check if a scalar is valid.
264	*/
265	@@ -1481,7 +1538,6 @@
266	int space_break = 0;
267
268	int preceeded_by_whitespace = 0;
269	- int followed_by_whitespace = 0;
270	int previous_space = 0;
271	int previous_break = 0;
272
273	@@ -1510,10 +1566,17 @@
274	}
275
276	preceeded_by_whitespace = 1;
277	- followed_by_whitespace = IS_BLANKZ_AT(string, WIDTH(string));
278
279	while (string.pointer != string.end)
280	{
281	+ unsigned int width = utf8_width(string.pointer, string.end);
282	+
283	+ if (!width) {
284	+ special_characters = 1;
285	+ string.pointer++;
286	+ continue;
287	+ }
288	+
289	if (string.start == string.pointer)
290	{
291	if (CHECK(string, '#') \|\| CHECK(string, ',')
292	@@ -1530,12 +1593,12 @@
293
294	if (CHECK(string, '?') \|\| CHECK(string, ':')) {
295	flow_indicators = 1;
296	- if (followed_by_whitespace) {
297	+ if (IS_BLANKZ_AT(string, 1)) {
298	block_indicators = 1;
299	}
300	}
301
302	- if (CHECK(string, '-') && followed_by_whitespace) {
303	+ if (CHECK(string, '-') && IS_BLANKZ_AT(string, 1)) {
304	flow_indicators = 1;
305	block_indicators = 1;
306	}
307	@@ -1550,7 +1613,7 @@
308
309	if (CHECK(string, ':')) {
310	flow_indicators = 1;
311	- if (followed_by_whitespace) {
312	+ if (IS_BLANKZ_AT(string, 1)) {
313	block_indicators = 1;
314	}
315	}
316	@@ -1566,10 +1629,6 @@
317	special_characters = 1;
318	}
319
320	- if (IS_BREAK(string)) {
321	- line_breaks = 1;
322	- }
323	-
324	if (IS_SPACE(string))
325	{
326	if (string.start == string.pointer) {
327	@@ -1586,6 +1645,7 @@
328	}
329	else if (IS_BREAK(string))
330	{
331	+ line_breaks = 1;
332	if (string.start == string.pointer) {
333	leading_break = 1;
334	}
335	@@ -1605,10 +1665,7 @@
336	}
337
338	preceeded_by_whitespace = IS_BLANKZ(string);
339	- MOVE(string);
340	- if (string.pointer != string.end) {
341	- followed_by_whitespace = IS_BLANKZ_AT(string, WIDTH(string));
342	- }
343	+ MOVEN(string, width);
344	}
345
346	emitter->scalar_data.multiline = line_breaks;
347	@@ -1851,18 +1908,15 @@
348	if (!WRITE(emitter, string)) return 0;
349	}
350	else {
351	- int width = WIDTH(string);
352	- unsigned int value;
353	- while (width --) {
354	- value = *(string.pointer++);
355	- if (!PUT(emitter, '%')) return 0;
356	- if (!PUT(emitter, (value >> 4)
357	- + ((value >> 4) < 10 ? '0' : 'A' - 10)))
358	- return 0;
359	- if (!PUT(emitter, (value & 0x0F)
360	- + ((value & 0x0F) < 10 ? '0' : 'A' - 10)))
361	- return 0;
362	- }
363	+ /* %-encode all other bytes, including valid and invalid UTF-8 */
364	+ unsigned char value = *(string.pointer++);
365	+ if (!PUT(emitter, '%')) return 0;
366	+ if (!PUT(emitter, (value >> 4)
367	+ + ((value >> 4) < 10 ? '0' : 'A' - 10)))
368	+ return 0;
369	+ if (!PUT(emitter, (value & 0x0F)
370	+ + ((value & 0x0F) < 10 ? '0' : 'A' - 10)))
371	+ return 0;
372	}
373	}
374
375	@@ -1991,6 +2045,8 @@
376	return 1;
377	}
378
379	+static unsigned char utf8_mask[5] = {0xFF, 0x7F, 0x1F, 0x0F, 0x07};
380	+
381	static int
382	yaml_emitter_write_double_quoted_scalar(yaml_emitter_t *emitter,
383	yaml_char_t *value, size_t length, int allow_breaks)
384	@@ -2003,27 +2059,40 @@
385
386	while (string.pointer != string.end)
387	{
388	+ unsigned int width = utf8_width(string.pointer, string.end);
389	+
390	+ if (width == 0) {
391	+ /*
392	+ UTF-8 encoding error. This is a byte with the high bit
393	+ set. The parser has been altered to read \XNN as a raw
394	+ byte. I would prefer to use lowercase x but the old
395	+ writer produces that for legal UTF-8 encodings of
396	+ U+0080..U+00FF. It is not clear what other parsers will
397	+ do with this, though previous libyaml threw an error.
398	+ */
399	+ unsigned int value = string.pointer[0];
400	+ int digit;
401	+ if (!PUT(emitter, '\\')) return 0;
402	+ if (!PUT(emitter, 'X')) return 0;
403	+ digit = (value >> 4) & 0x0F;
404	+ if (!PUT(emitter, digit + (digit < 10 ? '0' : 'A'-10))) return 0;
405	+ digit = value & 0x0F;
406	+ if (!PUT(emitter, digit + (digit < 10 ? '0' : 'A'-10))) return 0;
407	+ MOVE(string);
408	+ spaces = 0;
409	+ continue;
410	+ }
411	+
412	if (!IS_PRINTABLE(string) \|\| (!emitter->unicode && !IS_ASCII(string))
413	\|\| IS_BOM(string) \|\| IS_BREAK(string)
414	\|\| CHECK(string, '"') \|\| CHECK(string, '\\'))
415	{
416	- unsigned char octet;
417	- unsigned int width;
418	unsigned int value;
419	int k;
420
421	- octet = string.pointer[0];
422	- width = (octet & 0x80) == 0x00 ? 1 :
423	- (octet & 0xE0) == 0xC0 ? 2 :
424	- (octet & 0xF0) == 0xE0 ? 3 :
425	- (octet & 0xF8) == 0xF0 ? 4 : 0;
426	- value = (octet & 0x80) == 0x00 ? octet & 0x7F :
427	- (octet & 0xE0) == 0xC0 ? octet & 0x1F :
428	- (octet & 0xF0) == 0xE0 ? octet & 0x0F :
429	- (octet & 0xF8) == 0xF0 ? octet & 0x07 : 0;
430	+ value = string.pointer[0] & utf8_mask[width];
431	for (k = 1; k < (int)width; k ++) {
432	- octet = string.pointer[k];
433	- value = (value << 6) + (octet & 0x3F);
434	+ value = (value << 6) + (string.pointer[k] & 0x3F);
435	}
436	string.pointer += width;
437
438	@@ -2092,11 +2161,13 @@
439	break;
440
441	default:
442	- if (value <= 0xFF) {
443	+ // \xNN sequence is disabled so that it may be used in the
444	+ // future for invalid byte sequences
445	+ /*if (value <= 0xFF) {
446	if (!PUT(emitter, 'x')) return 0;
447	width = 2;
448	}
449	- else if (value <= 0xFFFF) {
450	+ else*/ if (value <= 0xFFFF) {
451	if (!PUT(emitter, 'u')) return 0;
452	width = 4;
453	}
454	@@ -2131,7 +2202,7 @@
455	}
456	else
457	{
458	- if (!WRITE(emitter, string)) return 0;
459	+ if (!WRITEN(emitter, string, width)) return 0;
460	spaces = 0;
461	}
462	}
463	diff -ur original_src/reader.c src/reader.c
464	--- original_src/reader.c 2008-12-27 15:18:21.000000000 -0800
465	+++ src/reader.c 2009-08-18 16:06:10.000000000 -0700
466	@@ -187,7 +187,6 @@
467	while (parser->raw_buffer.pointer != parser->raw_buffer.last)
468	{
469	unsigned int value = 0, value2 = 0;
470	- int incomplete = 0;
471	unsigned char octet;
472	unsigned int width = 0;
473	int low, high;
474	@@ -199,96 +198,22 @@
475	switch (parser->encoding)
476	{
477	case YAML_UTF8_ENCODING:
478	-
479	- /*
480	- * Decode a UTF-8 character. Check RFC 3629
481	- * (http://www.ietf.org/rfc/rfc3629.txt) for more details.
482	- *
483	- * The following table (taken from the RFC) is used for
484	- * decoding.
485	- *
486	- * Char. number range \| UTF-8 octet sequence
487	- * (hexadecimal) \| (binary)
488	- * --------------------+------------------------------------
489	- * 0000 0000-0000 007F \| 0xxxxxxx
490	- * 0000 0080-0000 07FF \| 110xxxxx 10xxxxxx
491	- * 0000 0800-0000 FFFF \| 1110xxxx 10xxxxxx 10xxxxxx
492	- * 0001 0000-0010 FFFF \| 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
493	- *
494	- * Additionally, the characters in the range 0xD800-0xDFFF
495	- * are prohibited as they are reserved for use with UTF-16
496	- * surrogate pairs.
497	- */
498	-
499	- /* Determine the length of the UTF-8 sequence. */
500	-
501	octet = parser->raw_buffer.pointer[0];
502	- width = (octet & 0x80) == 0x00 ? 1 :
503	- (octet & 0xE0) == 0xC0 ? 2 :
504	- (octet & 0xF0) == 0xE0 ? 3 :
505	- (octet & 0xF8) == 0xF0 ? 4 : 0;
506	-
507	- /* Check if the leading octet is valid. */
508	-
509	- if (!width)
510	- return yaml_parser_set_reader_error(parser,
511	- "Invalid leading UTF-8 octet",
512	- parser->offset, octet);
513	-
514	- /* Check if the raw buffer contains an incomplete character. */
515	-
516	- if (width > raw_unread) {
517	- if (parser->eof) {
518	- return yaml_parser_set_reader_error(parser,
519	- "Incomplete UTF-8 octet sequence",
520	- parser->offset, -1);
521	- }
522	- incomplete = 1;
523	- break;
524	- }
525	-
526	- /* Decode the leading octet. */
527	-
528	- value = (octet & 0x80) == 0x00 ? octet & 0x7F :
529	- (octet & 0xE0) == 0xC0 ? octet & 0x1F :
530	- (octet & 0xF0) == 0xE0 ? octet & 0x0F :
531	- (octet & 0xF8) == 0xF0 ? octet & 0x07 : 0;
532	-
533	- /* Check and decode the trailing octets. */
534	-
535	- for (k = 1; k < width; k ++)
536	- {
537	- octet = parser->raw_buffer.pointer[k];
538	-
539	- /* Check if the octet is valid. */
540	-
541	- if ((octet & 0xC0) != 0x80)
542	- return yaml_parser_set_reader_error(parser,
543	- "Invalid trailing UTF-8 octet",
544	- parser->offset+k, octet);
545	-
546	- /* Decode the octet. */
547	-
548	- value = (value << 6) + (octet & 0x3F);
549	- }
550	-
551	- /* Check the length of the sequence against the value. */
552	-
553	- if (!((width == 1) \|\|
554	- (width == 2 && value >= 0x80) \|\|
555	- (width == 3 && value >= 0x800) \|\|
556	- (width == 4 && value >= 0x10000)))
557	- return yaml_parser_set_reader_error(parser,
558	- "Invalid length of a UTF-8 sequence",
559	- parser->offset, -1);
560	-
561	- /* Check the range of the value. */
562	-
563	- if ((value >= 0xD800 && value <= 0xDFFF) \|\| value > 0x10FFFF)
564	+ /*
565	+ We can only disallow characters without the high
566	+ bit set. Characters with the high bit set are required
567	+ for invalid UTF-8 strings to be encoded, as we
568	+ cannot rely on any backslash sequences working.
569	+ */
570	+ if (! (octet == 0x09 \|\| octet == 0x0A \|\| octet == 0x0D
571	+ \|\| (octet >= 0x20 && octet != 0x7F) ))
572	return yaml_parser_set_reader_error(parser,
573	- "Invalid Unicode character",
574	+ "Control characters are not allowed",
575	parser->offset, value);
576
577	+ parser->raw_buffer.pointer++;
578	+ parser->offset++;
579	+ *(parser->buffer.last++) = octet;
580	break;
581
582	case YAML_UTF16LE_ENCODING:
583	@@ -331,7 +256,6 @@
584	"Incomplete UTF-16 character",
585	parser->offset, -1);
586	}
587	- incomplete = 1;
588	break;
589	}
590
591	@@ -339,109 +263,88 @@
592
593	value = parser->raw_buffer.pointer[low]
594	+ (parser->raw_buffer.pointer[high] << 8);
595	-
596	- /* Check for unexpected low surrogate area. */
597	-
598	- if ((value & 0xFC00) == 0xDC00)
599	- return yaml_parser_set_reader_error(parser,
600	- "Unexpected low surrogate area",
601	- parser->offset, value);
602	+ width = 2;
603
604	/* Check for a high surrogate area. */
605
606	if ((value & 0xFC00) == 0xD800) {
607
608	- width = 4;
609	-
610	/* Check for incomplete surrogate pair. */
611
612	if (raw_unread < 4) {
613	- if (parser->eof) {
614	- return yaml_parser_set_reader_error(parser,
615	- "Incomplete UTF-16 surrogate pair",
616	- parser->offset, -1);
617	+ if (parser->eof) { /* trailing high surrogate */
618	+ width = 2;
619	+ } else {
620	+ break; /* Can't tell until we have more raw characters */
621	+ }
622	+ } else {
623	+
624	+ /* Get the next character. */
625	+
626	+ value2 = parser->raw_buffer.pointer[low+2]
627	+ + (parser->raw_buffer.pointer[high+2] << 8);
628	+
629	+ /* Check for a low surrogate area. */
630	+ if ((value2 & 0xFC00) == 0xDC00) {
631	+ width = 4;
632	+ /* Generate the value of the surrogate pair. */
633	+ value = 0x10000 + ((value & 0x3FF) << 10) + (value2 & 0x3FF);
634	}
635	- incomplete = 1;
636	- break;
637	}
638	+ }
639
640	- /* Get the next character. */
641	+ /* Check if the raw buffer contains enough bytes to form a character. */
642
643	- value2 = parser->raw_buffer.pointer[low+2]
644	- + (parser->raw_buffer.pointer[high+2] << 8);
645	+ /*
646	+ * Check if the character is in the allowed range:
647	+ * #x9 \| #xA \| #xD \| [#x20-#x7E] (8 bit)
648	+ * \| #x85 \| [#xA0-#xD7FF] \| [#xE000-#xFFFD] (16 bit)
649	+ * \| [#x10000-#x10FFFF] (32 bit)
650	+ */
651	+ /* Modified to allow all 16-bit values as \uNNNN may not
652	+ work for some parsers for these values. */
653	+ if (! (value == 0x09 \|\| value == 0x0A \|\| value == 0x0D
654	+ \|\| (value >= 0x20 && value <= 0x7E)
655	+ \|\| (value == 0x85) \|\| value >= 0xA0))
656	+ return yaml_parser_set_reader_error(parser,
657	+ "Control characters are not allowed",
658	+ parser->offset, value);
659
660	- /* Check for a low surrogate area. */
661	+ /* Move the raw pointers. */
662
663	- if ((value2 & 0xFC00) != 0xDC00)
664	- return yaml_parser_set_reader_error(parser,
665	- "Expected low surrogate area",
666	- parser->offset+2, value2);
667	+ parser->raw_buffer.pointer += width;
668	+ parser->offset += width;
669
670	- /* Generate the value of the surrogate pair. */
671	+ /* Finally put the character into the buffer. */
672
673	- value = 0x10000 + ((value & 0x3FF) << 10) + (value2 & 0x3FF);
674	+ /* 0000 0000-0000 007F -> 0xxxxxxx */
675	+ if (value <= 0x7F) {
676	+ *(parser->buffer.last++) = value;
677	}
678	-
679	+ /* 0000 0080-0000 07FF -> 110xxxxx 10xxxxxx */
680	+ else if (value <= 0x7FF) {
681	+ *(parser->buffer.last++) = 0xC0 + (value >> 6);
682	+ *(parser->buffer.last++) = 0x80 + (value & 0x3F);
683	+ }
684	+ /* 0000 0800-0000 FFFF -> 1110xxxx 10xxxxxx 10xxxxxx */
685	+ else if (value <= 0xFFFF) {
686	+ *(parser->buffer.last++) = 0xE0 + (value >> 12);
687	+ *(parser->buffer.last++) = 0x80 + ((value >> 6) & 0x3F);
688	+ *(parser->buffer.last++) = 0x80 + (value & 0x3F);
689	+ }
690	+ /* 0001 0000-0010 FFFF -> 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */
691	else {
692	- width = 2;
693	+ *(parser->buffer.last++) = 0xF0 + (value >> 18);
694	+ *(parser->buffer.last++) = 0x80 + ((value >> 12) & 0x3F);
695	+ *(parser->buffer.last++) = 0x80 + ((value >> 6) & 0x3F);
696	+ *(parser->buffer.last++) = 0x80 + (value & 0x3F);
697	}
698	-
699	break;
700
701	default:
702	assert(1); /* Impossible. */
703	}
704
705	- /* Check if the raw buffer contains enough bytes to form a character. */
706	-
707	- if (incomplete) break;
708	-
709	- /*
710	- * Check if the character is in the allowed range:
711	- * #x9 \| #xA \| #xD \| [#x20-#x7E] (8 bit)
712	- * \| #x85 \| [#xA0-#xD7FF] \| [#xE000-#xFFFD] (16 bit)
713	- * \| [#x10000-#x10FFFF] (32 bit)
714	- */
715	-
716	- if (! (value == 0x09 \|\| value == 0x0A \|\| value == 0x0D
717	- \|\| (value >= 0x20 && value <= 0x7E)
718	- \|\| (value == 0x85) \|\| (value >= 0xA0 && value <= 0xD7FF)
719	- \|\| (value >= 0xE000 && value <= 0xFFFD)
720	- \|\| (value >= 0x10000 && value <= 0x10FFFF)))
721	- return yaml_parser_set_reader_error(parser,
722	- "Control characters are not allowed",
723	- parser->offset, value);
724	-
725	- /* Move the raw pointers. */
726	-
727	- parser->raw_buffer.pointer += width;
728	- parser->offset += width;
729	-
730	- /* Finally put the character into the buffer. */
731	-
732	- /* 0000 0000-0000 007F -> 0xxxxxxx */
733	- if (value <= 0x7F) {
734	- *(parser->buffer.last++) = value;
735	- }
736	- /* 0000 0080-0000 07FF -> 110xxxxx 10xxxxxx */
737	- else if (value <= 0x7FF) {
738	- *(parser->buffer.last++) = 0xC0 + (value >> 6);
739	- *(parser->buffer.last++) = 0x80 + (value & 0x3F);
740	- }
741	- /* 0000 0800-0000 FFFF -> 1110xxxx 10xxxxxx 10xxxxxx */
742	- else if (value <= 0xFFFF) {
743	- *(parser->buffer.last++) = 0xE0 + (value >> 12);
744	- *(parser->buffer.last++) = 0x80 + ((value >> 6) & 0x3F);
745	- *(parser->buffer.last++) = 0x80 + (value & 0x3F);
746	- }
747	- /* 0001 0000-0010 FFFF -> 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */
748	- else {
749	- *(parser->buffer.last++) = 0xF0 + (value >> 18);
750	- *(parser->buffer.last++) = 0x80 + ((value >> 12) & 0x3F);
751	- *(parser->buffer.last++) = 0x80 + ((value >> 6) & 0x3F);
752	- *(parser->buffer.last++) = 0x80 + (value & 0x3F);
753	- }
754	-
755	parser->unread ++;
756	}
757
758	diff -ur original_src/scanner.c src/scanner.c
759	--- original_src/scanner.c 2008-12-27 15:18:21.000000000 -0800
760	+++ src/scanner.c 2009-08-18 17:56:35.000000000 -0700
761	@@ -495,8 +495,13 @@
762	(parser->mark.index ++, \
763	parser->mark.column ++, \
764	parser->unread --, \
765	- parser->buffer.pointer += WIDTH(parser->buffer))
766	+ parser->buffer.pointer ++)
767
768	+#define SKIPN(parser,n) \
769	+ (parser->mark.index ++, \
770	+ parser->mark.column ++, \
771	+ parser->unread --, \
772	+ parser->buffer.pointer += (n))
773	#define SKIP_LINE(parser) \
774	(IS_CRLF(parser->buffer) ? \
775	(parser->mark.index += 2, \
776	@@ -523,6 +528,14 @@
777	parser->unread --, \
778	1) : 0)
779
780	+#define READN(parser,string,n) \
781	+ (STRING_EXTEND(parser,string) ? \
782	+ (COPYN(string,parser->buffer,n), \
783	+ parser->mark.index ++, \
784	+ parser->mark.column ++, \
785	+ parser->unread --, \
786	+ 1) : 0)
787	+
788	/*
789	* Copy a line break character to a string buffer and advance pointers.
790	*/
791	@@ -1924,7 +1937,7 @@
792	if (!CACHE(parser, 1)) return 0;
793
794	if (parser->mark.column == 0 && IS_BOM(parser->buffer))
795	- SKIP(parser);
796	+ SKIPN(parser,3); /* UTF-8 BOM is 3 bytes */
797
798	/*
799	* Eat whitespaces.
800	@@ -2657,63 +2670,30 @@
801	yaml_parser_scan_uri_escapes(yaml_parser_t *parser, int directive,
802	yaml_mark_t start_mark, yaml_string_t *string)
803	{
804	- int width = 0;
805	-
806	- /* Decode the required number of characters. */
807	-
808	- do {
809	-
810	- unsigned char octet = 0;
811	-
812	- /* Check for a URI-escaped octet. */
813	-
814	- if (!CACHE(parser, 3)) return 0;
815	-
816	- if (!(CHECK(parser->buffer, '%')
817	- && IS_HEX_AT(parser->buffer, 1)
818	- && IS_HEX_AT(parser->buffer, 2))) {
819	- return yaml_parser_set_scanner_error(parser, directive ?
820	- "while parsing a %TAG directive" : "while parsing a tag",
821	- start_mark, "did not find URI escaped octet");
822	- }
823	-
824	- /* Get the octet. */
825	+ unsigned char octet = 0;
826
827	- octet = (AS_HEX_AT(parser->buffer, 1) << 4) + AS_HEX_AT(parser->buffer, 2);
828	+ /* Check for a URI-escaped octet. */
829
830	- /* If it is the leading octet, determine the length of the UTF-8 sequence. */
831	+ if (!CACHE(parser, 3)) return 0;
832
833	- if (!width)
834	- {
835	- width = (octet & 0x80) == 0x00 ? 1 :
836	- (octet & 0xE0) == 0xC0 ? 2 :
837	- (octet & 0xF0) == 0xE0 ? 3 :
838	- (octet & 0xF8) == 0xF0 ? 4 : 0;
839	- if (!width) {
840	- return yaml_parser_set_scanner_error(parser, directive ?
841	- "while parsing a %TAG directive" : "while parsing a tag",
842	- start_mark, "found an incorrect leading UTF-8 octet");
843	- }
844	- }
845	- else
846	- {
847	- /* Check if the trailing octet is correct. */
848	+ if (!(CHECK(parser->buffer, '%')
849	+ && IS_HEX_AT(parser->buffer, 1)
850	+ && IS_HEX_AT(parser->buffer, 2))) {
851	+ return yaml_parser_set_scanner_error(parser, directive ?
852	+ "while parsing a %TAG directive" : "while parsing a tag",
853	+ start_mark, "did not find URI escaped octet");
854	+ }
855
856	- if ((octet & 0xC0) != 0x80) {
857	- return yaml_parser_set_scanner_error(parser, directive ?
858	- "while parsing a %TAG directive" : "while parsing a tag",
859	- start_mark, "found an incorrect trailing UTF-8 octet");
860	- }
861	- }
862	+ /* Get the octet. */
863
864	- /* Copy the octet and move the pointers. */
865	+ octet = (AS_HEX_AT(parser->buffer, 1) << 4) + AS_HEX_AT(parser->buffer, 2);
866
867	- *(string->pointer++) = octet;
868	- SKIP(parser);
869	- SKIP(parser);
870	- SKIP(parser);
871	+ /* Copy the octet and move the pointers. */
872
873	- } while (--width);
874	+ *(string->pointer++) = octet;
875	+ SKIP(parser);
876	+ SKIP(parser);
877	+ SKIP(parser);
878
879	return 1;
880	}
881	@@ -3102,6 +3082,7 @@
882	else if (!single && CHECK(parser->buffer, '\\'))
883	{
884	size_t code_length = 0;
885	+ int raw_code = 0;
886
887	if (!STRING_EXTEND(parser, string)) goto error;
888
889	@@ -3188,6 +3169,11 @@
890	code_length = 2;
891	break;
892
893	+ case 'X':
894	+ code_length = 2;
895	+ raw_code = 1;
896	+ break;
897	+
898	case 'u':
899	code_length = 4;
900	break;
901	@@ -3233,7 +3219,7 @@
902	goto error;
903	}
904
905	- if (value <= 0x7F) {
906	+ if (value <= 0x7F \|\| raw_code) {
907	*(string.pointer++) = value;
908	}
909	else if (value <= 0x7FF) {
910	@@ -3262,7 +3248,7 @@
911
912	else
913	{
914	- /* It is a non-escaped non-blank character. */
915	+ /* It is a non-escaped non-blank byte. */
916
917	if (!READ(parser, string)) goto error;
918	}
919	@@ -3474,7 +3460,7 @@
920	}
921	}
922
923	- /* Copy the character. */
924	+ /* Copy the byte. */
925
926	if (!READ(parser, string)) goto error;
927
928	diff -ur original_src/yaml_private.h src/yaml_private.h
929	--- original_src/yaml_private.h 2008-12-27 15:18:21.000000000 -0800
930	+++ src/yaml_private.h 2009-08-18 17:57:05.000000000 -0700
931	@@ -25,7 +25,7 @@
932	yaml_strdup(const yaml_char_t *);
933
934	/*
935	- * Reader: Ensure that the buffer contains at least `length` characters.
936	+ * Reader: Ensure that the buffer contains at least `length` bytes.
937	*/
938
939	YAML_DECLARE(int)
940	@@ -237,6 +237,7 @@
941
942	/*
943	* Check if the character can be printed unescaped.
944	+ * Only correct if you know you are looking at valid UTF-8!
945	*/
946
947	#define IS_PRINTABLE_AT(string,offset) \
948	@@ -355,6 +356,7 @@
949
950	/*
951	* Determine the width of the character.
952	+ * Only correct if you know you are looking at valid UTF-8!
953	*/
954
955	#define WIDTH_AT(string,offset) \
956	@@ -366,30 +368,23 @@
957	#define WIDTH(string) WIDTH_AT((string),0)
958
959	/*
960	- * Move the string pointer to the next character.
961	+ * Move the string pointer to the next byte.
962	*/
963
964	-#define MOVE(string) ((string).pointer += WIDTH((string)))
965	+#define MOVE(string) ((string).pointer++)
966	+
967	+#define MOVEN(string,n) ((string).pointer += n)
968
969	/*
970	- * Copy a character and move the pointers of both strings.
971	+ * Copy a byte and move the pointers of both strings.
972	*/
973
974	#define COPY(string_a,string_b) \
975	- ((*(string_b).pointer & 0x80) == 0x00 ? \
976	- (((string_a).pointer++) = ((string_b).pointer++)) : \
977	- (*(string_b).pointer & 0xE0) == 0xC0 ? \
978	- (((string_a).pointer++) = ((string_b).pointer++), \
979	- ((string_a).pointer++) = ((string_b).pointer++)) : \
980	- (*(string_b).pointer & 0xF0) == 0xE0 ? \
981	- (((string_a).pointer++) = ((string_b).pointer++), \
982	- ((string_a).pointer++) = ((string_b).pointer++), \
983	- ((string_a).pointer++) = ((string_b).pointer++)) : \
984	- (*(string_b).pointer & 0xF8) == 0xF0 ? \
985	- (((string_a).pointer++) = ((string_b).pointer++), \
986	- ((string_a).pointer++) = ((string_b).pointer++), \
987	- ((string_a).pointer++) = ((string_b).pointer++), \
988	- ((string_a).pointer++) = ((string_b).pointer++)) : 0)
989	+ (((string_a).pointer++) = ((string_b).pointer++))
990	+
991	+#define COPYN(string_a,string_b,n) \
992	+ (memcpy(string_a.pointer, string_b.pointer, n), \
993	+ string_a.pointer += n, string_b.pointer += n)
994
995	/*
996	* Stack and queue management.

Download in other formats:

Original Format