source: trunk/sandbox/my-parser/parser2_test.py @ 37

Revision 37, 15.1 KB checked in by xi, 9 years ago (diff)

Add the skeleton of a pure Python parser. It's surprisingly simple.

Line 
1
2import unittest
3import parser2
4
5EX1 = r"""
6- Mark McGwire
7- Sammy Sosa
8- Ken Griffey
9"""
10
11TOKENS1 = """
12BLOCK_SEQ_START
13ENTRY SCALAR
14ENTRY SCALAR
15ENTRY SCALAR
16BLOCK_END
17"""
18
19NODES1 = [True, True, True]
20
21EX2 = r"""
22hr:  65    # Home runs
23avg: 0.278 # Batting average
24rbi: 147   # Runs Batted In
25"""
26
27TOKENS2 = """
28BLOCK_MAP_START
29KEY SCALAR VALUE SCALAR
30KEY SCALAR VALUE SCALAR
31KEY SCALAR VALUE SCALAR
32BLOCK_END
33"""
34
35NODES2 = [(True, True), (True, True), (True, True)]
36
37EX3 = r"""
38american:
39  - Boston Red Sox
40  - Detroit Tigers
41  - New York Yankees
42national:
43  - New York Mets
44  - Chicago Cubs
45  - Atlanta Braves
46"""
47
48TOKENS3 = """
49BLOCK_MAP_START
50KEY SCALAR VALUE
51    BLOCK_SEQ_START
52    ENTRY SCALAR
53    ENTRY SCALAR
54    ENTRY SCALAR
55    BLOCK_END
56KEY SCALAR VALUE
57    BLOCK_SEQ_START
58    ENTRY SCALAR
59    ENTRY SCALAR
60    ENTRY SCALAR
61    BLOCK_END
62BLOCK_END
63"""
64
65NODES3 = [(True, [True, True, True]), (True, [True, True, True])]
66
67EX4 = r"""
68-
69  name: Mark McGwire
70  hr:   65
71  avg:  0.278
72-
73  name: Sammy Sosa
74  hr:   63
75  avg:  0.288
76"""
77
78TOKENS4 = """
79BLOCK_SEQ_START
80ENTRY
81    BLOCK_MAP_START
82    KEY SCALAR VALUE SCALAR
83    KEY SCALAR VALUE SCALAR
84    KEY SCALAR VALUE SCALAR
85    BLOCK_END
86ENTRY
87    BLOCK_MAP_START
88    KEY SCALAR VALUE SCALAR
89    KEY SCALAR VALUE SCALAR
90    KEY SCALAR VALUE SCALAR
91    BLOCK_END
92BLOCK_END
93"""
94
95NODES4 = [[(True, True), (True, True), (True, True)], [(True, True), (True, True), (True, True)]]
96
97EX5 = r"""
98- [name        , hr, avg  ]
99- [Mark McGwire, 65, 0.278]
100- [Sammy Sosa  , 63, 0.288]
101"""
102
103TOKENS5 = """
104BLOCK_SEQ_START
105ENTRY FLOW_SEQ_START SCALAR ENTRY SCALAR ENTRY SCALAR FLOW_SEQ_END
106ENTRY FLOW_SEQ_START SCALAR ENTRY SCALAR ENTRY SCALAR FLOW_SEQ_END
107ENTRY FLOW_SEQ_START SCALAR ENTRY SCALAR ENTRY SCALAR FLOW_SEQ_END
108BLOCK_END
109"""
110
111NODES5 = [[True, True, True], [True, True, True], [True, True, True]]
112
113EX6 = r"""
114Mark McGwire: {hr: 65, avg: 0.278}
115Sammy Sosa: {
116    hr: 63,
117    avg: 0.288
118  }
119"""
120
121TOKENS6 = """
122BLOCK_MAP_START
123KEY SCALAR VALUE
124    FLOW_MAP_START KEY SCALAR VALUE SCALAR ENTRY KEY SCALAR VALUE SCALAR FLOW_MAP_END
125KEY SCALAR VALUE
126    FLOW_MAP_START KEY SCALAR VALUE SCALAR ENTRY KEY SCALAR VALUE SCALAR FLOW_MAP_END
127BLOCK_END   
128"""
129
130NODES6 = [(True, [(True, True), (True, True)]), (True, [(True, True), (True, True)])]
131
132EX7 = r"""
133# Ranking of 1998 home runs
134---
135- Mark McGwire
136- Sammy Sosa
137- Ken Griffey
138
139# Team ranking
140---
141- Chicago Cubs
142- St Louis Cardinals
143"""
144
145TOKENS7 = """
146DOCUMENT_START
147BLOCK_SEQ_START
148ENTRY SCALAR
149ENTRY SCALAR
150ENTRY SCALAR
151BLOCK_END
152
153DOCUMENT_START
154BLOCK_SEQ_START
155ENTRY SCALAR
156ENTRY SCALAR
157BLOCK_END
158"""
159
160NODES7 = ([True, True, True], [True, True])
161
162EX8 = r"""
163---
164time: 20:03:20
165player: Sammy Sosa
166action: strike (miss)
167...
168---
169time: 20:03:47
170player: Sammy Sosa
171action: grand slam
172...
173"""
174
175TOKENS8 = """
176DOCUMENT_START
177BLOCK_MAP_START
178KEY SCALAR VALUE SCALAR
179KEY SCALAR VALUE SCALAR
180KEY SCALAR VALUE SCALAR
181BLOCK_END
182DOCUMENT_END
183
184DOCUMENT_START
185BLOCK_MAP_START
186KEY SCALAR VALUE SCALAR
187KEY SCALAR VALUE SCALAR
188KEY SCALAR VALUE SCALAR
189BLOCK_END
190DOCUMENT_END
191"""
192
193NODES8 = ([(True, True), (True, True), (True, True)], [(True, True), (True, True), (True, True)])
194
195EX9 = r"""
196---
197hr: # 1998 hr ranking
198  - Mark McGwire
199  - Sammy Sosa
200rbi:
201  # 1998 rbi ranking
202  - Sammy Sosa
203  - Ken Griffey
204"""
205
206TOKENS9 = """
207DOCUMENT_START
208BLOCK_MAP_START
209KEY SCALAR VALUE
210    BLOCK_SEQ_START
211    ENTRY SCALAR
212    ENTRY SCALAR
213    BLOCK_END
214KEY SCALAR VALUE
215    BLOCK_SEQ_START
216    ENTRY SCALAR
217    ENTRY SCALAR
218    BLOCK_END
219BLOCK_END
220"""
221
222NODES9 = [(True, [True, True]), (True, [True, True])]
223
224EX10 = r"""
225---
226hr:
227  - Mark McGwire
228  # Following node labeled SS
229  - &SS Sammy Sosa
230rbi:
231  - *SS # Subsequent occurrence
232  - Ken Griffey
233"""
234
235TOKENS10 = """
236DOCUMENT_START
237BLOCK_MAP_START
238KEY SCALAR VALUE
239    BLOCK_SEQ_START
240    ENTRY SCALAR
241    ENTRY ANCHOR SCALAR
242    BLOCK_END
243KEY SCALAR VALUE
244    BLOCK_SEQ_START
245    ENTRY ALIAS
246    ENTRY SCALAR
247    BLOCK_END
248BLOCK_END
249"""
250
251NODES10 = [(True, [True, True]), (True, ['*', True])]
252
253EX11 = r"""
254? - Detroit Tigers
255  - Chicago cubs
256:
257  - 2001-07-23
258
259? [ New York Yankees,
260    Atlanta Braves ]
261: [ 2001-07-02, 2001-08-12,
262    2001-08-14 ]
263"""
264
265TOKENS11 = """
266BLOCK_MAP_START
267KEY
268    BLOCK_SEQ_START
269    ENTRY SCALAR
270    ENTRY SCALAR
271    BLOCK_END
272VALUE
273    BLOCK_SEQ_START
274    ENTRY SCALAR
275    BLOCK_END
276KEY
277    FLOW_SEQ_START SCALAR ENTRY SCALAR FLOW_SEQ_END
278VALUE
279    FLOW_SEQ_START SCALAR ENTRY SCALAR ENTRY SCALAR FLOW_SEQ_END
280BLOCK_END
281"""
282
283NODES11 = [([True, True], [True]), ([True, True], [True, True, True])]
284
285EX12 = r"""
286---
287# products purchased
288- item    : Super Hoop
289  quantity: 1
290- item    : Basketball
291  quantity: 4
292- item    : Big Shoes
293  quantity: 1
294"""
295
296TOKENS12 = """
297DOCUMENT_START
298BLOCK_SEQ_START
299ENTRY
300    BLOCK_MAP_START
301    KEY SCALAR VALUE SCALAR
302    KEY SCALAR VALUE SCALAR
303    BLOCK_END
304ENTRY
305    BLOCK_MAP_START
306    KEY SCALAR VALUE SCALAR
307    KEY SCALAR VALUE SCALAR
308    BLOCK_END
309ENTRY
310    BLOCK_MAP_START
311    KEY SCALAR VALUE SCALAR
312    KEY SCALAR VALUE SCALAR
313    BLOCK_END
314BLOCK_END
315"""
316
317NODES12 = [[(True, True), (True, True)], [(True, True), (True, True)], [(True, True), (True, True)]]
318
319EX13 = r"""
320# ASCII Art
321--- |
322  \//||\/||
323  // ||  ||__
324"""
325
326TOKENS13 = """
327DOCUMENT_START SCALAR
328"""
329
330NODES13 = True
331
332EX14 = r"""
333---
334  Mark McGwire's
335  year was crippled
336  by a knee injury.
337"""
338
339TOKENS14 = """
340DOCUMENT_START SCALAR
341"""
342
343NODES14 = True
344
345EX15 = r"""
346>
347 Sammy Sosa completed another
348 fine season with great stats.
349
350   63 Home Runs
351   0.288 Batting Average
352
353 What a year!
354"""
355
356TOKENS15 = """
357SCALAR
358"""
359
360NODES15 = True
361
362EX16 = r"""
363name: Mark McGwire
364accomplishment: >
365  Mark set a major league
366  home run record in 1998.
367stats: |
368  65 Home Runs
369  0.278 Batting Average
370"""
371
372TOKENS16 = """
373BLOCK_MAP_START
374KEY SCALAR VALUE SCALAR
375KEY SCALAR VALUE SCALAR
376KEY SCALAR VALUE SCALAR
377BLOCK_END
378"""
379
380NODES16 = [(True, True), (True, True), (True, True)]
381
382EX17 = r"""
383unicode: "Sosa did fine.\u263A"
384control: "\b1998\t1999\t2000\n"
385hexesc:  "\x13\x10 is \r\n"
386
387single: '"Howdy!" he cried.'
388quoted: ' # not a ''comment''.'
389tie-fighter: '|\-*-/|'
390"""
391
392TOKENS17 = """
393BLOCK_MAP_START
394KEY SCALAR VALUE SCALAR
395KEY SCALAR VALUE SCALAR
396KEY SCALAR VALUE SCALAR
397KEY SCALAR VALUE SCALAR
398KEY SCALAR VALUE SCALAR
399KEY SCALAR VALUE SCALAR
400BLOCK_END
401"""
402
403NODES17 = [(True, True), (True, True), (True, True), (True, True), (True, True), (True, True)]
404
405EX18 = r"""
406plain:
407  This unquoted scalar
408  spans many lines.
409
410quoted: "So does this
411  quoted scalar.\n"
412"""
413
414TOKENS18 = """
415BLOCK_MAP_START
416KEY SCALAR VALUE SCALAR
417KEY SCALAR VALUE SCALAR
418BLOCK_END
419"""
420
421NODES18 = [(True, True), (True, True)]
422
423EX19 = r"""
424canonical: 12345
425decimal: +12,345
426sexagesimal: 3:25:45
427octal: 014
428hexadecimal: 0xC
429"""
430
431TOKENS19 = """
432BLOCK_MAP_START
433KEY SCALAR VALUE SCALAR
434KEY SCALAR VALUE SCALAR
435KEY SCALAR VALUE SCALAR
436KEY SCALAR VALUE SCALAR
437KEY SCALAR VALUE SCALAR
438BLOCK_END
439"""
440
441NODES19 = [(True, True), (True, True), (True, True), (True, True), (True, True)]
442
443EX20 = r"""
444canonical: 1.23015e+3
445exponential: 12.3015e+02
446sexagesimal: 20:30.15
447fixed: 1,230.15
448negative infinity: -.inf
449not a number: .NaN
450"""
451
452TOKENS20 = """
453BLOCK_MAP_START
454KEY SCALAR VALUE SCALAR
455KEY SCALAR VALUE SCALAR
456KEY SCALAR VALUE SCALAR
457KEY SCALAR VALUE SCALAR
458KEY SCALAR VALUE SCALAR
459KEY SCALAR VALUE SCALAR
460BLOCK_END
461"""
462
463NODES20 = [(True, True), (True, True), (True, True), (True, True), (True, True), (True, True)]
464
465EX21 = r"""
466null: ~
467true: y
468false: n
469string: '12345'
470"""
471
472TOKENS21 = """
473BLOCK_MAP_START
474KEY SCALAR VALUE SCALAR
475KEY SCALAR VALUE SCALAR
476KEY SCALAR VALUE SCALAR
477KEY SCALAR VALUE SCALAR
478BLOCK_END
479"""
480
481NODES21 = [(True, True), (True, True), (True, True), (True, True)]
482
483EX22 = r"""
484canonical: 2001-12-15T02:59:43.1Z
485iso8601: 2001-12-14t21:59:43.10-05:00
486spaced: 2001-12-14 21:59:43.10 -5
487date: 2002-12-14
488"""
489
490TOKENS22 = """
491BLOCK_MAP_START
492KEY SCALAR VALUE SCALAR
493KEY SCALAR VALUE SCALAR
494KEY SCALAR VALUE SCALAR
495KEY SCALAR VALUE SCALAR
496BLOCK_END
497"""
498
499NODES22 = [(True, True), (True, True), (True, True), (True, True)]
500
501EX23 = r"""
502---
503not-date: !!str 2002-04-28
504
505picture: !!binary |
506 R0lGODlhDAAMAIQAAP//9/X
507 17unp5WZmZgAAAOfn515eXv
508 Pz7Y6OjuDg4J+fn5OTk6enp
509 56enmleECcgggoBADs=
510
511application specific tag: !something |
512 The semantics of the tag
513 above may be different for
514 different documents.
515"""
516
517TOKENS23 = """
518DOCUMENT_START
519BLOCK_MAP_START
520KEY SCALAR VALUE TAG SCALAR
521KEY SCALAR VALUE TAG SCALAR
522KEY SCALAR VALUE TAG SCALAR
523BLOCK_END
524"""
525
526NODES23 = [(True, True), (True, True), (True, True)]
527
528EX24 = r"""
529%TAG ! tag:clarkevans.com,2002:
530--- !shape
531  # Use the ! handle for presenting
532  # tag:clarkevans.com,2002:circle
533- !circle
534  center: &ORIGIN {x: 73, y: 129}
535  radius: 7
536- !line
537  start: *ORIGIN
538  finish: { x: 89, y: 102 }
539- !label
540  start: *ORIGIN
541  color: 0xFFEEBB
542  text: Pretty vector drawing.
543"""
544
545TOKENS24 = """
546DIRECTIVE
547DOCUMENT_START TAG
548BLOCK_SEQ_START
549ENTRY TAG
550    BLOCK_MAP_START
551    KEY SCALAR VALUE ANCHOR
552        FLOW_MAP_START KEY SCALAR VALUE SCALAR ENTRY KEY SCALAR VALUE SCALAR FLOW_MAP_END
553    KEY SCALAR VALUE SCALAR
554    BLOCK_END
555ENTRY TAG
556    BLOCK_MAP_START
557    KEY SCALAR VALUE ALIAS
558    KEY SCALAR VALUE
559        FLOW_MAP_START KEY SCALAR VALUE SCALAR ENTRY KEY SCALAR VALUE SCALAR FLOW_MAP_END
560    BLOCK_END
561ENTRY TAG
562    BLOCK_MAP_START
563    KEY SCALAR VALUE ALIAS
564    KEY SCALAR VALUE SCALAR
565    KEY SCALAR VALUE SCALAR
566    BLOCK_END
567BLOCK_END
568"""
569
570NODES24 = [[(True, [(True, True), (True, True)]), (True, True)],
571    [(True, '*'), (True, [(True, True), (True, True)])],
572    [(True, '*'), (True, True), (True, True)]]
573
574EX25 = r"""
575# sets are represented as a
576# mapping where each key is
577# associated with the empty string
578--- !!set
579? Mark McGwire
580? Sammy Sosa
581? Ken Griff
582"""
583
584TOKENS25 = """
585DOCUMENT_START TAG
586BLOCK_MAP_START
587KEY SCALAR
588KEY SCALAR
589KEY SCALAR
590BLOCK_END
591"""
592
593NODES25 = [(True, None), (True, None), (True, None)]
594
595EX26 = r"""
596# ordered maps are represented as
597# a sequence of mappings, with
598# each mapping having one key
599--- !!omap
600- Mark McGwire: 65
601- Sammy Sosa: 63
602- Ken Griffy: 58
603"""
604
605TOKENS26 = """
606DOCUMENT_START TAG
607BLOCK_SEQ_START
608ENTRY
609    BLOCK_MAP_START
610    KEY SCALAR VALUE SCALAR
611    BLOCK_END
612ENTRY
613    BLOCK_MAP_START
614    KEY SCALAR VALUE SCALAR
615    BLOCK_END
616ENTRY
617    BLOCK_MAP_START
618    KEY SCALAR VALUE SCALAR
619    BLOCK_END
620BLOCK_END
621"""
622
623NODES26 = [[(True, True)], [(True, True)], [(True, True)]]
624
625EX27 = r"""
626--- !<tag:clarkevans.com,2002:invoice>
627invoice: 34843
628date   : 2001-01-23
629bill-to: &id001
630    given  : Chris
631    family : Dumars
632    address:
633        lines: |
634            458 Walkman Dr.
635            Suite #292
636        city    : Royal Oak
637        state   : MI
638        postal  : 48046
639ship-to: *id001
640product:
641    - sku         : BL394D
642      quantity    : 4
643      description : Basketball
644      price       : 450.00
645    - sku         : BL4438H
646      quantity    : 1
647      description : Super Hoop
648      price       : 2392.00
649tax  : 251.42
650total: 4443.52
651comments:
652    Late afternoon is best.
653    Backup contact is Nancy
654    Billsmer @ 338-4338.
655"""
656
657TOKENS27 = """
658DOCUMENT_START TAG
659BLOCK_MAP_START
660KEY SCALAR VALUE SCALAR
661KEY SCALAR VALUE SCALAR
662KEY SCALAR VALUE ANCHOR
663    BLOCK_MAP_START
664    KEY SCALAR VALUE SCALAR
665    KEY SCALAR VALUE SCALAR
666    KEY SCALAR VALUE
667        BLOCK_MAP_START
668        KEY SCALAR VALUE SCALAR
669        KEY SCALAR VALUE SCALAR
670        KEY SCALAR VALUE SCALAR
671        KEY SCALAR VALUE SCALAR
672        BLOCK_END
673    BLOCK_END
674KEY SCALAR VALUE ALIAS
675KEY SCALAR VALUE
676    BLOCK_SEQ_START
677    ENTRY
678        BLOCK_MAP_START
679        KEY SCALAR VALUE SCALAR
680        KEY SCALAR VALUE SCALAR
681        KEY SCALAR VALUE SCALAR
682        KEY SCALAR VALUE SCALAR
683        BLOCK_END
684    ENTRY
685        BLOCK_MAP_START
686        KEY SCALAR VALUE SCALAR
687        KEY SCALAR VALUE SCALAR
688        KEY SCALAR VALUE SCALAR
689        KEY SCALAR VALUE SCALAR
690        BLOCK_END
691    BLOCK_END
692KEY SCALAR VALUE SCALAR
693KEY SCALAR VALUE SCALAR
694KEY SCALAR VALUE SCALAR
695BLOCK_END
696"""
697
698NODES27 = [
699    (True, True), (True, True), (True, [(True, True), (True, True), (True, [(True, True), (True, True), (True, True), (True, True)])]), (True, '*'),
700    (True, [[(True, True), (True, True), (True, True), (True, True)], [(True, True), (True, True), (True, True), (True, True)]]), (True, True), (True, True), (True, True),
701]
702
703EX28 = r"""
704---
705Time: 2001-11-23 15:01:42 -5
706User: ed
707Warning:
708  This is an error message
709  for the log file
710---
711Time: 2001-11-23 15:02:31 -5
712User: ed
713Warning:
714  A slightly different error
715  message.
716---
717Date: 2001-11-23 15:03:17 -5
718User: ed
719Fatal:
720  Unknown variable "bar"
721Stack:
722  - file: TopClass.py
723    line: 23
724    code: |
725      x = MoreObject("345\n")
726  - file: MoreClass.py
727    line: 58
728    code: |-
729      foo = bar
730"""
731
732TOKENS28 = """
733DOCUMENT_START
734BLOCK_MAP_START
735KEY SCALAR VALUE SCALAR
736KEY SCALAR VALUE SCALAR
737KEY SCALAR VALUE SCALAR
738BLOCK_END
739
740DOCUMENT_START
741BLOCK_MAP_START
742KEY SCALAR VALUE SCALAR
743KEY SCALAR VALUE SCALAR
744KEY SCALAR VALUE SCALAR
745BLOCK_END
746
747DOCUMENT_START
748BLOCK_MAP_START
749KEY SCALAR VALUE SCALAR
750KEY SCALAR VALUE SCALAR
751KEY SCALAR VALUE SCALAR
752KEY SCALAR VALUE
753    BLOCK_SEQ_START
754    ENTRY
755        BLOCK_MAP_START
756        KEY SCALAR VALUE SCALAR
757        KEY SCALAR VALUE SCALAR
758        KEY SCALAR VALUE SCALAR
759        BLOCK_END
760    ENTRY
761        BLOCK_MAP_START
762        KEY SCALAR VALUE SCALAR
763        KEY SCALAR VALUE SCALAR
764        KEY SCALAR VALUE SCALAR
765        BLOCK_END
766    BLOCK_END
767BLOCK_END
768"""
769
770NODES28 = (
771    [(True, True), (True, True), (True, True)], [(True, True), (True, True), (True, True)],
772    [(True, True), (True, True), (True, True), (True, [[(True, True), (True, True), (True, True)], [(True, True), (True, True), (True, True)]])],
773)
774
775MAX_TESTS = 100
776
777class TestParser2(unittest.TestCase):
778
779    def _testTokens(self, index, EX, TOKENS):
780        try:
781            tokens = None
782            scanner = parser2.Scanner()
783            tokens = scanner.scan('EX'+str(index), EX)
784            self.failUnlessEqual(tokens, TOKENS.split())
785        except:
786            print "EXAMPLE #%s" % index
787            print "EX:"
788            print EX
789            print "TOKENS:"
790            print TOKENS
791            print "RESULT:", tokens
792            print "EXPECT:", TOKENS.split()
793            raise
794
795    def _testNodes(self, index, EX, NODES):
796        try:
797            nodes = None
798            parser = parser2.Parser()
799            nodes = parser.parse('EX'+str(index), EX)
800            self.failUnlessEqual(nodes, NODES)
801        except:
802            print "EXAMPLE #%s" % index
803            print "EX:"
804            print EX
805            print "RESULT:", nodes
806            print "EXPECT:", NODES
807            raise
808
809    @classmethod
810    def add_tests(cls, test_method_name, *tests):
811        for index in range(1, MAX_TESTS):
812            args = []
813            for name in tests:
814                if name+str(index) in globals():
815                    args.append(globals()[name+str(index)])
816                else:
817                    break
818            else:
819                def test_method(self, index=index, args=args):
820                    getattr(self, '_'+test_method_name)(index, *args)
821                test_method.__name__ = '%s%02d' % (test_method_name, index)
822                setattr(cls, test_method.__name__, test_method)
823
824TestParser2.add_tests('testTokens', 'EX', 'TOKENS')
825TestParser2.add_tests('testNodes', 'EX', 'NODES')
826
827if __name__ == '__main__':
828    unittest.main()
829
Note: See TracBrowser for help on using the repository browser.