1 #!/usr/bin/env python2
2 """
3 lexer_test.py: Tests for lexer.py
4 """
5
6 import unittest
7
8 from _devbuild.gen.id_kind_asdl import Id, Id_str
9 from _devbuild.gen.types_asdl import lex_mode_e
10 from core import test_lib
11 from core.test_lib import Tok
12 from mycpp.mylib import log
13 from frontend.lexer_def import LEXER_DEF
14 from frontend import reader
15
16
17 def _PrintfOuterTokens(fmt):
18 log('PrintfOuter lexing %r', fmt)
19
20 parse_ctx = test_lib.InitParseContext()
21 arena = test_lib.MakeArena('<lexer_test.py>')
22 line_reader = reader.StringLineReader(fmt, arena)
23 lexer = parse_ctx.MakeLexer(line_reader)
24
25 while True:
26 t = lexer.Read(lex_mode_e.PrintfOuter)
27 print(t)
28 if t.id in (Id.Eof_Real, Id.Eol_Tok):
29 break
30
31 log('')
32
33
34 def _PrintToken(t):
35 #print(t)
36 print('%20s %r' % (Id_str(t.id), t.tval))
37
38
39 def _PrintAllTokens(lx, lex_mode):
40 while True:
41 t = lx.Read(lex_mode)
42 _PrintToken(t)
43 if t.id in (Id.Eof_Real, Id.Eol_Tok):
44 break
45
46
47 class TokenTest(unittest.TestCase):
48 def testToken(self):
49 t = Tok(Id.Lit_Chars, 'abc')
50 print(t)
51
52 # This redundancy is OK I guess.
53 t = Tok(Id.Lit_LBrace, '{')
54 print(t)
55
56 t = Tok(Id.Op_Semi, ';')
57 print(t)
58
59 def testPrintStats(self):
60 states = sorted(LEXER_DEF.items(),
61 key=lambda pair: len(pair[1]),
62 reverse=True)
63 total = 0
64 for state, re_list in states:
65 n = len(re_list)
66 print(n, state)
67 total += n
68
69 print("Number of lex states: %d" % len(LEXER_DEF))
70 print("Number of token dispatches: %d" % total)
71
72 def testMoveToNextLine(self):
73 """Test that it doesn't mess up invariants."""
74 arena = test_lib.MakeArena('<lexer_test.py>')
75 code_str = '''cd {
76 }'''
77
78 print('=== Printing all tokens')
79 if 1:
80 _, lx = test_lib.InitLexer(code_str, arena)
81 _PrintAllTokens(lx, lex_mode_e.ShCommand)
82
83 print()
84 print('=== MoveToNextLine() and LookAheadOne()')
85 _, lx = test_lib.InitLexer(code_str, arena)
86
87 t = lx.Read(lex_mode_e.ShCommand)
88 _PrintToken(t)
89 self.assertEqual(Id.Lit_Chars, t.id)
90
91 t = lx.Read(lex_mode_e.ShCommand)
92 _PrintToken(t)
93 self.assertEqual(Id.WS_Space, t.id)
94
95 t = lx.Read(lex_mode_e.ShCommand)
96 _PrintToken(t)
97 self.assertEqual(Id.Lit_LBrace, t.id)
98
99 try:
100 lx.MoveToNextLine()
101 except AssertionError:
102 pass
103 else:
104 self.fail('Should have asserted')
105
106 t = lx.Read(lex_mode_e.ShCommand)
107 _PrintToken(t)
108 self.assertEqual(Id.Op_Newline, t.id)
109
110 look_ahead_id = lx.LookAheadOne(lex_mode_e.ShCommand)
111 self.assertEqual(Id.Unknown_Tok, look_ahead_id)
112
113 # Method being tested
114 lx.MoveToNextLine()
115
116 # Lookahead
117 print('Lookahead')
118 look_ahead_id = lx.LookAheadOne(lex_mode_e.ShCommand)
119 self.assertEqual(Id.Lit_RBrace, look_ahead_id)
120
121 # Lookahead again
122 print('Lookahead 2')
123 look_ahead_id = lx.LookAheadOne(lex_mode_e.ShCommand)
124 self.assertEqual(Id.Lit_RBrace, look_ahead_id)
125
126 t = lx.Read(lex_mode_e.ShCommand)
127 _PrintToken(t)
128 self.assertEqual(Id.Lit_RBrace, t.id)
129
130 t = lx.Read(lex_mode_e.ShCommand)
131 _PrintToken(t)
132 self.assertEqual(Id.Eof_Real, t.id)
133
134 def testMaybeUnreadOne(self):
135 arena = test_lib.MakeArena('<lexer_test.py>')
136 _, lx = test_lib.InitLexer('()', arena)
137
138 t = lx.Read(lex_mode_e.ShCommand)
139 print(t)
140 self.assertEqual(Id.Op_LParen, t.id)
141
142 t = lx.Read(lex_mode_e.ShCommand)
143 print(t)
144 self.assertEqual(Id.Op_RParen, t.id)
145
146 # Go back
147 lx.MaybeUnreadOne()
148
149 # Push Hint
150 lx.PushHint(Id.Op_RParen, Id.Right_CasePat)
151
152 # Now we see it again another a Id
153 t = lx.Read(lex_mode_e.ShCommand)
154 print(t)
155 self.assertEqual(Id.Right_CasePat, t.id)
156
157 def testPrintf(self):
158 # Demonstrate input handling quirk
159
160 # Get Id.Eof_Real because len('') == 0
161 _PrintfOuterTokens('')
162
163 # Get Id.Eol_Tok because len('\0') == 1
164 _PrintfOuterTokens('\0')
165
166 # Get x, then Id.Eof_Real because there are no more lines
167 _PrintfOuterTokens('x\0')
168
169
170 if __name__ == '__main__':
171 unittest.main()