1 """Token constants (from "token.h")."""
2
3 # Taken from Python (r53757) and modified to include some tokens
4 # originally monkeypatched in by pgen2.tokenize
5
6 #--start constants--
7 ENDMARKER = 0
8 NAME = 1
9 NUMBER = 2
10 STRING = 3
11 NEWLINE = 4
12 INDENT = 5
13 DEDENT = 6
14 LPAR = 7
15 RPAR = 8
16 LSQB = 9
17 RSQB = 10
18 COLON = 11
19 COMMA = 12
20 SEMI = 13
21 PLUS = 14
22 MINUS = 15
23 STAR = 16
24 SLASH = 17
25 VBAR = 18
26 AMPER = 19
27 LESS = 20
28 GREATER = 21
29 EQUAL = 22
30 DOT = 23
31 PERCENT = 24
32 BACKQUOTE = 25
33 LBRACE = 26
34 RBRACE = 27
35 EQEQUAL = 28
36 NOTEQUAL = 29
37 LESSEQUAL = 30
38 GREATEREQUAL = 31
39 TILDE = 32
40 CIRCUMFLEX = 33
41 LEFTSHIFT = 34
42 RIGHTSHIFT = 35
43 DOUBLESTAR = 36
44 PLUSEQUAL = 37
45 MINEQUAL = 38
46 STAREQUAL = 39
47 SLASHEQUAL = 40
48 PERCENTEQUAL = 41
49 AMPEREQUAL = 42
50 VBAREQUAL = 43
51 CIRCUMFLEXEQUAL = 44
52 LEFTSHIFTEQUAL = 45
53 RIGHTSHIFTEQUAL = 46
54 DOUBLESTAREQUAL = 47
55 DOUBLESLASH = 48
56 DOUBLESLASHEQUAL = 49
57 AT = 50
58 ATEQUAL = 51
59 OP = 52
60 COMMENT = 53
61 NL = 54
62 RARROW = 55
63 AWAIT = 56
64 ASYNC = 57
65 ERRORTOKEN = 58
66 N_TOKENS = 59
67 NT_OFFSET = 256
68 #--end constants--
69
70 tok_name = {}
71 for _name, _value in list(globals().items()):
72 if type(_value) is type(0):
73 tok_name[_value] = _name
74
75
76 def ISTERMINAL(x):
77 # type: (int) -> bool
78 return x < NT_OFFSET
79
80 def ISNONTERMINAL(x):
81 # type: (int) -> bool
82 return x >= NT_OFFSET
83
84 def ISEOF(x):
85 # type: (int) -> bool
86 return x == ENDMARKER
87
88
89 # Map from operator to number. NOTE: This was moved here from
90 # pgen2/grammar.py. Oil doesn't need this; only parsers that use Python's
91 # tokenizer (like pgen itself) need it.
92
93 opmap_raw = """
94 ( LPAR
95 ) RPAR
96 [ LSQB
97 ] RSQB
98 : COLON
99 , COMMA
100 ; SEMI
101 + PLUS
102 - MINUS
103 * STAR
104 / SLASH
105 | VBAR
106 & AMPER
107 < LESS
108 > GREATER
109 = EQUAL
110 . DOT
111 % PERCENT
112 ` BACKQUOTE
113 { LBRACE
114 } RBRACE
115 @ AT
116 @= ATEQUAL
117 == EQEQUAL
118 != NOTEQUAL
119 <> NOTEQUAL
120 <= LESSEQUAL
121 >= GREATEREQUAL
122 ~ TILDE
123 ^ CIRCUMFLEX
124 << LEFTSHIFT
125 >> RIGHTSHIFT
126 ** DOUBLESTAR
127 += PLUSEQUAL
128 -= MINEQUAL
129 *= STAREQUAL
130 /= SLASHEQUAL
131 %= PERCENTEQUAL
132 &= AMPEREQUAL
133 |= VBAREQUAL
134 ^= CIRCUMFLEXEQUAL
135 <<= LEFTSHIFTEQUAL
136 >>= RIGHTSHIFTEQUAL
137 **= DOUBLESTAREQUAL
138 // DOUBLESLASH
139 //= DOUBLESLASHEQUAL
140 -> RARROW
141 """
142
143 import sys
144 this_module = sys.modules[__name__]
145 opmap = {}
146 for line in opmap_raw.splitlines():
147 if line:
148 op, name = line.split()
149 opmap[op] = getattr(this_module, name)