1 # Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved.
2 # Licensed to PSF under a Contributor Agreement.
3
4 # Modifications:
5 # Copyright 2006 Google, Inc. All Rights Reserved.
6 # Licensed to PSF under a Contributor Agreement.
7 from __future__ import print_function
8
9 """Parser driver.
10
11 A high-level interface to parse a file into a syntax tree.
12 """
13
14 __author__ = "Guido van Rossum <guido@python.org>"
15
16
17 import sys
18
19 from . import parse, pnode, token, tokenize
20
21
22 def log(msg, *args):
23 if args:
24 msg = msg % args
25 print(msg, file=sys.stderr)
26
27
28 def classify(gr, typ, value):
29 """Turn a token into a label. (Internal)"""
30 if typ == token.NAME:
31 # Keep a listing of all used names
32 # OIL note: removed because it's only used by lib2to3
33 #self.used_names.add(value)
34
35 # Check for reserved words
36 ilabel = gr.keywords.get(value)
37 if ilabel is not None:
38 return ilabel
39 ilabel = gr.tokens.get(typ)
40 if ilabel is None:
41 raise parse.ParseError("bad token", typ, value)
42 return ilabel
43
44
45 def PushTokens(p, tokens, gr, start_symbol, opmap=token.opmap, debug=False):
46 """Parse a series of tokens and return the syntax tree.
47
48 NOTE: This function is specific to Python's lexer.
49 """
50 # XXX Move the prefix computation into a wrapper around tokenize.
51 # NOTE: It's mainly for lib2to3.
52
53 p.setup(gr.symbol2number[start_symbol], pnode.PNodeAllocator())
54
55 lineno = 1
56 column = 0
57 type_ = value = start = end = line_text = None
58 prefix = ""
59 for quintuple in tokens:
60 type_, value, start, end, line_text = quintuple
61 #log('token %s %r', type_, value)
62 if start != (lineno, column):
63 assert (lineno, column) <= start, ((lineno, column), start)
64 s_lineno, s_column = start
65 if lineno < s_lineno:
66 prefix += "\n" * (s_lineno - lineno)
67 lineno = s_lineno
68 column = 0
69 if column < s_column:
70 prefix += line_text[column:s_column]
71 column = s_column
72 if type_ in (tokenize.COMMENT, tokenize.NL):
73 prefix += value
74 lineno, column = end
75 if value.endswith("\n"):
76 lineno += 1
77 column = 0
78 continue
79
80 if type_ == token.OP:
81 type_ = opmap[value]
82
83 if debug:
84 log("%s %r (prefix=%r)", token.tok_name[type_], value, prefix)
85
86 ilabel = classify(gr, type_, value)
87 opaque = (value, prefix, start)
88 if p.addtoken(type_, opaque, ilabel):
89 if debug:
90 log("Stop.")
91 break
92 prefix = ""
93 lineno, column = end
94 if value.endswith("\n"):
95 lineno += 1
96 column = 0
97 else:
98 # We never broke out -- EOF is too soon (how can this happen???)
99 opaque = (value, prefix, start)
100 raise parse.ParseError("incomplete input", type_, opaque)
101 return p.rootnode