1 #include "Python.h"
2 #include "internal/pystate.h"
3 #ifdef Yield
4 #undef Yield /* undefine conflicting macro from winbase.h */
5 #endif
6 #include "Python-ast.h"
7 #include "code.h"
8 #include "symtable.h"
9 #include "structmember.h"
10
11 /* error strings used for warnings */
12 #define GLOBAL_PARAM \
13 "name '%U' is parameter and global"
14
15 #define NONLOCAL_PARAM \
16 "name '%U' is parameter and nonlocal"
17
18 #define GLOBAL_AFTER_ASSIGN \
19 "name '%U' is assigned to before global declaration"
20
21 #define NONLOCAL_AFTER_ASSIGN \
22 "name '%U' is assigned to before nonlocal declaration"
23
24 #define GLOBAL_AFTER_USE \
25 "name '%U' is used prior to global declaration"
26
27 #define NONLOCAL_AFTER_USE \
28 "name '%U' is used prior to nonlocal declaration"
29
30 #define GLOBAL_ANNOT \
31 "annotated name '%U' can't be global"
32
33 #define NONLOCAL_ANNOT \
34 "annotated name '%U' can't be nonlocal"
35
36 #define IMPORT_STAR_WARNING "import * only allowed at module level"
37
38 static PySTEntryObject *
ste_new(struct symtable * st,identifier name,_Py_block_ty block,void * key,int lineno,int col_offset)39 ste_new(struct symtable *st, identifier name, _Py_block_ty block,
40 void *key, int lineno, int col_offset)
41 {
42 PySTEntryObject *ste = NULL;
43 PyObject *k = NULL;
44
45 k = PyLong_FromVoidPtr(key);
46 if (k == NULL)
47 goto fail;
48 ste = PyObject_New(PySTEntryObject, &PySTEntry_Type);
49 if (ste == NULL) {
50 Py_DECREF(k);
51 goto fail;
52 }
53 ste->ste_table = st;
54 ste->ste_id = k; /* ste owns reference to k */
55
56 Py_INCREF(name);
57 ste->ste_name = name;
58
59 ste->ste_symbols = NULL;
60 ste->ste_varnames = NULL;
61 ste->ste_children = NULL;
62
63 ste->ste_directives = NULL;
64
65 ste->ste_type = block;
66 ste->ste_nested = 0;
67 ste->ste_free = 0;
68 ste->ste_varargs = 0;
69 ste->ste_varkeywords = 0;
70 ste->ste_opt_lineno = 0;
71 ste->ste_opt_col_offset = 0;
72 ste->ste_lineno = lineno;
73 ste->ste_col_offset = col_offset;
74
75 if (st->st_cur != NULL &&
76 (st->st_cur->ste_nested ||
77 st->st_cur->ste_type == FunctionBlock))
78 ste->ste_nested = 1;
79 ste->ste_child_free = 0;
80 ste->ste_generator = 0;
81 ste->ste_coroutine = 0;
82 ste->ste_returns_value = 0;
83 ste->ste_needs_class_closure = 0;
84
85 ste->ste_symbols = PyDict_New();
86 ste->ste_varnames = PyList_New(0);
87 ste->ste_children = PyList_New(0);
88 if (ste->ste_symbols == NULL
89 || ste->ste_varnames == NULL
90 || ste->ste_children == NULL)
91 goto fail;
92
93 if (PyDict_SetItem(st->st_blocks, ste->ste_id, (PyObject *)ste) < 0)
94 goto fail;
95
96 return ste;
97 fail:
98 Py_XDECREF(ste);
99 return NULL;
100 }
101
102 static PyObject *
ste_repr(PySTEntryObject * ste)103 ste_repr(PySTEntryObject *ste)
104 {
105 return PyUnicode_FromFormat("<symtable entry %U(%ld), line %d>",
106 ste->ste_name,
107 PyLong_AS_LONG(ste->ste_id), ste->ste_lineno);
108 }
109
110 static void
ste_dealloc(PySTEntryObject * ste)111 ste_dealloc(PySTEntryObject *ste)
112 {
113 ste->ste_table = NULL;
114 Py_XDECREF(ste->ste_id);
115 Py_XDECREF(ste->ste_name);
116 Py_XDECREF(ste->ste_symbols);
117 Py_XDECREF(ste->ste_varnames);
118 Py_XDECREF(ste->ste_children);
119 Py_XDECREF(ste->ste_directives);
120 PyObject_Del(ste);
121 }
122
123 #define OFF(x) offsetof(PySTEntryObject, x)
124
125 static PyMemberDef ste_memberlist[] = {
126 {"id", T_OBJECT, OFF(ste_id), READONLY},
127 {"name", T_OBJECT, OFF(ste_name), READONLY},
128 {"symbols", T_OBJECT, OFF(ste_symbols), READONLY},
129 {"varnames", T_OBJECT, OFF(ste_varnames), READONLY},
130 {"children", T_OBJECT, OFF(ste_children), READONLY},
131 {"nested", T_INT, OFF(ste_nested), READONLY},
132 {"type", T_INT, OFF(ste_type), READONLY},
133 {"lineno", T_INT, OFF(ste_lineno), READONLY},
134 {NULL}
135 };
136
137 PyTypeObject PySTEntry_Type = {
138 PyVarObject_HEAD_INIT(&PyType_Type, 0)
139 "symtable entry",
140 sizeof(PySTEntryObject),
141 0,
142 (destructor)ste_dealloc, /* tp_dealloc */
143 0, /* tp_print */
144 0, /* tp_getattr */
145 0, /* tp_setattr */
146 0, /* tp_reserved */
147 (reprfunc)ste_repr, /* tp_repr */
148 0, /* tp_as_number */
149 0, /* tp_as_sequence */
150 0, /* tp_as_mapping */
151 0, /* tp_hash */
152 0, /* tp_call */
153 0, /* tp_str */
154 PyObject_GenericGetAttr, /* tp_getattro */
155 0, /* tp_setattro */
156 0, /* tp_as_buffer */
157 Py_TPFLAGS_DEFAULT, /* tp_flags */
158 0, /* tp_doc */
159 0, /* tp_traverse */
160 0, /* tp_clear */
161 0, /* tp_richcompare */
162 0, /* tp_weaklistoffset */
163 0, /* tp_iter */
164 0, /* tp_iternext */
165 0, /* tp_methods */
166 ste_memberlist, /* tp_members */
167 0, /* tp_getset */
168 0, /* tp_base */
169 0, /* tp_dict */
170 0, /* tp_descr_get */
171 0, /* tp_descr_set */
172 0, /* tp_dictoffset */
173 0, /* tp_init */
174 0, /* tp_alloc */
175 0, /* tp_new */
176 };
177
178 static int symtable_analyze(struct symtable *st);
179 static int symtable_enter_block(struct symtable *st, identifier name,
180 _Py_block_ty block, void *ast, int lineno,
181 int col_offset);
182 static int symtable_exit_block(struct symtable *st, void *ast);
183 static int symtable_visit_stmt(struct symtable *st, stmt_ty s);
184 static int symtable_visit_expr(struct symtable *st, expr_ty s);
185 static int symtable_visit_genexp(struct symtable *st, expr_ty s);
186 static int symtable_visit_listcomp(struct symtable *st, expr_ty s);
187 static int symtable_visit_setcomp(struct symtable *st, expr_ty s);
188 static int symtable_visit_dictcomp(struct symtable *st, expr_ty s);
189 static int symtable_visit_arguments(struct symtable *st, arguments_ty);
190 static int symtable_visit_excepthandler(struct symtable *st, excepthandler_ty);
191 static int symtable_visit_alias(struct symtable *st, alias_ty);
192 static int symtable_visit_comprehension(struct symtable *st, comprehension_ty);
193 static int symtable_visit_keyword(struct symtable *st, keyword_ty);
194 static int symtable_visit_slice(struct symtable *st, slice_ty);
195 static int symtable_visit_params(struct symtable *st, asdl_seq *args);
196 static int symtable_visit_argannotations(struct symtable *st, asdl_seq *args);
197 static int symtable_implicit_arg(struct symtable *st, int pos);
198 static int symtable_visit_annotations(struct symtable *st, stmt_ty s, arguments_ty, expr_ty);
199 static int symtable_visit_withitem(struct symtable *st, withitem_ty item);
200
201
202 static identifier top = NULL, lambda = NULL, genexpr = NULL,
203 listcomp = NULL, setcomp = NULL, dictcomp = NULL,
204 __class__ = NULL;
205
206 #define GET_IDENTIFIER(VAR) \
207 ((VAR) ? (VAR) : ((VAR) = PyUnicode_InternFromString(# VAR)))
208
209 #define DUPLICATE_ARGUMENT \
210 "duplicate argument '%U' in function definition"
211
212 static struct symtable *
symtable_new(void)213 symtable_new(void)
214 {
215 struct symtable *st;
216
217 st = (struct symtable *)PyMem_Malloc(sizeof(struct symtable));
218 if (st == NULL) {
219 PyErr_NoMemory();
220 return NULL;
221 }
222
223 st->st_filename = NULL;
224 st->st_blocks = NULL;
225
226 if ((st->st_stack = PyList_New(0)) == NULL)
227 goto fail;
228 if ((st->st_blocks = PyDict_New()) == NULL)
229 goto fail;
230 st->st_cur = NULL;
231 st->st_private = NULL;
232 return st;
233 fail:
234 PySymtable_Free(st);
235 return NULL;
236 }
237
238 /* When compiling the use of C stack is probably going to be a lot
239 lighter than when executing Python code but still can overflow
240 and causing a Python crash if not checked (e.g. eval("()"*300000)).
241 Using the current recursion limit for the compiler seems too
242 restrictive (it caused at least one test to fail) so a factor is
243 used to allow deeper recursion when compiling an expression.
244
245 Using a scaling factor means this should automatically adjust when
246 the recursion limit is adjusted for small or large C stack allocations.
247 */
248 #define COMPILER_STACK_FRAME_SCALE 3
249
250 struct symtable *
PySymtable_BuildObject(mod_ty mod,PyObject * filename,PyFutureFeatures * future)251 PySymtable_BuildObject(mod_ty mod, PyObject *filename, PyFutureFeatures *future)
252 {
253 struct symtable *st = symtable_new();
254 asdl_seq *seq;
255 int i;
256 PyThreadState *tstate;
257 int recursion_limit = Py_GetRecursionLimit();
258
259 if (st == NULL)
260 return NULL;
261 if (filename == NULL) {
262 PySymtable_Free(st);
263 return NULL;
264 }
265 Py_INCREF(filename);
266 st->st_filename = filename;
267 st->st_future = future;
268
269 /* Setup recursion depth check counters */
270 tstate = PyThreadState_GET();
271 if (!tstate) {
272 PySymtable_Free(st);
273 return NULL;
274 }
275 /* Be careful here to prevent overflow. */
276 st->recursion_depth = (tstate->recursion_depth < INT_MAX / COMPILER_STACK_FRAME_SCALE) ?
277 tstate->recursion_depth * COMPILER_STACK_FRAME_SCALE : tstate->recursion_depth;
278 st->recursion_limit = (recursion_limit < INT_MAX / COMPILER_STACK_FRAME_SCALE) ?
279 recursion_limit * COMPILER_STACK_FRAME_SCALE : recursion_limit;
280
281 /* Make the initial symbol information gathering pass */
282 if (!GET_IDENTIFIER(top) ||
283 !symtable_enter_block(st, top, ModuleBlock, (void *)mod, 0, 0)) {
284 PySymtable_Free(st);
285 return NULL;
286 }
287
288 st->st_top = st->st_cur;
289 switch (mod->kind) {
290 case Module_kind:
291 seq = mod->v.Module.body;
292 for (i = 0; i < asdl_seq_LEN(seq); i++)
293 if (!symtable_visit_stmt(st,
294 (stmt_ty)asdl_seq_GET(seq, i)))
295 goto error;
296 break;
297 case Expression_kind:
298 if (!symtable_visit_expr(st, mod->v.Expression.body))
299 goto error;
300 break;
301 case Interactive_kind:
302 seq = mod->v.Interactive.body;
303 for (i = 0; i < asdl_seq_LEN(seq); i++)
304 if (!symtable_visit_stmt(st,
305 (stmt_ty)asdl_seq_GET(seq, i)))
306 goto error;
307 break;
308 case Suite_kind:
309 PyErr_SetString(PyExc_RuntimeError,
310 "this compiler does not handle Suites");
311 goto error;
312 }
313 if (!symtable_exit_block(st, (void *)mod)) {
314 PySymtable_Free(st);
315 return NULL;
316 }
317 /* Make the second symbol analysis pass */
318 if (symtable_analyze(st))
319 return st;
320 PySymtable_Free(st);
321 return NULL;
322 error:
323 (void) symtable_exit_block(st, (void *)mod);
324 PySymtable_Free(st);
325 return NULL;
326 }
327
328 struct symtable *
PySymtable_Build(mod_ty mod,const char * filename_str,PyFutureFeatures * future)329 PySymtable_Build(mod_ty mod, const char *filename_str, PyFutureFeatures *future)
330 {
331 PyObject *filename;
332 struct symtable *st;
333 filename = PyUnicode_DecodeFSDefault(filename_str);
334 if (filename == NULL)
335 return NULL;
336 st = PySymtable_BuildObject(mod, filename, future);
337 Py_DECREF(filename);
338 return st;
339 }
340
341 void
PySymtable_Free(struct symtable * st)342 PySymtable_Free(struct symtable *st)
343 {
344 Py_XDECREF(st->st_filename);
345 Py_XDECREF(st->st_blocks);
346 Py_XDECREF(st->st_stack);
347 PyMem_Free((void *)st);
348 }
349
350 PySTEntryObject *
PySymtable_Lookup(struct symtable * st,void * key)351 PySymtable_Lookup(struct symtable *st, void *key)
352 {
353 PyObject *k, *v;
354
355 k = PyLong_FromVoidPtr(key);
356 if (k == NULL)
357 return NULL;
358 v = PyDict_GetItem(st->st_blocks, k);
359 if (v) {
360 assert(PySTEntry_Check(v));
361 Py_INCREF(v);
362 }
363 else {
364 PyErr_SetString(PyExc_KeyError,
365 "unknown symbol table entry");
366 }
367
368 Py_DECREF(k);
369 return (PySTEntryObject *)v;
370 }
371
372 int
PyST_GetScope(PySTEntryObject * ste,PyObject * name)373 PyST_GetScope(PySTEntryObject *ste, PyObject *name)
374 {
375 PyObject *v = PyDict_GetItem(ste->ste_symbols, name);
376 if (!v)
377 return 0;
378 assert(PyLong_Check(v));
379 return (PyLong_AS_LONG(v) >> SCOPE_OFFSET) & SCOPE_MASK;
380 }
381
382 static int
error_at_directive(PySTEntryObject * ste,PyObject * name)383 error_at_directive(PySTEntryObject *ste, PyObject *name)
384 {
385 Py_ssize_t i;
386 PyObject *data;
387 assert(ste->ste_directives);
388 for (i = 0; i < PyList_GET_SIZE(ste->ste_directives); i++) {
389 data = PyList_GET_ITEM(ste->ste_directives, i);
390 assert(PyTuple_CheckExact(data));
391 assert(PyUnicode_CheckExact(PyTuple_GET_ITEM(data, 0)));
392 if (PyUnicode_Compare(PyTuple_GET_ITEM(data, 0), name) == 0) {
393 PyErr_SyntaxLocationObject(ste->ste_table->st_filename,
394 PyLong_AsLong(PyTuple_GET_ITEM(data, 1)),
395 PyLong_AsLong(PyTuple_GET_ITEM(data, 2)));
396
397 return 0;
398 }
399 }
400 PyErr_SetString(PyExc_RuntimeError,
401 "BUG: internal directive bookkeeping broken");
402 return 0;
403 }
404
405
406 /* Analyze raw symbol information to determine scope of each name.
407
408 The next several functions are helpers for symtable_analyze(),
409 which determines whether a name is local, global, or free. In addition,
410 it determines which local variables are cell variables; they provide
411 bindings that are used for free variables in enclosed blocks.
412
413 There are also two kinds of global variables, implicit and explicit. An
414 explicit global is declared with the global statement. An implicit
415 global is a free variable for which the compiler has found no binding
416 in an enclosing function scope. The implicit global is either a global
417 or a builtin. Python's module and class blocks use the xxx_NAME opcodes
418 to handle these names to implement slightly odd semantics. In such a
419 block, the name is treated as global until it is assigned to; then it
420 is treated as a local.
421
422 The symbol table requires two passes to determine the scope of each name.
423 The first pass collects raw facts from the AST via the symtable_visit_*
424 functions: the name is a parameter here, the name is used but not defined
425 here, etc. The second pass analyzes these facts during a pass over the
426 PySTEntryObjects created during pass 1.
427
428 When a function is entered during the second pass, the parent passes
429 the set of all name bindings visible to its children. These bindings
430 are used to determine if non-local variables are free or implicit globals.
431 Names which are explicitly declared nonlocal must exist in this set of
432 visible names - if they do not, a syntax error is raised. After doing
433 the local analysis, it analyzes each of its child blocks using an
434 updated set of name bindings.
435
436 The children update the free variable set. If a local variable is added to
437 the free variable set by the child, the variable is marked as a cell. The
438 function object being defined must provide runtime storage for the variable
439 that may outlive the function's frame. Cell variables are removed from the
440 free set before the analyze function returns to its parent.
441
442 During analysis, the names are:
443 symbols: dict mapping from symbol names to flag values (including offset scope values)
444 scopes: dict mapping from symbol names to scope values (no offset)
445 local: set of all symbol names local to the current scope
446 bound: set of all symbol names local to a containing function scope
447 free: set of all symbol names referenced but not bound in child scopes
448 global: set of all symbol names explicitly declared as global
449 */
450
451 #define SET_SCOPE(DICT, NAME, I) { \
452 PyObject *o = PyLong_FromLong(I); \
453 if (!o) \
454 return 0; \
455 if (PyDict_SetItem((DICT), (NAME), o) < 0) { \
456 Py_DECREF(o); \
457 return 0; \
458 } \
459 Py_DECREF(o); \
460 }
461
462 /* Decide on scope of name, given flags.
463
464 The namespace dictionaries may be modified to record information
465 about the new name. For example, a new global will add an entry to
466 global. A name that was global can be changed to local.
467 */
468
469 static int
analyze_name(PySTEntryObject * ste,PyObject * scopes,PyObject * name,long flags,PyObject * bound,PyObject * local,PyObject * free,PyObject * global)470 analyze_name(PySTEntryObject *ste, PyObject *scopes, PyObject *name, long flags,
471 PyObject *bound, PyObject *local, PyObject *free,
472 PyObject *global)
473 {
474 if (flags & DEF_GLOBAL) {
475 if (flags & DEF_NONLOCAL) {
476 PyErr_Format(PyExc_SyntaxError,
477 "name '%U' is nonlocal and global",
478 name);
479 return error_at_directive(ste, name);
480 }
481 SET_SCOPE(scopes, name, GLOBAL_EXPLICIT);
482 if (PySet_Add(global, name) < 0)
483 return 0;
484 if (bound && (PySet_Discard(bound, name) < 0))
485 return 0;
486 return 1;
487 }
488 if (flags & DEF_NONLOCAL) {
489 if (!bound) {
490 PyErr_Format(PyExc_SyntaxError,
491 "nonlocal declaration not allowed at module level");
492 return error_at_directive(ste, name);
493 }
494 if (!PySet_Contains(bound, name)) {
495 PyErr_Format(PyExc_SyntaxError,
496 "no binding for nonlocal '%U' found",
497 name);
498
499 return error_at_directive(ste, name);
500 }
501 SET_SCOPE(scopes, name, FREE);
502 ste->ste_free = 1;
503 return PySet_Add(free, name) >= 0;
504 }
505 if (flags & DEF_BOUND) {
506 SET_SCOPE(scopes, name, LOCAL);
507 if (PySet_Add(local, name) < 0)
508 return 0;
509 if (PySet_Discard(global, name) < 0)
510 return 0;
511 return 1;
512 }
513 /* If an enclosing block has a binding for this name, it
514 is a free variable rather than a global variable.
515 Note that having a non-NULL bound implies that the block
516 is nested.
517 */
518 if (bound && PySet_Contains(bound, name)) {
519 SET_SCOPE(scopes, name, FREE);
520 ste->ste_free = 1;
521 return PySet_Add(free, name) >= 0;
522 }
523 /* If a parent has a global statement, then call it global
524 explicit? It could also be global implicit.
525 */
526 if (global && PySet_Contains(global, name)) {
527 SET_SCOPE(scopes, name, GLOBAL_IMPLICIT);
528 return 1;
529 }
530 if (ste->ste_nested)
531 ste->ste_free = 1;
532 SET_SCOPE(scopes, name, GLOBAL_IMPLICIT);
533 return 1;
534 }
535
536 #undef SET_SCOPE
537
538 /* If a name is defined in free and also in locals, then this block
539 provides the binding for the free variable. The name should be
540 marked CELL in this block and removed from the free list.
541
542 Note that the current block's free variables are included in free.
543 That's safe because no name can be free and local in the same scope.
544 */
545
546 static int
analyze_cells(PyObject * scopes,PyObject * free)547 analyze_cells(PyObject *scopes, PyObject *free)
548 {
549 PyObject *name, *v, *v_cell;
550 int success = 0;
551 Py_ssize_t pos = 0;
552
553 v_cell = PyLong_FromLong(CELL);
554 if (!v_cell)
555 return 0;
556 while (PyDict_Next(scopes, &pos, &name, &v)) {
557 long scope;
558 assert(PyLong_Check(v));
559 scope = PyLong_AS_LONG(v);
560 if (scope != LOCAL)
561 continue;
562 if (!PySet_Contains(free, name))
563 continue;
564 /* Replace LOCAL with CELL for this name, and remove
565 from free. It is safe to replace the value of name
566 in the dict, because it will not cause a resize.
567 */
568 if (PyDict_SetItem(scopes, name, v_cell) < 0)
569 goto error;
570 if (PySet_Discard(free, name) < 0)
571 goto error;
572 }
573 success = 1;
574 error:
575 Py_DECREF(v_cell);
576 return success;
577 }
578
579 static int
drop_class_free(PySTEntryObject * ste,PyObject * free)580 drop_class_free(PySTEntryObject *ste, PyObject *free)
581 {
582 int res;
583 if (!GET_IDENTIFIER(__class__))
584 return 0;
585 res = PySet_Discard(free, __class__);
586 if (res < 0)
587 return 0;
588 if (res)
589 ste->ste_needs_class_closure = 1;
590 return 1;
591 }
592
593 /* Enter the final scope information into the ste_symbols dict.
594 *
595 * All arguments are dicts. Modifies symbols, others are read-only.
596 */
597 static int
update_symbols(PyObject * symbols,PyObject * scopes,PyObject * bound,PyObject * free,int classflag)598 update_symbols(PyObject *symbols, PyObject *scopes,
599 PyObject *bound, PyObject *free, int classflag)
600 {
601 PyObject *name = NULL, *itr = NULL;
602 PyObject *v = NULL, *v_scope = NULL, *v_new = NULL, *v_free = NULL;
603 Py_ssize_t pos = 0;
604
605 /* Update scope information for all symbols in this scope */
606 while (PyDict_Next(symbols, &pos, &name, &v)) {
607 long scope, flags;
608 assert(PyLong_Check(v));
609 flags = PyLong_AS_LONG(v);
610 v_scope = PyDict_GetItem(scopes, name);
611 assert(v_scope && PyLong_Check(v_scope));
612 scope = PyLong_AS_LONG(v_scope);
613 flags |= (scope << SCOPE_OFFSET);
614 v_new = PyLong_FromLong(flags);
615 if (!v_new)
616 return 0;
617 if (PyDict_SetItem(symbols, name, v_new) < 0) {
618 Py_DECREF(v_new);
619 return 0;
620 }
621 Py_DECREF(v_new);
622 }
623
624 /* Record not yet resolved free variables from children (if any) */
625 v_free = PyLong_FromLong(FREE << SCOPE_OFFSET);
626 if (!v_free)
627 return 0;
628
629 itr = PyObject_GetIter(free);
630 if (itr == NULL) {
631 Py_DECREF(v_free);
632 return 0;
633 }
634
635 while ((name = PyIter_Next(itr))) {
636 v = PyDict_GetItem(symbols, name);
637
638 /* Handle symbol that already exists in this scope */
639 if (v) {
640 /* Handle a free variable in a method of
641 the class that has the same name as a local
642 or global in the class scope.
643 */
644 if (classflag &&
645 PyLong_AS_LONG(v) & (DEF_BOUND | DEF_GLOBAL)) {
646 long flags = PyLong_AS_LONG(v) | DEF_FREE_CLASS;
647 v_new = PyLong_FromLong(flags);
648 if (!v_new) {
649 goto error;
650 }
651 if (PyDict_SetItem(symbols, name, v_new) < 0) {
652 Py_DECREF(v_new);
653 goto error;
654 }
655 Py_DECREF(v_new);
656 }
657 /* It's a cell, or already free in this scope */
658 Py_DECREF(name);
659 continue;
660 }
661 /* Handle global symbol */
662 if (bound && !PySet_Contains(bound, name)) {
663 Py_DECREF(name);
664 continue; /* it's a global */
665 }
666 /* Propagate new free symbol up the lexical stack */
667 if (PyDict_SetItem(symbols, name, v_free) < 0) {
668 goto error;
669 }
670 Py_DECREF(name);
671 }
672 Py_DECREF(itr);
673 Py_DECREF(v_free);
674 return 1;
675 error:
676 Py_XDECREF(v_free);
677 Py_XDECREF(itr);
678 Py_XDECREF(name);
679 return 0;
680 }
681
682 /* Make final symbol table decisions for block of ste.
683
684 Arguments:
685 ste -- current symtable entry (input/output)
686 bound -- set of variables bound in enclosing scopes (input). bound
687 is NULL for module blocks.
688 free -- set of free variables in enclosed scopes (output)
689 globals -- set of declared global variables in enclosing scopes (input)
690
691 The implementation uses two mutually recursive functions,
692 analyze_block() and analyze_child_block(). analyze_block() is
693 responsible for analyzing the individual names defined in a block.
694 analyze_child_block() prepares temporary namespace dictionaries
695 used to evaluated nested blocks.
696
697 The two functions exist because a child block should see the name
698 bindings of its enclosing blocks, but those bindings should not
699 propagate back to a parent block.
700 */
701
702 static int
703 analyze_child_block(PySTEntryObject *entry, PyObject *bound, PyObject *free,
704 PyObject *global, PyObject* child_free);
705
706 static int
analyze_block(PySTEntryObject * ste,PyObject * bound,PyObject * free,PyObject * global)707 analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free,
708 PyObject *global)
709 {
710 PyObject *name, *v, *local = NULL, *scopes = NULL, *newbound = NULL;
711 PyObject *newglobal = NULL, *newfree = NULL, *allfree = NULL;
712 PyObject *temp;
713 int i, success = 0;
714 Py_ssize_t pos = 0;
715
716 local = PySet_New(NULL); /* collect new names bound in block */
717 if (!local)
718 goto error;
719 scopes = PyDict_New(); /* collect scopes defined for each name */
720 if (!scopes)
721 goto error;
722
723 /* Allocate new global and bound variable dictionaries. These
724 dictionaries hold the names visible in nested blocks. For
725 ClassBlocks, the bound and global names are initialized
726 before analyzing names, because class bindings aren't
727 visible in methods. For other blocks, they are initialized
728 after names are analyzed.
729 */
730
731 /* TODO(jhylton): Package these dicts in a struct so that we
732 can write reasonable helper functions?
733 */
734 newglobal = PySet_New(NULL);
735 if (!newglobal)
736 goto error;
737 newfree = PySet_New(NULL);
738 if (!newfree)
739 goto error;
740 newbound = PySet_New(NULL);
741 if (!newbound)
742 goto error;
743
744 /* Class namespace has no effect on names visible in
745 nested functions, so populate the global and bound
746 sets to be passed to child blocks before analyzing
747 this one.
748 */
749 if (ste->ste_type == ClassBlock) {
750 /* Pass down known globals */
751 temp = PyNumber_InPlaceOr(newglobal, global);
752 if (!temp)
753 goto error;
754 Py_DECREF(temp);
755 /* Pass down previously bound symbols */
756 if (bound) {
757 temp = PyNumber_InPlaceOr(newbound, bound);
758 if (!temp)
759 goto error;
760 Py_DECREF(temp);
761 }
762 }
763
764 while (PyDict_Next(ste->ste_symbols, &pos, &name, &v)) {
765 long flags = PyLong_AS_LONG(v);
766 if (!analyze_name(ste, scopes, name, flags,
767 bound, local, free, global))
768 goto error;
769 }
770
771 /* Populate global and bound sets to be passed to children. */
772 if (ste->ste_type != ClassBlock) {
773 /* Add function locals to bound set */
774 if (ste->ste_type == FunctionBlock) {
775 temp = PyNumber_InPlaceOr(newbound, local);
776 if (!temp)
777 goto error;
778 Py_DECREF(temp);
779 }
780 /* Pass down previously bound symbols */
781 if (bound) {
782 temp = PyNumber_InPlaceOr(newbound, bound);
783 if (!temp)
784 goto error;
785 Py_DECREF(temp);
786 }
787 /* Pass down known globals */
788 temp = PyNumber_InPlaceOr(newglobal, global);
789 if (!temp)
790 goto error;
791 Py_DECREF(temp);
792 }
793 else {
794 /* Special-case __class__ */
795 if (!GET_IDENTIFIER(__class__))
796 goto error;
797 if (PySet_Add(newbound, __class__) < 0)
798 goto error;
799 }
800
801 /* Recursively call analyze_child_block() on each child block.
802
803 newbound, newglobal now contain the names visible in
804 nested blocks. The free variables in the children will
805 be collected in allfree.
806 */
807 allfree = PySet_New(NULL);
808 if (!allfree)
809 goto error;
810 for (i = 0; i < PyList_GET_SIZE(ste->ste_children); ++i) {
811 PyObject *c = PyList_GET_ITEM(ste->ste_children, i);
812 PySTEntryObject* entry;
813 assert(c && PySTEntry_Check(c));
814 entry = (PySTEntryObject*)c;
815 if (!analyze_child_block(entry, newbound, newfree, newglobal,
816 allfree))
817 goto error;
818 /* Check if any children have free variables */
819 if (entry->ste_free || entry->ste_child_free)
820 ste->ste_child_free = 1;
821 }
822
823 temp = PyNumber_InPlaceOr(newfree, allfree);
824 if (!temp)
825 goto error;
826 Py_DECREF(temp);
827
828 /* Check if any local variables must be converted to cell variables */
829 if (ste->ste_type == FunctionBlock && !analyze_cells(scopes, newfree))
830 goto error;
831 else if (ste->ste_type == ClassBlock && !drop_class_free(ste, newfree))
832 goto error;
833 /* Records the results of the analysis in the symbol table entry */
834 if (!update_symbols(ste->ste_symbols, scopes, bound, newfree,
835 ste->ste_type == ClassBlock))
836 goto error;
837
838 temp = PyNumber_InPlaceOr(free, newfree);
839 if (!temp)
840 goto error;
841 Py_DECREF(temp);
842 success = 1;
843 error:
844 Py_XDECREF(scopes);
845 Py_XDECREF(local);
846 Py_XDECREF(newbound);
847 Py_XDECREF(newglobal);
848 Py_XDECREF(newfree);
849 Py_XDECREF(allfree);
850 if (!success)
851 assert(PyErr_Occurred());
852 return success;
853 }
854
855 static int
analyze_child_block(PySTEntryObject * entry,PyObject * bound,PyObject * free,PyObject * global,PyObject * child_free)856 analyze_child_block(PySTEntryObject *entry, PyObject *bound, PyObject *free,
857 PyObject *global, PyObject* child_free)
858 {
859 PyObject *temp_bound = NULL, *temp_global = NULL, *temp_free = NULL;
860 PyObject *temp;
861
862 /* Copy the bound and global dictionaries.
863
864 These dictionaries are used by all blocks enclosed by the
865 current block. The analyze_block() call modifies these
866 dictionaries.
867
868 */
869 temp_bound = PySet_New(bound);
870 if (!temp_bound)
871 goto error;
872 temp_free = PySet_New(free);
873 if (!temp_free)
874 goto error;
875 temp_global = PySet_New(global);
876 if (!temp_global)
877 goto error;
878
879 if (!analyze_block(entry, temp_bound, temp_free, temp_global))
880 goto error;
881 temp = PyNumber_InPlaceOr(child_free, temp_free);
882 if (!temp)
883 goto error;
884 Py_DECREF(temp);
885 Py_DECREF(temp_bound);
886 Py_DECREF(temp_free);
887 Py_DECREF(temp_global);
888 return 1;
889 error:
890 Py_XDECREF(temp_bound);
891 Py_XDECREF(temp_free);
892 Py_XDECREF(temp_global);
893 return 0;
894 }
895
896 static int
symtable_analyze(struct symtable * st)897 symtable_analyze(struct symtable *st)
898 {
899 PyObject *free, *global;
900 int r;
901
902 free = PySet_New(NULL);
903 if (!free)
904 return 0;
905 global = PySet_New(NULL);
906 if (!global) {
907 Py_DECREF(free);
908 return 0;
909 }
910 r = analyze_block(st->st_top, NULL, free, global);
911 Py_DECREF(free);
912 Py_DECREF(global);
913 return r;
914 }
915
916 /* symtable_enter_block() gets a reference via ste_new.
917 This reference is released when the block is exited, via the DECREF
918 in symtable_exit_block().
919 */
920
921 static int
symtable_exit_block(struct symtable * st,void * ast)922 symtable_exit_block(struct symtable *st, void *ast)
923 {
924 Py_ssize_t size;
925
926 st->st_cur = NULL;
927 size = PyList_GET_SIZE(st->st_stack);
928 if (size) {
929 if (PyList_SetSlice(st->st_stack, size - 1, size, NULL) < 0)
930 return 0;
931 if (--size)
932 st->st_cur = (PySTEntryObject *)PyList_GET_ITEM(st->st_stack, size - 1);
933 }
934 return 1;
935 }
936
937 static int
symtable_enter_block(struct symtable * st,identifier name,_Py_block_ty block,void * ast,int lineno,int col_offset)938 symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block,
939 void *ast, int lineno, int col_offset)
940 {
941 PySTEntryObject *prev = NULL, *ste;
942
943 ste = ste_new(st, name, block, ast, lineno, col_offset);
944 if (ste == NULL)
945 return 0;
946 if (PyList_Append(st->st_stack, (PyObject *)ste) < 0) {
947 Py_DECREF(ste);
948 return 0;
949 }
950 prev = st->st_cur;
951 /* The entry is owned by the stack. Borrow it for st_cur. */
952 Py_DECREF(ste);
953 st->st_cur = ste;
954 if (block == ModuleBlock)
955 st->st_global = st->st_cur->ste_symbols;
956 if (prev) {
957 if (PyList_Append(prev->ste_children, (PyObject *)ste) < 0) {
958 return 0;
959 }
960 }
961 return 1;
962 }
963
964 static long
symtable_lookup(struct symtable * st,PyObject * name)965 symtable_lookup(struct symtable *st, PyObject *name)
966 {
967 PyObject *o;
968 PyObject *mangled = _Py_Mangle(st->st_private, name);
969 if (!mangled)
970 return 0;
971 o = PyDict_GetItem(st->st_cur->ste_symbols, mangled);
972 Py_DECREF(mangled);
973 if (!o)
974 return 0;
975 return PyLong_AsLong(o);
976 }
977
978 static int
symtable_add_def(struct symtable * st,PyObject * name,int flag)979 symtable_add_def(struct symtable *st, PyObject *name, int flag)
980 {
981 PyObject *o;
982 PyObject *dict;
983 long val;
984 PyObject *mangled = _Py_Mangle(st->st_private, name);
985
986
987 if (!mangled)
988 return 0;
989 dict = st->st_cur->ste_symbols;
990 if ((o = PyDict_GetItem(dict, mangled))) {
991 val = PyLong_AS_LONG(o);
992 if ((flag & DEF_PARAM) && (val & DEF_PARAM)) {
993 /* Is it better to use 'mangled' or 'name' here? */
994 PyErr_Format(PyExc_SyntaxError, DUPLICATE_ARGUMENT, name);
995 PyErr_SyntaxLocationObject(st->st_filename,
996 st->st_cur->ste_lineno,
997 st->st_cur->ste_col_offset);
998 goto error;
999 }
1000 val |= flag;
1001 } else
1002 val = flag;
1003 o = PyLong_FromLong(val);
1004 if (o == NULL)
1005 goto error;
1006 if (PyDict_SetItem(dict, mangled, o) < 0) {
1007 Py_DECREF(o);
1008 goto error;
1009 }
1010 Py_DECREF(o);
1011
1012 if (flag & DEF_PARAM) {
1013 if (PyList_Append(st->st_cur->ste_varnames, mangled) < 0)
1014 goto error;
1015 } else if (flag & DEF_GLOBAL) {
1016 /* XXX need to update DEF_GLOBAL for other flags too;
1017 perhaps only DEF_FREE_GLOBAL */
1018 val = flag;
1019 if ((o = PyDict_GetItem(st->st_global, mangled))) {
1020 val |= PyLong_AS_LONG(o);
1021 }
1022 o = PyLong_FromLong(val);
1023 if (o == NULL)
1024 goto error;
1025 if (PyDict_SetItem(st->st_global, mangled, o) < 0) {
1026 Py_DECREF(o);
1027 goto error;
1028 }
1029 Py_DECREF(o);
1030 }
1031 Py_DECREF(mangled);
1032 return 1;
1033
1034 error:
1035 Py_DECREF(mangled);
1036 return 0;
1037 }
1038
1039 /* VISIT, VISIT_SEQ and VIST_SEQ_TAIL take an ASDL type as their second argument.
1040 They use the ASDL name to synthesize the name of the C type and the visit
1041 function.
1042
1043 VISIT_SEQ_TAIL permits the start of an ASDL sequence to be skipped, which is
1044 useful if the first node in the sequence requires special treatment.
1045
1046 VISIT_QUIT macro returns the specified value exiting from the function but
1047 first adjusts current recursion counter depth.
1048 */
1049
1050 #define VISIT_QUIT(ST, X) \
1051 return --(ST)->recursion_depth,(X)
1052
1053 #define VISIT(ST, TYPE, V) \
1054 if (!symtable_visit_ ## TYPE((ST), (V))) \
1055 VISIT_QUIT((ST), 0);
1056
1057 #define VISIT_SEQ(ST, TYPE, SEQ) { \
1058 int i; \
1059 asdl_seq *seq = (SEQ); /* avoid variable capture */ \
1060 for (i = 0; i < asdl_seq_LEN(seq); i++) { \
1061 TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \
1062 if (!symtable_visit_ ## TYPE((ST), elt)) \
1063 VISIT_QUIT((ST), 0); \
1064 } \
1065 }
1066
1067 #define VISIT_SEQ_TAIL(ST, TYPE, SEQ, START) { \
1068 int i; \
1069 asdl_seq *seq = (SEQ); /* avoid variable capture */ \
1070 for (i = (START); i < asdl_seq_LEN(seq); i++) { \
1071 TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \
1072 if (!symtable_visit_ ## TYPE((ST), elt)) \
1073 VISIT_QUIT((ST), 0); \
1074 } \
1075 }
1076
1077 #define VISIT_SEQ_WITH_NULL(ST, TYPE, SEQ) { \
1078 int i = 0; \
1079 asdl_seq *seq = (SEQ); /* avoid variable capture */ \
1080 for (i = 0; i < asdl_seq_LEN(seq); i++) { \
1081 TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \
1082 if (!elt) continue; /* can be NULL */ \
1083 if (!symtable_visit_ ## TYPE((ST), elt)) \
1084 VISIT_QUIT((ST), 0); \
1085 } \
1086 }
1087
1088 static int
symtable_record_directive(struct symtable * st,identifier name,stmt_ty s)1089 symtable_record_directive(struct symtable *st, identifier name, stmt_ty s)
1090 {
1091 PyObject *data, *mangled;
1092 int res;
1093 if (!st->st_cur->ste_directives) {
1094 st->st_cur->ste_directives = PyList_New(0);
1095 if (!st->st_cur->ste_directives)
1096 return 0;
1097 }
1098 mangled = _Py_Mangle(st->st_private, name);
1099 if (!mangled)
1100 return 0;
1101 data = Py_BuildValue("(Nii)", mangled, s->lineno, s->col_offset);
1102 if (!data)
1103 return 0;
1104 res = PyList_Append(st->st_cur->ste_directives, data);
1105 Py_DECREF(data);
1106 return res == 0;
1107 }
1108
1109
1110 static int
symtable_visit_stmt(struct symtable * st,stmt_ty s)1111 symtable_visit_stmt(struct symtable *st, stmt_ty s)
1112 {
1113 if (++st->recursion_depth > st->recursion_limit) {
1114 PyErr_SetString(PyExc_RecursionError,
1115 "maximum recursion depth exceeded during compilation");
1116 VISIT_QUIT(st, 0);
1117 }
1118 switch (s->kind) {
1119 case FunctionDef_kind:
1120 if (!symtable_add_def(st, s->v.FunctionDef.name, DEF_LOCAL))
1121 VISIT_QUIT(st, 0);
1122 if (s->v.FunctionDef.args->defaults)
1123 VISIT_SEQ(st, expr, s->v.FunctionDef.args->defaults);
1124 if (s->v.FunctionDef.args->kw_defaults)
1125 VISIT_SEQ_WITH_NULL(st, expr, s->v.FunctionDef.args->kw_defaults);
1126 if (!symtable_visit_annotations(st, s, s->v.FunctionDef.args,
1127 s->v.FunctionDef.returns))
1128 VISIT_QUIT(st, 0);
1129 if (s->v.FunctionDef.decorator_list)
1130 VISIT_SEQ(st, expr, s->v.FunctionDef.decorator_list);
1131 if (!symtable_enter_block(st, s->v.FunctionDef.name,
1132 FunctionBlock, (void *)s, s->lineno,
1133 s->col_offset))
1134 VISIT_QUIT(st, 0);
1135 VISIT(st, arguments, s->v.FunctionDef.args);
1136 VISIT_SEQ(st, stmt, s->v.FunctionDef.body);
1137 if (!symtable_exit_block(st, s))
1138 VISIT_QUIT(st, 0);
1139 break;
1140 case ClassDef_kind: {
1141 PyObject *tmp;
1142 if (!symtable_add_def(st, s->v.ClassDef.name, DEF_LOCAL))
1143 VISIT_QUIT(st, 0);
1144 VISIT_SEQ(st, expr, s->v.ClassDef.bases);
1145 VISIT_SEQ(st, keyword, s->v.ClassDef.keywords);
1146 if (s->v.ClassDef.decorator_list)
1147 VISIT_SEQ(st, expr, s->v.ClassDef.decorator_list);
1148 if (!symtable_enter_block(st, s->v.ClassDef.name, ClassBlock,
1149 (void *)s, s->lineno, s->col_offset))
1150 VISIT_QUIT(st, 0);
1151 tmp = st->st_private;
1152 st->st_private = s->v.ClassDef.name;
1153 VISIT_SEQ(st, stmt, s->v.ClassDef.body);
1154 st->st_private = tmp;
1155 if (!symtable_exit_block(st, s))
1156 VISIT_QUIT(st, 0);
1157 break;
1158 }
1159 case Return_kind:
1160 if (s->v.Return.value) {
1161 VISIT(st, expr, s->v.Return.value);
1162 st->st_cur->ste_returns_value = 1;
1163 }
1164 break;
1165 case Delete_kind:
1166 VISIT_SEQ(st, expr, s->v.Delete.targets);
1167 break;
1168 case Assign_kind:
1169 VISIT_SEQ(st, expr, s->v.Assign.targets);
1170 VISIT(st, expr, s->v.Assign.value);
1171 break;
1172 case AnnAssign_kind:
1173 if (s->v.AnnAssign.target->kind == Name_kind) {
1174 expr_ty e_name = s->v.AnnAssign.target;
1175 long cur = symtable_lookup(st, e_name->v.Name.id);
1176 if (cur < 0) {
1177 VISIT_QUIT(st, 0);
1178 }
1179 if ((cur & (DEF_GLOBAL | DEF_NONLOCAL))
1180 && s->v.AnnAssign.simple) {
1181 PyErr_Format(PyExc_SyntaxError,
1182 cur & DEF_GLOBAL ? GLOBAL_ANNOT : NONLOCAL_ANNOT,
1183 e_name->v.Name.id);
1184 PyErr_SyntaxLocationObject(st->st_filename,
1185 s->lineno,
1186 s->col_offset);
1187 VISIT_QUIT(st, 0);
1188 }
1189 if (s->v.AnnAssign.simple &&
1190 !symtable_add_def(st, e_name->v.Name.id,
1191 DEF_ANNOT | DEF_LOCAL)) {
1192 VISIT_QUIT(st, 0);
1193 }
1194 else {
1195 if (s->v.AnnAssign.value
1196 && !symtable_add_def(st, e_name->v.Name.id, DEF_LOCAL)) {
1197 VISIT_QUIT(st, 0);
1198 }
1199 }
1200 }
1201 else {
1202 VISIT(st, expr, s->v.AnnAssign.target);
1203 }
1204 VISIT(st, expr, s->v.AnnAssign.annotation);
1205 if (s->v.AnnAssign.value) {
1206 VISIT(st, expr, s->v.AnnAssign.value);
1207 }
1208 break;
1209 case AugAssign_kind:
1210 VISIT(st, expr, s->v.AugAssign.target);
1211 VISIT(st, expr, s->v.AugAssign.value);
1212 break;
1213 case For_kind:
1214 VISIT(st, expr, s->v.For.target);
1215 VISIT(st, expr, s->v.For.iter);
1216 VISIT_SEQ(st, stmt, s->v.For.body);
1217 if (s->v.For.orelse)
1218 VISIT_SEQ(st, stmt, s->v.For.orelse);
1219 break;
1220 case While_kind:
1221 VISIT(st, expr, s->v.While.test);
1222 VISIT_SEQ(st, stmt, s->v.While.body);
1223 if (s->v.While.orelse)
1224 VISIT_SEQ(st, stmt, s->v.While.orelse);
1225 break;
1226 case If_kind:
1227 /* XXX if 0: and lookup_yield() hacks */
1228 VISIT(st, expr, s->v.If.test);
1229 VISIT_SEQ(st, stmt, s->v.If.body);
1230 if (s->v.If.orelse)
1231 VISIT_SEQ(st, stmt, s->v.If.orelse);
1232 break;
1233 case Raise_kind:
1234 if (s->v.Raise.exc) {
1235 VISIT(st, expr, s->v.Raise.exc);
1236 if (s->v.Raise.cause) {
1237 VISIT(st, expr, s->v.Raise.cause);
1238 }
1239 }
1240 break;
1241 case Try_kind:
1242 VISIT_SEQ(st, stmt, s->v.Try.body);
1243 VISIT_SEQ(st, stmt, s->v.Try.orelse);
1244 VISIT_SEQ(st, excepthandler, s->v.Try.handlers);
1245 VISIT_SEQ(st, stmt, s->v.Try.finalbody);
1246 break;
1247 case Assert_kind:
1248 VISIT(st, expr, s->v.Assert.test);
1249 if (s->v.Assert.msg)
1250 VISIT(st, expr, s->v.Assert.msg);
1251 break;
1252 case Import_kind:
1253 VISIT_SEQ(st, alias, s->v.Import.names);
1254 break;
1255 case ImportFrom_kind:
1256 VISIT_SEQ(st, alias, s->v.ImportFrom.names);
1257 break;
1258 case Global_kind: {
1259 int i;
1260 asdl_seq *seq = s->v.Global.names;
1261 for (i = 0; i < asdl_seq_LEN(seq); i++) {
1262 identifier name = (identifier)asdl_seq_GET(seq, i);
1263 long cur = symtable_lookup(st, name);
1264 if (cur < 0)
1265 VISIT_QUIT(st, 0);
1266 if (cur & (DEF_PARAM | DEF_LOCAL | USE | DEF_ANNOT)) {
1267 const char* msg;
1268 if (cur & DEF_PARAM) {
1269 msg = GLOBAL_PARAM;
1270 } else if (cur & USE) {
1271 msg = GLOBAL_AFTER_USE;
1272 } else if (cur & DEF_ANNOT) {
1273 msg = GLOBAL_ANNOT;
1274 } else { /* DEF_LOCAL */
1275 msg = GLOBAL_AFTER_ASSIGN;
1276 }
1277 PyErr_Format(PyExc_SyntaxError,
1278 msg, name);
1279 PyErr_SyntaxLocationObject(st->st_filename,
1280 s->lineno,
1281 s->col_offset);
1282 VISIT_QUIT(st, 0);
1283 }
1284 if (!symtable_add_def(st, name, DEF_GLOBAL))
1285 VISIT_QUIT(st, 0);
1286 if (!symtable_record_directive(st, name, s))
1287 VISIT_QUIT(st, 0);
1288 }
1289 break;
1290 }
1291 case Nonlocal_kind: {
1292 int i;
1293 asdl_seq *seq = s->v.Nonlocal.names;
1294 for (i = 0; i < asdl_seq_LEN(seq); i++) {
1295 identifier name = (identifier)asdl_seq_GET(seq, i);
1296 long cur = symtable_lookup(st, name);
1297 if (cur < 0)
1298 VISIT_QUIT(st, 0);
1299 if (cur & (DEF_PARAM | DEF_LOCAL | USE | DEF_ANNOT)) {
1300 const char* msg;
1301 if (cur & DEF_PARAM) {
1302 msg = NONLOCAL_PARAM;
1303 } else if (cur & USE) {
1304 msg = NONLOCAL_AFTER_USE;
1305 } else if (cur & DEF_ANNOT) {
1306 msg = NONLOCAL_ANNOT;
1307 } else { /* DEF_LOCAL */
1308 msg = NONLOCAL_AFTER_ASSIGN;
1309 }
1310 PyErr_Format(PyExc_SyntaxError, msg, name);
1311 PyErr_SyntaxLocationObject(st->st_filename,
1312 s->lineno,
1313 s->col_offset);
1314 VISIT_QUIT(st, 0);
1315 }
1316 if (!symtable_add_def(st, name, DEF_NONLOCAL))
1317 VISIT_QUIT(st, 0);
1318 if (!symtable_record_directive(st, name, s))
1319 VISIT_QUIT(st, 0);
1320 }
1321 break;
1322 }
1323 case Expr_kind:
1324 VISIT(st, expr, s->v.Expr.value);
1325 break;
1326 case Pass_kind:
1327 case Break_kind:
1328 case Continue_kind:
1329 /* nothing to do here */
1330 break;
1331 case With_kind:
1332 VISIT_SEQ(st, withitem, s->v.With.items);
1333 VISIT_SEQ(st, stmt, s->v.With.body);
1334 break;
1335 case AsyncFunctionDef_kind:
1336 if (!symtable_add_def(st, s->v.AsyncFunctionDef.name, DEF_LOCAL))
1337 VISIT_QUIT(st, 0);
1338 if (s->v.AsyncFunctionDef.args->defaults)
1339 VISIT_SEQ(st, expr, s->v.AsyncFunctionDef.args->defaults);
1340 if (s->v.AsyncFunctionDef.args->kw_defaults)
1341 VISIT_SEQ_WITH_NULL(st, expr,
1342 s->v.AsyncFunctionDef.args->kw_defaults);
1343 if (!symtable_visit_annotations(st, s, s->v.AsyncFunctionDef.args,
1344 s->v.AsyncFunctionDef.returns))
1345 VISIT_QUIT(st, 0);
1346 if (s->v.AsyncFunctionDef.decorator_list)
1347 VISIT_SEQ(st, expr, s->v.AsyncFunctionDef.decorator_list);
1348 if (!symtable_enter_block(st, s->v.AsyncFunctionDef.name,
1349 FunctionBlock, (void *)s, s->lineno,
1350 s->col_offset))
1351 VISIT_QUIT(st, 0);
1352 st->st_cur->ste_coroutine = 1;
1353 VISIT(st, arguments, s->v.AsyncFunctionDef.args);
1354 VISIT_SEQ(st, stmt, s->v.AsyncFunctionDef.body);
1355 if (!symtable_exit_block(st, s))
1356 VISIT_QUIT(st, 0);
1357 break;
1358 case AsyncWith_kind:
1359 VISIT_SEQ(st, withitem, s->v.AsyncWith.items);
1360 VISIT_SEQ(st, stmt, s->v.AsyncWith.body);
1361 break;
1362 case AsyncFor_kind:
1363 VISIT(st, expr, s->v.AsyncFor.target);
1364 VISIT(st, expr, s->v.AsyncFor.iter);
1365 VISIT_SEQ(st, stmt, s->v.AsyncFor.body);
1366 if (s->v.AsyncFor.orelse)
1367 VISIT_SEQ(st, stmt, s->v.AsyncFor.orelse);
1368 break;
1369 }
1370 VISIT_QUIT(st, 1);
1371 }
1372
1373 static int
symtable_visit_expr(struct symtable * st,expr_ty e)1374 symtable_visit_expr(struct symtable *st, expr_ty e)
1375 {
1376 if (++st->recursion_depth > st->recursion_limit) {
1377 PyErr_SetString(PyExc_RecursionError,
1378 "maximum recursion depth exceeded during compilation");
1379 VISIT_QUIT(st, 0);
1380 }
1381 switch (e->kind) {
1382 case BoolOp_kind:
1383 VISIT_SEQ(st, expr, e->v.BoolOp.values);
1384 break;
1385 case BinOp_kind:
1386 VISIT(st, expr, e->v.BinOp.left);
1387 VISIT(st, expr, e->v.BinOp.right);
1388 break;
1389 case UnaryOp_kind:
1390 VISIT(st, expr, e->v.UnaryOp.operand);
1391 break;
1392 case Lambda_kind: {
1393 if (!GET_IDENTIFIER(lambda))
1394 VISIT_QUIT(st, 0);
1395 if (e->v.Lambda.args->defaults)
1396 VISIT_SEQ(st, expr, e->v.Lambda.args->defaults);
1397 if (e->v.Lambda.args->kw_defaults)
1398 VISIT_SEQ_WITH_NULL(st, expr, e->v.Lambda.args->kw_defaults);
1399 if (!symtable_enter_block(st, lambda,
1400 FunctionBlock, (void *)e, e->lineno,
1401 e->col_offset))
1402 VISIT_QUIT(st, 0);
1403 VISIT(st, arguments, e->v.Lambda.args);
1404 VISIT(st, expr, e->v.Lambda.body);
1405 if (!symtable_exit_block(st, (void *)e))
1406 VISIT_QUIT(st, 0);
1407 break;
1408 }
1409 case IfExp_kind:
1410 VISIT(st, expr, e->v.IfExp.test);
1411 VISIT(st, expr, e->v.IfExp.body);
1412 VISIT(st, expr, e->v.IfExp.orelse);
1413 break;
1414 case Dict_kind:
1415 VISIT_SEQ_WITH_NULL(st, expr, e->v.Dict.keys);
1416 VISIT_SEQ(st, expr, e->v.Dict.values);
1417 break;
1418 case Set_kind:
1419 VISIT_SEQ(st, expr, e->v.Set.elts);
1420 break;
1421 case GeneratorExp_kind:
1422 if (!symtable_visit_genexp(st, e))
1423 VISIT_QUIT(st, 0);
1424 break;
1425 case ListComp_kind:
1426 if (!symtable_visit_listcomp(st, e))
1427 VISIT_QUIT(st, 0);
1428 break;
1429 case SetComp_kind:
1430 if (!symtable_visit_setcomp(st, e))
1431 VISIT_QUIT(st, 0);
1432 break;
1433 case DictComp_kind:
1434 if (!symtable_visit_dictcomp(st, e))
1435 VISIT_QUIT(st, 0);
1436 break;
1437 case Yield_kind:
1438 if (e->v.Yield.value)
1439 VISIT(st, expr, e->v.Yield.value);
1440 st->st_cur->ste_generator = 1;
1441 break;
1442 case YieldFrom_kind:
1443 VISIT(st, expr, e->v.YieldFrom.value);
1444 st->st_cur->ste_generator = 1;
1445 break;
1446 case Await_kind:
1447 VISIT(st, expr, e->v.Await.value);
1448 st->st_cur->ste_coroutine = 1;
1449 break;
1450 case Compare_kind:
1451 VISIT(st, expr, e->v.Compare.left);
1452 VISIT_SEQ(st, expr, e->v.Compare.comparators);
1453 break;
1454 case Call_kind:
1455 VISIT(st, expr, e->v.Call.func);
1456 VISIT_SEQ(st, expr, e->v.Call.args);
1457 VISIT_SEQ_WITH_NULL(st, keyword, e->v.Call.keywords);
1458 break;
1459 case FormattedValue_kind:
1460 VISIT(st, expr, e->v.FormattedValue.value);
1461 if (e->v.FormattedValue.format_spec)
1462 VISIT(st, expr, e->v.FormattedValue.format_spec);
1463 break;
1464 case JoinedStr_kind:
1465 VISIT_SEQ(st, expr, e->v.JoinedStr.values);
1466 break;
1467 case Constant_kind:
1468 case Num_kind:
1469 case Str_kind:
1470 case Bytes_kind:
1471 case Ellipsis_kind:
1472 case NameConstant_kind:
1473 /* Nothing to do here. */
1474 break;
1475 /* The following exprs can be assignment targets. */
1476 case Attribute_kind:
1477 VISIT(st, expr, e->v.Attribute.value);
1478 break;
1479 case Subscript_kind:
1480 VISIT(st, expr, e->v.Subscript.value);
1481 VISIT(st, slice, e->v.Subscript.slice);
1482 break;
1483 case Starred_kind:
1484 VISIT(st, expr, e->v.Starred.value);
1485 break;
1486 case Name_kind:
1487 if (!symtable_add_def(st, e->v.Name.id,
1488 e->v.Name.ctx == Load ? USE : DEF_LOCAL))
1489 VISIT_QUIT(st, 0);
1490 /* Special-case super: it counts as a use of __class__ */
1491 if (e->v.Name.ctx == Load &&
1492 st->st_cur->ste_type == FunctionBlock &&
1493 _PyUnicode_EqualToASCIIString(e->v.Name.id, "super")) {
1494 if (!GET_IDENTIFIER(__class__) ||
1495 !symtable_add_def(st, __class__, USE))
1496 VISIT_QUIT(st, 0);
1497 }
1498 break;
1499 /* child nodes of List and Tuple will have expr_context set */
1500 case List_kind:
1501 VISIT_SEQ(st, expr, e->v.List.elts);
1502 break;
1503 case Tuple_kind:
1504 VISIT_SEQ(st, expr, e->v.Tuple.elts);
1505 break;
1506 }
1507 VISIT_QUIT(st, 1);
1508 }
1509
1510 static int
symtable_implicit_arg(struct symtable * st,int pos)1511 symtable_implicit_arg(struct symtable *st, int pos)
1512 {
1513 PyObject *id = PyUnicode_FromFormat(".%d", pos);
1514 if (id == NULL)
1515 return 0;
1516 if (!symtable_add_def(st, id, DEF_PARAM)) {
1517 Py_DECREF(id);
1518 return 0;
1519 }
1520 Py_DECREF(id);
1521 return 1;
1522 }
1523
1524 static int
symtable_visit_params(struct symtable * st,asdl_seq * args)1525 symtable_visit_params(struct symtable *st, asdl_seq *args)
1526 {
1527 int i;
1528
1529 if (!args)
1530 return -1;
1531
1532 for (i = 0; i < asdl_seq_LEN(args); i++) {
1533 arg_ty arg = (arg_ty)asdl_seq_GET(args, i);
1534 if (!symtable_add_def(st, arg->arg, DEF_PARAM))
1535 return 0;
1536 }
1537
1538 return 1;
1539 }
1540
1541 static int
symtable_visit_argannotations(struct symtable * st,asdl_seq * args)1542 symtable_visit_argannotations(struct symtable *st, asdl_seq *args)
1543 {
1544 int i;
1545
1546 if (!args)
1547 return -1;
1548
1549 for (i = 0; i < asdl_seq_LEN(args); i++) {
1550 arg_ty arg = (arg_ty)asdl_seq_GET(args, i);
1551 if (arg->annotation)
1552 VISIT(st, expr, arg->annotation);
1553 }
1554
1555 return 1;
1556 }
1557
1558 static int
symtable_visit_annotations(struct symtable * st,stmt_ty s,arguments_ty a,expr_ty returns)1559 symtable_visit_annotations(struct symtable *st, stmt_ty s,
1560 arguments_ty a, expr_ty returns)
1561 {
1562 if (a->args && !symtable_visit_argannotations(st, a->args))
1563 return 0;
1564 if (a->vararg && a->vararg->annotation)
1565 VISIT(st, expr, a->vararg->annotation);
1566 if (a->kwarg && a->kwarg->annotation)
1567 VISIT(st, expr, a->kwarg->annotation);
1568 if (a->kwonlyargs && !symtable_visit_argannotations(st, a->kwonlyargs))
1569 return 0;
1570 if (returns)
1571 VISIT(st, expr, returns);
1572 return 1;
1573 }
1574
1575 static int
symtable_visit_arguments(struct symtable * st,arguments_ty a)1576 symtable_visit_arguments(struct symtable *st, arguments_ty a)
1577 {
1578 /* skip default arguments inside function block
1579 XXX should ast be different?
1580 */
1581 if (a->args && !symtable_visit_params(st, a->args))
1582 return 0;
1583 if (a->kwonlyargs && !symtable_visit_params(st, a->kwonlyargs))
1584 return 0;
1585 if (a->vararg) {
1586 if (!symtable_add_def(st, a->vararg->arg, DEF_PARAM))
1587 return 0;
1588 st->st_cur->ste_varargs = 1;
1589 }
1590 if (a->kwarg) {
1591 if (!symtable_add_def(st, a->kwarg->arg, DEF_PARAM))
1592 return 0;
1593 st->st_cur->ste_varkeywords = 1;
1594 }
1595 return 1;
1596 }
1597
1598
1599 static int
symtable_visit_excepthandler(struct symtable * st,excepthandler_ty eh)1600 symtable_visit_excepthandler(struct symtable *st, excepthandler_ty eh)
1601 {
1602 if (eh->v.ExceptHandler.type)
1603 VISIT(st, expr, eh->v.ExceptHandler.type);
1604 if (eh->v.ExceptHandler.name)
1605 if (!symtable_add_def(st, eh->v.ExceptHandler.name, DEF_LOCAL))
1606 return 0;
1607 VISIT_SEQ(st, stmt, eh->v.ExceptHandler.body);
1608 return 1;
1609 }
1610
1611 static int
symtable_visit_withitem(struct symtable * st,withitem_ty item)1612 symtable_visit_withitem(struct symtable *st, withitem_ty item)
1613 {
1614 VISIT(st, expr, item->context_expr);
1615 if (item->optional_vars) {
1616 VISIT(st, expr, item->optional_vars);
1617 }
1618 return 1;
1619 }
1620
1621
1622 static int
symtable_visit_alias(struct symtable * st,alias_ty a)1623 symtable_visit_alias(struct symtable *st, alias_ty a)
1624 {
1625 /* Compute store_name, the name actually bound by the import
1626 operation. It is different than a->name when a->name is a
1627 dotted package name (e.g. spam.eggs)
1628 */
1629 PyObject *store_name;
1630 PyObject *name = (a->asname == NULL) ? a->name : a->asname;
1631 Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0,
1632 PyUnicode_GET_LENGTH(name), 1);
1633 if (dot != -1) {
1634 store_name = PyUnicode_Substring(name, 0, dot);
1635 if (!store_name)
1636 return 0;
1637 }
1638 else {
1639 store_name = name;
1640 Py_INCREF(store_name);
1641 }
1642 if (!_PyUnicode_EqualToASCIIString(name, "*")) {
1643 int r = symtable_add_def(st, store_name, DEF_IMPORT);
1644 Py_DECREF(store_name);
1645 return r;
1646 }
1647 else {
1648 if (st->st_cur->ste_type != ModuleBlock) {
1649 int lineno = st->st_cur->ste_lineno;
1650 int col_offset = st->st_cur->ste_col_offset;
1651 PyErr_SetString(PyExc_SyntaxError, IMPORT_STAR_WARNING);
1652 PyErr_SyntaxLocationObject(st->st_filename, lineno, col_offset);
1653 Py_DECREF(store_name);
1654 return 0;
1655 }
1656 Py_DECREF(store_name);
1657 return 1;
1658 }
1659 }
1660
1661
1662 static int
symtable_visit_comprehension(struct symtable * st,comprehension_ty lc)1663 symtable_visit_comprehension(struct symtable *st, comprehension_ty lc)
1664 {
1665 VISIT(st, expr, lc->target);
1666 VISIT(st, expr, lc->iter);
1667 VISIT_SEQ(st, expr, lc->ifs);
1668 if (lc->is_async) {
1669 st->st_cur->ste_coroutine = 1;
1670 }
1671 return 1;
1672 }
1673
1674
1675 static int
symtable_visit_keyword(struct symtable * st,keyword_ty k)1676 symtable_visit_keyword(struct symtable *st, keyword_ty k)
1677 {
1678 VISIT(st, expr, k->value);
1679 return 1;
1680 }
1681
1682
1683 static int
symtable_visit_slice(struct symtable * st,slice_ty s)1684 symtable_visit_slice(struct symtable *st, slice_ty s)
1685 {
1686 switch (s->kind) {
1687 case Slice_kind:
1688 if (s->v.Slice.lower)
1689 VISIT(st, expr, s->v.Slice.lower)
1690 if (s->v.Slice.upper)
1691 VISIT(st, expr, s->v.Slice.upper)
1692 if (s->v.Slice.step)
1693 VISIT(st, expr, s->v.Slice.step)
1694 break;
1695 case ExtSlice_kind:
1696 VISIT_SEQ(st, slice, s->v.ExtSlice.dims)
1697 break;
1698 case Index_kind:
1699 VISIT(st, expr, s->v.Index.value)
1700 break;
1701 }
1702 return 1;
1703 }
1704
1705 static int
symtable_handle_comprehension(struct symtable * st,expr_ty e,identifier scope_name,asdl_seq * generators,expr_ty elt,expr_ty value)1706 symtable_handle_comprehension(struct symtable *st, expr_ty e,
1707 identifier scope_name, asdl_seq *generators,
1708 expr_ty elt, expr_ty value)
1709 {
1710 int is_generator = (e->kind == GeneratorExp_kind);
1711 comprehension_ty outermost = ((comprehension_ty)
1712 asdl_seq_GET(generators, 0));
1713 /* Outermost iterator is evaluated in current scope */
1714 VISIT(st, expr, outermost->iter);
1715 /* Create comprehension scope for the rest */
1716 if (!scope_name ||
1717 !symtable_enter_block(st, scope_name, FunctionBlock, (void *)e,
1718 e->lineno, e->col_offset)) {
1719 return 0;
1720 }
1721 if (outermost->is_async) {
1722 st->st_cur->ste_coroutine = 1;
1723 }
1724 /* Outermost iter is received as an argument */
1725 if (!symtable_implicit_arg(st, 0)) {
1726 symtable_exit_block(st, (void *)e);
1727 return 0;
1728 }
1729 VISIT(st, expr, outermost->target);
1730 VISIT_SEQ(st, expr, outermost->ifs);
1731 VISIT_SEQ_TAIL(st, comprehension, generators, 1);
1732 if (value)
1733 VISIT(st, expr, value);
1734 VISIT(st, expr, elt);
1735 if (st->st_cur->ste_generator) {
1736 PyObject *msg = PyUnicode_FromString(
1737 (e->kind == ListComp_kind) ? "'yield' inside list comprehension" :
1738 (e->kind == SetComp_kind) ? "'yield' inside set comprehension" :
1739 (e->kind == DictComp_kind) ? "'yield' inside dict comprehension" :
1740 "'yield' inside generator expression");
1741 if (msg == NULL) {
1742 symtable_exit_block(st, (void *)e);
1743 return 0;
1744 }
1745 if (PyErr_WarnExplicitObject(PyExc_DeprecationWarning,
1746 msg, st->st_filename, st->st_cur->ste_lineno,
1747 NULL, NULL) == -1)
1748 {
1749 if (PyErr_ExceptionMatches(PyExc_DeprecationWarning)) {
1750 /* Replace the DeprecationWarning exception with a SyntaxError
1751 to get a more accurate error report */
1752 PyErr_Clear();
1753 PyErr_SetObject(PyExc_SyntaxError, msg);
1754 PyErr_SyntaxLocationObject(st->st_filename,
1755 st->st_cur->ste_lineno,
1756 st->st_cur->ste_col_offset);
1757 }
1758 Py_DECREF(msg);
1759 symtable_exit_block(st, (void *)e);
1760 return 0;
1761 }
1762 Py_DECREF(msg);
1763 }
1764 st->st_cur->ste_generator |= is_generator;
1765 return symtable_exit_block(st, (void *)e);
1766 }
1767
1768 static int
symtable_visit_genexp(struct symtable * st,expr_ty e)1769 symtable_visit_genexp(struct symtable *st, expr_ty e)
1770 {
1771 return symtable_handle_comprehension(st, e, GET_IDENTIFIER(genexpr),
1772 e->v.GeneratorExp.generators,
1773 e->v.GeneratorExp.elt, NULL);
1774 }
1775
1776 static int
symtable_visit_listcomp(struct symtable * st,expr_ty e)1777 symtable_visit_listcomp(struct symtable *st, expr_ty e)
1778 {
1779 return symtable_handle_comprehension(st, e, GET_IDENTIFIER(listcomp),
1780 e->v.ListComp.generators,
1781 e->v.ListComp.elt, NULL);
1782 }
1783
1784 static int
symtable_visit_setcomp(struct symtable * st,expr_ty e)1785 symtable_visit_setcomp(struct symtable *st, expr_ty e)
1786 {
1787 return symtable_handle_comprehension(st, e, GET_IDENTIFIER(setcomp),
1788 e->v.SetComp.generators,
1789 e->v.SetComp.elt, NULL);
1790 }
1791
1792 static int
symtable_visit_dictcomp(struct symtable * st,expr_ty e)1793 symtable_visit_dictcomp(struct symtable *st, expr_ty e)
1794 {
1795 return symtable_handle_comprehension(st, e, GET_IDENTIFIER(dictcomp),
1796 e->v.DictComp.generators,
1797 e->v.DictComp.key,
1798 e->v.DictComp.value);
1799 }
1800