1#! /usr/bin/env python 2"""Generate C code from an ASDL description.""" 3 4# TO DO 5# handle fields that have a type but no name 6 7import os, sys 8 9import asdl 10 11TABSIZE = 8 12MAX_COL = 80 13 14def get_c_type(name): 15 """Return a string for the C name of the type. 16 17 This function special cases the default types provided by asdl: 18 identifier, string, int, bool. 19 """ 20 # XXX ack! need to figure out where Id is useful and where string 21 if isinstance(name, asdl.Id): 22 name = name.value 23 if name in asdl.builtin_types: 24 return name 25 else: 26 return "%s_ty" % name 27 28def reflow_lines(s, depth): 29 """Reflow the line s indented depth tabs. 30 31 Return a sequence of lines where no line extends beyond MAX_COL 32 when properly indented. The first line is properly indented based 33 exclusively on depth * TABSIZE. All following lines -- these are 34 the reflowed lines generated by this function -- start at the same 35 column as the first character beyond the opening { in the first 36 line. 37 """ 38 size = MAX_COL - depth * TABSIZE 39 if len(s) < size: 40 return [s] 41 42 lines = [] 43 cur = s 44 padding = "" 45 while len(cur) > size: 46 i = cur.rfind(' ', 0, size) 47 # XXX this should be fixed for real 48 if i == -1 and 'GeneratorExp' in cur: 49 i = size + 3 50 assert i != -1, "Impossible line %d to reflow: %r" % (size, s) 51 lines.append(padding + cur[:i]) 52 if len(lines) == 1: 53 # find new size based on brace 54 j = cur.find('{', 0, i) 55 if j >= 0: 56 j += 2 # account for the brace and the space after it 57 size -= j 58 padding = " " * j 59 else: 60 j = cur.find('(', 0, i) 61 if j >= 0: 62 j += 1 # account for the paren (no space after it) 63 size -= j 64 padding = " " * j 65 cur = cur[i+1:] 66 else: 67 lines.append(padding + cur) 68 return lines 69 70def is_simple(sum): 71 """Return True if a sum is a simple. 72 73 A sum is simple if its types have no fields, e.g. 74 unaryop = Invert | Not | UAdd | USub 75 """ 76 for t in sum.types: 77 if t.fields: 78 return False 79 return True 80 81 82class EmitVisitor(asdl.VisitorBase): 83 """Visit that emits lines""" 84 85 def __init__(self, file): 86 self.file = file 87 super(EmitVisitor, self).__init__() 88 89 def emit(self, s, depth, reflow=True): 90 # XXX reflow long lines? 91 if reflow: 92 lines = reflow_lines(s, depth) 93 else: 94 lines = [s] 95 for line in lines: 96 line = (" " * TABSIZE * depth) + line + "\n" 97 self.file.write(line) 98 99 100class TypeDefVisitor(EmitVisitor): 101 def visitModule(self, mod): 102 for dfn in mod.dfns: 103 self.visit(dfn) 104 105 def visitType(self, type, depth=0): 106 self.visit(type.value, type.name, depth) 107 108 def visitSum(self, sum, name, depth): 109 if is_simple(sum): 110 self.simple_sum(sum, name, depth) 111 else: 112 self.sum_with_constructors(sum, name, depth) 113 114 def simple_sum(self, sum, name, depth): 115 enum = [] 116 for i in range(len(sum.types)): 117 type = sum.types[i] 118 enum.append("%s=%d" % (type.name, i + 1)) 119 enums = ", ".join(enum) 120 ctype = get_c_type(name) 121 s = "typedef enum _%s { %s } %s;" % (name, enums, ctype) 122 self.emit(s, depth) 123 self.emit("", depth) 124 125 def sum_with_constructors(self, sum, name, depth): 126 ctype = get_c_type(name) 127 s = "typedef struct _%(name)s *%(ctype)s;" % locals() 128 self.emit(s, depth) 129 self.emit("", depth) 130 131 def visitProduct(self, product, name, depth): 132 ctype = get_c_type(name) 133 s = "typedef struct _%(name)s *%(ctype)s;" % locals() 134 self.emit(s, depth) 135 self.emit("", depth) 136 137 138class StructVisitor(EmitVisitor): 139 """Visitor to generate typdefs for AST.""" 140 141 def visitModule(self, mod): 142 for dfn in mod.dfns: 143 self.visit(dfn) 144 145 def visitType(self, type, depth=0): 146 self.visit(type.value, type.name, depth) 147 148 def visitSum(self, sum, name, depth): 149 if not is_simple(sum): 150 self.sum_with_constructors(sum, name, depth) 151 152 def sum_with_constructors(self, sum, name, depth): 153 def emit(s, depth=depth): 154 self.emit(s % sys._getframe(1).f_locals, depth) 155 enum = [] 156 for i in range(len(sum.types)): 157 type = sum.types[i] 158 enum.append("%s_kind=%d" % (type.name, i + 1)) 159 160 emit("enum _%(name)s_kind {" + ", ".join(enum) + "};") 161 162 emit("struct _%(name)s {") 163 emit("enum _%(name)s_kind kind;", depth + 1) 164 emit("union {", depth + 1) 165 for t in sum.types: 166 self.visit(t, depth + 2) 167 emit("} v;", depth + 1) 168 for field in sum.attributes: 169 # rudimentary attribute handling 170 type = str(field.type) 171 assert type in asdl.builtin_types, type 172 emit("%s %s;" % (type, field.name), depth + 1); 173 emit("};") 174 emit("") 175 176 def visitConstructor(self, cons, depth): 177 if cons.fields: 178 self.emit("struct {", depth) 179 for f in cons.fields: 180 self.visit(f, depth + 1) 181 self.emit("} %s;" % cons.name, depth) 182 self.emit("", depth) 183 else: 184 # XXX not sure what I want here, nothing is probably fine 185 pass 186 187 def visitField(self, field, depth): 188 # XXX need to lookup field.type, because it might be something 189 # like a builtin... 190 ctype = get_c_type(field.type) 191 name = field.name 192 if field.seq: 193 if field.type.value in ('cmpop',): 194 self.emit("asdl_int_seq *%(name)s;" % locals(), depth) 195 else: 196 self.emit("asdl_seq *%(name)s;" % locals(), depth) 197 else: 198 self.emit("%(ctype)s %(name)s;" % locals(), depth) 199 200 def visitProduct(self, product, name, depth): 201 self.emit("struct _%(name)s {" % locals(), depth) 202 for f in product.fields: 203 self.visit(f, depth + 1) 204 self.emit("};", depth) 205 self.emit("", depth) 206 207 208class PrototypeVisitor(EmitVisitor): 209 """Generate function prototypes for the .h file""" 210 211 def visitModule(self, mod): 212 for dfn in mod.dfns: 213 self.visit(dfn) 214 215 def visitType(self, type): 216 self.visit(type.value, type.name) 217 218 def visitSum(self, sum, name): 219 if is_simple(sum): 220 pass # XXX 221 else: 222 for t in sum.types: 223 self.visit(t, name, sum.attributes) 224 225 def get_args(self, fields): 226 """Return list of C argument into, one for each field. 227 228 Argument info is 3-tuple of a C type, variable name, and flag 229 that is true if type can be NULL. 230 """ 231 args = [] 232 unnamed = {} 233 for f in fields: 234 if f.name is None: 235 name = f.type 236 c = unnamed[name] = unnamed.get(name, 0) + 1 237 if c > 1: 238 name = "name%d" % (c - 1) 239 else: 240 name = f.name 241 # XXX should extend get_c_type() to handle this 242 if f.seq: 243 if f.type.value in ('cmpop',): 244 ctype = "asdl_int_seq *" 245 else: 246 ctype = "asdl_seq *" 247 else: 248 ctype = get_c_type(f.type) 249 args.append((ctype, name, f.opt or f.seq)) 250 return args 251 252 def visitConstructor(self, cons, type, attrs): 253 args = self.get_args(cons.fields) 254 attrs = self.get_args(attrs) 255 ctype = get_c_type(type) 256 self.emit_function(cons.name, ctype, args, attrs) 257 258 def emit_function(self, name, ctype, args, attrs, union=True): 259 args = args + attrs 260 if args: 261 argstr = ", ".join(["%s %s" % (atype, aname) 262 for atype, aname, opt in args]) 263 argstr += ", PyArena *arena" 264 else: 265 argstr = "PyArena *arena" 266 margs = "a0" 267 for i in range(1, len(args)+1): 268 margs += ", a%d" % i 269 self.emit("#define %s(%s) _Py_%s(%s)" % (name, margs, name, margs), 0, 270 reflow=False) 271 self.emit("%s _Py_%s(%s);" % (ctype, name, argstr), False) 272 273 def visitProduct(self, prod, name): 274 self.emit_function(name, get_c_type(name), 275 self.get_args(prod.fields), [], union=False) 276 277 278class FunctionVisitor(PrototypeVisitor): 279 """Visitor to generate constructor functions for AST.""" 280 281 def emit_function(self, name, ctype, args, attrs, union=True): 282 def emit(s, depth=0, reflow=True): 283 self.emit(s, depth, reflow) 284 argstr = ", ".join(["%s %s" % (atype, aname) 285 for atype, aname, opt in args + attrs]) 286 if argstr: 287 argstr += ", PyArena *arena" 288 else: 289 argstr = "PyArena *arena" 290 self.emit("%s" % ctype, 0) 291 emit("%s(%s)" % (name, argstr)) 292 emit("{") 293 emit("%s p;" % ctype, 1) 294 for argtype, argname, opt in args: 295 # XXX hack alert: false is allowed for a bool 296 if not opt and not (argtype == "bool" or argtype == "int"): 297 emit("if (!%s) {" % argname, 1) 298 emit("PyErr_SetString(PyExc_ValueError,", 2) 299 msg = "field %s is required for %s" % (argname, name) 300 emit(' "%s");' % msg, 301 2, reflow=False) 302 emit('return NULL;', 2) 303 emit('}', 1) 304 305 emit("p = (%s)PyArena_Malloc(arena, sizeof(*p));" % ctype, 1); 306 emit("if (!p)", 1) 307 emit("return NULL;", 2) 308 if union: 309 self.emit_body_union(name, args, attrs) 310 else: 311 self.emit_body_struct(name, args, attrs) 312 emit("return p;", 1) 313 emit("}") 314 emit("") 315 316 def emit_body_union(self, name, args, attrs): 317 def emit(s, depth=0, reflow=True): 318 self.emit(s, depth, reflow) 319 emit("p->kind = %s_kind;" % name, 1) 320 for argtype, argname, opt in args: 321 emit("p->v.%s.%s = %s;" % (name, argname, argname), 1) 322 for argtype, argname, opt in attrs: 323 emit("p->%s = %s;" % (argname, argname), 1) 324 325 def emit_body_struct(self, name, args, attrs): 326 def emit(s, depth=0, reflow=True): 327 self.emit(s, depth, reflow) 328 for argtype, argname, opt in args: 329 emit("p->%s = %s;" % (argname, argname), 1) 330 assert not attrs 331 332 333class PickleVisitor(EmitVisitor): 334 335 def visitModule(self, mod): 336 for dfn in mod.dfns: 337 self.visit(dfn) 338 339 def visitType(self, type): 340 self.visit(type.value, type.name) 341 342 def visitSum(self, sum, name): 343 pass 344 345 def visitProduct(self, sum, name): 346 pass 347 348 def visitConstructor(self, cons, name): 349 pass 350 351 def visitField(self, sum): 352 pass 353 354 355class Obj2ModPrototypeVisitor(PickleVisitor): 356 def visitProduct(self, prod, name): 357 code = "static int obj2ast_%s(PyObject* obj, %s* out, PyArena* arena);" 358 self.emit(code % (name, get_c_type(name)), 0) 359 360 visitSum = visitProduct 361 362 363class Obj2ModVisitor(PickleVisitor): 364 def funcHeader(self, name): 365 ctype = get_c_type(name) 366 self.emit("int", 0) 367 self.emit("obj2ast_%s(PyObject* obj, %s* out, PyArena* arena)" % (name, ctype), 0) 368 self.emit("{", 0) 369 self.emit("PyObject* tmp = NULL;", 1) 370 self.emit("int isinstance;", 1) 371 self.emit("", 0) 372 373 def sumTrailer(self, name): 374 self.emit("", 0) 375 self.emit("tmp = PyObject_Repr(obj);", 1) 376 # there's really nothing more we can do if this fails ... 377 self.emit("if (tmp == NULL) goto failed;", 1) 378 error = "expected some sort of %s, but got %%.400s" % name 379 format = "PyErr_Format(PyExc_TypeError, \"%s\", PyString_AS_STRING(tmp));" 380 self.emit(format % error, 1, reflow=False) 381 self.emit("failed:", 0) 382 self.emit("Py_XDECREF(tmp);", 1) 383 self.emit("return 1;", 1) 384 self.emit("}", 0) 385 self.emit("", 0) 386 387 def simpleSum(self, sum, name): 388 self.funcHeader(name) 389 for t in sum.types: 390 line = ("isinstance = PyObject_IsInstance(obj, " 391 "(PyObject *)%s_type);") 392 self.emit(line % (t.name,), 1) 393 self.emit("if (isinstance == -1) {", 1) 394 self.emit("return 1;", 2) 395 self.emit("}", 1) 396 self.emit("if (isinstance) {", 1) 397 self.emit("*out = %s;" % t.name, 2) 398 self.emit("return 0;", 2) 399 self.emit("}", 1) 400 self.sumTrailer(name) 401 402 def buildArgs(self, fields): 403 return ", ".join(fields + ["arena"]) 404 405 def complexSum(self, sum, name): 406 self.funcHeader(name) 407 for a in sum.attributes: 408 self.visitAttributeDeclaration(a, name, sum=sum) 409 self.emit("", 0) 410 # XXX: should we only do this for 'expr'? 411 self.emit("if (obj == Py_None) {", 1) 412 self.emit("*out = NULL;", 2) 413 self.emit("return 0;", 2) 414 self.emit("}", 1) 415 for a in sum.attributes: 416 self.visitField(a, name, sum=sum, depth=1) 417 for t in sum.types: 418 line = "isinstance = PyObject_IsInstance(obj, (PyObject*)%s_type);" 419 self.emit(line % (t.name,), 1) 420 self.emit("if (isinstance == -1) {", 1) 421 self.emit("return 1;", 2) 422 self.emit("}", 1) 423 self.emit("if (isinstance) {", 1) 424 for f in t.fields: 425 self.visitFieldDeclaration(f, t.name, sum=sum, depth=2) 426 self.emit("", 0) 427 for f in t.fields: 428 self.visitField(f, t.name, sum=sum, depth=2) 429 args = [f.name.value for f in t.fields] + [a.name.value for a in sum.attributes] 430 self.emit("*out = %s(%s);" % (t.name, self.buildArgs(args)), 2) 431 self.emit("if (*out == NULL) goto failed;", 2) 432 self.emit("return 0;", 2) 433 self.emit("}", 1) 434 self.sumTrailer(name) 435 436 def visitAttributeDeclaration(self, a, name, sum=sum): 437 ctype = get_c_type(a.type) 438 self.emit("%s %s;" % (ctype, a.name), 1) 439 440 def visitSum(self, sum, name): 441 if is_simple(sum): 442 self.simpleSum(sum, name) 443 else: 444 self.complexSum(sum, name) 445 446 def visitProduct(self, prod, name): 447 ctype = get_c_type(name) 448 self.emit("int", 0) 449 self.emit("obj2ast_%s(PyObject* obj, %s* out, PyArena* arena)" % (name, ctype), 0) 450 self.emit("{", 0) 451 self.emit("PyObject* tmp = NULL;", 1) 452 for f in prod.fields: 453 self.visitFieldDeclaration(f, name, prod=prod, depth=1) 454 self.emit("", 0) 455 for f in prod.fields: 456 self.visitField(f, name, prod=prod, depth=1) 457 args = [f.name.value for f in prod.fields] 458 self.emit("*out = %s(%s);" % (name, self.buildArgs(args)), 1) 459 self.emit("return 0;", 1) 460 self.emit("failed:", 0) 461 self.emit("Py_XDECREF(tmp);", 1) 462 self.emit("return 1;", 1) 463 self.emit("}", 0) 464 self.emit("", 0) 465 466 def visitFieldDeclaration(self, field, name, sum=None, prod=None, depth=0): 467 ctype = get_c_type(field.type) 468 if field.seq: 469 if self.isSimpleType(field): 470 self.emit("asdl_int_seq* %s;" % field.name, depth) 471 else: 472 self.emit("asdl_seq* %s;" % field.name, depth) 473 else: 474 ctype = get_c_type(field.type) 475 self.emit("%s %s;" % (ctype, field.name), depth) 476 477 def isSimpleSum(self, field): 478 # XXX can the members of this list be determined automatically? 479 return field.type.value in ('expr_context', 'boolop', 'operator', 480 'unaryop', 'cmpop') 481 482 def isNumeric(self, field): 483 return get_c_type(field.type) in ("int", "bool") 484 485 def isSimpleType(self, field): 486 return self.isSimpleSum(field) or self.isNumeric(field) 487 488 def visitField(self, field, name, sum=None, prod=None, depth=0): 489 ctype = get_c_type(field.type) 490 self.emit("if (PyObject_HasAttrString(obj, \"%s\")) {" % field.name, depth) 491 self.emit("int res;", depth+1) 492 if field.seq: 493 self.emit("Py_ssize_t len;", depth+1) 494 self.emit("Py_ssize_t i;", depth+1) 495 self.emit("tmp = PyObject_GetAttrString(obj, \"%s\");" % field.name, depth+1) 496 self.emit("if (tmp == NULL) goto failed;", depth+1) 497 if field.seq: 498 self.emit("if (!PyList_Check(tmp)) {", depth+1) 499 self.emit("PyErr_Format(PyExc_TypeError, \"%s field \\\"%s\\\" must " 500 "be a list, not a %%.200s\", tmp->ob_type->tp_name);" % 501 (name, field.name), 502 depth+2, reflow=False) 503 self.emit("goto failed;", depth+2) 504 self.emit("}", depth+1) 505 self.emit("len = PyList_GET_SIZE(tmp);", depth+1) 506 if self.isSimpleType(field): 507 self.emit("%s = asdl_int_seq_new(len, arena);" % field.name, depth+1) 508 else: 509 self.emit("%s = asdl_seq_new(len, arena);" % field.name, depth+1) 510 self.emit("if (%s == NULL) goto failed;" % field.name, depth+1) 511 self.emit("for (i = 0; i < len; i++) {", depth+1) 512 self.emit("%s value;" % ctype, depth+2) 513 self.emit("res = obj2ast_%s(PyList_GET_ITEM(tmp, i), &value, arena);" % 514 field.type, depth+2, reflow=False) 515 self.emit("if (res != 0) goto failed;", depth+2) 516 self.emit("asdl_seq_SET(%s, i, value);" % field.name, depth+2) 517 self.emit("}", depth+1) 518 else: 519 self.emit("res = obj2ast_%s(tmp, &%s, arena);" % 520 (field.type, field.name), depth+1) 521 self.emit("if (res != 0) goto failed;", depth+1) 522 523 self.emit("Py_XDECREF(tmp);", depth+1) 524 self.emit("tmp = NULL;", depth+1) 525 self.emit("} else {", depth) 526 if not field.opt: 527 message = "required field \\\"%s\\\" missing from %s" % (field.name, name) 528 format = "PyErr_SetString(PyExc_TypeError, \"%s\");" 529 self.emit(format % message, depth+1, reflow=False) 530 self.emit("return 1;", depth+1) 531 else: 532 if self.isNumeric(field): 533 self.emit("%s = 0;" % field.name, depth+1) 534 elif not self.isSimpleType(field): 535 self.emit("%s = NULL;" % field.name, depth+1) 536 else: 537 raise TypeError("could not determine the default value for %s" % field.name) 538 self.emit("}", depth) 539 540 541class MarshalPrototypeVisitor(PickleVisitor): 542 543 def prototype(self, sum, name): 544 ctype = get_c_type(name) 545 self.emit("static int marshal_write_%s(PyObject **, int *, %s);" 546 % (name, ctype), 0) 547 548 visitProduct = visitSum = prototype 549 550 551class PyTypesDeclareVisitor(PickleVisitor): 552 553 def visitProduct(self, prod, name): 554 self.emit("static PyTypeObject *%s_type;" % name, 0) 555 self.emit("static PyObject* ast2obj_%s(void*);" % name, 0) 556 if prod.fields: 557 self.emit("static char *%s_fields[]={" % name,0) 558 for f in prod.fields: 559 self.emit('"%s",' % f.name, 1) 560 self.emit("};", 0) 561 562 def visitSum(self, sum, name): 563 self.emit("static PyTypeObject *%s_type;" % name, 0) 564 if sum.attributes: 565 self.emit("static char *%s_attributes[] = {" % name, 0) 566 for a in sum.attributes: 567 self.emit('"%s",' % a.name, 1) 568 self.emit("};", 0) 569 ptype = "void*" 570 if is_simple(sum): 571 ptype = get_c_type(name) 572 tnames = [] 573 for t in sum.types: 574 tnames.append(str(t.name)+"_singleton") 575 tnames = ", *".join(tnames) 576 self.emit("static PyObject *%s;" % tnames, 0) 577 self.emit("static PyObject* ast2obj_%s(%s);" % (name, ptype), 0) 578 for t in sum.types: 579 self.visitConstructor(t, name) 580 581 def visitConstructor(self, cons, name): 582 self.emit("static PyTypeObject *%s_type;" % cons.name, 0) 583 if cons.fields: 584 self.emit("static char *%s_fields[]={" % cons.name, 0) 585 for t in cons.fields: 586 self.emit('"%s",' % t.name, 1) 587 self.emit("};",0) 588 589class PyTypesVisitor(PickleVisitor): 590 591 def visitModule(self, mod): 592 self.emit(""" 593static int 594ast_type_init(PyObject *self, PyObject *args, PyObject *kw) 595{ 596 Py_ssize_t i, numfields = 0; 597 int res = -1; 598 PyObject *key, *value, *fields; 599 fields = PyObject_GetAttrString((PyObject*)Py_TYPE(self), "_fields"); 600 if (!fields) 601 PyErr_Clear(); 602 if (fields) { 603 numfields = PySequence_Size(fields); 604 if (numfields == -1) 605 goto cleanup; 606 } 607 res = 0; /* if no error occurs, this stays 0 to the end */ 608 if (PyTuple_GET_SIZE(args) > 0) { 609 if (numfields != PyTuple_GET_SIZE(args)) { 610 PyErr_Format(PyExc_TypeError, "%.400s constructor takes %s" 611 "%zd positional argument%s", 612 Py_TYPE(self)->tp_name, 613 numfields == 0 ? "" : "either 0 or ", 614 numfields, numfields == 1 ? "" : "s"); 615 res = -1; 616 goto cleanup; 617 } 618 for (i = 0; i < PyTuple_GET_SIZE(args); i++) { 619 /* cannot be reached when fields is NULL */ 620 PyObject *name = PySequence_GetItem(fields, i); 621 if (!name) { 622 res = -1; 623 goto cleanup; 624 } 625 res = PyObject_SetAttr(self, name, PyTuple_GET_ITEM(args, i)); 626 Py_DECREF(name); 627 if (res < 0) 628 goto cleanup; 629 } 630 } 631 if (kw) { 632 i = 0; /* needed by PyDict_Next */ 633 while (PyDict_Next(kw, &i, &key, &value)) { 634 res = PyObject_SetAttr(self, key, value); 635 if (res < 0) 636 goto cleanup; 637 } 638 } 639 cleanup: 640 Py_XDECREF(fields); 641 return res; 642} 643 644/* Pickling support */ 645static PyObject * 646ast_type_reduce(PyObject *self, PyObject *unused) 647{ 648 PyObject *res; 649 PyObject *dict = PyObject_GetAttrString(self, "__dict__"); 650 if (dict == NULL) { 651 if (PyErr_ExceptionMatches(PyExc_AttributeError)) 652 PyErr_Clear(); 653 else 654 return NULL; 655 } 656 if (dict) { 657 res = Py_BuildValue("O()O", Py_TYPE(self), dict); 658 Py_DECREF(dict); 659 return res; 660 } 661 return Py_BuildValue("O()", Py_TYPE(self)); 662} 663 664static PyMethodDef ast_type_methods[] = { 665 {"__reduce__", ast_type_reduce, METH_NOARGS, NULL}, 666 {NULL} 667}; 668 669static PyTypeObject AST_type = { 670 PyVarObject_HEAD_INIT(&PyType_Type, 0) 671 "_ast.AST", 672 sizeof(PyObject), 673 0, 674 0, /* tp_dealloc */ 675 0, /* tp_print */ 676 0, /* tp_getattr */ 677 0, /* tp_setattr */ 678 0, /* tp_compare */ 679 0, /* tp_repr */ 680 0, /* tp_as_number */ 681 0, /* tp_as_sequence */ 682 0, /* tp_as_mapping */ 683 0, /* tp_hash */ 684 0, /* tp_call */ 685 0, /* tp_str */ 686 PyObject_GenericGetAttr, /* tp_getattro */ 687 PyObject_GenericSetAttr, /* tp_setattro */ 688 0, /* tp_as_buffer */ 689 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ 690 0, /* tp_doc */ 691 0, /* tp_traverse */ 692 0, /* tp_clear */ 693 0, /* tp_richcompare */ 694 0, /* tp_weaklistoffset */ 695 0, /* tp_iter */ 696 0, /* tp_iternext */ 697 ast_type_methods, /* tp_methods */ 698 0, /* tp_members */ 699 0, /* tp_getset */ 700 0, /* tp_base */ 701 0, /* tp_dict */ 702 0, /* tp_descr_get */ 703 0, /* tp_descr_set */ 704 0, /* tp_dictoffset */ 705 (initproc)ast_type_init, /* tp_init */ 706 PyType_GenericAlloc, /* tp_alloc */ 707 PyType_GenericNew, /* tp_new */ 708 PyObject_Del, /* tp_free */ 709}; 710 711 712static PyTypeObject* make_type(char *type, PyTypeObject* base, char**fields, int num_fields) 713{ 714 PyObject *fnames, *result; 715 int i; 716 fnames = PyTuple_New(num_fields); 717 if (!fnames) return NULL; 718 for (i = 0; i < num_fields; i++) { 719 PyObject *field = PyString_FromString(fields[i]); 720 if (!field) { 721 Py_DECREF(fnames); 722 return NULL; 723 } 724 PyTuple_SET_ITEM(fnames, i, field); 725 } 726 result = PyObject_CallFunction((PyObject*)&PyType_Type, "s(O){sOss}", 727 type, base, "_fields", fnames, "__module__", "_ast"); 728 Py_DECREF(fnames); 729 return (PyTypeObject*)result; 730} 731 732static int add_attributes(PyTypeObject* type, char**attrs, int num_fields) 733{ 734 int i, result; 735 PyObject *s, *l = PyTuple_New(num_fields); 736 if (!l) 737 return 0; 738 for (i = 0; i < num_fields; i++) { 739 s = PyString_FromString(attrs[i]); 740 if (!s) { 741 Py_DECREF(l); 742 return 0; 743 } 744 PyTuple_SET_ITEM(l, i, s); 745 } 746 result = PyObject_SetAttrString((PyObject*)type, "_attributes", l) >= 0; 747 Py_DECREF(l); 748 return result; 749} 750 751/* Conversion AST -> Python */ 752 753static PyObject* ast2obj_list(asdl_seq *seq, PyObject* (*func)(void*)) 754{ 755 int i, n = asdl_seq_LEN(seq); 756 PyObject *result = PyList_New(n); 757 PyObject *value; 758 if (!result) 759 return NULL; 760 for (i = 0; i < n; i++) { 761 value = func(asdl_seq_GET(seq, i)); 762 if (!value) { 763 Py_DECREF(result); 764 return NULL; 765 } 766 PyList_SET_ITEM(result, i, value); 767 } 768 return result; 769} 770 771static PyObject* ast2obj_object(void *o) 772{ 773 if (!o) 774 o = Py_None; 775 Py_INCREF((PyObject*)o); 776 return (PyObject*)o; 777} 778#define ast2obj_identifier ast2obj_object 779#define ast2obj_string ast2obj_object 780static PyObject* ast2obj_bool(bool b) 781{ 782 return PyBool_FromLong(b); 783} 784 785static PyObject* ast2obj_int(long b) 786{ 787 return PyInt_FromLong(b); 788} 789 790/* Conversion Python -> AST */ 791 792static int obj2ast_object(PyObject* obj, PyObject** out, PyArena* arena) 793{ 794 if (obj == Py_None) 795 obj = NULL; 796 if (obj) 797 PyArena_AddPyObject(arena, obj); 798 Py_XINCREF(obj); 799 *out = obj; 800 return 0; 801} 802 803#define obj2ast_identifier obj2ast_object 804#define obj2ast_string obj2ast_object 805 806static int obj2ast_int(PyObject* obj, int* out, PyArena* arena) 807{ 808 int i; 809 if (!PyInt_Check(obj) && !PyLong_Check(obj)) { 810 PyObject *s = PyObject_Repr(obj); 811 if (s == NULL) return 1; 812 PyErr_Format(PyExc_ValueError, "invalid integer value: %.400s", 813 PyString_AS_STRING(s)); 814 Py_DECREF(s); 815 return 1; 816 } 817 818 i = (int)PyLong_AsLong(obj); 819 if (i == -1 && PyErr_Occurred()) 820 return 1; 821 *out = i; 822 return 0; 823} 824 825static int obj2ast_bool(PyObject* obj, bool* out, PyArena* arena) 826{ 827 if (!PyBool_Check(obj)) { 828 PyObject *s = PyObject_Repr(obj); 829 if (s == NULL) return 1; 830 PyErr_Format(PyExc_ValueError, "invalid boolean value: %.400s", 831 PyString_AS_STRING(s)); 832 Py_DECREF(s); 833 return 1; 834 } 835 836 *out = (obj == Py_True); 837 return 0; 838} 839 840static int add_ast_fields(void) 841{ 842 PyObject *empty_tuple, *d; 843 if (PyType_Ready(&AST_type) < 0) 844 return -1; 845 d = AST_type.tp_dict; 846 empty_tuple = PyTuple_New(0); 847 if (!empty_tuple || 848 PyDict_SetItemString(d, "_fields", empty_tuple) < 0 || 849 PyDict_SetItemString(d, "_attributes", empty_tuple) < 0) { 850 Py_XDECREF(empty_tuple); 851 return -1; 852 } 853 Py_DECREF(empty_tuple); 854 return 0; 855} 856 857""", 0, reflow=False) 858 859 self.emit("static int init_types(void)",0) 860 self.emit("{", 0) 861 self.emit("static int initialized;", 1) 862 self.emit("if (initialized) return 1;", 1) 863 self.emit("if (add_ast_fields() < 0) return 0;", 1) 864 for dfn in mod.dfns: 865 self.visit(dfn) 866 self.emit("initialized = 1;", 1) 867 self.emit("return 1;", 1); 868 self.emit("}", 0) 869 870 def visitProduct(self, prod, name): 871 if prod.fields: 872 fields = name.value+"_fields" 873 else: 874 fields = "NULL" 875 self.emit('%s_type = make_type("%s", &AST_type, %s, %d);' % 876 (name, name, fields, len(prod.fields)), 1) 877 self.emit("if (!%s_type) return 0;" % name, 1) 878 879 def visitSum(self, sum, name): 880 self.emit('%s_type = make_type("%s", &AST_type, NULL, 0);' % 881 (name, name), 1) 882 self.emit("if (!%s_type) return 0;" % name, 1) 883 if sum.attributes: 884 self.emit("if (!add_attributes(%s_type, %s_attributes, %d)) return 0;" % 885 (name, name, len(sum.attributes)), 1) 886 else: 887 self.emit("if (!add_attributes(%s_type, NULL, 0)) return 0;" % name, 1) 888 simple = is_simple(sum) 889 for t in sum.types: 890 self.visitConstructor(t, name, simple) 891 892 def visitConstructor(self, cons, name, simple): 893 if cons.fields: 894 fields = cons.name.value+"_fields" 895 else: 896 fields = "NULL" 897 self.emit('%s_type = make_type("%s", %s_type, %s, %d);' % 898 (cons.name, cons.name, name, fields, len(cons.fields)), 1) 899 self.emit("if (!%s_type) return 0;" % cons.name, 1) 900 if simple: 901 self.emit("%s_singleton = PyType_GenericNew(%s_type, NULL, NULL);" % 902 (cons.name, cons.name), 1) 903 self.emit("if (!%s_singleton) return 0;" % cons.name, 1) 904 905 906def parse_version(mod): 907 return mod.version.value[12:-3] 908 909class ASTModuleVisitor(PickleVisitor): 910 911 def visitModule(self, mod): 912 self.emit("PyMODINIT_FUNC", 0) 913 self.emit("init_ast(void)", 0) 914 self.emit("{", 0) 915 self.emit("PyObject *m, *d;", 1) 916 self.emit("if (!init_types()) return;", 1) 917 self.emit('m = Py_InitModule3("_ast", NULL, NULL);', 1) 918 self.emit("if (!m) return;", 1) 919 self.emit("d = PyModule_GetDict(m);", 1) 920 self.emit('if (PyDict_SetItemString(d, "AST", (PyObject*)&AST_type) < 0) return;', 1) 921 self.emit('if (PyModule_AddIntConstant(m, "PyCF_ONLY_AST", PyCF_ONLY_AST) < 0)', 1) 922 self.emit("return;", 2) 923 # Value of version: "$Revision$" 924 self.emit('if (PyModule_AddStringConstant(m, "__version__", "%s") < 0)' 925 % parse_version(mod), 1) 926 self.emit("return;", 2) 927 for dfn in mod.dfns: 928 self.visit(dfn) 929 self.emit("}", 0) 930 931 def visitProduct(self, prod, name): 932 self.addObj(name) 933 934 def visitSum(self, sum, name): 935 self.addObj(name) 936 for t in sum.types: 937 self.visitConstructor(t, name) 938 939 def visitConstructor(self, cons, name): 940 self.addObj(cons.name) 941 942 def addObj(self, name): 943 self.emit('if (PyDict_SetItemString(d, "%s", (PyObject*)%s_type) < 0) return;' % (name, name), 1) 944 945 946_SPECIALIZED_SEQUENCES = ('stmt', 'expr') 947 948def find_sequence(fields, doing_specialization): 949 """Return True if any field uses a sequence.""" 950 for f in fields: 951 if f.seq: 952 if not doing_specialization: 953 return True 954 if str(f.type) not in _SPECIALIZED_SEQUENCES: 955 return True 956 return False 957 958def has_sequence(types, doing_specialization): 959 for t in types: 960 if find_sequence(t.fields, doing_specialization): 961 return True 962 return False 963 964 965class StaticVisitor(PickleVisitor): 966 CODE = '''Very simple, always emit this static code. Overide CODE''' 967 968 def visit(self, object): 969 self.emit(self.CODE, 0, reflow=False) 970 971 972class ObjVisitor(PickleVisitor): 973 974 def func_begin(self, name): 975 ctype = get_c_type(name) 976 self.emit("PyObject*", 0) 977 self.emit("ast2obj_%s(void* _o)" % (name), 0) 978 self.emit("{", 0) 979 self.emit("%s o = (%s)_o;" % (ctype, ctype), 1) 980 self.emit("PyObject *result = NULL, *value = NULL;", 1) 981 self.emit('if (!o) {', 1) 982 self.emit("Py_INCREF(Py_None);", 2) 983 self.emit('return Py_None;', 2) 984 self.emit("}", 1) 985 self.emit('', 0) 986 987 def func_end(self): 988 self.emit("return result;", 1) 989 self.emit("failed:", 0) 990 self.emit("Py_XDECREF(value);", 1) 991 self.emit("Py_XDECREF(result);", 1) 992 self.emit("return NULL;", 1) 993 self.emit("}", 0) 994 self.emit("", 0) 995 996 def visitSum(self, sum, name): 997 if is_simple(sum): 998 self.simpleSum(sum, name) 999 return 1000 self.func_begin(name) 1001 self.emit("switch (o->kind) {", 1) 1002 for i in range(len(sum.types)): 1003 t = sum.types[i] 1004 self.visitConstructor(t, i + 1, name) 1005 self.emit("}", 1) 1006 for a in sum.attributes: 1007 self.emit("value = ast2obj_%s(o->%s);" % (a.type, a.name), 1) 1008 self.emit("if (!value) goto failed;", 1) 1009 self.emit('if (PyObject_SetAttrString(result, "%s", value) < 0)' % a.name, 1) 1010 self.emit('goto failed;', 2) 1011 self.emit('Py_DECREF(value);', 1) 1012 self.func_end() 1013 1014 def simpleSum(self, sum, name): 1015 self.emit("PyObject* ast2obj_%s(%s_ty o)" % (name, name), 0) 1016 self.emit("{", 0) 1017 self.emit("switch(o) {", 1) 1018 for t in sum.types: 1019 self.emit("case %s:" % t.name, 2) 1020 self.emit("Py_INCREF(%s_singleton);" % t.name, 3) 1021 self.emit("return %s_singleton;" % t.name, 3) 1022 self.emit("default:" % name, 2) 1023 self.emit('/* should never happen, but just in case ... */', 3) 1024 code = "PyErr_Format(PyExc_SystemError, \"unknown %s found\");" % name 1025 self.emit(code, 3, reflow=False) 1026 self.emit("return NULL;", 3) 1027 self.emit("}", 1) 1028 self.emit("}", 0) 1029 1030 def visitProduct(self, prod, name): 1031 self.func_begin(name) 1032 self.emit("result = PyType_GenericNew(%s_type, NULL, NULL);" % name, 1); 1033 self.emit("if (!result) return NULL;", 1) 1034 for field in prod.fields: 1035 self.visitField(field, name, 1, True) 1036 self.func_end() 1037 1038 def visitConstructor(self, cons, enum, name): 1039 self.emit("case %s_kind:" % cons.name, 1) 1040 self.emit("result = PyType_GenericNew(%s_type, NULL, NULL);" % cons.name, 2); 1041 self.emit("if (!result) goto failed;", 2) 1042 for f in cons.fields: 1043 self.visitField(f, cons.name, 2, False) 1044 self.emit("break;", 2) 1045 1046 def visitField(self, field, name, depth, product): 1047 def emit(s, d): 1048 self.emit(s, depth + d) 1049 if product: 1050 value = "o->%s" % field.name 1051 else: 1052 value = "o->v.%s.%s" % (name, field.name) 1053 self.set(field, value, depth) 1054 emit("if (!value) goto failed;", 0) 1055 emit('if (PyObject_SetAttrString(result, "%s", value) == -1)' % field.name, 0) 1056 emit("goto failed;", 1) 1057 emit("Py_DECREF(value);", 0) 1058 1059 def emitSeq(self, field, value, depth, emit): 1060 emit("seq = %s;" % value, 0) 1061 emit("n = asdl_seq_LEN(seq);", 0) 1062 emit("value = PyList_New(n);", 0) 1063 emit("if (!value) goto failed;", 0) 1064 emit("for (i = 0; i < n; i++) {", 0) 1065 self.set("value", field, "asdl_seq_GET(seq, i)", depth + 1) 1066 emit("if (!value1) goto failed;", 1) 1067 emit("PyList_SET_ITEM(value, i, value1);", 1) 1068 emit("value1 = NULL;", 1) 1069 emit("}", 0) 1070 1071 def set(self, field, value, depth): 1072 if field.seq: 1073 # XXX should really check for is_simple, but that requires a symbol table 1074 if field.type.value == "cmpop": 1075 # While the sequence elements are stored as void*, 1076 # ast2obj_cmpop expects an enum 1077 self.emit("{", depth) 1078 self.emit("int i, n = asdl_seq_LEN(%s);" % value, depth+1) 1079 self.emit("value = PyList_New(n);", depth+1) 1080 self.emit("if (!value) goto failed;", depth+1) 1081 self.emit("for(i = 0; i < n; i++)", depth+1) 1082 # This cannot fail, so no need for error handling 1083 self.emit("PyList_SET_ITEM(value, i, ast2obj_cmpop((cmpop_ty)asdl_seq_GET(%s, i)));" % value, 1084 depth+2, reflow=False) 1085 self.emit("}", depth) 1086 else: 1087 self.emit("value = ast2obj_list(%s, ast2obj_%s);" % (value, field.type), depth) 1088 else: 1089 ctype = get_c_type(field.type) 1090 self.emit("value = ast2obj_%s(%s);" % (field.type, value), depth, reflow=False) 1091 1092 1093class PartingShots(StaticVisitor): 1094 1095 CODE = """ 1096PyObject* PyAST_mod2obj(mod_ty t) 1097{ 1098 init_types(); 1099 return ast2obj_mod(t); 1100} 1101 1102/* mode is 0 for "exec", 1 for "eval" and 2 for "single" input */ 1103mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode) 1104{ 1105 mod_ty res; 1106 PyObject *req_type[] = {(PyObject*)Module_type, (PyObject*)Expression_type, 1107 (PyObject*)Interactive_type}; 1108 char *req_name[] = {"Module", "Expression", "Interactive"}; 1109 int isinstance; 1110 assert(0 <= mode && mode <= 2); 1111 1112 init_types(); 1113 1114 isinstance = PyObject_IsInstance(ast, req_type[mode]); 1115 if (isinstance == -1) 1116 return NULL; 1117 if (!isinstance) { 1118 PyErr_Format(PyExc_TypeError, "expected %s node, got %.400s", 1119 req_name[mode], Py_TYPE(ast)->tp_name); 1120 return NULL; 1121 } 1122 if (obj2ast_mod(ast, &res, arena) != 0) 1123 return NULL; 1124 else 1125 return res; 1126} 1127 1128int PyAST_Check(PyObject* obj) 1129{ 1130 init_types(); 1131 return PyObject_IsInstance(obj, (PyObject*)&AST_type); 1132} 1133""" 1134 1135class ChainOfVisitors: 1136 def __init__(self, *visitors): 1137 self.visitors = visitors 1138 1139 def visit(self, object): 1140 for v in self.visitors: 1141 v.visit(object) 1142 v.emit("", 0) 1143 1144common_msg = "/* File automatically generated by %s. */\n\n" 1145 1146c_file_msg = """ 1147/* 1148 __version__ %s. 1149 1150 This module must be committed separately after each AST grammar change; 1151 The __version__ number is set to the revision number of the commit 1152 containing the grammar change. 1153*/ 1154 1155""" 1156 1157def main(srcfile): 1158 argv0 = sys.argv[0] 1159 components = argv0.split(os.sep) 1160 argv0 = os.sep.join(components[-2:]) 1161 auto_gen_msg = common_msg % argv0 1162 mod = asdl.parse(srcfile) 1163 if not asdl.check(mod): 1164 sys.exit(1) 1165 if INC_DIR: 1166 p = "%s/%s-ast.h" % (INC_DIR, mod.name) 1167 f = open(p, "wb") 1168 f.write(auto_gen_msg) 1169 f.write('#include "asdl.h"\n\n') 1170 c = ChainOfVisitors(TypeDefVisitor(f), 1171 StructVisitor(f), 1172 PrototypeVisitor(f), 1173 ) 1174 c.visit(mod) 1175 f.write("PyObject* PyAST_mod2obj(mod_ty t);\n") 1176 f.write("mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);\n") 1177 f.write("int PyAST_Check(PyObject* obj);\n") 1178 f.close() 1179 1180 if SRC_DIR: 1181 p = os.path.join(SRC_DIR, str(mod.name) + "-ast.c") 1182 f = open(p, "wb") 1183 f.write(auto_gen_msg) 1184 f.write(c_file_msg % parse_version(mod)) 1185 f.write('#include "Python.h"\n') 1186 f.write('#include "%s-ast.h"\n' % mod.name) 1187 f.write('\n') 1188 f.write("static PyTypeObject AST_type;\n") 1189 v = ChainOfVisitors( 1190 PyTypesDeclareVisitor(f), 1191 PyTypesVisitor(f), 1192 Obj2ModPrototypeVisitor(f), 1193 FunctionVisitor(f), 1194 ObjVisitor(f), 1195 Obj2ModVisitor(f), 1196 ASTModuleVisitor(f), 1197 PartingShots(f), 1198 ) 1199 v.visit(mod) 1200 f.close() 1201 1202if __name__ == "__main__": 1203 import sys 1204 import getopt 1205 1206 INC_DIR = '' 1207 SRC_DIR = '' 1208 opts, args = getopt.getopt(sys.argv[1:], "h:c:") 1209 if len(opts) != 1: 1210 print "Must specify exactly one output file" 1211 sys.exit(1) 1212 for o, v in opts: 1213 if o == '-h': 1214 INC_DIR = v 1215 if o == '-c': 1216 SRC_DIR = v 1217 if len(args) != 1: 1218 print "Must specify single input file" 1219 sys.exit(1) 1220 main(args[0]) 1221