1 """This module includes tests of the code object representation. 2 3 >>> def f(x): 4 ... def g(y): 5 ... return x + y 6 ... return g 7 ... 8 9 >>> dump(f.__code__) 10 name: f 11 argcount: 1 12 posonlyargcount: 0 13 kwonlyargcount: 0 14 names: () 15 varnames: ('x', 'g') 16 cellvars: ('x',) 17 freevars: () 18 nlocals: 2 19 flags: 3 20 consts: ('None', '<code object g>') 21 22 >>> dump(f(4).__code__) 23 name: g 24 argcount: 1 25 posonlyargcount: 0 26 kwonlyargcount: 0 27 names: () 28 varnames: ('y',) 29 cellvars: () 30 freevars: ('x',) 31 nlocals: 1 32 flags: 19 33 consts: ('None',) 34 35 >>> def h(x, y): 36 ... a = x + y 37 ... b = x - y 38 ... c = a * b 39 ... return c 40 ... 41 42 >>> dump(h.__code__) 43 name: h 44 argcount: 2 45 posonlyargcount: 0 46 kwonlyargcount: 0 47 names: () 48 varnames: ('x', 'y', 'a', 'b', 'c') 49 cellvars: () 50 freevars: () 51 nlocals: 5 52 flags: 3 53 consts: ('None',) 54 55 >>> def attrs(obj): 56 ... print(obj.attr1) 57 ... print(obj.attr2) 58 ... print(obj.attr3) 59 60 >>> dump(attrs.__code__) 61 name: attrs 62 argcount: 1 63 posonlyargcount: 0 64 kwonlyargcount: 0 65 names: ('print', 'attr1', 'attr2', 'attr3') 66 varnames: ('obj',) 67 cellvars: () 68 freevars: () 69 nlocals: 1 70 flags: 3 71 consts: ('None',) 72 73 >>> def optimize_away(): 74 ... 'doc string' 75 ... 'not a docstring' 76 ... 53 77 ... 0x53 78 79 >>> dump(optimize_away.__code__) 80 name: optimize_away 81 argcount: 0 82 posonlyargcount: 0 83 kwonlyargcount: 0 84 names: () 85 varnames: () 86 cellvars: () 87 freevars: () 88 nlocals: 0 89 flags: 3 90 consts: ("'doc string'", 'None') 91 92 >>> def keywordonly_args(a,b,*,k1): 93 ... return a,b,k1 94 ... 95 96 >>> dump(keywordonly_args.__code__) 97 name: keywordonly_args 98 argcount: 2 99 posonlyargcount: 0 100 kwonlyargcount: 1 101 names: () 102 varnames: ('a', 'b', 'k1') 103 cellvars: () 104 freevars: () 105 nlocals: 3 106 flags: 3 107 consts: ('None',) 108 109 >>> def posonly_args(a,b,/,c): 110 ... return a,b,c 111 ... 112 113 >>> dump(posonly_args.__code__) 114 name: posonly_args 115 argcount: 3 116 posonlyargcount: 2 117 kwonlyargcount: 0 118 names: () 119 varnames: ('a', 'b', 'c') 120 cellvars: () 121 freevars: () 122 nlocals: 3 123 flags: 3 124 consts: ('None',) 125 126 """ 127 128 import inspect 129 import sys 130 import threading 131 import doctest 132 import unittest 133 import textwrap 134 import weakref 135 import dis 136 137 try: 138 import ctypes 139 except ImportError: 140 ctypes = None 141 from test.support import (cpython_only, 142 check_impl_detail, requires_debug_ranges, 143 gc_collect) 144 from test.support.script_helper import assert_python_ok 145 from test.support import threading_helper 146 from opcode import opmap 147 COPY_FREE_VARS = opmap['COPY_FREE_VARS'] 148 149 150 def consts(t): 151 """Yield a doctest-safe sequence of object reprs.""" 152 for elt in t: 153 r = repr(elt) 154 if r.startswith("<code object"): 155 yield "<code object %s>" % elt.co_name 156 else: 157 yield r 158 159 def dump(co): 160 """Print out a text representation of a code object.""" 161 for attr in ["name", "argcount", "posonlyargcount", 162 "kwonlyargcount", "names", "varnames", 163 "cellvars", "freevars", "nlocals", "flags"]: 164 print("%s: %s" % (attr, getattr(co, "co_" + attr))) 165 print("consts:", tuple(consts(co.co_consts))) 166 167 # Needed for test_closure_injection below 168 # Defined at global scope to avoid implicitly closing over __class__ 169 def external_getitem(self, i): 170 return f"Foreign getitem: {super().__getitem__(i)}" 171 172 class CodeTest(unittest.TestCase): 173 174 @cpython_only 175 def test_newempty(self): 176 import _testcapi 177 co = _testcapi.code_newempty("filename", "funcname", 15) 178 self.assertEqual(co.co_filename, "filename") 179 self.assertEqual(co.co_name, "funcname") 180 self.assertEqual(co.co_firstlineno, 15) 181 #Empty code object should raise, but not crash the VM 182 with self.assertRaises(Exception): 183 exec(co) 184 185 @cpython_only 186 def test_closure_injection(self): 187 # From https://bugs.python.org/issue32176 188 from types import FunctionType 189 190 def create_closure(__class__): 191 return (lambda: __class__).__closure__ 192 193 def new_code(c): 194 '''A new code object with a __class__ cell added to freevars''' 195 return c.replace(co_freevars=c.co_freevars + ('__class__',), co_code=bytes([COPY_FREE_VARS, 1])+c.co_code) 196 197 def add_foreign_method(cls, name, f): 198 code = new_code(f.__code__) 199 assert not f.__closure__ 200 closure = create_closure(cls) 201 defaults = f.__defaults__ 202 setattr(cls, name, FunctionType(code, globals(), name, defaults, closure)) 203 204 class List(list): 205 pass 206 207 add_foreign_method(List, "__getitem__", external_getitem) 208 209 # Ensure the closure injection actually worked 210 function = List.__getitem__ 211 class_ref = function.__closure__[0].cell_contents 212 self.assertIs(class_ref, List) 213 214 # Ensure the zero-arg super() call in the injected method works 215 obj = List([1, 2, 3]) 216 self.assertEqual(obj[0], "Foreign getitem: 1") 217 218 def test_constructor(self): 219 def func(): pass 220 co = func.__code__ 221 CodeType = type(co) 222 223 # test code constructor 224 CodeType(co.co_argcount, 225 co.co_posonlyargcount, 226 co.co_kwonlyargcount, 227 co.co_nlocals, 228 co.co_stacksize, 229 co.co_flags, 230 co.co_code, 231 co.co_consts, 232 co.co_names, 233 co.co_varnames, 234 co.co_filename, 235 co.co_name, 236 co.co_qualname, 237 co.co_firstlineno, 238 co.co_linetable, 239 co.co_exceptiontable, 240 co.co_freevars, 241 co.co_cellvars) 242 243 def test_qualname(self): 244 self.assertEqual( 245 CodeTest.test_qualname.__code__.co_qualname, 246 CodeTest.test_qualname.__qualname__ 247 ) 248 249 def test_replace(self): 250 def func(): 251 x = 1 252 return x 253 code = func.__code__ 254 255 # different co_name, co_varnames, co_consts 256 def func2(): 257 y = 2 258 z = 3 259 return y 260 code2 = func2.__code__ 261 262 for attr, value in ( 263 ("co_argcount", 0), 264 ("co_posonlyargcount", 0), 265 ("co_kwonlyargcount", 0), 266 ("co_nlocals", 1), 267 ("co_stacksize", 0), 268 ("co_flags", code.co_flags | inspect.CO_COROUTINE), 269 ("co_firstlineno", 100), 270 ("co_code", code2.co_code), 271 ("co_consts", code2.co_consts), 272 ("co_names", ("myname",)), 273 ("co_varnames", ('spam',)), 274 ("co_freevars", ("freevar",)), 275 ("co_cellvars", ("cellvar",)), 276 ("co_filename", "newfilename"), 277 ("co_name", "newname"), 278 ("co_linetable", code2.co_linetable), 279 ): 280 with self.subTest(attr=attr, value=value): 281 new_code = code.replace(**{attr: value}) 282 self.assertEqual(getattr(new_code, attr), value) 283 284 new_code = code.replace(co_varnames=code2.co_varnames, 285 co_nlocals=code2.co_nlocals) 286 self.assertEqual(new_code.co_varnames, code2.co_varnames) 287 self.assertEqual(new_code.co_nlocals, code2.co_nlocals) 288 289 def test_nlocals_mismatch(self): 290 def func(): 291 x = 1 292 return x 293 co = func.__code__ 294 assert co.co_nlocals > 0; 295 296 # First we try the constructor. 297 CodeType = type(co) 298 for diff in (-1, 1): 299 with self.assertRaises(ValueError): 300 CodeType(co.co_argcount, 301 co.co_posonlyargcount, 302 co.co_kwonlyargcount, 303 # This is the only change. 304 co.co_nlocals + diff, 305 co.co_stacksize, 306 co.co_flags, 307 co.co_code, 308 co.co_consts, 309 co.co_names, 310 co.co_varnames, 311 co.co_filename, 312 co.co_name, 313 co.co_qualname, 314 co.co_firstlineno, 315 co.co_linetable, 316 co.co_exceptiontable, 317 co.co_freevars, 318 co.co_cellvars, 319 ) 320 # Then we try the replace method. 321 with self.assertRaises(ValueError): 322 co.replace(co_nlocals=co.co_nlocals - 1) 323 with self.assertRaises(ValueError): 324 co.replace(co_nlocals=co.co_nlocals + 1) 325 326 def test_shrinking_localsplus(self): 327 # Check that PyCode_NewWithPosOnlyArgs resizes both 328 # localsplusnames and localspluskinds, if an argument is a cell. 329 def func(arg): 330 return lambda: arg 331 code = func.__code__ 332 newcode = code.replace(co_name="func") # Should not raise SystemError 333 self.assertEqual(code, newcode) 334 335 def test_empty_linetable(self): 336 def func(): 337 pass 338 new_code = code = func.__code__.replace(co_linetable=b'') 339 self.assertEqual(list(new_code.co_lines()), []) 340 341 @requires_debug_ranges() 342 def test_co_positions_artificial_instructions(self): 343 import dis 344 345 namespace = {} 346 exec(textwrap.dedent("""\ 347 try: 348 1/0 349 except Exception as e: 350 exc = e 351 """), namespace) 352 353 exc = namespace['exc'] 354 traceback = exc.__traceback__ 355 code = traceback.tb_frame.f_code 356 357 artificial_instructions = [] 358 for instr, positions in zip( 359 dis.get_instructions(code, show_caches=True), 360 code.co_positions(), 361 strict=True 362 ): 363 # If any of the positions is None, then all have to 364 # be None as well for the case above. There are still 365 # some places in the compiler, where the artificial instructions 366 # get assigned the first_lineno but they don't have other positions. 367 # There is no easy way of inferring them at that stage, so for now 368 # we don't support it. 369 self.assertIn(positions.count(None), [0, 3, 4]) 370 371 if not any(positions): 372 artificial_instructions.append(instr) 373 374 self.assertEqual( 375 [ 376 (instruction.opname, instruction.argval) 377 for instruction in artificial_instructions 378 ], 379 [ 380 ("PUSH_EXC_INFO", None), 381 ("LOAD_CONST", None), # artificial 'None' 382 ("STORE_NAME", "e"), # XX: we know the location for this 383 ("DELETE_NAME", "e"), 384 ("RERAISE", 1), 385 ("COPY", 3), 386 ("POP_EXCEPT", None), 387 ("RERAISE", 1) 388 ] 389 ) 390 391 def test_endline_and_columntable_none_when_no_debug_ranges(self): 392 # Make sure that if `-X no_debug_ranges` is used, there is 393 # minimal debug info 394 code = textwrap.dedent(""" 395 def f(): 396 pass 397 398 positions = f.__code__.co_positions() 399 for line, end_line, column, end_column in positions: 400 assert line == end_line 401 assert column is None 402 assert end_column is None 403 """) 404 assert_python_ok('-X', 'no_debug_ranges', '-c', code) 405 406 def test_endline_and_columntable_none_when_no_debug_ranges_env(self): 407 # Same as above but using the environment variable opt out. 408 code = textwrap.dedent(""" 409 def f(): 410 pass 411 412 positions = f.__code__.co_positions() 413 for line, end_line, column, end_column in positions: 414 assert line == end_line 415 assert column is None 416 assert end_column is None 417 """) 418 assert_python_ok('-c', code, PYTHONNODEBUGRANGES='1') 419 420 # co_positions behavior when info is missing. 421 422 @requires_debug_ranges() 423 def test_co_positions_empty_linetable(self): 424 def func(): 425 x = 1 426 new_code = func.__code__.replace(co_linetable=b'') 427 positions = new_code.co_positions() 428 for line, end_line, column, end_column in positions: 429 self.assertIsNone(line) 430 self.assertEqual(end_line, new_code.co_firstlineno + 1) 431 432 def test_code_equality(self): 433 def f(): 434 try: 435 a() 436 except: 437 b() 438 else: 439 c() 440 finally: 441 d() 442 code_a = f.__code__ 443 code_b = code_a.replace(co_linetable=b"") 444 code_c = code_a.replace(co_exceptiontable=b"") 445 code_d = code_b.replace(co_exceptiontable=b"") 446 self.assertNotEqual(code_a, code_b) 447 self.assertNotEqual(code_a, code_c) 448 self.assertNotEqual(code_a, code_d) 449 self.assertNotEqual(code_b, code_c) 450 self.assertNotEqual(code_b, code_d) 451 self.assertNotEqual(code_c, code_d) 452 453 454 def isinterned(s): 455 return s is sys.intern(('_' + s + '_')[1:-1]) 456 457 class CodeConstsTest(unittest.TestCase): 458 459 def find_const(self, consts, value): 460 for v in consts: 461 if v == value: 462 return v 463 self.assertIn(value, consts) # raises an exception 464 self.fail('Should never be reached') 465 466 def assertIsInterned(self, s): 467 if not isinterned(s): 468 self.fail('String %r is not interned' % (s,)) 469 470 def assertIsNotInterned(self, s): 471 if isinterned(s): 472 self.fail('String %r is interned' % (s,)) 473 474 @cpython_only 475 def test_interned_string(self): 476 co = compile('res = "str_value"', '?', 'exec') 477 v = self.find_const(co.co_consts, 'str_value') 478 self.assertIsInterned(v) 479 480 @cpython_only 481 def test_interned_string_in_tuple(self): 482 co = compile('res = ("str_value",)', '?', 'exec') 483 v = self.find_const(co.co_consts, ('str_value',)) 484 self.assertIsInterned(v[0]) 485 486 @cpython_only 487 def test_interned_string_in_frozenset(self): 488 co = compile('res = a in {"str_value"}', '?', 'exec') 489 v = self.find_const(co.co_consts, frozenset(('str_value',))) 490 self.assertIsInterned(tuple(v)[0]) 491 492 @cpython_only 493 def test_interned_string_default(self): 494 def f(a='str_value'): 495 return a 496 self.assertIsInterned(f()) 497 498 @cpython_only 499 def test_interned_string_with_null(self): 500 co = compile(r'res = "str\0value!"', '?', 'exec') 501 v = self.find_const(co.co_consts, 'str\0value!') 502 self.assertIsNotInterned(v) 503 504 505 class CodeWeakRefTest(unittest.TestCase): 506 507 def test_basic(self): 508 # Create a code object in a clean environment so that we know we have 509 # the only reference to it left. 510 namespace = {} 511 exec("def f(): pass", globals(), namespace) 512 f = namespace["f"] 513 del namespace 514 515 self.called = False 516 def callback(code): 517 self.called = True 518 519 # f is now the last reference to the function, and through it, the code 520 # object. While we hold it, check that we can create a weakref and 521 # deref it. Then delete it, and check that the callback gets called and 522 # the reference dies. 523 coderef = weakref.ref(f.__code__, callback) 524 self.assertTrue(bool(coderef())) 525 del f 526 gc_collect() # For PyPy or other GCs. 527 self.assertFalse(bool(coderef())) 528 self.assertTrue(self.called) 529 530 # Python implementation of location table parsing algorithm 531 def read(it): 532 return next(it) 533 534 def read_varint(it): 535 b = read(it) 536 val = b & 63; 537 shift = 0; 538 while b & 64: 539 b = read(it) 540 shift += 6 541 val |= (b&63) << shift 542 return val 543 544 def read_signed_varint(it): 545 uval = read_varint(it) 546 if uval & 1: 547 return -(uval >> 1) 548 else: 549 return uval >> 1 550 551 def parse_location_table(code): 552 line = code.co_firstlineno 553 it = iter(code.co_linetable) 554 while True: 555 try: 556 first_byte = read(it) 557 except StopIteration: 558 return 559 code = (first_byte >> 3) & 15 560 length = (first_byte & 7) + 1 561 if code == 15: 562 yield (code, length, None, None, None, None) 563 elif code == 14: 564 line_delta = read_signed_varint(it) 565 line += line_delta 566 end_line = line + read_varint(it) 567 col = read_varint(it) 568 if col == 0: 569 col = None 570 else: 571 col -= 1 572 end_col = read_varint(it) 573 if end_col == 0: 574 end_col = None 575 else: 576 end_col -= 1 577 yield (code, length, line, end_line, col, end_col) 578 elif code == 13: # No column 579 line_delta = read_signed_varint(it) 580 line += line_delta 581 yield (code, length, line, line, None, None) 582 elif code in (10, 11, 12): # new line 583 line_delta = code - 10 584 line += line_delta 585 column = read(it) 586 end_column = read(it) 587 yield (code, length, line, line, column, end_column) 588 else: 589 assert (0 <= code < 10) 590 second_byte = read(it) 591 column = code << 3 | (second_byte >> 4) 592 yield (code, length, line, line, column, column + (second_byte & 15)) 593 594 def positions_from_location_table(code): 595 for _, length, line, end_line, col, end_col in parse_location_table(code): 596 for _ in range(length): 597 yield (line, end_line, col, end_col) 598 599 def dedup(lst, prev=object()): 600 for item in lst: 601 if item != prev: 602 yield item 603 prev = item 604 605 def lines_from_postions(positions): 606 return dedup(l for (l, _, _, _) in positions) 607 608 def misshappen(): 609 """ 610 611 612 613 614 615 """ 616 x = ( 617 618 619 4 620 621 + 622 623 y 624 625 ) 626 y = ( 627 a 628 + 629 b 630 + 631 632 d 633 ) 634 return q if ( 635 636 x 637 638 ) else p 639 640 def bug93662(): 641 example_report_generation_message= ( 642 """ 643 """ 644 ).strip() 645 raise ValueError() 646 647 648 class CodeLocationTest(unittest.TestCase): 649 650 def check_positions(self, func): 651 pos1 = list(func.__code__.co_positions()) 652 pos2 = list(positions_from_location_table(func.__code__)) 653 for l1, l2 in zip(pos1, pos2): 654 self.assertEqual(l1, l2) 655 self.assertEqual(len(pos1), len(pos2)) 656 657 def test_positions(self): 658 self.check_positions(parse_location_table) 659 self.check_positions(misshappen) 660 self.check_positions(bug93662) 661 662 def check_lines(self, func): 663 co = func.__code__ 664 lines1 = list(dedup(l for (_, _, l) in co.co_lines())) 665 lines2 = list(lines_from_postions(positions_from_location_table(co))) 666 for l1, l2 in zip(lines1, lines2): 667 self.assertEqual(l1, l2) 668 self.assertEqual(len(lines1), len(lines2)) 669 670 def test_lines(self): 671 self.check_lines(parse_location_table) 672 self.check_lines(misshappen) 673 self.check_lines(bug93662) 674 675 @cpython_only 676 def test_code_new_empty(self): 677 # If this test fails, it means that the construction of PyCode_NewEmpty 678 # needs to be modified! Please update this test *and* PyCode_NewEmpty, 679 # so that they both stay in sync. 680 def f(): 681 pass 682 PY_CODE_LOCATION_INFO_NO_COLUMNS = 13 683 f.__code__ = f.__code__.replace( 684 co_firstlineno=42, 685 co_code=bytes( 686 [ 687 dis.opmap["RESUME"], 0, 688 dis.opmap["LOAD_ASSERTION_ERROR"], 0, 689 dis.opmap["RAISE_VARARGS"], 1, 690 ] 691 ), 692 co_linetable=bytes( 693 [ 694 (1 << 7) 695 | (PY_CODE_LOCATION_INFO_NO_COLUMNS << 3) 696 | (3 - 1), 697 0, 698 ] 699 ), 700 ) 701 self.assertRaises(AssertionError, f) 702 self.assertEqual( 703 list(f.__code__.co_positions()), 704 3 * [(42, 42, None, None)], 705 ) 706 707 708 if check_impl_detail(cpython=True) and ctypes is not None: 709 py = ctypes.pythonapi 710 freefunc = ctypes.CFUNCTYPE(None,ctypes.c_voidp) 711 712 RequestCodeExtraIndex = py._PyEval_RequestCodeExtraIndex 713 RequestCodeExtraIndex.argtypes = (freefunc,) 714 RequestCodeExtraIndex.restype = ctypes.c_ssize_t 715 716 SetExtra = py._PyCode_SetExtra 717 SetExtra.argtypes = (ctypes.py_object, ctypes.c_ssize_t, ctypes.c_voidp) 718 SetExtra.restype = ctypes.c_int 719 720 GetExtra = py._PyCode_GetExtra 721 GetExtra.argtypes = (ctypes.py_object, ctypes.c_ssize_t, 722 ctypes.POINTER(ctypes.c_voidp)) 723 GetExtra.restype = ctypes.c_int 724 725 LAST_FREED = None 726 def myfree(ptr): 727 global LAST_FREED 728 LAST_FREED = ptr 729 730 FREE_FUNC = freefunc(myfree) 731 FREE_INDEX = RequestCodeExtraIndex(FREE_FUNC) 732 733 class CoExtra(unittest.TestCase): 734 def get_func(self): 735 # Defining a function causes the containing function to have a 736 # reference to the code object. We need the code objects to go 737 # away, so we eval a lambda. 738 return eval('lambda:42') 739 740 def test_get_non_code(self): 741 f = self.get_func() 742 743 self.assertRaises(SystemError, SetExtra, 42, FREE_INDEX, 744 ctypes.c_voidp(100)) 745 self.assertRaises(SystemError, GetExtra, 42, FREE_INDEX, 746 ctypes.c_voidp(100)) 747 748 def test_bad_index(self): 749 f = self.get_func() 750 self.assertRaises(SystemError, SetExtra, f.__code__, 751 FREE_INDEX+100, ctypes.c_voidp(100)) 752 self.assertEqual(GetExtra(f.__code__, FREE_INDEX+100, 753 ctypes.c_voidp(100)), 0) 754 755 def test_free_called(self): 756 # Verify that the provided free function gets invoked 757 # when the code object is cleaned up. 758 f = self.get_func() 759 760 SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(100)) 761 del f 762 self.assertEqual(LAST_FREED, 100) 763 764 def test_get_set(self): 765 # Test basic get/set round tripping. 766 f = self.get_func() 767 768 extra = ctypes.c_voidp() 769 770 SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(200)) 771 # reset should free... 772 SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(300)) 773 self.assertEqual(LAST_FREED, 200) 774 775 extra = ctypes.c_voidp() 776 GetExtra(f.__code__, FREE_INDEX, extra) 777 self.assertEqual(extra.value, 300) 778 del f 779 780 @threading_helper.requires_working_threading() 781 def test_free_different_thread(self): 782 # Freeing a code object on a different thread then 783 # where the co_extra was set should be safe. 784 f = self.get_func() 785 class ThreadTest(threading.Thread): 786 def __init__(self, f, test): 787 super().__init__() 788 self.f = f 789 self.test = test 790 def run(self): 791 del self.f 792 self.test.assertEqual(LAST_FREED, 500) 793 794 SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(500)) 795 tt = ThreadTest(f, self) 796 del f 797 tt.start() 798 tt.join() 799 self.assertEqual(LAST_FREED, 500) 800 801 802 def load_tests(loader, tests, pattern): 803 tests.addTest(doctest.DocTestSuite()) 804 return tests 805 806 807 if __name__ == "__main__": 808 unittest.main() 809