test_gc.py revision 0c5958b1636c47ed7c284f859c8e805fd06a0e63 
1import unittest
2from test.test_support import verbose, run_unittest
3import sys
4import time
5import gc
6import weakref
7
8try:
9    import threading
10except ImportError:
11    threading = None
12
13### Support code
14###############################################################################
15
16# Bug 1055820 has several tests of longstanding bugs involving weakrefs and
17# cyclic gc.
18
19# An instance of C1055820 has a self-loop, so becomes cyclic trash when
20# unreachable.
21class C1055820(object):
22    def __init__(self, i):
23        self.i = i
24        self.loop = self
25
26class GC_Detector(object):
27    # Create an instance I.  Then gc hasn't happened again so long as
28    # I.gc_happened is false.
29
30    def __init__(self):
31        self.gc_happened = False
32
33        def it_happened(ignored):
34            self.gc_happened = True
35
36        # Create a piece of cyclic trash that triggers it_happened when
37        # gc collects it.
38        self.wr = weakref.ref(C1055820(666), it_happened)
39
40
41### Tests
42###############################################################################
43
44class GCTests(unittest.TestCase):
45    def test_list(self):
46        l = []
47        l.append(l)
48        gc.collect()
49        del l
50        self.assertEqual(gc.collect(), 1)
51
52    def test_dict(self):
53        d = {}
54        d[1] = d
55        gc.collect()
56        del d
57        self.assertEqual(gc.collect(), 1)
58
59    def test_tuple(self):
60        # since tuples are immutable we close the loop with a list
61        l = []
62        t = (l,)
63        l.append(t)
64        gc.collect()
65        del t
66        del l
67        self.assertEqual(gc.collect(), 2)
68
69    def test_class(self):
70        class A:
71            pass
72        A.a = A
73        gc.collect()
74        del A
75        self.assertNotEqual(gc.collect(), 0)
76
77    def test_newstyleclass(self):
78        class A(object):
79            pass
80        gc.collect()
81        del A
82        self.assertNotEqual(gc.collect(), 0)
83
84    def test_instance(self):
85        class A:
86            pass
87        a = A()
88        a.a = a
89        gc.collect()
90        del a
91        self.assertNotEqual(gc.collect(), 0)
92
93    def test_newinstance(self):
94        class A(object):
95            pass
96        a = A()
97        a.a = a
98        gc.collect()
99        del a
100        self.assertNotEqual(gc.collect(), 0)
101        class B(list):
102            pass
103        class C(B, A):
104            pass
105        a = C()
106        a.a = a
107        gc.collect()
108        del a
109        self.assertNotEqual(gc.collect(), 0)
110        del B, C
111        self.assertNotEqual(gc.collect(), 0)
112        A.a = A()
113        del A
114        self.assertNotEqual(gc.collect(), 0)
115        self.assertEqual(gc.collect(), 0)
116
117    def test_method(self):
118        # Tricky: self.__init__ is a bound method, it references the instance.
119        class A:
120            def __init__(self):
121                self.init = self.__init__
122        a = A()
123        gc.collect()
124        del a
125        self.assertNotEqual(gc.collect(), 0)
126
127    def test_finalizer(self):
128        # A() is uncollectable if it is part of a cycle, make sure it shows up
129        # in gc.garbage.
130        class A:
131            def __del__(self): pass
132        class B:
133            pass
134        a = A()
135        a.a = a
136        id_a = id(a)
137        b = B()
138        b.b = b
139        gc.collect()
140        del a
141        del b
142        self.assertNotEqual(gc.collect(), 0)
143        for obj in gc.garbage:
144            if id(obj) == id_a:
145                del obj.a
146                break
147        else:
148            self.fail("didn't find obj in garbage (finalizer)")
149        gc.garbage.remove(obj)
150
151    def test_finalizer_newclass(self):
152        # A() is uncollectable if it is part of a cycle, make sure it shows up
153        # in gc.garbage.
154        class A(object):
155            def __del__(self): pass
156        class B(object):
157            pass
158        a = A()
159        a.a = a
160        id_a = id(a)
161        b = B()
162        b.b = b
163        gc.collect()
164        del a
165        del b
166        self.assertNotEqual(gc.collect(), 0)
167        for obj in gc.garbage:
168            if id(obj) == id_a:
169                del obj.a
170                break
171        else:
172            self.fail("didn't find obj in garbage (finalizer)")
173        gc.garbage.remove(obj)
174
175    def test_function(self):
176        # Tricky: f -> d -> f, code should call d.clear() after the exec to
177        # break the cycle.
178        d = {}
179        exec("def f(): pass\n") in d
180        gc.collect()
181        del d
182        self.assertEqual(gc.collect(), 2)
183
184    def test_frame(self):
185        def f():
186            frame = sys._getframe()
187        gc.collect()
188        f()
189        self.assertEqual(gc.collect(), 1)
190
191    def test_saveall(self):
192        # Verify that cyclic garbage like lists show up in gc.garbage if the
193        # SAVEALL option is enabled.
194
195        # First make sure we don't save away other stuff that just happens to
196        # be waiting for collection.
197        gc.collect()
198        # if this fails, someone else created immortal trash
199        self.assertEqual(gc.garbage, [])
200
201        L = []
202        L.append(L)
203        id_L = id(L)
204
205        debug = gc.get_debug()
206        gc.set_debug(debug | gc.DEBUG_SAVEALL)
207        del L
208        gc.collect()
209        gc.set_debug(debug)
210
211        self.assertEqual(len(gc.garbage), 1)
212        obj = gc.garbage.pop()
213        self.assertEqual(id(obj), id_L)
214
215    def test_del(self):
216        # __del__ methods can trigger collection, make this to happen
217        thresholds = gc.get_threshold()
218        gc.enable()
219        gc.set_threshold(1)
220
221        class A:
222            def __del__(self):
223                dir(self)
224        a = A()
225        del a
226
227        gc.disable()
228        gc.set_threshold(*thresholds)
229
230    def test_del_newclass(self):
231        # __del__ methods can trigger collection, make this to happen
232        thresholds = gc.get_threshold()
233        gc.enable()
234        gc.set_threshold(1)
235
236        class A(object):
237            def __del__(self):
238                dir(self)
239        a = A()
240        del a
241
242        gc.disable()
243        gc.set_threshold(*thresholds)
244
245    # The following two tests are fragile:
246    # They precisely count the number of allocations,
247    # which is highly implementation-dependent.
248    # For example:
249    # - disposed tuples are not freed, but reused
250    # - the call to assertEqual somehow avoids building its args tuple
251    def test_get_count(self):
252        # Avoid future allocation of method object
253        assertEqual = self._baseAssertEqual
254        gc.collect()
255        assertEqual(gc.get_count(), (0, 0, 0))
256        a = dict()
257        # since gc.collect(), we created two objects:
258        # the dict, and the tuple returned by get_count()
259        assertEqual(gc.get_count(), (2, 0, 0))
260
261    def test_collect_generations(self):
262        # Avoid future allocation of method object
263        assertEqual = self.assertEqual
264        gc.collect()
265        a = dict()
266        gc.collect(0)
267        assertEqual(gc.get_count(), (0, 1, 0))
268        gc.collect(1)
269        assertEqual(gc.get_count(), (0, 0, 1))
270        gc.collect(2)
271        assertEqual(gc.get_count(), (0, 0, 0))
272
273    def test_trashcan(self):
274        class Ouch:
275            n = 0
276            def __del__(self):
277                Ouch.n = Ouch.n + 1
278                if Ouch.n % 17 == 0:
279                    gc.collect()
280
281        # "trashcan" is a hack to prevent stack overflow when deallocating
282        # very deeply nested tuples etc.  It works in part by abusing the
283        # type pointer and refcount fields, and that can yield horrible
284        # problems when gc tries to traverse the structures.
285        # If this test fails (as it does in 2.0, 2.1 and 2.2), it will
286        # most likely die via segfault.
287
288        # Note:  In 2.3 the possibility for compiling without cyclic gc was
289        # removed, and that in turn allows the trashcan mechanism to work
290        # via much simpler means (e.g., it never abuses the type pointer or
291        # refcount fields anymore).  Since it's much less likely to cause a
292        # problem now, the various constants in this expensive (we force a lot
293        # of full collections) test are cut back from the 2.2 version.
294        gc.enable()
295        N = 150
296        for count in range(2):
297            t = []
298            for i in range(N):
299                t = [t, Ouch()]
300            u = []
301            for i in range(N):
302                u = [u, Ouch()]
303            v = {}
304            for i in range(N):
305                v = {1: v, 2: Ouch()}
306        gc.disable()
307
308    @unittest.skipUnless(threading, "test meaningless on builds without threads")
309    def test_trashcan_threads(self):
310        # Issue #13992: trashcan mechanism should be thread-safe
311        NESTING = 60
312        N_THREADS = 2
313
314        def sleeper_gen():
315            """A generator that releases the GIL when closed or dealloc'ed."""
316            try:
317                yield
318            finally:
319                time.sleep(0.000001)
320
321        class C(list):
322            # Appending to a list is atomic, which avoids the use of a lock.
323            inits = []
324            dels = []
325            def __init__(self, alist):
326                self[:] = alist
327                C.inits.append(None)
328            def __del__(self):
329                # This __del__ is called by subtype_dealloc().
330                C.dels.append(None)
331                # `g` will release the GIL when garbage-collected.  This
332                # helps assert subtype_dealloc's behaviour when threads
333                # switch in the middle of it.
334                g = sleeper_gen()
335                next(g)
336                # Now that __del__ is finished, subtype_dealloc will proceed
337                # to call list_dealloc, which also uses the trashcan mechanism.
338
339        def make_nested():
340            """Create a sufficiently nested container object so that the
341            trashcan mechanism is invoked when deallocating it."""
342            x = C([])
343            for i in range(NESTING):
344                x = [C([x])]
345            del x
346
347        def run_thread():
348            """Exercise make_nested() in a loop."""
349            while not exit:
350                make_nested()
351
352        old_checkinterval = sys.getcheckinterval()
353        sys.setcheckinterval(3)
354        try:
355            exit = False
356            threads = []
357            for i in range(N_THREADS):
358                t = threading.Thread(target=run_thread)
359                threads.append(t)
360            for t in threads:
361                t.start()
362            time.sleep(1.0)
363            exit = True
364            for t in threads:
365                t.join()
366        finally:
367            sys.setcheckinterval(old_checkinterval)
368        gc.collect()
369        self.assertEqual(len(C.inits), len(C.dels))
370
371    def test_boom(self):
372        class Boom:
373            def __getattr__(self, someattribute):
374                del self.attr
375                raise AttributeError
376
377        a = Boom()
378        b = Boom()
379        a.attr = b
380        b.attr = a
381
382        gc.collect()
383        garbagelen = len(gc.garbage)
384        del a, b
385        # a<->b are in a trash cycle now.  Collection will invoke
386        # Boom.__getattr__ (to see whether a and b have __del__ methods), and
387        # __getattr__ deletes the internal "attr" attributes as a side effect.
388        # That causes the trash cycle to get reclaimed via refcounts falling to
389        # 0, thus mutating the trash graph as a side effect of merely asking
390        # whether __del__ exists.  This used to (before 2.3b1) crash Python.
391        # Now __getattr__ isn't called.
392        self.assertEqual(gc.collect(), 4)
393        self.assertEqual(len(gc.garbage), garbagelen)
394
395    def test_boom2(self):
396        class Boom2:
397            def __init__(self):
398                self.x = 0
399
400            def __getattr__(self, someattribute):
401                self.x += 1
402                if self.x > 1:
403                    del self.attr
404                raise AttributeError
405
406        a = Boom2()
407        b = Boom2()
408        a.attr = b
409        b.attr = a
410
411        gc.collect()
412        garbagelen = len(gc.garbage)
413        del a, b
414        # Much like test_boom(), except that __getattr__ doesn't break the
415        # cycle until the second time gc checks for __del__.  As of 2.3b1,
416        # there isn't a second time, so this simply cleans up the trash cycle.
417        # We expect a, b, a.__dict__ and b.__dict__ (4 objects) to get
418        # reclaimed this way.
419        self.assertEqual(gc.collect(), 4)
420        self.assertEqual(len(gc.garbage), garbagelen)
421
422    def test_boom_new(self):
423        # boom__new and boom2_new are exactly like boom and boom2, except use
424        # new-style classes.
425
426        class Boom_New(object):
427            def __getattr__(self, someattribute):
428                del self.attr
429                raise AttributeError
430
431        a = Boom_New()
432        b = Boom_New()
433        a.attr = b
434        b.attr = a
435
436        gc.collect()
437        garbagelen = len(gc.garbage)
438        del a, b
439        self.assertEqual(gc.collect(), 4)
440        self.assertEqual(len(gc.garbage), garbagelen)
441
442    def test_boom2_new(self):
443        class Boom2_New(object):
444            def __init__(self):
445                self.x = 0
446
447            def __getattr__(self, someattribute):
448                self.x += 1
449                if self.x > 1:
450                    del self.attr
451                raise AttributeError
452
453        a = Boom2_New()
454        b = Boom2_New()
455        a.attr = b
456        b.attr = a
457
458        gc.collect()
459        garbagelen = len(gc.garbage)
460        del a, b
461        self.assertEqual(gc.collect(), 4)
462        self.assertEqual(len(gc.garbage), garbagelen)
463
464    def test_get_referents(self):
465        alist = [1, 3, 5]
466        got = gc.get_referents(alist)
467        got.sort()
468        self.assertEqual(got, alist)
469
470        atuple = tuple(alist)
471        got = gc.get_referents(atuple)
472        got.sort()
473        self.assertEqual(got, alist)
474
475        adict = {1: 3, 5: 7}
476        expected = [1, 3, 5, 7]
477        got = gc.get_referents(adict)
478        got.sort()
479        self.assertEqual(got, expected)
480
481        got = gc.get_referents([1, 2], {3: 4}, (0, 0, 0))
482        got.sort()
483        self.assertEqual(got, [0, 0] + range(5))
484
485        self.assertEqual(gc.get_referents(1, 'a', 4j), [])
486
487    def test_is_tracked(self):
488        # Atomic built-in types are not tracked, user-defined objects and
489        # mutable containers are.
490        # NOTE: types with special optimizations (e.g. tuple) have tests
491        # in their own test files instead.
492        self.assertFalse(gc.is_tracked(None))
493        self.assertFalse(gc.is_tracked(1))
494        self.assertFalse(gc.is_tracked(1.0))
495        self.assertFalse(gc.is_tracked(1.0 + 5.0j))
496        self.assertFalse(gc.is_tracked(True))
497        self.assertFalse(gc.is_tracked(False))
498        self.assertFalse(gc.is_tracked("a"))
499        self.assertFalse(gc.is_tracked(u"a"))
500        self.assertFalse(gc.is_tracked(bytearray("a")))
501        self.assertFalse(gc.is_tracked(type))
502        self.assertFalse(gc.is_tracked(int))
503        self.assertFalse(gc.is_tracked(object))
504        self.assertFalse(gc.is_tracked(object()))
505
506        class OldStyle:
507            pass
508        class NewStyle(object):
509            pass
510        self.assertTrue(gc.is_tracked(gc))
511        self.assertTrue(gc.is_tracked(OldStyle))
512        self.assertTrue(gc.is_tracked(OldStyle()))
513        self.assertTrue(gc.is_tracked(NewStyle))
514        self.assertTrue(gc.is_tracked(NewStyle()))
515        self.assertTrue(gc.is_tracked([]))
516        self.assertTrue(gc.is_tracked(set()))
517
518    def test_bug1055820b(self):
519        # Corresponds to temp2b.py in the bug report.
520
521        ouch = []
522        def callback(ignored):
523            ouch[:] = [wr() for wr in WRs]
524
525        Cs = [C1055820(i) for i in range(2)]
526        WRs = [weakref.ref(c, callback) for c in Cs]
527        c = None
528
529        gc.collect()
530        self.assertEqual(len(ouch), 0)
531        # Make the two instances trash, and collect again.  The bug was that
532        # the callback materialized a strong reference to an instance, but gc
533        # cleared the instance's dict anyway.
534        Cs = None
535        gc.collect()
536        self.assertEqual(len(ouch), 2)  # else the callbacks didn't run
537        for x in ouch:
538            # If the callback resurrected one of these guys, the instance
539            # would be damaged, with an empty __dict__.
540            self.assertEqual(x, None)
541
542class GCTogglingTests(unittest.TestCase):
543    def setUp(self):
544        gc.enable()
545
546    def tearDown(self):
547        gc.disable()
548
549    def test_bug1055820c(self):
550        # Corresponds to temp2c.py in the bug report.  This is pretty
551        # elaborate.
552
553        c0 = C1055820(0)
554        # Move c0 into generation 2.
555        gc.collect()
556
557        c1 = C1055820(1)
558        c1.keep_c0_alive = c0
559        del c0.loop # now only c1 keeps c0 alive
560
561        c2 = C1055820(2)
562        c2wr = weakref.ref(c2) # no callback!
563
564        ouch = []
565        def callback(ignored):
566            ouch[:] = [c2wr()]
567
568        # The callback gets associated with a wr on an object in generation 2.
569        c0wr = weakref.ref(c0, callback)
570
571        c0 = c1 = c2 = None
572
573        # What we've set up:  c0, c1, and c2 are all trash now.  c0 is in
574        # generation 2.  The only thing keeping it alive is that c1 points to
575        # it. c1 and c2 are in generation 0, and are in self-loops.  There's a
576        # global weakref to c2 (c2wr), but that weakref has no callback.
577        # There's also a global weakref to c0 (c0wr), and that does have a
578        # callback, and that callback references c2 via c2wr().
579        #
580        #               c0 has a wr with callback, which references c2wr
581        #               ^
582        #               |
583        #               |     Generation 2 above dots
584        #. . . . . . . .|. . . . . . . . . . . . . . . . . . . . . . . .
585        #               |     Generation 0 below dots
586        #               |
587        #               |
588        #            ^->c1   ^->c2 has a wr but no callback
589        #            |  |    |  |
590        #            <--v    <--v
591        #
592        # So this is the nightmare:  when generation 0 gets collected, we see
593        # that c2 has a callback-free weakref, and c1 doesn't even have a
594        # weakref.  Collecting generation 0 doesn't see c0 at all, and c0 is
595        # the only object that has a weakref with a callback.  gc clears c1
596        # and c2.  Clearing c1 has the side effect of dropping the refcount on
597        # c0 to 0, so c0 goes away (despite that it's in an older generation)
598        # and c0's wr callback triggers.  That in turn materializes a reference
599        # to c2 via c2wr(), but c2 gets cleared anyway by gc.
600
601        # We want to let gc happen "naturally", to preserve the distinction
602        # between generations.
603        junk = []
604        i = 0
605        detector = GC_Detector()
606        while not detector.gc_happened:
607            i += 1
608            if i > 10000:
609                self.fail("gc didn't happen after 10000 iterations")
610            self.assertEqual(len(ouch), 0)
611            junk.append([])  # this will eventually trigger gc
612
613        self.assertEqual(len(ouch), 1)  # else the callback wasn't invoked
614        for x in ouch:
615            # If the callback resurrected c2, the instance would be damaged,
616            # with an empty __dict__.
617            self.assertEqual(x, None)
618
619    def test_bug1055820d(self):
620        # Corresponds to temp2d.py in the bug report.  This is very much like
621        # test_bug1055820c, but uses a __del__ method instead of a weakref
622        # callback to sneak in a resurrection of cyclic trash.
623
624        ouch = []
625        class D(C1055820):
626            def __del__(self):
627                ouch[:] = [c2wr()]
628
629        d0 = D(0)
630        # Move all the above into generation 2.
631        gc.collect()
632
633        c1 = C1055820(1)
634        c1.keep_d0_alive = d0
635        del d0.loop # now only c1 keeps d0 alive
636
637        c2 = C1055820(2)
638        c2wr = weakref.ref(c2) # no callback!
639
640        d0 = c1 = c2 = None
641
642        # What we've set up:  d0, c1, and c2 are all trash now.  d0 is in
643        # generation 2.  The only thing keeping it alive is that c1 points to
644        # it.  c1 and c2 are in generation 0, and are in self-loops.  There's
645        # a global weakref to c2 (c2wr), but that weakref has no callback.
646        # There are no other weakrefs.
647        #
648        #               d0 has a __del__ method that references c2wr
649        #               ^
650        #               |
651        #               |     Generation 2 above dots
652        #. . . . . . . .|. . . . . . . . . . . . . . . . . . . . . . . .
653        #               |     Generation 0 below dots
654        #               |
655        #               |
656        #            ^->c1   ^->c2 has a wr but no callback
657        #            |  |    |  |
658        #            <--v    <--v
659        #
660        # So this is the nightmare:  when generation 0 gets collected, we see
661        # that c2 has a callback-free weakref, and c1 doesn't even have a
662        # weakref.  Collecting generation 0 doesn't see d0 at all.  gc clears
663        # c1 and c2.  Clearing c1 has the side effect of dropping the refcount
664        # on d0 to 0, so d0 goes away (despite that it's in an older
665        # generation) and d0's __del__ triggers.  That in turn materializes
666        # a reference to c2 via c2wr(), but c2 gets cleared anyway by gc.
667
668        # We want to let gc happen "naturally", to preserve the distinction
669        # between generations.
670        detector = GC_Detector()
671        junk = []
672        i = 0
673        while not detector.gc_happened:
674            i += 1
675            if i > 10000:
676                self.fail("gc didn't happen after 10000 iterations")
677            self.assertEqual(len(ouch), 0)
678            junk.append([])  # this will eventually trigger gc
679
680        self.assertEqual(len(ouch), 1)  # else __del__ wasn't invoked
681        for x in ouch:
682            # If __del__ resurrected c2, the instance would be damaged, with an
683            # empty __dict__.
684            self.assertEqual(x, None)
685
686def test_main():
687    enabled = gc.isenabled()
688    gc.disable()
689    assert not gc.isenabled()
690    debug = gc.get_debug()
691    gc.set_debug(debug & ~gc.DEBUG_LEAK) # this test is supposed to leak
692
693    try:
694        gc.collect() # Delete 2nd generation garbage
695        run_unittest(GCTests, GCTogglingTests)
696    finally:
697        gc.set_debug(debug)
698        # test gc.enable() even if GC is disabled by default
699        if verbose:
700            print "restoring automatic collection"
701        # make sure to always test gc.enable()
702        gc.enable()
703        assert gc.isenabled()
704        if not enabled:
705            gc.disable()
706
707if __name__ == "__main__":
708    test_main()
709