Skip to content

Commit c2a9d57

Browse files
jeff5jeff5
committed
Refactor PyType etc. to use a thread-safe non-blocking look-up.
This change re-works the apparatus behind PyType.fromClass and addBuilder. It moves the static API of PyType to an inner "Registry", where we use a ClassValue as a non-blocking cache in PyType look-up. The construction of PyType and PyJavaType entries, bootstrapping of the type system, and the initialisation of some core types have all been re-thought, aiming for clearer logic. The "state journey" of a class and its PyType from loading to exposure has been carefully considered. The need to manage an explicit hard reference to exposed types has been eliminated. The ClassValue classToType holds a plain (hard) reference, and a re-worked test in test_jy_internals demonstrates that PyTypes are correctly GC'd with the Java classes they represent to Python.
1 parent 812b62e commit c2a9d57

9 files changed

Lines changed: 831 additions & 378 deletions

File tree

‎Lib/test/test_java_visibility.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import array
2+
import os
23
import unittest
34
import subprocess
45
import sys
@@ -260,7 +261,9 @@ def test_overriding(self):
260261
class ClassloaderTest(unittest.TestCase):
261262

262263
def test_loading_classes_without_import(self):
263-
cl = test_support.make_jar_classloader("../callbacker_test.jar")
264+
# Look for the Callbacker class ONLY in the special JAR
265+
jar = os.path.join(sys.prefix, "callbacker_test.jar")
266+
cl = test_support.make_jar_classloader(jar, None)
264267
X = cl.loadClass("org.python.tests.Callbacker")
265268
called = []
266269
class Blah(X.Callback):

‎Lib/test/test_jy_internals.py‎

Lines changed: 89 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -2,33 +2,24 @@
22
test some jython internals
33
"""
44
import unittest
5-
import time
65
from test import test_support
76

7+
import datetime
88
import java
99
import jarray
10+
import os
11+
import sys
12+
import time
13+
import weakref
1014

1115
from org.python.core import Py
1216
from org.python.util import PythonInterpreter
1317
from javatests.TestSupport import invokePyTypeMethod
1418
from java.sql import Date, Time, Timestamp
15-
import datetime
16-
17-
18-
class MemoryLeakTests(unittest.TestCase):
19-
20-
def test_class_to_test_weakness(self):
21-
# regrtest for bug 1522, adapted from test code submitted by Matt Brinkley
2219

23-
# work around the fact that we can't look at PyType directly
24-
# by using this helper function that reflects on PyType (and
25-
# demonstrates here that it's the same as the builtin function
26-
# `type`!)
27-
class_to_type_map = invokePyTypeMethod(type, 'getClassToType')
2820

29-
def create_proxies():
30-
pi = PythonInterpreter()
31-
pi.exec("""
21+
# Script for MemoryLeakTests.test_class_to_test_weakness
22+
DOG = """
3223
from java.lang import Comparable
3324
3425
class Dog(Comparable):
@@ -37,22 +28,83 @@ def compareTo(self, o):
3728
def bark(self):
3829
return 'woof woof'
3930
40-
Dog().bark()
41-
""")
42-
# get to steady state first, then verify we don't create new proxies
43-
for i in xrange(2):
44-
create_proxies()
45-
# Ensure the reaper thread can run and clear out weak refs, so
46-
# use this supporting function
31+
dog = Dog()
32+
dog.bark()
33+
breed = dog.getClass()
34+
"""
35+
36+
def run_script(script, names):
37+
"""Run the script and return a weak list of the values named"""
38+
pi = PythonInterpreter()
39+
pi.exec(script)
40+
if isinstance(names, str):
41+
names = (names, )
42+
result = []
43+
for n in names:
44+
obj = pi.getLocals()[n]
45+
result.append(weakref.ref(obj))
46+
return result
47+
48+
def survivors(weak_list):
49+
"""Set of all objects on the weak list that are still live."""
50+
s = {ref() for ref in weak_list}
51+
s.discard(None)
52+
return s
53+
54+
55+
class MemoryLeakTests(unittest.TestCase):
56+
57+
def test_class_to_test_weakness(self):
58+
# regrtest for bug 1522, adapted from test code submitted by Matt Brinkley
59+
60+
# We wish to demonstrate that the proxy created by a Python class and
61+
# its PyType are GC'd when no longer in use, and therefore that the
62+
# Jython type system does not keep a PyType alive gratuitously. We do
63+
# this by holding weak references, then checking they're dead.
64+
65+
# Get to steady state by creating >1 Dog and then GC-ing homeless ones.
66+
battersea = []
67+
for i in range(2):
68+
battersea.extend(run_script(DOG, ('Dog', 'dog', 'breed')))
69+
test_support.gc_collect()
70+
71+
# This is the steady-state, GC number of Dog objects alive after GC:
72+
start_size = len(survivors(battersea)) # probably = 1
73+
74+
# Add more Dogs and GC the homeless ones again.
75+
for i in range(5):
76+
battersea.extend(run_script(DOG, ('Dog', 'dog', 'breed')))
4777
test_support.gc_collect()
48-
# Given that taking the len (or size()) of Guava weak maps is
49-
# eventually consistent, we should instead take a len of its
50-
# keys.
51-
start_size = len(list(class_to_type_map))
52-
for i in xrange(5):
53-
create_proxies()
78+
#print "\nDogs home =", battersea
79+
#print "\nDogs alive =", survivors(battersea)
80+
81+
# Post-GC number of Dogs should be as before
82+
self.assertEqual(start_size, len(survivors(battersea)))
83+
84+
def test_loading_classes_weakness(self):
85+
# Show that classes loaded via a class loader are collectible once the
86+
# class loader is not strongly reachable.
87+
88+
# Reference all species of object on a weak list:
89+
zoo = []
90+
91+
def activity():
92+
# Look for the Callbacker class ONLY in the special JAR
93+
jar = os.path.join(sys.prefix, "callbacker_test.jar")
94+
cldr = test_support.make_jar_classloader(jar, None)
95+
CB = cldr.loadClass("org.python.tests.Callbacker")
96+
cb = CB()
97+
# Save locals as weak references (that we hope will go dead)
98+
for obj in (cb, type(cb), cldr):
99+
zoo.append(weakref.ref(obj))
100+
101+
# Load and use a class: objects created in zoo.
102+
activity()
103+
104+
#print "\nzoo =", zoo
54105
test_support.gc_collect()
55-
self.assertEqual(start_size, len(list(class_to_type_map)))
106+
#print "\nsurvivors =", survivors(zoo)
107+
self.assertEqual(0, len(survivors(zoo)))
56108

57109

58110
class WeakIdentityMapTests(unittest.TestCase):
@@ -93,6 +145,7 @@ def test_functionality(self):
93145
assert widmap.get(i) == 'i' # triggers stale weak refs cleanup
94146
assert widmap._internal_map_size() == 1
95147

148+
96149
class LongAsScaledDoubleValueTests(unittest.TestCase):
97150

98151
def setUp(self):
@@ -143,6 +196,7 @@ def test_no_worse_than_doubleValue(self):
143196
assert float((v+d)*256+y) == sdv(((v+d)*256+y)*256, e)
144197
assert e[0] == 1
145198

199+
146200
class ExtraMathTests(unittest.TestCase):
147201
def test_epsilon(self):
148202
from org.python.core.util import ExtraMath
@@ -162,6 +216,7 @@ def test_closeFloor(self):
162216
ExtraMath.closeFloor(3.0 - 3.0 * ExtraMath.CLOSE), 2.0)
163217
self.assertEquals(ExtraMath.closeFloor(math.log10(10**3)), 3.0)
164218

219+
165220
class DatetimeTypeMappingTest(unittest.TestCase):
166221
def test_date(self):
167222
self.assertEquals(datetime.date(2008, 5, 29),
@@ -183,6 +238,7 @@ def test_datetime(self):
183238
self.assertEquals(datetime.datetime(2008, 5, 29, 16, 50, 1, 1),
184239
Py.newDatetime(Timestamp(108, 4, 29, 16, 50, 1, 1000)))
185240

241+
186242
class IdTest(unittest.TestCase):
187243
def test_unique_ids(self):
188244
d = {}
@@ -197,6 +253,7 @@ def test_unique_ids(self):
197253

198254
self.assertEquals(cnt, 0)
199255

256+
200257
class FrameTest(unittest.TestCase):
201258
def test_stack_frame_locals(self):
202259
def h():
@@ -268,6 +325,7 @@ def baz(self):
268325
foo()
269326
Bar().baz()
270327

328+
271329
class ModuleTest(unittest.TestCase):
272330
def test_create_module(self):
273331
from org.python.core import PyModule, PyInstance
@@ -279,6 +337,7 @@ def test_create_module(self):
279337
exec "b = 3" in test.__dict__
280338
self.assertEquals(len(test.__dict__), 5)
281339

340+
282341
def test_main():
283342
test_support.run_unittest(__name__)
284343

‎Lib/test/test_support.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -487,7 +487,7 @@ def u(s):
487487
return unicode(s, 'unicode-escape')
488488

489489
if is_jython:
490-
def make_jar_classloader(jar):
490+
def make_jar_classloader(jar, parent=False):
491491
import os
492492
from java.net import URL, URLClassLoader
493493
from java.io import File
@@ -508,7 +508,10 @@ def make_jar_classloader(jar):
508508
# better fix
509509
conn.setDefaultUseCaches(False)
510510

511-
return URLClassLoader([url])
511+
if parent is False:
512+
return URLClassLoader([url])
513+
else:
514+
return URLClassLoader([url], parent)
512515

513516
# Filename used for testing
514517
if is_jython:

‎src/org/python/core/Py.java‎

Lines changed: 2 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -41,28 +41,6 @@
4141
import jnr.posix.POSIXFactory;
4242
import jnr.posix.util.Platform;
4343

44-
/** Builtin types that are used to setup PyObject.
45-
*
46-
* Resolve circular dependency with some laziness. */
47-
class BootstrapTypesSingleton {
48-
private final Set<Class<?>> BOOTSTRAP_TYPES;
49-
private BootstrapTypesSingleton() {
50-
BOOTSTRAP_TYPES = Generic.set();
51-
BOOTSTRAP_TYPES.add(PyObject.class);
52-
BOOTSTRAP_TYPES.add(PyType.class);
53-
BOOTSTRAP_TYPES.add(PyBuiltinCallable.class);
54-
BOOTSTRAP_TYPES.add(PyDataDescr.class);
55-
}
56-
57-
private static class LazyHolder {
58-
private static final BootstrapTypesSingleton INSTANCE = new BootstrapTypesSingleton();
59-
}
60-
61-
public static Set<Class<?>> getInstance() {
62-
return LazyHolder.INSTANCE.BOOTSTRAP_TYPES;
63-
}
64-
}
65-
6644
public final class Py {
6745

6846
static class SingletonResolver implements Serializable {
@@ -87,7 +65,7 @@ private Object readResolve() throws ObjectStreamException {
8765

8866
/* Holds the singleton None and Ellipsis objects */
8967
/** The singleton None Python object **/
90-
public final static PyObject None = new PyNone();
68+
public final static PyObject None = PyNone.getInstance();
9169
/** The singleton Ellipsis Python object - written as ... when indexing */
9270
public final static PyObject Ellipsis = new PyEllipsis();
9371
/** The singleton NotImplemented Python object. Used in rich comparison */
@@ -566,6 +544,7 @@ private Py() {
566544
@param o the <code>PyObject</code> to convert.
567545
@param c the class to convert it to.
568546
**/
547+
@SuppressWarnings("unchecked")
569548
public static <T> T tojava(PyObject o, Class<T> c) {
570549
Object obj = o.__tojava__(c);
571550
if (obj == Py.NoConversion) {
@@ -786,10 +765,6 @@ private static List<String> fileSystemDecode(PyList path) {
786765
}
787766

788767
public static PyStringMap newStringMap() {
789-
// enable lazy bootstrapping (see issue #1671)
790-
if (!PyType.hasBuilder(PyStringMap.class)) {
791-
BootstrapTypesSingleton.getInstance().add(PyStringMap.class);
792-
}
793768
return new PyStringMap();
794769
}
795770

0 commit comments

Comments
 (0)