diff --git a/graalpython/com.oracle.graal.python.benchmarks/python/micro/object-dir-custom-bases.py b/graalpython/com.oracle.graal.python.benchmarks/python/micro/object-dir-custom-bases.py new file mode 100644 index 0000000000..0a5e83f92f --- /dev/null +++ b/graalpython/com.oracle.graal.python.benchmarks/python/micro/object-dir-custom-bases.py @@ -0,0 +1,118 @@ +# Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +"""Same workload as object-dir, rejected only by the final namespace-access guard.""" + + +_type_bases = type.__dict__["__bases__"] + + +class Meta(type): + @property + def __bases__(cls): + # Preserve the hierarchy, but require Python descriptor dispatch. + # Calling type.__getattribute__(cls, "__bases__") would recurse here. + return _type_bases.__get__(cls) + + +class Base(metaclass=Meta): + pass + + +class Config(Base): + pass + + +# Overlapping class and instance attributes exercise duplicate elimination. +# Use setattr, not a replacement __dict__, to retain DynamicObjectStorage. +for i in range(32): + setattr(Base, "attr_%02d" % i, i) + setattr(Config, "attr_%02d" % (i + 16), i) + +objects = [] +results = [] +expected = [] + + +def meta_attribute(name): + for cls in Meta.__mro__: + if name in cls.__dict__: + return cls.__dict__[name] + return None + + +def __setup__(num=10000): + if num < 1: + raise ValueError("num must be positive") + # Mirror hasStandardNamespaceAccess without invoking the bases property. + assert meta_attribute("__getattribute__") is type.__getattribute__ + assert meta_attribute("__getattr__") is None + assert meta_attribute("__dict__") is type.__dict__["__dict__"] + assert meta_attribute("__bases__") is not type.__dict__["__bases__"] + + objects.clear() + results.clear() + expected.clear() + for index in range(4): + obj = Config() + for i in range(32, 64): + setattr(obj, "attr_%02d" % i, i) + setattr(obj, "instance_%d" % index, index) + obj.removed = None + del obj.removed + objects.append(obj) + results.append(None) + names = set(obj.__dict__) + for cls in Config.__mro__: + names.update(cls.__dict__) + expected.append(sorted(names)) + + +def __benchmark__(num=10000): + # Retain the actual lists, not just their lengths, so names must be produced. + # Object/class construction and result validation are outside the timed loop. + for _ in range(num): + for index, obj in enumerate(objects): + results[index] = dir(obj) + return results + + +def __cleanup__(num=10000): + assert results == expected diff --git a/graalpython/com.oracle.graal.python.benchmarks/python/micro/object-dir.py b/graalpython/com.oracle.graal.python.benchmarks/python/micro/object-dir.py new file mode 100644 index 0000000000..6962055359 --- /dev/null +++ b/graalpython/com.oracle.graal.python.benchmarks/python/micro/object-dir.py @@ -0,0 +1,111 @@ +# Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +"""Repeated instance dir() with an eligible custom metaclass.""" + + +class Meta(type): + pass + + +class Base(metaclass=Meta): + pass + + +class Config(Base): + pass + + +# Overlapping class and instance attributes exercise duplicate elimination. +# Use setattr, not a replacement __dict__, to retain DynamicObjectStorage. +for i in range(32): + setattr(Base, "attr_%02d" % i, i) + setattr(Config, "attr_%02d" % (i + 16), i) + +objects = [] +results = [] +expected = [] + + +def meta_attribute(name): + for cls in Meta.__mro__: + if name in cls.__dict__: + return cls.__dict__[name] + return None + + +def __setup__(num=10000): + if num < 1: + raise ValueError("num must be positive") + # Mirror hasStandardNamespaceAccess without invoking the bases property. + assert meta_attribute("__getattribute__") is type.__getattribute__ + assert meta_attribute("__getattr__") is None + assert meta_attribute("__dict__") is type.__dict__["__dict__"] + assert meta_attribute("__bases__") is type.__dict__["__bases__"] + + objects.clear() + results.clear() + expected.clear() + for index in range(4): + obj = Config() + for i in range(32, 64): + setattr(obj, "attr_%02d" % i, i) + setattr(obj, "instance_%d" % index, index) + obj.removed = None + del obj.removed + objects.append(obj) + results.append(None) + names = set(obj.__dict__) + for cls in Config.__mro__: + names.update(cls.__dict__) + expected.append(sorted(names)) + + +def __benchmark__(num=10000): + # Retain the actual lists, not just their lengths, so names must be produced. + # Object/class construction and result validation are outside the timed loop. + for _ in range(num): + for index, obj in enumerate(objects): + results[index] = dir(obj) + return results + + +def __cleanup__(num=10000): + assert results == expected diff --git a/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/objects/ObjectHashMapTests.java b/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/objects/ObjectHashMapTests.java index d209b42c56..aff03f616f 100644 --- a/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/objects/ObjectHashMapTests.java +++ b/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/objects/ObjectHashMapTests.java @@ -96,6 +96,44 @@ public boolean execute(Frame frame, Node inliningTarget, Object a, Object b, Ric } } + @Test + public void testEnsureCapacity() { + ObjectHashMap map = new ObjectHashMap(); + LinkedHashMap expected = new LinkedHashMap<>(); + map.ensureCapacity(0); + assertEqual("empty reservation", expected, map); + + map.ensureCapacity(32); + for (long key = 0; key < 32; key++) { + Object value = newValue(); + put(map, key, getKeyHash(key), value); + expected.put(key, value); + } + map.ensureCapacity(map.size()); + map.ensureCapacity(33); + assertEqual("reservation within existing capacity", expected, map); + + removeValues(map, expected, new Random(42), 4); + map.ensureCapacity(map.size()); + map.ensureCapacity(map.size() + 20); + assertEqual("reservation after deletion", expected, map); + for (long key = 32; key < 52; key++) { + Object value = newValue(); + put(map, key, getKeyHash(key), value); + expected.put(key, value); + } + assertEqual("insertions after reservation", expected, map); + + ObjectHashMap fullMap = new ObjectHashMap(); + Object value = newValue(); + put(fullMap, 0L, getKeyHash(0L), value); + LinkedHashMap singleEntry = new LinkedHashMap<>(); + singleEntry.put(0L, value); + // Clamp speculative capacity, including requests exceeding Integer.MAX_VALUE. + fullMap.ensureCapacity((long) fullMap.size() + Integer.MAX_VALUE); + assertEqual("bounded reservation", singleEntry, fullMap); + } + @Test public void testCollisionsByPuttingManyKeysWithSameHash() { ObjectHashMap map = new ObjectHashMap(); diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_dict.py b/graalpython/com.oracle.graal.python.test/src/tests/test_dict.py index 8433c583ef..63ce7afee6 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_dict.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_dict.py @@ -1978,3 +1978,32 @@ class Test: del o.foo assert "foo" not in o.__dict__ + + +def test_bulk_update_after_deletions(): + d = {i: i for i in range(128)} + for i in range(0, 128, 2): + del d[i] + incoming = {i: -i for i in range(64, 256)} + expected_order = list(d) + [i for i in incoming if i not in d] + d.update(incoming) + assert list(d) == expected_order + assert all(d[i] == (-i if i >= 64 else i) for i in d) + snapshot = list(d.items()) + d.update(d) + d.update({}) + d.update(incoming) + assert list(d.items()) == snapshot + + +def test_bulk_value_update_preserves_iterators(): + d = {i: i for i in range(128)} + del d[0] + expected = list(d) + it = iter(d) + reverse = reversed(d) + assert next(it) == expected[0] + assert next(reverse) == expected[-1] + d.update({k: -v for k, v in d.items()}) + assert list(it) == expected[1:] + assert list(reverse) == expected[-2::-1] diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_object.py b/graalpython/com.oracle.graal.python.test/src/tests/test_object.py index e4873ee22a..d964c2eedb 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_object.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_object.py @@ -130,6 +130,121 @@ class MyList(list): assert mlst.a == 10 +def test_dir_many_attributes(): + class Meta(type): + pass + + cls = object + for depth in range(4): + cls = Meta(f"Level{depth}", (cls,), {f"attr_{depth}_{i}": i for i in range(200)}) + obj = cls() + for i in range(200): + setattr(obj, f"instance_{i}", i) + obj.attr_0_0 = "shadowed" + del obj.instance_5 + del cls.attr_3_5 + before = obj.__dict__.copy() + expected = set(before) + for base in cls.__mro__: + expected.update(base.__dict__) + names = object.__dir__(obj) + assert len(names) == len(expected) + assert set(names) == expected + assert dir(obj) == sorted(expected) + assert obj.__dict__ == before + + +def test_dir_replaced_dict(): + class C: + class_attr = 1 + + obj = C() + obj.old = 1 + assert "old" in object.__dir__(obj) + obj.__dict__ = {"new": 2, 42: "non-string key", "class_attr": 3} + expected = set(obj.__dict__) | set(C.__dict__) | set(object.__dict__) + names = object.__dir__(obj) + assert len(names) == len(expected) + assert set(names) == expected + assert obj.__dict__ == {"new": 2, 42: "non-string key", "class_attr": 3} + assert_raises(TypeError, dir, obj) + + +def test_dir_dict_subclass(): + class Dict(dict): + def keys(self): + raise AssertionError("must copy the dict, not call keys") + + class C: + @property + def __dict__(self): + return Dict(instance_attr=1) + + assert set(object.__dir__(C())) == {"instance_attr"} | set(C.__dict__) | set(object.__dict__) + + +def test_dir_custom_attribute_lookup(): + class FakeClass: + def __init__(self, namespace, bases): + self.namespace = namespace + self.__bases__ = bases + + @property + def __dict__(self): + return self.namespace + + fake_base = FakeClass({"base_attr": 1}, ()) + fake_class = FakeClass({"class_attr": 1}, (fake_base,)) + lookups = [] + + class C: + def __getattribute__(self, name): + lookups.append(name) + if name == "__dict__": + return {"instance_attr": 1, "class_attr": 2} + if name == "__class__": + return fake_class + raise AssertionError(name) + + assert set(object.__dir__(C())) == {"instance_attr", "class_attr", "base_attr"} + assert lookups == ["__dict__", "__class__"] + + +def test_dir_missing_or_invalid_attributes(): + from types import MappingProxyType + + class C: + def __getattribute__(self, name): + if name == "__dict__": + return namespace + raise AttributeError(name) + + for namespace in (None, {"present": 1}, MappingProxyType({"ignored": 1}), ["ignored"]): + assert object.__dir__(C()) == (["present"] if isinstance(namespace, dict) else []) + + class Missing: + def __getattribute__(self, name): + raise AttributeError(name) + + assert object.__dir__(Missing()) == [] + + class Slots: + __slots__ = ("slot",) + + assert set(object.__dir__(Slots())) == set(Slots.__dict__) | set(object.__dict__) + + +def test_dir_propagates_lookup_errors(): + for failing_name in ("__dict__", "__class__"): + class C: + def __getattribute__(self, name): + if name == failing_name: + raise RuntimeError(name) + return object.__getattribute__(self, name) + + assert_raises(RuntimeError, object.__dir__, C()) + + def test_set_dict_attr_with_getattr_defined(): class MyOtherClass(object): def __getattribute__(self, item): diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/common/ObjectHashMap.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/common/ObjectHashMap.java index 182211f628..f07f0076a3 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/common/ObjectHashMap.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/common/ObjectHashMap.java @@ -45,6 +45,8 @@ import com.oracle.graal.python.builtins.objects.common.ObjectHashMapFactory.PutNodeGen; import com.oracle.graal.python.builtins.objects.common.ObjectHashMapFactory.RemoveNodeGen; import com.oracle.graal.python.lib.PyObjectRichCompareBool; +import com.oracle.graal.python.runtime.sequence.storage.ObjectSequenceStorage; +import com.oracle.graal.python.util.ArrayBuilder; import com.oracle.graal.python.util.PythonUtils; import com.oracle.truffle.api.CompilerDirectives; import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; @@ -1007,14 +1009,52 @@ private boolean keysEqual(byte[] originalMetadata, Frame frame, Node inliningTar } /** - * Called when we need space for new entry. It determines the new size from the number of slots - * occupied by real values (i.e., does not count dummy entries), so the new size may be actually - * smaller than the old size if there were many dummy entries. The rehashing also removes the - * dummy entries. + * Called when we need space for new entry, it also inserts that entry. */ - @TruffleBoundary private void rehashAndPut(Object newKey, long newKeyHash, Object newValue) { - int newSize = size + 1; + rehash(size + 1, newKey, newKeyHash, newValue); + } + + /** + * Reserve for a bulk insertion whose keys may overlap existing keys. Like the constructor, cap + * speculative allocation so an overestimate does not cause an unnecessarily early memory error. + * Larger maps and maps with deleted entries continue to grow on demand. Accept a long so callers + * can add the current and incoming sizes without integer overflow. + */ + public void ensureCapacity(long newCapacity) { + assert newCapacity >= size; + if (newCapacity == size) { + return; + } + // Rehashing compacts deleted entries. A duplicate-only update must not move entries + // underneath live iterators, so leave maps with holes to grow on actual insertion. + if (usedHashes != size) { + return; + } + int limit = getUsableSize(MAX_PREALLOCATED_INDICES_SIZE); + if (size >= limit) { + return; + } + int capacity = (int) Math.min(newCapacity, limit); + int additionalEntries = capacity - size; + int entryCapacity = getEntryCapacity(); + int bucketsCount = getBucketsCount(metadata, entryCapacity, getIndexByteSize(entryCapacity)); + if (additionalEntries <= entryCapacity - usedHashes && additionalEntries <= getUsableSize(bucketsCount) - usedIndices) { + return; + } + rehash(Math.max(capacity, entryCapacity), null, -1, null); + } + + /** + * Called when we need space for new entry/entries. It determines the new size from the number of + * slots occupied by real values (i.e., does not count dummy entries), so the new size may be + * actually smaller than the old size if there were many dummy entries. The rehashing also removes + * the dummy entries. If the {@code newKey} argument is non-{@code null}, it also inserts given + * key/value with given hash. + */ + @TruffleBoundary + private void rehash(int newSize, Object newKey, long newKeyHash, Object newValue) { + assert newSize > size; int indicesCapacity = getMinBucketsCount(newSize); byte[] oldMetadata = metadata; Object[] oldKeysAndValues = keysAndValues; @@ -1038,7 +1078,9 @@ private void rehashAndPut(Object newKey, long newKeyHash, Object newValue) { } assert size == oldSize : String.format("size=%d, oldSize=%d, oldUsedSize=%d, usedHashes=%d, usedIndices=%d", size, oldSize, oldUsedSize, usedHashes, usedIndices); - insertNewKey(localMetadata, indicesLen, indicesOffset, indexByteSize, physicalCollisionMask, newKey, newKeyHash, newValue); + if (newKey != null) { + insertNewKey(localMetadata, indicesLen, indicesOffset, indexByteSize, physicalCollisionMask, newKey, newKeyHash, newValue); + } } private static int getRequestedEntryCapacity(int requestedCapacity, int bucketsCount) { @@ -1136,6 +1178,20 @@ public Object getKey(int index) { return getKey(index, keysAndValues); } + public ObjectSequenceStorage getKeys(Node inliningTarget) { + int index = 0; + ArrayBuilder keys = new ArrayBuilder<>(size); + while (index < usedHashes) { + Object val = getValue(index); + if (val != null) { + keys.add(getKey(index)); + } + index++; + } + LoopNode.reportLoopCount(inliningTarget, index); + return keys.toObjectSequenceStorage(); + } + public Object getValue(int index) { return getValue(index, keysAndValues); } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/object/ObjectBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/object/ObjectBuiltins.java index 3c8f75e7c1..b4a7fcb738 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/object/ObjectBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/object/ObjectBuiltins.java @@ -78,6 +78,7 @@ import com.oracle.graal.python.builtins.objects.cext.capi.transitions.CApiTransitions; import com.oracle.graal.python.builtins.objects.cext.structs.CFields; import com.oracle.graal.python.builtins.objects.cext.structs.CStructAccess; +import com.oracle.graal.python.builtins.objects.common.EconomicMapStorage; import com.oracle.graal.python.builtins.objects.dict.PDict; import com.oracle.graal.python.builtins.objects.function.PKeyword; import com.oracle.graal.python.builtins.objects.getsetdescriptor.DescriptorBuiltins.DescrDeleteNode; @@ -90,8 +91,6 @@ import com.oracle.graal.python.builtins.objects.object.ObjectBuiltinsClinicProviders.ReduceExNodeClinicProviderGen; import com.oracle.graal.python.builtins.objects.object.ObjectBuiltinsFactory.DictNodeFactory; import com.oracle.graal.python.builtins.objects.object.ObjectBuiltinsFactory.GetAttributeNodeFactory; -import com.oracle.graal.python.builtins.objects.set.PSet; -import com.oracle.graal.python.builtins.objects.set.SetBuiltins; import com.oracle.graal.python.builtins.objects.str.StringNodes.CastToTruffleStringChecked0Node; import com.oracle.graal.python.builtins.objects.str.StringNodes.CastToTruffleStringChecked1Node; import com.oracle.graal.python.builtins.objects.type.PythonBuiltinClass; @@ -99,7 +98,8 @@ import com.oracle.graal.python.builtins.objects.type.TpSlots; import com.oracle.graal.python.builtins.objects.type.TpSlots.GetCachedTpSlotsNode; import com.oracle.graal.python.builtins.objects.type.TpSlots.GetObjectSlotsNode; -import com.oracle.graal.python.builtins.objects.type.TypeBuiltins; +import com.oracle.graal.python.builtins.objects.type.TypeBuiltins.AddKeysToHashMap; +import com.oracle.graal.python.builtins.objects.type.TypeBuiltins.CollectKeysNode; import com.oracle.graal.python.builtins.objects.type.TypeFlags; import com.oracle.graal.python.builtins.objects.type.TypeNodes; import com.oracle.graal.python.builtins.objects.type.TypeNodes.CheckCompatibleForAssigmentNode; @@ -112,6 +112,7 @@ import com.oracle.graal.python.builtins.objects.type.slots.TpSlotRepr.CallSlotReprNode; import com.oracle.graal.python.builtins.objects.type.slots.TpSlotRichCompare; import com.oracle.graal.python.builtins.objects.type.slots.TpSlotSetAttr.SetAttrBuiltinNode; +import com.oracle.graal.python.lib.PyDictCheckNode; import com.oracle.graal.python.lib.PyObjectCallMethodObjArgs; import com.oracle.graal.python.lib.PyObjectIsTrueNode; import com.oracle.graal.python.lib.PyObjectLookupAttr; @@ -126,9 +127,7 @@ import com.oracle.graal.python.nodes.attributes.ReadAttributeFromObjectNode; import com.oracle.graal.python.nodes.attributes.WriteAttributeToObjectNode; import com.oracle.graal.python.nodes.builtins.ListNodes; -import com.oracle.graal.python.nodes.builtins.ListNodes.ConstructListNode; import com.oracle.graal.python.nodes.call.CallNode; -import com.oracle.graal.python.nodes.classes.IsSubtypeNode; import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode; import com.oracle.graal.python.nodes.function.PythonBuiltinNode; import com.oracle.graal.python.nodes.function.builtins.PythonBinaryBuiltinNode; @@ -146,7 +145,6 @@ import com.oracle.graal.python.nodes.object.IsNode; import com.oracle.graal.python.nodes.object.SetDictNode; import com.oracle.graal.python.nodes.util.CastToTruffleStringNode; -import com.oracle.graal.python.runtime.ExecutionContext.BoundaryCallContext; import com.oracle.graal.python.runtime.IndirectCallData.BoundaryCallData; import com.oracle.graal.python.runtime.PythonContext; import com.oracle.graal.python.runtime.exception.PException; @@ -870,28 +868,22 @@ public abstract static class DirNode extends PythonBuiltinNode { @Specialization static Object dir(VirtualFrame frame, Object obj, @Bind Node inliningTarget, - @Cached("createFor($node)") BoundaryCallData boundaryCallData, - @Cached SetBuiltins.UpdateSingleNode updateSetNode, + @Cached AddKeysToHashMap addKeys, + @Cached CollectKeysNode collectKeys, @Cached PyObjectLookupAttr lookupAttrNode, - @Cached GetClassNode getClassNode, - @Cached IsSubtypeNode isSubtypeNode, - @Cached ConstructListNode constructListNode, + @Cached PyDictCheckNode dictCheck, @Bind PythonLanguage language) { - PSet names = PFactory.createSet(language); + EconomicMapStorage map = EconomicMapStorage.create(); + PDict names = PFactory.createDict(language, map); Object ns = lookupAttrNode.execute(frame, inliningTarget, obj, T___DICT__); - if (isSubtypeNode.execute(getClassNode.execute(inliningTarget, ns), PythonBuiltinClassType.PDict)) { - updateSetNode.execute(frame, names, ns); + if (dictCheck.execute(inliningTarget, ns)) { + addKeys.execute(frame, inliningTarget, ns, names, map); } Object klass = lookupAttrNode.execute(frame, inliningTarget, obj, T___CLASS__); if (klass != PNone.NO_VALUE) { - Object state = BoundaryCallContext.enter(frame, boundaryCallData); - try { - TypeBuiltins.DirNode.dir(names, klass); - } finally { - BoundaryCallContext.exit(frame, boundaryCallData, state); - } + collectKeys.execute(frame, inliningTarget, klass, names, map); } - return constructListNode.execute(frame, names); + return PFactory.createList(language, map.getKeys(inliningTarget)); } } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TypeBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TypeBuiltins.java index 5eaf163740..a7f65a0181 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TypeBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TypeBuiltins.java @@ -31,6 +31,7 @@ import static com.oracle.graal.python.builtins.objects.cext.structs.CFields.PyHeapTypeObject__ht_qualname; import static com.oracle.graal.python.builtins.objects.cext.structs.CFields.PyTypeObject__tp_name; import static com.oracle.graal.python.builtins.objects.cext.structs.CStructAccess.writePtrField; +import static com.oracle.graal.python.builtins.objects.object.PythonObject.HAS_NO_VALUE_PROPERTIES; import static com.oracle.graal.python.nodes.BuiltinNames.T_BUILTINS; import static com.oracle.graal.python.nodes.ErrorMessages.ATTR_NAME_MUST_BE_STRING; import static com.oracle.graal.python.nodes.SpecialAttributeNames.J___ABSTRACTMETHODS__; @@ -77,7 +78,6 @@ import static com.oracle.graal.python.util.PythonUtils.tsLiteral; import java.util.Arrays; -import java.util.LinkedHashSet; import java.util.List; import com.oracle.graal.python.PythonLanguage; @@ -95,14 +95,16 @@ import com.oracle.graal.python.builtins.objects.cext.structs.CFields; import com.oracle.graal.python.builtins.objects.cext.structs.CStructAccess; import com.oracle.graal.python.builtins.objects.common.DynamicObjectStorage; +import com.oracle.graal.python.builtins.objects.common.EconomicMapStorage; +import com.oracle.graal.python.builtins.objects.common.ObjectHashMap; import com.oracle.graal.python.builtins.objects.common.SequenceNodes.GetObjectArrayNode; import com.oracle.graal.python.builtins.objects.common.SequenceStorageNodes.ToArrayNode; import com.oracle.graal.python.builtins.objects.dict.PDict; import com.oracle.graal.python.builtins.objects.function.PKeyword; import com.oracle.graal.python.builtins.objects.getsetdescriptor.DescriptorDeleteMarker; import com.oracle.graal.python.builtins.objects.list.PList; +import com.oracle.graal.python.builtins.objects.mappingproxy.PMappingproxy; import com.oracle.graal.python.builtins.objects.object.ObjectNodes; -import com.oracle.graal.python.builtins.objects.set.PSet; import com.oracle.graal.python.builtins.objects.set.SetBuiltins.UpdateSingleNode; import com.oracle.graal.python.builtins.objects.str.PString; import com.oracle.graal.python.builtins.objects.str.StringNodes.CastToTruffleStringChecked0Node; @@ -130,6 +132,7 @@ import com.oracle.graal.python.builtins.objects.type.slots.TpSlotVarargs.CallSlotTpInitNode; import com.oracle.graal.python.builtins.objects.type.slots.TpSlotVarargs.CallSlotTpNewNode; import com.oracle.graal.python.builtins.objects.types.GenericTypeNodes; +import com.oracle.graal.python.lib.PyObjectHashNode; import com.oracle.graal.python.lib.PyObjectIsTrueNode; import com.oracle.graal.python.lib.PyObjectLookupAttr; import com.oracle.graal.python.lib.PyObjectReprAsTruffleStringNode; @@ -144,7 +147,6 @@ import com.oracle.graal.python.nodes.attributes.LookupAttributeInMRONode; import com.oracle.graal.python.nodes.attributes.ReadAttributeFromObjectNode; import com.oracle.graal.python.nodes.attributes.WriteAttributeToObjectNode; -import com.oracle.graal.python.nodes.builtins.ListNodes.ConstructListNode; import com.oracle.graal.python.nodes.builtins.TupleNodes.ConstructTupleNode; import com.oracle.graal.python.nodes.classes.IsSubtypeNode; import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode; @@ -163,6 +165,7 @@ import com.oracle.graal.python.runtime.exception.PException; import com.oracle.graal.python.runtime.exception.PythonErrorType; import com.oracle.graal.python.runtime.object.PFactory; +import com.oracle.graal.python.util.ArrayBuilder; import com.oracle.graal.python.util.PythonUtils; import com.oracle.truffle.api.CompilerDirectives; import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; @@ -180,11 +183,14 @@ import com.oracle.truffle.api.dsl.NodeFactory; import com.oracle.truffle.api.dsl.Specialization; import com.oracle.truffle.api.frame.VirtualFrame; +import com.oracle.truffle.api.nodes.LoopNode; import com.oracle.truffle.api.nodes.Node; -import com.oracle.truffle.api.object.DynamicObject; +import com.oracle.truffle.api.object.DynamicObject.GetKeyArrayNode; +import com.oracle.truffle.api.object.DynamicObject.GetNode; import com.oracle.truffle.api.profiles.BranchProfile; import com.oracle.truffle.api.profiles.InlinedBranchProfile; import com.oracle.truffle.api.profiles.InlinedConditionProfile; +import com.oracle.truffle.api.profiles.InlinedLoopConditionProfile; import com.oracle.truffle.api.strings.TruffleString; @CoreFunctions(extendClasses = PythonBuiltinClassType.PythonClass) @@ -1243,103 +1249,104 @@ public abstract static class DirNode extends PythonUnaryBuiltinNode { static PList dir(VirtualFrame frame, Object klass, @Bind Node inliningTarget, @Bind PythonLanguage language, - @Bind PythonContext context, - @Cached ConstructListNode constructListNode, - @Cached InlinedBranchProfile slowPathBranch, - @Cached("createFor($node)") BoundaryCallData boundaryCallData) { - Object[] fastNames = dirFast(language, context, klass); - if (fastNames != null) { - return PFactory.createList(language, fastNames); - } - - slowPathBranch.enter(inliningTarget); - PSet names = PFactory.createSet(language); - Object state = BoundaryCallContext.enter(frame, boundaryCallData); - try { - dir(names, klass); - } finally { - BoundaryCallContext.exit(frame, boundaryCallData, state); - } - return constructListNode.execute(frame, names); + @Cached CollectKeysNode collectKeys) { + EconomicMapStorage map = EconomicMapStorage.create(); + PDict dict = PFactory.createDict(language, map); + collectKeys.execute(frame, inliningTarget, klass, dict, map); + return PFactory.createList(language, map.getKeys(inliningTarget)); } - @TruffleBoundary - private static Object[] dirFast(PythonLanguage language, PythonContext context, Object klass) { - if (!isFastPathEligible(context, klass)) { - return null; - } - LinkedHashSet names = new LinkedHashSet<>(); - collectDynamicObjectStorageKeys(names, context, klass); - return names.toArray(); + @NeverDefault + public static DirNode create() { + return TypeBuiltinsFactory.DirNodeFactory.create(); } + } - private static boolean isFastPathEligible(PythonContext context, Object klass) { - PythonManagedClass managedClass = asManagedClass(context, klass); - if (managedClass == null || PGuards.hasMaterializedDict(managedClass.getShape())) { - return false; - } - for (PythonAbstractClass base : managedClass.getBaseClasses()) { - if (!isFastPathEligible(context, base)) { - return false; + @GenerateInline + @GenerateCached(false) + public abstract static class CollectKeysNode extends Node { + public abstract void execute(VirtualFrame frame, Node inliningTarget, Object klass, PDict dict, ObjectHashMap map); + + @Specialization + static void collect(VirtualFrame frame, Node inliningTarget, Object klass, PDict dict, ObjectHashMap map, + @Cached PyObjectLookupAttr lookupDict, + @Cached PyObjectLookupAttr lookupBases, + @Cached AddKeysToHashMap addKeysToHashMap, + @Cached PyTupleCheckNode pyTupleCheckNode, + @Cached ConstructTupleNode constructTupleNode, + @Cached ToArrayNode tupleToArrayNode) { + ArrayBuilder worklist = new ArrayBuilder<>(); + worklist.add(klass); + + while (!worklist.isEmpty()) { + Object currentType = worklist.pop(); + Object ns = lookupDict.execute(frame, inliningTarget, currentType, T___DICT__); + addKeysToHashMap.execute(frame, inliningTarget, ns, dict, map); + Object basesAttr = lookupBases.execute(frame, inliningTarget, currentType, T___BASES__); + if (pyTupleCheckNode.execute(inliningTarget, basesAttr)) { + PTuple basesTuple = constructTupleNode.execute(frame, basesAttr); + Object[] bases = tupleToArrayNode.execute(inliningTarget, basesTuple.getSequenceStorage()); + // Push in reverse so the leftmost base is processed first. Later bases remain + // pending and need not be added again if they are also bases of an earlier base. + for (int i = bases.length - 1; i >= 0; i--) { + Object base = bases[i]; + if (!worklist.containsIdentical(base)) { + worklist.add(base); + } + } } } - return true; } + } - private static PythonManagedClass asManagedClass(PythonContext context, Object klass) { - if (GetClassNode.executeUncached(klass) != PythonBuiltinClassType.PythonClass) { - return null; + @GenerateInline + @GenerateCached(false) + public abstract static class AddKeysToHashMap extends Node { + public final void execute(VirtualFrame frame, Node inliningTarget, Object ns, PDict dict, ObjectHashMap map) { + assert dict.getDictStorage() == map; + if (ns == NO_VALUE) { + return; } - if (klass instanceof PythonManagedClass pythonManagedClass) { - return pythonManagedClass; - } else if (klass instanceof PythonBuiltinClassType builtinClassType) { - return context.lookupType(builtinClassType); + Object mapping = ns; + if (ns instanceof PMappingproxy proxy && proxy.getMapping() instanceof PDict source && PGuards.isBuiltinDict(source)) { + mapping = source; } - return null; - } - - private static void collectDynamicObjectStorageKeys(LinkedHashSet names, PythonContext context, Object klass) { - PythonManagedClass managedClass = asManagedClass(context, klass); - assert managedClass != null && !PGuards.hasMaterializedDict(managedClass.getShape()); - DynamicObject.GetNode getNode = DynamicObject.GetNode.getUncached(); - for (Object key : DynamicObject.GetKeyArrayNode.getUncached().execute(managedClass)) { - if (key instanceof TruffleString stringKey && getNode.execute(managedClass, stringKey, NO_VALUE) != NO_VALUE) { - names.add(stringKey); - } - } - - for (PythonAbstractClass base : managedClass.getBaseClasses()) { - collectDynamicObjectStorageKeys(names, context, base); + if (mapping instanceof PDict source && source.getDictStorage() instanceof DynamicObjectStorage storage) { + ns = storage; } + executeImpl(frame, inliningTarget, ns, dict, map); } - @TruffleBoundary - public static void dir(PSet names, Object klass) { - Object ns = PyObjectLookupAttr.executeUncached(klass, T___DICT__); - UpdateSingleNode updateSingleNode = UpdateSingleNode.getUncached(); - if (ns != NO_VALUE) { - updateSingleNode.execute(null, names, ns); - } - Object basesAttr = PyObjectLookupAttr.executeUncached(klass, T___BASES__); - if (basesAttr instanceof PTuple || PyTupleCheckNode.executeUncached(basesAttr)) { - PTuple basesTuple = basesAttr instanceof PTuple ? (PTuple) basesAttr : ConstructTupleNode.getUncached().execute(null, basesAttr); - Object[] bases = ToArrayNode.executeUncached(basesTuple.getSequenceStorage()); - for (Object cls : bases) { - // Note that since we are only interested in the keys, the order - // we merge classes is unimportant - dir(names, cls); + public abstract void executeImpl(VirtualFrame frame, Node inliningTarget, Object ns, PDict dict, ObjectHashMap map); + + @Specialization + static void doDynamicObjectStorage(VirtualFrame frame, Node inliningTarget, DynamicObjectStorage storage, @SuppressWarnings("unused") PDict dict, ObjectHashMap map, + @Cached GetKeyArrayNode keyDomKeyArrayNode, + @Cached TruffleString.HashCodeNode hashCodeNode, + @Cached InlinedConditionProfile mayHaveNoValueProfile, + @Cached InlinedLoopConditionProfile loopProfile, + @Cached ObjectHashMap.PutNode putNode) { + boolean mayHaveNoValue = mayHaveNoValueProfile.profile(inliningTarget, !(storage.getStore() instanceof PythonManagedClass) || + (storage.getStore().getShape().getFlags() & HAS_NO_VALUE_PROPERTIES) != 0); + Object[] keys = keyDomKeyArrayNode.execute(storage.getStore()); + loopProfile.profileCounted(inliningTarget, keys.length); + map.ensureCapacity((long) map.size() + keys.length); + // The DOM access is almost always going to be polymorphic, + // so save the cache invalidation dance and cached node footprint + GetNode getNode = GetNode.getUncached(); + for (int i = 0; loopProfile.inject(inliningTarget, i < keys.length); i++) { + Object key = keys[i]; + if (key instanceof TruffleString strKey && (!mayHaveNoValue || getNode.execute(storage.getStore(), strKey, NO_VALUE) != NO_VALUE)) { + putNode.put(frame, inliningTarget, map, strKey, PyObjectHashNode.hash(strKey, hashCodeNode), PNone.NONE); } } + LoopNode.reportLoopCount(inliningTarget, keys.length); } - @NeverDefault - protected GetFixedAttributeNode createGetAttrNode() { - return GetFixedAttributeNode.create(T___BASES__); - } - - @NeverDefault - public static DirNode create() { - return TypeBuiltinsFactory.DirNodeFactory.create(); + @Fallback + static void doGeneric(VirtualFrame frame, @SuppressWarnings("unused") Node inliningTarget, Object ns, PDict dict, @SuppressWarnings("unused") ObjectHashMap map, + @Cached UpdateSingleNode updateSingleNode) { + updateSingleNode.execute(frame, dict, ns); } } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/lib/PyDictMerge.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/lib/PyDictMerge.java index facda4999e..69550330c5 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/lib/PyDictMerge.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/lib/PyDictMerge.java @@ -45,6 +45,7 @@ import static com.oracle.graal.python.runtime.exception.PythonErrorType.TypeError; import static com.oracle.graal.python.runtime.exception.PythonErrorType.ValueError; +import com.oracle.graal.python.builtins.objects.common.EconomicMapStorage; import com.oracle.graal.python.builtins.objects.common.HashingCollectionNodes; import com.oracle.graal.python.builtins.objects.common.HashingStorage; import com.oracle.graal.python.builtins.objects.common.HashingStorageNodes.HashingStorageGetIterator; @@ -77,6 +78,7 @@ import com.oracle.truffle.api.frame.VirtualFrame; import com.oracle.truffle.api.nodes.Node; import com.oracle.truffle.api.profiles.InlinedBranchProfile; +import com.oracle.truffle.api.profiles.InlinedConditionProfile; import com.oracle.truffle.api.profiles.InlinedLoopConditionProfile; /** Equivalent to {@code PyDict_Merge(target, mapping, 1)}. */ @@ -116,6 +118,7 @@ static void doMerge(VirtualFrame frame, Node inliningTarget, Object target, PDic @Cached HashingStorageGetIterator getMappingIter, @Cached HashingStorageIteratorNext iterNext, @Cached HashingStorageLen mappingLenNode, + @Cached InlinedConditionProfile targetIsMapProfile, @Cached PRaiseNode raiseNode) { if (target == mapping) { return; @@ -123,6 +126,10 @@ static void doMerge(VirtualFrame frame, Node inliningTarget, Object target, PDic HashingStorage targetStorage = getStorageNode.execute(inliningTarget, target); HashingStorage mappingStorage = mapping.getDictStorage(); int initialSize = mappingLenNode.execute(inliningTarget, mappingStorage); + if (targetIsMapProfile.profile(inliningTarget, targetStorage instanceof EconomicMapStorage)) { + EconomicMapStorage map = (EconomicMapStorage) targetStorage; + map.ensureCapacity((long) map.size() + initialSize); + } HashingStorageIterator iterator = getMappingIter.execute(inliningTarget, mappingStorage); HashingStorage newStorage = targetStorage; while (iterNext.execute(inliningTarget, mappingStorage, iterator)) { diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/util/ArrayBuilder.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/util/ArrayBuilder.java index 31578a8685..149e5a1606 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/util/ArrayBuilder.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/util/ArrayBuilder.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -42,6 +42,7 @@ import java.util.Arrays; +import com.oracle.graal.python.runtime.sequence.storage.ObjectSequenceStorage; import com.oracle.truffle.api.CompilerDirectives; /** @@ -94,6 +95,15 @@ public T peek() { return (T) data[size - 1]; } + public boolean containsIdentical(Object o) { + for (int i = 0; i < size; i++) { + if (data[i] == o) { + return true; + } + } + return false; + } + @SuppressWarnings("unchecked") public T[] toArray(T[] newArray) { return (T[]) arrayCopyOf(data, size, newArray.getClass()); @@ -104,6 +114,13 @@ public Object[] toObjectArray() { return arrayCopyOf(data, size, newArray.getClass()); } + /** + * The backing array may be overallocated! + */ + public ObjectSequenceStorage toObjectSequenceStorage() { + return new ObjectSequenceStorage(data, size); + } + public int size() { return size; } diff --git a/mx.graalpython/mx_graalpython_bench_param.py b/mx.graalpython/mx_graalpython_bench_param.py index 2ef13cce7d..476301f69b 100644 --- a/mx.graalpython/mx_graalpython_bench_param.py +++ b/mx.graalpython/mx_graalpython_bench_param.py @@ -97,6 +97,8 @@ 'dict-getitem-sized': ITER_10 + ['50_000_000'], 'math-sqrt': ITER_10 + ['500000000'], 'object-allocate': ITER_10 + ['5000'], + 'object-dir': ITER_10 + ['10_000'], + 'object-dir-custom-bases': ITER_10 + ['10_000'], 'object-layout-change': ITER_10 + ['1000000'], 'special-add-int-sized': ITER_10 + ['20_000'], 'special-add-sized': ITER_10 + ['20_000'], @@ -175,6 +177,8 @@ 'dict-getitem-sized': ITER_10 + WARMUP_2 + ['1_000_000'], 'math-sqrt': ITER_6 + WARMUP_2 + ['20_000_000'], 'object-allocate': ITER_6 + WARMUP_2 + ['50'], + 'object-dir': ITER_6 + WARMUP_2 + ['100'], + 'object-dir-custom-bases': ITER_6 + WARMUP_2 + ['100'], 'object-layout-change': ITER_6 + WARMUP_2 + ['10_000'], 'special-add-int-sized': ITER_6 + WARMUP_2 + ['1_000'], 'special-add-sized': ITER_6 + WARMUP_2 + ['1_000'],