From fe4cf0b08cf4b44ea2e07affa78a4b09ce34d347 Mon Sep 17 00:00:00 2001 From: Danilo Ansaloni Date: Thu, 23 Jan 2025 19:38:03 +0100 Subject: [PATCH 01/82] Restart the 24.2 dev cycle. Begin of the feature freeze for the 24.2 release. Revert "Start 25.0.0 dev cycle." This reverts commit f4105848044ea7cb58318895c7da184c3af64217. --- graalpython/com.oracle.graal.python.test.integration/pom.xml | 2 +- .../src/tests/standalone/gradle/build/build.gradle | 2 +- .../src/tests/standalone/gradle/build/build.gradle.kts | 2 +- .../src/tests/standalone/jbang/EmptyPIPComments.j | 2 +- .../src/tests/standalone/jbang/EmptyPythonResourceComment.j | 2 +- .../standalone/jbang/EmptyPythonResourceCommentWithBlanks.j | 2 +- .../src/tests/standalone/jbang/NoPackagesResourcesDir.j | 2 +- .../src/tests/standalone/jbang/TwoPythonResourceComments.j | 2 +- graalpython/graalpy-archetype-polyglot-app/pom.xml | 2 +- .../src/main/resources/archetype-resources/pom.xml | 2 +- graalpython/graalpy-jbang/examples/hello.java | 2 +- .../graalpy-jbang/templates/graalpy-template.java.qute | 2 +- .../templates/graalpy-template_local_repo.java.qute | 2 +- graalpython/graalpy-maven-plugin/pom.xml | 4 ++-- mx.graalpython/suite.py | 2 +- 15 files changed, 16 insertions(+), 16 deletions(-) diff --git a/graalpython/com.oracle.graal.python.test.integration/pom.xml b/graalpython/com.oracle.graal.python.test.integration/pom.xml index bad3f27502..0f55209ba9 100644 --- a/graalpython/com.oracle.graal.python.test.integration/pom.xml +++ b/graalpython/com.oracle.graal.python.test.integration/pom.xml @@ -64,7 +64,7 @@ Additionally, one can change the polyglot artifacts version with 17 17 UTF-8 - 25.0.0 + 24.2.0 diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/build.gradle b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/build.gradle index c9b31ffe75..15c3c89c12 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/build.gradle +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/build.gradle @@ -1,6 +1,6 @@ plugins { id "application" - id 'org.graalvm.python' version '25.0.0' + id 'org.graalvm.python' version '24.2.0' id "org.graalvm.buildtools.native" version "0.10.2" } diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/build.gradle.kts b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/build.gradle.kts index ee72dfe3de..a086b6635e 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/build.gradle.kts +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/build.gradle.kts @@ -1,6 +1,6 @@ plugins { application - id("org.graalvm.python") version "25.0.0" + id("org.graalvm.python") version "24.2.0" id("org.graalvm.buildtools.native") version "0.10.2" } diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/jbang/EmptyPIPComments.j b/graalpython/com.oracle.graal.python.test/src/tests/standalone/jbang/EmptyPIPComments.j index 098695dccb..6b140a59b8 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/standalone/jbang/EmptyPIPComments.j +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/jbang/EmptyPIPComments.j @@ -1,6 +1,6 @@ ///usr/bin/env jbang "$0" "$@" ; exit $? -//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:25.0.0} +//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:24.2.0} //PIP // one blank after PIP //PIP diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/jbang/EmptyPythonResourceComment.j b/graalpython/com.oracle.graal.python.test/src/tests/standalone/jbang/EmptyPythonResourceComment.j index 89f5758c5e..768e188e9e 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/standalone/jbang/EmptyPythonResourceComment.j +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/jbang/EmptyPythonResourceComment.j @@ -1,6 +1,6 @@ ///usr/bin/env jbang "$0" "$@" ; exit $? -//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:25.0.0} +//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:24.2.0} //PYTHON_RESOURCES_DIRECTORY public class EmptyPythonResourceComment { diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/jbang/EmptyPythonResourceCommentWithBlanks.j b/graalpython/com.oracle.graal.python.test/src/tests/standalone/jbang/EmptyPythonResourceCommentWithBlanks.j index fcb93bdc33..f4b7556997 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/standalone/jbang/EmptyPythonResourceCommentWithBlanks.j +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/jbang/EmptyPythonResourceCommentWithBlanks.j @@ -1,6 +1,6 @@ ///usr/bin/env jbang "$0" "$@" ; exit $? -//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:25.0.0} +//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:24.2.0} // resource dir with blanks //PYTHON_RESOURCES_DIRECTORY diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/jbang/NoPackagesResourcesDir.j b/graalpython/com.oracle.graal.python.test/src/tests/standalone/jbang/NoPackagesResourcesDir.j index c2255aca6c..749ae89606 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/standalone/jbang/NoPackagesResourcesDir.j +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/jbang/NoPackagesResourcesDir.j @@ -1,6 +1,6 @@ ///usr/bin/env jbang "$0" "$@" ; exit $? -//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:25.0.0} +//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:24.2.0} //PYTHON_RESOURCES_DIRECTORY python-resources public class NoPackagesResourcesDir { diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/jbang/TwoPythonResourceComments.j b/graalpython/com.oracle.graal.python.test/src/tests/standalone/jbang/TwoPythonResourceComments.j index 3bd4381f51..c2d902d358 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/standalone/jbang/TwoPythonResourceComments.j +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/jbang/TwoPythonResourceComments.j @@ -1,6 +1,6 @@ ///usr/bin/env jbang "$0" "$@" ; exit $? -//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:25.0.0} +//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:24.2.0} //PYTHON_RESOURCES_DIRECTORY //PYTHON_RESOURCES_DIRECTORY diff --git a/graalpython/graalpy-archetype-polyglot-app/pom.xml b/graalpython/graalpy-archetype-polyglot-app/pom.xml index d08b5631bb..04b27f55e3 100644 --- a/graalpython/graalpy-archetype-polyglot-app/pom.xml +++ b/graalpython/graalpy-archetype-polyglot-app/pom.xml @@ -45,7 +45,7 @@ SOFTWARE. org.graalvm.python graalpy-archetype-polyglot-app - 25.0.0 + 24.2.0 http://www.graalvm.org/python Maven archetype providing a skeleton GraalPy - Java polyglot application. maven-archetype diff --git a/graalpython/graalpy-archetype-polyglot-app/src/main/resources/archetype-resources/pom.xml b/graalpython/graalpy-archetype-polyglot-app/src/main/resources/archetype-resources/pom.xml index eb03184b9f..90be4e32ca 100644 --- a/graalpython/graalpy-archetype-polyglot-app/src/main/resources/archetype-resources/pom.xml +++ b/graalpython/graalpy-archetype-polyglot-app/src/main/resources/archetype-resources/pom.xml @@ -10,7 +10,7 @@ #set( $symbol_dollar = '$' ) - 25.0.0 + 24.2.0 python-community 0.10.4 17 diff --git a/graalpython/graalpy-jbang/examples/hello.java b/graalpython/graalpy-jbang/examples/hello.java index 2afc0c82b3..b005158ef6 100644 --- a/graalpython/graalpy-jbang/examples/hello.java +++ b/graalpython/graalpy-jbang/examples/hello.java @@ -40,7 +40,7 @@ */ ///usr/bin/env jbang "$0" "$@" ; exit $? //JAVA 17+ -//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:25.0.0} +//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:24.2.0} // specify python packages and their versions as if used with pip //PIP termcolor==2.2 diff --git a/graalpython/graalpy-jbang/templates/graalpy-template.java.qute b/graalpython/graalpy-jbang/templates/graalpy-template.java.qute index 5258d8cb30..5a61e3dea9 100644 --- a/graalpython/graalpy-jbang/templates/graalpy-template.java.qute +++ b/graalpython/graalpy-jbang/templates/graalpy-template.java.qute @@ -5,7 +5,7 @@ {/for} {#if dependencies.isEmpty()}// //DEPS {/if} {| -//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:25.0.0} +//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:24.2.0} // specify python packages and their versions as if used with pip //PIP termcolor==2.2 |} diff --git a/graalpython/graalpy-jbang/templates/graalpy-template_local_repo.java.qute b/graalpython/graalpy-jbang/templates/graalpy-template_local_repo.java.qute index 9db2304494..2fb3914dc1 100644 --- a/graalpython/graalpy-jbang/templates/graalpy-template_local_repo.java.qute +++ b/graalpython/graalpy-jbang/templates/graalpy-template_local_repo.java.qute @@ -8,7 +8,7 @@ //REPOS mc=https://repo1.maven.org/maven2/ //REPOS local=file://{path_to_local_repo} {| -//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:25.0.0} +//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:24.2.0} // specify python packages and their versions as if used with pip //PIP termcolor==2.2 |} diff --git a/graalpython/graalpy-maven-plugin/pom.xml b/graalpython/graalpy-maven-plugin/pom.xml index 7a8715415f..4838e2cec7 100644 --- a/graalpython/graalpy-maven-plugin/pom.xml +++ b/graalpython/graalpy-maven-plugin/pom.xml @@ -48,7 +48,7 @@ SOFTWARE. graalpy-maven-plugin maven-plugin - 25.0.0 + 24.2.0 http://www.graalvm.org/python graalpy-maven-plugin Handles python related resources in a maven GraalPy - Java polyglot application. @@ -57,7 +57,7 @@ SOFTWARE. 17 17 UTF-8 - 25.0.0 + 24.2.0 diff --git a/mx.graalpython/suite.py b/mx.graalpython/suite.py index 92ae8a88d7..efb3e6a4ee 100644 --- a/mx.graalpython/suite.py +++ b/mx.graalpython/suite.py @@ -9,7 +9,7 @@ "name": "graalpython", "versionConflictResolution": "latest", - "version": "25.0.0", + "version": "24.2.0", "graalpython:pythonVersion": "3.11.7", "release": False, "groupId": "org.graalvm.python", From b94c39b17b87b69dfa53683f5f2cb334a4857dae Mon Sep 17 00:00:00 2001 From: Danilo Ansaloni Date: Thu, 23 Jan 2025 19:40:45 +0100 Subject: [PATCH 02/82] Update imports --- ci.jsonnet | 2 +- mx.graalpython/suite.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ci.jsonnet b/ci.jsonnet index 48bf915fd3..f4fbcec95d 100644 --- a/ci.jsonnet +++ b/ci.jsonnet @@ -1 +1 @@ -{ "overlay": "840dea7f9575e2e96e2143685751932513fd0c78" } +{ "overlay": "da4fbb453006eea9997babadc69b70c2e0cee0ab" } diff --git a/mx.graalpython/suite.py b/mx.graalpython/suite.py index efb3e6a4ee..763a0fd5f6 100644 --- a/mx.graalpython/suite.py +++ b/mx.graalpython/suite.py @@ -45,7 +45,7 @@ }, { "name": "sdk", - "version": "0f61c409358b6ecdb637d69410c8d3960b289e9c", + "version": "826d284d76d4e3c2379e952824f4ab7e96c805ee", "subdir": True, "urls": [ {"url": "https://github.com/oracle/graal", "kind": "git"}, @@ -53,7 +53,7 @@ }, { "name": "tools", - "version": "0f61c409358b6ecdb637d69410c8d3960b289e9c", + "version": "826d284d76d4e3c2379e952824f4ab7e96c805ee", "subdir": True, "urls": [ {"url": "https://github.com/oracle/graal", "kind": "git"}, @@ -61,7 +61,7 @@ }, { "name": "sulong", - "version": "0f61c409358b6ecdb637d69410c8d3960b289e9c", + "version": "826d284d76d4e3c2379e952824f4ab7e96c805ee", "subdir": True, "urls": [ {"url": "https://github.com/oracle/graal", "kind": "git"}, @@ -69,7 +69,7 @@ }, { "name": "regex", - "version": "0f61c409358b6ecdb637d69410c8d3960b289e9c", + "version": "826d284d76d4e3c2379e952824f4ab7e96c805ee", "subdir": True, "urls": [ {"url": "https://github.com/oracle/graal", "kind": "git"}, From 9095026c26d47b96e20762b12a94340a482ed727 Mon Sep 17 00:00:00 2001 From: stepan Date: Wed, 29 Jan 2025 21:40:54 +0100 Subject: [PATCH 03/82] Move the longest running Gradle Kotlin tests to the 'long running' batch (cherry picked from commit 1fa8957d0236aed27dd4b7d9a381cef87af09bfc) --- .../src/tests/standalone/test_gradle_plugin.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/test_gradle_plugin.py b/graalpython/com.oracle.graal.python.test/src/tests/standalone/test_gradle_plugin.py index 2b15001db5..52d95fc704 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/standalone/test_gradle_plugin.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/test_gradle_plugin.py @@ -677,11 +677,11 @@ def setUpClass(cls): cls.build_file_name = "build.gradle.kts" cls.settings_file_name = "settings.gradle.kts" - @unittest.skipUnless(util.is_gradle_plugin_test_enabled, "ENABLE_GRADLE_PLUGIN_UNITTESTS is not true") + @unittest.skipUnless(util.is_gradle_plugin_long_running_test_enabled, "ENABLE_GRADLE_PLUGIN_LONG_RUNNING_UNITTESTS is not true") def test_gradle_generated_app(self): self.check_gradle_generated_app(community=True) - @unittest.skipUnless(util.is_gradle_plugin_test_enabled, "ENABLE_GRADLE_PLUGIN_UNITTESTS is not true") + @unittest.skipUnless(util.is_gradle_plugin_long_running_test_enabled, "ENABLE_GRADLE_PLUGIN_LONG_RUNNING_UNITTESTS is not true") def test_gradle_generated_app_external_resources(self): self.check_gradle_generated_app_external_resources() @@ -701,7 +701,7 @@ def test_gradle_check_home(self): def test_gradle_empty_packages(self): self.check_gradle_empty_packages() - @unittest.skipUnless(util.is_gradle_plugin_test_enabled, "ENABLE_GRADLE_PLUGIN_UNITTESTS is not true") + @unittest.skipUnless(util.is_gradle_plugin_long_running_test_enabled, "ENABLE_GRADLE_PLUGIN_LONG_RUNNING_UNITTESTS is not true") def test_gradle_namespaced_vfs(self): self.check_gradle_namespaced_vfs() From 99bc51c61f38c4a5ecbc0b19a59909178c530382 Mon Sep 17 00:00:00 2001 From: stepan Date: Wed, 29 Jan 2025 21:57:12 +0100 Subject: [PATCH 04/82] Update CI overlay (cherry picked from commit f25f7853affa54cf53e93e45699dd94b761de48f) --- ci.jsonnet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci.jsonnet b/ci.jsonnet index f4fbcec95d..a3a4e1743e 100644 --- a/ci.jsonnet +++ b/ci.jsonnet @@ -1 +1 @@ -{ "overlay": "da4fbb453006eea9997babadc69b70c2e0cee0ab" } +{ "overlay": "585265d3c6bafc1d12054374377589b360307957" } From 77a8b4168109e94e605ff3d5482673f511a465e4 Mon Sep 17 00:00:00 2001 From: stepan Date: Wed, 29 Jan 2025 14:34:09 +0100 Subject: [PATCH 05/82] Change the log level of 'Unexpected executable for slot pointer' messages to fine (cherry picked from commit b95a25d3910688780a94c5f89639a5fbbcd1df64) --- .../com/oracle/graal/python/builtins/objects/type/TpSlots.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TpSlots.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TpSlots.java index 497d8f0470..0685be32b0 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TpSlots.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TpSlots.java @@ -842,7 +842,7 @@ public static TpSlots fromNative(PythonAbstractNativeObject pythonClass, PythonC existingSlotWrapper = execWrapper; } else if (executable != null) { // This can happen for legacy slots where the delegate would be a PFunction - LOGGER.warning(() -> String.format("Unexpected executable for slot pointer: %s", executable)); + LOGGER.fine(() -> String.format("Unexpected executable for slot pointer: %s", executable)); } } catch (UnsupportedMessageException e) { throw new IllegalStateException(e); From 195d4bb37d421be7ce29522ebb33b07dae8427fa Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Fri, 17 Jan 2025 14:59:17 +0100 Subject: [PATCH 06/82] Properly order traits in GetForeignObjectClassNode (cherry picked from commit 8922dc627a1da6cc1bc7f4cbe684889112db5eff) --- .../src/tests/test_interop.py | 6 ++--- .../object/GetForeignObjectClassNode.java | 25 +++++++++++-------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_interop.py b/graalpython/com.oracle.graal.python.test/src/tests/test_interop.py index 8533ed3835..d3b511aea6 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_interop.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_interop.py @@ -150,7 +150,7 @@ def t(obj): # ForeignInstantiable self.assertEqual(t((e for e in [1])), polyglot.ForeignIteratorIterable) self.assertEqual(t(iter([1])), polyglot.ForeignIteratorIterable) - self.assertEqual(t(object), polyglot.ForeignExecutableClass) + self.assertEqual(t(object), polyglot.ForeignClassExecutable) self.assertEqual(t(None), polyglot.ForeignNone) self.assertEqual(t(1), polyglot.ForeignNumber) self.assertEqual(t("abc"), polyglot.ForeignString) @@ -472,9 +472,9 @@ def test_java_import_from_jar(self): def test_java_class(self): from java.lang import Integer, Number, NumberFormatException - self.assertEqual(type(Integer).mro(), [polyglot.ForeignClass, polyglot.ForeignInstantiable, polyglot.ForeignAbstractClass, polyglot.ForeignObject, object]) + self.assertEqual(type(Integer).mro(), [polyglot.ForeignClass, polyglot.ForeignAbstractClass, polyglot.ForeignInstantiable, polyglot.ForeignObject, object]) self.assertEqual(type(Number).mro(), [polyglot.ForeignAbstractClass, polyglot.ForeignObject, object]) - self.assertEqual(type(NumberFormatException).mro(), [polyglot.ForeignClass, polyglot.ForeignInstantiable, polyglot.ForeignAbstractClass, polyglot.ForeignObject, object]) + self.assertEqual(type(NumberFormatException).mro(), [polyglot.ForeignClass, polyglot.ForeignAbstractClass, polyglot.ForeignInstantiable, polyglot.ForeignObject, object]) def test_java_exceptions(self): # TODO: more tests diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/object/GetForeignObjectClassNode.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/object/GetForeignObjectClassNode.java index b40c4afbcf..9bf4d52ea1 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/object/GetForeignObjectClassNode.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/object/GetForeignObjectClassNode.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -87,21 +87,26 @@ public enum Trait { // The type field is only set for cases which are already implemented. // First in MRO + // Interop types first as they are the most concrete/specific types + NULL("None", PythonBuiltinClassType.PNone), BOOLEAN("Boolean", PythonBuiltinClassType.ForeignBoolean), NUMBER("Number", PythonBuiltinClassType.ForeignNumber), // int, float, complex STRING("String", PythonBuiltinClassType.PString), + EXCEPTION("Exception", PythonBuiltinClassType.PBaseException), + META_OBJECT("AbstractClass"), // PythonBuiltinClassType.PythonClass ? + + // Interop traits + EXECUTABLE("Executable"), + INSTANTIABLE("Instantiable"), + + // Container traits/types must be last, see comment above // Hash before Array so that foreign dict+list prefers dict.[] HASH("Dict", PythonBuiltinClassType.PDict), // Array before Iterable so that foreign list+iterable prefers list.__iter__ ARRAY("List", PythonBuiltinClassType.PList), - EXCEPTION("Exception", PythonBuiltinClassType.PBaseException), - EXECUTABLE("Executable"), - INSTANTIABLE("Instantiable"), // Iterator before Iterable so that foreign iterator+iterable prefers iterator.__iter__ ITERATOR("Iterator", PythonBuiltinClassType.PIterator), - ITERABLE("Iterable"), - META_OBJECT("AbstractClass"), // PythonBuiltinClassType.PythonClass ? - NULL("None", PythonBuiltinClassType.PNone); + ITERABLE("Iterable"); // Last in MRO public static final Trait[] VALUES = Trait.values(); @@ -217,14 +222,14 @@ private PythonManagedClass resolvePolyglotForeignClass(int traits) { traitsList.add(classForTraits(trait.bit)); } - if (trait == Trait.INSTANTIABLE && Trait.META_OBJECT.isSet(traits)) { - // Deal with it when we are at trait META_OBJECT - } else if (trait == Trait.META_OBJECT) { + if (trait == Trait.META_OBJECT) { if (Trait.INSTANTIABLE.isSet(traits)) { nameBuilder.append("Class"); } else { nameBuilder.append("AbstractClass"); } + } else if (trait == Trait.INSTANTIABLE && Trait.META_OBJECT.isSet(traits)) { + // Dealt with above } else { nameBuilder.append(trait.name); } From 17c4bf0a37d29af3e72c99336c15de4cfc3f9cbf Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Fri, 17 Jan 2025 16:54:29 +0100 Subject: [PATCH 07/82] Move __bases__ and __instancecheck__ from ForeignObjectBuiltins to ForeignAbstractClassBuiltins (cherry picked from commit f69bb094614ffc48fcc0a32fda8b7d69b354b8fb) --- .../graal/python/builtins/Python3Core.java | 2 + .../builtins/PythonBuiltinClassType.java | 2 + .../foreign/ForeignAbstractClassBuiltins.java | 94 +++++++++++++++++++ .../foreign/ForeignObjectBuiltins.java | 51 +--------- .../object/GetForeignObjectClassNode.java | 2 +- mx.graalpython/copyrights/overrides | 1 + 6 files changed, 101 insertions(+), 51 deletions(-) create mode 100644 graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignAbstractClassBuiltins.java diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java index 633acb7e4c..8351695719 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java @@ -259,6 +259,7 @@ import com.oracle.graal.python.builtins.objects.exception.UnicodeTranslateErrorBuiltins; import com.oracle.graal.python.builtins.objects.floats.FloatBuiltins; import com.oracle.graal.python.builtins.objects.floats.PFloat; +import com.oracle.graal.python.builtins.objects.foreign.ForeignAbstractClassBuiltins; import com.oracle.graal.python.builtins.objects.foreign.ForeignBooleanBuiltins; import com.oracle.graal.python.builtins.objects.foreign.ForeignNumberBuiltins; import com.oracle.graal.python.builtins.objects.foreign.ForeignObjectBuiltins; @@ -487,6 +488,7 @@ private static PythonBuiltins[] initializeBuiltins(boolean nativeAccessAllowed, new ForeignObjectBuiltins(), new ForeignNumberBuiltins(), new ForeignBooleanBuiltins(), + new ForeignAbstractClassBuiltins(), new ListBuiltins(), new DictBuiltins(), new DictReprBuiltin(), diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/PythonBuiltinClassType.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/PythonBuiltinClassType.java index cc3d1de44a..b4d755082a 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/PythonBuiltinClassType.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/PythonBuiltinClassType.java @@ -304,6 +304,7 @@ public enum PythonBuiltinClassType implements TruffleObject { ForeignObject("ForeignObject", J_POLYGLOT, Flags.PUBLIC_BASE_WDICT, ForeignObjectBuiltins.SLOTS), ForeignNumber("ForeignNumber", J_POLYGLOT, Flags.PUBLIC_BASE_WDICT, FOREIGNNUMBER_M_FLAGS, ForeignNumberBuiltins.SLOTS), ForeignBoolean("ForeignBoolean", J_POLYGLOT, Flags.PUBLIC_BASE_WDICT, FOREIGNNUMBER_M_FLAGS, ForeignBooleanBuiltins.SLOTS), + ForeignAbstractClass("ForeignAbstractClass", J_POLYGLOT, Flags.PUBLIC_BASE_WDICT), // bz2 BZ2Compressor("BZ2Compressor", "_bz2"), @@ -839,6 +840,7 @@ public final Shape getInstanceShape(PythonLanguage lang) { ForeignNumber.base = ForeignObject; ForeignBoolean.base = ForeignNumber; + ForeignAbstractClass.base = ForeignObject; PBaseExceptionGroup.base = PBaseException; SystemExit.base = PBaseException; diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignAbstractClassBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignAbstractClassBuiltins.java new file mode 100644 index 0000000000..adde80b51e --- /dev/null +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignAbstractClassBuiltins.java @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2017, 2025, Oracle and/or its affiliates. + * Copyright (c) 2014, Regents of the University of California + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are + * permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS + * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE + * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package com.oracle.graal.python.builtins.objects.foreign; + +import com.oracle.graal.python.builtins.Builtin; +import com.oracle.graal.python.builtins.CoreFunctions; +import com.oracle.graal.python.builtins.PythonBuiltinClassType; +import com.oracle.graal.python.builtins.PythonBuiltins; +import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode; +import com.oracle.graal.python.nodes.function.builtins.PythonBinaryBuiltinNode; +import com.oracle.graal.python.nodes.function.builtins.PythonUnaryBuiltinNode; +import com.oracle.graal.python.runtime.GilNode; +import com.oracle.graal.python.runtime.object.PythonObjectFactory; +import com.oracle.graal.python.util.PythonUtils; +import com.oracle.truffle.api.CompilerDirectives; +import com.oracle.truffle.api.dsl.Cached; +import com.oracle.truffle.api.dsl.GenerateNodeFactory; +import com.oracle.truffle.api.dsl.NodeFactory; +import com.oracle.truffle.api.dsl.Specialization; +import com.oracle.truffle.api.interop.InteropLibrary; +import com.oracle.truffle.api.interop.UnsupportedMessageException; +import com.oracle.truffle.api.library.CachedLibrary; + +import java.util.List; + +import static com.oracle.graal.python.nodes.SpecialAttributeNames.J___BASES__; +import static com.oracle.graal.python.nodes.SpecialMethodNames.J___INSTANCECHECK__; + +/* + * NOTE: We are not using IndirectCallContext here in this file (except for CallNode) + * because it seems unlikely that these interop messages would call back to Python + * and that we would also need precise frame info for that case. + * Adding it shouldn't hurt peak, but might be a non-trivial overhead in interpreter. + */ +@CoreFunctions(extendClasses = PythonBuiltinClassType.ForeignAbstractClass) +public final class ForeignAbstractClassBuiltins extends PythonBuiltins { + @Override + protected List> getNodeFactories() { + return ForeignAbstractClassBuiltinsFactory.getFactories(); + } + + @Builtin(name = J___BASES__, minNumOfPositionalArgs = 1, isGetter = true, isSetter = false) + @GenerateNodeFactory + abstract static class BasesNode extends PythonUnaryBuiltinNode { + @Specialization + static Object getBases(Object self, + @Cached PythonObjectFactory factory) { + return factory.createTuple(PythonUtils.EMPTY_OBJECT_ARRAY); + } + } + + @Builtin(name = J___INSTANCECHECK__, minNumOfPositionalArgs = 2) + @GenerateNodeFactory + abstract static class InstancecheckNode extends PythonBinaryBuiltinNode { + @Specialization(limit = "3") + static Object check(Object self, Object instance, + @CachedLibrary("self") InteropLibrary lib, + @Cached GilNode gil) { + gil.release(true); + try { + return lib.isMetaInstance(self, instance); + } catch (UnsupportedMessageException e) { + throw CompilerDirectives.shouldNotReachHere(); + } finally { + gil.acquire(); + } + } + } + +} diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignObjectBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignObjectBuiltins.java index 617458040f..19ad62ff6f 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignObjectBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignObjectBuiltins.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2024, Oracle and/or its affiliates. + * Copyright (c) 2017, 2025, Oracle and/or its affiliates. * Copyright (c) 2014, Regents of the University of California * * All rights reserved. @@ -29,17 +29,13 @@ import static com.oracle.graal.python.builtins.PythonBuiltinClassType.AttributeError; import static com.oracle.graal.python.builtins.PythonBuiltinClassType.TypeError; import static com.oracle.graal.python.builtins.objects.str.StringUtils.simpleTruffleStringFormatUncached; -import static com.oracle.graal.python.nodes.SpecialAttributeNames.J___BASES__; -import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___BASES__; import static com.oracle.graal.python.nodes.SpecialMethodNames.J___CALL__; import static com.oracle.graal.python.nodes.SpecialMethodNames.J___DIR__; import static com.oracle.graal.python.nodes.SpecialMethodNames.J___HASH__; -import static com.oracle.graal.python.nodes.SpecialMethodNames.J___INSTANCECHECK__; import static com.oracle.graal.python.nodes.SpecialMethodNames.J___ITER__; import static com.oracle.graal.python.nodes.SpecialMethodNames.J___NEW__; import static com.oracle.graal.python.nodes.SpecialMethodNames.J___REPR__; import static com.oracle.graal.python.nodes.SpecialMethodNames.J___STR__; -import static com.oracle.graal.python.nodes.SpecialMethodNames.T___INSTANCECHECK__; import static com.oracle.graal.python.util.PythonUtils.TS_ENCODING; import java.util.List; @@ -66,7 +62,6 @@ import com.oracle.graal.python.nodes.call.special.LookupAndCallUnaryNode; import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode; import com.oracle.graal.python.nodes.function.PythonBuiltinNode; -import com.oracle.graal.python.nodes.function.builtins.PythonBinaryBuiltinNode; import com.oracle.graal.python.nodes.function.builtins.PythonUnaryBuiltinNode; import com.oracle.graal.python.nodes.interop.PForeignToPTypeNode; import com.oracle.graal.python.nodes.object.BuiltinClassProfiles.IsBuiltinObjectProfile; @@ -82,7 +77,6 @@ import com.oracle.graal.python.runtime.exception.PException; import com.oracle.graal.python.runtime.exception.PythonErrorType; import com.oracle.graal.python.runtime.object.PythonObjectFactory; -import com.oracle.graal.python.util.PythonUtils; import com.oracle.truffle.api.CompilerDirectives; import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; import com.oracle.truffle.api.dsl.Bind; @@ -497,47 +491,4 @@ protected TruffleString defaultConversion(VirtualFrame frame, InteropLibrary lib } } - @Builtin(name = J___BASES__, minNumOfPositionalArgs = 1, isGetter = true, isSetter = false) - @GenerateNodeFactory - @ImportStatic(PGuards.class) - abstract static class BasesNode extends PythonUnaryBuiltinNode { - @Specialization(limit = "3") - static Object getBases(Object self, - @Bind("this") Node inliningTarget, - @CachedLibrary("self") InteropLibrary lib, - @Cached PythonObjectFactory factory, - @Cached PRaiseNode.Lazy raiseNode) { - if (lib.isMetaObject(self)) { - return factory.createTuple(PythonUtils.EMPTY_OBJECT_ARRAY); - } else { - throw raiseNode.get(inliningTarget).raise(AttributeError, ErrorMessages.FOREIGN_OBJ_HAS_NO_ATTR_S, T___BASES__); - } - } - } - - @Builtin(name = J___INSTANCECHECK__, minNumOfPositionalArgs = 2) - @GenerateNodeFactory - @ImportStatic(PGuards.class) - abstract static class InstancecheckNode extends PythonBinaryBuiltinNode { - @Specialization(limit = "3") - static Object check(Object self, Object instance, - @Bind("this") Node inliningTarget, - @CachedLibrary("self") InteropLibrary lib, - @Cached GilNode gil, - @Cached PRaiseNode.Lazy raiseNode) { - if (lib.isMetaObject(self)) { - gil.release(true); - try { - return lib.isMetaInstance(self, instance); - } catch (UnsupportedMessageException e) { - throw CompilerDirectives.shouldNotReachHere(); - } finally { - gil.acquire(); - } - } else { - throw raiseNode.get(inliningTarget).raise(AttributeError, ErrorMessages.FOREIGN_OBJ_HAS_NO_ATTR_S, T___INSTANCECHECK__); - } - } - } - } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/object/GetForeignObjectClassNode.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/object/GetForeignObjectClassNode.java index 9bf4d52ea1..e7cb56977d 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/object/GetForeignObjectClassNode.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/object/GetForeignObjectClassNode.java @@ -93,7 +93,7 @@ public enum Trait { NUMBER("Number", PythonBuiltinClassType.ForeignNumber), // int, float, complex STRING("String", PythonBuiltinClassType.PString), EXCEPTION("Exception", PythonBuiltinClassType.PBaseException), - META_OBJECT("AbstractClass"), // PythonBuiltinClassType.PythonClass ? + META_OBJECT("AbstractClass", PythonBuiltinClassType.ForeignAbstractClass), // Interop traits EXECUTABLE("Executable"), diff --git a/mx.graalpython/copyrights/overrides b/mx.graalpython/copyrights/overrides index 9b09b5c78f..0a1351458c 100644 --- a/mx.graalpython/copyrights/overrides +++ b/mx.graalpython/copyrights/overrides @@ -617,6 +617,7 @@ graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/floats/FloatBuiltins.java,zippy.copyright graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/floats/PFloat.java,zippy.copyright graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignObjectBuiltins.java,zippy.copyright +graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignAbstractClassBuiltins.java,zippy.copyright graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignBooleanBuiltins.java,zippy.copyright graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignNumberBuiltins.java,zippy.copyright graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/frame/FrameBuiltins.java,zippy.copyright From c5ab7ea52787e1c05318bc4aba10686868973439 Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Mon, 27 Jan 2025 16:17:22 +0100 Subject: [PATCH 08/82] Make it easier to set the base in PythonBuiltinClassType (cherry picked from commit bcd576ff22da723adb7c90a6188dce6b4a943198) --- .../builtins/PythonBuiltinClassType.java | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/PythonBuiltinClassType.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/PythonBuiltinClassType.java index b4d755082a..b7a4e04a27 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/PythonBuiltinClassType.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/PythonBuiltinClassType.java @@ -302,9 +302,9 @@ public enum PythonBuiltinClassType implements TruffleObject { // Foreign ForeignObject("ForeignObject", J_POLYGLOT, Flags.PUBLIC_BASE_WDICT, ForeignObjectBuiltins.SLOTS), - ForeignNumber("ForeignNumber", J_POLYGLOT, Flags.PUBLIC_BASE_WDICT, FOREIGNNUMBER_M_FLAGS, ForeignNumberBuiltins.SLOTS), - ForeignBoolean("ForeignBoolean", J_POLYGLOT, Flags.PUBLIC_BASE_WDICT, FOREIGNNUMBER_M_FLAGS, ForeignBooleanBuiltins.SLOTS), - ForeignAbstractClass("ForeignAbstractClass", J_POLYGLOT, Flags.PUBLIC_BASE_WDICT), + ForeignNumber("ForeignNumber", J_POLYGLOT, ForeignObject, Flags.PUBLIC_BASE_WDICT, FOREIGNNUMBER_M_FLAGS, ForeignNumberBuiltins.SLOTS), + ForeignBoolean("ForeignBoolean", J_POLYGLOT, ForeignNumber, Flags.PUBLIC_BASE_WDICT, FOREIGNNUMBER_M_FLAGS, ForeignBooleanBuiltins.SLOTS), + ForeignAbstractClass("ForeignAbstractClass", J_POLYGLOT, ForeignObject, Flags.PUBLIC_BASE_WDICT), // bz2 BZ2Compressor("BZ2Compressor", "_bz2"), @@ -628,6 +628,11 @@ private static class Flags { this(name, module, module, flags); } + PythonBuiltinClassType(String name, String module, PythonBuiltinClassType base, Flags flags) { + this(name, module, module, flags); + this.base = base; + } + PythonBuiltinClassType(String name, String module, Flags flags, TpSlots slots) { this(name, module, module, flags, DEFAULT_M_FLAGS, slots); } @@ -640,6 +645,11 @@ private static class Flags { this(name, module, module, flags, methodsFlags, slots); } + PythonBuiltinClassType(String name, String module, PythonBuiltinClassType base, Flags flags, long methodsFlags, TpSlots slots) { + this(name, module, module, flags, methodsFlags, slots); + this.base = base; + } + PythonBuiltinClassType(String name, String publishInModule, String moduleName, Flags flags) { this(name, publishInModule, moduleName, flags, DEFAULT_M_FLAGS, TpSlots.createEmpty()); } @@ -838,10 +848,6 @@ public final Shape getInstanceShape(PythonLanguage lang) { Boolean.base = PInt; - ForeignNumber.base = ForeignObject; - ForeignBoolean.base = ForeignNumber; - ForeignAbstractClass.base = ForeignObject; - PBaseExceptionGroup.base = PBaseException; SystemExit.base = PBaseException; KeyboardInterrupt.base = PBaseException; From 05ec2d779f8a7fe6abec0d73bbb268202624774c Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Mon, 27 Jan 2025 16:49:54 +0100 Subject: [PATCH 09/82] Move the remaining trait-specific methods from ForeignObject to the dedicated trait classes (cherry picked from commit 60b82ae3527d9737d7fd41ca0c0eda02f2ba7b62) --- .../src/tests/test_interop.py | 4 +- .../graal/python/builtins/Python3Core.java | 6 + .../builtins/PythonBuiltinClassType.java | 3 + .../foreign/ForeignAbstractClassBuiltins.java | 2 +- .../foreign/ForeignExecutableBuiltins.java | 110 ++++++++++++++ .../foreign/ForeignInstantiableBuiltins.java | 143 ++++++++++++++++++ .../foreign/ForeignIterableBuiltins.java | 83 ++++++++++ .../foreign/ForeignObjectBuiltins.java | 117 +------------- .../graal/python/nodes/ErrorMessages.java | 1 - .../object/GetForeignObjectClassNode.java | 10 +- mx.graalpython/copyrights/overrides | 3 + 11 files changed, 356 insertions(+), 126 deletions(-) create mode 100644 graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignExecutableBuiltins.java create mode 100644 graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignInstantiableBuiltins.java create mode 100644 graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignIterableBuiltins.java diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_interop.py b/graalpython/com.oracle.graal.python.test/src/tests/test_interop.py index d3b511aea6..c065cacbd8 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_interop.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_interop.py @@ -1040,7 +1040,9 @@ def test_java_map(self): h.__init__(a=1, b=2) assert h == {'a': 1, 'b': 2} - with self.assertRaisesRegex(TypeError, 'invalid instantiation of foreign object'): + # Because it tries to call ForeignDict.__call__, but ForeignDict is not executable/instantiable, + # so it resolves to type.__call__, which cannot create a ForeignDict + with self.assertRaisesRegex(TypeError, "descriptor requires a 'dict' object but received a 'ForeignDict'"): type(h).fromkeys(['a', 'b'], 42) def test_java_iterator(self): diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java index 8351695719..6f6254b9d8 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java @@ -59,6 +59,9 @@ import java.util.ServiceLoader; import java.util.logging.Level; +import com.oracle.graal.python.builtins.objects.foreign.ForeignExecutableBuiltins; +import com.oracle.graal.python.builtins.objects.foreign.ForeignInstantiableBuiltins; +import com.oracle.graal.python.builtins.objects.foreign.ForeignIterableBuiltins; import org.graalvm.nativeimage.ImageInfo; import com.oracle.graal.python.PythonLanguage; @@ -489,6 +492,9 @@ private static PythonBuiltins[] initializeBuiltins(boolean nativeAccessAllowed, new ForeignNumberBuiltins(), new ForeignBooleanBuiltins(), new ForeignAbstractClassBuiltins(), + new ForeignExecutableBuiltins(), + new ForeignInstantiableBuiltins(), + new ForeignIterableBuiltins(), new ListBuiltins(), new DictBuiltins(), new DictReprBuiltin(), diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/PythonBuiltinClassType.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/PythonBuiltinClassType.java index b7a4e04a27..8d586cc758 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/PythonBuiltinClassType.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/PythonBuiltinClassType.java @@ -305,6 +305,9 @@ public enum PythonBuiltinClassType implements TruffleObject { ForeignNumber("ForeignNumber", J_POLYGLOT, ForeignObject, Flags.PUBLIC_BASE_WDICT, FOREIGNNUMBER_M_FLAGS, ForeignNumberBuiltins.SLOTS), ForeignBoolean("ForeignBoolean", J_POLYGLOT, ForeignNumber, Flags.PUBLIC_BASE_WDICT, FOREIGNNUMBER_M_FLAGS, ForeignBooleanBuiltins.SLOTS), ForeignAbstractClass("ForeignAbstractClass", J_POLYGLOT, ForeignObject, Flags.PUBLIC_BASE_WDICT), + ForeignExecutable("ForeignExecutable", J_POLYGLOT, ForeignObject, Flags.PUBLIC_BASE_WDICT), + ForeignInstantiable("ForeignInstantiable", J_POLYGLOT, ForeignObject, Flags.PUBLIC_BASE_WDICT), + ForeignIterable("ForeignIterable", J_POLYGLOT, ForeignObject, Flags.PUBLIC_BASE_WDICT), // bz2 BZ2Compressor("BZ2Compressor", "_bz2"), diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignAbstractClassBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignAbstractClassBuiltins.java index adde80b51e..48cfd68373 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignAbstractClassBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignAbstractClassBuiltins.java @@ -51,7 +51,7 @@ import static com.oracle.graal.python.nodes.SpecialMethodNames.J___INSTANCECHECK__; /* - * NOTE: We are not using IndirectCallContext here in this file (except for CallNode) + * NOTE: We are not using IndirectCallContext here in this file * because it seems unlikely that these interop messages would call back to Python * and that we would also need precise frame info for that case. * Adding it shouldn't hurt peak, but might be a non-trivial overhead in interpreter. diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignExecutableBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignExecutableBuiltins.java new file mode 100644 index 0000000000..22c632fac0 --- /dev/null +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignExecutableBuiltins.java @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2017, 2025, Oracle and/or its affiliates. + * Copyright (c) 2014, Regents of the University of California + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are + * permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS + * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE + * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package com.oracle.graal.python.builtins.objects.foreign; + +import static com.oracle.graal.python.nodes.SpecialMethodNames.J___CALL__; + +import java.util.List; + +import com.oracle.graal.python.PythonLanguage; +import com.oracle.graal.python.builtins.Builtin; +import com.oracle.graal.python.builtins.CoreFunctions; +import com.oracle.graal.python.builtins.PythonBuiltinClassType; +import com.oracle.graal.python.builtins.PythonBuiltins; +import com.oracle.graal.python.builtins.objects.function.PKeyword; +import com.oracle.graal.python.nodes.ErrorMessages; +import com.oracle.graal.python.nodes.PRaiseNode; +import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode; +import com.oracle.graal.python.nodes.function.PythonBuiltinNode; +import com.oracle.graal.python.nodes.interop.PForeignToPTypeNode; +import com.oracle.graal.python.nodes.object.IsForeignObjectNode; +import com.oracle.graal.python.runtime.ExecutionContext.IndirectCallContext; +import com.oracle.graal.python.runtime.GilNode; +import com.oracle.graal.python.runtime.IndirectCallData; +import com.oracle.graal.python.runtime.PythonContext; +import com.oracle.graal.python.runtime.exception.PythonErrorType; +import com.oracle.truffle.api.CompilerDirectives; +import com.oracle.truffle.api.dsl.Bind; +import com.oracle.truffle.api.dsl.Cached; +import com.oracle.truffle.api.dsl.Fallback; +import com.oracle.truffle.api.dsl.GenerateNodeFactory; +import com.oracle.truffle.api.dsl.NodeFactory; +import com.oracle.truffle.api.dsl.Specialization; +import com.oracle.truffle.api.frame.VirtualFrame; +import com.oracle.truffle.api.interop.ArityException; +import com.oracle.truffle.api.interop.InteropLibrary; +import com.oracle.truffle.api.interop.UnsupportedMessageException; +import com.oracle.truffle.api.interop.UnsupportedTypeException; +import com.oracle.truffle.api.library.CachedLibrary; +import com.oracle.truffle.api.nodes.Node; + +@CoreFunctions(extendClasses = PythonBuiltinClassType.ForeignExecutable) +public final class ForeignExecutableBuiltins extends PythonBuiltins { + @Override + protected List> getNodeFactories() { + return ForeignExecutableBuiltinsFactory.getFactories(); + } + + @Builtin(name = J___CALL__, minNumOfPositionalArgs = 1, takesVarArgs = true, takesVarKeywordArgs = true) + @GenerateNodeFactory + public abstract static class CallNode extends PythonBuiltinNode { + @Specialization(guards = {"isForeignObjectNode.execute(inliningTarget, callee)", "!isNoValue(callee)", "keywords.length == 0"}, limit = "1") + static Object doInteropCall(VirtualFrame frame, Object callee, Object[] arguments, @SuppressWarnings("unused") PKeyword[] keywords, + @SuppressWarnings("unused") @Bind("this") Node inliningTarget, + @Cached("createFor(this)") IndirectCallData indirectCallData, + @SuppressWarnings("unused") @Cached IsForeignObjectNode isForeignObjectNode, + @CachedLibrary(limit = "4") InteropLibrary lib, + @Cached PForeignToPTypeNode toPTypeNode, + @Cached GilNode gil, + @Cached PRaiseNode.Lazy raiseNode) { + PythonLanguage language = PythonLanguage.get(inliningTarget); + PythonContext context = PythonContext.get(inliningTarget); + try { + Object state = IndirectCallContext.enter(frame, language, context, indirectCallData); + gil.release(true); + try { + return toPTypeNode.executeConvert(lib.execute(callee, arguments)); + } finally { + gil.acquire(); + IndirectCallContext.exit(frame, language, context, state); + } + } catch (ArityException | UnsupportedTypeException e) { + throw raiseNode.get(inliningTarget).raise(PythonErrorType.TypeError, ErrorMessages.INVALID_INSTANTIATION_OF_FOREIGN_OBJ); + } catch (UnsupportedMessageException e) { + throw CompilerDirectives.shouldNotReachHere(e); + } + } + + @Fallback + @SuppressWarnings("unused") + static Object doGeneric(Object callee, Object arguments, Object keywords, + @Cached PRaiseNode raiseNode) { + throw raiseNode.raise(PythonErrorType.TypeError, ErrorMessages.INVALID_INSTANTIATION_OF_FOREIGN_OBJ); + } + } + +} diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignInstantiableBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignInstantiableBuiltins.java new file mode 100644 index 0000000000..7ab0b0d664 --- /dev/null +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignInstantiableBuiltins.java @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2017, 2025, Oracle and/or its affiliates. + * Copyright (c) 2014, Regents of the University of California + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are + * permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS + * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE + * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package com.oracle.graal.python.builtins.objects.foreign; + +import static com.oracle.graal.python.nodes.SpecialMethodNames.J___CALL__; +import static com.oracle.graal.python.nodes.SpecialMethodNames.J___NEW__; + +import java.util.List; + +import com.oracle.graal.python.PythonLanguage; +import com.oracle.graal.python.builtins.Builtin; +import com.oracle.graal.python.builtins.CoreFunctions; +import com.oracle.graal.python.builtins.PythonBuiltinClassType; +import com.oracle.graal.python.builtins.PythonBuiltins; +import com.oracle.graal.python.builtins.objects.function.PKeyword; +import com.oracle.graal.python.nodes.ErrorMessages; +import com.oracle.graal.python.nodes.PRaiseNode; +import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode; +import com.oracle.graal.python.nodes.function.PythonBuiltinNode; +import com.oracle.graal.python.nodes.interop.PForeignToPTypeNode; +import com.oracle.graal.python.nodes.object.IsForeignObjectNode; +import com.oracle.graal.python.runtime.ExecutionContext.IndirectCallContext; +import com.oracle.graal.python.runtime.GilNode; +import com.oracle.graal.python.runtime.IndirectCallData; +import com.oracle.graal.python.runtime.PythonContext; +import com.oracle.graal.python.runtime.exception.PythonErrorType; +import com.oracle.truffle.api.CompilerDirectives; +import com.oracle.truffle.api.dsl.Bind; +import com.oracle.truffle.api.dsl.Cached; +import com.oracle.truffle.api.dsl.Fallback; +import com.oracle.truffle.api.dsl.GenerateNodeFactory; +import com.oracle.truffle.api.dsl.NodeFactory; +import com.oracle.truffle.api.dsl.Specialization; +import com.oracle.truffle.api.frame.VirtualFrame; +import com.oracle.truffle.api.interop.ArityException; +import com.oracle.truffle.api.interop.InteropLibrary; +import com.oracle.truffle.api.interop.UnsupportedMessageException; +import com.oracle.truffle.api.interop.UnsupportedTypeException; +import com.oracle.truffle.api.library.CachedLibrary; +import com.oracle.truffle.api.nodes.Node; + +@CoreFunctions(extendClasses = PythonBuiltinClassType.ForeignInstantiable) +public final class ForeignInstantiableBuiltins extends PythonBuiltins { + @Override + protected List> getNodeFactories() { + return ForeignInstantiableBuiltinsFactory.getFactories(); + } + + @Builtin(name = J___NEW__, minNumOfPositionalArgs = 1, takesVarArgs = true, takesVarKeywordArgs = true) + @GenerateNodeFactory + abstract static class NewNode extends PythonBuiltinNode { + @Specialization(guards = {"isForeignObjectNode.execute(inliningTarget, callee)", "!isNoValue(callee)", "keywords.length == 0"}, limit = "1") + static Object doInteropCall(Object callee, Object[] arguments, @SuppressWarnings("unused") PKeyword[] keywords, + @SuppressWarnings("unused") @Bind("this") Node inliningTarget, + @SuppressWarnings("unused") @Cached IsForeignObjectNode isForeignObjectNode, + @CachedLibrary(limit = "3") InteropLibrary lib, + @Cached PForeignToPTypeNode toPTypeNode, + @Cached GilNode gil, + @Cached PRaiseNode.Lazy raiseNode) { + gil.release(true); + try { + Object res = lib.instantiate(callee, arguments); + return toPTypeNode.executeConvert(res); + } catch (ArityException | UnsupportedTypeException e) { + throw raiseNode.get(inliningTarget).raise(PythonErrorType.TypeError, ErrorMessages.INVALID_INSTANTIATION_OF_FOREIGN_OBJ); + } catch (UnsupportedMessageException e) { + throw CompilerDirectives.shouldNotReachHere(e); + } finally { + gil.acquire(); + } + } + + @Fallback + @SuppressWarnings("unused") + static Object doGeneric(Object callee, Object arguments, Object keywords, + @Cached PRaiseNode raiseNode) { + throw raiseNode.raise(PythonErrorType.TypeError, ErrorMessages.INVALID_INSTANTIATION_OF_FOREIGN_OBJ); + } + } + + @Builtin(name = J___CALL__, minNumOfPositionalArgs = 1, takesVarArgs = true, takesVarKeywordArgs = true) + @GenerateNodeFactory + public abstract static class CallNode extends PythonBuiltinNode { + @Specialization(guards = {"isForeignObjectNode.execute(inliningTarget, callee)", "!isNoValue(callee)", "keywords.length == 0"}, limit = "1") + static Object doInteropCall(VirtualFrame frame, Object callee, Object[] arguments, @SuppressWarnings("unused") PKeyword[] keywords, + @SuppressWarnings("unused") @Bind("this") Node inliningTarget, + @Cached("createFor(this)") IndirectCallData indirectCallData, + @SuppressWarnings("unused") @Cached IsForeignObjectNode isForeignObjectNode, + @CachedLibrary(limit = "4") InteropLibrary lib, + @Cached PForeignToPTypeNode toPTypeNode, + @Cached GilNode gil, + @Cached PRaiseNode.Lazy raiseNode) { + PythonLanguage language = PythonLanguage.get(inliningTarget); + PythonContext context = PythonContext.get(inliningTarget); + try { + Object state = IndirectCallContext.enter(frame, language, context, indirectCallData); + gil.release(true); + try { + return toPTypeNode.executeConvert(lib.instantiate(callee, arguments)); + } finally { + gil.acquire(); + IndirectCallContext.exit(frame, language, context, state); + } + } catch (ArityException | UnsupportedTypeException e) { + throw raiseNode.get(inliningTarget).raise(PythonErrorType.TypeError, ErrorMessages.INVALID_INSTANTIATION_OF_FOREIGN_OBJ); + } catch (UnsupportedMessageException e) { + throw CompilerDirectives.shouldNotReachHere(e); + } + } + + @Fallback + @SuppressWarnings("unused") + static Object doGeneric(Object callee, Object arguments, Object keywords, + @Cached PRaiseNode raiseNode) { + throw raiseNode.raise(PythonErrorType.TypeError, ErrorMessages.INVALID_INSTANTIATION_OF_FOREIGN_OBJ); + } + } + +} diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignIterableBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignIterableBuiltins.java new file mode 100644 index 0000000000..647eee6b2f --- /dev/null +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignIterableBuiltins.java @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2017, 2025, Oracle and/or its affiliates. + * Copyright (c) 2014, Regents of the University of California + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are + * permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS + * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE + * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package com.oracle.graal.python.builtins.objects.foreign; + +import static com.oracle.graal.python.nodes.SpecialMethodNames.J___ITER__; + +import java.util.List; + +import com.oracle.graal.python.builtins.Builtin; +import com.oracle.graal.python.builtins.CoreFunctions; +import com.oracle.graal.python.builtins.PythonBuiltinClassType; +import com.oracle.graal.python.builtins.PythonBuiltins; +import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode; +import com.oracle.graal.python.nodes.function.builtins.PythonUnaryBuiltinNode; +import com.oracle.graal.python.nodes.interop.PForeignToPTypeNode; +import com.oracle.graal.python.runtime.GilNode; +import com.oracle.truffle.api.CompilerDirectives; +import com.oracle.truffle.api.dsl.Cached; +import com.oracle.truffle.api.dsl.GenerateNodeFactory; +import com.oracle.truffle.api.dsl.NodeFactory; +import com.oracle.truffle.api.dsl.Specialization; +import com.oracle.truffle.api.interop.InteropLibrary; +import com.oracle.truffle.api.interop.UnsupportedMessageException; +import com.oracle.truffle.api.library.CachedLibrary; + +/* + * NOTE: We are not using IndirectCallContext here in this file + * because it seems unlikely that these interop messages would call back to Python + * and that we would also need precise frame info for that case. + * Adding it shouldn't hurt peak, but might be a non-trivial overhead in interpreter. + */ +@CoreFunctions(extendClasses = PythonBuiltinClassType.ForeignIterable) +public final class ForeignIterableBuiltins extends PythonBuiltins { + @Override + protected List> getNodeFactories() { + return ForeignIterableBuiltinsFactory.getFactories(); + } + + @Builtin(name = J___ITER__, minNumOfPositionalArgs = 1) + @GenerateNodeFactory + public abstract static class IterNode extends PythonUnaryBuiltinNode { + + @Specialization(limit = "3") + static Object doGeneric(Object object, + @CachedLibrary("object") InteropLibrary lib, + @Cached PForeignToPTypeNode convertNode, + @Cached GilNode gil) { + gil.release(true); + try { + return convertNode.executeConvert(lib.getIterator(object)); + } catch (UnsupportedMessageException e) { + throw CompilerDirectives.shouldNotReachHere(e); + } finally { + gil.acquire(); + } + } + } + +} diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignObjectBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignObjectBuiltins.java index 19ad62ff6f..ea386f7fed 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignObjectBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignObjectBuiltins.java @@ -29,11 +29,8 @@ import static com.oracle.graal.python.builtins.PythonBuiltinClassType.AttributeError; import static com.oracle.graal.python.builtins.PythonBuiltinClassType.TypeError; import static com.oracle.graal.python.builtins.objects.str.StringUtils.simpleTruffleStringFormatUncached; -import static com.oracle.graal.python.nodes.SpecialMethodNames.J___CALL__; import static com.oracle.graal.python.nodes.SpecialMethodNames.J___DIR__; import static com.oracle.graal.python.nodes.SpecialMethodNames.J___HASH__; -import static com.oracle.graal.python.nodes.SpecialMethodNames.J___ITER__; -import static com.oracle.graal.python.nodes.SpecialMethodNames.J___NEW__; import static com.oracle.graal.python.nodes.SpecialMethodNames.J___REPR__; import static com.oracle.graal.python.nodes.SpecialMethodNames.J___STR__; import static com.oracle.graal.python.util.PythonUtils.TS_ENCODING; @@ -49,7 +46,6 @@ import com.oracle.graal.python.builtins.PythonBuiltins; import com.oracle.graal.python.builtins.objects.PNone; import com.oracle.graal.python.builtins.objects.PythonAbstractObject; -import com.oracle.graal.python.builtins.objects.function.PKeyword; import com.oracle.graal.python.builtins.objects.object.ObjectBuiltins; import com.oracle.graal.python.builtins.objects.object.ObjectNodes; import com.oracle.graal.python.builtins.objects.type.TpSlots; @@ -61,17 +57,13 @@ import com.oracle.graal.python.nodes.attributes.LookupAttributeInMRONode; import com.oracle.graal.python.nodes.call.special.LookupAndCallUnaryNode; import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode; -import com.oracle.graal.python.nodes.function.PythonBuiltinNode; import com.oracle.graal.python.nodes.function.builtins.PythonUnaryBuiltinNode; import com.oracle.graal.python.nodes.interop.PForeignToPTypeNode; import com.oracle.graal.python.nodes.object.BuiltinClassProfiles.IsBuiltinObjectProfile; import com.oracle.graal.python.nodes.object.GetClassNode; -import com.oracle.graal.python.nodes.object.IsForeignObjectNode; import com.oracle.graal.python.nodes.util.CannotCastException; import com.oracle.graal.python.nodes.util.CastToJavaStringNode; -import com.oracle.graal.python.runtime.ExecutionContext.IndirectCallContext; import com.oracle.graal.python.runtime.GilNode; -import com.oracle.graal.python.runtime.IndirectCallData; import com.oracle.graal.python.runtime.PythonContext; import com.oracle.graal.python.runtime.PythonOptions; import com.oracle.graal.python.runtime.exception.PException; @@ -82,12 +74,10 @@ import com.oracle.truffle.api.dsl.Bind; import com.oracle.truffle.api.dsl.Cached; import com.oracle.truffle.api.dsl.Cached.Shared; -import com.oracle.truffle.api.dsl.Fallback; import com.oracle.truffle.api.dsl.GenerateCached; import com.oracle.truffle.api.dsl.GenerateInline; import com.oracle.truffle.api.dsl.GenerateNodeFactory; import com.oracle.truffle.api.dsl.ImportStatic; -import com.oracle.truffle.api.dsl.NeverDefault; import com.oracle.truffle.api.dsl.NodeFactory; import com.oracle.truffle.api.dsl.Specialization; import com.oracle.truffle.api.frame.VirtualFrame; @@ -102,7 +92,7 @@ import com.oracle.truffle.api.strings.TruffleString; /* - * NOTE: We are not using IndirectCallContext here in this file (except for CallNode) + * NOTE: We are not using IndirectCallContext here in this file * because it seems unlikely that these interop messages would call back to Python * and that we would also need precise frame info for that case. * Adding it shouldn't hurt peak, but might be a non-trivial overhead in interpreter. @@ -138,111 +128,6 @@ private static int hashCodeBoundary(Object self) { } } - @Builtin(name = J___ITER__, minNumOfPositionalArgs = 1) - @GenerateNodeFactory - public abstract static class IterNode extends PythonUnaryBuiltinNode { - - @Specialization(limit = "3") - static Object doGeneric(Object object, - @Cached PRaiseNode raiseNode, - @CachedLibrary("object") InteropLibrary lib, - @Cached PForeignToPTypeNode convertNode, - @Cached GilNode gil) { - gil.release(true); - try { - if (lib.hasIterator(object)) { - return convertNode.executeConvert(lib.getIterator(object)); - } - } catch (UnsupportedMessageException e) { - throw CompilerDirectives.shouldNotReachHere(e); - } finally { - gil.acquire(); - } - throw raiseNode.raise(TypeError, ErrorMessages.FOREIGN_OBJ_ISNT_ITERABLE); - } - } - - @Builtin(name = J___NEW__, minNumOfPositionalArgs = 1, takesVarArgs = true, takesVarKeywordArgs = true) - @GenerateNodeFactory - abstract static class NewNode extends PythonBuiltinNode { - @Specialization(guards = {"isForeignObjectNode.execute(inliningTarget, callee)", "!isNoValue(callee)", "keywords.length == 0"}, limit = "1") - static Object doInteropCall(Object callee, Object[] arguments, @SuppressWarnings("unused") PKeyword[] keywords, - @SuppressWarnings("unused") @Bind("this") Node inliningTarget, - @SuppressWarnings("unused") @Cached IsForeignObjectNode isForeignObjectNode, - @CachedLibrary(limit = "3") InteropLibrary lib, - @Cached PForeignToPTypeNode toPTypeNode, - @Cached GilNode gil, - @Cached PRaiseNode.Lazy raiseNode) { - gil.release(true); - try { - Object res = lib.instantiate(callee, arguments); - return toPTypeNode.executeConvert(res); - } catch (ArityException | UnsupportedTypeException | UnsupportedMessageException e) { - throw raiseNode.get(inliningTarget).raise(PythonErrorType.TypeError, ErrorMessages.INVALID_INSTANTIATION_OF_FOREIGN_OBJ); - } finally { - gil.acquire(); - } - } - - @Fallback - @SuppressWarnings("unused") - static Object doGeneric(Object callee, Object arguments, Object keywords, - @Cached PRaiseNode raiseNode) { - throw raiseNode.raise(PythonErrorType.TypeError, ErrorMessages.INVALID_INSTANTIATION_OF_FOREIGN_OBJ); - } - } - - @Builtin(name = J___CALL__, minNumOfPositionalArgs = 1, takesVarArgs = true, takesVarKeywordArgs = true) - @GenerateNodeFactory - public abstract static class CallNode extends PythonBuiltinNode { - public final Object executeWithArgs(VirtualFrame frame, Object callee, Object[] arguments) { - return execute(frame, callee, arguments, PKeyword.EMPTY_KEYWORDS); - } - - public abstract Object execute(VirtualFrame frame, Object callee, Object[] arguments, PKeyword[] keywords); - - @Specialization(guards = {"isForeignObjectNode.execute(inliningTarget, callee)", "!isNoValue(callee)", "keywords.length == 0"}, limit = "1") - static Object doInteropCall(VirtualFrame frame, Object callee, Object[] arguments, @SuppressWarnings("unused") PKeyword[] keywords, - @SuppressWarnings("unused") @Bind("this") Node inliningTarget, - @Cached("createFor(this)") IndirectCallData indirectCallData, - @SuppressWarnings("unused") @Cached IsForeignObjectNode isForeignObjectNode, - @CachedLibrary(limit = "4") InteropLibrary lib, - @Cached PForeignToPTypeNode toPTypeNode, - @Cached GilNode gil, - @Cached PRaiseNode.Lazy raiseNode) { - PythonLanguage language = PythonLanguage.get(inliningTarget); - PythonContext context = PythonContext.get(inliningTarget); - try { - Object state = IndirectCallContext.enter(frame, language, context, indirectCallData); - gil.release(true); - try { - if (lib.isExecutable(callee)) { - return toPTypeNode.executeConvert(lib.execute(callee, arguments)); - } else { - return toPTypeNode.executeConvert(lib.instantiate(callee, arguments)); - } - } finally { - gil.acquire(); - IndirectCallContext.exit(frame, language, context, state); - } - } catch (ArityException | UnsupportedTypeException | UnsupportedMessageException e) { - throw raiseNode.get(inliningTarget).raise(PythonErrorType.TypeError, ErrorMessages.INVALID_INSTANTIATION_OF_FOREIGN_OBJ); - } - } - - @Fallback - @SuppressWarnings("unused") - static Object doGeneric(Object callee, Object arguments, Object keywords, - @Cached PRaiseNode raiseNode) { - throw raiseNode.raise(PythonErrorType.TypeError, ErrorMessages.INVALID_INSTANTIATION_OF_FOREIGN_OBJ); - } - - @NeverDefault - public static CallNode create() { - return ForeignObjectBuiltinsFactory.CallNodeFactory.create(null); - } - } - @Slot(value = SlotKind.tp_getattro, isComplex = true) @GenerateNodeFactory abstract static class GetAttributeNode extends GetAttrBuiltinNode { diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/ErrorMessages.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/ErrorMessages.java index f15c1e4196..37dbb36db7 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/ErrorMessages.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/ErrorMessages.java @@ -334,7 +334,6 @@ public abstract class ErrorMessages { public static final TruffleString FAILED_TO_CONVERT_SEQ = tsLiteral("failed to convert sequence"); public static final TruffleString FLOAT_ARG_REQUIRED = tsLiteral("float argument required, not %p"); public static final TruffleString FOREIGN_OBJ_HAS_NO_ATTR_S = tsLiteral("foreign object has no attribute '%s'"); - public static final TruffleString FOREIGN_OBJ_ISNT_ITERABLE = tsLiteral("foreign object is not iterable"); public static final TruffleString FOREIGN_OBJ_ISNT_REVERSE_ITERABLE = tsLiteral("foreign object cannot be iterated in reverse"); public static final TruffleString FORMAT_REQUIRES_MAPPING = tsLiteral("format requires a mapping"); public static final TruffleString FORMAT_STR_CONTAINS_POS_FIELDS = tsLiteral("Format string contains positional fields"); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/object/GetForeignObjectClassNode.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/object/GetForeignObjectClassNode.java index e7cb56977d..b7b041b290 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/object/GetForeignObjectClassNode.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/object/GetForeignObjectClassNode.java @@ -96,8 +96,8 @@ public enum Trait { META_OBJECT("AbstractClass", PythonBuiltinClassType.ForeignAbstractClass), // Interop traits - EXECUTABLE("Executable"), - INSTANTIABLE("Instantiable"), + EXECUTABLE("Executable", PythonBuiltinClassType.ForeignExecutable), + INSTANTIABLE("Instantiable", PythonBuiltinClassType.ForeignInstantiable), // Container traits/types must be last, see comment above // Hash before Array so that foreign dict+list prefers dict.[] @@ -106,7 +106,7 @@ public enum Trait { ARRAY("List", PythonBuiltinClassType.PList), // Iterator before Iterable so that foreign iterator+iterable prefers iterator.__iter__ ITERATOR("Iterator", PythonBuiltinClassType.PIterator), - ITERABLE("Iterable"); + ITERABLE("Iterable", PythonBuiltinClassType.ForeignIterable); // Last in MRO public static final Trait[] VALUES = Trait.values(); @@ -116,10 +116,6 @@ public enum Trait { final int bit; final PythonBuiltinClassType type; - Trait(String name) { - this(name, null); - } - Trait(String name, PythonBuiltinClassType type) { this.name = name; this.bit = 1 << ordinal(); diff --git a/mx.graalpython/copyrights/overrides b/mx.graalpython/copyrights/overrides index 0a1351458c..1a30d74d25 100644 --- a/mx.graalpython/copyrights/overrides +++ b/mx.graalpython/copyrights/overrides @@ -619,6 +619,9 @@ graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignObjectBuiltins.java,zippy.copyright graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignAbstractClassBuiltins.java,zippy.copyright graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignBooleanBuiltins.java,zippy.copyright +graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignExecutableBuiltins.java,zippy.copyright +graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignInstantiableBuiltins.java,zippy.copyright +graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignIterableBuiltins.java,zippy.copyright graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignNumberBuiltins.java,zippy.copyright graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/frame/FrameBuiltins.java,zippy.copyright graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/function/AbstractFunctionBuiltins.java,zippy.copyright From 38c69a533e60b1517874c4daf5ba0eaa1a3fcb0a Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Mon, 27 Jan 2025 17:22:31 +0100 Subject: [PATCH 10/82] Simplify logic for foreign __new__ and __call__: they do not accept keyword arguments * So the error is now: TypeError: ForeignInstantiable.__call__() got an unexpected keyword argument 'kwarg' vs before: TypeError: invalid instantiation of foreign object (cherry picked from commit 95b076038980628431dc08d661dc74e0b4bd5deb) --- .../src/tests/test_interop.py | 15 ++++-- .../foreign/ForeignExecutableBuiltins.java | 17 ++----- .../foreign/ForeignInstantiableBuiltins.java | 50 ++----------------- 3 files changed, 19 insertions(+), 63 deletions(-) diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_interop.py b/graalpython/com.oracle.graal.python.test/src/tests/test_interop.py index c065cacbd8..6ea5c7e6f6 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_interop.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_interop.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # The Universal Permissive License (UPL), Version 1.0 @@ -471,11 +471,20 @@ def test_java_import_from_jar(self): os.unlink(tempname) def test_java_class(self): - from java.lang import Integer, Number, NumberFormatException - self.assertEqual(type(Integer).mro(), [polyglot.ForeignClass, polyglot.ForeignAbstractClass, polyglot.ForeignInstantiable, polyglot.ForeignObject, object]) + from java.lang import Number, NumberFormatException + from java.util import ArrayList + self.assertEqual(type(ArrayList).mro(), [polyglot.ForeignClass, polyglot.ForeignAbstractClass, polyglot.ForeignInstantiable, polyglot.ForeignObject, object]) self.assertEqual(type(Number).mro(), [polyglot.ForeignAbstractClass, polyglot.ForeignObject, object]) self.assertEqual(type(NumberFormatException).mro(), [polyglot.ForeignClass, polyglot.ForeignAbstractClass, polyglot.ForeignInstantiable, polyglot.ForeignObject, object]) + from java.util import ArrayList + l = ArrayList() + assert isinstance(l, ArrayList) + self.assertEqual(getattr(ArrayList, 'class'), l.getClass()) + + with self.assertRaisesRegex(TypeError, "ForeignInstantiable.__call__\(\) got an unexpected keyword argument 'kwarg'"): + ArrayList(kwarg=42) + def test_java_exceptions(self): # TODO: more tests diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignExecutableBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignExecutableBuiltins.java index 22c632fac0..ea02d004c7 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignExecutableBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignExecutableBuiltins.java @@ -35,13 +35,11 @@ import com.oracle.graal.python.builtins.CoreFunctions; import com.oracle.graal.python.builtins.PythonBuiltinClassType; import com.oracle.graal.python.builtins.PythonBuiltins; -import com.oracle.graal.python.builtins.objects.function.PKeyword; import com.oracle.graal.python.nodes.ErrorMessages; import com.oracle.graal.python.nodes.PRaiseNode; import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode; import com.oracle.graal.python.nodes.function.PythonBuiltinNode; import com.oracle.graal.python.nodes.interop.PForeignToPTypeNode; -import com.oracle.graal.python.nodes.object.IsForeignObjectNode; import com.oracle.graal.python.runtime.ExecutionContext.IndirectCallContext; import com.oracle.graal.python.runtime.GilNode; import com.oracle.graal.python.runtime.IndirectCallData; @@ -50,7 +48,6 @@ import com.oracle.truffle.api.CompilerDirectives; import com.oracle.truffle.api.dsl.Bind; import com.oracle.truffle.api.dsl.Cached; -import com.oracle.truffle.api.dsl.Fallback; import com.oracle.truffle.api.dsl.GenerateNodeFactory; import com.oracle.truffle.api.dsl.NodeFactory; import com.oracle.truffle.api.dsl.Specialization; @@ -69,14 +66,13 @@ protected List> getNodeFa return ForeignExecutableBuiltinsFactory.getFactories(); } - @Builtin(name = J___CALL__, minNumOfPositionalArgs = 1, takesVarArgs = true, takesVarKeywordArgs = true) + @Builtin(name = J___CALL__, minNumOfPositionalArgs = 1, takesVarArgs = true) @GenerateNodeFactory public abstract static class CallNode extends PythonBuiltinNode { - @Specialization(guards = {"isForeignObjectNode.execute(inliningTarget, callee)", "!isNoValue(callee)", "keywords.length == 0"}, limit = "1") - static Object doInteropCall(VirtualFrame frame, Object callee, Object[] arguments, @SuppressWarnings("unused") PKeyword[] keywords, + @Specialization + static Object doInteropCall(VirtualFrame frame, Object callee, Object[] arguments, @SuppressWarnings("unused") @Bind("this") Node inliningTarget, @Cached("createFor(this)") IndirectCallData indirectCallData, - @SuppressWarnings("unused") @Cached IsForeignObjectNode isForeignObjectNode, @CachedLibrary(limit = "4") InteropLibrary lib, @Cached PForeignToPTypeNode toPTypeNode, @Cached GilNode gil, @@ -98,13 +94,6 @@ static Object doInteropCall(VirtualFrame frame, Object callee, Object[] argument throw CompilerDirectives.shouldNotReachHere(e); } } - - @Fallback - @SuppressWarnings("unused") - static Object doGeneric(Object callee, Object arguments, Object keywords, - @Cached PRaiseNode raiseNode) { - throw raiseNode.raise(PythonErrorType.TypeError, ErrorMessages.INVALID_INSTANTIATION_OF_FOREIGN_OBJ); - } } } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignInstantiableBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignInstantiableBuiltins.java index 7ab0b0d664..d5b095145e 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignInstantiableBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignInstantiableBuiltins.java @@ -36,13 +36,11 @@ import com.oracle.graal.python.builtins.CoreFunctions; import com.oracle.graal.python.builtins.PythonBuiltinClassType; import com.oracle.graal.python.builtins.PythonBuiltins; -import com.oracle.graal.python.builtins.objects.function.PKeyword; import com.oracle.graal.python.nodes.ErrorMessages; import com.oracle.graal.python.nodes.PRaiseNode; import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode; import com.oracle.graal.python.nodes.function.PythonBuiltinNode; import com.oracle.graal.python.nodes.interop.PForeignToPTypeNode; -import com.oracle.graal.python.nodes.object.IsForeignObjectNode; import com.oracle.graal.python.runtime.ExecutionContext.IndirectCallContext; import com.oracle.graal.python.runtime.GilNode; import com.oracle.graal.python.runtime.IndirectCallData; @@ -51,7 +49,6 @@ import com.oracle.truffle.api.CompilerDirectives; import com.oracle.truffle.api.dsl.Bind; import com.oracle.truffle.api.dsl.Cached; -import com.oracle.truffle.api.dsl.Fallback; import com.oracle.truffle.api.dsl.GenerateNodeFactory; import com.oracle.truffle.api.dsl.NodeFactory; import com.oracle.truffle.api.dsl.Specialization; @@ -70,46 +67,14 @@ protected List> getNodeFa return ForeignInstantiableBuiltinsFactory.getFactories(); } - @Builtin(name = J___NEW__, minNumOfPositionalArgs = 1, takesVarArgs = true, takesVarKeywordArgs = true) - @GenerateNodeFactory - abstract static class NewNode extends PythonBuiltinNode { - @Specialization(guards = {"isForeignObjectNode.execute(inliningTarget, callee)", "!isNoValue(callee)", "keywords.length == 0"}, limit = "1") - static Object doInteropCall(Object callee, Object[] arguments, @SuppressWarnings("unused") PKeyword[] keywords, - @SuppressWarnings("unused") @Bind("this") Node inliningTarget, - @SuppressWarnings("unused") @Cached IsForeignObjectNode isForeignObjectNode, - @CachedLibrary(limit = "3") InteropLibrary lib, - @Cached PForeignToPTypeNode toPTypeNode, - @Cached GilNode gil, - @Cached PRaiseNode.Lazy raiseNode) { - gil.release(true); - try { - Object res = lib.instantiate(callee, arguments); - return toPTypeNode.executeConvert(res); - } catch (ArityException | UnsupportedTypeException e) { - throw raiseNode.get(inliningTarget).raise(PythonErrorType.TypeError, ErrorMessages.INVALID_INSTANTIATION_OF_FOREIGN_OBJ); - } catch (UnsupportedMessageException e) { - throw CompilerDirectives.shouldNotReachHere(e); - } finally { - gil.acquire(); - } - } - - @Fallback - @SuppressWarnings("unused") - static Object doGeneric(Object callee, Object arguments, Object keywords, - @Cached PRaiseNode raiseNode) { - throw raiseNode.raise(PythonErrorType.TypeError, ErrorMessages.INVALID_INSTANTIATION_OF_FOREIGN_OBJ); - } - } - - @Builtin(name = J___CALL__, minNumOfPositionalArgs = 1, takesVarArgs = true, takesVarKeywordArgs = true) + @Builtin(name = J___NEW__, minNumOfPositionalArgs = 1, takesVarArgs = true) + @Builtin(name = J___CALL__, minNumOfPositionalArgs = 1, takesVarArgs = true) @GenerateNodeFactory public abstract static class CallNode extends PythonBuiltinNode { - @Specialization(guards = {"isForeignObjectNode.execute(inliningTarget, callee)", "!isNoValue(callee)", "keywords.length == 0"}, limit = "1") - static Object doInteropCall(VirtualFrame frame, Object callee, Object[] arguments, @SuppressWarnings("unused") PKeyword[] keywords, + @Specialization + static Object doInteropCall(VirtualFrame frame, Object callee, Object[] arguments, @SuppressWarnings("unused") @Bind("this") Node inliningTarget, @Cached("createFor(this)") IndirectCallData indirectCallData, - @SuppressWarnings("unused") @Cached IsForeignObjectNode isForeignObjectNode, @CachedLibrary(limit = "4") InteropLibrary lib, @Cached PForeignToPTypeNode toPTypeNode, @Cached GilNode gil, @@ -131,13 +96,6 @@ static Object doInteropCall(VirtualFrame frame, Object callee, Object[] argument throw CompilerDirectives.shouldNotReachHere(e); } } - - @Fallback - @SuppressWarnings("unused") - static Object doGeneric(Object callee, Object arguments, Object keywords, - @Cached PRaiseNode raiseNode) { - throw raiseNode.raise(PythonErrorType.TypeError, ErrorMessages.INVALID_INSTANTIATION_OF_FOREIGN_OBJ); - } } } From 7c829c5f0ece9d7a56e08863240e9ac66778561b Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Fri, 17 Jan 2025 16:39:17 +0100 Subject: [PATCH 11/82] Rethrow internal errors during PythonLanguage#initializeContext * Such as AssertionError, etc. * Ensure to not print Python exceptions twice by always throwing an exit exception after printing stacktraces with AlwaysRunExcepthook. (cherry picked from commit 53180fecb184d4b89b1765527f39f99423d399c9) --- .../graal/python/shell/GraalPythonMain.java | 2 + .../exception/TopLevelExceptionHandler.java | 37 +++++++++++-------- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java b/graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java index f7838f7d5e..f0a6328d1f 100644 --- a/graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java +++ b/graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java @@ -843,6 +843,8 @@ protected void launch(Builder contextBuilder) { } catch (PolyglotException e) { if (e.isExit()) { rc = e.getExitStatus(); + } else { + throw e; } } catch (NoSuchFileException e) { printFileNotFoundException(e); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/exception/TopLevelExceptionHandler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/exception/TopLevelExceptionHandler.java index f20a4ea9cd..53f1fe05d0 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/exception/TopLevelExceptionHandler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/exception/TopLevelExceptionHandler.java @@ -195,12 +195,7 @@ private AbstractTruffleException handlePythonException(AbstractTruffleException if (PythonOptions.isPExceptionWithJavaStacktrace(getPythonLanguage()) && e instanceof PException pe) { ExceptionUtils.printJavaStackTrace(pe); } - if (!getSourceSection().getSource().isInteractive()) { - if (getContext().isChildContext()) { - getContext().getChildContextData().setExitCode(1); - } - throw new PythonExitException(this, 1); - } + exit(1); } // Before we leave Python, format the message since outside the context if (e instanceof PException pe) { @@ -209,6 +204,15 @@ private AbstractTruffleException handlePythonException(AbstractTruffleException throw e; } + private void exit(int exitCode) { + if (!getSourceSection().getSource().isInteractive()) { + if (getContext().isChildContext()) { + getContext().getChildContextData().setExitCode(1); + } + throw new PythonExitException(this, exitCode); + } + } + private static boolean isSystemExit(PBaseException pythonException) { return IsBuiltinClassProfile.profileClassSlowPath(GetPythonObjectClassNode.executeUncached(pythonException), SystemExit); } @@ -227,6 +231,7 @@ private void handleJavaException(Throwable e) { if (PythonOptions.shouldPrintJavaStacktrace(getPythonLanguage(), e)) { e.printStackTrace(); } + exit(1); } } catch (UnsupportedMessageException unsupportedMessageException) { throw CompilerDirectives.shouldNotReachHere(); @@ -250,12 +255,12 @@ private void handleSystemExit(PBaseException pythonException) { int exitcode = getExitCode(pythonException); throw new PythonExitException(this, exitcode); } catch (CannotCastException e) { - // fall through - } - if (handleAlwaysRunExceptHook(theContext, pythonException)) { - throw new PythonExitException(this, 1); + if (handleAlwaysRunExceptHook(theContext, pythonException)) { + throw new PythonExitException(this, 1); + } else { + throw pythonException.getExceptionForReraise(pythonException.getTraceback()); + } } - throw pythonException.getExceptionForReraise(pythonException.getTraceback()); } @TruffleBoundary @@ -264,12 +269,12 @@ private Object handleChildContextExit(PBaseException pythonException) throws PEx try { return getExitCode(pythonException); } catch (CannotCastException cce) { - // fall through - } - if (handleAlwaysRunExceptHook(getContext(), pythonException)) { - return 1; + if (handleAlwaysRunExceptHook(getContext(), pythonException)) { + return 1; + } else { + throw pythonException.getExceptionForReraise(pythonException.getTraceback()); + } } - throw pythonException.getExceptionForReraise(pythonException.getTraceback()); } private static int getExitCode(PBaseException pythonException) throws CannotCastException { From 96d33d879b537c0c5f733c9cf00e82f355e1f7e3 Mon Sep 17 00:00:00 2001 From: Danilo Ansaloni Date: Tue, 3 Sep 2024 19:40:34 +0200 Subject: [PATCH 12/82] Update XZ version. (cherry picked from commit 10f8ae3547ff2abba2ed100d7116355155a51985) --- mx.graalpython/suite.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mx.graalpython/suite.py b/mx.graalpython/suite.py index 763a0fd5f6..8cc53b105a 100644 --- a/mx.graalpython/suite.py +++ b/mx.graalpython/suite.py @@ -108,12 +108,12 @@ ], "digest": "sha512:16920fd41f398696c563417049472c0d81abb2d293ecb45bbbe97c12651669833e34eac238e2e4a6f8761ea58fb39806425d2741e88e8c3097fe2b5457ebf488", }, - "XZ-5.2.6": { + "XZ-5.6.2": { "urls": [ - "https://lafo.ssw.uni-linz.ac.at/pub/graal-external-deps/xz-5.2.6.tar.gz", + "https://lafo.ssw.uni-linz.ac.at/pub/graal-external-deps/xz-5.6.2.tar.gz", ], "packedResource": True, - "digest": "sha512:090958dd6c202c989746686094c86707ad4ae835026640080fc0a9d0fad699821b7d5cb3a67e6700661a0938818ba153662366f89ab8ec47e0bae4a3fe9b1961", + "digest": "sha512:c32c32c95e3541b906e0284e66a953ace677e0ce6af2084e7b122600047bf7542c1b0fabb5909b19ff79fba6def530be674df1c675b22a47a8d57f3f0b736a82", }, "BOUNCYCASTLE-PROVIDER": { "digest": "sha512:fb10c3c089921c8173ad285329f730e0e78de175d1b50b9bdd79c6a85a265af9b3331caa0c1ed57e5f47047319ce3b0f3bb5def0a3db9cccf2755cc95e145e52", @@ -636,10 +636,10 @@ "bin/", ], "cmakeConfig": { - "XZ_SRC": "", + "XZ_SRC": "", "XZ_VERSION_MAJOR": "5", - "XZ_VERSION_MINOR": "2", - "XZ_VERSION_PATCH": "6", + "XZ_VERSION_MINOR": "6", + "XZ_VERSION_PATCH": "2", }, "os_arch": { "windows": { @@ -654,7 +654,7 @@ }, }, "buildDependencies": [ - "XZ-5.2.6", + "XZ-5.6.2", ], }, From ce7f385fd261f0f8f75122d6964045c3065ad04d Mon Sep 17 00:00:00 2001 From: stepan Date: Wed, 19 Feb 2025 14:27:35 +0100 Subject: [PATCH 13/82] Fix: Maven plugin incorrectly prints warning about default resources directory. Fix few typos in warning messages. --- .../src/tests/standalone/test_gradle_plugin.py | 6 +++++- .../src/tests/standalone/test_maven_plugin.py | 2 ++ .../python/maven/plugin/ManageResourcesMojo.java | 4 ++-- .../java/org/graalvm/python/GraalPyGradlePlugin.java | 10 +++++----- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/test_gradle_plugin.py b/graalpython/com.oracle.graal.python.test/src/tests/standalone/test_gradle_plugin.py index 52d95fc704..b50be878dc 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/standalone/test_gradle_plugin.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/test_gradle_plugin.py @@ -133,6 +133,8 @@ def check_gradle_generated_app(self, community): cmd = gradlew_cmd + ["build"] out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir, logger=log) util.check_ouput("BUILD SUCCESS", out, logger=log) + util.check_ouput("Virtual filesystem is deployed to default resources directory", out, logger=log) + util.check_ouput("This can cause conflicts if used with other Java libraries that also deploy GraalPy virtual filesystem", out, logger=log) self.check_filelist(target_dir, log) cmd = gradlew_cmd + ["nativeCompile"] @@ -413,7 +415,7 @@ def check_gradle_python_resources_dir_and_external_dir_error(self): gradle_cmd = util.get_gradle_wrapper(target_dir, self.env) cmd = gradle_cmd + ["graalPyResources"] out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir) - util.check_ouput("Cannot set both 'externalDirectory' and 'resourcesDirectory' at the same time", out) + util.check_ouput("Cannot set both 'externalDirectory' and 'resourceDirectory' at the same time", out) assert return_code != 0, out @@ -478,6 +480,8 @@ def check_gradle_namespaced_vfs(self): app1_gradle_cmd = util.get_gradle_wrapper(app1_dir, self.env) out, return_code = util.run_cmd(app1_gradle_cmd + ['publishToMavenLocal'], self.env, cwd=app1_dir) + util.check_ouput("Virtual filesystem is deployed to default resources directory", out, contains=False) + util.check_ouput("This can cause conflicts if used with other Java libraries that also deploy GraalPy virtual filesystem", out, contains=False) assert return_code == 0, out app2_gradle_cmd = util.get_gradle_wrapper(app2_dir, self.env) diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/test_maven_plugin.py b/graalpython/com.oracle.graal.python.test/src/tests/standalone/test_maven_plugin.py index 4b2cdb5c04..5c4f9e34cb 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/standalone/test_maven_plugin.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/test_maven_plugin.py @@ -115,6 +115,8 @@ def check_generated_app(self, use_default_vfs_path, use_utils_pkg=False): cmd = mvnw_cmd + ["package", "-Pnative", "-DmainClass=it.pkg.GraalPy"] out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir) util.check_ouput("BUILD SUCCESS", out) + util.check_ouput("Virtual filesystem is deployed to default resources directory", out, contains=use_default_vfs_path) + util.check_ouput("This can cause conflicts if used with other Java libraries that also deploy GraalPy virtual filesystem.", out, contains=use_default_vfs_path) # check fileslist.txt fl_path = os.path.join(target_dir, "target", "classes", vfs_prefix, "fileslist.txt") diff --git a/graalpython/graalpy-maven-plugin/src/main/java/org/graalvm/python/maven/plugin/ManageResourcesMojo.java b/graalpython/graalpy-maven-plugin/src/main/java/org/graalvm/python/maven/plugin/ManageResourcesMojo.java index bc2b69ceae..d81b78b861 100644 --- a/graalpython/graalpy-maven-plugin/src/main/java/org/graalvm/python/maven/plugin/ManageResourcesMojo.java +++ b/graalpython/graalpy-maven-plugin/src/main/java/org/graalvm/python/maven/plugin/ManageResourcesMojo.java @@ -145,7 +145,7 @@ public void execute() throws MojoExecutionException { } if (resourceDirectory == null) { - if (externalDirectory != null) { + if (externalDirectory == null) { getLog().info(String.format("Virtual filesystem is deployed to default resources directory '%s'. " + "This can cause conflicts if used with other Java libraries that also deploy GraalPy virtual filesystem. " + "Consider adding GRAALPY-VFS/${project.groupId}/${project.artifactId} to your pom.xml, " + @@ -162,7 +162,7 @@ public void execute() throws MojoExecutionException { getLog().warn("The GraalPy plugin configuration setting was deprecated and has no effect anymore.\n" + "For execution in jvm mode, the python language home is always available.\n" + "When building a native executable using GraalVM Native Image, then the full python language home is by default embedded into the native executable.\n" + - "For more details, please refer to the documentation of GraalVM Native Image options IncludeLanguageResources and CopyLanguageResources documentation."); + "For more details, please refer to the documentation of GraalVM Native Image options IncludeLanguageResources and CopyLanguageResources."); } manageVenv(); diff --git a/graalpython/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/GraalPyGradlePlugin.java b/graalpython/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/GraalPyGradlePlugin.java index f0ff56f0e1..0b964a9720 100644 --- a/graalpython/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/GraalPyGradlePlugin.java +++ b/graalpython/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/GraalPyGradlePlugin.java @@ -117,11 +117,11 @@ public void apply(Project project) { if (extension.getPythonResourcesDirectory().isPresent() && extension.getExternalDirectory().isPresent()) { throw new GradleException( - "Cannot set both 'externalDirectory' and 'resourcesDirectory' at the same time. " + + "Cannot set both 'externalDirectory' and 'resourceDirectory' at the same time. " + "New property 'externalDirectory' is a replacement for deprecated 'pythonResourcesDirectory'. " + "If you want to deploy the virtual environment into physical filesystem, use 'externalDirectory'. " + "The deployment of the external directory alongside the application is not handled by the GraalPy Maven plugin in such case." + - "If you wish to bundle the virtual filesystem in Java resources, use 'resourcesDirectory'. " + + "If you wish to bundle the virtual filesystem in Java resources, use 'resourceDirectory'. " + "For more details, please refer to https://www.graalvm.org/latest/reference-manual/python/Embedding-Build-Tools. "); } @@ -132,9 +132,9 @@ public void apply(Project project) { // Run the vfsFilesListTask conditionally only if 'externalDirectory' is not set if (!extension.getPythonResourcesDirectory().isPresent() && !extension.getExternalDirectory().isPresent()) { if (!extension.getResourceDirectory().isPresent()) { - proj.getLogger().info(String.format("Virtual filesystem is deployed to default resources directory '%s'. " + + proj.getLogger().warn(String.format("Virtual filesystem is deployed to default resources directory '%s'. " + "This can cause conflicts if used with other Java libraries that also deploy GraalPy virtual filesystem. " + - "Consider adding `resourcesDirectory = \"GRAALPY-VFS/${groupId}/${artifactId}\"` to your build.gradle script " + + "Consider adding `resourceDirectory = \"GRAALPY-VFS/${groupId}/${artifactId}\"` to your build.gradle script " + "(replace the placeholders with values specific to your project), " + "moving any existing sources from '%s' to '%s', and using VirtualFileSystem$Builder#resourceDirectory." + "For more details, please refer to https://www.graalvm.org/latest/reference-manual/python/Embedding-Build-Tools. ", @@ -192,7 +192,7 @@ private TaskProvider registerResourcesTask(Project project, Confi t.getLogger().warn("The GraalPy plugin pythonHome configuration setting was deprecated and has no effect anymore.\n" + "For execution in jvm mode, the python language home is always available.\n" + "When building a native executable using GraalVM Native Image, then the full python language home is by default embedded into the native executable.\n" + - "For more details, please refer to the documentation of GraalVM Native Image options IncludeLanguageResources and CopyLanguageResources documentation."); + "For more details, please refer to the documentation of GraalVM Native Image options IncludeLanguageResources and CopyLanguageResources."); } t.getPackages().set(extension.getPackages()); From 3b46423445fef929fc1546c5d88efc8a46551935 Mon Sep 17 00:00:00 2001 From: stepan Date: Wed, 19 Feb 2025 14:51:56 +0100 Subject: [PATCH 14/82] Improve the GraalPyResources docs --- .../python/embedding/GraalPyResources.java | 46 ++++++++++++------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/graalpython/org.graalvm.python.embedding/src/org/graalvm/python/embedding/GraalPyResources.java b/graalpython/org.graalvm.python.embedding/src/org/graalvm/python/embedding/GraalPyResources.java index 7efc01cd3a..fa26d06389 100644 --- a/graalpython/org.graalvm.python.embedding/src/org/graalvm/python/embedding/GraalPyResources.java +++ b/graalpython/org.graalvm.python.embedding/src/org/graalvm/python/embedding/GraalPyResources.java @@ -198,7 +198,12 @@ private GraalPyResources() { * location *
  • /org.graalvm.python.vfs/src - is set as the python sources location
    • * - *

    + *

    + * When the virtual filesystem is located in other than the default resource directory, + * {@code org.graalvm.python.vfs}, i.e., using Maven or Gradle option {@code resourceDirectory}, + * use {@link #contextBuilder(VirtualFileSystem)} and + * {@link VirtualFileSystem.Builder#resourceDirectory(String)} when building the + * {@link VirtualFileSystem}. * * @return a new {@link Context} instance * @since 24.2.0 @@ -233,6 +238,12 @@ public static Context createContext() { * } * } * + *

    + * When the virtual filesystem is located in other than the default resource directory, + * {@code org.graalvm.python.vfs}, i.e., using Maven or Gradle option {@code resourceDirectory}, + * use {@link #contextBuilder(VirtualFileSystem)} and + * {@link VirtualFileSystem.Builder#resourceDirectory(String)} when building the + * {@link VirtualFileSystem}. * * @see PythonOptions @@ -308,13 +319,14 @@ public static Context.Builder contextBuilder(VirtualFileSystem vfs) { /** * Creates a GraalPy context preconfigured with GraalPy and polyglot Context configuration - * options for use with resources located in a real filesystem. + * options for use with resources located in an external directory in real filesystem. *

    * Following resource paths are preconfigured: *

      - *
    • ${resourcesDirectory}/venv - is set as the python virtual environment + *
    • ${externalResourcesDirectory}/venv - is set as the python virtual + * environment location
      • + *
    • ${externalResourcesDirectory}/src - is set as the python sources * location
      • - *
    • ${resourcesDirectory}/src - is set as the python sources location
      • *
    *

    *

    @@ -343,19 +355,20 @@ public static Context.Builder contextBuilder(VirtualFileSystem vfs) { * *

    * - * @param resourcesDirectory the root directory with GraalPy specific embedding resources + * @param externalResourcesDirectory the root directory with GraalPy specific embedding + * resources * @return a new {@link org.graalvm.polyglot.Context.Builder} instance * @since 24.2.0 */ - public static Context.Builder contextBuilder(Path resourcesDirectory) { + public static Context.Builder contextBuilder(Path externalResourcesDirectory) { String execPath; if (VirtualFileSystemImpl.isWindows()) { - execPath = resourcesDirectory.resolve(VirtualFileSystemImpl.VFS_VENV).resolve("Scripts").resolve("python.exe").toAbsolutePath().toString(); + execPath = externalResourcesDirectory.resolve(VirtualFileSystemImpl.VFS_VENV).resolve("Scripts").resolve("python.exe").toAbsolutePath().toString(); } else { - execPath = resourcesDirectory.resolve(VirtualFileSystemImpl.VFS_VENV).resolve("bin").resolve("python").toAbsolutePath().toString(); + execPath = externalResourcesDirectory.resolve(VirtualFileSystemImpl.VFS_VENV).resolve("bin").resolve("python").toAbsolutePath().toString(); } - String srcPath = resourcesDirectory.resolve(VirtualFileSystemImpl.VFS_SRC).toAbsolutePath().toString(); + String srcPath = externalResourcesDirectory.resolve(VirtualFileSystemImpl.VFS_SRC).toAbsolutePath().toString(); return createContextBuilder(). // allow all IO access allowIO(IOAccess.ALL). @@ -437,8 +450,9 @@ public static Path getNativeExecutablePath() { * The structure of the created resource directory will stay the same like the embedded Python * resources structure: *
      - *
    • ${resourcesDirectory}/venv - the python virtual environment location
      • - *
    • ${resourcesDirectory}/src - the python sources location
      • + *
    • ${externalResourcesDirectory}/venv - the python virtual environment + * location
      • + *
    • ${externalResourcesDirectory}/src - the python sources location
      • *
    *

    *

    @@ -456,17 +470,17 @@ public static Path getNativeExecutablePath() { *

    * * @param vfs the {@link VirtualFileSystem} from which resources are to be extracted - * @param resourcesDirectory the target directory to extract the resources to + * @param externalResourcesDirectory the target directory to extract the resources to * @throws IOException if resources isn't a directory * @see #contextBuilder(Path) * @see VirtualFileSystem.Builder#resourceLoadingClass(Class) * * @since 24.2.0 */ - public static void extractVirtualFileSystemResources(VirtualFileSystem vfs, Path resourcesDirectory) throws IOException { - if (Files.exists(resourcesDirectory) && !Files.isDirectory(resourcesDirectory)) { - throw new IOException(String.format("%s has to be a directory", resourcesDirectory.toString())); + public static void extractVirtualFileSystemResources(VirtualFileSystem vfs, Path externalResourcesDirectory) throws IOException { + if (Files.exists(externalResourcesDirectory) && !Files.isDirectory(externalResourcesDirectory)) { + throw new IOException(String.format("%s has to be a directory", externalResourcesDirectory.toString())); } - vfs.impl.extractResources(resourcesDirectory); + vfs.impl.extractResources(externalResourcesDirectory); } } From 875be24f018563e373e3da5452e4ea725c6108fa Mon Sep 17 00:00:00 2001 From: Danilo Ansaloni Date: Tue, 4 Mar 2025 20:29:50 +0100 Subject: [PATCH 15/82] Release GraalVM 24.2.0. --- mx.graalpython/suite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mx.graalpython/suite.py b/mx.graalpython/suite.py index 8cc53b105a..16abd25b65 100644 --- a/mx.graalpython/suite.py +++ b/mx.graalpython/suite.py @@ -11,7 +11,7 @@ "version": "24.2.0", "graalpython:pythonVersion": "3.11.7", - "release": False, + "release": True, "groupId": "org.graalvm.python", "url": "http://www.graalvm.org/python", From 0d0fb1665a3a9451276accb3d665ed5800329eb3 Mon Sep 17 00:00:00 2001 From: Danilo Ansaloni Date: Tue, 4 Mar 2025 20:30:23 +0100 Subject: [PATCH 16/82] Start 24.2.1 dev cycle. --- ci.jsonnet | 2 +- .../com.oracle.graal.python.test.integration/pom.xml | 2 +- .../src/tests/standalone/gradle/build/build.gradle | 2 +- .../tests/standalone/gradle/build/build.gradle.kts | 2 +- .../src/tests/standalone/jbang/EmptyPIPComments.j | 2 +- .../standalone/jbang/EmptyPythonResourceComment.j | 2 +- .../jbang/EmptyPythonResourceCommentWithBlanks.j | 2 +- .../tests/standalone/jbang/NoPackagesResourcesDir.j | 2 +- .../standalone/jbang/TwoPythonResourceComments.j | 2 +- graalpython/graalpy-archetype-polyglot-app/pom.xml | 2 +- .../src/main/resources/archetype-resources/pom.xml | 2 +- graalpython/graalpy-jbang/examples/hello.java | 2 +- .../templates/graalpy-template.java.qute | 2 +- .../templates/graalpy-template_local_repo.java.qute | 2 +- graalpython/graalpy-maven-plugin/pom.xml | 4 ++-- mx.graalpython/suite.py | 12 ++++++------ 16 files changed, 22 insertions(+), 22 deletions(-) diff --git a/ci.jsonnet b/ci.jsonnet index a3a4e1743e..7d5f235fe3 100644 --- a/ci.jsonnet +++ b/ci.jsonnet @@ -1 +1 @@ -{ "overlay": "585265d3c6bafc1d12054374377589b360307957" } +{ "overlay": "16d0e566573ebdcf380bf6f309c02724baf4172b" } diff --git a/graalpython/com.oracle.graal.python.test.integration/pom.xml b/graalpython/com.oracle.graal.python.test.integration/pom.xml index 0f55209ba9..6251cedd21 100644 --- a/graalpython/com.oracle.graal.python.test.integration/pom.xml +++ b/graalpython/com.oracle.graal.python.test.integration/pom.xml @@ -64,7 +64,7 @@ Additionally, one can change the polyglot artifacts version with 17 17 UTF-8 - 24.2.0 + 24.2.1 diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/build.gradle b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/build.gradle index 15c3c89c12..2dfdd3f5ce 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/build.gradle +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/build.gradle @@ -1,6 +1,6 @@ plugins { id "application" - id 'org.graalvm.python' version '24.2.0' + id 'org.graalvm.python' version '24.2.1' id "org.graalvm.buildtools.native" version "0.10.2" } diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/build.gradle.kts b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/build.gradle.kts index a086b6635e..8ccf16421d 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/build.gradle.kts +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/build.gradle.kts @@ -1,6 +1,6 @@ plugins { application - id("org.graalvm.python") version "24.2.0" + id("org.graalvm.python") version "24.2.1" id("org.graalvm.buildtools.native") version "0.10.2" } diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/jbang/EmptyPIPComments.j b/graalpython/com.oracle.graal.python.test/src/tests/standalone/jbang/EmptyPIPComments.j index 6b140a59b8..08561a0c15 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/standalone/jbang/EmptyPIPComments.j +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/jbang/EmptyPIPComments.j @@ -1,6 +1,6 @@ ///usr/bin/env jbang "$0" "$@" ; exit $? -//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:24.2.0} +//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:24.2.1} //PIP // one blank after PIP //PIP diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/jbang/EmptyPythonResourceComment.j b/graalpython/com.oracle.graal.python.test/src/tests/standalone/jbang/EmptyPythonResourceComment.j index 768e188e9e..31330e96fd 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/standalone/jbang/EmptyPythonResourceComment.j +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/jbang/EmptyPythonResourceComment.j @@ -1,6 +1,6 @@ ///usr/bin/env jbang "$0" "$@" ; exit $? -//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:24.2.0} +//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:24.2.1} //PYTHON_RESOURCES_DIRECTORY public class EmptyPythonResourceComment { diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/jbang/EmptyPythonResourceCommentWithBlanks.j b/graalpython/com.oracle.graal.python.test/src/tests/standalone/jbang/EmptyPythonResourceCommentWithBlanks.j index f4b7556997..6f71f88213 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/standalone/jbang/EmptyPythonResourceCommentWithBlanks.j +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/jbang/EmptyPythonResourceCommentWithBlanks.j @@ -1,6 +1,6 @@ ///usr/bin/env jbang "$0" "$@" ; exit $? -//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:24.2.0} +//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:24.2.1} // resource dir with blanks //PYTHON_RESOURCES_DIRECTORY diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/jbang/NoPackagesResourcesDir.j b/graalpython/com.oracle.graal.python.test/src/tests/standalone/jbang/NoPackagesResourcesDir.j index 749ae89606..24a4866b26 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/standalone/jbang/NoPackagesResourcesDir.j +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/jbang/NoPackagesResourcesDir.j @@ -1,6 +1,6 @@ ///usr/bin/env jbang "$0" "$@" ; exit $? -//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:24.2.0} +//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:24.2.1} //PYTHON_RESOURCES_DIRECTORY python-resources public class NoPackagesResourcesDir { diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/jbang/TwoPythonResourceComments.j b/graalpython/com.oracle.graal.python.test/src/tests/standalone/jbang/TwoPythonResourceComments.j index c2d902d358..ab31f0a738 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/standalone/jbang/TwoPythonResourceComments.j +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/jbang/TwoPythonResourceComments.j @@ -1,6 +1,6 @@ ///usr/bin/env jbang "$0" "$@" ; exit $? -//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:24.2.0} +//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:24.2.1} //PYTHON_RESOURCES_DIRECTORY //PYTHON_RESOURCES_DIRECTORY diff --git a/graalpython/graalpy-archetype-polyglot-app/pom.xml b/graalpython/graalpy-archetype-polyglot-app/pom.xml index 04b27f55e3..e525172eb1 100644 --- a/graalpython/graalpy-archetype-polyglot-app/pom.xml +++ b/graalpython/graalpy-archetype-polyglot-app/pom.xml @@ -45,7 +45,7 @@ SOFTWARE. org.graalvm.python graalpy-archetype-polyglot-app - 24.2.0 + 24.2.1 http://www.graalvm.org/python Maven archetype providing a skeleton GraalPy - Java polyglot application. maven-archetype diff --git a/graalpython/graalpy-archetype-polyglot-app/src/main/resources/archetype-resources/pom.xml b/graalpython/graalpy-archetype-polyglot-app/src/main/resources/archetype-resources/pom.xml index 90be4e32ca..e67d93e879 100644 --- a/graalpython/graalpy-archetype-polyglot-app/src/main/resources/archetype-resources/pom.xml +++ b/graalpython/graalpy-archetype-polyglot-app/src/main/resources/archetype-resources/pom.xml @@ -10,7 +10,7 @@ #set( $symbol_dollar = '$' ) - 24.2.0 + 24.2.1 python-community 0.10.4 17 diff --git a/graalpython/graalpy-jbang/examples/hello.java b/graalpython/graalpy-jbang/examples/hello.java index b005158ef6..11049392fc 100644 --- a/graalpython/graalpy-jbang/examples/hello.java +++ b/graalpython/graalpy-jbang/examples/hello.java @@ -40,7 +40,7 @@ */ ///usr/bin/env jbang "$0" "$@" ; exit $? //JAVA 17+ -//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:24.2.0} +//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:24.2.1} // specify python packages and their versions as if used with pip //PIP termcolor==2.2 diff --git a/graalpython/graalpy-jbang/templates/graalpy-template.java.qute b/graalpython/graalpy-jbang/templates/graalpy-template.java.qute index 5a61e3dea9..9a94477922 100644 --- a/graalpython/graalpy-jbang/templates/graalpy-template.java.qute +++ b/graalpython/graalpy-jbang/templates/graalpy-template.java.qute @@ -5,7 +5,7 @@ {/for} {#if dependencies.isEmpty()}// //DEPS {/if} {| -//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:24.2.0} +//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:24.2.1} // specify python packages and their versions as if used with pip //PIP termcolor==2.2 |} diff --git a/graalpython/graalpy-jbang/templates/graalpy-template_local_repo.java.qute b/graalpython/graalpy-jbang/templates/graalpy-template_local_repo.java.qute index 2fb3914dc1..6b41f4e288 100644 --- a/graalpython/graalpy-jbang/templates/graalpy-template_local_repo.java.qute +++ b/graalpython/graalpy-jbang/templates/graalpy-template_local_repo.java.qute @@ -8,7 +8,7 @@ //REPOS mc=https://repo1.maven.org/maven2/ //REPOS local=file://{path_to_local_repo} {| -//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:24.2.0} +//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:24.2.1} // specify python packages and their versions as if used with pip //PIP termcolor==2.2 |} diff --git a/graalpython/graalpy-maven-plugin/pom.xml b/graalpython/graalpy-maven-plugin/pom.xml index 4838e2cec7..5a8fe1acde 100644 --- a/graalpython/graalpy-maven-plugin/pom.xml +++ b/graalpython/graalpy-maven-plugin/pom.xml @@ -48,7 +48,7 @@ SOFTWARE. graalpy-maven-plugin maven-plugin - 24.2.0 + 24.2.1 http://www.graalvm.org/python graalpy-maven-plugin Handles python related resources in a maven GraalPy - Java polyglot application. @@ -57,7 +57,7 @@ SOFTWARE. 17 17 UTF-8 - 24.2.0 + 24.2.1 diff --git a/mx.graalpython/suite.py b/mx.graalpython/suite.py index 16abd25b65..af58122c58 100644 --- a/mx.graalpython/suite.py +++ b/mx.graalpython/suite.py @@ -9,9 +9,9 @@ "name": "graalpython", "versionConflictResolution": "latest", - "version": "24.2.0", + "version": "24.2.1", "graalpython:pythonVersion": "3.11.7", - "release": True, + "release": False, "groupId": "org.graalvm.python", "url": "http://www.graalvm.org/python", @@ -45,7 +45,7 @@ }, { "name": "sdk", - "version": "826d284d76d4e3c2379e952824f4ab7e96c805ee", + "version": "c9096be682f7aa67f5133fb098762e2152ff355f", "subdir": True, "urls": [ {"url": "https://github.com/oracle/graal", "kind": "git"}, @@ -53,7 +53,7 @@ }, { "name": "tools", - "version": "826d284d76d4e3c2379e952824f4ab7e96c805ee", + "version": "c9096be682f7aa67f5133fb098762e2152ff355f", "subdir": True, "urls": [ {"url": "https://github.com/oracle/graal", "kind": "git"}, @@ -61,7 +61,7 @@ }, { "name": "sulong", - "version": "826d284d76d4e3c2379e952824f4ab7e96c805ee", + "version": "c9096be682f7aa67f5133fb098762e2152ff355f", "subdir": True, "urls": [ {"url": "https://github.com/oracle/graal", "kind": "git"}, @@ -69,7 +69,7 @@ }, { "name": "regex", - "version": "826d284d76d4e3c2379e952824f4ab7e96c805ee", + "version": "c9096be682f7aa67f5133fb098762e2152ff355f", "subdir": True, "urls": [ {"url": "https://github.com/oracle/graal", "kind": "git"}, From 10573cbf474ce2981d7114bc0f90ced4aaaf9dfa Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Thu, 20 Mar 2025 11:42:01 +0100 Subject: [PATCH 17/82] Pass NULL instead of None as arg to METHNOARGS functions (cherry picked from commit 935e46a1e7f5ed7e965407ccbdcf26fe35a6cc9e) --- .../builtins/objects/cext/capi/ExternalFunctionNodes.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/ExternalFunctionNodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/ExternalFunctionNodes.java index 18528ad4a6..8d236077eb 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/ExternalFunctionNodes.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/ExternalFunctionNodes.java @@ -1309,7 +1309,7 @@ public MethNoargsRoot(PythonLanguage language, TruffleString name, boolean isSta @Override protected Object[] prepareCArguments(VirtualFrame frame) { - return new Object[]{readSelf(frame), PNone.NONE}; + return new Object[]{readSelf(frame), PNone.NO_VALUE}; } @Override From 11936fb04188f8ab0b75d6b09379baa934cc701a Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Thu, 20 Mar 2025 11:58:24 +0100 Subject: [PATCH 18/82] Support calling PyType_Modified on any Python class (cherry picked from commit cef2e89e85d4d8681092968c477574153b3fd9e7) --- .../modules/cext/PythonCextTypeBuiltins.java | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextTypeBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextTypeBuiltins.java index eea71e9ebd..a7e76df0a2 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextTypeBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextTypeBuiltins.java @@ -228,20 +228,24 @@ static Object doIt(PythonAbstractNativeObject clazz, @CApiBuiltin(ret = ArgDescriptor.Void, args = {PyTypeObject}, call = Direct) abstract static class PyType_Modified extends CApiUnaryBuiltinNode { - @TruffleBoundary @Specialization - static Object doIt(PythonAbstractNativeObject clazz, + static Object doIt(PythonAbstractClass object, @Bind("this") Node inliningTarget) { - PythonContext context = PythonContext.get(inliningTarget); - CyclicAssumption nativeClassStableAssumption = context.getNativeClassStableAssumption(clazz, false); - if (nativeClassStableAssumption != null) { - nativeClassStableAssumption.invalidate("PyType_Modified(\"" + TypeNodes.GetNameNode.executeUncached(clazz).toJavaStringUncached() + "\") called"); + if (object instanceof PythonAbstractNativeObject clazz) { + PythonContext context = PythonContext.get(inliningTarget); + CyclicAssumption nativeClassStableAssumption = context.getNativeClassStableAssumption(clazz, false); + if (nativeClassStableAssumption != null) { + nativeClassStableAssumption.invalidate("PyType_Modified(\"" + TypeNodes.GetNameNode.executeUncached(clazz).toJavaStringUncached() + "\") called"); + } + MroSequenceStorage mroStorage = TypeNodes.GetMroStorageNode.executeUncached(clazz); + mroStorage.lookupChanged(); + // Reload slots from native, which also invalidates cached slot lookups + clazz.setTpSlots(TpSlots.fromNative(clazz, context)); + } else { + MroSequenceStorage mroStorage = TypeNodes.GetMroStorageNode.executeUncached(object); + mroStorage.lookupChanged(); } - MroSequenceStorage mroStorage = TypeNodes.GetMroStorageNode.executeUncached(clazz); - mroStorage.lookupChanged(); - // Reload slots from native, which also invalidates cached slot lookups - clazz.setTpSlots(TpSlots.fromNative(clazz, context)); return PNone.NO_VALUE; } } From 9b12f6854fd3f5e09448603a8c36293c8f769ee2 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Thu, 20 Mar 2025 13:30:40 +0100 Subject: [PATCH 19/82] Enable PyUnicode_DecodeRawUnicodeEscape (cherry picked from commit e36f7d45604e1aad0678cf43964958022dbb001d) --- .../com.oracle.graal.python.cext/src/unicodeobject.c | 9 +++++---- .../python/builtins/objects/cext/capi/CApiFunction.java | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/graalpython/com.oracle.graal.python.cext/src/unicodeobject.c b/graalpython/com.oracle.graal.python.cext/src/unicodeobject.c index 42699b743a..852906637e 100644 --- a/graalpython/com.oracle.graal.python.cext/src/unicodeobject.c +++ b/graalpython/com.oracle.graal.python.cext/src/unicodeobject.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, 2024, Oracle and/or its affiliates. +/* Copyright (c) 2018, 2025, Oracle and/or its affiliates. * Copyright (C) 1996-2020 Python Software Foundation * * Licensed under the PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 @@ -6220,16 +6220,17 @@ _PyUnicode_DecodeRawUnicodeEscapeStateful(const char *s, Py_XDECREF(exc); return NULL; } - +#endif // GraalPy change PyObject * PyUnicode_DecodeRawUnicodeEscape(const char *s, Py_ssize_t size, const char *errors) { - return _PyUnicode_DecodeRawUnicodeEscapeStateful(s, size, errors, NULL); + return PyUnicode_Decode(s, size, "raw_unicode_escape", errors); + // return _PyUnicode_DecodeRawUnicodeEscapeStateful(s, size, errors, NULL); } - +#if 0 // GraalPy change PyObject * PyUnicode_AsRawUnicodeEscapeString(PyObject *unicode) { diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiFunction.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiFunction.java index 5909dc1427..ae89ccf769 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiFunction.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiFunction.java @@ -504,6 +504,7 @@ public final class CApiFunction { @CApiBuiltin(name = "PyUnicode_DecodeLatin1", ret = PyObject, args = {ConstCharPtrAsTruffleString, Py_ssize_t, ConstCharPtrAsTruffleString}, call = CImpl) @CApiBuiltin(name = "PyUnicode_DecodeLocale", ret = PyObject, args = {ConstCharPtrAsTruffleString, ConstCharPtrAsTruffleString}, call = CImpl) @CApiBuiltin(name = "PyUnicode_DecodeLocaleAndSize", ret = PyObject, args = {ConstCharPtrAsTruffleString, Py_ssize_t, ConstCharPtrAsTruffleString}, call = CImpl) + @CApiBuiltin(name = "PyUnicode_DecodeRawUnicodeEscape", ret = PyObject, args = {ConstCharPtrAsTruffleString, Py_ssize_t, ConstCharPtrAsTruffleString}, call = CImpl) @CApiBuiltin(name = "PyUnicode_DecodeUTF16", ret = PyObject, args = {ConstCharPtrAsTruffleString, Py_ssize_t, ConstCharPtrAsTruffleString, INT_LIST}, call = CImpl) @CApiBuiltin(name = "PyUnicode_DecodeUTF16Stateful", ret = PyObject, args = {ConstCharPtrAsTruffleString, Py_ssize_t, ConstCharPtrAsTruffleString, INT_LIST, PY_SSIZE_T_PTR}, call = CImpl) @CApiBuiltin(name = "PyUnicode_DecodeUTF32", ret = PyObject, args = {ConstCharPtrAsTruffleString, Py_ssize_t, ConstCharPtrAsTruffleString, INT_LIST}, call = CImpl) @@ -942,7 +943,6 @@ public final class CApiFunction { @CApiBuiltin(name = "PyUnicode_BuildEncodingMap", ret = PyObject, args = {PyObject}, call = NotImplemented) @CApiBuiltin(name = "PyUnicode_CopyCharacters", ret = Py_ssize_t, args = {PyObject, Py_ssize_t, PyObject, Py_ssize_t, Py_ssize_t}, call = NotImplemented) @CApiBuiltin(name = "PyUnicode_DecodeCharmap", ret = PyObject, args = {ConstCharPtrAsTruffleString, Py_ssize_t, PyObject, ConstCharPtrAsTruffleString}, call = NotImplemented) - @CApiBuiltin(name = "PyUnicode_DecodeRawUnicodeEscape", ret = PyObject, args = {ConstCharPtrAsTruffleString, Py_ssize_t, ConstCharPtrAsTruffleString}, call = NotImplemented) @CApiBuiltin(name = "PyUnicode_DecodeUTF7", ret = PyObject, args = {ConstCharPtrAsTruffleString, Py_ssize_t, ConstCharPtrAsTruffleString}, call = NotImplemented) @CApiBuiltin(name = "PyUnicode_DecodeUTF7Stateful", ret = PyObject, args = {ConstCharPtrAsTruffleString, Py_ssize_t, ConstCharPtrAsTruffleString, PY_SSIZE_T_PTR}, call = NotImplemented) @CApiBuiltin(name = "PyUnicode_DecodeUnicodeEscape", ret = PyObject, args = {ConstCharPtrAsTruffleString, Py_ssize_t, ConstCharPtrAsTruffleString}, call = NotImplemented) From dc7165228972aa20e2edd533e02ddebd6cffa833 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Fri, 21 Mar 2025 06:31:18 +0100 Subject: [PATCH 20/82] Add patch to install pymupdf (cherry picked from commit 98e11fb717901205a4d3a25b4307afe6c7eafa52) --- .../modules/cext/PythonCextTypeBuiltins.java | 2 +- .../objects/cext/capi/CApiFunction.java | 2 +- .../cext/capi/ExternalFunctionNodes.java | 2 +- .../lib-graalpython/patches/metadata.toml | 9 ++ .../lib-graalpython/patches/pymupdf.patch | 124 ++++++++++++++++++ mx.graalpython/verify_patches.py | 3 +- 6 files changed, 138 insertions(+), 4 deletions(-) create mode 100644 graalpython/lib-graalpython/patches/pymupdf.patch diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextTypeBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextTypeBuiltins.java index a7e76df0a2..a997a8a2aa 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextTypeBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextTypeBuiltins.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiFunction.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiFunction.java index ae89ccf769..97ad1090b1 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiFunction.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiFunction.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/ExternalFunctionNodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/ExternalFunctionNodes.java index 8d236077eb..3111ccef04 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/ExternalFunctionNodes.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/ExternalFunctionNodes.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 diff --git a/graalpython/lib-graalpython/patches/metadata.toml b/graalpython/lib-graalpython/patches/metadata.toml index ac1ccb7262..2d414e4baa 100644 --- a/graalpython/lib-graalpython/patches/metadata.toml +++ b/graalpython/lib-graalpython/patches/metadata.toml @@ -476,6 +476,15 @@ version = ">= 4.8.0" patch = 'pymongo-4.8.0.patch' license = 'Apache-2.0' +[[PyMuPDF.rules]] +version = "== 1.25.4" +patch = "pymupdf.patch" +# That project is AGPL, so do not actually include *any* code of pymupdf in the patch, not even an +# empty line, in the diff context. The code we write in the patch is UPL - that is compatible with +# AGPL in the sense that if someone were to apply it and distribute *that*, our patch is now part +# of the AGPL'd codebase +license = 'UPL' + [[pyOpenSSL.rules]] # Pin this version to avoid pulling newer cryptography than we have patch for version = "== 23.2.0" diff --git a/graalpython/lib-graalpython/patches/pymupdf.patch b/graalpython/lib-graalpython/patches/pymupdf.patch new file mode 100644 index 0000000000..27869eccd6 --- /dev/null +++ b/graalpython/lib-graalpython/patches/pymupdf.patch @@ -0,0 +1,124 @@ +diff --git a/graalpy-config b/graalpy-config +new file mode 100755 +index 00000000..1f69f726 +--- /dev/null ++++ b/graalpy-config +@@ -0,0 +1,78 @@ ++#!/bin/sh ++ ++# Adapted from CPython but deferring to GraalPy ++ ++exit_with_usage () ++{ ++ echo "Usage: $0 --prefix|--exec-prefix|--includes|--libs|--cflags|--ldflags|--extension-suffix|--help|--abiflags|--configdir|--embed" ++ exit $1 ++} ++ ++if [ "$1" = "" ] ; then ++ exit_with_usage 1 ++fi ++ ++# Returns the actual prefix where this script was installed to. ++EXE=$(cd $(dirname "$0") && pwd -P) ++if which readlink >/dev/null 2>&1 ; then ++ if readlink -f "$RESULT" >/dev/null 2>&1; then ++ EXE=$(readlink -f "$RESULT") ++ fi ++fi ++EXE=$EXE/graalpy ++ ++if ! test -x "$EXE" ; then ++ EXE=graalpy ++fi ++ ++# Scan for --help or unknown argument. ++for ARG in $* ++do ++ case $ARG in ++ --help) ++ exit_with_usage 0 ++ ;; ++ --embed) ++ echo "graalpy-config does not print embedding flags" ++ exit 1 ++ ;; ++ --prefix|--exec-prefix|--includes|--libs|--cflags|--ldflags|--extension-suffix|--abiflags|--configdir) ++ ;; ++ *) ++ exit_with_usage 1 ++ ;; ++ esac ++done ++ ++for ARG in "$@" ++do ++ case "$ARG" in ++ --prefix) ++ $EXE -c "print(__import__('sysconfig').get_config_var('prefix'))" ++ ;; ++ --exec-prefix) ++ $EXE -c "print(__import__('sysconfig').get_config_var('exec_prefix'))" ++ ;; ++ --includes) ++ $EXE -c "from sysconfig import get_path; print('-I'+get_path('include'), '-I'+get_path('platinclude'))" ++ ;; ++ --cflags) ++ $EXE -c "import sysconfig as s; print('-I' + s.get_path('include'), '-I' + s.get_path('platinclude'), s.get_config_var('CFLAGS').replace('NDEBUG', 'DEBUG'), s.get_config_var('OPT').replace('NDEBUG', 'DEBUG'))" ++ ;; ++ --libs) ++ $EXE -c "import sysconfig as s; print('-L' + s.get_config_var('LIBDIR'))" ++ ;; ++ --ldflags) ++ $EXE -c "import sysconfig as s; print('-L' + s.get_config_var('LIBDIR'))" ++ ;; ++ --extension-suffix) ++ $EXE -c "import sysconfig as s; print(s.get_config_var('EXT_SUFFIX'))" ++ ;; ++ --abiflags) ++ $EXE -c "import sysconfig as s; print(s.get_config_var('ABIFLAGS'))" ++ ;; ++ --configdir) ++ echo "" ++ ;; ++esac ++done +diff --git a/setup.py b/setup.py +index 5fba2c97..3fe63b07 100755 +--- a/setup.py ++++ b/setup.py +@@ -1452,0 +1452,35 @@ ++if sys.implementation.name == "graalpy": ++ import os ++ import re ++ import subprocess ++ import shutil ++ import sysconfig ++ from pathlib import Path ++ ++ def build_wheel(wheel_directory, config_settings=None, metadata_directory=None): ++ wheel_directory = Path(wheel_directory).absolute() ++ sdir = Path(__file__).absolute().parent ++ python311 = shutil.which("python3.11") ++ if not python311: ++ raise RuntimeError("python3.11 must be available on the PATH for cross-compilation") ++ env = os.environ.copy() ++ env["PIPCL_PYTHON_CONFIG"] = str(sdir / "graalpy-config") ++ env["PYMUPDF_SETUP_PY_LIMITED_API"] = "1" ++ subprocess.run( ++ [python311, "setup.py", "bdist_wheel"], ++ env=env, ++ cwd=sdir, ++ check=True, ++ ) ++ wheels = list((sdir / 'dist').glob('*.whl')) ++ assert len(wheels) == 1, f"Expected 1 wheel, found {len(wheels)}" ++ wheel = wheels[0] ++ assert "-cp311-abi3" in wheel.name, f"Expected wheel to be for CPython 3.11 ABI 3, got {wheel.name}" ++ graalpy_ext_suffix = sysconfig.get_config_var("EXT_SUFFIX") ++ m = re.match(r"\.graalpy(\d+[^\-]*)-(\d+)", sysconfig.get_config_var("EXT_SUFFIX")) ++ gpver = m[1] ++ cpver = m[2] ++ graalpy_wheel_tag = f"graalpy{cpver}-graalpy{gpver}_{cpver}_native" ++ name = wheel.name.replace("cp311-abi3", graalpy_wheel_tag) ++ shutil.copyfile(wheel, wheel_directory / name) ++ return str(name) diff --git a/mx.graalpython/verify_patches.py b/mx.graalpython/verify_patches.py index 1ebc371694..8aa4e84c10 100644 --- a/mx.graalpython/verify_patches.py +++ b/mx.graalpython/verify_patches.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2023, 2025, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # The Universal Permissive License (UPL), Version 1.0 @@ -45,6 +45,7 @@ # Approved license identifiers in SPDX "short identifier" format ALLOWED_LICENSES = { + 'UPL', # https://spdx.org/licenses/UPL-1.0.html 'MIT', # https://spdx.org/licenses/MIT.html 'BSD-3-Clause', # https://spdx.org/licenses/BSD-3-Clause.html 'BSD-2-Clause', # https://spdx.org/licenses/BSD-2-Clause.html From beb30c8fc33209b7f33c523850df05c45d4ba891 Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Mon, 10 Mar 2025 09:45:46 +0100 Subject: [PATCH 21/82] Skip time-dependent test (cherry picked from commit 438997b0c0db5ec59f9944a13f6b5b061dd765f8) --- .../src/tests/unittest_tags/test_strptime.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_strptime.txt b/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_strptime.txt index 019ffaf499..220ef38651 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_strptime.txt +++ b/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_strptime.txt @@ -35,7 +35,8 @@ test.test_strptime.StrptimeTests.test_percent @ darwin-arm64,darwin-x86_64,linux test.test_strptime.StrptimeTests.test_second @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64,win32-AMD64 test.test_strptime.StrptimeTests.test_strptime_exception_context @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64,win32-AMD64 test.test_strptime.StrptimeTests.test_time @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64,win32-AMD64 -test.test_strptime.StrptimeTests.test_timezone @ darwin-arm64,darwin-x86_64,win32-AMD64 +# Seems to be dependent on the actual time/date/timezone of the machine, at least on GraalPy. Needs investigation +!test.test_strptime.StrptimeTests.test_timezone test.test_strptime.StrptimeTests.test_unconverteddata @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64,win32-AMD64 test.test_strptime.StrptimeTests.test_year @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64,win32-AMD64 test.test_strptime.TimeRETests.test_blankpattern @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64,win32-AMD64 From abd391210a10593c7e35c132a2894dcfe8755798 Mon Sep 17 00:00:00 2001 From: Marouane El Hallaoui Date: Wed, 2 Apr 2025 18:58:53 +0000 Subject: [PATCH 22/82] release GraalVM 24.2.1 --- mx.graalpython/suite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mx.graalpython/suite.py b/mx.graalpython/suite.py index af58122c58..05c541dd3e 100644 --- a/mx.graalpython/suite.py +++ b/mx.graalpython/suite.py @@ -11,7 +11,7 @@ "version": "24.2.1", "graalpython:pythonVersion": "3.11.7", - "release": False, + "release": True, "groupId": "org.graalvm.python", "url": "http://www.graalvm.org/python", From c25c88dfc60b8f4312276cae65fc741aeac565ee Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Thu, 3 Apr 2025 09:35:01 +0200 Subject: [PATCH 23/82] Set overlay to graalpy-patch-branch branch --- ci.jsonnet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci.jsonnet b/ci.jsonnet index a3a4e1743e..f3fbf47e80 100644 --- a/ci.jsonnet +++ b/ci.jsonnet @@ -1 +1 @@ -{ "overlay": "585265d3c6bafc1d12054374377589b360307957" } +{ "overlay": "ea6007b968048485d43905914816208f870ac88b" } From 83b9de9d33b366c7f0bd734cf184485954608f34 Mon Sep 17 00:00:00 2001 From: Ivo Horak Date: Thu, 30 Jan 2025 13:31:35 +0100 Subject: [PATCH 24/82] Add pyarrow v19.0.0 patch --- .../lib-graalpython/patches/metadata.toml | 5 + .../patches/pyarrow-19.0.0.patch | 164 ++++++++++++++++++ 2 files changed, 169 insertions(+) create mode 100644 graalpython/lib-graalpython/patches/pyarrow-19.0.0.patch diff --git a/graalpython/lib-graalpython/patches/metadata.toml b/graalpython/lib-graalpython/patches/metadata.toml index ac1ccb7262..450bbfe206 100644 --- a/graalpython/lib-graalpython/patches/metadata.toml +++ b/graalpython/lib-graalpython/patches/metadata.toml @@ -416,6 +416,11 @@ version = '== 12.0.0' patch = 'pyarrow-12.0.0.patch' license = 'Apache-2.0' +[[pyarrow.rules]] +version = '== 19.0.0' +patch = 'pyarrow-19.0.0.patch' +license = 'Apache-2.0' + [[pybind11.rules]] # Upstreamed install-priority = 0 diff --git a/graalpython/lib-graalpython/patches/pyarrow-19.0.0.patch b/graalpython/lib-graalpython/patches/pyarrow-19.0.0.patch new file mode 100644 index 0000000000..70adda0ede --- /dev/null +++ b/graalpython/lib-graalpython/patches/pyarrow-19.0.0.patch @@ -0,0 +1,164 @@ +diff --git a/pyarrow/error.pxi b/pyarrow/error.pxi +index cbe2552..8d0d9d9 100644 +--- a/pyarrow/error.pxi ++++ b/pyarrow/error.pxi +@@ -248,7 +248,7 @@ cdef class SignalStopHandler: + if exc_value.signum: + # Re-emit the exact same signal. We restored the Python signal + # handler above, so it should receive it. +- if os.name == 'nt': ++ if os.name == 'nt' or sys.implementation.name == 'graalpy': + SendSignal(exc_value.signum) + else: + SendSignalToThread(exc_value.signum, +diff --git a/pyarrow/memory.pxi b/pyarrow/memory.pxi +index 1ddcb01..6805e42 100644 +--- a/pyarrow/memory.pxi ++++ b/pyarrow/memory.pxi +@@ -20,6 +20,10 @@ + # cython: embedsignature = True + + ++cdef extern from "Python.h": ++ void Py_INCREF(object) ++ ++ + cdef class MemoryPool(_Weakrefable): + """ + Base class for memory allocation. +@@ -35,6 +39,13 @@ cdef class MemoryPool(_Weakrefable): + + cdef void init(self, CMemoryPool* pool): + self.pool = pool ++ # GraalPy change: pyarrow doesn't maintain python references from ++ # buffers to pools, but they dereference the pointer to the pool in the ++ # destructor. They just assume buffers will get GC'ed before their ++ # pools. You can easily get a segfault even on CPython if you make ++ # a buffer outlive its pool. Since we can't guarantee destruction ++ # order, we just leak the pool. ++ Py_INCREF(self) + + def release_unused(self): + """ +diff --git a/pyarrow_build_backend.py b/pyarrow_build_backend.py +new file mode 100644 +index 0000000..0929c5f +--- /dev/null ++++ b/pyarrow_build_backend.py +@@ -0,0 +1,93 @@ ++import os ++import re ++import sys ++import tarfile ++import subprocess ++import tempfile ++import shutil ++import tarfile ++import urllib.request ++from pathlib import Path ++ ++VERSION = '19.0.0' ++ ++ ++def build_sdist(sdist_directory, config_settings=None): ++ nv = f'pyarrow-{VERSION}' ++ srcdir = Path(__file__).parent ++ archive_path = Path(sdist_directory) / f'{nv}.tar.gz' ++ ++ def tarfilter(info): ++ if re.match(r'\./(?:.git|venv|[^-/]+-venv|dist)', info.name): ++ return None ++ info.name = f'./{nv}/{info.name}' ++ return info ++ ++ with tarfile.open(archive_path, 'w:gz') as tar: ++ tar.add('.', filter=tarfilter) ++ return archive_path.name ++ ++ ++def build_wheel(wheel_directory, config_settings=None, metadata_directory=None): ++ wheel_directory = Path(wheel_directory).absolute() ++ with tempfile.TemporaryDirectory() as tmpdir: ++ tmpdir = Path(tmpdir).absolute() ++ tarname = f'apache-arrow-{VERSION}.tar.gz' ++ tarpath = tmpdir / tarname ++ urllib.request.urlretrieve(f"https://github.com/apache/arrow/archive/refs/tags/{tarname}", tarpath) ++ with tarfile.open(tarpath) as tar: ++ tar.extractall(tmpdir) ++ arrow_dir = tmpdir / f'arrow-apache-arrow-{VERSION}' ++ assert arrow_dir.is_dir() ++ arrow_dist = tmpdir / 'arrow-dist' ++ build_dir = tmpdir / 'arrow-build' ++ subprocess.check_call([ ++ 'cmake', '-S', str(arrow_dir / 'cpp'), '-B', str(build_dir), ++ '-DCMAKE_INSTALL_LIBDIR=lib', ++ f'-DCMAKE_INSTALL_PREFIX={arrow_dist}', ++ '-DCMAKE_BUILD_TYPE=Release', ++ '-DARROW_RPATH_ORIGIN=ON', ++ '-DARROW_BUILD_TESTS=OFF', ++ '-DARROW_BUILD_SHARED=ON', ++ '-DARROW_BUILD_STATIC=OFF', ++ # Features ++ '-DARROW_COMPUTE=ON', ++ '-DARROW_CSV=ON', ++ '-DARROW_JSON=ON', ++ '-DARROW_FILESYSTEM=ON', ++ '-DARROW_DATASET=ON', ++ '-DARROW_PARQUET=ON', ++ '-DPARQUET_REQUIRE_ENCRYPTION=ON', ++ '-DARROW_GANDIVA=ON', ++ '-DARROW_WITH_BZ2=ON', ++ '-DARROW_WITH_ZLIB=ON', ++ '-DARROW_WITH_ZSTD=ON', ++ '-DARROW_WITH_LZ4=ON', ++ '-DARROW_WITH_SNAPPY=ON', ++ '-DARROW_WITH_BROTLI=ON', ++ ]) ++ subprocess.check_call([ ++ 'cmake', '--build', str(build_dir), '--parallel', ++ ]) ++ subprocess.check_call([ ++ 'cmake', '--install', str(build_dir), ++ ]) ++ env = os.environ.copy() ++ env['ARROW_HOME'] = str(arrow_dist) ++ env['CMAKE_PREFIX_PATH'] = str(arrow_dist) ++ env['PYARROW_WITH_DATASET'] = '1' ++ env['PYARROW_WITH_PARQUET'] = '1' ++ env['PYARROW_WITH_PARQUET_ENCRYPTION'] = '1' ++ env['PYARROW_WITH_GANDIVA'] = '1' ++ env['PYARROW_BUNDLE_ARROW_CPP'] = '1' ++ env['PYARROW_BUNDLE_CYTHON_CPP'] = '1' ++ subprocess.run( ++ [sys.executable, 'setup.py', 'bdist_wheel'], ++ env=env, ++ check=True, ++ ) ++ wheels = list(Path('dist').glob('*.whl')) ++ assert len(wheels) == 1, f"Expected 1 wheel, found {len(wheels)}" ++ wheel = wheels[0] ++ shutil.copyfile(wheel, wheel_directory / wheel.name) ++ return str(wheel.name) +diff --git a/pyproject.toml b/pyproject.toml +index ef2043f..cb08a86 100644 +--- a/pyproject.toml ++++ b/pyproject.toml +@@ -17,7 +17,7 @@ + + [build-system] + requires = [ +- "cython >= 0.29.31", ++ "cython >= 0.29.31, < 3", + # Starting with NumPy 1.25, NumPy is (by default) as far back compatible + # as oldest-support-numpy was (customizable with a NPY_TARGET_VERSION + # define). For older Python versions (where NumPy 1.25 is not yet available) +@@ -29,7 +29,8 @@ requires = [ + "setuptools_scm[toml]>=8", + "setuptools>=64", + ] +-build-backend = "setuptools.build_meta" ++build-backend = "pyarrow_build_backend" ++backend-path = ["."] + + [project] + name = "pyarrow" From 3fce96f2d48cfebb5884b08abba0fbd1a806e2ba Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Wed, 29 Jan 2025 14:14:41 +0100 Subject: [PATCH 25/82] Add patch for psycopg2-2.9.10 --- .../lib-graalpython/patches/metadata.toml | 16 ++++++++++++++-- .../{psycopg2.patch => psycopg2-2.9.10.patch} | 12 ++++++------ ...sycopg2-binary.patch => psycopg2-2.9.9.patch} | 0 3 files changed, 20 insertions(+), 8 deletions(-) rename graalpython/lib-graalpython/patches/{psycopg2.patch => psycopg2-2.9.10.patch} (51%) rename graalpython/lib-graalpython/patches/{psycopg2-binary.patch => psycopg2-2.9.9.patch} (100%) diff --git a/graalpython/lib-graalpython/patches/metadata.toml b/graalpython/lib-graalpython/patches/metadata.toml index 450bbfe206..7356d3a129 100644 --- a/graalpython/lib-graalpython/patches/metadata.toml +++ b/graalpython/lib-graalpython/patches/metadata.toml @@ -394,11 +394,23 @@ patch = 'protobuf-3.21.9.patch' license = 'BSD-3-Clause' [[psycopg2.rules]] -patch = "psycopg2.patch" +version = '<= 2.9.9' +patch = "psycopg2-2.9.9.patch" +license = "LGPL-3.0-or-later WITH openssl-exception" + +[[psycopg2.rules]] +version = '>= 2.9.10' +patch = "psycopg2-2.9.10.patch" +license = "LGPL-3.0-or-later WITH openssl-exception" + +[[psycopg2-binary.rules]] +version = '<= 2.9.9' +patch = "psycopg2-2.9.9.patch" license = "LGPL-3.0-or-later WITH openssl-exception" [[psycopg2-binary.rules]] -patch = "psycopg2-binary.patch" +version = '>= 2.9.10' +patch = "psycopg2-2.9.10.patch" license = "LGPL-3.0-or-later WITH openssl-exception" [[py4j.rules]] diff --git a/graalpython/lib-graalpython/patches/psycopg2.patch b/graalpython/lib-graalpython/patches/psycopg2-2.9.10.patch similarity index 51% rename from graalpython/lib-graalpython/patches/psycopg2.patch rename to graalpython/lib-graalpython/patches/psycopg2-2.9.10.patch index a3d7e3aa2d..1b35cdc301 100644 --- a/graalpython/lib-graalpython/patches/psycopg2.patch +++ b/graalpython/lib-graalpython/patches/psycopg2-2.9.10.patch @@ -1,15 +1,15 @@ diff --git a/psycopg/utils.c b/psycopg/utils.c -index 16be906..c78a24b 100644 +index 1dfb87d..28cc6c1 100644 --- a/psycopg/utils.c +++ b/psycopg/utils.c @@ -392,7 +392,9 @@ psyco_set_error(PyObject *exc, cursorObject *curs, const char *msg) static int psyco_is_main_interp(void) { --#if PY_VERSION_HEX >= 0x03080000 +-#if PY_VERSION_HEX >= 0x030d0000 +#if GRAALVM_PYTHON + return 1; -+#elif PY_VERSION_HEX >= 0x03080000 - /* tested with Python 3.8.0a2 */ - return _PyInterpreterState_Get() == PyInterpreterState_Main(); - #else ++#elif PY_VERSION_HEX >= 0x030d0000 + /* tested with Python 3.13.0a6 */ + return PyInterpreterState_Get() == PyInterpreterState_Main(); + #elif PY_VERSION_HEX >= 0x03080000 diff --git a/graalpython/lib-graalpython/patches/psycopg2-binary.patch b/graalpython/lib-graalpython/patches/psycopg2-2.9.9.patch similarity index 100% rename from graalpython/lib-graalpython/patches/psycopg2-binary.patch rename to graalpython/lib-graalpython/patches/psycopg2-2.9.9.patch From 92d279e435d8cb77482b03091f60f5ad4b8309f3 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Tue, 25 Feb 2025 10:35:26 +0100 Subject: [PATCH 26/82] [GR-62557] Add patch for jq to recythonize the C code. --- .../lib-graalpython/patches/jq-1.8.0.patch | 59 +++++++++++++++++++ .../lib-graalpython/patches/metadata.toml | 9 +++ 2 files changed, 68 insertions(+) create mode 100644 graalpython/lib-graalpython/patches/jq-1.8.0.patch diff --git a/graalpython/lib-graalpython/patches/jq-1.8.0.patch b/graalpython/lib-graalpython/patches/jq-1.8.0.patch new file mode 100644 index 0000000000..c62c51759a --- /dev/null +++ b/graalpython/lib-graalpython/patches/jq-1.8.0.patch @@ -0,0 +1,59 @@ +--- a/pyproject.toml ++++ b/pyproject.toml +@@ -2,13 +2,10 @@ + requires = [ + "setuptools>=43", + "wheel", ++ "Cython==3.0.10", + ] + build-backend = "setuptools.build_meta" + + [tool.cibuildwheel] +-before-build = [ +- "pip install cython==3.0.10", +- "cython {project}/jq.pyx", +-] + test-requires = "-r test-requirements.txt" + test-command = "pytest {project}/tests" +diff --git a/setup.py b/setup.py +index 0b97097..54ed7b3 100644 +--- a/setup.py ++++ b/setup.py +@@ -94,15 +94,27 @@ else: + os.path.join(jq_lib_dir, "modules/oniguruma/src/.libs/libonig.a"), + ] + ++ ++try: ++ # Follow recommendation from https://cython.readthedocs.io/en/latest/src/userguide/source_files_and_compilation.html#distributing-cython-modules ++ from Cython.Build import cythonize ++except ImportError: ++ cythonize = lambda o: o ++ ext = ".c" ++else: ++ ext = ".pyx" ++ ++ + jq_extension = Extension( + "jq", +- sources=["jq.c"], ++ sources=[_path_in_dir(f"jq{ext}")], + define_macros=[("MS_WIN64" , 1)] if os.name == "nt" and sys.maxsize > 2**32 else None, # https://github.com/cython/cython/issues/2670 + include_dirs=[os.path.join(jq_lib_dir, "src")], + extra_link_args=["-lm"] + (["-Wl,-Bstatic", "-lpthread", "-lshlwapi", "-static-libgcc"] if os.name == 'nt' else []) + link_args_deps, + extra_objects=extra_objects, + ) + ++ + setup( + name='jq', + version='1.8.0', +@@ -112,7 +124,7 @@ setup( + url='https://github.com/mwilliamson/jq.py', + python_requires='>=3.6', + license='BSD 2-Clause', +- ext_modules = [jq_extension], ++ ext_modules = cythonize([jq_extension]), + cmdclass={"build_ext": jq_build_ext}, + classifiers=[ + 'Development Status :: 5 - Production/Stable', diff --git a/graalpython/lib-graalpython/patches/metadata.toml b/graalpython/lib-graalpython/patches/metadata.toml index 7356d3a129..1f3a11255e 100644 --- a/graalpython/lib-graalpython/patches/metadata.toml +++ b/graalpython/lib-graalpython/patches/metadata.toml @@ -200,6 +200,15 @@ version = '<=1.3.2' patch = 'joblib-1.3.2.patch' license = 'BSD-3-Clause' +[[jq.rules]] +version = '==1.8.0' +patch = 'jq-1.8.0.patch' +license = 'BSD-2-Clause' + +[[jq.add-sources]] +version = '1.8.0' +url = 'https://github.com/mwilliamson/jq.py/archive/refs/tags/1.8.0.tar.gz' + [[jupyter_server.rules]] patch = 'jupyter_server.patch' license = 'BSD-3-Clause' From 5c17862c8db7d67f8c8582490a11c916b19a2f62 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Tue, 25 Feb 2025 15:03:47 +0100 Subject: [PATCH 27/82] Add pandas 2.2.3 patch --- .../lib-graalpython/patches/metadata.toml | 6 +++ .../patches/pandas-2.2.3.patch | 52 +++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 graalpython/lib-graalpython/patches/pandas-2.2.3.patch diff --git a/graalpython/lib-graalpython/patches/metadata.toml b/graalpython/lib-graalpython/patches/metadata.toml index 1f3a11255e..55e95f5619 100644 --- a/graalpython/lib-graalpython/patches/metadata.toml +++ b/graalpython/lib-graalpython/patches/metadata.toml @@ -367,6 +367,12 @@ patch = 'pandas-2.2.2.patch' license = 'BSD-3-Clause' dist-type = 'sdist' +[[pandas.rules]] +version = '== 2.2.3' +patch = 'pandas-2.2.3.patch' +license = 'BSD-3-Clause' +dist-type = 'sdist' + [[pandas.rules]] version = '== 2.0.3' patch = 'pandas-2.0.3.patch' diff --git a/graalpython/lib-graalpython/patches/pandas-2.2.3.patch b/graalpython/lib-graalpython/patches/pandas-2.2.3.patch new file mode 100644 index 0000000000..1312dfbb7d --- /dev/null +++ b/graalpython/lib-graalpython/patches/pandas-2.2.3.patch @@ -0,0 +1,52 @@ +diff --git a/pandas/_libs/include/pandas/vendored/klib/khash_python.h b/pandas/_libs/include/pandas/vendored/klib/khash_python.h +index 5a933b4..f579fc6 100644 +--- a/pandas/_libs/include/pandas/vendored/klib/khash_python.h ++++ b/pandas/_libs/include/pandas/vendored/klib/khash_python.h +@@ -173,13 +173,15 @@ static inline int floatobject_cmp(PyFloatObject *a, PyFloatObject *b) { + // PyObject_RichCompareBool for complexobjects has a different behavior + // needs to be replaced + static inline int complexobject_cmp(PyComplexObject *a, PyComplexObject *b) { +- return (Py_IS_NAN(a->cval.real) && Py_IS_NAN(b->cval.real) && +- Py_IS_NAN(a->cval.imag) && Py_IS_NAN(b->cval.imag)) || +- (Py_IS_NAN(a->cval.real) && Py_IS_NAN(b->cval.real) && +- a->cval.imag == b->cval.imag) || +- (a->cval.real == b->cval.real && Py_IS_NAN(a->cval.imag) && +- Py_IS_NAN(b->cval.imag)) || +- (a->cval.real == b->cval.real && a->cval.imag == b->cval.imag); ++ Py_complex a_cval = PyComplex_AsCComplex((PyObject*)a); ++ Py_complex b_cval = PyComplex_AsCComplex((PyObject*)b); ++ return (Py_IS_NAN(a_cval.real) && Py_IS_NAN(b_cval.real) && ++ Py_IS_NAN(a_cval.imag) && Py_IS_NAN(b_cval.imag)) || ++ (Py_IS_NAN(a_cval.real) && Py_IS_NAN(b_cval.real) && ++ a_cval.imag == b_cval.imag) || ++ (a_cval.real == b_cval.real && Py_IS_NAN(a_cval.imag) && ++ Py_IS_NAN(b_cval.imag)) || ++ (a_cval.real == b_cval.real && a_cval.imag == b_cval.imag); + } + + static inline int pyobject_cmp(PyObject *a, PyObject *b); +@@ -250,8 +252,9 @@ static inline Py_hash_t floatobject_hash(PyFloatObject *key) { + + // replaces _Py_HashDouble with _Pandas_HashDouble + static inline Py_hash_t complexobject_hash(PyComplexObject *key) { +- Py_uhash_t realhash = (Py_uhash_t)_Pandas_HashDouble(key->cval.real); +- Py_uhash_t imaghash = (Py_uhash_t)_Pandas_HashDouble(key->cval.imag); ++ Py_complex cval = PyComplex_AsCComplex((PyObject*)key); ++ Py_uhash_t realhash = (Py_uhash_t)_Pandas_HashDouble(cval.real); ++ Py_uhash_t imaghash = (Py_uhash_t)_Pandas_HashDouble(cval.imag); + if (realhash == (Py_uhash_t)-1 || imaghash == (Py_uhash_t)-1) { + return -1; + } +diff --git a/pyproject.toml b/pyproject.toml +index db9f055..c191232 100644 +--- a/pyproject.toml ++++ b/pyproject.toml +@@ -5,7 +5,7 @@ requires = [ + "meson-python==0.13.1", + "meson==1.2.1", + "wheel", +- "Cython~=3.0.5", # Note: sync with setup.py, environment.yml and asv.conf.json ++ "Cython==3.0.10", # Note: sync with setup.py, environment.yml and asv.conf.json + # Force numpy higher than 2.0, so that built wheels are compatible + # with both numpy 1 and 2 + "numpy>=2.0", From 0432e9c039ad5774c77389646e20b1e860c88621 Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Fri, 21 Mar 2025 13:15:17 +0100 Subject: [PATCH 28/82] Add patch for trio --- .../lib-graalpython/patches/metadata.toml | 4 + .../lib-graalpython/patches/trio.patch | 103 ++++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100644 graalpython/lib-graalpython/patches/trio.patch diff --git a/graalpython/lib-graalpython/patches/metadata.toml b/graalpython/lib-graalpython/patches/metadata.toml index 55e95f5619..e6bd9477b7 100644 --- a/graalpython/lib-graalpython/patches/metadata.toml +++ b/graalpython/lib-graalpython/patches/metadata.toml @@ -749,6 +749,10 @@ version = '== 3.*' # transformers tends to depend on a specific version of tokenizers. Pin it to avoid pulling unpatched tokenizers version = '== 4.33.3' +[[trio.rules]] +patch = 'trio.patch' +license = 'Apache-2.0 OR MIT' + [[typing_extensions.rules]] patch = 'typing_extensions.patch' license = 'PSF-2.0' diff --git a/graalpython/lib-graalpython/patches/trio.patch b/graalpython/lib-graalpython/patches/trio.patch new file mode 100644 index 0000000000..dee5d3d252 --- /dev/null +++ b/graalpython/lib-graalpython/patches/trio.patch @@ -0,0 +1,103 @@ +diff --git a/trio/_socket.py b/trio/_socket.py +index 003f6c4..da7411c 100644 +--- a/trio/_socket.py ++++ b/trio/_socket.py +@@ -317,9 +317,7 @@ def fromfd( + return from_stdlib_socket(_stdlib_socket.fromfd(fd, family, type_, proto)) + + +-if sys.platform == "win32" or ( +- not TYPE_CHECKING and hasattr(_stdlib_socket, "fromshare") +-): ++if hasattr(_stdlib_socket, "fromshare"): + + @_wraps(_stdlib_socket.fromshare, assigned=(), updated=()) + def fromshare(info: bytes) -> SocketType: +@@ -606,9 +604,7 @@ class SocketType: + def set_inheritable(self, inheritable: bool) -> None: + raise NotImplementedError + +- if sys.platform == "win32" or ( +- not TYPE_CHECKING and hasattr(_stdlib_socket.socket, "share") +- ): ++ if hasattr(_stdlib_socket.socket, "share"): + + def share(self, process_id: int) -> bytes: + raise NotImplementedError +@@ -699,9 +695,7 @@ class SocketType: + ) -> Awaitable[tuple[int, AddressFormat]]: + raise NotImplementedError + +- if sys.platform != "win32" or ( +- not TYPE_CHECKING and hasattr(_stdlib_socket.socket, "recvmsg") +- ): ++ if hasattr(_stdlib_socket.socket, "recvmsg"): + + def recvmsg( + self, +@@ -712,9 +706,7 @@ class SocketType: + ) -> Awaitable[tuple[bytes, list[tuple[int, int, bytes]], int, object]]: + raise NotImplementedError + +- if sys.platform != "win32" or ( +- not TYPE_CHECKING and hasattr(_stdlib_socket.socket, "recvmsg_into") +- ): ++ if hasattr(_stdlib_socket.socket, "recvmsg_into"): + + def recvmsg_into( + self, +@@ -748,9 +740,7 @@ class SocketType: + async def sendto(self, *args: object) -> int: + raise NotImplementedError + +- if sys.platform != "win32" or ( +- not TYPE_CHECKING and hasattr(_stdlib_socket.socket, "sendmsg") +- ): ++ if hasattr(_stdlib_socket.socket, "sendmsg"): + + @_wraps(_stdlib_socket.socket.sendmsg, assigned=(), updated=()) + async def sendmsg( +@@ -867,9 +857,7 @@ class _SocketType(SocketType): + def set_inheritable(self, inheritable: bool) -> None: + return self._sock.set_inheritable(inheritable) + +- if sys.platform == "win32" or ( +- not TYPE_CHECKING and hasattr(_stdlib_socket.socket, "share") +- ): ++ if hasattr(_stdlib_socket.socket, "share"): + + def share(self, process_id: int) -> bytes: + return self._sock.share(process_id) +@@ -1181,9 +1169,7 @@ class _SocketType(SocketType): + # recvmsg + ################################################################ + +- if sys.platform != "win32" or ( +- not TYPE_CHECKING and hasattr(_stdlib_socket.socket, "recvmsg") +- ): ++ if hasattr(_stdlib_socket.socket, "recvmsg"): + if TYPE_CHECKING: + + def recvmsg( +@@ -1204,9 +1190,7 @@ class _SocketType(SocketType): + # recvmsg_into + ################################################################ + +- if sys.platform != "win32" or ( +- not TYPE_CHECKING and hasattr(_stdlib_socket.socket, "recvmsg_into") +- ): ++ if hasattr(_stdlib_socket.socket, "recvmsg_into"): + if TYPE_CHECKING: + + def recvmsg_into( +@@ -1276,9 +1260,7 @@ class _SocketType(SocketType): + # sendmsg + ################################################################ + +- if sys.platform != "win32" or ( +- not TYPE_CHECKING and hasattr(_stdlib_socket.socket, "sendmsg") +- ): ++ if hasattr(_stdlib_socket.socket, "sendmsg"): + + @_wraps(_stdlib_socket.socket.sendmsg, assigned=(), updated=()) + async def sendmsg( From 3481bf2a210898aaa6674356b5c005b34e88bddc Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Fri, 21 Mar 2025 13:19:05 +0100 Subject: [PATCH 29/82] Pin uvloop --- graalpython/lib-graalpython/patches/metadata.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/graalpython/lib-graalpython/patches/metadata.toml b/graalpython/lib-graalpython/patches/metadata.toml index e6bd9477b7..95e4d84a06 100644 --- a/graalpython/lib-graalpython/patches/metadata.toml +++ b/graalpython/lib-graalpython/patches/metadata.toml @@ -768,6 +768,7 @@ version = '>= 2, < 2.0.3' install-priority = 0 [[uvloop.rules]] +version = '<= 0.19.0' patch = 'uvloop.patch' license = 'MIT' From 490978a97bb192b95c32be13e2b7bbb616805ea3 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Tue, 25 Mar 2025 13:38:45 +0100 Subject: [PATCH 30/82] Expand range for Cython patch --- graalpython/lib-graalpython/patches/metadata.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graalpython/lib-graalpython/patches/metadata.toml b/graalpython/lib-graalpython/patches/metadata.toml index 105675338a..75a63b2145 100644 --- a/graalpython/lib-graalpython/patches/metadata.toml +++ b/graalpython/lib-graalpython/patches/metadata.toml @@ -105,7 +105,7 @@ patch = 'cryptography-42.0.5.patch' license = 'Apache-2.0 OR BSD-3-Clause' [[Cython.rules]] -version = '>= 3.0.10, < 3.0.12' +version = '>= 3.0.10, <= 3.0.12' patch = 'Cython-3.0.10.patch' license = 'Apache-2.0' From 810f9b6b603525fc460d8ca36090dca047b2e15e Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Mon, 31 Mar 2025 10:45:55 +0200 Subject: [PATCH 31/82] Extend patch for torch 2.4.1 to sidestep set_extra_state bit --- .../lib-graalpython/patches/torch-2.4.1.patch | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/graalpython/lib-graalpython/patches/torch-2.4.1.patch b/graalpython/lib-graalpython/patches/torch-2.4.1.patch index 5f7724d365..caebd3aa86 100644 --- a/graalpython/lib-graalpython/patches/torch-2.4.1.patch +++ b/graalpython/lib-graalpython/patches/torch-2.4.1.patch @@ -537,6 +537,20 @@ index c301da982..a2668be20 100644 +const int THP_PyOpcode_Caches_size = 0; + +#endif // GraalPy change +diff --git a/torch/csrc/dynamo/extra_state.c b/torch/csrc/dynamo/extra_state.c +index cbe9ab37a..18740a0d8 100644 +--- a/torch/csrc/dynamo/extra_state.c ++++ b/torch/csrc/dynamo/extra_state.c +@@ -100,9 +100,6 @@ void destroy_extra_state(void* obj) { + } + + void set_extra_state(PyCodeObject* code, ExtraState* extra_state) { +- ExtraState* old_extra_state = get_extra_state(code); +- CHECK(extra_state == nullptr || old_extra_state != extra_state); +- _PyCode_SetExtra((PyObject*)code, extra_index, extra_state); + } + + ExtraState* init_and_set_extra_state(PyCodeObject* code) { diff --git a/torch/csrc/dynamo/eval_frame.c b/torch/csrc/dynamo/eval_frame.c index cbe9ab37a..18740a0d8 100644 --- a/torch/csrc/dynamo/eval_frame.c From 479174b8f09e54706804f3d05cd03640c517cceb Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Tue, 15 Apr 2025 09:14:20 +0200 Subject: [PATCH 32/82] Widen range of numpy patch --- graalpython/lib-graalpython/patches/metadata.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graalpython/lib-graalpython/patches/metadata.toml b/graalpython/lib-graalpython/patches/metadata.toml index 75a63b2145..ac29e68dcc 100644 --- a/graalpython/lib-graalpython/patches/metadata.toml +++ b/graalpython/lib-graalpython/patches/metadata.toml @@ -296,7 +296,7 @@ patch = 'numba-0.59.1.patch' license = 'BSD-2-Clause' [[numpy.rules]] -version = '>= 2.0.0rc1, < 2.1' +version = '>= 2.0.0rc1, < 2.3' patch = 'numpy-2.0.0.patch' license = 'BSD-3-Clause' dist-type = 'sdist' From eb40b18b6055eb069d1ca46a4e4bb65642bff6ad Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Tue, 15 Apr 2025 12:39:08 +0200 Subject: [PATCH 33/82] Add patch for ormsgpack --- .../lib-graalpython/patches/metadata.toml | 5 + .../patches/ormsgpack-1.8.0-1.9.1.patch | 364 ++++++++++++++++++ 2 files changed, 369 insertions(+) create mode 100644 graalpython/lib-graalpython/patches/ormsgpack-1.8.0-1.9.1.patch diff --git a/graalpython/lib-graalpython/patches/metadata.toml b/graalpython/lib-graalpython/patches/metadata.toml index ac29e68dcc..3c331d1657 100644 --- a/graalpython/lib-graalpython/patches/metadata.toml +++ b/graalpython/lib-graalpython/patches/metadata.toml @@ -346,6 +346,11 @@ version = '== 3.10.5' patch = 'orjson-3.10.5.patch' license = 'Apache-2.0 OR MIT' +[[ormsgpack.rules]] +version = '>= 1.8.0, <= 1.9.1' +patch = 'ormsgpack-1.8.0-1.9.1.patch' +license = 'Apache-2.0 OR MIT' + [[overrides.rules]] version = '== 7.4.0' # Important: This patch esentially breaks the package, it's not upstreamable. The package relies on bytecode parsing diff --git a/graalpython/lib-graalpython/patches/ormsgpack-1.8.0-1.9.1.patch b/graalpython/lib-graalpython/patches/ormsgpack-1.8.0-1.9.1.patch new file mode 100644 index 0000000000..3a9542a058 --- /dev/null +++ b/graalpython/lib-graalpython/patches/ormsgpack-1.8.0-1.9.1.patch @@ -0,0 +1,364 @@ +diff --git a/src/deserialize/deserializer.rs b/src/deserialize/deserializer.rs +index 41cf7f1..99cd68e 100644 +--- a/src/deserialize/deserializer.rs ++++ b/src/deserialize/deserializer.rs +@@ -292,7 +292,10 @@ impl<'de> Deserializer<'de> { + marker => Err(Error::InvalidType(marker)), + }?; + let value = self.deserialize()?; ++ #[cfg(not(GraalPy))] + let pyhash = unsafe { (*key.as_ptr().cast::()).hash }; ++ #[cfg(GraalPy)] ++ let pyhash = unsafe { pyo3::ffi::PyObject_Hash(key.as_ptr()) }; + let _ = ffi!(_PyDict_SetItem_KnownHash( + dict_ptr, + key.as_ptr(), +@@ -471,7 +474,7 @@ impl<'de> Deserializer<'de> { + let ptr = ffi!(PyTuple_New(len as pyo3::ffi::Py_ssize_t)); + for i in 0..len { + let elem = self.deserialize_map_key()?; +- ffi!(PyTuple_SET_ITEM( ++ ffi!(PyTuple_SetItem( + ptr, + i as pyo3::ffi::Py_ssize_t, + elem.as_ptr() +diff --git a/src/ext.rs b/src/ext.rs +index b2573b4..9668d4f 100644 +--- a/src/ext.rs ++++ b/src/ext.rs +@@ -22,7 +22,7 @@ unsafe extern "C" fn ext_new( + ); + return null_mut(); + } +- let tag = PyTuple_GET_ITEM(args, 0); ++ let tag = PyTuple_GetItem(args, 0); + if PyLong_Check(tag) == 0 { + PyErr_SetString( + PyExc_TypeError, +@@ -30,7 +30,7 @@ unsafe extern "C" fn ext_new( + ); + return null_mut(); + } +- let data = PyTuple_GET_ITEM(args, 1); ++ let data = PyTuple_GetItem(args, 1); + if PyBytes_Check(data) == 0 { + PyErr_SetString( + PyExc_TypeError, +diff --git a/src/ffi.rs b/src/ffi.rs +index 4e5ddc3..20c9db4 100644 +--- a/src/ffi.rs ++++ b/src/ffi.rs +@@ -7,13 +7,16 @@ use std::ptr::NonNull; + #[allow(non_snake_case)] + #[inline(always)] + pub unsafe fn PyBytes_AS_STRING(op: *mut PyObject) -> *const c_char { +- &(*op.cast::()).ob_sval as *const c_char ++ #[cfg(not(any(PyPy, GraalPy, Py_LIMITED_API)))] ++ return &(*op.cast::()).ob_sval as *const c_char; ++ #[cfg(any(PyPy, GraalPy, Py_LIMITED_API))] ++ return crate::PyBytes_AsString(op); + } + + #[allow(non_snake_case)] + #[inline(always)] + pub unsafe fn PyBytes_GET_SIZE(op: *mut PyObject) -> Py_ssize_t { +- (*op.cast::()).ob_size ++ Py_SIZE(op) + } + + #[repr(C)] +@@ -63,11 +66,21 @@ pub fn pylong_is_positive(op: *mut PyObject) -> bool { + unsafe { (*(op as *mut PyLongObject)).long_value.lv_tag & SIGN_MASK == 0 } + } + +-#[cfg(not(Py_3_12))] ++#[cfg(not(any(Py_3_12, GraalPy)))] + pub fn pylong_is_positive(op: *mut PyObject) -> bool { + unsafe { (*(op as *mut PyVarObject)).ob_size > 0 } + } + ++extern "C" { ++ #[cfg(not(PyPy))] ++ pub fn _PyLong_Sign(v: *mut PyObject) -> c_int; ++} ++ ++#[cfg(GraalPy)] ++pub fn pylong_is_positive(op: *mut PyObject) -> bool { ++ unsafe { _PyLong_Sign(op) > 0 } ++} ++ + pub struct PyDictIter { + op: *mut PyObject, + pos: isize, +diff --git a/src/lib.rs b/src/lib.rs +index f10b1c4..1a9768b 100644 +--- a/src/lib.rs ++++ b/src/lib.rs +@@ -143,7 +143,7 @@ fn raise_unpackb_exception(msg: &str) -> *mut PyObject { + let err_msg = + PyUnicode_FromStringAndSize(msg.as_ptr() as *const c_char, msg.len() as isize); + let args = PyTuple_New(1); +- PyTuple_SET_ITEM(args, 0, err_msg); ++ PyTuple_SetItem(args, 0, err_msg); + PyErr_SetObject(typeref::MsgpackDecodeError, args); + Py_DECREF(args); + }; +@@ -199,10 +199,10 @@ pub unsafe extern "C" fn unpackb( + if !kwnames.is_null() { + let tuple_size = PyTuple_GET_SIZE(kwnames); + for i in 0..tuple_size { +- let arg = PyTuple_GET_ITEM(kwnames, i as Py_ssize_t); +- if arg == typeref::EXT_HOOK { ++ let arg = PyTuple_GetItem(kwnames, i as Py_ssize_t); ++ if PyUnicode_Compare(arg, typeref::EXT_HOOK) == 0 { + ext_hook = Some(NonNull::new_unchecked(*args.offset(num_args + i))); +- } else if arg == typeref::OPTION { ++ } else if PyUnicode_Compare(arg, typeref::OPTION) == 0 { + optsptr = Some(NonNull::new_unchecked(*args.offset(num_args + i))); + } else { + return raise_unpackb_exception("unpackb() got an unexpected keyword argument"); +@@ -247,15 +247,15 @@ pub unsafe extern "C" fn packb( + if !kwnames.is_null() { + let tuple_size = PyTuple_GET_SIZE(kwnames); + for i in 0..tuple_size { +- let arg = PyTuple_GET_ITEM(kwnames, i as Py_ssize_t); +- if arg == typeref::DEFAULT { ++ let arg = PyTuple_GetItem(kwnames, i as Py_ssize_t); ++ if PyUnicode_Compare(arg, typeref::DEFAULT) == 0 { + if unlikely!(default.is_some()) { + return raise_packb_exception( + "packb() got multiple values for argument: 'default'", + ); + } + default = Some(NonNull::new_unchecked(*args.offset(num_args + i))); +- } else if arg == typeref::OPTION { ++ } else if PyUnicode_Compare(arg, typeref::OPTION) == 0 { + if unlikely!(optsptr.is_some()) { + return raise_packb_exception( + "packb() got multiple values for argument: 'option'", +diff --git a/src/serialize/datetime.rs b/src/serialize/datetime.rs +index 63212d6..5ac2b2b 100644 +--- a/src/serialize/datetime.rs ++++ b/src/serialize/datetime.rs +@@ -61,9 +61,14 @@ pub struct Time { + + impl Time { + pub fn new(ptr: *mut pyo3::ffi::PyObject, opts: Opt) -> Result { ++ #[cfg(not(GraalPy))] + if unsafe { (*(ptr as *mut pyo3::ffi::PyDateTime_Time)).hastzinfo != 0 } { + return Err(TimeError::HasTimezone); + } ++ #[cfg(GraalPy)] ++ if unsafe { pyo3::ffi::PyDateTime_TIME_GET_TZINFO(ptr) != crate::typeref::NONE } { ++ return Err(TimeError::HasTimezone); ++ } + Ok(Time { + ptr: ptr, + opts: opts, +@@ -114,23 +119,28 @@ impl std::fmt::Display for DateTimeError { + } + + fn utcoffset(ptr: *mut pyo3::ffi::PyObject) -> Result { ++ #[cfg(not(GraalPy))] + if !unsafe { (*(ptr as *mut pyo3::ffi::PyDateTime_DateTime)).hastzinfo == 1 } { + return Ok(Offset::default()); + } + + let tzinfo = ffi!(PyDateTime_DATE_GET_TZINFO(ptr)); ++ #[cfg(GraalPy)] ++ if unsafe { tzinfo == crate::typeref::NONE } { ++ return Ok(Offset::default()); ++ } + let py_offset: *mut pyo3::ffi::PyObject; + if ffi!(PyObject_HasAttr(tzinfo, CONVERT_METHOD_STR)) == 1 { + // pendulum +- py_offset = ffi!(PyObject_CallMethodNoArgs(ptr, UTCOFFSET_METHOD_STR)); ++ py_offset = unsafe { pyo3::ffi::compat::PyObject_CallMethodNoArgs(ptr, UTCOFFSET_METHOD_STR) }; + } else if ffi!(PyObject_HasAttr(tzinfo, NORMALIZE_METHOD_STR)) == 1 { + // pytz +- let normalized = ffi!(PyObject_CallMethodOneArg(tzinfo, NORMALIZE_METHOD_STR, ptr)); +- py_offset = ffi!(PyObject_CallMethodNoArgs(normalized, UTCOFFSET_METHOD_STR)); ++ let normalized = ffi!(PyObject_CallMethodObjArgs(tzinfo, NORMALIZE_METHOD_STR, ptr, std::ptr::null_mut::())); ++ py_offset = unsafe { pyo3::ffi::compat::PyObject_CallMethodNoArgs(normalized, UTCOFFSET_METHOD_STR) }; + ffi!(Py_DECREF(normalized)); + } else if ffi!(PyObject_HasAttr(tzinfo, DST_STR)) == 1 { + // dateutil/arrow, datetime.timezone.utc +- py_offset = ffi!(PyObject_CallMethodOneArg(tzinfo, UTCOFFSET_METHOD_STR, ptr)); ++ py_offset = ffi!(PyObject_CallMethodObjArgs(tzinfo, UTCOFFSET_METHOD_STR, ptr, std::ptr::null_mut::())); + } else { + return Err(DateTimeError::LibraryUnsupported); + } +@@ -193,7 +203,10 @@ impl TimeLike for DateTime { + + impl DateTimeLike for DateTime { + fn has_tz(&self) -> bool { +- unsafe { (*(self.ptr as *mut pyo3::ffi::PyDateTime_DateTime)).hastzinfo == 1 } ++ #[cfg(not(GraalPy))] ++ return unsafe { (*(self.ptr as *mut pyo3::ffi::PyDateTime_DateTime)).hastzinfo == 1 }; ++ #[cfg(GraalPy)] ++ return unsafe { pyo3::ffi::PyDateTime_TIME_GET_TZINFO(self.ptr) != crate::typeref::NONE }; + } + + fn offset(&self) -> Offset { +diff --git a/src/serialize/numpy.rs b/src/serialize/numpy.rs +index afc5cdf..4d007bd 100644 +--- a/src/serialize/numpy.rs ++++ b/src/serialize/numpy.rs +@@ -392,8 +392,8 @@ impl NumpyDatetimeUnit { + fn from_pyobject(ptr: *mut PyObject) -> Self { + let dtype = ffi!(PyObject_GetAttr(ptr, DTYPE_STR)); + let descr = ffi!(PyObject_GetAttr(dtype, DESCR_STR)); +- let el0 = ffi!(PyList_GET_ITEM(descr, 0)); +- let descr_str = ffi!(PyTuple_GET_ITEM(el0, 1)); ++ let el0 = ffi!(PyList_GetItem(descr, 0)); ++ let descr_str = ffi!(PyTuple_GetItem(el0, 1)); + let uni = crate::unicode::unicode_to_str(descr_str).unwrap(); + if uni.len() < 5 { + return Self::NaT; +diff --git a/src/serialize/serializer.rs b/src/serialize/serializer.rs +index 309e6e1..6f7dec7 100644 +--- a/src/serialize/serializer.rs ++++ b/src/serialize/serializer.rs +@@ -864,7 +864,7 @@ impl Serialize for DictTupleKey { + let len = ffi!(PyTuple_GET_SIZE(self.ptr)) as usize; + let mut seq = serializer.serialize_seq(Some(len)).unwrap(); + for i in 0..len { +- let item = ffi!(PyTuple_GET_ITEM(self.ptr, i as isize)); ++ let item = ffi!(PyTuple_GetItem(self.ptr, i as isize)); + let value = DictKey::new(item, self.opts, self.recursion + 1); + seq.serialize_element(&value)?; + } +diff --git a/src/serialize/tuple.rs b/src/serialize/tuple.rs +index fa81cb6..9b66019 100644 +--- a/src/serialize/tuple.rs ++++ b/src/serialize/tuple.rs +@@ -41,7 +41,7 @@ impl Serialize for Tuple { + let len = ffi!(PyTuple_GET_SIZE(self.ptr)) as usize; + let mut seq = serializer.serialize_seq(Some(len)).unwrap(); + for i in 0..len { +- let item = ffi!(PyTuple_GET_ITEM(self.ptr, i as isize)); ++ let item = ffi!(PyTuple_GetItem(self.ptr, i as isize)); + let value = PyObject::new( + item, + self.opts, +diff --git a/src/serialize/writer.rs b/src/serialize/writer.rs +index a790bdd..35346d9 100644 +--- a/src/serialize/writer.rs ++++ b/src/serialize/writer.rs +@@ -27,7 +27,6 @@ impl BytesWriter { + pub fn finish(&mut self) -> NonNull { + unsafe { + std::ptr::write(self.buffer_ptr(), 0); +- (*self.bytes.cast::()).ob_size = self.len as Py_ssize_t; + self.resize(self.len); + NonNull::new_unchecked(self.bytes as *mut PyObject) + } +@@ -35,10 +34,14 @@ impl BytesWriter { + + fn buffer_ptr(&self) -> *mut u8 { + unsafe { +- std::mem::transmute::<*mut [c_char; 1], *mut u8>(std::ptr::addr_of_mut!( ++ #[cfg(not(GraalPy))] ++ return std::mem::transmute::<*mut [c_char; 1], *mut u8>(std::ptr::addr_of_mut!( + (*self.bytes).ob_sval + )) +- .add(self.len) ++ .add(self.len); ++ #[cfg(GraalPy)] ++ return std::mem::transmute::<*mut i8, *mut u8>(PyBytes_AsString(self.bytes.cast::())) ++ .add(self.len); + } + } + +diff --git a/src/unicode.rs b/src/unicode.rs +index 53aca09..552fa6c 100644 +--- a/src/unicode.rs ++++ b/src/unicode.rs +@@ -6,6 +6,7 @@ use pyo3::ffi::*; + + // see unicodeobject.h for documentation + ++#[cfg(not(GraalPy))] + pub fn unicode_from_str(buf: &str) -> *mut PyObject { + if buf.is_empty() { + ffi!(Py_INCREF(EMPTY_UNICODE)); +@@ -27,6 +28,13 @@ pub fn unicode_from_str(buf: &str) -> *mut PyObject { + } + } + ++#[cfg(GraalPy)] ++pub fn unicode_from_str(buf: &str) -> *mut PyObject { ++ unsafe { ++ PyUnicode_FromStringAndSize(buf.as_ptr() as *const i8, buf.len() as isize) ++ } ++} ++ + fn pyunicode_ascii(buf: &str) -> *mut PyObject { + unsafe { + let ptr = ffi!(PyUnicode_New(buf.len() as isize, 127)); +@@ -80,6 +88,7 @@ fn pyunicode_fourbyte(buf: &str, num_chars: usize) -> *mut PyObject { + + #[inline] + pub fn hash_str(op: *mut PyObject) -> Py_hash_t { ++ #[cfg(not(GraalPy))] + unsafe { + let data_ptr: *mut c_void = if (*op.cast::()).compact() == 1 + && (*op.cast::()).ascii() == 1 +@@ -92,7 +101,11 @@ pub fn hash_str(op: *mut PyObject) -> Py_hash_t { + (*(op as *mut PyASCIIObject)).length * ((*(op as *mut PyASCIIObject)).kind()) as isize; + let hash = _Py_HashBytes(data_ptr, num_bytes); + (*op.cast::()).hash = hash; +- hash ++ return hash; ++ } ++ #[cfg(GraalPy)] ++ unsafe { ++ return PyObject_Hash(op); + } + } + +@@ -109,19 +122,24 @@ pub fn unicode_to_str_via_ffi(op: *mut PyObject) -> Option<&'static str> { + + #[inline] + pub fn unicode_to_str(op: *mut PyObject) -> Option<&'static str> { ++ #[cfg(not(GraalPy))] + unsafe { + if unlikely!((*op.cast::()).compact() == 0) { +- unicode_to_str_via_ffi(op) ++ return unicode_to_str_via_ffi(op); + } else if (*op.cast::()).ascii() == 1 { + let ptr = op.cast::().offset(1) as *const u8; + let len = (*op.cast::()).length as usize; +- Some(str_from_slice!(ptr, len)) ++ return Some(str_from_slice!(ptr, len)); + } else if (*op.cast::()).utf8_length != 0 { + let ptr = (*op.cast::()).utf8 as *const u8; + let len = (*op.cast::()).utf8_length as usize; +- Some(str_from_slice!(ptr, len)) ++ return Some(str_from_slice!(ptr, len)); + } else { +- unicode_to_str_via_ffi(op) ++ return unicode_to_str_via_ffi(op); + } + } ++ #[cfg(GraalPy)] ++ unsafe { ++ return unicode_to_str_via_ffi(op); ++ } + } +diff --git a/src/util.rs b/src/util.rs +index 2bcc32d..89faf1a 100644 +--- a/src/util.rs ++++ b/src/util.rs +@@ -8,7 +8,7 @@ macro_rules! py_is { + + macro_rules! ob_type { + ($obj:expr) => { +- unsafe { (*($obj as *mut pyo3::ffi::PyObject)).ob_type } ++ unsafe { pyo3::ffi::Py_TYPE($obj as *mut pyo3::ffi::PyObject) } + }; + } + +-- +2.43.0 + From b50b6bc525f7ced037aaf3941286be9c9724477c Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Tue, 6 May 2025 12:54:14 +0200 Subject: [PATCH 34/82] Add patch for pythran 0.16+ Related #501 --- .../lib-graalpython/patches/metadata.toml | 7 +- .../patches/pythran-0.16.patch | 70 +++++++++++++++++++ 2 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 graalpython/lib-graalpython/patches/pythran-0.16.patch diff --git a/graalpython/lib-graalpython/patches/metadata.toml b/graalpython/lib-graalpython/patches/metadata.toml index 3c331d1657..7be5bef267 100644 --- a/graalpython/lib-graalpython/patches/metadata.toml +++ b/graalpython/lib-graalpython/patches/metadata.toml @@ -564,10 +564,15 @@ patch = 'pythran-0.13.patch' license = 'BSD-3-Clause' [[pythran.rules]] -version = '>= 0.14' +version = '>= 0.14, < 0.16' patch = 'pythran-0.15.patch' license = 'BSD-3-Clause' +[[pythran.rules]] +version = '>= 0.16' +patch = 'pythran-0.16.patch' +license = 'BSD-3-Clause' + [[pyzmq.rules]] # 26+ needs Cython 3 version = '< 26' diff --git a/graalpython/lib-graalpython/patches/pythran-0.16.patch b/graalpython/lib-graalpython/patches/pythran-0.16.patch new file mode 100644 index 0000000000..3fda5ec7ab --- /dev/null +++ b/graalpython/lib-graalpython/patches/pythran-0.16.patch @@ -0,0 +1,70 @@ +diff --git a/pythran/pythonic/python/core.hpp b/pythran/pythonic/python/core.hpp +index 4cbe3e0..24340b0 100644 +--- a/pythran/pythonic/python/core.hpp ++++ b/pythran/pythonic/python/core.hpp +@@ -53,7 +53,14 @@ namespace python + { + + #ifndef PyString_AS_STRING +-#define PyString_AS_STRING (char *)_PyUnicode_COMPACT_DATA ++ static inline const char* PyString_AS_STRING(PyObject* obj) { ++ const char* str = PyUnicode_AsUTF8(obj); ++ if (!str) { ++ PyErr_Clear(); ++ str = ""; ++ } ++ return str; ++ } + #endif + + inline void PyObject_TypePrettyPrinter(std::ostream &oss, PyObject *obj) +diff --git a/pythran/pythonic/types/str.hpp b/pythran/pythonic/types/str.hpp +index e5dbe60..41d1658 100644 +--- a/pythran/pythonic/types/str.hpp ++++ b/pythran/pythonic/types/str.hpp +@@ -741,10 +741,17 @@ namespace std + #define PyString_FromStringAndSize PyUnicode_FromStringAndSize + + #ifndef PyString_Check +-#define PyString_Check(x) PyUnicode_Check(x) && PyUnicode_IS_COMPACT_ASCII(x) ++#define PyString_Check(x) PyUnicode_Check(x) + #endif + #ifndef PyString_AS_STRING +-#define PyString_AS_STRING (char *)_PyUnicode_COMPACT_DATA ++ static inline const char* PyString_AS_STRING(PyObject* obj) { ++ const char* str = PyUnicode_AsUTF8(obj); ++ if (!str) { ++ PyErr_Clear(); ++ str = ""; ++ } ++ return str; ++ } + #endif + #ifndef PyString_GET_SIZE + #define PyString_GET_SIZE PyUnicode_GET_LENGTH +diff --git a/pythran/tables.py b/pythran/tables.py +index d62abe1..4ba279d 100644 +--- a/pythran/tables.py ++++ b/pythran/tables.py +@@ -4617,7 +4617,10 @@ def save_arguments(module_name, elements): + # some function are actually forward function, detect those + # and accept to use our description instead. + if looks_like_a_forward_function(spec): +- assert signature.args.args, "{} require an explicit description".format(elem) ++ # GraalPy change: we have signatures for more builtins than ++ # CPython and this trips up on type constructors like `dict` or ++ # `BaseException`. ++ # assert signature.args.args, "{} require an explicit description".format(elem) + continue + + args = [ast.Name(arg, ast.Param(), None, None) +@@ -4630,7 +4633,8 @@ def save_arguments(module_name, elements): + defaults = list(spec.defaults or []) + args += [ast.Name(arg, ast.Param(), None, None) + for arg in spec.kwonlyargs] +- defaults += [spec.kwonlydefaults[kw] for kw in spec.kwonlyargs] ++ if spec.kwonlydefaults: ++ defaults += [spec.kwonlydefaults[kw] for kw in spec.kwonlyargs] + + # Check if we already have a pythran description for that object + if signature.args.args: From 0ba7a7f2384cbfb68415d972e422284eb73cc854 Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Wed, 21 May 2025 17:54:34 +0200 Subject: [PATCH 35/82] Work around missing AttributeError kwargs --- .../lib-graalpython/patches/metadata.toml | 8 +- .../lib-graalpython/patches/numpy-2.2.4.patch | 182 ++++++++++++++++++ 2 files changed, 189 insertions(+), 1 deletion(-) create mode 100644 graalpython/lib-graalpython/patches/numpy-2.2.4.patch diff --git a/graalpython/lib-graalpython/patches/metadata.toml b/graalpython/lib-graalpython/patches/metadata.toml index 7be5bef267..b9d99a732b 100644 --- a/graalpython/lib-graalpython/patches/metadata.toml +++ b/graalpython/lib-graalpython/patches/metadata.toml @@ -296,7 +296,13 @@ patch = 'numba-0.59.1.patch' license = 'BSD-2-Clause' [[numpy.rules]] -version = '>= 2.0.0rc1, < 2.3' +version = '== 2.2.4' +patch = 'numpy-2.2.4.patch' +license = 'BSD-3-Clause' +dist-type = 'sdist' + +[[numpy.rules]] +version = '>= 2.0.0rc1, < 2.1' patch = 'numpy-2.0.0.patch' license = 'BSD-3-Clause' dist-type = 'sdist' diff --git a/graalpython/lib-graalpython/patches/numpy-2.2.4.patch b/graalpython/lib-graalpython/patches/numpy-2.2.4.patch new file mode 100644 index 0000000000..49760fcb51 --- /dev/null +++ b/graalpython/lib-graalpython/patches/numpy-2.2.4.patch @@ -0,0 +1,182 @@ +diff --git a/numpy/__init__.py b/numpy/__init__.py +index 2a4fd03..4df4f2d 100644 +--- a/numpy/__init__.py ++++ b/numpy/__init__.py +@@ -371,7 +371,7 @@ def __getattr__(attr): + return char + elif attr == "array_api": + raise AttributeError("`numpy.array_api` is not available from " +- "numpy 2.0 onwards", name=None) ++ "numpy 2.0 onwards") + elif attr == "core": + import numpy.core as core + return core +@@ -384,7 +384,7 @@ def __getattr__(attr): + return distutils + else: + raise AttributeError("`numpy.distutils` is not available from " +- "Python 3.12 onwards", name=None) ++ "Python 3.12 onwards") + + if attr in __future_scalars__: + # And future warnings for those that will change, but also give +@@ -394,13 +394,12 @@ def __getattr__(attr): + "corresponding NumPy scalar.", FutureWarning, stacklevel=2) + + if attr in __former_attrs__: +- raise AttributeError(__former_attrs__[attr], name=None) ++ raise AttributeError(__former_attrs__[attr]) + + if attr in __expired_attributes__: + raise AttributeError( + f"`np.{attr}` was removed in the NumPy 2.0 release. " + f"{__expired_attributes__[attr]}", +- name=None + ) + + if attr == "chararray": +diff --git a/numpy/_core/include/numpy/ndarrayobject.h b/numpy/_core/include/numpy/ndarrayobject.h +index 0462625..3625e34 100644 +--- a/numpy/_core/include/numpy/ndarrayobject.h ++++ b/numpy/_core/include/numpy/ndarrayobject.h +@@ -220,7 +220,7 @@ NPY_TITLE_KEY_check(PyObject *key, PyObject *value) + if (key == title) { + return 1; + } +-#ifdef PYPY_VERSION ++#if defined(PYPY_VERSION) || defined(GRAALVM_PYTHON) + /* + * On PyPy, dictionary keys do not always preserve object identity. + * Fall back to comparison by value. +diff --git a/numpy/_core/src/multiarray/compiled_base.c b/numpy/_core/src/multiarray/compiled_base.c +index e3af951..48a7030 100644 +--- a/numpy/_core/src/multiarray/compiled_base.c ++++ b/numpy/_core/src/multiarray/compiled_base.c +@@ -1465,6 +1465,7 @@ fail: + NPY_NO_EXPORT PyObject * + arr_add_docstring(PyObject *NPY_UNUSED(dummy), PyObject *const *args, Py_ssize_t len_args) + { ++#if 0 // GraalPy change + PyObject *obj; + PyObject *str; + const char *docstr; +@@ -1569,6 +1570,7 @@ arr_add_docstring(PyObject *NPY_UNUSED(dummy), PyObject *const *args, Py_ssize_t + } + + #undef _ADDDOC ++#endif // GraalPy change + + Py_RETURN_NONE; + } +diff --git a/numpy/_core/src/multiarray/shape.c b/numpy/_core/src/multiarray/shape.c +index 340fe72..8f13674 100644 +--- a/numpy/_core/src/multiarray/shape.c ++++ b/numpy/_core/src/multiarray/shape.c +@@ -97,6 +97,11 @@ PyArray_Resize(PyArrayObject *self, PyArray_Dims *newshape, int refcheck, + "cannot resize an array with refcheck=True on PyPy.\n" + "Use the np.resize function or refcheck=False"); + return NULL; ++#elif defined(GRAALVM_PYTHON) ++ PyErr_SetString(PyExc_ValueError, ++ "cannot resize an array with refcheck=True on GraalPy.\n" ++ "Use the np.resize function or refcheck=False"); ++ return NULL; + #else + refcnt = Py_REFCNT(self); + #endif /* PYPY_VERSION */ +diff --git a/numpy/_core/src/multiarray/stringdtype/dtype.c b/numpy/_core/src/multiarray/stringdtype/dtype.c +index 0350375..8b254cb 100644 +--- a/numpy/_core/src/multiarray/stringdtype/dtype.c ++++ b/numpy/_core/src/multiarray/stringdtype/dtype.c +@@ -841,7 +841,7 @@ init_string_dtype(void) + }; + + /* Loaded dynamically, so needs to be set here: */ +- Py_TYPE(((PyObject *)&PyArray_StringDType)) = &PyArrayDTypeMeta_Type; ++ Py_SET_TYPE(((PyObject *)&PyArray_StringDType), &PyArrayDTypeMeta_Type); + ((PyTypeObject *)&PyArray_StringDType)->tp_base = &PyArrayDescr_Type; + if (PyType_Ready((PyTypeObject *)&PyArray_StringDType) < 0) { + return -1; +diff --git a/numpy/_core/src/multiarray/temp_elide.c b/numpy/_core/src/multiarray/temp_elide.c +index 662a2fa..791ede8 100644 +--- a/numpy/_core/src/multiarray/temp_elide.c ++++ b/numpy/_core/src/multiarray/temp_elide.c +@@ -58,7 +58,7 @@ + * supported too by using the appropriate Windows APIs. + */ + +-#if defined HAVE_BACKTRACE && defined HAVE_DLFCN_H && ! defined PYPY_VERSION ++#if defined HAVE_BACKTRACE && defined HAVE_DLFCN_H && ! defined PYPY_VERSION && !defined(GRAALVM_PYTHON) + + #include + +diff --git a/numpy/_core/src/npymath/ieee754.c.src b/numpy/_core/src/npymath/ieee754.c.src +index 8fccc9a..3bb9cf0 100644 +--- a/numpy/_core/src/npymath/ieee754.c.src ++++ b/numpy/_core/src/npymath/ieee754.c.src +@@ -362,6 +362,11 @@ int npy_get_floatstatus_barrier(char* param) + * By using a volatile, the compiler cannot reorder this call + */ + if (param != NULL) { ++ // GraalPy change: the pointer needs to be dereferenced to establish ++ // a data dependency to to ensure the compiler won't reorder the call ++ if (points_to_py_handle_space(param)) { ++ param = (char*)pointer_to_stub(param); ++ } + volatile char NPY_UNUSED(c) = *(char*)param; + } + +diff --git a/numpy/_core/src/npymath/ieee754.cpp b/numpy/_core/src/npymath/ieee754.cpp +index 1c59bf3..519fabc 100644 +--- a/numpy/_core/src/npymath/ieee754.cpp ++++ b/numpy/_core/src/npymath/ieee754.cpp +@@ -428,6 +428,11 @@ npy_get_floatstatus_barrier(char *param) + * By using a volatile, the compiler cannot reorder this call + */ + if (param != NULL) { ++ // GraalPy change: the pointer needs to be dereferenced to establish ++ // a data dependency to to ensure the compiler won't reorder the call ++ if (points_to_py_handle_space(param)) { ++ param = (char*)pointer_to_stub(param); ++ } + volatile char NPY_UNUSED(c) = *(char *)param; + } + +diff --git a/numpy/lib/__init__.py b/numpy/lib/__init__.py +index 928121c..bbe6a7e 100644 +--- a/numpy/lib/__init__.py ++++ b/numpy/lib/__init__.py +@@ -70,7 +70,6 @@ def __getattr__(attr): + "numpy.lib.emath was an alias for emath module that was removed " + "in NumPy 2.0. Replace usages of numpy.lib.emath with " + "numpy.emath.", +- name=None + ) + elif attr in ( + "histograms", "type_check", "nanfunctions", "function_base", +@@ -81,13 +80,11 @@ def __getattr__(attr): + f"numpy.lib.{attr} is now private. If you are using a public " + "function, it should be available in the main numpy namespace, " + "otherwise check the NumPy 2.0 migration guide.", +- name=None + ) + elif attr == "arrayterator": + raise AttributeError( + "numpy.lib.arrayterator submodule is now private. To access " + "Arrayterator class use numpy.lib.Arrayterator.", +- name=None + ) + else: + raise AttributeError("module {!r} has no attribute " +diff --git a/vendored-meson/meson/mesonbuild/utils/universal.py b/vendored-meson/meson/mesonbuild/utils/universal.py +index 6aee268..539be8b 100644 +--- a/vendored-meson/meson/mesonbuild/utils/universal.py ++++ b/vendored-meson/meson/mesonbuild/utils/universal.py +@@ -728,6 +728,7 @@ def windows_detect_native_arch() -> str: + """ + if sys.platform != 'win32': + return '' ++ return 'amd64' # Workaround for GraalPy bug on Windows with kernel32.GetCurrentProcess() + try: + import ctypes + process_arch = ctypes.c_ushort() From 6c98b10d65acfa01cfdc112a669e492ce642d21d Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Tue, 20 May 2025 15:05:20 +0200 Subject: [PATCH 36/82] Fix torch 2.4.1 patch --- .../lib-graalpython/patches/torch-2.4.1.patch | 45 +++++++++++++------ 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/graalpython/lib-graalpython/patches/torch-2.4.1.patch b/graalpython/lib-graalpython/patches/torch-2.4.1.patch index caebd3aa86..3dc4d354e3 100644 --- a/graalpython/lib-graalpython/patches/torch-2.4.1.patch +++ b/graalpython/lib-graalpython/patches/torch-2.4.1.patch @@ -178,6 +178,19 @@ index f5fdbf155..d76176cb6 100644 class TestWrapperSubclassAliasing(TestCase): +diff --git a/third_party/fbgemm/CMakeLists.txt b/third_party/fbgemm/CMakeLists.txt +index 134523e7d..a00538e3c 100644 +--- a/third_party/fbgemm/CMakeLists.txt ++++ b/third_party/fbgemm/CMakeLists.txt +@@ -10,6 +10,8 @@ + + cmake_minimum_required(VERSION 3.16 FATAL_ERROR) + ++add_compile_options(-Wno-error=maybe-uninitialized -Wno-error=uninitialized -Wno-error=restrict) ++ + list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules") + + # Define function to extract filelists from defs.bzl file diff --git a/third_party/pybind11/include/pybind11/detail/common.h b/third_party/pybind11/include/pybind11/detail/common.h index 454e6061b..7feafc7d7 100644 --- a/third_party/pybind11/include/pybind11/detail/common.h @@ -537,20 +550,6 @@ index c301da982..a2668be20 100644 +const int THP_PyOpcode_Caches_size = 0; + +#endif // GraalPy change -diff --git a/torch/csrc/dynamo/extra_state.c b/torch/csrc/dynamo/extra_state.c -index cbe9ab37a..18740a0d8 100644 ---- a/torch/csrc/dynamo/extra_state.c -+++ b/torch/csrc/dynamo/extra_state.c -@@ -100,9 +100,6 @@ void destroy_extra_state(void* obj) { - } - - void set_extra_state(PyCodeObject* code, ExtraState* extra_state) { -- ExtraState* old_extra_state = get_extra_state(code); -- CHECK(extra_state == nullptr || old_extra_state != extra_state); -- _PyCode_SetExtra((PyObject*)code, extra_index, extra_state); - } - - ExtraState* init_and_set_extra_state(PyCodeObject* code) { diff --git a/torch/csrc/dynamo/eval_frame.c b/torch/csrc/dynamo/eval_frame.c index cbe9ab37a..18740a0d8 100644 --- a/torch/csrc/dynamo/eval_frame.c @@ -628,6 +627,24 @@ index cbe9ab37a..18740a0d8 100644 return module; } +diff --git a/torch/csrc/dynamo/extra_state.cpp b/torch/csrc/dynamo/extra_state.cpp +index 7c9b4be00..b8edbcfda 100644 +--- a/torch/csrc/dynamo/extra_state.cpp ++++ b/torch/csrc/dynamo/extra_state.cpp +@@ -65,11 +65,13 @@ void destroy_extra_state(void* obj) { + } + + void set_extra_state(PyCodeObject* code, ExtraState* extra_state) { ++#if 0 // GraalPy change + ExtraState* old_extra_state = get_extra_state(code); + CHECK( + old_extra_state == nullptr || old_extra_state == SKIP_CODE || + old_extra_state != extra_state); + _PyCode_SetExtra((PyObject*)code, extra_index, extra_state); ++#endif // GraalPy change + } + + ExtraState* init_and_set_extra_state(PyCodeObject* code) { diff --git a/torch/csrc/jit/python/python_tracer.cpp b/torch/csrc/jit/python/python_tracer.cpp index 92e6e2d3a..4d2ec0bfe 100644 --- a/torch/csrc/jit/python/python_tracer.cpp From 7027419eea35d0e96c51fe9c1b91682230b4f613 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Thu, 31 Jul 2025 11:02:52 +0200 Subject: [PATCH 37/82] Backport orjson 3.10.18 patch 24.2.2 --- .../lib-graalpython/patches/metadata.toml | 5 + .../patches/orjson-3.10.18.patch | 933 ++++++++++++++++++ 2 files changed, 938 insertions(+) create mode 100644 graalpython/lib-graalpython/patches/orjson-3.10.18.patch diff --git a/graalpython/lib-graalpython/patches/metadata.toml b/graalpython/lib-graalpython/patches/metadata.toml index b9d99a732b..1ff2415c92 100644 --- a/graalpython/lib-graalpython/patches/metadata.toml +++ b/graalpython/lib-graalpython/patches/metadata.toml @@ -352,6 +352,11 @@ version = '== 3.10.5' patch = 'orjson-3.10.5.patch' license = 'Apache-2.0 OR MIT' +[[orjson.rules]] +version = '== 3.10.18' +patch = 'orjson-3.10.18.patch' +license = 'Apache-2.0 OR MIT' + [[ormsgpack.rules]] version = '>= 1.8.0, <= 1.9.1' patch = 'ormsgpack-1.8.0-1.9.1.patch' diff --git a/graalpython/lib-graalpython/patches/orjson-3.10.18.patch b/graalpython/lib-graalpython/patches/orjson-3.10.18.patch new file mode 100644 index 0000000000..3677bf64b9 --- /dev/null +++ b/graalpython/lib-graalpython/patches/orjson-3.10.18.patch @@ -0,0 +1,933 @@ +commit a889953c08d33854e574ab06e9593e0484c4c147 +Author: stepan +Date: Wed Jul 9 18:50:37 2025 +0200 + + Adapt for GraalPy + +diff --git a/build.rs b/build.rs +index 43538c0..85857ba 100644 +--- a/build.rs ++++ b/build.rs +@@ -40,7 +40,8 @@ fn main() { + + #[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))] + if is_64_bit_python { +- println!("cargo:rustc-cfg=feature=\"inline_int\""); ++ // cannot serialize integers that don't fit into 64b ++ // println!("cargo:rustc-cfg=feature=\"inline_int\""); + } + + if env::var("ORJSON_DISABLE_YYJSON").is_ok() { +diff --git a/include/pyo3/pyo3-ffi/src/cpython/abstract_.rs b/include/pyo3/pyo3-ffi/src/cpython/abstract_.rs +index 477ad02..0b649f3 100644 +--- a/include/pyo3/pyo3-ffi/src/cpython/abstract_.rs ++++ b/include/pyo3/pyo3-ffi/src/cpython/abstract_.rs +@@ -1,12 +1,12 @@ + use crate::{PyObject, Py_ssize_t}; +-#[cfg(not(all(Py_3_11, GraalPy)))] ++#[cfg(any(all(Py_3_8, not(PyPy)), not(Py_3_11)))] + use std::os::raw::c_char; + use std::os::raw::c_int; + + #[cfg(not(Py_3_11))] + use crate::Py_buffer; + +-#[cfg(all(Py_3_8, not(any(PyPy, GraalPy))))] ++#[cfg(all(Py_3_8, not(PyPy)))] + use crate::{ + vectorcallfunc, PyCallable_Check, PyThreadState, PyThreadState_GET, PyTuple_Check, + PyType_HasFeature, Py_TPFLAGS_HAVE_VECTORCALL, +@@ -23,7 +23,7 @@ extern "C" { + const _PY_FASTCALL_SMALL_STACK: size_t = 5; + + extern "C" { +- #[cfg(all(Py_3_8, not(any(PyPy, GraalPy))))] ++ #[cfg(all(Py_3_8, not(PyPy)))] + pub fn _Py_CheckFunctionResult( + tstate: *mut PyThreadState, + callable: *mut PyObject, +@@ -31,7 +31,7 @@ extern "C" { + where_: *const c_char, + ) -> *mut PyObject; + +- #[cfg(all(Py_3_8, not(any(PyPy, GraalPy))))] ++ #[cfg(all(Py_3_8, not(PyPy)))] + pub fn _PyObject_MakeTpCall( + tstate: *mut PyThreadState, + callable: *mut PyObject, +@@ -52,7 +52,7 @@ pub unsafe fn PyVectorcall_NARGS(n: size_t) -> Py_ssize_t { + n.try_into().expect("cannot fail due to mask") + } + +-#[cfg(all(Py_3_8, not(any(PyPy, GraalPy))))] ++#[cfg(all(Py_3_8, not(PyPy)))] + #[inline(always)] + pub unsafe fn PyVectorcall_Function(callable: *mut PyObject) -> Option { + assert!(!callable.is_null()); +@@ -67,7 +67,7 @@ pub unsafe fn PyVectorcall_Function(callable: *mut PyObject) -> Option *m + } + + extern "C" { +- #[cfg(all(Py_3_9, not(any(PyPy, GraalPy))))] ++ #[cfg(all(Py_3_9, not(PyPy)))] + pub fn PyObject_VectorcallMethod( + name: *mut PyObject, + args: *const *mut PyObject, +@@ -197,7 +197,7 @@ extern "C" { + ) -> *mut PyObject; + } + +-#[cfg(all(Py_3_9, not(any(PyPy, GraalPy))))] ++#[cfg(all(Py_3_9, not(PyPy)))] + #[inline(always)] + pub unsafe fn PyObject_CallMethodNoArgs( + self_: *mut PyObject, +@@ -211,7 +211,7 @@ pub unsafe fn PyObject_CallMethodNoArgs( + ) + } + +-#[cfg(all(Py_3_9, not(any(PyPy, GraalPy))))] ++#[cfg(all(Py_3_9, not(any(PyPy))))] + #[inline(always)] + pub unsafe fn PyObject_CallMethodOneArg( + self_: *mut PyObject, +diff --git a/include/pyo3/pyo3-ffi/src/cpython/bytesobject.rs b/include/pyo3/pyo3-ffi/src/cpython/bytesobject.rs +index 306702d..f55bc70 100644 +--- a/include/pyo3/pyo3-ffi/src/cpython/bytesobject.rs ++++ b/include/pyo3/pyo3-ffi/src/cpython/bytesobject.rs +@@ -1,6 +1,6 @@ + use crate::object::*; + use crate::Py_ssize_t; +-#[cfg(not(any(PyPy, GraalPy, Py_LIMITED_API)))] ++#[cfg(not(Py_LIMITED_API))] + use std::os::raw::c_char; + use std::os::raw::c_int; + +@@ -23,3 +23,12 @@ extern "C" { + #[cfg_attr(PyPy, link_name = "_PyPyBytes_Resize")] + pub fn _PyBytes_Resize(bytes: *mut *mut PyObject, newsize: Py_ssize_t) -> c_int; + } ++ ++#[cfg(not(Py_LIMITED_API))] ++#[inline] ++pub unsafe fn PyBytes_AS_STRING(op: *mut PyObject) -> *const c_char { ++ #[cfg(not(any(PyPy, GraalPy)))] ++ return &(*op.cast::()).ob_sval as *const c_char; ++ #[cfg(any(PyPy, GraalPy))] ++ return crate::PyBytes_AsString(op); ++} +\ No newline at end of file +diff --git a/include/pyo3/pyo3-ffi/src/cpython/tupleobject.rs b/include/pyo3/pyo3-ffi/src/cpython/tupleobject.rs +index c06609f..29cc8fd 100644 +--- a/include/pyo3/pyo3-ffi/src/cpython/tupleobject.rs ++++ b/include/pyo3/pyo3-ffi/src/cpython/tupleobject.rs +@@ -28,6 +28,13 @@ pub unsafe fn PyTuple_GET_ITEM(op: *mut PyObject, i: Py_ssize_t) -> *mut PyObjec + *(*(op as *mut PyTupleObject)).ob_item.as_ptr().offset(i) + } + ++#[inline] ++#[cfg(any(PyPy, GraalPy))] ++pub unsafe fn PyTuple_GET_ITEM(op: *mut PyObject, i: Py_ssize_t) -> *mut PyObject { ++ use crate::PyTuple_GetItem; ++ PyTuple_GetItem(op, i) ++} ++ + /// Macro, *only* to be used to fill in brand new tuples + #[inline] + #[cfg(not(any(PyPy, GraalPy)))] +@@ -35,4 +42,11 @@ pub unsafe fn PyTuple_SET_ITEM(op: *mut PyObject, i: Py_ssize_t, v: *mut PyObjec + *(*(op as *mut PyTupleObject)).ob_item.as_mut_ptr().offset(i) = v; + } + ++#[inline] ++#[cfg(any(PyPy, GraalPy))] ++pub unsafe fn PyTuple_SET_ITEM(op: *mut PyObject, i: Py_ssize_t, v: *mut PyObject) { ++ use crate::PyTuple_SetItem; ++ PyTuple_SetItem(op, i, v); ++} ++ + // skipped _PyTuple_DebugMallocStats +diff --git a/include/pyo3/pyo3-ffi/src/cpython/unicodeobject.rs b/include/pyo3/pyo3-ffi/src/cpython/unicodeobject.rs +index 72da46c..1f03e3e 100644 +--- a/include/pyo3/pyo3-ffi/src/cpython/unicodeobject.rs ++++ b/include/pyo3/pyo3-ffi/src/cpython/unicodeobject.rs +@@ -401,7 +401,7 @@ pub struct PyUnicodeObject { + } + + extern "C" { +- #[cfg(not(any(PyPy, GraalPy)))] ++ #[cfg(not(PyPy))] + pub fn _PyUnicode_CheckConsistency(op: *mut PyObject, check_content: c_int) -> c_int; + } + +diff --git a/include/pyo3/pyo3-ffi/src/object.rs b/include/pyo3/pyo3-ffi/src/object.rs +index 51083a8..9a5971f 100644 +--- a/include/pyo3/pyo3-ffi/src/object.rs ++++ b/include/pyo3/pyo3-ffi/src/object.rs +@@ -168,7 +168,7 @@ pub unsafe fn Py_REFCNT(ob: *mut PyObject) -> Py_ssize_t { + local as Py_ssize_t + Py_ssize_t::from(shared >> _Py_REF_SHARED_SHIFT) + } + +- #[cfg(all(not(Py_GIL_DISABLED), Py_3_12))] ++ #[cfg(all(not(Py_GIL_DISABLED), Py_3_12, not(GraalPy)))] + { + (*ob).ob_refcnt.ob_refcnt + } +@@ -178,7 +178,7 @@ pub unsafe fn Py_REFCNT(ob: *mut PyObject) -> Py_ssize_t { + (*ob).ob_refcnt + } + +- #[cfg(all(not(Py_GIL_DISABLED), not(Py_3_12), GraalPy))] ++ #[cfg(any(all(not(Py_GIL_DISABLED), not(Py_3_12)), GraalPy))] + { + _Py_REFCNT(ob) + } +@@ -215,14 +215,10 @@ pub unsafe fn Py_SIZE(ob: *mut PyObject) -> Py_ssize_t { + #[inline(always)] + #[cfg(all(Py_3_12, not(Py_GIL_DISABLED)))] + pub unsafe fn _Py_IsImmortal(op: *mut PyObject) -> c_int { +- #[cfg(target_pointer_width = "64")] +- { +- (((*op).ob_refcnt.ob_refcnt as crate::PY_INT32_T) < 0) as c_int +- } +- +- #[cfg(target_pointer_width = "32")] +- { +- ((*op).ob_refcnt.ob_refcnt == _Py_IMMORTAL_REFCNT) as c_int ++ if _Py_REFCNT(op) == _Py_IMMORTAL_REFCNT { ++ 1 ++ } else { ++ 0 + } + } + +diff --git a/src/deserialize/backend/yyjson.rs b/src/deserialize/backend/yyjson.rs +index 24d247d..fc83957 100644 +--- a/src/deserialize/backend/yyjson.rs ++++ b/src/deserialize/backend/yyjson.rs +@@ -200,15 +200,14 @@ fn populate_yy_array(list: *mut pyo3_ffi::PyObject, elem: *mut yyjson_val) { + let len = unsafe_yyjson_get_len(elem); + assume!(len >= 1); + let mut next = unsafe_yyjson_get_first(elem); +- let mut dptr = (*list.cast::()).ob_item; + +- for _ in 0..len { ++ for idx in 0..len { + let val = next; + if unlikely!(unsafe_yyjson_is_ctn(val)) { + next = unsafe_yyjson_get_next_container(val); + if is_yyjson_tag!(val, TAG_ARRAY) { + let pyval = ffi!(PyList_New(usize_to_isize(unsafe_yyjson_get_len(val)))); +- append_to_list!(dptr, pyval); ++ pyo3_ffi::PyList_SetItem(list, idx as isize, pyval); + if unsafe_yyjson_get_len(val) > 0 { + populate_yy_array(pyval, val); + } +@@ -216,7 +215,7 @@ fn populate_yy_array(list: *mut pyo3_ffi::PyObject, elem: *mut yyjson_val) { + let pyval = ffi!(_PyDict_NewPresized(usize_to_isize(unsafe_yyjson_get_len( + val + )))); +- append_to_list!(dptr, pyval); ++ pyo3_ffi::PyList_SetItem(list, idx as isize, pyval); + if unsafe_yyjson_get_len(val) > 0 { + populate_yy_object(pyval, val); + } +@@ -234,7 +233,7 @@ fn populate_yy_array(list: *mut pyo3_ffi::PyObject, elem: *mut yyjson_val) { + ElementType::Array => unreachable_unchecked!(), + ElementType::Object => unreachable_unchecked!(), + }; +- append_to_list!(dptr, pyval.as_ptr()); ++ pyo3_ffi::PyList_SetItem(list, idx as isize, pyval.as_ptr()); + } + } + } +diff --git a/src/ffi/bytes.rs b/src/ffi/bytes.rs +index 94eb36b..a81b8b7 100644 +--- a/src/ffi/bytes.rs ++++ b/src/ffi/bytes.rs +@@ -1,16 +1,16 @@ + // SPDX-License-Identifier: (Apache-2.0 OR MIT) + + use core::ffi::c_char; +-use pyo3_ffi::{PyBytesObject, PyObject, PyVarObject, Py_ssize_t}; ++use pyo3_ffi::{PyBytesObject, PyObject, PyVarObject, Py_SIZE, Py_ssize_t}; + + #[allow(non_snake_case)] + #[inline(always)] + pub unsafe fn PyBytes_AS_STRING(op: *mut PyObject) -> *const c_char { +- unsafe { &(*op.cast::()).ob_sval as *const c_char } ++ pyo3_ffi::PyBytes_AS_STRING(op) + } + + #[allow(non_snake_case)] + #[inline(always)] + pub unsafe fn PyBytes_GET_SIZE(op: *mut PyObject) -> Py_ssize_t { +- unsafe { (*op.cast::()).ob_size } ++ Py_SIZE(op) + } +diff --git a/src/ffi/fragment.rs b/src/ffi/fragment.rs +index 9a7fe09..ba17899 100644 +--- a/src/ffi/fragment.rs ++++ b/src/ffi/fragment.rs +@@ -136,7 +136,7 @@ pub unsafe extern "C" fn orjson_fragmenttype_new() -> *mut PyTypeObject { + ob_refcnt: 0, + ob_type: &raw mut PyType_Type, + }, +- ob_size: 0, ++ _ob_size_graalpy: 0, + }, + tp_name: c"orjson.Fragment".as_ptr(), + tp_basicsize: core::mem::size_of::() as isize, +diff --git a/src/ffi/long.rs b/src/ffi/long.rs +index 438143b..9599e83 100644 +--- a/src/ffi/long.rs ++++ b/src/ffi/long.rs +@@ -37,10 +37,16 @@ pub struct PyLongObject { + pub ob_digit: u32, + } + ++extern "C" { ++ pub fn GraalPyPrivate_Long_lv_tag( ++ v: *mut pyo3_ffi::PyObject ++ ) -> usize; ++} ++ + #[cfg(Py_3_12)] + #[inline(always)] + pub fn pylong_is_unsigned(ptr: *mut pyo3_ffi::PyObject) -> bool { +- unsafe { (*ptr.cast::()).long_value.lv_tag & SIGN_MASK == 0 } ++ unsafe { GraalPyPrivate_Long_lv_tag(ptr) & SIGN_MASK == 0 } + } + + #[cfg(not(Py_3_12))] +diff --git a/src/lib.rs b/src/lib.rs +index 21f72d2..488ab2a 100644 +--- a/src/lib.rs ++++ b/src/lib.rs +@@ -75,7 +75,7 @@ use pyo3_ffi::{ + PyMethodDefPointer, PyModuleDef, PyModuleDef_HEAD_INIT, PyModuleDef_Slot, PyObject, + PyTuple_GET_ITEM, PyTuple_New, PyTuple_SET_ITEM, PyUnicode_FromStringAndSize, + PyUnicode_InternFromString, PyVectorcall_NARGS, Py_DECREF, Py_SIZE, Py_ssize_t, METH_KEYWORDS, +- METH_O, ++ METH_O,Py_TYPE + }; + + use crate::util::{isize_to_usize, usize_to_isize}; +@@ -275,7 +275,7 @@ fn raise_loads_exception(err: deserialize::DeserializeError) -> *mut PyObject { + PyTuple_SET_ITEM(args, 1, doc); + PyTuple_SET_ITEM(args, 2, pos); + PyErr_SetObject(typeref::JsonDecodeError, args); +- debug_assert!(ffi!(Py_REFCNT(args)) <= 2); ++ // debug_assert!(ffi!(Py_REFCNT(args)) <= 2); + Py_DECREF(args); + } + null_mut() +@@ -289,7 +289,7 @@ fn raise_dumps_exception_fixed(msg: &str) -> *mut PyObject { + let err_msg = + PyUnicode_FromStringAndSize(msg.as_ptr().cast::(), usize_to_isize(msg.len())); + PyErr_SetObject(typeref::JsonEncodeError, err_msg); +- debug_assert!(ffi!(Py_REFCNT(err_msg)) <= 2); ++ // debug_assert!(ffi!(Py_REFCNT(err_msg)) <= 2); + Py_DECREF(err_msg); + } + null_mut() +@@ -306,7 +306,7 @@ fn raise_dumps_exception_dynamic(err: &str) -> *mut PyObject { + let err_msg = + PyUnicode_FromStringAndSize(err.as_ptr().cast::(), usize_to_isize(err.len())); + PyErr_SetObject(typeref::JsonEncodeError, err_msg); +- debug_assert!(ffi!(Py_REFCNT(err_msg)) <= 2); ++ // debug_assert!(ffi!(Py_REFCNT(err_msg)) <= 2); + Py_DECREF(err_msg); + + if !cause_exc.is_null() { +@@ -388,14 +388,14 @@ pub unsafe extern "C" fn dumps( + if unlikely!(!kwnames.is_null()) { + for i in 0..=Py_SIZE(kwnames).saturating_sub(1) { + let arg = PyTuple_GET_ITEM(kwnames, i as Py_ssize_t); +- if core::ptr::eq(arg, typeref::DEFAULT) { ++ if pyo3_ffi::PyUnicode_Compare(arg, typeref::DEFAULT) == 0 { + if unlikely!(num_args & 2 == 2) { + return raise_dumps_exception_fixed( + "dumps() got multiple values for argument: 'default'", + ); + } + default = Some(NonNull::new_unchecked(*args.offset(num_args + i))); +- } else if core::ptr::eq(arg, typeref::OPTION) { ++ } else if pyo3_ffi::PyUnicode_Compare(arg, typeref::OPTION) == 0 { + if unlikely!(num_args & 3 == 3) { + return raise_dumps_exception_fixed( + "dumps() got multiple values for argument: 'option'", +@@ -413,7 +413,7 @@ pub unsafe extern "C" fn dumps( + let mut optsbits: i32 = 0; + if unlikely!(optsptr.is_some()) { + let opts = optsptr.unwrap(); +- if core::ptr::eq((*opts.as_ptr()).ob_type, typeref::INT_TYPE) { ++ if core::ptr::eq(Py_TYPE(opts.as_ptr()), typeref::INT_TYPE) { + #[allow(clippy::cast_possible_truncation)] + let tmp = PyLong_AsLong(optsptr.unwrap().as_ptr()) as i32; // stmt_expr_attributes + optsbits = tmp; +diff --git a/src/serialize/per_type/datetime.rs b/src/serialize/per_type/datetime.rs +index 0cad0ad..ae2e8fe 100644 +--- a/src/serialize/per_type/datetime.rs ++++ b/src/serialize/per_type/datetime.rs +@@ -101,7 +101,7 @@ impl Time { + + #[inline(never)] + pub fn write_buf(&self, buf: &mut SmallFixedBuffer) -> Result<(), TimeError> { +- if unsafe { (*self.ptr.cast::()).hastzinfo == 1 } { ++ if unsafe { pyo3_ffi::Py_IsNone(GraalPyDateTime_DATE_GET_TZINFO(self.ptr)) != 0 } { + return Err(TimeError::HasTimezone); + } + let hour = ffi!(PyDateTime_TIME_GET_HOUR(self.ptr)) as u8; +@@ -159,6 +159,10 @@ macro_rules! pydatetime_get { + }; + } + ++extern "C" { ++ pub fn GraalPyDateTime_DATE_GET_TZINFO(obj: *mut pyo3_ffi::PyObject) -> *mut pyo3_ffi::PyObject; ++} ++ + impl DateTimeLike for DateTime { + pydatetime_get!(year, PyDateTime_GET_YEAR, i32); + pydatetime_get!(month, PyDateTime_GET_MONTH, u8); +@@ -173,7 +177,7 @@ impl DateTimeLike for DateTime { + } + + fn has_tz(&self) -> bool { +- unsafe { (*(self.ptr.cast::())).hastzinfo == 1 } ++ unsafe { pyo3_ffi::Py_IsNone(GraalPyDateTime_DATE_GET_TZINFO(self.ptr)) == 0 } + } + + #[inline(never)] +diff --git a/src/serialize/per_type/int.rs b/src/serialize/per_type/int.rs +index 00ef947..6e45475 100644 +--- a/src/serialize/per_type/int.rs ++++ b/src/serialize/per_type/int.rs +@@ -31,9 +31,6 @@ impl Serialize for IntSerializer { + S: Serializer, + { + unsafe { +- if crate::ffi::pylong_is_zero(self.ptr) { +- return serializer.serialize_bytes(b"0"); +- } + let is_signed = i32::from(!crate::ffi::pylong_is_unsigned(self.ptr)); + if crate::ffi::pylong_fits_in_i32(self.ptr) { + if is_signed == 0 { +diff --git a/src/serialize/per_type/list.rs b/src/serialize/per_type/list.rs +index 505840e..e984191 100644 +--- a/src/serialize/per_type/list.rs ++++ b/src/serialize/per_type/list.rs +@@ -34,7 +34,8 @@ impl Serialize for ZeroListSerializer { + } + + pub struct ListTupleSerializer { +- data_ptr: *const *mut pyo3_ffi::PyObject, ++ data_ptr: *mut pyo3_ffi::PyObject, ++ is_list: bool, + state: SerializerState, + default: Option>, + len: usize, +@@ -50,10 +51,10 @@ impl ListTupleSerializer { + is_type!(ob_type!(ptr), LIST_TYPE) + || is_subclass_by_flag!(tp_flags!(ob_type!(ptr)), Py_TPFLAGS_LIST_SUBCLASS) + ); +- let data_ptr = unsafe { (*ptr.cast::()).ob_item }; + let len = isize_to_usize(ffi!(Py_SIZE(ptr))); + Self { +- data_ptr: data_ptr, ++ data_ptr: ptr, ++ is_list: true, + len: len, + state: state.copy_for_recursive_call(), + default: default, +@@ -69,10 +70,10 @@ impl ListTupleSerializer { + is_type!(ob_type!(ptr), TUPLE_TYPE) + || is_subclass_by_flag!(tp_flags!(ob_type!(ptr)), Py_TPFLAGS_TUPLE_SUBCLASS) + ); +- let data_ptr = unsafe { (*ptr.cast::()).ob_item.as_ptr() }; + let len = isize_to_usize(ffi!(Py_SIZE(ptr))); + Self { +- data_ptr: data_ptr, ++ data_ptr: ptr, ++ is_list: false, + len: len, + state: state.copy_for_recursive_call(), + default: default, +@@ -92,7 +93,11 @@ impl Serialize for ListTupleSerializer { + debug_assert!(self.len >= 1); + let mut seq = serializer.serialize_seq(None).unwrap(); + for idx in 0..self.len { +- let value = unsafe { *((self.data_ptr).add(idx)) }; ++ let value = if self.is_list { ++ unsafe { pyo3_ffi::PyList_GET_ITEM(self.data_ptr, idx as isize) } ++ } else { ++ unsafe { pyo3_ffi::PyTuple_GET_ITEM(self.data_ptr, idx as isize) } ++ }; + match pyobject_to_obtype(value, self.state.opts()) { + ObType::Str => { + seq.serialize_element(&StrSerializer::new(value))?; +diff --git a/src/serialize/serializer.rs b/src/serialize/serializer.rs +index 7b9a5de..e291912 100644 +--- a/src/serialize/serializer.rs ++++ b/src/serialize/serializer.rs +@@ -34,7 +34,6 @@ pub fn serialize( + Ok(buf.finish()) + } + Err(err) => { +- ffi!(Py_DECREF(buf.bytes_ptr().as_ptr())); + Err(err.to_string()) + } + } +diff --git a/src/serialize/writer/byteswriter.rs b/src/serialize/writer/byteswriter.rs +index b2d13d7..d1a1dd2 100644 +--- a/src/serialize/writer/byteswriter.rs ++++ b/src/serialize/writer/byteswriter.rs +@@ -8,59 +8,46 @@ use std::io::Error; + const BUFFER_LENGTH: usize = 1024; + + pub struct BytesWriter { +- cap: usize, ++ // cap: usize, // cap for us is the bytes.len() + len: usize, +- bytes: *mut PyBytesObject, ++ bytes: Vec::, + } + + impl BytesWriter { + pub fn default() -> Self { ++ let mut bytes = Vec::with_capacity(BUFFER_LENGTH); ++ bytes.resize(BUFFER_LENGTH, 0); + BytesWriter { +- cap: BUFFER_LENGTH, + len: 0, +- bytes: unsafe { +- PyBytes_FromStringAndSize(core::ptr::null_mut(), usize_to_isize(BUFFER_LENGTH)) +- .cast::() +- }, ++ bytes: bytes, + } + } + +- pub fn bytes_ptr(&mut self) -> NonNull { +- unsafe { NonNull::new_unchecked(self.bytes.cast::()) } +- } +- + pub fn finish(&mut self) -> NonNull { + unsafe { +- core::ptr::write(self.buffer_ptr(), 0); +- (*self.bytes.cast::()).ob_size = usize_to_isize(self.len); +- self.resize(self.len); +- self.bytes_ptr() ++ let bytes_obj = PyBytes_FromStringAndSize(self.bytes.as_mut_ptr() as *const i8, usize_to_isize(self.len)) ++ .cast::(); ++ NonNull::new_unchecked(bytes_obj as *mut PyObject) + } + } + +- fn buffer_ptr(&self) -> *mut u8 { +- unsafe { (&raw mut (*self.bytes).ob_sval).cast::().add(self.len) } ++ fn buffer_ptr(&mut self) -> *mut u8 { ++ unsafe { self.bytes.as_mut_ptr().byte_add(self.len) } ++ } ++ ++ fn cap(&self) -> usize { ++ self.bytes.len() + } + + #[inline] + pub fn resize(&mut self, len: usize) { +- self.cap = len; +- unsafe { +- _PyBytes_Resize( +- (&raw mut self.bytes).cast::<*mut PyObject>(), +- usize_to_isize(len), +- ); +- } ++ self.bytes.resize(len, 0); + } + + #[cold] + #[inline(never)] + fn grow(&mut self, len: usize) { +- let mut cap = self.cap; +- while len >= cap { +- cap *= 2; +- } +- self.resize(cap); ++ self.resize(len + 1); + } + } + +@@ -73,7 +60,7 @@ impl std::io::Write for BytesWriter { + fn write_all(&mut self, buf: &[u8]) -> Result<(), Error> { + let to_write = buf.len(); + let end_length = self.len + to_write; +- if unlikely!(end_length >= self.cap) { ++ if unlikely!(end_length >= self.cap()) { + self.grow(end_length); + } + unsafe { +@@ -144,14 +131,14 @@ impl WriteExt for &mut BytesWriter { + #[inline(always)] + fn reserve(&mut self, len: usize) { + let end_length = self.len + len; +- if unlikely!(end_length >= self.cap) { ++ if unlikely!(end_length >= self.cap()) { + self.grow(end_length); + } + } + + #[inline] + fn has_capacity(&mut self, len: usize) -> bool { +- self.len + len <= self.cap ++ self.len + len <= self.cap() + } + + #[inline(always)] +@@ -162,7 +149,7 @@ impl WriteExt for &mut BytesWriter { + fn write_str(&mut self, val: &str) -> Result<(), Error> { + let to_write = val.len(); + let end_length = self.len + to_write + 2; +- if unlikely!(end_length >= self.cap) { ++ if unlikely!(end_length >= self.cap()) { + self.grow(end_length); + } + unsafe { +diff --git a/src/str/avx512.rs b/src/str/avx512.rs +index c52dc21..828aa1f 100644 +--- a/src/str/avx512.rs ++++ b/src/str/avx512.rs +@@ -101,7 +101,8 @@ static mut STR_CREATE_FN: StrDeserializer = super::scalar::str_impl_kind_scalar; + pub fn set_str_create_fn() { + unsafe { + if std::is_x86_feature_detected!("avx512vl") { +- STR_CREATE_FN = create_str_impl_avx512vl; ++ // This won't be faster for GraalPy ++ // STR_CREATE_FN = create_str_impl_avx512vl; + } + } + } +diff --git a/src/str/ffi.rs b/src/str/ffi.rs +index 4f1c76b..3d2c7e9 100644 +--- a/src/str/ffi.rs ++++ b/src/str/ffi.rs +@@ -2,31 +2,13 @@ + + use crate::util::isize_to_usize; + use core::ffi::c_void; +-use pyo3_ffi::{PyASCIIObject, PyCompactUnicodeObject, PyObject, Py_hash_t}; ++use pyo3_ffi::{PyASCIIObject, PyCompactUnicodeObject, PyObject, PyObject_Hash, Py_hash_t}; + + // see unicodeobject.h for documentation + + #[inline] + pub fn hash_str(op: *mut PyObject) -> Py_hash_t { +- unsafe { +- let data_ptr: *mut c_void = if (*op.cast::()).compact() == 1 +- && (*op.cast::()).ascii() == 1 +- { +- op.cast::().offset(1).cast::() +- } else { +- op.cast::() +- .offset(1) +- .cast::() +- }; +- let num_bytes = +- (*op.cast::()).length * ((*op.cast::()).kind()) as isize; +- #[cfg(Py_3_14)] +- let hash = pyo3_ffi::Py_HashBuffer(data_ptr, num_bytes); +- #[cfg(not(Py_3_14))] +- let hash = pyo3_ffi::_Py_HashBytes(data_ptr, num_bytes); +- (*op.cast::()).hash = hash; +- hash +- } ++ unsafe { PyObject_Hash(op) } + } + + #[inline(never)] +@@ -43,18 +25,6 @@ pub fn unicode_to_str_via_ffi(op: *mut PyObject) -> Option<&'static str> { + #[inline] + pub fn unicode_to_str(op: *mut PyObject) -> Option<&'static str> { + unsafe { +- if unlikely!((*op.cast::()).compact() == 0) { +- unicode_to_str_via_ffi(op) +- } else if (*op.cast::()).ascii() == 1 { +- let ptr = op.cast::().offset(1) as *const u8; +- let len = isize_to_usize((*op.cast::()).length); +- Some(str_from_slice!(ptr, len)) +- } else if (*op.cast::()).utf8_length != 0 { +- let ptr = (*op.cast::()).utf8 as *const u8; +- let len = isize_to_usize((*op.cast::()).utf8_length); +- Some(str_from_slice!(ptr, len)) +- } else { + unicode_to_str_via_ffi(op) +- } + } + } +diff --git a/src/str/pyunicode_new.rs b/src/str/pyunicode_new.rs +index 7b1c2df..5ac8dec 100644 +--- a/src/str/pyunicode_new.rs ++++ b/src/str/pyunicode_new.rs +@@ -1,75 +1,30 @@ + // SPDX-License-Identifier: (Apache-2.0 OR MIT) + + use crate::util::usize_to_isize; +-use pyo3_ffi::{PyASCIIObject, PyCompactUnicodeObject, PyObject}; +- +-macro_rules! validate_str { +- ($ptr:expr) => { +- #[cfg(not(Py_3_12))] +- debug_assert!((*($ptr.cast::())).ready() == 1); +- +- debug_assert!((*($ptr.cast::())).compact() == 1); +- debug_assert!((*($ptr.cast::())).interned() == 0); +- +- debug_assert!(ffi!(_PyUnicode_CheckConsistency($ptr.cast::(), 1)) == 1); +- }; +-} ++use std::ffi::CString; ++use pyo3_ffi::{PyASCIIObject, PyCompactUnicodeObject, PyObject, PyUnicode_FromStringAndSize}; + + #[inline(never)] + pub fn pyunicode_ascii(buf: *const u8, num_chars: usize) -> *mut pyo3_ffi::PyObject { + unsafe { +- let ptr = ffi!(PyUnicode_New(usize_to_isize(num_chars), 127)); +- let data_ptr = ptr.cast::().offset(1).cast::(); +- core::ptr::copy_nonoverlapping(buf, data_ptr, num_chars); +- core::ptr::write(data_ptr.add(num_chars), 0); +- debug_assert!((*(ptr.cast::())).ascii() == 1); +- validate_str!(ptr); +- ptr.cast::() ++ PyUnicode_FromStringAndSize(buf as *const i8, usize_to_isize(num_chars)) + } + } + + #[cold] + #[inline(never)] + pub fn pyunicode_onebyte(buf: &str, num_chars: usize) -> *mut pyo3_ffi::PyObject { +- unsafe { +- let ptr = ffi!(PyUnicode_New(usize_to_isize(num_chars), 255)); +- let mut data_ptr = ptr.cast::().offset(1).cast::(); +- for each in buf.chars().fuse() { +- core::ptr::write(data_ptr, each as u8); +- data_ptr = data_ptr.offset(1); +- } +- core::ptr::write(data_ptr, 0); +- validate_str!(ptr); +- ptr.cast::() +- } ++ pyunicode_fourbyte(buf, num_chars) + } + + #[inline(never)] + pub fn pyunicode_twobyte(buf: &str, num_chars: usize) -> *mut pyo3_ffi::PyObject { +- unsafe { +- let ptr = ffi!(PyUnicode_New(usize_to_isize(num_chars), 65535)); +- let mut data_ptr = ptr.cast::().offset(1).cast::(); +- for each in buf.chars().fuse() { +- core::ptr::write(data_ptr, each as u16); +- data_ptr = data_ptr.offset(1); +- } +- core::ptr::write(data_ptr, 0); +- validate_str!(ptr); +- ptr.cast::() +- } ++ pyunicode_fourbyte(buf, num_chars) + } + + #[inline(never)] + pub fn pyunicode_fourbyte(buf: &str, num_chars: usize) -> *mut pyo3_ffi::PyObject { + unsafe { +- let ptr = ffi!(PyUnicode_New(usize_to_isize(num_chars), 1114111)); +- let mut data_ptr = ptr.cast::().offset(1).cast::(); +- for each in buf.chars().fuse() { +- core::ptr::write(data_ptr, each as u32); +- data_ptr = data_ptr.offset(1); +- } +- core::ptr::write(data_ptr, 0); +- validate_str!(ptr); +- ptr.cast::() ++ PyUnicode_FromStringAndSize(buf.as_ptr() as *const i8, usize_to_isize(buf.len())) + } + } +diff --git a/src/str/scalar.rs b/src/str/scalar.rs +index 12adb63..99fce8f 100644 +--- a/src/str/scalar.rs ++++ b/src/str/scalar.rs +@@ -1,11 +1,17 @@ + // SPDX-License-Identifier: (Apache-2.0 OR MIT) + ++use pyo3_ffi::PyUnicode_FromStringAndSize; + use crate::str::pyunicode_new::{ + pyunicode_ascii, pyunicode_fourbyte, pyunicode_onebyte, pyunicode_twobyte, + }; ++use crate::util::usize_to_isize; + + #[inline(never)] + pub fn str_impl_kind_scalar(buf: &str) -> *mut pyo3_ffi::PyObject { ++ // all the optimizations below don't make sense for GraalPy ++ return unsafe { ++ PyUnicode_FromStringAndSize(buf.as_ptr() as *const i8, usize_to_isize(buf.len())) ++ }; + let num_chars = bytecount::num_chars(buf.as_bytes()); + if buf.len() == num_chars { + return pyunicode_ascii(buf.as_ptr(), num_chars); +@@ -40,7 +46,9 @@ pub fn str_impl_kind_scalar(buf: &str) -> *mut pyo3_ffi::PyObject { + #[inline(always)] + pub fn unicode_from_str(buf: &str) -> *mut pyo3_ffi::PyObject { + if unlikely!(buf.is_empty()) { +- return use_immortal!(crate::typeref::EMPTY_UNICODE); ++ // Not immortal on GraalPy ++ ffi!(Py_INCREF(crate::typeref::EMPTY_UNICODE)); ++ return unsafe { crate::typeref::EMPTY_UNICODE }; + } + str_impl_kind_scalar(buf) + } +diff --git a/src/typeref.rs b/src/typeref.rs +index bf925a4..388c9f4 100644 +--- a/src/typeref.rs ++++ b/src/typeref.rs +@@ -154,7 +154,7 @@ fn _init_typerefs_impl() -> bool { + DICT_TYPE = &raw mut PyDict_Type; + LIST_TYPE = &raw mut PyList_Type; + TUPLE_TYPE = &raw mut PyTuple_Type; +- NONE_TYPE = (*NONE).ob_type; ++ NONE_TYPE = Py_TYPE(NONE); + BOOL_TYPE = &raw mut PyBool_Type; + INT_TYPE = &raw mut PyLong_Type; + FLOAT_TYPE = &raw mut PyFloat_Type; +@@ -290,7 +290,7 @@ unsafe fn look_up_uuid_type() -> *mut PyTypeObject { + let uuid_mod = PyImport_ImportModule(c"uuid".as_ptr()); + let uuid_mod_dict = PyObject_GenericGetDict(uuid_mod, null_mut()); + let uuid = PyMapping_GetItemString(uuid_mod_dict, c"NAMESPACE_DNS".as_ptr()); +- let ptr = (*uuid).ob_type; ++ let ptr = Py_TYPE(uuid); + Py_DECREF(uuid); + Py_DECREF(uuid_mod_dict); + Py_DECREF(uuid_mod); +@@ -313,7 +313,7 @@ unsafe fn look_up_datetime_type() -> *mut PyTypeObject { + NONE, + (*(PyDateTimeAPI())).DateTimeType, + ); +- let ptr = (*datetime).ob_type; ++ let ptr = Py_TYPE(datetime); + Py_DECREF(datetime); + ptr + } +@@ -324,7 +324,7 @@ unsafe fn look_up_datetime_type() -> *mut PyTypeObject { + unsafe fn look_up_date_type() -> *mut PyTypeObject { + unsafe { + let date = ((*PyDateTimeAPI()).Date_FromDate)(1, 1, 1, (*(PyDateTimeAPI())).DateType); +- let ptr = (*date).ob_type; ++ let ptr = Py_TYPE(date); + Py_DECREF(date); + ptr + } +@@ -336,7 +336,7 @@ unsafe fn look_up_time_type() -> *mut PyTypeObject { + unsafe { + let time = + ((*PyDateTimeAPI()).Time_FromTime)(0, 0, 0, 0, NONE, (*(PyDateTimeAPI())).TimeType); +- let ptr = (*time).ob_type; ++ let ptr = Py_TYPE(time); + Py_DECREF(time); + ptr + } +diff --git a/src/util.rs b/src/util.rs +index ccf425c..bc9268d 100644 +--- a/src/util.rs ++++ b/src/util.rs +@@ -10,7 +10,7 @@ macro_rules! is_type { + + macro_rules! ob_type { + ($obj:expr) => { +- unsafe { (*$obj).ob_type } ++ unsafe { pyo3_ffi::Py_TYPE($obj) } + }; + } + +@@ -121,8 +121,8 @@ macro_rules! reverse_pydict_incref { + ($op:expr) => { + unsafe { + if pyo3_ffi::_Py_IsImmortal($op) == 0 { +- debug_assert!(ffi!(Py_REFCNT($op)) >= 2); +- (*$op).ob_refcnt.ob_refcnt -= 1; ++ // debug_assert!(ffi!(Py_REFCNT($op)) >= 2); ++ ffi!(Py_DECREF($op)) + } + } + }; +@@ -183,13 +183,12 @@ macro_rules! str_hash { + }; + } + +-#[cfg(Py_3_13)] + macro_rules! pydict_contains { + ($obj1:expr, $obj2:expr) => { + unsafe { pyo3_ffi::PyDict_Contains(pyo3_ffi::PyType_GetDict($obj1), $obj2) == 1 } + }; + } +- ++/* + #[cfg(all(Py_3_12, not(Py_3_13)))] + macro_rules! pydict_contains { + ($obj1:expr, $obj2:expr) => { +@@ -222,6 +221,7 @@ macro_rules! pydict_contains { + unsafe { pyo3_ffi::PyDict_Contains((*$obj1).tp_dict, $obj2) == 1 } + }; + } ++*/ + + #[cfg(Py_3_12)] + macro_rules! use_immortal { +@@ -234,6 +234,9 @@ macro_rules! use_immortal { + macro_rules! use_immortal { + ($op:expr) => { + unsafe { ++ // orjson makes assumption about something being immortal that is not immortal ++ // on GraalPy: update the caller to do proper refcounting instead ++ debug_assert!(ffi!(_Py_IsImmortal($op)) == 1); + ffi!(Py_INCREF($op)); + $op + } +@@ -256,20 +259,11 @@ macro_rules! pydict_next { + + macro_rules! pydict_setitem { + ($dict:expr, $pykey:expr, $pyval:expr) => { +- debug_assert!(ffi!(Py_REFCNT($dict)) == 1); +- debug_assert!(str_hash!($pykey) != -1); ++ // debug_assert!(ffi!(Py_REFCNT($dict)) == 1); ++ // debug_assert!(str_hash!($pykey) != -1); + #[cfg(not(Py_3_13))] + unsafe { +- let _ = pyo3_ffi::_PyDict_SetItem_KnownHash($dict, $pykey, $pyval, str_hash!($pykey)); +- } +- #[cfg(Py_3_13)] +- unsafe { +- let _ = pyo3_ffi::_PyDict_SetItem_KnownHash_LockHeld( +- $dict.cast::(), +- $pykey, +- $pyval, +- str_hash!($pykey), +- ); ++ let _ = pyo3_ffi::PyDict_SetItem($dict, $pykey, $pyval); + } + reverse_pydict_incref!($pykey); + reverse_pydict_incref!($pyval); From 5457f23ce8b19130256029528fa24e98c3179218 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Thu, 31 Jul 2025 13:38:46 +0200 Subject: [PATCH 38/82] Adapt orjson 3.10.18 patch to 24.2.2 --- .../patches/orjson-3.10.18.patch | 110 +++++++++++++----- 1 file changed, 79 insertions(+), 31 deletions(-) diff --git a/graalpython/lib-graalpython/patches/orjson-3.10.18.patch b/graalpython/lib-graalpython/patches/orjson-3.10.18.patch index 3677bf64b9..41cc7f44a6 100644 --- a/graalpython/lib-graalpython/patches/orjson-3.10.18.patch +++ b/graalpython/lib-graalpython/patches/orjson-3.10.18.patch @@ -300,24 +300,31 @@ diff --git a/src/ffi/long.rs b/src/ffi/long.rs index 438143b..9599e83 100644 --- a/src/ffi/long.rs +++ b/src/ffi/long.rs -@@ -37,10 +37,16 @@ pub struct PyLongObject { +@@ -37,13 +37,21 @@ pub struct PyLongObject { pub ob_digit: u32, } -+extern "C" { -+ pub fn GraalPyPrivate_Long_lv_tag( -+ v: *mut pyo3_ffi::PyObject -+ ) -> usize; -+} -+ - #[cfg(Py_3_12)] +-#[cfg(Py_3_12)] ++#[cfg(any(Py_3_12, GraalPy))] #[inline(always)] pub fn pylong_is_unsigned(ptr: *mut pyo3_ffi::PyObject) -> bool { - unsafe { (*ptr.cast::()).long_value.lv_tag & SIGN_MASK == 0 } -+ unsafe { GraalPyPrivate_Long_lv_tag(ptr) & SIGN_MASK == 0 } ++ unsafe { ++ ffi!(PyLong_AsUnsignedLongLong(ptr)); ++ if (ffi!(PyErr_Occurred()).is_null()) { ++ return true; ++ } else { ++ ffi!(PyErr_Clear()); ++ return false; ++ } ++ } } - #[cfg(not(Py_3_12))] +-#[cfg(not(Py_3_12))] ++#[cfg(not(any(Py_3_12, GraalPy)))] + #[inline(always)] + pub fn pylong_is_unsigned(ptr: *mut pyo3_ffi::PyObject) -> bool { + unsafe { (*ptr.cast::()).ob_size > 0 } diff --git a/src/lib.rs b/src/lib.rs index 21f72d2..488ab2a 100644 --- a/src/lib.rs @@ -358,6 +365,15 @@ index 21f72d2..488ab2a 100644 Py_DECREF(err_msg); if !cause_exc.is_null() { +@@ -332,7 +332,7 @@ fn raise_dumps_exception_dynamic(err: &str) -> *mut PyObject { + let err_msg = + PyUnicode_FromStringAndSize(err.as_ptr().cast::(), usize_to_isize(err.len())); + PyErr_SetObject(typeref::JsonEncodeError, err_msg); +- debug_assert!(ffi!(Py_REFCNT(err_msg)) == 2); ++ // debug_assert!(ffi!(Py_REFCNT(err_msg)) == 2); + Py_DECREF(err_msg); + let mut tp: *mut PyObject = null_mut(); + let mut val: *mut PyObject = null_mut(); @@ -388,14 +388,14 @@ pub unsafe extern "C" fn dumps( if unlikely!(!kwnames.is_null()) { for i in 0..=Py_SIZE(kwnames).saturating_sub(1) { @@ -393,7 +409,7 @@ index 0cad0ad..ae2e8fe 100644 #[inline(never)] pub fn write_buf(&self, buf: &mut SmallFixedBuffer) -> Result<(), TimeError> { - if unsafe { (*self.ptr.cast::()).hastzinfo == 1 } { -+ if unsafe { pyo3_ffi::Py_IsNone(GraalPyDateTime_DATE_GET_TZINFO(self.ptr)) != 0 } { ++ if unsafe { pyo3_ffi::Py_IsNone(PyTruffle_PyDateTime_GET_TZINFO(self.ptr)) != 0 } { return Err(TimeError::HasTimezone); } let hour = ffi!(PyDateTime_TIME_GET_HOUR(self.ptr)) as u8; @@ -402,7 +418,7 @@ index 0cad0ad..ae2e8fe 100644 } +extern "C" { -+ pub fn GraalPyDateTime_DATE_GET_TZINFO(obj: *mut pyo3_ffi::PyObject) -> *mut pyo3_ffi::PyObject; ++ pub fn PyTruffle_PyDateTime_GET_TZINFO(obj: *mut pyo3_ffi::PyObject) -> *mut pyo3_ffi::PyObject; +} + impl DateTimeLike for DateTime { @@ -413,7 +429,7 @@ index 0cad0ad..ae2e8fe 100644 fn has_tz(&self) -> bool { - unsafe { (*(self.ptr.cast::())).hastzinfo == 1 } -+ unsafe { pyo3_ffi::Py_IsNone(GraalPyDateTime_DATE_GET_TZINFO(self.ptr)) == 0 } ++ unsafe { pyo3_ffi::Py_IsNone(PyTruffle_PyDateTime_GET_TZINFO(self.ptr)) == 0 } } #[inline(never)] @@ -484,6 +500,20 @@ index 505840e..e984191 100644 match pyobject_to_obtype(value, self.state.opts()) { ObType::Str => { seq.serialize_element(&StrSerializer::new(value))?; +diff --git a/src/serialize/per_type/numpy.rs b/src/serialize/per_type/numpy.rs +index f888029..4a0fe0e 100644 +--- a/src/serialize/per_type/numpy.rs ++++ b/src/serialize/per_type/numpy.rs +@@ -196,8 +196,7 @@ impl NumpyArray { + pub fn new(ptr: *mut PyObject, opts: Opt) -> Result { + let capsule = ffi!(PyObject_GetAttr(ptr, ARRAY_STRUCT_STR)); + let array = unsafe { +- (*capsule.cast::()) +- .pointer ++ ffi!(PyCapsule_GetPointer(capsule, core::ptr::null_mut())) + .cast::() + }; + if unsafe { (*array).two != 2 } { diff --git a/src/serialize/serializer.rs b/src/serialize/serializer.rs index 7b9a5de..e291912 100644 --- a/src/serialize/serializer.rs @@ -863,51 +893,69 @@ index ccf425c..bc9268d 100644 }; } -@@ -121,8 +121,8 @@ macro_rules! reverse_pydict_incref { +@@ -116,14 +116,14 @@ macro_rules! str_from_slice { + }; + } + +-#[cfg(all(Py_3_12, not(Py_GIL_DISABLED)))] ++#[cfg(any(GraalPy, all(Py_3_12, not(Py_GIL_DISABLED))))] + macro_rules! reverse_pydict_incref { ($op:expr) => { unsafe { - if pyo3_ffi::_Py_IsImmortal($op) == 0 { +- if pyo3_ffi::_Py_IsImmortal($op) == 0 { - debug_assert!(ffi!(Py_REFCNT($op)) >= 2); - (*$op).ob_refcnt.ob_refcnt -= 1; +- } ++ // if pyo3_ffi::_Py_IsImmortal($op) == 0 { + // debug_assert!(ffi!(Py_REFCNT($op)) >= 2); + ffi!(Py_DECREF($op)) - } ++ // } } }; -@@ -183,13 +183,12 @@ macro_rules! str_hash { + } +@@ -136,7 +136,7 @@ macro_rules! reverse_pydict_incref { }; } --#[cfg(Py_3_13)] - macro_rules! pydict_contains { - ($obj1:expr, $obj2:expr) => { - unsafe { pyo3_ffi::PyDict_Contains(pyo3_ffi::PyType_GetDict($obj1), $obj2) == 1 } +-#[cfg(not(Py_3_12))] ++#[cfg(not(any(GraalPy, Py_3_12)))] + macro_rules! reverse_pydict_incref { + ($op:expr) => { + unsafe { +@@ -183,10 +183,11 @@ macro_rules! str_hash { }; } -- + +/* - #[cfg(all(Py_3_12, not(Py_3_13)))] + #[cfg(Py_3_13)] macro_rules! pydict_contains { ($obj1:expr, $obj2:expr) => { -@@ -222,6 +221,7 @@ macro_rules! pydict_contains { - unsafe { pyo3_ffi::PyDict_Contains((*$obj1).tp_dict, $obj2) == 1 } +- unsafe { pyo3_ffi::PyDict_Contains(pyo3_ffi::PyType_GetDict($obj1), $obj2) == 1 } ++ unsafe { pyo3_ffi::PyDict_Contains(pyo3_ffi::PyObject_GenericGetDict($obj1.cast::(), core::ptr::null_mut()), $obj2) == 1 } }; } -+*/ - #[cfg(Py_3_12)] - macro_rules! use_immortal { -@@ -234,6 +234,9 @@ macro_rules! use_immortal { +@@ -215,8 +216,8 @@ macro_rules! pydict_contains { + } + }; + } +- + #[cfg(not(Py_3_10))] ++*/ + macro_rules! pydict_contains { + ($obj1:expr, $obj2:expr) => { + unsafe { pyo3_ffi::PyDict_Contains((*$obj1).tp_dict, $obj2) == 1 } +@@ -234,6 +235,9 @@ macro_rules! use_immortal { macro_rules! use_immortal { ($op:expr) => { unsafe { + // orjson makes assumption about something being immortal that is not immortal + // on GraalPy: update the caller to do proper refcounting instead -+ debug_assert!(ffi!(_Py_IsImmortal($op)) == 1); ++ // debug_assert!(ffi!(_Py_IsImmortal($op)) == 1); ffi!(Py_INCREF($op)); $op } -@@ -256,20 +259,11 @@ macro_rules! pydict_next { +@@ -256,20 +260,11 @@ macro_rules! pydict_next { macro_rules! pydict_setitem { ($dict:expr, $pykey:expr, $pyval:expr) => { From 37280e19c5e229edaf22d238993053a923d90462 Mon Sep 17 00:00:00 2001 From: Ivo Horak Date: Thu, 31 Jul 2025 21:30:22 +0200 Subject: [PATCH 39/82] Backport gensim fix to 24.2.2 --- graalpython/lib-graalpython/patches/metadata.toml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/graalpython/lib-graalpython/patches/metadata.toml b/graalpython/lib-graalpython/patches/metadata.toml index b9d99a732b..ca3dd47caa 100644 --- a/graalpython/lib-graalpython/patches/metadata.toml +++ b/graalpython/lib-graalpython/patches/metadata.toml @@ -863,3 +863,10 @@ install-priority = 0 [[zstandard.rules]] patch = 'zstandard.patch' license = 'BSD-3-Clause' + +[[gensim.add-sources]] +version = '4.3.3' +url = 'https://github.com/piskvorky/gensim/archive/refs/tags/4.3.3.tar.gz' + +[[gensim.rules]] +version = '== 4.3.3' From 586dfc41014e88d9c223a4110fa33c013f7b3a5f Mon Sep 17 00:00:00 2001 From: Ivo Horak Date: Mon, 18 Aug 2025 18:33:54 +0000 Subject: [PATCH 40/82] [GR-68827]: Adjusting onnxruntime-1.17.1 patch to work on MacOS PullRequest: graalpython/3959 --- .../patches/onnxruntime-1.17.1.patch | 31 ++++++++++++++++--- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/graalpython/lib-graalpython/patches/onnxruntime-1.17.1.patch b/graalpython/lib-graalpython/patches/onnxruntime-1.17.1.patch index 1b302d5053..44772bc782 100644 --- a/graalpython/lib-graalpython/patches/onnxruntime-1.17.1.patch +++ b/graalpython/lib-graalpython/patches/onnxruntime-1.17.1.patch @@ -1,19 +1,42 @@ diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt -index 94d650f..175a6b8 100644 +index 94d650f..31dc0bc 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt -@@ -54,6 +54,8 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON) +@@ -54,6 +54,16 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON) enable_testing() include(Dart) -+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=array-bounds -Wno-error=maybe-uninitialized -Wno-error=uninitialized -Wno-error=restrict -Wno-error=unused-but-set-variable -Wno-error=overloaded-virtual") ++# For Linux ++if(CMAKE_SYSTEM_NAME STREQUAL "Linux") ++ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=array-bounds -Wno-error=maybe-uninitialized -Wno-error=uninitialized -Wno-error=restrict -Wno-error=unused-but-set-variable -Wno-error=overloaded-virtual") ++endif() ++ ++# For macOS ++if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") ++ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-deprecated") ++endif() + if (NOT CMAKE_BUILD_TYPE) message(STATUS "Build type not set - using RelWithDebInfo") set(CMAKE_BUILD_TYPE "RelWithDebInfo" CACHE STRING "Choose build type: Debug Release RelWithDebInfo MinSizeRel." FORCE) +diff --git a/cmake/deps.txt b/cmake/deps.txt +index ba9c2bb..c2813aa 100644 +--- a/cmake/deps.txt ++++ b/cmake/deps.txt +@@ -21,7 +21,9 @@ dlpack;https://github.com/dmlc/dlpack/archive/refs/tags/v0.6.zip;4d565dd2e5b3132 + # it contains changes on top of 3.4.0 which are required to fix build issues. + # Until the 3.4.1 release this is the best option we have. + # Issue link: https://gitlab.com/libeigen/eigen/-/issues/2744 +-eigen;https://gitlab.com/libeigen/eigen/-/archive/e7248b26a1ed53fa030c5c459f7ea095dfd276ac/eigen-e7248b26a1ed53fa030c5c459f7ea095dfd276ac.zip;be8be39fdbc6e60e94fa7870b280707069b5b81a ++# Moved to github mirror to avoid gitlab issues. ++# Issue link: https://github.com/bazelbuild/bazel-central-registry/issues/4355 ++eigen;https://github.com/eigen-mirror/eigen/archive/1d8b82b0740839c0de7f1242a3585e3390ff5f33/eigen-1d8b82b0740839c0de7f1242a3585e3390ff5f33.zip;05b19b49e6fbb91246be711d801160528c135e34 + flatbuffers;https://github.com/google/flatbuffers/archive/refs/tags/v1.12.0.zip;ba0a75fd12dbef8f6557a74e611b7a3d0c5fe7bf + fp16;https://github.com/Maratyszcza/FP16/archive/0a92994d729ff76a58f692d3028ca1b64b145d91.zip;b985f6985a05a1c03ff1bb71190f66d8f98a1494 + fxdiv;https://github.com/Maratyszcza/FXdiv/archive/63058eff77e11aa15bf531df5dd34395ec3017c8.zip;a5658f4036402dbca7cebee32be57fb8149811e1 diff --git a/onnxruntime_build_backend.py b/onnxruntime_build_backend.py new file mode 100644 -index 0000000..33fc235 +index 0000000..156e70a --- /dev/null +++ b/onnxruntime_build_backend.py @@ -0,0 +1,59 @@ From 83bcd82c5572c5e3b096261e658b7f507eecf6c3 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Tue, 18 Mar 2025 10:23:41 +0100 Subject: [PATCH 41/82] Add more packages to build with Github actions --- scripts/wheelbuilder/darwin/cryptography.sh | 48 ++++++++ scripts/wheelbuilder/darwin/pandas.sh | 49 ++++++++ scripts/wheelbuilder/linux/cryptography.sh | 42 +++++++ scripts/wheelbuilder/packages.txt | 129 +++++++++++++++++++- 4 files changed, 267 insertions(+), 1 deletion(-) create mode 100755 scripts/wheelbuilder/darwin/cryptography.sh create mode 100755 scripts/wheelbuilder/darwin/pandas.sh create mode 100755 scripts/wheelbuilder/linux/cryptography.sh diff --git a/scripts/wheelbuilder/darwin/cryptography.sh b/scripts/wheelbuilder/darwin/cryptography.sh new file mode 100755 index 0000000000..aa224941ef --- /dev/null +++ b/scripts/wheelbuilder/darwin/cryptography.sh @@ -0,0 +1,48 @@ +# Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +if [ -n "$GITHUB_RUN_ID" ]; then + brew install libffi openssl +fi +export PKG_CONFIG_PATH=/opt/homebrew/opt/openssl/lib/pkgconfig +if [ -n "$1" ]; then + pip wheel "cryptography==$1" +else + pip wheel cryptography +fi diff --git a/scripts/wheelbuilder/darwin/pandas.sh b/scripts/wheelbuilder/darwin/pandas.sh new file mode 100755 index 0000000000..9217005d7b --- /dev/null +++ b/scripts/wheelbuilder/darwin/pandas.sh @@ -0,0 +1,49 @@ +# Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +if [ -n "$GITHUB_RUN_ID" ]; then + brew install gcc openblas pkg-config + export PKG_CONFIG_PATH=/opt/homebrew/opt/openblas/lib/pkgconfig +fi +export FFLAGS=-fallow-argument-mismatch +if [ -n "$1" ]; then + pip wheel "pandas==$1" +else + pip wheel pandas +fi diff --git a/scripts/wheelbuilder/linux/cryptography.sh b/scripts/wheelbuilder/linux/cryptography.sh new file mode 100755 index 0000000000..ef0138706f --- /dev/null +++ b/scripts/wheelbuilder/linux/cryptography.sh @@ -0,0 +1,42 @@ +# Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +if command -v manylinux-interpreters 2>&1 >/dev/null; then + dnf install -y libffi-devel openssl-devel +fi diff --git a/scripts/wheelbuilder/packages.txt b/scripts/wheelbuilder/packages.txt index aa4de2a2f1..890c395feb 100644 --- a/scripts/wheelbuilder/packages.txt +++ b/scripts/wheelbuilder/packages.txt @@ -1,6 +1,133 @@ -numpy==1.26.4 +aiohappyeyeballs==2.4.6 +aiohttp==3.11.13 +aiosignal==1.3.2 +alembic==1.14.1 +annotated-types==0.7.0 +anyio==4.8.0 +asyncer==0.0.8 +attrs==25.1.0 +backoff==2.2.1 +bcrypt==4.2.1 +certifi==2025.1.31 +cffi==1.17.1 +charset-normalizer==3.4.1 +click==8.1.8 +colorama==0.4.6 +colorlog==6.9.0 +cryptography==44.0.1 +datasets==3.3.2 +deprecated==1.2.18 +dill==0.3.8 +diskcache==5.6.3 +distro==1.9.0 +dspy-ai==2.5.41 +dspy==2.5.41 +fastapi==0.115.6 +filelock==3.17.0 +frozenlist==1.5.0 +fsspec==2024.12.0 +fsspec[http]==2024.12.0 +googleapis-common-protos==1.56.1 +greenlet==3.1.1 +grpcio==1.70.0 +h11==0.14.0 +httpcore==1.0.7 httptools==0.6.1 +httpx==0.28.1 +huggingface-hub==0.29.1 +idna==3.10 +importlib-metadata==8.5.0 +javalang==0.13.0 +jinja2==3.1.5 +jiter==0.9.0 +joblib==1.4.2 +json-repair==0.39.1 +json5==0.10.0 +jsonpatch==1.33 +jsonpointer==3.0.0 +jsonschema-specifications==2024.10.1 +jsonschema==4.23.0 kiwisolver==1.4.5 +langchain-core==0.3.41 +langgraph-checkpoint==2.0.17 +langgraph-sdk==0.1.55 +langgraph==0.2.76 +langsmith==0.3.12 +levenshtein==0.26.1 +litellm==1.51.0 +magicattr==0.1.6 +mako==1.3.9 +markupsafe==3.0.2 +msgpack==1.1.0 +multidict==6.1.0 +multiprocess==0.70.16 +nest-asyncio==1.6.0 +numpy==1.26.4 +numpy==2.2.3 +openai==1.64.0 +openinference-instrumentation-dspy==0.1.9 +openinference-instrumentation-litellm==0.1.6 +openinference-instrumentation==0.1.23 +openinference-semantic-conventions==0.1.14 +opentelemetry-api==1.30.0 +opentelemetry-exporter-jaeger-proto-grpc==1.21.0 +opentelemetry-exporter-jaeger-thrift==1.21.0 +opentelemetry-exporter-jaeger==1.21.0 +opentelemetry-exporter-otlp-proto-common==1.30.0 +opentelemetry-exporter-otlp-proto-grpc==1.30.0 +opentelemetry-exporter-otlp-proto-http==1.30.0 +opentelemetry-exporter-otlp==1.30.0 +opentelemetry-exporter-prometheus==0.51b0 +opentelemetry-instrumentation==0.51b0 +opentelemetry-propagator-b3==1.30.0 +opentelemetry-proto==1.30.0 +opentelemetry-sdk==1.30.0 +opentelemetry-semantic-conventions==0.51b0 +optuna==4.2.1 +oracledb==2.5.1 +orjson==3.10.15 +packaging==24.2 +pandas==2.2.3 +paramiko==3.5.1 +prometheus-client==0.21.1 +propcache==0.3.0 +protobuf==5.29.3 psutil==5.9.8 +pyarrow==19.0.1 +pycparser==2.22 +pydantic-core==2.27.1 +pydantic==2.10.1 +pynacl==1.5.0 +python-dateutil==2.9.0.post0 +python-dotenv==1.0.1 +pytz==2025.1 +pyyaml==6.0.2 +rapidfuzz==3.12.1 +referencing==0.36.2 +regex==2024.11.6 +requests-toolbelt==1.0.0 +requests==2.32.3 +rpds-py==0.23.1 +setuptools==76.0.0 +six==1.17.0 +sniffio==1.3.1 +sqlalchemy==2.0.38 +sshtunnel==0.4.0 +starlette==0.41.3 +tenacity==9.0.0 +thrift==0.21.0 +tiktoken==0.9.0 +tokenizers==0.21.0 +tqdm==4.67.1 +typing-extensions==4.12.2 +tzdata==2025.1 +ujson==5.10.0 ujson==5.10.0 +urllib3==2.3.0 +uvicorn==0.34.0 +wrapt==1.17.2 xxhash==3.4.1 +xxhash==3.5.0 +yarl==1.18.3 +zipp==3.21.0 +zstandard==0.23.0 From 5e093302dd6135321f90522be29c6e936a47a378 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Tue, 18 Mar 2025 11:12:05 +0100 Subject: [PATCH 42/82] Run wheel scripts in subshell --- scripts/wheelbuilder/build_wheels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wheelbuilder/build_wheels.py b/scripts/wheelbuilder/build_wheels.py index 605032e50d..2c1a008d7b 100644 --- a/scripts/wheelbuilder/build_wheels.py +++ b/scripts/wheelbuilder/build_wheels.py @@ -136,7 +136,7 @@ def build_wheels(pip): if sys.platform == "win32": cmd = [script, version] # Python's subprocess.py does the quoting we need else: - cmd = f"{shlex.quote(script)} {version}" + cmd = f"{os.environ.get('SHELL', '/bin/sh')} {shlex.quote(script)} {version}" subprocess.check_call(cmd, shell=True, env=env) if not len(glob("*.whl")) > whl_count: print("Building wheel for", name, version, "after", script, "did not", flush=True) From e69ad45cea004a65aa7e63678fed590afc9fb3e6 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Tue, 18 Mar 2025 20:09:20 +0100 Subject: [PATCH 43/82] Iterate over wheels to build until none can be built anymore This helps with wheels depending on other wheels when the scripts for these dependencies have not run, yet. This way we will eventually either be able to build the dependencies, or we will reach a point where no more wheels can be built, and then we give up. --- scripts/wheelbuilder/build_wheels.py | 61 +++++++++++++++++----------- 1 file changed, 38 insertions(+), 23 deletions(-) diff --git a/scripts/wheelbuilder/build_wheels.py b/scripts/wheelbuilder/build_wheels.py index 2c1a008d7b..fdb4161a78 100644 --- a/scripts/wheelbuilder/build_wheels.py +++ b/scripts/wheelbuilder/build_wheels.py @@ -121,29 +121,42 @@ def build_wheels(pip): available_scripts = {s.lower(): s for s in os.listdir(scriptdir)} else: available_scripts = {} - for spec in packages_to_build: - name, version = spec.split("==") - whl_count = len(glob("*.whl")) - script = f"{name}.{version}.{script_ext}".lower() - if script not in available_scripts: - script = f"{name}.{script_ext}".lower() - if script in available_scripts: - script = join(scriptdir, available_scripts[script]) - env = os.environ.copy() - env["PATH"] = abspath(dirname(pip)) + os.pathsep + env["PATH"] - env["VIRTUAL_ENV"] = abspath(dirname(dirname(pip))) - print("Building", name, version, "with", script, flush=True) - if sys.platform == "win32": - cmd = [script, version] # Python's subprocess.py does the quoting we need - else: - cmd = f"{os.environ.get('SHELL', '/bin/sh')} {shlex.quote(script)} {version}" - subprocess.check_call(cmd, shell=True, env=env) - if not len(glob("*.whl")) > whl_count: - print("Building wheel for", name, version, "after", script, "did not", flush=True) - subprocess.check_call([pip, "wheel", spec]) - else: + remaining_packages = 0 + while remaining_packages != len(packages_to_build): + remaining_packages = len(packages_to_build) + for spec in packages_to_build.copy(): + name, version = spec.split("==") + whl_count = len(glob("*.whl")) + script = f"{name}.{version}.{script_ext}".lower() + if script not in available_scripts: + script = f"{name}.{script_ext}".lower() + if script in available_scripts: + script = join(scriptdir, available_scripts[script]) + env = os.environ.copy() + env["PATH"] = abspath(dirname(pip)) + os.pathsep + env["PATH"] + env["VIRTUAL_ENV"] = abspath(dirname(dirname(pip))) + print("Building", name, version, "with", script, flush=True) + if sys.platform == "win32": + cmd = [script, version] # Python's subprocess.py does the quoting we need + else: + cmd = f"{os.environ.get('SHELL', '/bin/sh')} {shlex.quote(script)} {version}" + p = subprocess.run(cmd, shell=True, env=env) + if p.returncode != 0: + continue + if len(glob("*.whl")) > whl_count: + packages_to_build.remove(spec) + continue + print(script, "did not build a wheel, we will do so now", flush=True) print("Building", name, version, flush=True) - subprocess.check_call([pip, "wheel", spec]) + p = subprocess.run([pip, "wheel", spec]) + if p.returncode == 0: + packages_to_build.remove(spec) + if packages_to_build: + print("Failed to build all packages, the following packages failed") + print(packages_to_build) + return False + else: + return True def repair_wheels(): @@ -188,5 +201,7 @@ def repair_wheels(): download(args.graalpy_url, outpath) extract(outpath) pip = create_venv() - build_wheels(pip) + success = build_wheels(pip) repair_wheels() + if not success: + sys.exit(1) From 74836f3617342a7defbc13d51c84be1a393c307f Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Tue, 18 Mar 2025 21:05:26 +0100 Subject: [PATCH 44/82] Only attempt to repair graalpy platform specific wheels --- scripts/wheelbuilder/build_wheels.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/wheelbuilder/build_wheels.py b/scripts/wheelbuilder/build_wheels.py index fdb4161a78..44f9e14026 100644 --- a/scripts/wheelbuilder/build_wheels.py +++ b/scripts/wheelbuilder/build_wheels.py @@ -160,6 +160,7 @@ def build_wheels(pip): def repair_wheels(): + whls = glob("*graalpy*.whl") if sys.platform == "win32": ensure_installed("delvewheel") env = os.environ.copy() @@ -175,19 +176,19 @@ def repair_wheels(): "python-native.dll", "-w", "wheelhouse", - *glob("*.whl"), + *whls, ], env=env, ) elif sys.platform == "linux": ensure_installed("auditwheel") subprocess.check_call( - [join(dirname(sys.executable), "auditwheel"), "repair", "-w", "wheelhouse", *glob("*.whl")] + [join(dirname(sys.executable), "auditwheel"), "repair", "-w", "wheelhouse", *whls] ) elif sys.platform == "darwin": ensure_installed("delocate") subprocess.check_call( - [join(dirname(sys.executable), "delocate-wheel"), "-v", "-w", "wheelhouse", *glob("*.whl")] + [join(dirname(sys.executable), "delocate-wheel"), "-v", "-w", "wheelhouse", *whls] ) From dd4ad6baae1da17f81a8195b041dffb4f070a787 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Wed, 19 Mar 2025 09:17:10 +0100 Subject: [PATCH 45/82] Fix multiple wheel build recipes --- scripts/wheelbuilder/linux/lxml.sh | 42 +++++++++++++++++++++ scripts/wheelbuilder/linux/numpy.1.23.2.sh | 43 ++++++++++++++++++++++ scripts/wheelbuilder/linux/pynacl.sh | 42 +++++++++++++++++++++ scripts/wheelbuilder/linux/zstandard.sh | 42 +++++++++++++++++++++ 4 files changed, 169 insertions(+) create mode 100644 scripts/wheelbuilder/linux/lxml.sh create mode 100644 scripts/wheelbuilder/linux/numpy.1.23.2.sh create mode 100644 scripts/wheelbuilder/linux/pynacl.sh create mode 100644 scripts/wheelbuilder/linux/zstandard.sh diff --git a/scripts/wheelbuilder/linux/lxml.sh b/scripts/wheelbuilder/linux/lxml.sh new file mode 100644 index 0000000000..5335db4335 --- /dev/null +++ b/scripts/wheelbuilder/linux/lxml.sh @@ -0,0 +1,42 @@ +# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +if command -v manylinux-interpreters 2>&1 >/dev/null; then + dnf install -y libxml2-devel libxslt-devel +fi diff --git a/scripts/wheelbuilder/linux/numpy.1.23.2.sh b/scripts/wheelbuilder/linux/numpy.1.23.2.sh new file mode 100644 index 0000000000..a9378be59d --- /dev/null +++ b/scripts/wheelbuilder/linux/numpy.1.23.2.sh @@ -0,0 +1,43 @@ +# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +if command -v manylinux-interpreters 2>&1 >/dev/null; then + dnf install -y gcc-toolset-9 gcc-toolset-9-gcc-gfortran openblas-devel + scl enable gcc-toolset-9 'pip wheel "numpy==1.23.2"' +fi diff --git a/scripts/wheelbuilder/linux/pynacl.sh b/scripts/wheelbuilder/linux/pynacl.sh new file mode 100644 index 0000000000..f4e5359ec6 --- /dev/null +++ b/scripts/wheelbuilder/linux/pynacl.sh @@ -0,0 +1,42 @@ +# Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +if command -v manylinux-interpreters 2>&1 >/dev/null; then + dnf install -y libffi-devel +fi diff --git a/scripts/wheelbuilder/linux/zstandard.sh b/scripts/wheelbuilder/linux/zstandard.sh new file mode 100644 index 0000000000..f4e5359ec6 --- /dev/null +++ b/scripts/wheelbuilder/linux/zstandard.sh @@ -0,0 +1,42 @@ +# Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +if command -v manylinux-interpreters 2>&1 >/dev/null; then + dnf install -y libffi-devel +fi From 9438cfb22fc6f319d314b1a8d0c817e0ca4f02cd Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Wed, 19 Mar 2025 15:26:17 +0100 Subject: [PATCH 46/82] Fix errors with rust toolchain and act, add 1.24.3 script for numpy --- scripts/wheelbuilder/linux/numpy.1.24.3.sh | 43 ++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 scripts/wheelbuilder/linux/numpy.1.24.3.sh diff --git a/scripts/wheelbuilder/linux/numpy.1.24.3.sh b/scripts/wheelbuilder/linux/numpy.1.24.3.sh new file mode 100644 index 0000000000..7d50924cea --- /dev/null +++ b/scripts/wheelbuilder/linux/numpy.1.24.3.sh @@ -0,0 +1,43 @@ +# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +if command -v manylinux-interpreters 2>&1 >/dev/null; then + dnf install -y gcc-toolset-9 gcc-toolset-9-gcc-gfortran openblas-devel + scl enable gcc-toolset-9 'pip wheel "numpy==1.24.3"' +fi From 3275c4b3a7417b3e0052fbd1ec13dd2a915fe573 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Thu, 20 Mar 2025 09:45:10 +0100 Subject: [PATCH 47/82] Fix pyarrow 0.19.1 wheel building and build sequentially, parallel OOMs on a 16GB container --- scripts/wheelbuilder/linux/pyarrow.sh | 58 +++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 scripts/wheelbuilder/linux/pyarrow.sh diff --git a/scripts/wheelbuilder/linux/pyarrow.sh b/scripts/wheelbuilder/linux/pyarrow.sh new file mode 100644 index 0000000000..ed554ae5c4 --- /dev/null +++ b/scripts/wheelbuilder/linux/pyarrow.sh @@ -0,0 +1,58 @@ +# Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +if command -v manylinux-interpreters 2>&1 >/dev/null; then + dnf install -y libffi-devel \ + boost-devel \ + snappy-devel \ + brotli-devel \ + openssl-devel \ + thrift-devel \ + jemalloc-devel \ + xsimd-devel \ + libzstd-devel \ + re2-devel \ + mimalloc-devel \ + lz4-devel \ + bzip2-devel \ + llvm llvm-libs llvm-devel \ + llvm-cmake-utils \ + lld lld-devel \ + clang clang-libs clang-devel +fi From 2824a3d9cd91fa2776366a492a81f68c849798f7 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Fri, 21 Mar 2025 10:53:39 +0100 Subject: [PATCH 48/82] Switch to an orjson version we can build --- scripts/wheelbuilder/packages.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wheelbuilder/packages.txt b/scripts/wheelbuilder/packages.txt index 890c395feb..352261afe0 100644 --- a/scripts/wheelbuilder/packages.txt +++ b/scripts/wheelbuilder/packages.txt @@ -85,7 +85,7 @@ opentelemetry-sdk==1.30.0 opentelemetry-semantic-conventions==0.51b0 optuna==4.2.1 oracledb==2.5.1 -orjson==3.10.15 +orjson==3.10.5 packaging==24.2 pandas==2.2.3 paramiko==3.5.1 From 0695934f5e893eee8e9f2923c3536cfc45a2a634 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Fri, 21 Mar 2025 15:06:36 +0100 Subject: [PATCH 49/82] Add argument to wheelbuilder to ignore failures in generating exit code --- scripts/wheelbuilder/build_wheels.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/wheelbuilder/build_wheels.py b/scripts/wheelbuilder/build_wheels.py index 44f9e14026..909d1f5c01 100644 --- a/scripts/wheelbuilder/build_wheels.py +++ b/scripts/wheelbuilder/build_wheels.py @@ -195,6 +195,7 @@ def repair_wheels(): if __name__ == "__main__": parser = ArgumentParser() parser.add_argument("graalpy_url") + parser.add_argument("--ignore-failures", action="store_true", default=False) args = parser.parse_args() ext = splitext(args.graalpy_url)[1] outpath = f"graalpy{ext}" @@ -204,5 +205,5 @@ def repair_wheels(): pip = create_venv() success = build_wheels(pip) repair_wheels() - if not success: + if not success and not args.ignore_failures: sys.exit(1) From 4892dc35d95c7b44f4a7da5af9d76926da74aabd Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Fri, 21 Mar 2025 22:41:34 +0100 Subject: [PATCH 50/82] If repairing a wheel fails, keep the original --- scripts/wheelbuilder/build_wheels.py | 65 +++++++++++++++------------- 1 file changed, 36 insertions(+), 29 deletions(-) diff --git a/scripts/wheelbuilder/build_wheels.py b/scripts/wheelbuilder/build_wheels.py index 909d1f5c01..4297163c9d 100644 --- a/scripts/wheelbuilder/build_wheels.py +++ b/scripts/wheelbuilder/build_wheels.py @@ -161,35 +161,42 @@ def build_wheels(pip): def repair_wheels(): whls = glob("*graalpy*.whl") - if sys.platform == "win32": - ensure_installed("delvewheel") - env = os.environ.copy() - env["PYTHONUTF8"] = "1" - subprocess.check_call( - [ - sys.executable, - "-m", - "delvewheel", - "repair", - "-v", - "--exclude", - "python-native.dll", - "-w", - "wheelhouse", - *whls, - ], - env=env, - ) - elif sys.platform == "linux": - ensure_installed("auditwheel") - subprocess.check_call( - [join(dirname(sys.executable), "auditwheel"), "repair", "-w", "wheelhouse", *whls] - ) - elif sys.platform == "darwin": - ensure_installed("delocate") - subprocess.check_call( - [join(dirname(sys.executable), "delocate-wheel"), "-v", "-w", "wheelhouse", *whls] - ) + for whl in whls: + if sys.platform == "win32": + ensure_installed("delvewheel") + env = os.environ.copy() + env["PYTHONUTF8"] = "1" + p = subprocess.run( + [ + sys.executable, + "-m", + "delvewheel", + "repair", + "-v", + "--exclude", + "python-native.dll", + "-w", + "wheelhouse", + whl, + ], + env=env, + ) + elif sys.platform == "linux": + ensure_installed("auditwheel") + p = subprocess.run( + [join(dirname(sys.executable), "auditwheel"), "repair", "-w", "wheelhouse", whl] + ) + elif sys.platform == "darwin": + ensure_installed("delocate") + p = subprocess.run( + [join(dirname(sys.executable), "delocate-wheel"), "-v", "-w", "wheelhouse", whl] + ) + if p.returncode != 0: + print("Repairing", whl, "failed, copying as is.") + try: + shutil.copy(whl, "wheelhouse") + except: + pass if __name__ == "__main__": From 9a84d9599aa3693be6407bd5f99e8c1c22f9c404 Mon Sep 17 00:00:00 2001 From: Ivo Horak Date: Tue, 25 Mar 2025 08:56:29 +0100 Subject: [PATCH 51/82] Adding pynacl.sh --- scripts/wheelbuilder/darwin/pynacl.sh | 51 +++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 scripts/wheelbuilder/darwin/pynacl.sh diff --git a/scripts/wheelbuilder/darwin/pynacl.sh b/scripts/wheelbuilder/darwin/pynacl.sh new file mode 100644 index 0000000000..cb10d556a6 --- /dev/null +++ b/scripts/wheelbuilder/darwin/pynacl.sh @@ -0,0 +1,51 @@ +# Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +if [ -n "$GITHUB_RUN_ID" ]; then + brew install libsodium pkg-config + export PKG_CONFIG_PATH=/opt/homebrew/opt/openblas/lib/pkgconfig +fi +export SODIUM_INSTALL=system +export CFLAGS="-arch arm64 $(pkg-config libsodium --cflags)" +export LDFLAGS="$(pkg-config libsodium --libs)" +if [ -n "$1" ]; then + pip wheel "pynacl==$1" +else + pip wheel pynacl +fi From f4f7c263e461f15f4eac789072e60bdab40efff2 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Mon, 24 Mar 2025 10:02:52 +0100 Subject: [PATCH 52/82] Make sure to install pkg-config when using it --- scripts/wheelbuilder/darwin/cryptography.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wheelbuilder/darwin/cryptography.sh b/scripts/wheelbuilder/darwin/cryptography.sh index aa224941ef..500059fb52 100755 --- a/scripts/wheelbuilder/darwin/cryptography.sh +++ b/scripts/wheelbuilder/darwin/cryptography.sh @@ -38,7 +38,7 @@ # SOFTWARE. if [ -n "$GITHUB_RUN_ID" ]; then - brew install libffi openssl + brew install pkg-config libffi openssl fi export PKG_CONFIG_PATH=/opt/homebrew/opt/openssl/lib/pkgconfig if [ -n "$1" ]; then From 3b62902c83583afa5ad204ec4e8a9efc93308f35 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Mon, 24 Mar 2025 17:42:54 +0100 Subject: [PATCH 53/82] Try our best with delocate on macos --- scripts/wheelbuilder/build_wheels.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/scripts/wheelbuilder/build_wheels.py b/scripts/wheelbuilder/build_wheels.py index 4297163c9d..036bbe1de4 100644 --- a/scripts/wheelbuilder/build_wheels.py +++ b/scripts/wheelbuilder/build_wheels.py @@ -50,6 +50,7 @@ import hashlib import importlib import os +import platform import re import shlex import shutil @@ -189,7 +190,16 @@ def repair_wheels(): elif sys.platform == "darwin": ensure_installed("delocate") p = subprocess.run( - [join(dirname(sys.executable), "delocate-wheel"), "-v", "-w", "wheelhouse", whl] + [ + join(dirname(sys.executable), "delocate-wheel"), + "-v", + "--ignore-missing-dependencies", + "--require-archs", + "arm64" if platform.processor() == "arm" else "x86_64", + "-w", + "wheelhouse", + whl, + ], ) if p.returncode != 0: print("Repairing", whl, "failed, copying as is.") From 0823d812caa36ed05ff6f284acb05cdb54ece385 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Mon, 24 Mar 2025 20:56:13 +0100 Subject: [PATCH 54/82] Expand readme for wheelbuilder --- scripts/wheelbuilder/README.md | 48 ++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/scripts/wheelbuilder/README.md b/scripts/wheelbuilder/README.md index c12d7c5643..8f314373fa 100644 --- a/scripts/wheelbuilder/README.md +++ b/scripts/wheelbuilder/README.md @@ -33,3 +33,51 @@ That can be useful to put things like package installations specific to GitHub A Just run the `build_wheels.py` script. It expects a URL to download the GraalPy release from. You can set the environment variable `PACKAGES_TO_BUILD` to a comma-separated list of package build scripts you want to consider. + +You can build wheels locally. +The way to go about it is to make sure to include the packages you care about in the packages.txt. + +For Linux/amd64, I use [act](https://github.com/nektos/act/releases) on a Linux machine: +```shell +git clone https://github.com/oracle/graalpython +cd graalpython +VERSION=24.2.0 +BINDIR=. curl --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/nektos/act/master/install.sh | bash +echo "graalpy_url=https://github.com/oracle/graalpython/releases/download/graal-$VERSION/graalpy-$VERSION-linux-amd64.tar.gz" > .input +podman system service -t 0 unix:///tmp/podman.sock & +export DOCKER_HOST=unix:///tmp/podman.sock +./act --env http_proxy=$http_proxy --env https_proxy=$https_proxy -W .github/workflows/build-linux-amd64-wheels.yml --artifact-server-path=$(pwd)/artifacts +``` + +For Linux/aarch64, I use act on a mac, those are usually beefy ARM machines that a developer has in front of them, if we make sure the podman VM has enough memory. +```shell +git clone https://github.com/oracle/graalpython +cd graalpython +VERSION=24.2.0 +brew install act +echo "graalpy_url=https://github.com/oracle/graalpython/releases/download/graal-$VERSION/graalpy-$VERSION-linux-aarch64.tar.gz" > .input +podman machine init -m 16384 --now +act --env http_proxy=$http_proxy --env https_proxy=$https_proxy -W .github/workflows/build-linux-aarch64-wheels.yml --artifact-server-path=$(pwd)/artifacts --container-architecture linux/aarch64 +``` + +For macOS/aarch64, you get no isolation from act, so I just run it directly. +```shell +git clone https://github.com/oracle/graalpython +VERSION=24.2.0 +export GITHUB_RUN_ID=doesntMatterJustTriggerBrewInstallScripts +export PIP_GRAALPY_PATCHES_URL="$(pwd)/graalpython/graalpython/lib-graalpython/patches" +python3 -m venv wheelbuilder-venv +. wheelbuilder-venv/bin/activate +python3 graalpython/scripts/wheelbuilder/build_wheels.py https://github.com/oracle/graalpython/releases/download/graal-$VERSION/graalpy-$VERSION-macos-aarch64.tar.gz +``` + +For Windows/amd64, you get no isolation from act, so I just run it directly in Visual Studio powershell. +```shell +git clone https://github.com/oracle/graalpython +$VERSION="24.2.0" +$env:GITHUB_RUN_ID="doesntMatterJustTriggerBrewInstallScripts" +$env:PIP_GRAALPY_PATCHES_URL="$PWD/graalpython/graalpython/lib-graalpython/patches" +python3 -m venv wheelbuilder-venv +wheelbuilder-venv/scripts/activate +python3 graalpython/scripts/wheelbuilder/build_wheels.py https://github.com/oracle/graalpython/releases/download/graal-$VERSION/graalpy-$VERSION-windows-amd64.zip +``` From 807927675c1a47255539ca47e37bdc53914588f6 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Tue, 25 Mar 2025 12:36:44 +0100 Subject: [PATCH 55/82] Get python-patch-ng to provide patch.exe on windows --- scripts/wheelbuilder/build_wheels.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/wheelbuilder/build_wheels.py b/scripts/wheelbuilder/build_wheels.py index 036bbe1de4..00e4fe13e5 100644 --- a/scripts/wheelbuilder/build_wheels.py +++ b/scripts/wheelbuilder/build_wheels.py @@ -101,6 +101,11 @@ def create_venv(): subprocess.check_call([binary, "-m", "venv", "graalpy"]) print("Installing wheel with", pip, flush=True) subprocess.check_call([pip, "install", "wheel"]) + if sys.platform == "win32": + print("Installing python-patch-ng to provide patch.exe", flush=True) + p = subprocess.run([pip, "install", "git+https://github.com/timfel/python-patch-ng.git"]) + if p.returncode != 0: + print("Installing python-patch-ng failed, assuming patch.exe is on PATH", flush=True) return pip From d441e37b472edf52d6eec4c22b5d27e0bd96cf32 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Tue, 25 Mar 2025 13:39:06 +0100 Subject: [PATCH 56/82] Avoid redownload in build_wheels.py --- scripts/wheelbuilder/build_wheels.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/wheelbuilder/build_wheels.py b/scripts/wheelbuilder/build_wheels.py index 00e4fe13e5..9b462d3196 100644 --- a/scripts/wheelbuilder/build_wheels.py +++ b/scripts/wheelbuilder/build_wheels.py @@ -75,8 +75,11 @@ def ensure_installed(name): def download(url, out): - print("Downloading", url, flush=True) - urlretrieve(url, out) + if not os.path.exists(out): + print("Downloading", url, flush=True) + urlretrieve(url, out) + else: + print("Using previously downloaded", out, flush=True) def extract(archive): From c0821851c1b716d9176b6fa23cd824c9fcccb868 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Tue, 25 Mar 2025 13:59:16 +0100 Subject: [PATCH 57/82] Make sure we recythonize rapidfuzz 3.10 - 3.12 --- .../wheelbuilder/darwin/levenshtein.0.26.1.sh | 62 +++++++++++++++++++ .../wheelbuilder/darwin/rapidfuzz.3.12.1.sh | 41 ++++++++++++ .../wheelbuilder/linux/levenshtein.0.26.1.sh | 62 +++++++++++++++++++ .../wheelbuilder/linux/rapidfuzz.3.12.1.sh | 41 ++++++++++++ 4 files changed, 206 insertions(+) create mode 100644 scripts/wheelbuilder/darwin/levenshtein.0.26.1.sh create mode 100644 scripts/wheelbuilder/darwin/rapidfuzz.3.12.1.sh create mode 100644 scripts/wheelbuilder/linux/levenshtein.0.26.1.sh create mode 100644 scripts/wheelbuilder/linux/rapidfuzz.3.12.1.sh diff --git a/scripts/wheelbuilder/darwin/levenshtein.0.26.1.sh b/scripts/wheelbuilder/darwin/levenshtein.0.26.1.sh new file mode 100644 index 0000000000..04be0e91b7 --- /dev/null +++ b/scripts/wheelbuilder/darwin/levenshtein.0.26.1.sh @@ -0,0 +1,62 @@ +# Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +if [ -n "$GITHUB_RUN_ID" ]; then + git clone --single-branch --branch v3.3.2 --depth 1 https://github.com/rapidfuzz/rapidfuzz-cpp.git rapidfuzz-cpp + cd rapidfuzz-cpp + mkdir build && cd build + cmake .. -DCMAKE_BUILD_TYPE=Release + cmake --build . + sudo cmake --build . --target install + cd .. + cd .. +fi + +pip install 'Cython==3.0.11' 'scikit_build_core==0.11.1' +pip install --no-build-isolation 'rapidfuzz==3.12.1' +pip wheel --no-build-isolation 'levenshtein==0.26.1' + +if [ -n "$GITHUB_RUN_ID" ]; then + cd rapidfuzz-cpp + cd build + xargs rm < install_manifest.txt + cd .. + cd .. + rm -rf rapidfuzz-cpp +fi diff --git a/scripts/wheelbuilder/darwin/rapidfuzz.3.12.1.sh b/scripts/wheelbuilder/darwin/rapidfuzz.3.12.1.sh new file mode 100644 index 0000000000..8c083b59b0 --- /dev/null +++ b/scripts/wheelbuilder/darwin/rapidfuzz.3.12.1.sh @@ -0,0 +1,41 @@ +# Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +pip install 'Cython==3.0.11' 'scikit_build_core==0.11.1' +pip wheel --no-build-isolation 'rapidfuzz==3.12.1' diff --git a/scripts/wheelbuilder/linux/levenshtein.0.26.1.sh b/scripts/wheelbuilder/linux/levenshtein.0.26.1.sh new file mode 100644 index 0000000000..9fcc8842de --- /dev/null +++ b/scripts/wheelbuilder/linux/levenshtein.0.26.1.sh @@ -0,0 +1,62 @@ +# Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +if command -v manylinux-interpreters 2>&1 >/dev/null; then + git clone --single-branch --branch v3.3.2 --depth 1 https://github.com/rapidfuzz/rapidfuzz-cpp.git rapidfuzz-cpp + cd rapidfuzz-cpp + mkdir build && cd build + cmake .. -DCMAKE_BUILD_TYPE=Release + cmake --build . + sudo cmake --build . --target install + cd .. + cd .. +fi + +pip install 'Cython==3.0.11' 'scikit_build_core==0.11.1' +pip install --no-build-isolation 'rapidfuzz==3.12.1' +pip wheel --no-build-isolation 'levenshtein==0.26.1' + +if command -v manylinux-interpreters 2>&1 >/dev/null; then + cd rapidfuzz-cpp + cd build + xargs rm < install_manifest.txt + cd .. + cd .. + rm -rf rapidfuzz-cpp +fi diff --git a/scripts/wheelbuilder/linux/rapidfuzz.3.12.1.sh b/scripts/wheelbuilder/linux/rapidfuzz.3.12.1.sh new file mode 100644 index 0000000000..8c083b59b0 --- /dev/null +++ b/scripts/wheelbuilder/linux/rapidfuzz.3.12.1.sh @@ -0,0 +1,41 @@ +# Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +pip install 'Cython==3.0.11' 'scikit_build_core==0.11.1' +pip wheel --no-build-isolation 'rapidfuzz==3.12.1' From c7d6a3415b14bb7417844a71d57467383953e1a8 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Thu, 27 Mar 2025 13:03:51 +0100 Subject: [PATCH 58/82] auditwheel repair for OL8 --- scripts/wheelbuilder/build_wheels.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/scripts/wheelbuilder/build_wheels.py b/scripts/wheelbuilder/build_wheels.py index 9b462d3196..883478ac45 100644 --- a/scripts/wheelbuilder/build_wheels.py +++ b/scripts/wheelbuilder/build_wheels.py @@ -193,7 +193,15 @@ def repair_wheels(): elif sys.platform == "linux": ensure_installed("auditwheel") p = subprocess.run( - [join(dirname(sys.executable), "auditwheel"), "repair", "-w", "wheelhouse", whl] + [ + join(dirname(sys.executable), "auditwheel"), + "repair", + "--plat", + "manylinux_2_28_x86_64", + "-w", + "wheelhouse", + whl, + ] ) elif sys.platform == "darwin": ensure_installed("delocate") From 6201debe3bfaac58bf5d39fd1918755bfdd0ff3a Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Thu, 27 Mar 2025 13:09:44 +0100 Subject: [PATCH 59/82] Install patchelf in addition to auditwheel into the venv --- scripts/wheelbuilder/build_wheels.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/scripts/wheelbuilder/build_wheels.py b/scripts/wheelbuilder/build_wheels.py index 883478ac45..24ae6628a7 100644 --- a/scripts/wheelbuilder/build_wheels.py +++ b/scripts/wheelbuilder/build_wheels.py @@ -66,11 +66,11 @@ from urllib.request import urlretrieve -def ensure_installed(name): +def ensure_installed(name, *extra): try: return importlib.import_module(name) except ImportError: - subprocess.check_call([sys.executable, "-m", "pip", "install", name]) + subprocess.check_call([sys.executable, "-m", "pip", "install", name, *extra]) return importlib.import_module(name) @@ -170,11 +170,12 @@ def build_wheels(pip): def repair_wheels(): whls = glob("*graalpy*.whl") + env = os.environ.copy() + env["PYTHONUTF8"] = "1" + env["PATH"] = abspath(dirname(sys.executable)) + os.pathsep + env["PATH"] for whl in whls: if sys.platform == "win32": ensure_installed("delvewheel") - env = os.environ.copy() - env["PYTHONUTF8"] = "1" p = subprocess.run( [ sys.executable, @@ -191,7 +192,7 @@ def repair_wheels(): env=env, ) elif sys.platform == "linux": - ensure_installed("auditwheel") + ensure_installed("auditwheel", "patchelf") p = subprocess.run( [ join(dirname(sys.executable), "auditwheel"), @@ -201,7 +202,8 @@ def repair_wheels(): "-w", "wheelhouse", whl, - ] + ], + env=env, ) elif sys.platform == "darwin": ensure_installed("delocate") @@ -216,6 +218,7 @@ def repair_wheels(): "wheelhouse", whl, ], + env=env, ) if p.returncode != 0: print("Repairing", whl, "failed, copying as is.") From c560393f4b6fed59166d49d07c9220ab6589c38e Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Thu, 27 Mar 2025 14:10:14 +0100 Subject: [PATCH 60/82] Use GITHUB_RUN_ID envvar everywhere to decide if build wheels shell scripts should run --- scripts/wheelbuilder/linux/cffi.sh | 2 +- scripts/wheelbuilder/linux/cryptography.sh | 2 +- scripts/wheelbuilder/linux/h5py.sh | 2 +- scripts/wheelbuilder/linux/levenshtein.0.26.1.sh | 4 ++-- scripts/wheelbuilder/linux/lxml.sh | 2 +- scripts/wheelbuilder/linux/numpy.1.23.2.sh | 2 +- scripts/wheelbuilder/linux/numpy.1.24.3.sh | 2 +- scripts/wheelbuilder/linux/numpy.sh | 2 +- scripts/wheelbuilder/linux/pillow.sh | 2 +- scripts/wheelbuilder/linux/pyarrow.sh | 2 +- scripts/wheelbuilder/linux/pynacl.sh | 2 +- scripts/wheelbuilder/linux/scipy.sh | 2 +- scripts/wheelbuilder/linux/tensorflow.sh | 2 +- scripts/wheelbuilder/linux/torch.sh | 2 +- scripts/wheelbuilder/linux/zstandard.sh | 2 +- 15 files changed, 16 insertions(+), 16 deletions(-) diff --git a/scripts/wheelbuilder/linux/cffi.sh b/scripts/wheelbuilder/linux/cffi.sh index 6770b98e27..f883a887ae 100755 --- a/scripts/wheelbuilder/linux/cffi.sh +++ b/scripts/wheelbuilder/linux/cffi.sh @@ -37,6 +37,6 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -if command -v manylinux-interpreters 2>&1 >/dev/null; then +if [ -n "$GITHUB_RUN_ID" ]; then dnf install -y libffi-devel fi diff --git a/scripts/wheelbuilder/linux/cryptography.sh b/scripts/wheelbuilder/linux/cryptography.sh index ef0138706f..eb9b402c81 100755 --- a/scripts/wheelbuilder/linux/cryptography.sh +++ b/scripts/wheelbuilder/linux/cryptography.sh @@ -37,6 +37,6 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -if command -v manylinux-interpreters 2>&1 >/dev/null; then +if [ -n "$GITHUB_RUN_ID" ]; then dnf install -y libffi-devel openssl-devel fi diff --git a/scripts/wheelbuilder/linux/h5py.sh b/scripts/wheelbuilder/linux/h5py.sh index 82172d55f0..f680b5f69e 100755 --- a/scripts/wheelbuilder/linux/h5py.sh +++ b/scripts/wheelbuilder/linux/h5py.sh @@ -37,6 +37,6 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -if command -v manylinux-interpreters 2>&1 >/dev/null; then +if [ -n "$GITHUB_RUN_ID" ]; then dnf install -y hdf5-devel fi diff --git a/scripts/wheelbuilder/linux/levenshtein.0.26.1.sh b/scripts/wheelbuilder/linux/levenshtein.0.26.1.sh index 9fcc8842de..04be0e91b7 100644 --- a/scripts/wheelbuilder/linux/levenshtein.0.26.1.sh +++ b/scripts/wheelbuilder/linux/levenshtein.0.26.1.sh @@ -37,7 +37,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -if command -v manylinux-interpreters 2>&1 >/dev/null; then +if [ -n "$GITHUB_RUN_ID" ]; then git clone --single-branch --branch v3.3.2 --depth 1 https://github.com/rapidfuzz/rapidfuzz-cpp.git rapidfuzz-cpp cd rapidfuzz-cpp mkdir build && cd build @@ -52,7 +52,7 @@ pip install 'Cython==3.0.11' 'scikit_build_core==0.11.1' pip install --no-build-isolation 'rapidfuzz==3.12.1' pip wheel --no-build-isolation 'levenshtein==0.26.1' -if command -v manylinux-interpreters 2>&1 >/dev/null; then +if [ -n "$GITHUB_RUN_ID" ]; then cd rapidfuzz-cpp cd build xargs rm < install_manifest.txt diff --git a/scripts/wheelbuilder/linux/lxml.sh b/scripts/wheelbuilder/linux/lxml.sh index 5335db4335..c22e81e04b 100644 --- a/scripts/wheelbuilder/linux/lxml.sh +++ b/scripts/wheelbuilder/linux/lxml.sh @@ -37,6 +37,6 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -if command -v manylinux-interpreters 2>&1 >/dev/null; then +if [ -n "$GITHUB_RUN_ID" ]; then dnf install -y libxml2-devel libxslt-devel fi diff --git a/scripts/wheelbuilder/linux/numpy.1.23.2.sh b/scripts/wheelbuilder/linux/numpy.1.23.2.sh index a9378be59d..eeafa70a2a 100644 --- a/scripts/wheelbuilder/linux/numpy.1.23.2.sh +++ b/scripts/wheelbuilder/linux/numpy.1.23.2.sh @@ -37,7 +37,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -if command -v manylinux-interpreters 2>&1 >/dev/null; then +if [ -n "$GITHUB_RUN_ID" ]; then dnf install -y gcc-toolset-9 gcc-toolset-9-gcc-gfortran openblas-devel scl enable gcc-toolset-9 'pip wheel "numpy==1.23.2"' fi diff --git a/scripts/wheelbuilder/linux/numpy.1.24.3.sh b/scripts/wheelbuilder/linux/numpy.1.24.3.sh index 7d50924cea..5b6d9e0613 100644 --- a/scripts/wheelbuilder/linux/numpy.1.24.3.sh +++ b/scripts/wheelbuilder/linux/numpy.1.24.3.sh @@ -37,7 +37,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -if command -v manylinux-interpreters 2>&1 >/dev/null; then +if [ -n "$GITHUB_RUN_ID" ]; then dnf install -y gcc-toolset-9 gcc-toolset-9-gcc-gfortran openblas-devel scl enable gcc-toolset-9 'pip wheel "numpy==1.24.3"' fi diff --git a/scripts/wheelbuilder/linux/numpy.sh b/scripts/wheelbuilder/linux/numpy.sh index a24e6a2aa3..57e2076985 100755 --- a/scripts/wheelbuilder/linux/numpy.sh +++ b/scripts/wheelbuilder/linux/numpy.sh @@ -37,6 +37,6 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -if command -v manylinux-interpreters 2>&1 >/dev/null; then +if [ -n "$GITHUB_RUN_ID" ]; then dnf install -y gcc-toolset-12-gcc-gfortran openblas-devel fi diff --git a/scripts/wheelbuilder/linux/pillow.sh b/scripts/wheelbuilder/linux/pillow.sh index e6d64d0863..668580ab54 100755 --- a/scripts/wheelbuilder/linux/pillow.sh +++ b/scripts/wheelbuilder/linux/pillow.sh @@ -37,6 +37,6 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -if command -v manylinux-interpreters 2>&1 >/dev/null; then +if [ -n "$GITHUB_RUN_ID" ]; then dnf install -y libtiff-devel libjpeg-devel openjpeg2-devel zlib-devel freetype-devel lcms2-devel fi diff --git a/scripts/wheelbuilder/linux/pyarrow.sh b/scripts/wheelbuilder/linux/pyarrow.sh index ed554ae5c4..1f56c7bee9 100644 --- a/scripts/wheelbuilder/linux/pyarrow.sh +++ b/scripts/wheelbuilder/linux/pyarrow.sh @@ -37,7 +37,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -if command -v manylinux-interpreters 2>&1 >/dev/null; then +if [ -n "$GITHUB_RUN_ID" ]; then dnf install -y libffi-devel \ boost-devel \ snappy-devel \ diff --git a/scripts/wheelbuilder/linux/pynacl.sh b/scripts/wheelbuilder/linux/pynacl.sh index f4e5359ec6..8d488965e9 100644 --- a/scripts/wheelbuilder/linux/pynacl.sh +++ b/scripts/wheelbuilder/linux/pynacl.sh @@ -37,6 +37,6 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -if command -v manylinux-interpreters 2>&1 >/dev/null; then +if [ -n "$GITHUB_RUN_ID" ]; then dnf install -y libffi-devel fi diff --git a/scripts/wheelbuilder/linux/scipy.sh b/scripts/wheelbuilder/linux/scipy.sh index cb127c5abc..53e2005eda 100755 --- a/scripts/wheelbuilder/linux/scipy.sh +++ b/scripts/wheelbuilder/linux/scipy.sh @@ -37,7 +37,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -if command -v manylinux-interpreters 2>&1 >/dev/null; then +if [ -n "$GITHUB_RUN_ID" ]; then dnf install -y gcc-toolset-12-gcc-gfortran openblas-devel fi export FFLAGS=-fallow-argument-mismatch diff --git a/scripts/wheelbuilder/linux/tensorflow.sh b/scripts/wheelbuilder/linux/tensorflow.sh index f67401d4bf..a7cfb98b4f 100755 --- a/scripts/wheelbuilder/linux/tensorflow.sh +++ b/scripts/wheelbuilder/linux/tensorflow.sh @@ -37,7 +37,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -if command -v manylinux-interpreters 2>&1 >/dev/null; then +if [ -n "$GITHUB_RUN_ID" ]; then dnf install -y openblas-devel /usr/bin/cmake /usr/bin/sudo /usr/bin/curl java-11-openjdk-devel fi pip install pip numpy wheel packaging requests opt_einsum diff --git a/scripts/wheelbuilder/linux/torch.sh b/scripts/wheelbuilder/linux/torch.sh index e931f26abd..7ae3d60110 100755 --- a/scripts/wheelbuilder/linux/torch.sh +++ b/scripts/wheelbuilder/linux/torch.sh @@ -37,7 +37,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -if command -v manylinux-interpreters 2>&1 >/dev/null; then +if [ -n "$GITHUB_RUN_ID" ]; then dnf install -y openblas-devel /usr/bin/cmake /usr/bin/sudo libffi-devel export USE_CUDA=0 fi diff --git a/scripts/wheelbuilder/linux/zstandard.sh b/scripts/wheelbuilder/linux/zstandard.sh index f4e5359ec6..8d488965e9 100644 --- a/scripts/wheelbuilder/linux/zstandard.sh +++ b/scripts/wheelbuilder/linux/zstandard.sh @@ -37,6 +37,6 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -if command -v manylinux-interpreters 2>&1 >/dev/null; then +if [ -n "$GITHUB_RUN_ID" ]; then dnf install -y libffi-devel fi From 72c21257e0546bf78acbed7225709e070b8898b9 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Thu, 27 Mar 2025 14:15:35 +0100 Subject: [PATCH 61/82] Select manylinux tag appropriate for processor architecture --- scripts/wheelbuilder/build_wheels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wheelbuilder/build_wheels.py b/scripts/wheelbuilder/build_wheels.py index 24ae6628a7..4a5181adb0 100644 --- a/scripts/wheelbuilder/build_wheels.py +++ b/scripts/wheelbuilder/build_wheels.py @@ -198,7 +198,7 @@ def repair_wheels(): join(dirname(sys.executable), "auditwheel"), "repair", "--plat", - "manylinux_2_28_x86_64", + "manylinux_2_28_x86_64" if platform.processor() == "x86_64" else "manylinux_2_28_aarch64", "-w", "wheelhouse", whl, From 532c1a41581d739179d30edee24fc93e637bba7e Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Fri, 28 Mar 2025 11:09:20 +0100 Subject: [PATCH 62/82] Update copyrights, fix MACOSX_DEPLOYMENT_TARGET to 11 --- scripts/wheelbuilder/build_wheels.py | 2 +- scripts/wheelbuilder/linux/cffi.sh | 2 +- scripts/wheelbuilder/linux/h5py.sh | 2 +- scripts/wheelbuilder/linux/lxml.sh | 2 +- scripts/wheelbuilder/linux/numpy.1.23.2.sh | 2 +- scripts/wheelbuilder/linux/numpy.1.24.3.sh | 2 +- scripts/wheelbuilder/linux/numpy.sh | 2 +- scripts/wheelbuilder/linux/pillow.sh | 2 +- scripts/wheelbuilder/linux/scipy.sh | 2 +- scripts/wheelbuilder/linux/tensorflow.sh | 2 +- scripts/wheelbuilder/linux/torch.sh | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/scripts/wheelbuilder/build_wheels.py b/scripts/wheelbuilder/build_wheels.py index 4a5181adb0..78756b84a8 100644 --- a/scripts/wheelbuilder/build_wheels.py +++ b/scripts/wheelbuilder/build_wheels.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2023, 2025, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # The Universal Permissive License (UPL), Version 1.0 diff --git a/scripts/wheelbuilder/linux/cffi.sh b/scripts/wheelbuilder/linux/cffi.sh index f883a887ae..21c79d86b7 100755 --- a/scripts/wheelbuilder/linux/cffi.sh +++ b/scripts/wheelbuilder/linux/cffi.sh @@ -1,4 +1,4 @@ -# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # The Universal Permissive License (UPL), Version 1.0 diff --git a/scripts/wheelbuilder/linux/h5py.sh b/scripts/wheelbuilder/linux/h5py.sh index f680b5f69e..7c70c6eba1 100755 --- a/scripts/wheelbuilder/linux/h5py.sh +++ b/scripts/wheelbuilder/linux/h5py.sh @@ -1,4 +1,4 @@ -# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # The Universal Permissive License (UPL), Version 1.0 diff --git a/scripts/wheelbuilder/linux/lxml.sh b/scripts/wheelbuilder/linux/lxml.sh index c22e81e04b..40cf8d251e 100644 --- a/scripts/wheelbuilder/linux/lxml.sh +++ b/scripts/wheelbuilder/linux/lxml.sh @@ -1,4 +1,4 @@ -# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # The Universal Permissive License (UPL), Version 1.0 diff --git a/scripts/wheelbuilder/linux/numpy.1.23.2.sh b/scripts/wheelbuilder/linux/numpy.1.23.2.sh index eeafa70a2a..473b91359d 100644 --- a/scripts/wheelbuilder/linux/numpy.1.23.2.sh +++ b/scripts/wheelbuilder/linux/numpy.1.23.2.sh @@ -1,4 +1,4 @@ -# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # The Universal Permissive License (UPL), Version 1.0 diff --git a/scripts/wheelbuilder/linux/numpy.1.24.3.sh b/scripts/wheelbuilder/linux/numpy.1.24.3.sh index 5b6d9e0613..f9e524e3d4 100644 --- a/scripts/wheelbuilder/linux/numpy.1.24.3.sh +++ b/scripts/wheelbuilder/linux/numpy.1.24.3.sh @@ -1,4 +1,4 @@ -# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # The Universal Permissive License (UPL), Version 1.0 diff --git a/scripts/wheelbuilder/linux/numpy.sh b/scripts/wheelbuilder/linux/numpy.sh index 57e2076985..8cd8c11077 100755 --- a/scripts/wheelbuilder/linux/numpy.sh +++ b/scripts/wheelbuilder/linux/numpy.sh @@ -1,4 +1,4 @@ -# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # The Universal Permissive License (UPL), Version 1.0 diff --git a/scripts/wheelbuilder/linux/pillow.sh b/scripts/wheelbuilder/linux/pillow.sh index 668580ab54..4b535f89df 100755 --- a/scripts/wheelbuilder/linux/pillow.sh +++ b/scripts/wheelbuilder/linux/pillow.sh @@ -1,4 +1,4 @@ -# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # The Universal Permissive License (UPL), Version 1.0 diff --git a/scripts/wheelbuilder/linux/scipy.sh b/scripts/wheelbuilder/linux/scipy.sh index 53e2005eda..19cffbffc4 100755 --- a/scripts/wheelbuilder/linux/scipy.sh +++ b/scripts/wheelbuilder/linux/scipy.sh @@ -1,4 +1,4 @@ -# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # The Universal Permissive License (UPL), Version 1.0 diff --git a/scripts/wheelbuilder/linux/tensorflow.sh b/scripts/wheelbuilder/linux/tensorflow.sh index a7cfb98b4f..03259d33cd 100755 --- a/scripts/wheelbuilder/linux/tensorflow.sh +++ b/scripts/wheelbuilder/linux/tensorflow.sh @@ -1,4 +1,4 @@ -# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # The Universal Permissive License (UPL), Version 1.0 diff --git a/scripts/wheelbuilder/linux/torch.sh b/scripts/wheelbuilder/linux/torch.sh index 7ae3d60110..b2c54a9644 100755 --- a/scripts/wheelbuilder/linux/torch.sh +++ b/scripts/wheelbuilder/linux/torch.sh @@ -1,4 +1,4 @@ -# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # The Universal Permissive License (UPL), Version 1.0 From 50035569f8703201268aa9e782f0c0e1c008f9a7 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Fri, 28 Mar 2025 14:40:18 +0100 Subject: [PATCH 63/82] Add wheelbuilder script for numpy-1.23.5 --- scripts/wheelbuilder/linux/numpy.1.23.5.sh | 43 ++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 scripts/wheelbuilder/linux/numpy.1.23.5.sh diff --git a/scripts/wheelbuilder/linux/numpy.1.23.5.sh b/scripts/wheelbuilder/linux/numpy.1.23.5.sh new file mode 100644 index 0000000000..2293b2c906 --- /dev/null +++ b/scripts/wheelbuilder/linux/numpy.1.23.5.sh @@ -0,0 +1,43 @@ +# Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +if [ -n "$GITHUB_RUN_ID" ]; then + dnf install -y gcc-toolset-9 gcc-toolset-9-gcc-gfortran openblas-devel + scl enable gcc-toolset-9 'pip wheel "numpy==1.23.5"' +fi From 2f3869eac77e7ddd94393c38c9479968ee501991 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Fri, 28 Mar 2025 15:18:09 +0100 Subject: [PATCH 64/82] Keep order of packages to build in wheelbuilder so we can optimize build times --- scripts/wheelbuilder/build_wheels.py | 37 +++++++++++++++++++--------- 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/scripts/wheelbuilder/build_wheels.py b/scripts/wheelbuilder/build_wheels.py index 78756b84a8..e61c23669c 100644 --- a/scripts/wheelbuilder/build_wheels.py +++ b/scripts/wheelbuilder/build_wheels.py @@ -113,14 +113,28 @@ def create_venv(): def build_wheels(pip): - packages_selected = [s for s in os.environ.get("PACKAGES_TO_BUILD", "").split(",") if s] - packages_to_build = set() with open(join(dirname(__file__), "packages.txt")) as f: - for line in f.readlines(): - line = line.strip() - name, version = line.split("==") - if not packages_selected or name in packages_selected or line in packages_selected: - packages_to_build.add(line) + packages_from_txt = [tuple(l.strip().split("==")) for l in f] + packages_to_build = [] + for s in os.environ.get("PACKAGES_TO_BUILD", "").split(","): + s = s.strip() + if not s: + continue + elif "==" in s: + name, version = s.split("==") + else: + name, version = s, None + for n, v in packages_from_txt: + if n == name: + version = v + break + if not version: + print("ERROR: Asked to build", s, "but no version given") + return False + packages_to_build.append((name, version)) + if not packages_to_build: + packages_to_build = packages_from_txt + print("About to build", packages_to_build) scriptdir = abspath(join(dirname(__file__), sys.platform)) if sys.platform == "win32": script_ext = "bat" @@ -133,8 +147,7 @@ def build_wheels(pip): remaining_packages = 0 while remaining_packages != len(packages_to_build): remaining_packages = len(packages_to_build) - for spec in packages_to_build.copy(): - name, version = spec.split("==") + for name, version in packages_to_build.copy(): whl_count = len(glob("*.whl")) script = f"{name}.{version}.{script_ext}".lower() if script not in available_scripts: @@ -153,13 +166,13 @@ def build_wheels(pip): if p.returncode != 0: continue if len(glob("*.whl")) > whl_count: - packages_to_build.remove(spec) + packages_to_build.remove((name, version)) continue print(script, "did not build a wheel, we will do so now", flush=True) print("Building", name, version, flush=True) - p = subprocess.run([pip, "wheel", spec]) + p = subprocess.run([pip, "wheel", f"{name}=={version}"]) if p.returncode == 0: - packages_to_build.remove(spec) + packages_to_build.remove((name, version)) if packages_to_build: print("Failed to build all packages, the following packages failed") print(packages_to_build) From af650325acbd0cf858a2984e26840f4fef17dae4 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Wed, 2 Apr 2025 16:52:53 +0200 Subject: [PATCH 65/82] New set of packages for AI workloads --- scripts/wheelbuilder/packages.txt | 163 ++++++++++-------------------- 1 file changed, 55 insertions(+), 108 deletions(-) diff --git a/scripts/wheelbuilder/packages.txt b/scripts/wheelbuilder/packages.txt index 352261afe0..d9504374aa 100644 --- a/scripts/wheelbuilder/packages.txt +++ b/scripts/wheelbuilder/packages.txt @@ -1,133 +1,80 @@ -aiohappyeyeballs==2.4.6 -aiohttp==3.11.13 -aiosignal==1.3.2 -alembic==1.14.1 +Deprecated==1.2.18 +FLAML==2.1.2 +aiofiles==24.1.0 annotated-types==0.7.0 +anyio==4.4.0 anyio==4.8.0 -asyncer==0.0.8 -attrs==25.1.0 -backoff==2.2.1 -bcrypt==4.2.1 +autogen-agentchat==0.4.6 +autogen-agentchat==0.4.9 +autogen-agentchat==0.4.9.3 +autogen-core==0.4.6 +autogen-core==0.4.9 +autogen-core==0.4.9.3 +autogen-ext==0.4.6 +autogen-ext==0.4.9.3 +autogen-ext[openai]==0.4.9 +blis==0.7.11 +certifi==2024.7.4 certifi==2025.1.31 cffi==1.17.1 +charset-normalizer==3.1.0 charset-normalizer==3.4.1 -click==8.1.8 -colorama==0.4.6 -colorlog==6.9.0 -cryptography==44.0.1 -datasets==3.3.2 +circuitbreaker==2.1.3 +cryptography==41.0.7 deprecated==1.2.18 -dill==0.3.8 diskcache==5.6.3 distro==1.9.0 -dspy-ai==2.5.41 -dspy==2.5.41 -fastapi==0.115.6 -filelock==3.17.0 -frozenlist==1.5.0 -fsspec==2024.12.0 -fsspec[http]==2024.12.0 -googleapis-common-protos==1.56.1 -greenlet==3.1.1 -grpcio==1.70.0 +docker==7.1.0 +exceptiongroup==1.2.1 h11==0.14.0 +httpcore==1.0.5 httpcore==1.0.7 -httptools==0.6.1 +httpx==0.27.0 httpx==0.28.1 -huggingface-hub==0.29.1 idna==3.10 -importlib-metadata==8.5.0 -javalang==0.13.0 -jinja2==3.1.5 -jiter==0.9.0 -joblib==1.4.2 -json-repair==0.39.1 -json5==0.10.0 -jsonpatch==1.33 -jsonpointer==3.0.0 -jsonschema-specifications==2024.10.1 -jsonschema==4.23.0 -kiwisolver==1.4.5 -langchain-core==0.3.41 -langgraph-checkpoint==2.0.17 -langgraph-sdk==0.1.55 -langgraph==0.2.76 -langsmith==0.3.12 -levenshtein==0.26.1 -litellm==1.51.0 -magicattr==0.1.6 -mako==1.3.9 -markupsafe==3.0.2 -msgpack==1.1.0 -multidict==6.1.0 -multiprocess==0.70.16 -nest-asyncio==1.6.0 +idna==3.7 +jiter==0.8.2 +jsonref==1.1.0 +marisa_trie==1.2.1 +numpy==1.23.2 +numpy==1.23.5 +numpy==1.24.3 numpy==1.26.4 -numpy==2.2.3 -openai==1.64.0 -openinference-instrumentation-dspy==0.1.9 -openinference-instrumentation-litellm==0.1.6 -openinference-instrumentation==0.1.23 -openinference-semantic-conventions==0.1.14 +oci==2.149.2 +openai==1.35.10 +openai==1.62.0 opentelemetry-api==1.30.0 -opentelemetry-exporter-jaeger-proto-grpc==1.21.0 -opentelemetry-exporter-jaeger-thrift==1.21.0 -opentelemetry-exporter-jaeger==1.21.0 -opentelemetry-exporter-otlp-proto-common==1.30.0 -opentelemetry-exporter-otlp-proto-grpc==1.30.0 -opentelemetry-exporter-otlp-proto-http==1.30.0 -opentelemetry-exporter-otlp==1.30.0 -opentelemetry-exporter-prometheus==0.51b0 -opentelemetry-instrumentation==0.51b0 -opentelemetry-propagator-b3==1.30.0 -opentelemetry-proto==1.30.0 -opentelemetry-sdk==1.30.0 -opentelemetry-semantic-conventions==0.51b0 -optuna==4.2.1 -oracledb==2.5.1 -orjson==3.10.5 -packaging==24.2 -pandas==2.2.3 -paramiko==3.5.1 -prometheus-client==0.21.1 -propcache==0.3.0 +opentelemetry-api==1.31.1 +packaging==24.1 +pandas==1.5.2 +pillow==11.1.0 +preshed==3.0.9 protobuf==5.29.3 -psutil==5.9.8 -pyarrow==19.0.1 +protobuf==5.29.4 +pyOpenSSL==23.2.0 +pyautogen==0.2.32 pycparser==2.22 -pydantic-core==2.27.1 -pydantic==2.10.1 -pynacl==1.5.0 -python-dateutil==2.9.0.post0 +pydantic-core==2.10.1 +pydantic-core==2.27.2 +pydantic-core==2.29.0 +pydantic==2.10.6 +pydantic==2.11.0a2 +pydantic==2.4.2 python-dotenv==1.0.1 -pytz==2025.1 -pyyaml==6.0.2 -rapidfuzz==3.12.1 -referencing==0.36.2 +pytz==2025.2 regex==2024.11.6 -requests-toolbelt==1.0.0 +regex==2024.5.15 requests==2.32.3 -rpds-py==0.23.1 -setuptools==76.0.0 -six==1.17.0 sniffio==1.3.1 -sqlalchemy==2.0.38 -sshtunnel==0.4.0 -starlette==0.41.3 -tenacity==9.0.0 -thrift==0.21.0 -tiktoken==0.9.0 -tokenizers==0.21.0 +spacy==3.6.1 +termcolor==2.4.0 +thinc==8.1.12 +tiktoken==0.7.0 +tiktoken==0.8.0 +tqdm==4.66.4 tqdm==4.67.1 typing-extensions==4.12.2 -tzdata==2025.1 -ujson==5.10.0 -ujson==5.10.0 +urllib3==2.2.2 urllib3==2.3.0 -uvicorn==0.34.0 wrapt==1.17.2 -xxhash==3.4.1 -xxhash==3.5.0 -yarl==1.18.3 zipp==3.21.0 -zstandard==0.23.0 From c8a81537442206e4371317da9d712238365602cd Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Thu, 3 Apr 2025 09:16:15 +0200 Subject: [PATCH 66/82] Try to put local rust on the PATH before building --- scripts/wheelbuilder/build_wheels.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/scripts/wheelbuilder/build_wheels.py b/scripts/wheelbuilder/build_wheels.py index e61c23669c..0f0f93134d 100644 --- a/scripts/wheelbuilder/build_wheels.py +++ b/scripts/wheelbuilder/build_wheels.py @@ -61,7 +61,7 @@ from argparse import ArgumentParser from glob import glob -from os.path import abspath, basename, dirname, exists, isabs, join, splitext +from os.path import abspath, basename, dirname, exists, expanduser, isabs, isdir, join, splitext from tempfile import TemporaryDirectory from urllib.request import urlretrieve @@ -112,6 +112,17 @@ def create_venv(): return pip +def prepare_environment(pip_exe): + env = os.environ.copy() + env["PATH"] = abspath(dirname(pip_exe)) + os.pathsep + env["PATH"] + env["VIRTUAL_ENV"] = abspath(dirname(dirname(pip_exe))) + if not shutil.which("cargo"): + cargo_bin = join(expanduser("~"), ".cargo", "bin") + if isdir(cargo_bin): + env["PATH"] += os.pathsep + cargo_bin + return env + + def build_wheels(pip): with open(join(dirname(__file__), "packages.txt")) as f: packages_from_txt = [tuple(l.strip().split("==")) for l in f] @@ -154,9 +165,7 @@ def build_wheels(pip): script = f"{name}.{script_ext}".lower() if script in available_scripts: script = join(scriptdir, available_scripts[script]) - env = os.environ.copy() - env["PATH"] = abspath(dirname(pip)) + os.pathsep + env["PATH"] - env["VIRTUAL_ENV"] = abspath(dirname(dirname(pip))) + env = prepare_environment(pip) print("Building", name, version, "with", script, flush=True) if sys.platform == "win32": cmd = [script, version] # Python's subprocess.py does the quoting we need From 6a1770819984bdeafe86b7adcff75284338601a6 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Thu, 3 Apr 2025 20:24:38 +0200 Subject: [PATCH 67/82] Wheelbuilder script for pymupdf --- scripts/wheelbuilder/darwin/pymupdf.sh | 89 ++++++++++++++++++++++++++ scripts/wheelbuilder/linux/pymupdf.sh | 68 ++++++++++++++++++++ 2 files changed, 157 insertions(+) create mode 100755 scripts/wheelbuilder/darwin/pymupdf.sh create mode 100755 scripts/wheelbuilder/linux/pymupdf.sh diff --git a/scripts/wheelbuilder/darwin/pymupdf.sh b/scripts/wheelbuilder/darwin/pymupdf.sh new file mode 100755 index 0000000000..f5d515da16 --- /dev/null +++ b/scripts/wheelbuilder/darwin/pymupdf.sh @@ -0,0 +1,89 @@ +# Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +if [ -n "$GITHUB_RUN_ID" ]; then + brew install python@3.11 + # Make sure homebrew Python 3.11 is first on PATH, not the GraalPy symlink + # Also, BSD patch fails to apply our no-context pymupdf patch, so use + # python-patch-ng + $HOMEBREW_PREFIX/bin/python3.11 -m venv 311venv + 311venv/bin/pip install git+https://github.com/timfel/python-patch-ng + 311venv/bin/pip uninstall -y pip + export PATH="$(pwd)/311venv/bin:$PATH" +fi + +mkdir cc_bin +export PATH="$(pwd)/cc_bin:$PATH" +# darwin's linker does not support --gc-sections but mupdf passes that we wrap +# cc and pass all arguments on, but we remove the argument -Wl,--gc-sections +original_cc=`which cc` +cat < cc_bin/cc +#!/bin/bash +# Wrapper for cc that removes --gc-sections from command line arguments if present +# Pass on all arguments, but remove -Wl,--gc-sections if it is given + +if [[ "\$@" == *"-Wl,--gc-sections"* ]]; then + echo "Removing -Wl,--gc-sections argument from command line..." + newargs=() + for arg in "\$@"; do + if [ \$arg != "-Wl,--gc-sections" ]; then + newargs+=("\$arg") + fi + done + exec $original_cc "\${newargs[@]}" +else + exec $original_cc "\$@" +fi +EOF +chmod +x cc_bin/cc +export CC="$(pwd)/cc_bin/cc" + +export USE_SONAME="no" + +if [ -n "$1" ]; then + pip wheel "pymupdf==$1" +else + pip wheel pymupdf +fi + +rm -rf cc_bin + +if [ -n "$GITHUB_RUN_ID" ]; then + rm -rf 311venv +fi diff --git a/scripts/wheelbuilder/linux/pymupdf.sh b/scripts/wheelbuilder/linux/pymupdf.sh new file mode 100755 index 0000000000..248a6f929a --- /dev/null +++ b/scripts/wheelbuilder/linux/pymupdf.sh @@ -0,0 +1,68 @@ +# Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +if [ -n "$GITHUB_RUN_ID" ]; then + dnf install -y /usr/bin/c++ /usr/bin/make /usr/bin/which + + # Make sure system Python is first on PATH, not the GraalPy symlink :( + # Installing a new Python3.11 might break the outer pip, so we just symlink + # whatever the outer python is :(( + current_path="$PATH" + builder_pip=`which pip` + builder_pip_dirname=`dirname $pip` + path_without_builder_venv=`echo $PATH | sed "s#$pip_dirname##"` + export PATH="$path_without_builder_venv" + outer_python3=`which python3` + outer_python3_dir=`dirname $outer_python3_dir` + rm -f "${pip_dirname}/python3.11" + if which python3.11; then + echo "Python3.11 is available" + else + echo "Symlinking $outer_python3 to be Python3.11" + ln -sf "$outer_python3" "${outer_python3_dir}/python3.11" + fi +fi + +export USE_SONAME="no" + +if [ -n "$1" ]; then + pip wheel "pymupdf==$1" +else + pip wheel pymupdf +fi From e1e35b8e318198949bb5b0b50efedd759ca1be46 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Fri, 4 Apr 2025 22:35:12 +0200 Subject: [PATCH 68/82] Update pymupdf patch --- scripts/wheelbuilder/darwin/pymupdf.sh | 20 ++++++-------------- scripts/wheelbuilder/linux/pymupdf.sh | 21 ++------------------- 2 files changed, 8 insertions(+), 33 deletions(-) diff --git a/scripts/wheelbuilder/darwin/pymupdf.sh b/scripts/wheelbuilder/darwin/pymupdf.sh index f5d515da16..c26844ef46 100755 --- a/scripts/wheelbuilder/darwin/pymupdf.sh +++ b/scripts/wheelbuilder/darwin/pymupdf.sh @@ -38,20 +38,16 @@ # SOFTWARE. if [ -n "$GITHUB_RUN_ID" ]; then - brew install python@3.11 - # Make sure homebrew Python 3.11 is first on PATH, not the GraalPy symlink - # Also, BSD patch fails to apply our no-context pymupdf patch, so use - # python-patch-ng - $HOMEBREW_PREFIX/bin/python3.11 -m venv 311venv - 311venv/bin/pip install git+https://github.com/timfel/python-patch-ng - 311venv/bin/pip uninstall -y pip - export PATH="$(pwd)/311venv/bin:$PATH" + export CPYTHON_EXE="$HOMEBREW_PREFIX/bin/python" fi -mkdir cc_bin -export PATH="$(pwd)/cc_bin:$PATH" +# BSD patch fails to apply our pymupdf.patch +pip install git+https://github.com/timfel/python-patch-ng + # darwin's linker does not support --gc-sections but mupdf passes that we wrap # cc and pass all arguments on, but we remove the argument -Wl,--gc-sections +mkdir cc_bin +export PATH="$(pwd)/cc_bin:$PATH" original_cc=`which cc` cat < cc_bin/cc #!/bin/bash @@ -83,7 +79,3 @@ else fi rm -rf cc_bin - -if [ -n "$GITHUB_RUN_ID" ]; then - rm -rf 311venv -fi diff --git a/scripts/wheelbuilder/linux/pymupdf.sh b/scripts/wheelbuilder/linux/pymupdf.sh index 248a6f929a..d949e7d332 100755 --- a/scripts/wheelbuilder/linux/pymupdf.sh +++ b/scripts/wheelbuilder/linux/pymupdf.sh @@ -38,25 +38,8 @@ # SOFTWARE. if [ -n "$GITHUB_RUN_ID" ]; then - dnf install -y /usr/bin/c++ /usr/bin/make /usr/bin/which - - # Make sure system Python is first on PATH, not the GraalPy symlink :( - # Installing a new Python3.11 might break the outer pip, so we just symlink - # whatever the outer python is :(( - current_path="$PATH" - builder_pip=`which pip` - builder_pip_dirname=`dirname $pip` - path_without_builder_venv=`echo $PATH | sed "s#$pip_dirname##"` - export PATH="$path_without_builder_venv" - outer_python3=`which python3` - outer_python3_dir=`dirname $outer_python3_dir` - rm -f "${pip_dirname}/python3.11" - if which python3.11; then - echo "Python3.11 is available" - else - echo "Symlinking $outer_python3 to be Python3.11" - ln -sf "$outer_python3" "${outer_python3_dir}/python3.11" - fi + dnf install -y /usr/bin/c++ /usr/bin/make /usr/bin/which /usr/bin/python3 + export CPYTHON_EXE=/usr/bin/python3 fi export USE_SONAME="no" From 2b5de3b1d6808a9682a4e56dfeeb0f21117633be Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Fri, 11 Apr 2025 16:09:02 +0200 Subject: [PATCH 69/82] Use paatch to apply patches the same way across all platforms --- scripts/wheelbuilder/build_wheels.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/scripts/wheelbuilder/build_wheels.py b/scripts/wheelbuilder/build_wheels.py index 0f0f93134d..4797fde2d1 100644 --- a/scripts/wheelbuilder/build_wheels.py +++ b/scripts/wheelbuilder/build_wheels.py @@ -104,11 +104,10 @@ def create_venv(): subprocess.check_call([binary, "-m", "venv", "graalpy"]) print("Installing wheel with", pip, flush=True) subprocess.check_call([pip, "install", "wheel"]) - if sys.platform == "win32": - print("Installing python-patch-ng to provide patch.exe", flush=True) - p = subprocess.run([pip, "install", "git+https://github.com/timfel/python-patch-ng.git"]) - if p.returncode != 0: - print("Installing python-patch-ng failed, assuming patch.exe is on PATH", flush=True) + print("Installing paatch to provide patch.exe", flush=True) + p = subprocess.run([pip, "install", "paatch"]) + if p.returncode != 0: + print("Installing paatch failed, assuming a GNU patch compatible binary is on PATH", flush=True) return pip From edaf95ca1521e7211d8234f322b84dd78d7b16d6 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Thu, 22 May 2025 14:48:40 +0200 Subject: [PATCH 70/82] [GR-65181] Add shapely build script. --- scripts/wheelbuilder/darwin/shapely.sh | 42 ++++++++++++++++++++++++++ scripts/wheelbuilder/linux/shapely.sh | 42 ++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100755 scripts/wheelbuilder/darwin/shapely.sh create mode 100644 scripts/wheelbuilder/linux/shapely.sh diff --git a/scripts/wheelbuilder/darwin/shapely.sh b/scripts/wheelbuilder/darwin/shapely.sh new file mode 100755 index 0000000000..4064af4fa0 --- /dev/null +++ b/scripts/wheelbuilder/darwin/shapely.sh @@ -0,0 +1,42 @@ +# Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +if [ -n "$GITHUB_RUN_ID" ]; then + brew install geos +fi diff --git a/scripts/wheelbuilder/linux/shapely.sh b/scripts/wheelbuilder/linux/shapely.sh new file mode 100644 index 0000000000..8fd92732b3 --- /dev/null +++ b/scripts/wheelbuilder/linux/shapely.sh @@ -0,0 +1,42 @@ +# Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +if [ -n "$GITHUB_RUN_ID" ]; then + dnf install -y libgeos-devel +fi From 60c8dc9e479b1408f33d57db435ed0137a412168 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Mon, 26 May 2025 11:36:27 +0200 Subject: [PATCH 71/82] Update unittest tags and imports --- scripts/wheelbuilder/linux/shapely.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wheelbuilder/linux/shapely.sh b/scripts/wheelbuilder/linux/shapely.sh index 8fd92732b3..7e9c3d4a9d 100644 --- a/scripts/wheelbuilder/linux/shapely.sh +++ b/scripts/wheelbuilder/linux/shapely.sh @@ -38,5 +38,5 @@ # SOFTWARE. if [ -n "$GITHUB_RUN_ID" ]; then - dnf install -y libgeos-devel + dnf install -y geos-devel fi From d3fb29617a2b7f0849d06db92b61fffce0f82483 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Thu, 12 Jun 2025 15:41:11 +0200 Subject: [PATCH 72/82] Update wheelbuilder shell script to build shapely with current GEOS version --- scripts/wheelbuilder/linux/shapely.sh | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/scripts/wheelbuilder/linux/shapely.sh b/scripts/wheelbuilder/linux/shapely.sh index 7e9c3d4a9d..f73c287d5f 100644 --- a/scripts/wheelbuilder/linux/shapely.sh +++ b/scripts/wheelbuilder/linux/shapely.sh @@ -37,6 +37,26 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +GEOS_VERSION=3.13.1 +GEOS_INSTALL="/usr" + if [ -n "$GITHUB_RUN_ID" ]; then - dnf install -y geos-devel + dnf install -y cmake + curl -OL --retry 5 "https://download.osgeo.org/geos/geos-${GEOS_VERSION}.tar.bz2" + tar xfj "geos-${GEOS_VERSION}.tar.bz2" && rm "geos-${GEOS_VERSION}.tar.bz2" + cmake -DCMAKE_INSTALL_LIBDIR=lib \ + "-DCMAKE_INSTALL_NAME_DIR=${GEOS_INSTALL}/lib" \ + "-DCMAKE_INSTALL_PREFIX=${GEOS_INSTALL}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_TESTING=OFF \ + -S "geos-${GEOS_VERSION}" -B build + cmake --build build -j 4 + cmake --install build + rm -rf build "geos-${GEOS_VERSION}" +fi + +if [ -n "$1" ]; then + pip wheel "shapely==$1" +else + pip wheel shapely fi From c2be04f0376f1880ba12c807a542a7bc8662eecf Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Fri, 13 Jun 2025 11:00:46 +0200 Subject: [PATCH 73/82] Update wheelbuilder scripts for grpcio and pillow on darwin --- scripts/wheelbuilder/darwin/grpcio.sh | 46 +++++++++++++++++++++++++++ scripts/wheelbuilder/darwin/pillow.sh | 2 +- 2 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 scripts/wheelbuilder/darwin/grpcio.sh diff --git a/scripts/wheelbuilder/darwin/grpcio.sh b/scripts/wheelbuilder/darwin/grpcio.sh new file mode 100644 index 0000000000..1498a5a43d --- /dev/null +++ b/scripts/wheelbuilder/darwin/grpcio.sh @@ -0,0 +1,46 @@ +# Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +export MACOSX_DEPLOYMENT_TARGET=11.0 + +if [ -n "$1" ]; then + pip wheel "grpcio==$1" +else + pip wheel grpcio +fi diff --git a/scripts/wheelbuilder/darwin/pillow.sh b/scripts/wheelbuilder/darwin/pillow.sh index 7058e6880f..3672b34084 100755 --- a/scripts/wheelbuilder/darwin/pillow.sh +++ b/scripts/wheelbuilder/darwin/pillow.sh @@ -38,5 +38,5 @@ # SOFTWARE. if [ -n "$GITHUB_RUN_ID" ]; then - brew install libjpeg libtiff little-cms2 openjpeg2 webp + brew install libjpeg libtiff little-cms2 openjpeg webp fi From b3f18ae496414df95898c0865e5c0b4dc905283f Mon Sep 17 00:00:00 2001 From: Ivo Horak Date: Fri, 13 Jun 2025 10:58:02 +0200 Subject: [PATCH 74/82] Fixing transient test --- scripts/wheelbuilder/darwin/pillow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wheelbuilder/darwin/pillow.sh b/scripts/wheelbuilder/darwin/pillow.sh index 3672b34084..b41fac495a 100755 --- a/scripts/wheelbuilder/darwin/pillow.sh +++ b/scripts/wheelbuilder/darwin/pillow.sh @@ -1,4 +1,4 @@ -# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # The Universal Permissive License (UPL), Version 1.0 From e86f61416eee119ccebf1128ff8e1c79f36350ac Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Mon, 16 Jun 2025 15:14:10 +0200 Subject: [PATCH 75/82] Add wheelbuilder for numpy 2 --- scripts/wheelbuilder/linux/numpy.2.2.4.sh | 43 +++++++++++++++++++++++ scripts/wheelbuilder/packages.txt | 4 +-- 2 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 scripts/wheelbuilder/linux/numpy.2.2.4.sh diff --git a/scripts/wheelbuilder/linux/numpy.2.2.4.sh b/scripts/wheelbuilder/linux/numpy.2.2.4.sh new file mode 100644 index 0000000000..6389f99413 --- /dev/null +++ b/scripts/wheelbuilder/linux/numpy.2.2.4.sh @@ -0,0 +1,43 @@ +# Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +if [ -n "$GITHUB_RUN_ID" ]; then + dnf install -y gcc-toolset-9 gcc-toolset-9-gcc-gfortran openblas-devel + scl enable gcc-toolset-9 'pip wheel "numpy==2.2.4"' +fi diff --git a/scripts/wheelbuilder/packages.txt b/scripts/wheelbuilder/packages.txt index d9504374aa..354316a27f 100644 --- a/scripts/wheelbuilder/packages.txt +++ b/scripts/wheelbuilder/packages.txt @@ -36,10 +36,8 @@ idna==3.7 jiter==0.8.2 jsonref==1.1.0 marisa_trie==1.2.1 -numpy==1.23.2 -numpy==1.23.5 -numpy==1.24.3 numpy==1.26.4 +numpy==2.2.4 oci==2.149.2 openai==1.35.10 openai==1.62.0 From 64cf7403b214cd31b601d12ac9143a1dda205cf6 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Wed, 18 Jun 2025 13:06:03 +0200 Subject: [PATCH 76/82] Copy and upload all wheels --- scripts/wheelbuilder/build_wheels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wheelbuilder/build_wheels.py b/scripts/wheelbuilder/build_wheels.py index 4797fde2d1..6ae666ece3 100644 --- a/scripts/wheelbuilder/build_wheels.py +++ b/scripts/wheelbuilder/build_wheels.py @@ -190,7 +190,7 @@ def build_wheels(pip): def repair_wheels(): - whls = glob("*graalpy*.whl") + whls = glob("*.whl") env = os.environ.copy() env["PYTHONUTF8"] = "1" env["PATH"] = abspath(dirname(sys.executable)) + os.pathsep + env["PATH"] From 93b45cabcc40e2253b0ad21a07ca649dae071e38 Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Fri, 15 Aug 2025 12:01:54 +0200 Subject: [PATCH 77/82] Extract repair_wheels to be testable --- scripts/wheelbuilder/build_wheels.py | 72 ++------------- scripts/wheelbuilder/repair_wheels.py | 125 ++++++++++++++++++++++++++ 2 files changed, 130 insertions(+), 67 deletions(-) create mode 100644 scripts/wheelbuilder/repair_wheels.py diff --git a/scripts/wheelbuilder/build_wheels.py b/scripts/wheelbuilder/build_wheels.py index 6ae666ece3..41baf63754 100644 --- a/scripts/wheelbuilder/build_wheels.py +++ b/scripts/wheelbuilder/build_wheels.py @@ -47,24 +47,22 @@ 4. Go over the build scripts and run them in the venv """ -import hashlib import importlib import os -import platform -import re import shlex import shutil import subprocess import sys import tarfile import zipfile - from argparse import ArgumentParser from glob import glob -from os.path import abspath, basename, dirname, exists, expanduser, isabs, isdir, join, splitext -from tempfile import TemporaryDirectory +from os.path import abspath, dirname, exists, expanduser, isdir, join, splitext from urllib.request import urlretrieve +sys.path.append(os.path.dirname(__file__)) +from repair_wheels import repair_wheels + def ensure_installed(name, *extra): try: @@ -189,66 +187,6 @@ def build_wheels(pip): return True -def repair_wheels(): - whls = glob("*.whl") - env = os.environ.copy() - env["PYTHONUTF8"] = "1" - env["PATH"] = abspath(dirname(sys.executable)) + os.pathsep + env["PATH"] - for whl in whls: - if sys.platform == "win32": - ensure_installed("delvewheel") - p = subprocess.run( - [ - sys.executable, - "-m", - "delvewheel", - "repair", - "-v", - "--exclude", - "python-native.dll", - "-w", - "wheelhouse", - whl, - ], - env=env, - ) - elif sys.platform == "linux": - ensure_installed("auditwheel", "patchelf") - p = subprocess.run( - [ - join(dirname(sys.executable), "auditwheel"), - "repair", - "--plat", - "manylinux_2_28_x86_64" if platform.processor() == "x86_64" else "manylinux_2_28_aarch64", - "-w", - "wheelhouse", - whl, - ], - env=env, - ) - elif sys.platform == "darwin": - ensure_installed("delocate") - p = subprocess.run( - [ - join(dirname(sys.executable), "delocate-wheel"), - "-v", - "--ignore-missing-dependencies", - "--require-archs", - "arm64" if platform.processor() == "arm" else "x86_64", - "-w", - "wheelhouse", - whl, - ], - env=env, - ) - if p.returncode != 0: - print("Repairing", whl, "failed, copying as is.") - try: - shutil.copy(whl, "wheelhouse") - except: - pass - - if __name__ == "__main__": parser = ArgumentParser() parser.add_argument("graalpy_url") @@ -261,6 +199,6 @@ def repair_wheels(): extract(outpath) pip = create_venv() success = build_wheels(pip) - repair_wheels() + repair_wheels("wheelhouse") if not success and not args.ignore_failures: sys.exit(1) diff --git a/scripts/wheelbuilder/repair_wheels.py b/scripts/wheelbuilder/repair_wheels.py new file mode 100644 index 0000000000..157283b761 --- /dev/null +++ b/scripts/wheelbuilder/repair_wheels.py @@ -0,0 +1,125 @@ +# Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import argparse +import os +import platform +import shutil +import subprocess +import sys +from glob import glob +from os.path import abspath, dirname, join + + +def ensure_installed(name, *extra): + import importlib + try: + return importlib.import_module(name) + except ImportError: + subprocess.check_call([sys.executable, "-m", "pip", "install", name, *extra]) + return importlib.import_module(name) + +def repair_wheels(wheelhouse): + whls = glob("*.whl") + env = os.environ.copy() + env["PYTHONUTF8"] = "1" + env["PATH"] = abspath(dirname(sys.executable)) + os.pathsep + env["PATH"] + os.makedirs(wheelhouse, exist_ok=True) + for whl in whls: + if sys.platform == "win32": + ensure_installed("delvewheel") + p = subprocess.run( + [ + sys.executable, + "-m", + "delvewheel", + "repair", + "-v", + "--exclude", + "python-native.dll", + "-w", + wheelhouse, + whl, + ], + env=env, + ) + elif sys.platform == "linux": + ensure_installed("auditwheel", "patchelf") + p = subprocess.run( + [ + join(dirname(sys.executable), "auditwheel"), + "repair", + "--plat", + "manylinux_2_28_x86_64" if platform.processor() == "x86_64" else "manylinux_2_28_aarch64", + "-w", + wheelhouse, + whl, + ], + env=env, + ) + elif sys.platform == "darwin": + ensure_installed("delocate") + p = subprocess.run( + [ + join(dirname(sys.executable), "delocate-wheel"), + "-v", + "--ignore-missing-dependencies", + "--require-archs", + "arm64" if platform.processor() == "arm" else "x86_64", + "-w", + wheelhouse, + whl, + ], + env=env, + ) + else: + print(f"Unsupported platform: {sys.platform}", file=sys.stderr) + continue + if p.returncode != 0: + print("Repairing", whl, "failed, copying as is.") + try: + shutil.copy(whl, wheelhouse) + except Exception as e: + print("Copy failed:", e, file=sys.stderr) + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Repair wheels using platform-specific tools.") + parser.add_argument("--wheelhouse", required=True, help="Output directory for repaired wheels") + args = parser.parse_args() + repair_wheels(args.wheelhouse) From cb00059a7722e0527f250fd6593ed46ee540576c Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Fri, 15 Aug 2025 12:11:22 +0200 Subject: [PATCH 78/82] Avoid using platform.processor() in repair_wheels --- scripts/wheelbuilder/repair_wheels.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/scripts/wheelbuilder/repair_wheels.py b/scripts/wheelbuilder/repair_wheels.py index 157283b761..be81f21676 100644 --- a/scripts/wheelbuilder/repair_wheels.py +++ b/scripts/wheelbuilder/repair_wheels.py @@ -81,12 +81,19 @@ def repair_wheels(wheelhouse): ) elif sys.platform == "linux": ensure_installed("auditwheel", "patchelf") + arch = platform.machine().lower() + if arch == "x86_64": + plat_arg = "manylinux_2_28_x86_64" + elif arch == "aarch64": + plat_arg = "manylinux_2_28_aarch64" + else: + raise RuntimeError(f"Unsupported architecture on Linux: {arch}") p = subprocess.run( [ join(dirname(sys.executable), "auditwheel"), "repair", "--plat", - "manylinux_2_28_x86_64" if platform.processor() == "x86_64" else "manylinux_2_28_aarch64", + plat_arg, "-w", wheelhouse, whl, @@ -95,13 +102,20 @@ def repair_wheels(wheelhouse): ) elif sys.platform == "darwin": ensure_installed("delocate") + arch = platform.machine().lower() + if arch == "arm64": + require_arch = "arm64" + elif arch == "x86_64": + require_arch = "x86_64" + else: + raise RuntimeError(f"Unsupported architecture on macOS: {arch}") p = subprocess.run( [ join(dirname(sys.executable), "delocate-wheel"), "-v", "--ignore-missing-dependencies", "--require-archs", - "arm64" if platform.processor() == "arm" else "x86_64", + require_arch, "-w", wheelhouse, whl, From b7eeefeb4c6d4b0307acf4621f4232647ceee36a Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Thu, 21 Aug 2025 15:50:44 +0200 Subject: [PATCH 79/82] Stop hardcoding manylinux glibc version in wheelbuilder --- scripts/wheelbuilder/repair_wheels.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/scripts/wheelbuilder/repair_wheels.py b/scripts/wheelbuilder/repair_wheels.py index be81f21676..c92c58aad1 100644 --- a/scripts/wheelbuilder/repair_wheels.py +++ b/scripts/wheelbuilder/repair_wheels.py @@ -55,6 +55,7 @@ def ensure_installed(name, *extra): subprocess.check_call([sys.executable, "-m", "pip", "install", name, *extra]) return importlib.import_module(name) + def repair_wheels(wheelhouse): whls = glob("*.whl") env = os.environ.copy() @@ -81,19 +82,10 @@ def repair_wheels(wheelhouse): ) elif sys.platform == "linux": ensure_installed("auditwheel", "patchelf") - arch = platform.machine().lower() - if arch == "x86_64": - plat_arg = "manylinux_2_28_x86_64" - elif arch == "aarch64": - plat_arg = "manylinux_2_28_aarch64" - else: - raise RuntimeError(f"Unsupported architecture on Linux: {arch}") p = subprocess.run( [ join(dirname(sys.executable), "auditwheel"), "repair", - "--plat", - plat_arg, "-w", wheelhouse, whl, @@ -132,6 +124,7 @@ def repair_wheels(wheelhouse): except Exception as e: print("Copy failed:", e, file=sys.stderr) + if __name__ == "__main__": parser = argparse.ArgumentParser(description="Repair wheels using platform-specific tools.") parser.add_argument("--wheelhouse", required=True, help="Output directory for repaired wheels") From 4dc3a1a0a8a0de1a823752cc50c6b05ed125451a Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Mon, 6 Oct 2025 20:47:22 +0200 Subject: [PATCH 80/82] Add patch for easyocr --- graalpython/lib-graalpython/patches/easyocr.patch | 13 +++++++++++++ graalpython/lib-graalpython/patches/metadata.toml | 10 ++++++++++ 2 files changed, 23 insertions(+) create mode 100644 graalpython/lib-graalpython/patches/easyocr.patch diff --git a/graalpython/lib-graalpython/patches/easyocr.patch b/graalpython/lib-graalpython/patches/easyocr.patch new file mode 100644 index 0000000000..0fe788f745 --- /dev/null +++ b/graalpython/lib-graalpython/patches/easyocr.patch @@ -0,0 +1,13 @@ +diff --git a/easyocr/utils.py b/easyocr/utils.py +index 5dee1b7..de7af43 100644 +--- a/easyocr/utils.py ++++ b/easyocr/utils.py +@@ -329,7 +329,7 @@ import stat + texts = [] + index = 0 + for l in length: +- t = text_index[index:index + l] ++ t = text_index[index:int(index + l)] + # Returns a boolean array where true is when the value is not repeated + a = np.insert(~((t[1:]==t[:-1])),0,True) + # Returns a boolean array where true is when the value is not in the ignore_idx list diff --git a/graalpython/lib-graalpython/patches/metadata.toml b/graalpython/lib-graalpython/patches/metadata.toml index 63017d669d..9ec3ba391b 100644 --- a/graalpython/lib-graalpython/patches/metadata.toml +++ b/graalpython/lib-graalpython/patches/metadata.toml @@ -875,3 +875,13 @@ url = 'https://github.com/piskvorky/gensim/archive/refs/tags/4.3.3.tar.gz' [[gensim.rules]] version = '== 4.3.3' + +[[easyocr.rules]] +version = '== 1.7.2' +patch = 'easyocr.patch' +license = 'Apache-2.0' + +[[opencv-python-headless.rules]] +version = '== 4.12.0.88' +patch = 'opencv-python-headless.patch' +dist-type = 'sdist' From 209f70fb27459d83f2b4446012e1139e8d6f8f03 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Mon, 6 Oct 2025 20:47:29 +0200 Subject: [PATCH 81/82] Add build script for torchvision --- scripts/wheelbuilder/linux/torchvision.sh | 47 +++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 scripts/wheelbuilder/linux/torchvision.sh diff --git a/scripts/wheelbuilder/linux/torchvision.sh b/scripts/wheelbuilder/linux/torchvision.sh new file mode 100644 index 0000000000..0cbb9fc16f --- /dev/null +++ b/scripts/wheelbuilder/linux/torchvision.sh @@ -0,0 +1,47 @@ +# Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +if [ -n "$GITHUB_RUN_ID" ]; then + dnf install -y gcc-toolset-12 + if [ -n "$1" ]; then + scl enable gcc-toolset-12 "pip wheel torchvision==$1" + else + scl enable gcc-toolset-12 "pip wheel torchvision" + fi +fi From 1de8760536f51f39c08e6c3fded5b8133608a2f4 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Mon, 6 Oct 2025 21:28:43 +0200 Subject: [PATCH 82/82] Add hacky opencv-python-headless patch --- graalpython/lib-graalpython/patches/metadata.toml | 1 + .../patches/opencv-python-headless.patch | 13 +++++++++++++ 2 files changed, 14 insertions(+) create mode 100644 graalpython/lib-graalpython/patches/opencv-python-headless.patch diff --git a/graalpython/lib-graalpython/patches/metadata.toml b/graalpython/lib-graalpython/patches/metadata.toml index 9ec3ba391b..5e63aa7665 100644 --- a/graalpython/lib-graalpython/patches/metadata.toml +++ b/graalpython/lib-graalpython/patches/metadata.toml @@ -884,4 +884,5 @@ license = 'Apache-2.0' [[opencv-python-headless.rules]] version = '== 4.12.0.88' patch = 'opencv-python-headless.patch' +license = 'MIT' dist-type = 'sdist' diff --git a/graalpython/lib-graalpython/patches/opencv-python-headless.patch b/graalpython/lib-graalpython/patches/opencv-python-headless.patch new file mode 100644 index 0000000000..022a0847f7 --- /dev/null +++ b/graalpython/lib-graalpython/patches/opencv-python-headless.patch @@ -0,0 +1,13 @@ +diff --git a/opencv/modules/python/package/cv2/__init__.py b/opencv/modules/python/package/cv2/__init__.py +index 5dee1b7..de7af43 100644 +--- a/opencv/modules/python/package/cv2/__init__.py ++++ b/opencv/modules/python/package/cv2/__init__.py +@@ -149,6 +149,8 @@ import stat + if DEBUG: print("Relink everything from native cv2 module to cv2 package") + + py_module = sys.modules.pop("cv2") ++ if os.path.exists(os.path.join(*py_module.__path__, "cv2.abi3.so")): ++ os.rename(os.path.join(*py_module.__path__, "cv2.abi3.so"), os.path.join(*py_module.__path__, "cv2.so")) + + native_module = importlib.import_module("cv2") +