diff --git a/.github/workflows/downstream-tests.yml b/.github/workflows/downstream-tests.yml index 02eb747bec..3bc35aeab4 100644 --- a/.github/workflows/downstream-tests.yml +++ b/.github/workflows/downstream-tests.yml @@ -13,6 +13,7 @@ jobs: - pybind11 - virtualenv - pyo3 + - jiter os: - id: ubuntu-latest platform: linux @@ -35,7 +36,7 @@ jobs: run: brew install cmake - name: Install Rust toolchain - if: ${{ matrix.name == 'pyo3' }} + if: ${{ matrix.name == 'pyo3' || matrix.name == 'jiter' }} run: | curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y echo "${HOME}/.cargo/bin" >> $GITHUB_PATH diff --git a/.gitignore b/.gitignore index bf2d8928b6..2d868481d7 100644 --- a/.gitignore +++ b/.gitignore @@ -97,3 +97,4 @@ pom-mx.xml !graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/gradle/wrapper/gradle-wrapper.jar /*-venv/ .aider* +*.iprof.gz diff --git a/ci.jsonnet b/ci.jsonnet index 19f1d5054d..9d8ed1e889 100644 --- a/ci.jsonnet +++ b/ci.jsonnet @@ -5,7 +5,7 @@ (import "ci/python-gate.libsonnet") + (import "ci/python-bench.libsonnet") + { - overlay: "a8df46e16d6fcae69e9a7c42c632131fdf6e043e", + overlay: "907f84a98fe5736824d2964e8815d75d0511d39d", specVersion: "6", // Until buildbot issues around CI tiers are resolved, we cannot use them // tierConfig: self.tierConfig, @@ -208,7 +208,7 @@ "windows:amd64:jdk-latest" : tier3 + require(GRAAL_JDK_LATEST), }), "python-junit-bytecode-dsl": gpgate + platform_spec(no_jobs) + bytecode_dsl_gate("python-junit") + platform_spec({ - "linux:amd64:jdk21" : tier3 + require(GRAAL_JDK_LATEST), + "linux:amd64:jdk21" : daily + require(GRAAL_JDK_LATEST), }), "python-junit-maven": gpgate_maven + platform_spec(no_jobs) + platform_spec({ "linux:amd64:jdk21" : daily + t("00:30:00"), @@ -493,7 +493,7 @@ ) + [ { name: "graalpy-website-build", - targets: ["gate"], + targets: ["tier1"], guard: { includes: ["docs/user/**", "docs/site/**"], }, diff --git a/ci/graal/ci/ci_common/common-utils.libsonnet b/ci/graal/ci/ci_common/common-utils.libsonnet index 794af17efc..a476dc8d31 100644 --- a/ci/graal/ci/ci_common/common-utils.libsonnet +++ b/ci/graal/ci/ci_common/common-utils.libsonnet @@ -25,7 +25,7 @@ # Adds a CI build predicate to `build` if it is a gate such that it is only # run if a top level CI file or a non-documentation file in any of `suites` has been updated add_gate_predicate(build, suites, extra_includes=[], extra_excludes=[]):: - if std.member(build.targets, "gate") || std.member(build.targets, "tier1") || std.member(build.targets, "tier2") || std.member(build.targets, "tier3") then + if std.member(build.targets, "gate") then build + { guard+: { includes+: [ suite + "/**" for suite in suites ] + extra_includes + $.top_level_ci, diff --git a/ci/graal/ci/ci_common/musl-common.libsonnet b/ci/graal/ci/ci_common/musl-common.libsonnet index b9a788fac6..3bae934d67 100644 --- a/ci/graal/ci/ci_common/musl-common.libsonnet +++ b/ci/graal/ci/ci_common/musl-common.libsonnet @@ -3,8 +3,8 @@ musl_dependency:: { downloads+: { MUSL_TOOLCHAIN: { - name: 'toolchain-gcc-10.2.1-musl', - version: '1.2.2', + name: 'toolchain-gcc-10.3.0-zlib-1.2.13-musl', + version: '1.2.5.1', platformspecific: true, }, }, diff --git a/ci/graal/ci/common.jsonnet b/ci/graal/ci/common.jsonnet index 4272ca2d15..74d4ac1dc9 100644 --- a/ci/graal/ci/common.jsonnet +++ b/ci/graal/ci/common.jsonnet @@ -57,9 +57,6 @@ local common_json = import "../common.json"; for name in ["oraclejdk21"] + variants("labsjdk-ce-21") + variants("labsjdk-ee-21") } + { 'oraclejdk24': jdk_base + common_json.jdks["oraclejdk24"] + { jdk_version:: 24 }, - } + { - [name]: jdk_base + common_json.jdks[name] + { jdk_version:: 25 } - for name in ["oraclejdk25"] + variants("labsjdk-ce-25") + variants("labsjdk-ee-25") } + { [name]: jdk_base + common_json.jdks[name] + { jdk_version:: parse_labsjdk_version(self), jdk_name:: "jdk-latest"} for name in ["oraclejdk-latest"] + variants("labsjdk-ce-latest") + variants("labsjdk-ee-latest") @@ -145,11 +142,6 @@ local common_json = import "../common.json"; common_catch_files: { catch_files+: [ - # There are additional catch_files-like patterns in buildbot/graal/catcher.py for: - # * hs_err_pid*.log files - # * Dumping IGV graphs to (?P.+(\.gv\.xml|\.bgv)) - # * CFGPrinter: Output to file (?P.*compilations-.+\.cfg) - # There are defined there for efficiency reasons. # Keep in sync with jdk.graal.compiler.debug.StandardPathUtilitiesProvider#DIAGNOSTIC_OUTPUT_DIRECTORY_MESSAGE_REGEXP "Graal diagnostic output saved in '(?P[^']+)'", # Keep in sync with jdk.graal.compiler.debug.DebugContext#DUMP_FILE_MESSAGE_REGEXP @@ -225,19 +217,15 @@ local common_json = import "../common.json"; } }, - # ProGuard does not yet run on JDK 25 - proguard: { - downloads+: if 'jdk_version' in self && self.jdk_version > 21 then { - TOOLS_JAVA_HOME: jdks_data['oraclejdk24'], - IGV_JAVA_HOME: jdks_data['oraclejdk21'], - } else {}, - }, - # GR-49566: SpotBugs does not yet run on JDK 22 - spotbugs: { + local code_tools = { downloads+: if 'jdk_version' in self && self.jdk_version > 21 then { TOOLS_JAVA_HOME: jdks_data['oraclejdk21'], } else {}, }, + # GR-46676: ProGuard does not yet run on JDK 22 + proguard: code_tools, + # GR-49566: SpotBugs does not yet run on JDK 22 + spotbugs: code_tools, sulong:: self.cmake + { packages+: if self.os == "windows" then { @@ -277,10 +265,9 @@ local common_json = import "../common.json"; graalpy:: self.gradle + self.cmake + { packages+: if (self.os == "linux") then { - libffi: '==3.2.1', - bzip2: '==1.0.6', - zlib: '==1.2.11', - maven: "==3.6.3", + libffi: '>=3.2.1', + bzip2: '>=1.0.6', + maven: ">=3.3.9", } else {}, }, @@ -366,15 +353,6 @@ local common_json = import "../common.json"; # Job frequencies # *************** frequencies: { - tier1: { - targets+: ["tier1"], - }, - tier2: { - targets+: ["tier2"], - }, - tier3: { - targets+: ["tier3"], - }, gate: { targets+: ["gate"], }, diff --git a/ci/graal/common.json b/ci/graal/common.json index 5b3c75de13..badb4abb1f 100644 --- a/ci/graal/common.json +++ b/ci/graal/common.json @@ -4,11 +4,11 @@ "Jsonnet files should not include this file directly but use ci/common.jsonnet instead." ], - "mx_version": "7.58.9", + "mx_version": "7.54.6.1", "COMMENT.jdks": "When adding or removing JDKs keep in sync with JDKs in ci/common.jsonnet", "jdks": { - "galahad-jdk": {"name": "jpg-jdk", "version": "25", "build_id": "jdk-26+6-570", "platformspecific": true, "extrabundles": ["static-libs"]}, + "galahad-jdk": {"name": "jpg-jdk", "version": "25", "build_id": "jdk-25+26-3156", "platformspecific": true, "extrabundles": ["static-libs"]}, "oraclejdk17": {"name": "jpg-jdk", "version": "17.0.7", "build_id": "jdk-17.0.7+8", "platformspecific": true, "extrabundles": ["static-libs"]}, "labsjdk-ce-17": {"name": "labsjdk", "version": "ce-17.0.7+4-jvmci-23.1-b02", "platformspecific": true }, @@ -45,21 +45,13 @@ "oraclejdk24": {"name": "jpg-jdk", "version": "24", "build_id": "jdk-24.0.1+9", "platformspecific": true, "extrabundles": ["static-libs"]}, - "oraclejdk25": {"name": "jpg-jdk", "version": "25", "build_id": "jdk-25+26", "platformspecific": true, "extrabundles": ["static-libs"]}, - "labsjdk-ce-25": {"name": "labsjdk", "version": "ce-25+26-jvmci-b01", "platformspecific": true }, - "labsjdk-ce-25Debug": {"name": "labsjdk", "version": "ce-25+26-jvmci-b01-debug", "platformspecific": true }, - "labsjdk-ce-25-llvm": {"name": "labsjdk", "version": "ce-25+26-jvmci-b01-sulong", "platformspecific": true }, - "labsjdk-ee-25": {"name": "labsjdk", "version": "ee-25+26-jvmci-b01", "platformspecific": true }, - "labsjdk-ee-25Debug": {"name": "labsjdk", "version": "ee-25+26-jvmci-b01-debug", "platformspecific": true }, - "labsjdk-ee-25-llvm": {"name": "labsjdk", "version": "ee-25+26-jvmci-b01-sulong", "platformspecific": true }, - - "oraclejdk-latest": {"name": "jpg-jdk", "version": "26", "build_id": "jdk-26+6", "platformspecific": true, "extrabundles": ["static-libs"]}, - "labsjdk-ce-latest": {"name": "labsjdk", "version": "ce-26+6-jvmci-b01", "platformspecific": true }, - "labsjdk-ce-latestDebug": {"name": "labsjdk", "version": "ce-26+6-jvmci-b01-debug", "platformspecific": true }, - "labsjdk-ce-latest-llvm": {"name": "labsjdk", "version": "ce-26+5-jvmci-b01-sulong", "platformspecific": true }, - "labsjdk-ee-latest": {"name": "labsjdk", "version": "ee-26+6-jvmci-b01", "platformspecific": true }, - "labsjdk-ee-latestDebug": {"name": "labsjdk", "version": "ee-26+6-jvmci-b01-debug", "platformspecific": true }, - "labsjdk-ee-latest-llvm": {"name": "labsjdk", "version": "ee-26+5-jvmci-b01-sulong", "platformspecific": true } + "oraclejdk-latest": {"name": "jpg-jdk", "version": "25", "build_id": "jdk-25.0.1+8", "platformspecific": true, "extrabundles": ["static-libs"]}, + "labsjdk-ce-latest": {"name": "labsjdk", "version": "ce-25.0.1+8-jvmci-b01", "platformspecific": true }, + "labsjdk-ce-latestDebug": {"name": "labsjdk", "version": "ce-25.0.1+8-jvmci-b01-debug", "platformspecific": true }, + "labsjdk-ce-latest-llvm": {"name": "labsjdk", "version": "ce-25.0.1+8-jvmci-b01-sulong", "platformspecific": true }, + "labsjdk-ee-latest": {"name": "labsjdk", "version": "ee-25.0.1+8-jvmci-b01", "platformspecific": true }, + "labsjdk-ee-latestDebug": {"name": "labsjdk", "version": "ee-25.0.1+8-jvmci-b01-debug", "platformspecific": true }, + "labsjdk-ee-latest-llvm": {"name": "labsjdk", "version": "ee-25.0.1+8-jvmci-b01-sulong", "platformspecific": true } }, "eclipse": { diff --git a/ci/python-gate.libsonnet b/ci/python-gate.libsonnet index dbe350b221..a024a8b933 100644 --- a/ci/python-gate.libsonnet +++ b/ci/python-gate.libsonnet @@ -262,7 +262,7 @@ tier1:: $.target("tier1"), tier2:: $.target("tier2"), tier3:: $.target("tier3"), - post_merge:: $.target("tier4") + task_spec({name_target:: "post_merge"}), + post_merge:: $.target("post-merge") + task_spec({name_target:: "post_merge"}), bench:: $.target(graal_common.frequencies.bench.targets[0]), on_demand:: $.target(graal_common.frequencies.on_demand.targets[0]) + task_spec({name_target:: "on_demand"}), diff --git a/graalpython/com.oracle.graal.python.pegparser.generator/main_parser_gen.py b/graalpython/com.oracle.graal.python.pegparser.generator/main_parser_gen.py index e4fc96faee..c4417f5c29 100644 --- a/graalpython/com.oracle.graal.python.pegparser.generator/main_parser_gen.py +++ b/graalpython/com.oracle.graal.python.pegparser.generator/main_parser_gen.py @@ -41,6 +41,7 @@ import os import sys import tokenize +import io from pegen.build import generate_token_definitions from pegen.grammar_parser import GeneratedParser as GrammarParser @@ -71,34 +72,29 @@ def main(): # Determine which line separator to use if os.path.exists(args.output_file): - stat_result = os.stat(args.output_file) - atime, mtime = stat_result.st_atime, stat_result.st_mtime - with open(args.output_file, "r", encoding="utf-8", newline=os.linesep) as f: - content = f.read() - if os.linesep != "\n": - unix_content = content.replace(os.linesep, "\n") - if unix_content == content: - # Windows file has Unix line endings - linesep = "\n" - content = unix_content + with open(args.output_file, "rb") as f: + raw_old_content = f.read() + if b'\r\n' in raw_old_content: + linesep = '\r\n' else: - linesep = os.linesep - else: - linesep = "\n" + linesep = '\n' + with open(args.output_file, "r", encoding="utf-8") as f: + old_content = f.read() else: - content = None + old_content = None linesep = os.linesep - with open(args.output_file, "w", encoding="utf-8", newline=linesep) as file: - gen = JavaParserGenerator(grammar, all_tokens, exact_tokens, non_exact_tokens, file, debug=args.debug) - gen.generate(os.path.basename(args.grammar_file)) + out = io.StringIO() + class_name = os.path.splitext(os.path.basename(args.output_file))[0] + gen = JavaParserGenerator(grammar, all_tokens, exact_tokens, non_exact_tokens, out, class_name, debug=args.debug) + gen.generate(os.path.basename(args.grammar_file)) - with open(args.output_file, "r", encoding="utf-8", newline=linesep) as file: - new_content = file.read() - - if content == new_content: + if out.getvalue() != old_content: + print(f"Writing new {args.output_file}") + with open(args.output_file, "w", encoding="utf-8", newline=linesep) as f: + f.write(out.getvalue()) + else: print(f"{args.output_file} not modified") - os.utime(args.output_file, (atime, mtime)) if __name__ == '__main__': diff --git a/graalpython/com.oracle.graal.python.pegparser.generator/pegjava/java_generator.py b/graalpython/com.oracle.graal.python.pegparser.generator/pegjava/java_generator.py index 45bf6fc4e0..f936050b3b 100644 --- a/graalpython/com.oracle.graal.python.pegparser.generator/pegjava/java_generator.py +++ b/graalpython/com.oracle.graal.python.pegparser.generator/pegjava/java_generator.py @@ -874,10 +874,12 @@ def __init__( exact_tokens: Dict[str, int], non_exact_tokens: Set[str], file: Optional[IO[Text]], + class_name: str, debug: bool = True, skip_actions: bool = False, ): super().__init__(grammar, set(tokens.values()), file) + self.class_name = class_name self.typingvisitor = TypingVisitor(self) # Java type hack self.callmakervisitor = JavaCallMakerVisitor(self, exact_tokens, non_exact_tokens, self.print) self.lookahead_functions: Dict[str, FunctionCall] = {} @@ -930,10 +932,9 @@ def generate(self, filename: str) -> None: self.print(f"// Generated from {filename} by pegen") self.print("package com.oracle.graal.python.pegparser;") self.print(IMPORTS) - className = os.path.splitext(os.path.basename(self.file.name))[0] self.print('@SuppressWarnings({"all", "cast"})') self.print('@SuppressFBWarnings') - self.print("public final class %s extends AbstractParser {" % className) + self.print("public final class %s extends AbstractParser {" % self.class_name) # Java needs a few fields declarations. Also, we're now in a class self.level += 1 self.print() @@ -945,12 +946,12 @@ def generate(self, filename: str) -> None: self.print(f"private static final int {rulename.upper()}_ID = {i};{comment}") self.print() # Java needs a constructor - self.print("public %s(String source, SourceRange sourceRange, ParserCallbacks parserCb, InputType startRule, EnumSet flags, int featureVersion) {" % className) + self.print("public %s(String source, SourceRange sourceRange, ParserCallbacks parserCb, InputType startRule, EnumSet flags, int featureVersion) {" % self.class_name) with self.indent(): self.print("super(source, sourceRange, parserCb, startRule, flags, featureVersion);") self.print("}") self.print() - self.print("public %s(String source, ParserCallbacks parserCb, InputType startRule, EnumSet flags, int featureVersion) {" % className) + self.print("public %s(String source, ParserCallbacks parserCb, InputType startRule, EnumSet flags, int featureVersion) {" % self.class_name) with self.indent(): self.print("super(source, null, parserCb, startRule, flags, featureVersion);") self.print("}") diff --git a/graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java b/graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java index 781ec18a3b..3f937cbc8c 100644 --- a/graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java +++ b/graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java @@ -182,6 +182,7 @@ protected List preprocessArguments(List givenArgs, Map(); boolean posixBackendSpecified = false; boolean sha3BackendSpecified = false; + boolean compressionBackendSpecified = false; boolean installSignalHandlersSpecified = false; boolean isolateNativeModulesSpecified = false; for (Iterator argumentIterator = arguments.iterator(); argumentIterator.hasNext();) { @@ -271,7 +272,8 @@ protected List preprocessArguments(List givenArgs, Map preprocessArguments(List givenArgs, Map preprocessArguments(List givenArgs, Map17 17 UTF-8 - 26.0.0 + 25.0.2 diff --git a/graalpython/com.oracle.graal.python.test.integration/src/com/oracle/graal/python/test/integration/interop/JavaInteropTest.java b/graalpython/com.oracle.graal.python.test.integration/src/com/oracle/graal/python/test/integration/interop/JavaInteropTest.java index 2e120a0f37..29f9462734 100644 --- a/graalpython/com.oracle.graal.python.test.integration/src/com/oracle/graal/python/test/integration/interop/JavaInteropTest.java +++ b/graalpython/com.oracle.graal.python.test.integration/src/com/oracle/graal/python/test/integration/interop/JavaInteropTest.java @@ -48,7 +48,6 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.math.BigInteger; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -107,6 +106,14 @@ public void tearDown() { context.getEngine().close(); } + private String getOutString() { + return out.toString(StandardCharsets.UTF_8).replaceAll("\r\n", "\n"); + } + + private String getErrString() { + return err.toString(StandardCharsets.UTF_8).replaceAll("\r\n", "\n"); + } + @Test public void evalFailsOnError() { boolean didFail = false; @@ -217,8 +224,8 @@ public void testHostException() { Assert.assertTrue(e.getMessage(), e.getMessage().contains("divide by zero")); } - String outString = out.toString(StandardCharsets.UTF_8); - String errString = err.toString(StandardCharsets.UTF_8); + String outString = getOutString(); + String errString = getErrString(); Assert.assertTrue(outString, outString.isEmpty()); Assert.assertTrue(errString, errString.isEmpty()); } @@ -257,7 +264,7 @@ def is_none(x): } @Test - public void testPassingFloats() throws UnsupportedEncodingException { + public void testPassingFloats() { String source = "import polyglot\n" + "@polyglot.export_value\n" + "def foo(x, y):\n" + @@ -266,11 +273,11 @@ public void testPassingFloats() throws UnsupportedEncodingException { context.eval(script); Value main = context.getPolyglotBindings().getMember("foo"); main.execute((float) 1.0, (float) 2.0); - assertEquals("2.0\n", out.toString("UTF-8")); + assertEquals("2.0\n", getOutString()); } @Test - public void testPassingBigIntegers() throws UnsupportedEncodingException { + public void testPassingBigIntegers() { String source = "import polyglot\n" + "@polyglot.export_value\n" + "def foo(x, y):\n" + @@ -279,20 +286,20 @@ public void testPassingBigIntegers() throws UnsupportedEncodingException { context.eval(script); Value main = context.getPolyglotBindings().getMember("foo"); main.execute(BigInteger.valueOf(Long.MAX_VALUE).multiply(BigInteger.TWO), BigInteger.valueOf(7)); - assertEquals("129127208515966861298\n", out.toString("UTF-8")); + assertEquals("129127208515966861298\n", getOutString()); out.reset(); main.execute(Long.MAX_VALUE, 14); - assertEquals("129127208515966861298\n", out.toString("UTF-8")); + assertEquals("129127208515966861298\n", getOutString()); out.reset(); main.execute(6, 7); - assertEquals("42\n", out.toString("UTF-8")); + assertEquals("42\n", getOutString()); out.reset(); main.execute(true, true); - assertEquals("1\n", out.toString("UTF-8")); + assertEquals("1\n", getOutString()); } @Test - public void testBigIntegersAdd() throws UnsupportedEncodingException { + public void testBigIntegersAdd() { String source = "import polyglot\n" + "@polyglot.export_value\n" + "def foo(x, y):\n" + @@ -301,20 +308,20 @@ public void testBigIntegersAdd() throws UnsupportedEncodingException { context.eval(script); Value main = context.getPolyglotBindings().getMember("foo"); main.execute(BigInteger.valueOf(24).shiftLeft(101), BigInteger.valueOf(7)); - assertEquals("60847228810955011271841753858055\n", out.toString("UTF-8")); + assertEquals("60847228810955011271841753858055\n", getOutString()); out.reset(); main.execute(BigInteger.valueOf(24).shiftLeft(101), BigInteger.valueOf(24).shiftLeft(101)); - assertEquals("121694457621910022543683507716096\n", out.toString("UTF-8")); + assertEquals("121694457621910022543683507716096\n", getOutString()); out.reset(); main.execute(6, 7); - assertEquals("13\n", out.toString("UTF-8")); + assertEquals("13\n", getOutString()); out.reset(); main.execute(true, true); - assertEquals("2\n", out.toString("UTF-8")); + assertEquals("2\n", getOutString()); } @Test - public void testBigIntegersEg() throws UnsupportedEncodingException { + public void testBigIntegersEg() { String source = "import polyglot\n" + "@polyglot.export_value\n" + "def foo(x, y):\n" + @@ -323,26 +330,26 @@ public void testBigIntegersEg() throws UnsupportedEncodingException { context.eval(script); Value main = context.getPolyglotBindings().getMember("foo"); main.execute(BigInteger.valueOf(24).shiftLeft(101), BigInteger.valueOf(7)); - assertEquals("False\n", out.toString("UTF-8")); + assertEquals("False\n", getOutString()); out.reset(); main.execute(BigInteger.valueOf(24).shiftLeft(101), BigInteger.valueOf(24).shiftLeft(101)); - assertEquals("True\n", out.toString("UTF-8")); + assertEquals("True\n", getOutString()); out.reset(); main.execute(6, 7); - assertEquals("False\n", out.toString("UTF-8")); + assertEquals("False\n", getOutString()); out.reset(); main.execute(6, 6); - assertEquals("True\n", out.toString("UTF-8")); + assertEquals("True\n", getOutString()); out.reset(); main.execute(true, BigInteger.ONE); - assertEquals("True\n", out.toString("UTF-8")); + assertEquals("True\n", getOutString()); out.reset(); main.execute(true, BigInteger.ZERO); - assertEquals("False\n", out.toString("UTF-8")); + assertEquals("False\n", getOutString()); } @Test - public void testAsFunction() throws UnsupportedEncodingException { + public void testAsFunction() { String source = "import polyglot\n" + "@polyglot.export_value\n" + "def foo():\n" + @@ -351,11 +358,11 @@ public void testAsFunction() throws UnsupportedEncodingException { context.eval(script); Value main = context.getPolyglotBindings().getMember("foo"); main.execute(); - assertEquals("Called\n", out.toString("UTF-8")); + assertEquals("Called\n", getOutString()); } @Test - public void testAsFunctionVarArgs() throws UnsupportedEncodingException { + public void testAsFunctionVarArgs() { String source = "import polyglot\n" + "@polyglot.export_value\n" + "def foo(a, b):\n" + @@ -364,28 +371,28 @@ public void testAsFunctionVarArgs() throws UnsupportedEncodingException { context.eval(script); Value main = context.getPolyglotBindings().getMember("foo"); main.execute("Hello", "World"); - assertEquals("Hello World\n", out.toString("UTF-8")); + assertEquals("Hello World\n", getOutString()); } @Test - public void mainFunctionsAreImplicitlyImporteable() throws UnsupportedEncodingException { + public void mainFunctionsAreImplicitlyImporteable() { String source = "def foo(a, b):\n" + " print(a, b)\n\n"; Source script = Source.create("python", source); context.eval(script); Value main = context.getBindings("python").getMember("foo"); main.execute("Hello", "World"); - assertEquals("Hello World\n", out.toString("UTF-8")); + assertEquals("Hello World\n", getOutString()); } @Test - public void builtinFunctionsAreImporteable() throws UnsupportedEncodingException { + public void builtinFunctionsAreImporteable() { String source = "pass"; Source script = Source.create("python", source); context.eval(script); Value main = context.getBindings("python").getMember("__builtins__").getMember("print"); main.execute("Hello", "World"); - assertEquals("Hello World\n", out.toString("UTF-8")); + assertEquals("Hello World\n", getOutString()); } @Test @@ -394,14 +401,14 @@ public void enumeratingMainBindingsWorks() throws Exception { } @Test - public void testMultipleInvocationsAreInSameScope() throws UnsupportedEncodingException { + public void testMultipleInvocationsAreInSameScope() { String source = "def foo(a, b):\n" + " print(a, b)\n" + "foo"; Source script = Source.create("python", source); Value foo = context.eval(script); foo.execute("Hello", "World"); - assertEquals("Hello World\n", out.toString("UTF-8")); + assertEquals("Hello World\n", getOutString()); source = "def bar(a, b):\n" + " foo(a, b)\n" + @@ -409,7 +416,7 @@ public void testMultipleInvocationsAreInSameScope() throws UnsupportedEncodingEx script = Source.create("python", source); Value bar = context.eval(script); bar.execute("Hello", "World"); - assertEquals("Hello World\nHello World\n", out.toString("UTF-8")); + assertEquals("Hello World\nHello World\n", getOutString()); source = "invalid syntax"; script = Source.create("python", source); @@ -418,9 +425,9 @@ public void testMultipleInvocationsAreInSameScope() throws UnsupportedEncodingEx } catch (Throwable t) { } bar.execute("Hello", "World"); - assertEquals("Hello World\nHello World\nHello World\n", out.toString("UTF-8")); + assertEquals("Hello World\nHello World\nHello World\n", getOutString()); foo.execute("Hello", "World"); - assertEquals("Hello World\nHello World\nHello World\nHello World\n", out.toString("UTF-8")); + assertEquals("Hello World\nHello World\nHello World\nHello World\n", getOutString()); } @Test @@ -568,7 +575,7 @@ public void accessJavaObjectFields() throws IOException { "5.0 \n" + "6.0 \n" + "True \n" + - "c \n", out.toString("UTF-8")); + "c \n", getOutString()); } @Test @@ -595,7 +602,7 @@ public void accessJavaObjectGetters() throws IOException { "5.0 \n" + "6.0 \n" + "True \n" + - "c \n", out.toString("UTF-8")); + "c \n", getOutString()); } @Test @@ -612,7 +619,7 @@ public void javaStringsAndPythonStrings() throws IOException { assertEquals("" + "\n" + "True\n" + - "True\n", out.toString("UTF-8")); + "True\n", getOutString()); } @Test diff --git a/graalpython/com.oracle.graal.python.test/src/runner.py b/graalpython/com.oracle.graal.python.test/src/runner.py index cd2589299c..5c0ea3b390 100644 --- a/graalpython/com.oracle.graal.python.test/src/runner.py +++ b/graalpython/com.oracle.graal.python.test/src/runner.py @@ -899,7 +899,7 @@ def parse_config(cls, config_path: Path): if config_tags_dir := settings.get('tags_dir'): tags_dir = (config_path.parent / config_tags_dir).resolve() # Temporary hack for Bytecode DSL development in master branch: - if IS_GRAALPY and __graalpython__.is_bytecode_dsl_interpreter and tags_dir: + if IS_GRAALPY and getattr(__graalpython__, 'is_bytecode_dsl_interpreter', False) and tags_dir: new_tags_dir = (config_path.parent / (config_tags_dir + '_bytecode_dsl')).resolve() if new_tags_dir.exists(): tags_dir = new_tags_dir diff --git a/graalpython/com.oracle.graal.python.test/src/tests/conftest.toml b/graalpython/com.oracle.graal.python.test/src/tests/conftest.toml index b8cdc367d6..95958d8594 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/conftest.toml +++ b/graalpython/com.oracle.graal.python.test/src/tests/conftest.toml @@ -22,7 +22,6 @@ partial_splits_individual_tests = true exclude_on = ['win32'] selector = [ "test_multiprocessing_graalpy.py", # import _winapi - "test_patched_pip.py", "test_pathlib.py", "test_posix.py", # import posix "test_pyio.py", # pyio imports msvcrt diff --git a/graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_mmap.py b/graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_mmap.py index af156b4aa3..6997f467bc 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_mmap.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_mmap.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # The Universal Permissive License (UPL), Version 1.0 @@ -36,7 +36,7 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. - +import io import mmap import sys import tempfile @@ -55,12 +55,11 @@ def create_and_map_file(): def _reference_buffer(args): if sys.implementation.name == 'graalpy' and __graalpython__.posix_module_backend() == 'java': # Cannot get mmap pointer under emulated backend - raise OSError + raise io.UnsupportedOperation return b"hello, world" class TestPyMmap(CPyExtTestCase): - test_buffer = CPyExtFunction( _reference_buffer, lambda: ( @@ -79,7 +78,7 @@ class TestPyMmap(CPyExtTestCase): for (i=0; i < buf.len; i++) { data[i] = ((char *) buf.buf)[i]; } - return PyBytes_FromStringAndSize(data, len); + return PyBytes_FromStringAndSize(data, len); } """, resultspec="O", diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_autopatch_capi.py b/graalpython/com.oracle.graal.python.test/src/tests/test_autopatch_capi.py index 0c53ab7f6c..b44a86890d 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_autopatch_capi.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_autopatch_capi.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # The Universal Permissive License (UPL), Version 1.0 @@ -54,7 +54,7 @@ def check_autopatched(source, expected): autopatch_capi.auto_patch(f.name, False) f.seek(0) actual = f.read() - assert actual == expected, f"Autopatch didn't make expected changes. Expected:\n{expected}\nActual:{actual}" + assert actual == expected, f"Autopatch didn't make expected changes. Expected:\n{expected}\nActual:\n{actual}" def test_replace_field_access(): @@ -106,3 +106,29 @@ def test_replace_field_access(): #endif ''', ) + check_autopatched( + ''' + #if SOME_MACRO + ((PyCFunctionObject *) func)->m_ml, + #else + ((PyCFunctionObject *) func)->m_ml, + #endif + ''', + ''' + #if SOME_MACRO + GraalPyCFunction_GetMethodDef((PyObject*)(((PyCFunctionObject *) func))), + #else + GraalPyCFunction_GetMethodDef((PyObject*)(((PyCFunctionObject *) func))), + #endif + ''', + ) + check_autopatched( + ''' + // PyList_SET_ITEM(). + L->ob_item[len] = x; + ''', + ''' + // PyList_SET_ITEM(). + PySequence_Fast_ITEMS((PyObject*)L)[len] = x; + ''', + ) diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_int.py b/graalpython/com.oracle.graal.python.test/src/tests/test_int.py index b173566af2..cc3ead0f84 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_int.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_int.py @@ -150,6 +150,10 @@ def test_bigint_mul(): assert 99999937497465632974931 * 1223432423545234234123123 == 122343165886896325043539375228725106116626429513 assert 99999937497465632974931 * (2**100) == 126764980791447734004805377032945185921379990352429056 +def test_floordiv(): + assert 0 // (-92233720368547394651) == 0 + assert -234 // (-92233720368547394651) == 0 + assert 234 // (-92233720368547394651) == -1 def test_pow(): assert 2 ** 10 == 1024 diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_io.py b/graalpython/com.oracle.graal.python.test/src/tests/test_io.py index eb43d642d1..0faeb0597c 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_io.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_io.py @@ -36,9 +36,11 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. - -import unittest import _io +import os +import sys +import tempfile +import unittest class IOBaseTests(unittest.TestCase): @@ -312,6 +314,17 @@ def read(self, n=None): t = _io.TextIOWrapper(MockRawIO(), newline="\r\n") self.assertEqual(["ab\r\n", "cd"], t.readlines()) + @unittest.skipIf(sys.platform != "win32", "Windows-only test") + def test_crlf_default_on_win32(self): + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "tmp") + with open(path, 'w') as f: + f.write("hello\nworld\n") + with open(path, 'rb') as f: + self.assertEqual(f.read(), b"hello\r\nworld\r\n") + with open(path, 'r') as f: + self.assertEqual(f.read(), "hello\nworld\n") + if __name__ == '__main__': unittest.main() diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_patched_pip.py b/graalpython/com.oracle.graal.python.test/src/tests/test_patched_pip.py index 91654a81b8..c7716f7117 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_patched_pip.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_patched_pip.py @@ -36,6 +36,7 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +import io import os import re import shutil @@ -87,7 +88,10 @@ class PipPatchingTest(unittest.TestCase): def setUpClass(cls): cls.venv_dir = Path(tempfile.mkdtemp()).resolve() subprocess.check_output([sys.executable, "-m", "venv", str(cls.venv_dir)]) - cls.venv_python = str(cls.venv_dir / 'bin' / 'python') + if sys.platform != 'win32': + cls.venv_python = str(cls.venv_dir / 'bin' / 'python') + else: + cls.venv_python = str(cls.venv_dir / 'Scripts' / 'python.exe') subprocess.check_output([cls.venv_python, '-m', 'pip', 'install', 'wheel', 'setuptools']) cls.venv_template_dir = f'{cls.venv_dir}.template' cls.venv_dir.rename(cls.venv_template_dir) @@ -157,11 +161,12 @@ def run_venv_pip_install(self, package, extra_env=None, assert_stderr_matches=No env.update(extra_env) proc = subprocess.run( [ - str(self.venv_dir / 'bin' / 'pip'), + str(self.venv_python), + '-m', 'pip', '--isolated', 'install', '--force-reinstall', - '--find-links', self.index_dir, + '--find-links', str(self.index_dir), '--no-index', '--no-cache-dir', package, @@ -171,6 +176,7 @@ def run_venv_pip_install(self, package, extra_env=None, assert_stderr_matches=No env=env, universal_newlines=True, ) + print(proc.stdout) print(proc.stderr) assert 'Applying GraalPy patch failed for' not in proc.stderr if assert_stderr_matches: @@ -423,10 +429,6 @@ def test_broken_patches_path(self): def test_patches_file_url(self): self.check_installing_with_patch_repo(urljoin('file:', pathname2url(str(self.patch_dir.absolute())))) - @unittest.skipIf( - __graalpython__.posix_module_backend() == 'java', - "Server doesn't work properly under Java posix backend" - ) def test_patches_http_url(self): patch_dir = self.patch_dir @@ -434,8 +436,24 @@ class Handler(SimpleHTTPRequestHandler): def __init__(self, *args, **kwargs): super().__init__(*args, directory=str(patch_dir), **kwargs) + def do_GET(self): + f = self.send_head() + if f: + try: + input_file = io.TextIOWrapper(f) + # Always serve UTF-8 with unix line endings regardless of platform + # to emulate what GitHub would serve from a patch branch + output_file = io.TextIOWrapper(self.wfile, encoding='utf-8', newline='\n') + shutil.copyfileobj(input_file, output_file) + output_file.flush() + finally: + f.close() + try: with HTTPServer(('localhost', 0), Handler) as server: + if not server.server_port: + # Workaround for java posix backend limitation + server.server_port = server.socket.getsockname()[1] thread = threading.Thread(target=server.serve_forever) thread.start() try: diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_posix.py b/graalpython/com.oracle.graal.python.test/src/tests/test_posix.py index 247f11b72e..0fd52da639 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_posix.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_posix.py @@ -173,6 +173,31 @@ def test_pipe(self): os.close(fd1) os.close(fd2) + @unittest.skipIf(sys.platform != 'linux', 'mkfifo is a linux command') + def test_seek_pipe(self): + new_file_path = './myscript.sh' + with io.open(new_file_path, 'w') as script: + script.write("""#!/bin/sh + mkfifo testpipe + echo "4" > testpipe & + """) + try: + st = os.stat(new_file_path) + os.chmod(new_file_path, st.st_mode | stat.S_IEXEC) + os.system(new_file_path) + with io.open("testpipe", "rb") as r: + out = r.read(1) + assert out == b"4", out + finally: + try: + os.remove(new_file_path) + except: + pass + try: + os.remove("testpipe") + except: + pass + def test_mkdir_rmdir(self): os.mkdir(TEST_FULL_PATH1) try: diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_subprocess.py b/graalpython/com.oracle.graal.python.test/src/tests/test_subprocess.py index dbfd881928..672468b57f 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_subprocess.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_subprocess.py @@ -162,6 +162,21 @@ def test_java_asserts(self): result = subprocess.run([sys.executable, "-c", "import __graalpython__; not __graalpython__.java_assert()"]) assert result.returncode == 0 + def test_subprocess_inherits_environ(self): + import os + import subprocess + prev = os.environ.get("FOOBAR") + try: + expected_value = f"42{prev}".strip() + os.environ["FOOBAR"] = expected_value + out = subprocess.check_output([sys.executable, '-c', "import os; print(os.environ['FOOBAR'])"]).decode().strip() + assert out == expected_value, f"{out!r} != {expected_value!r}" + finally: + if prev: + os.environ["FOOBAR"] = prev + else: + del os.environ["FOOBAR"] + @unittest.skipUnless(sys.implementation.name == 'graalpy', "GraalPy-specific test") @unittest.skipIf(sys.platform == 'win32', "TODO the cmd replacement breaks the test") def test_graal_python_args(self): diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_time.py b/graalpython/com.oracle.graal.python.test/src/tests/test_time.py index 30a6fda692..3d68826dce 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_time.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_time.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # The Universal Permissive License (UPL), Version 1.0 @@ -36,10 +36,14 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. - -import time import calendar +import os +import subprocess +import sys +import time import unittest +from textwrap import dedent + def test_sleep(): start = time.time() @@ -72,7 +76,7 @@ def test_new_struct_time(self): self.assertEqual(t.tm_mday, 26) self.assertEqual(t[2], 26) self.assertEqual(t.tm_zone, None) - + self.assertRaises(TypeError, time.struct_time, (2018, 11, 26, 17, 34, 12, 0, 340)) self.assertRaises(TypeError, time.struct_time, (2018, 11, 26, 17, 34, 12, 0, 340, 9, 10, 11, 12)) @@ -80,11 +84,11 @@ def test_from_times(self): gt = time.gmtime() self.assertNotEqual(gt.tm_zone, None) self.assertNotEqual(gt.tm_gmtoff, None) - + lt = time.localtime() self.assertNotEqual(lt.tm_zone, None) self.assertNotEqual(lt.tm_gmtoff, None) - + def test_destructuring_assignment(self): t = time.struct_time((1,2,3,4,5,6,7,8,9)) y,m,d,h,mi,s,wd,yd,dst = t @@ -93,7 +97,7 @@ def test_destructuring_assignment(self): self.assertEqual(dst, 9) self.assertEqual(t.tm_zone, None) self.assertEqual(t.tm_gmtoff, None) - + t = time.struct_time((11,12,13,14,15,16,17,18,19,20, 21)) y,m,d,h,mi,s,wd,yd,dst = t self.assertEqual(y, 11) @@ -234,7 +238,7 @@ def test_Yeary(self): self.check_format("%y", (2018, 11, 28, 10, 0, 0, -1, 1, 0), '18') self.check_format("%y", (18, 11, 28, 10, 0, 0, 0, 1, 0), '18') self.check_format("%y", (0, 11, 25, 11, 12, 2, 3, 1, 0), '00') - # This is failing on CPython, which return '35' + # This is failing on CPython, which return '35' #self.check_format("%y", (-365, 11, 24, 23, 20, 61, 6, 1, 0), '65') self.check_format("%y", (17829, 11, 24, 23, 20, 61, 7, 1, 0), '29') @@ -280,3 +284,28 @@ def test_padding(self): self.check_format("%Y-%b-%d-%j", (2018, 8, 8, 15, 24, 10, 3, 55, 0), '2018-Aug-08-055') self.check_format("%Y-%b-%-d-%-j", (2018, 8, 8, 15, 24, 10, 3, 5, 0), '2018-Aug-8-5') self.check_format("%Y-%b-%-d-%-j", (2018, 8, 8, 15, 24, 10, 3, 55, 0), '2018-Aug-8-55') + + @unittest.skipIf(sys.platform == 'win32', "Cannot change timezone per-process on Windows") + def test_default_tz_dst_no_dst(self): + env = os.environ.copy() + testdate = '1994-11-15 12:45:26' + for tz, expected in [ + ('UTC', f"tzname=('UTC', 'UTC'), timezone=0, altzone=0, daylight=0, testdate={testdate}"), + ('Europe/Prague', f"tzname=('CET', 'CEST'), timezone=-3600, altzone=-7200, daylight=1, testdate={testdate}"), + ('Australia/Sydney', f"tzname=('AEST', 'AEDT'), timezone=-36000, altzone=-39600, daylight=1, testdate={testdate}"), + ]: + env['TZ'] = tz + env['LC_ALL'] = 'C' + script = dedent('''\ + from time import * + from datetime import datetime + # Regression test for issue #532 + testdate = datetime.strptime('Tue, 15 Nov 1994 12:45:26 GMT', '%a, %d %b %Y %H:%M:%S %Z') + print(f'{tzname=}, {timezone=}, {altzone=}, {daylight=}, {testdate=!s}') + ''') + out = subprocess.check_output( + [sys.executable, '-c', script], + env=env, + text=True, + ).strip() + self.assertEqual(expected, out) diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_zlib.py b/graalpython/com.oracle.graal.python.test/src/tests/test_zlib.py index 8dd5aacc79..de9fd69d4b 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_zlib.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_zlib.py @@ -14,11 +14,12 @@ def _disable_native_zlib(self, flag): return None __graalpython__ = GP() +import binascii import os +import random +import sys import unittest import zlib -import binascii -import sys pintNumber = 98765432109876543210 longNumber = 9876543210 @@ -271,3 +272,35 @@ def test_GR65704(): __graalpython__._disable_native_zlib(False) assert decompressed == contents + +def test_large_chunk(): + contents = random.randbytes(5000) + wbits = 31 + + __graalpython__._disable_native_zlib(True) + + compressed = zlib.compress(contents, wbits=wbits) + decompressor = zlib.decompressobj(wbits=wbits) + + decompressed = decompressor.decompress(compressed) + + __graalpython__._disable_native_zlib(False) + + assert decompressed == contents + +def test_various_chunks(): + contents = random.randbytes(5000) + wbits = 31 + + __graalpython__._disable_native_zlib(True) + + compressed = zlib.compress(contents, wbits=wbits) + decompressor = zlib.decompressobj(wbits=wbits) + + decompressed = decompressor.decompress(compressed[:10]) + decompressed += decompressor.decompress(compressed[10:200]) + decompressed += decompressor.decompress(compressed[200:]) + + __graalpython__._disable_native_zlib(False) + + assert decompressed == contents diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PwdModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PwdModuleBuiltins.java index 887a037582..446b8ea993 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PwdModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PwdModuleBuiltins.java @@ -74,6 +74,7 @@ import com.oracle.graal.python.runtime.PosixSupportLibrary; import com.oracle.graal.python.runtime.PosixSupportLibrary.PosixException; import com.oracle.graal.python.runtime.PosixSupportLibrary.PwdResult; +import com.oracle.graal.python.runtime.PosixSupportLibrary.UnsupportedPosixFeatureException; import com.oracle.graal.python.runtime.PythonContext; import com.oracle.graal.python.runtime.exception.PException; import com.oracle.graal.python.runtime.object.PFactory; @@ -166,6 +167,8 @@ static Object doGetpwuid(VirtualFrame frame, Object uidObj, } } catch (PosixException e) { throw constructAndRaiseNode.get(inliningTarget).raiseOSErrorFromPosixException(frame, e); + } catch (UnsupportedPosixFeatureException e) { + throw constructAndRaiseNode.get(inliningTarget).raiseOSErrorUnsupported(frame, e); } if (pwd == null) { throw raiseUidNotFound(inliningTarget, raiseNode); @@ -213,6 +216,8 @@ static Object doGetpwname(VirtualFrame frame, TruffleString name, } } catch (PosixException e) { throw constructAndRaiseNode.get(inliningTarget).raiseOSErrorFromPosixException(frame, e); + } catch (UnsupportedPosixFeatureException e) { + throw constructAndRaiseNode.get(inliningTarget).raiseOSErrorUnsupported(frame, e); } if (pwd == null) { throw raiseNode.raise(inliningTarget, PythonBuiltinClassType.KeyError, ErrorMessages.GETPWNAM_NAME_NOT_FOUND, name); @@ -237,6 +242,8 @@ static Object doGetpall(VirtualFrame frame, entries = posixLib.getpwentries(context.getPosixSupport()); } catch (PosixException e) { throw constructAndRaiseNode.get(inliningTarget).raiseOSErrorFromPosixException(frame, e); + } catch (UnsupportedPosixFeatureException e) { + throw constructAndRaiseNode.get(inliningTarget).raiseOSErrorUnsupported(frame, e); } PythonLanguage language = context.getLanguage(inliningTarget); Object[] result = new Object[entries.length]; diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/TimeModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/TimeModuleBuiltins.java index 58663d04fe..553222deb7 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/TimeModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/TimeModuleBuiltins.java @@ -174,17 +174,10 @@ public void postInitialize(Python3Core core) { TruffleString noDaylightSavingZone = toTruffleStringUncached(defaultTimeZone.getDisplayName(false, TimeZone.SHORT)); TruffleString daylightSavingZone = toTruffleStringUncached(defaultTimeZone.getDisplayName(true, TimeZone.SHORT)); - boolean hasDaylightSaving = !noDaylightSavingZone.equalsUncached(daylightSavingZone, TS_ENCODING); - if (hasDaylightSaving) { - timeModule.setAttribute(T_TZNAME, PFactory.createTuple(core.getLanguage(), new Object[]{noDaylightSavingZone, daylightSavingZone})); - } else { - timeModule.setAttribute(T_TZNAME, PFactory.createTuple(core.getLanguage(), new Object[]{noDaylightSavingZone})); - } - - timeModule.setAttribute(T_DAYLIGHT, PInt.intValue(hasDaylightSaving)); - int rawOffsetSeconds = defaultTimeZone.getRawOffset() / -1000; - timeModule.setAttribute(T_TIMEZONE, rawOffsetSeconds); - timeModule.setAttribute(T_ALTZONE, rawOffsetSeconds - 3600); + timeModule.setAttribute(T_TZNAME, PFactory.createTuple(core.getLanguage(), new Object[]{noDaylightSavingZone, daylightSavingZone})); + timeModule.setAttribute(T_DAYLIGHT, PInt.intValue(defaultTimeZone.getDSTSavings() != 0)); + timeModule.setAttribute(T_TIMEZONE, defaultTimeZone.getRawOffset() / -1000); + timeModule.setAttribute(T_ALTZONE, (defaultTimeZone.getRawOffset() + defaultTimeZone.getDSTSavings()) / -1000); // register_interop_behavior() for time.struct_time AbstractImportNode.importModule(T_POLYGLOT_TIME); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/io/TextIOWrapperNodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/io/TextIOWrapperNodes.java index 34635c2162..02c1112611 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/io/TextIOWrapperNodes.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/io/TextIOWrapperNodes.java @@ -67,6 +67,7 @@ import static com.oracle.graal.python.nodes.ErrorMessages.S_SHOULD_HAVE_RETURNED_A_BYTES_LIKE_OBJECT_NOT_P; import static com.oracle.graal.python.nodes.PGuards.isPNone; import static com.oracle.graal.python.nodes.SpecialMethodNames.T_DECODE; +import static com.oracle.graal.python.nodes.StringLiterals.T_CRLF; import static com.oracle.graal.python.nodes.StringLiterals.T_EMPTY_STRING; import static com.oracle.graal.python.nodes.StringLiterals.T_NEWLINE; import static com.oracle.graal.python.nodes.StringLiterals.T_STRICT; @@ -77,6 +78,7 @@ import static com.oracle.graal.python.util.PythonUtils.tsLiteral; import com.oracle.graal.python.PythonLanguage; +import com.oracle.graal.python.builtins.PythonOS; import com.oracle.graal.python.builtins.modules.CodecsTruffleModuleBuiltins; import com.oracle.graal.python.builtins.modules.CodecsTruffleModuleBuiltins.MakeIncrementalcodecNode; import com.oracle.graal.python.builtins.modules.WarningsModuleBuiltins; @@ -153,7 +155,11 @@ protected static void setNewline(PTextIO self, TruffleString newline, TruffleStr self.setWriteNewline(self.getReadNewline()); } } else { - self.setWriteNewline(null); + if (PythonOS.getPythonOS() == PythonOS.PLATFORM_WIN32) { + self.setWriteNewline(T_CRLF); + } else { + self.setWriteNewline(null); + } } } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/zlib/JavaDecompress.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/zlib/JavaDecompress.java index ade0a03d6f..b9752be544 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/zlib/JavaDecompress.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/zlib/JavaDecompress.java @@ -47,6 +47,7 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.EOFException; import java.io.IOException; import java.io.InputStream; import java.util.zip.DataFormatException; @@ -112,7 +113,7 @@ public Inflater getInflater() { return inf; } - public void setInput() throws IOException { + public void fillInput() throws IOException { fill(); } } @@ -162,7 +163,7 @@ private static boolean isGZIPStreamReady(DecompressStream stream, byte[] data, i // GZIPInputStream will read the header during initialization stream.stream = new GZIPDecompressStream(stream.in); stream.inflater = stream.stream.getInflater(); - stream.stream.setInput(); + stream.stream.fillInput(); return true; } } catch (ZipException ze) { @@ -178,7 +179,7 @@ private static boolean isGZIPStreamFinishing(DecompressStream stream, byte[] dat stream.in.append(data, 0, length); try { if (stream.in.length() >= HEADER_TRAILER_SIZE) { - stream.stream.setInput(); + stream.stream.fillInput(); // this should trigger reading trailer stream.stream.read(); stream.stream = null; @@ -243,13 +244,25 @@ private byte[] createByteArray(byte[] bytes, int length, int maxLength, int bufS return EMPTY_BYTE_ARRAY; } - int maxLen = maxLength == 0 ? Integer.MAX_VALUE : maxLength; + int maxLen = maxLength <= 0 ? Integer.MAX_VALUE : maxLength; byte[] result = new byte[Math.min(maxLen, bufSize)]; - int bytesWritten = result.length; ByteArrayOutputStream baos = new ByteArrayOutputStream(); boolean zdictIsSet = false; - while (baos.size() < maxLen && bytesWritten == result.length) { + while (baos.size() < maxLen && !stream.inflater.finished()) { + if (stream.inflater.needsInput()) { + if (stream.stream == null) { + break; + } + try { + stream.stream.fillInput(); + } catch (EOFException e) { + break; + } catch (IOException e) { + throw CompilerDirectives.shouldNotReachHere(e); + } + } + int bytesWritten; try { int len = Math.min(maxLen - baos.size(), result.length); bytesWritten = stream.inflater.inflate(result, 0, len); @@ -257,8 +270,7 @@ private byte[] createByteArray(byte[] bytes, int length, int maxLength, int bufS if (getZdict().length > 0) { setDictionary(); zdictIsSet = true; - // we inflate again with a dictionary - bytesWritten = stream.inflater.inflate(result, 0, len); + continue; } else { throw PRaiseNode.raiseStatic(nodeForRaise, ZLibError, WHILE_SETTING_ZDICT); } @@ -320,7 +332,7 @@ protected static byte[] decompress(byte[] bytes, int length, int wbits, int bufs private void saveUnconsumedInput(byte[] data, int length, byte[] unusedDataBytes, int unconsumedTailLen, Node inliningTarget) { int unusedLen = getRemaining(); - byte[] tail = PythonUtils.arrayCopyOfRange(data, length - unusedLen, length); + byte[] tail = PythonUtils.arrayCopyOfRange(data, Math.max(0, length - unusedLen), length); PythonLanguage language = PythonLanguage.get(inliningTarget); if (isEof()) { if (unconsumedTailLen > 0) { diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/exception/OSErrorEnum.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/exception/OSErrorEnum.java index fc0f608b1e..135d39bcd9 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/exception/OSErrorEnum.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/exception/OSErrorEnum.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -312,7 +312,7 @@ public static ErrorAndMessagePair fromException(Exception e, TruffleString.Equal return new ErrorAndMessagePair(oserror, oserror.getMessage()); } } else { // Generic IOException - OSErrorEnum oserror = tryFindErrnoFromMessage(e); + OSErrorEnum oserror = tryFindErrnoFromMessage(e, eqNode); if (oserror == null) { return new ErrorAndMessagePair(OSErrorEnum.EIO, getMessage(e)); } else { @@ -360,15 +360,13 @@ private static TruffleString getReason(FileSystemException e) { } @TruffleBoundary - private static OSErrorEnum tryFindErrnoFromMessage(Exception e) { - if (e.getMessage().contains("Broken pipe")) { - return OSErrorEnum.EPIPE; - } - Matcher m = ERRNO_PATTERN.matcher(e.getMessage()); + private static OSErrorEnum tryFindErrnoFromMessage(Exception e, TruffleString.EqualNode eqNode) { + String message = e.getMessage(); + Matcher m = ERRNO_PATTERN.matcher(message); if (m.find()) { return fromNumber(Integer.parseInt(m.group(1))); } - return null; + return OSErrorEnum.fromMessage(toTruffleStringUncached(message), eqNode); } @ValueType diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/ints/IntBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/ints/IntBuiltins.java index 71fada76b6..7653a2f87e 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/ints/IntBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/ints/IntBuiltins.java @@ -41,8 +41,8 @@ package com.oracle.graal.python.builtins.objects.ints; import static com.oracle.graal.python.builtins.PythonBuiltinClassType.DeprecationWarning; -import static com.oracle.graal.python.nodes.ErrorMessages.BITWISE_INVERSION_OF_THE_UNDERLYING_INT; import static com.oracle.graal.python.nodes.BuiltinNames.J_INT; +import static com.oracle.graal.python.nodes.ErrorMessages.BITWISE_INVERSION_OF_THE_UNDERLYING_INT; import static com.oracle.graal.python.nodes.SpecialMethodNames.J___CEIL__; import static com.oracle.graal.python.nodes.SpecialMethodNames.J___FLOOR__; import static com.oracle.graal.python.nodes.SpecialMethodNames.J___FORMAT__; @@ -813,7 +813,7 @@ static Object doIPi(int left, PInt right, } return Math.floorDiv(left, rightValue); } catch (OverflowException e) { - return left < 0 == right.isNegative() ? 0 : -1; + return left == 0 || left < 0 == right.isNegative() ? 0 : -1; } } @@ -831,7 +831,7 @@ static Object doLPi(long left, PInt right, } return Math.floorDiv(left, rightValue); } catch (OverflowException e) { - return left < 0 == right.isNegative() ? 0 : -1; + return left == 0 || left < 0 == right.isNegative() ? 0 : -1; } } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/PConstructAndRaiseNode.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/PConstructAndRaiseNode.java index adad68b9d6..53b027b77d 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/PConstructAndRaiseNode.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/PConstructAndRaiseNode.java @@ -243,7 +243,7 @@ public final PException raiseOSErrorFromPosixException(VirtualFrame frame, Posix } public final PException raiseOSErrorUnsupported(VirtualFrame frame, UnsupportedPosixFeatureException e) { - return raiseOSError(frame, OSErrorEnum.EINVAL, createUnsupportedErrorMessage(e)); + return executeWithArgsOnly(frame, PythonBuiltinClassType.IOUnsupportedOperation, new Object[]{createUnsupportedErrorMessage(e)}); } @TruffleBoundary diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/EmulatedPosixSupport.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/EmulatedPosixSupport.java index 75773e27b0..9198f7c151 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/EmulatedPosixSupport.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/EmulatedPosixSupport.java @@ -527,10 +527,15 @@ public Buffer read(int fd, long length, } @TruffleBoundary - private static Buffer readBytesFromChannel(ReadableByteChannel channel, long size) throws IOException { + private static Buffer readBytesFromChannel(ReadableByteChannel channel, long sizeIn) throws IOException { + long size = sizeIn; if (channel instanceof SeekableByteChannel seekableByteChannel) { - long availableSize = seekableByteChannel.size() - seekableByteChannel.position(); - size = Math.min(size, availableSize); + try { + long availableSize = seekableByteChannel.size() - seekableByteChannel.position(); + size = Math.min(size, availableSize); + } catch (IOException e) { + // pass and read what we can + } } size = Math.min(size, MAX_READ); ByteBuffer dst = ByteBuffer.allocate((int) size); @@ -2310,6 +2315,8 @@ public int forkExec(Object[] executables, Object[] args, Object cwd, Object[] en throw createUnsupportedFeature("Only key=value environment variables are supported in fork_exec"); } } + } else { + envMap = new HashMap<>(environ); } String[] argStrings; diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NFIBz2Support.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NFIBz2Support.java index acb8b48353..70bcd84756 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NFIBz2Support.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NFIBz2Support.java @@ -153,7 +153,7 @@ public String signature() { @CompilerDirectives.CompilationFinal private boolean available; private NFIBz2Support(PythonContext context, NativeLibrary.NFIBackend backend, String noNativeAccessHelp) { - if (context.isNativeAccessAllowed()) { + if (context.useNativeCompressionModules()) { this.pythonContext = context; this.typedNativeLib = NativeLibrary.create(PythonContext.getSupportLibName(SUPPORTING_NATIVE_LIB_NAME), Bz2NativeFunctions.values(), backend, noNativeAccessHelp, false); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NFILZMASupport.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NFILZMASupport.java index 4155cfee22..c0595c9639 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NFILZMASupport.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NFILZMASupport.java @@ -293,7 +293,7 @@ public String signature() { @CompilerDirectives.CompilationFinal private boolean available; private NFILZMASupport(PythonContext context, NativeLibrary.NFIBackend backend, String noNativeAccessHelp) { - if (context.isNativeAccessAllowed()) { + if (context.useNativeCompressionModules()) { this.pythonContext = context; this.typedNativeLib = NativeLibrary.create(PythonContext.getSupportLibName(SUPPORTING_NATIVE_LIB_NAME), LZMANativeFunctions.values(), backend, noNativeAccessHelp, true); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NFIZlibSupport.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NFIZlibSupport.java index 8043d34878..97728360fb 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NFIZlibSupport.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NFIZlibSupport.java @@ -269,7 +269,7 @@ public String signature() { @CompilerDirectives.CompilationFinal private boolean available; private NFIZlibSupport(PythonContext context, NativeLibrary.NFIBackend backend, String noNativeAccessHelp) { - if (context.isNativeAccessAllowed()) { + if (context.useNativeCompressionModules()) { this.pythonContext = context; this.typedNativeLib = NativeLibrary.create(PythonContext.getSupportLibName(SUPPORTING_NATIVE_LIB_NAME), ZlibNativeFunctions.values(), backend, noNativeAccessHelp, true); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java index 5e93534742..975bad40be 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java @@ -2335,6 +2335,15 @@ public boolean isExecutableAccessAllowed() { return getEnv().isHostLookupAllowed() || isNativeAccessAllowed(); } + public boolean useNativeCompressionModules() { + if (isNativeAccessAllowed()) { + TruffleString option = getLanguage().getEngineOption(PythonOptions.CompressionModulesBackend); + TruffleString.EqualNode eqNode = TruffleString.EqualNode.getUncached(); + return !eqNode.execute(T_JAVA, option, TS_ENCODING); + } + return false; + } + /** * Trigger any pending asynchronous actions */ diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonOptions.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonOptions.java index 00adc12ad0..58e6f245b6 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonOptions.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonOptions.java @@ -195,12 +195,15 @@ public static void checkBytecodeDSLEnv() { } })); - @EngineOption @Option(category = OptionCategory.USER, help = "Choose the backend for the POSIX module.", usageSyntax = "java|native|llvm", stability = OptionStability.STABLE) // + @EngineOption @Option(category = OptionCategory.USER, help = "Choose the backend for the POSIX module.", usageSyntax = "java|native", stability = OptionStability.STABLE) // public static final OptionKey PosixModuleBackend = new OptionKey<>(T_JAVA, TS_OPTION_TYPE); @EngineOption @Option(category = OptionCategory.USER, help = "Choose the backend for the Sha3 module.", usageSyntax = "java|native", stability = OptionStability.STABLE) // public static final OptionKey Sha3ModuleBackend = new OptionKey<>(T_JAVA, TS_OPTION_TYPE); + @EngineOption @Option(category = OptionCategory.USER, help = "Choose the backend for the Zlib, Bz2, and LZMA modules.", usageSyntax = "java|native", stability = OptionStability.STABLE) // + public static final OptionKey CompressionModulesBackend = new OptionKey<>(T_JAVA, TS_OPTION_TYPE); + @Option(category = OptionCategory.USER, help = "Install default signal handlers on startup", usageSyntax = "true|false", stability = OptionStability.STABLE) // public static final OptionKey InstallSignalHandlers = new OptionKey<>(false); diff --git a/graalpython/graalpy-jbang/examples/hello.java b/graalpython/graalpy-jbang/examples/hello.java index 1cd6805367..d55d8734c2 100644 --- a/graalpython/graalpy-jbang/examples/hello.java +++ b/graalpython/graalpy-jbang/examples/hello.java @@ -40,7 +40,7 @@ */ ///usr/bin/env jbang "$0" "$@" ; exit $? //JAVA 17+ -//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:26.0.0} +//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:25.0.2} // specify python packages and their versions as if used with pip //PIP termcolor==2.2 diff --git a/graalpython/graalpy-jbang/templates/graalpy-template.java.qute b/graalpython/graalpy-jbang/templates/graalpy-template.java.qute index 1c397a3f35..bf7b2f3331 100644 --- a/graalpython/graalpy-jbang/templates/graalpy-template.java.qute +++ b/graalpython/graalpy-jbang/templates/graalpy-template.java.qute @@ -5,7 +5,7 @@ {/for} {#if dependencies.isEmpty()}// //DEPS {/if} {| -//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:26.0.0} +//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:25.0.2} // specify python packages and their versions as if used with pip //PIP termcolor==2.2 |} diff --git a/graalpython/graalpy-jbang/templates/graalpy-template_local_repo.java.qute b/graalpython/graalpy-jbang/templates/graalpy-template_local_repo.java.qute index 094182f80c..1b3ba2dd44 100644 --- a/graalpython/graalpy-jbang/templates/graalpy-template_local_repo.java.qute +++ b/graalpython/graalpy-jbang/templates/graalpy-template_local_repo.java.qute @@ -8,7 +8,7 @@ //REPOS mc=https://repo1.maven.org/maven2/ //REPOS local=file://{path_to_local_repo} {| -//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:26.0.0} +//DEPS org.graalvm.python:jbang:${env.GRAALPY_VERSION:25.0.2} // specify python packages and their versions as if used with pip //PIP termcolor==2.2 |} diff --git a/graalpython/lib-graalpython/modules/autopatch_capi.py b/graalpython/lib-graalpython/modules/autopatch_capi.py index 1551aa218c..49382c5fc6 100644 --- a/graalpython/lib-graalpython/modules/autopatch_capi.py +++ b/graalpython/lib-graalpython/modules/autopatch_capi.py @@ -65,8 +65,13 @@ def replace_field_access(contents, match, replacement, assignment): start, end = match.span(1) level = 0 - def consume_whitespace_backwards(idx): - while idx >= 0 and contents[idx].isspace(): + def consume_whitespace_backwards(idx, newlines=True): + while idx >= 0 and contents[idx].isspace() and (not newlines or contents[idx] != '\n'): + idx -= 1 + return idx + + def consume_line_backwards(idx): + while idx >= 0 and contents[idx] != '\n': idx -= 1 return idx @@ -124,6 +129,13 @@ def consume_identifier_backwards(idx): else: idx += 1 break + idx = consume_whitespace_backwards(idx, newlines=False) + if contents[idx + 1] == '\n': + # Get previous line. If it's a comment or a preprocessor directive, stop + line_start = consume_whitespace_forward(consume_line_backwards(idx)) + if contents[line_start: line_start + 2].startswith(('#', '//')): + idx += 1 + break idx = consume_whitespace_backwards(idx) receiver_start = consume_whitespace_forward(idx) diff --git a/graalpython/lib-graalpython/patches/pip-24.3.1.patch b/graalpython/lib-graalpython/patches/pip-24.3.1.patch index 325e7c1cfb..6a5afce19a 100644 --- a/graalpython/lib-graalpython/patches/pip-24.3.1.patch +++ b/graalpython/lib-graalpython/patches/pip-24.3.1.patch @@ -148,10 +148,10 @@ index 6617644..ad52082 100644 return self._ireq diff --git a/pip/_internal/utils/graalpy.py b/pip/_internal/utils/graalpy.py new file mode 100644 -index 0000000..9e79fb0 +index 0000000..cdc4852 --- /dev/null +++ b/pip/_internal/utils/graalpy.py -@@ -0,0 +1,334 @@ +@@ -0,0 +1,354 @@ +import abc +import logging +import os @@ -179,6 +179,7 @@ index 0000000..9e79fb0 +DEFAULT_PATCHES_URL = f'https://raw.githubusercontent.com/oracle/graalpython/refs/heads/github/patches/{VERSION_PARAMETER}/graalpython/lib-graalpython/patches/' + +PATCHES_URL = os.environ.get('PIP_GRAALPY_PATCHES_URL', DEFAULT_PATCHES_URL) ++DISABLED_PATCHES_URL = 'disabled' +DISABLE_PATCHING = os.environ.get('PIP_GRAALPY_DISABLE_PATCHING', '').lower() in ('true', '1') +DISABLE_VERSION_SELECTION = os.environ.get('PIP_GRAALPY_DISABLE_VERSION_SELECTION', '').lower() in ('true', '1') + @@ -197,6 +198,10 @@ index 0000000..9e79fb0 + pass + + ++class RepositoryNotFound(RepositoryException): ++ pass ++ ++ +class AbstractPatchRepository(metaclass=abc.ABCMeta): + def __init__(self, metadata: dict): + self._repository = metadata @@ -262,13 +267,14 @@ index 0000000..9e79fb0 + def __init__(self, patches_path: Path, repository_data: dict): + super().__init__(repository_data) + self.patches_path = patches_path -+ logger.debug("Loaded GraalPy patch repository from %s", patches_path) + + @classmethod + def from_path(cls, patches_path: Path): + try: + with open(patches_path / METADATA_FILENAME) as f: + metadata_content = f.read() ++ except FileNotFoundError: ++ raise RepositoryNotFound(f"'{METADATA_FILENAME}' does not exist") + except OSError as e: + raise RepositoryException(f"'{METADATA_FILENAME}' cannot be read: {e}") + return cls(patches_path, cls.metadata_from_string(metadata_content)) @@ -282,7 +288,6 @@ index 0000000..9e79fb0 + def __init__(self, patches_url: str, repository_data: dict): + super().__init__(repository_data) + self.patches_url = patches_url -+ logger.debug("Loaded GraalPy patch repository from %s", patches_url) + + @staticmethod + def get_session(): @@ -294,10 +299,14 @@ index 0000000..9e79fb0 + try: + url = url_for_file(patches_url, METADATA_FILENAME) + response = cls.get_session().get(url) ++ if response.status_code == 404: ++ raise RepositoryNotFound(f"'{METADATA_FILENAME}' not found") + response.raise_for_status() -+ metadata_content = response.content.decode('utf-8') ++ if not response.encoding: ++ response.encoding = 'utf-8' ++ metadata_content = response.text + except Exception as e: -+ raise RepositoryException(f"'{METADATA_FILENAME} cannot be retrieved': {e}") ++ raise RepositoryException(f"'{METADATA_FILENAME}' cannot be retrieved': {e}") + return cls(patches_url, cls.metadata_from_string(metadata_content)) + + @contextmanager @@ -310,9 +319,11 @@ index 0000000..9e79fb0 + yield None + else: + with tempfile.TemporaryDirectory() as tempdir: ++ if not response.encoding: ++ response.encoding = 'utf-8' + patch_file = Path(tempdir) / patch_name -+ with open(patch_file, 'wb') as f: -+ f.write(response.content) ++ with open(patch_file, 'w') as f: ++ f.write(response.text) + yield patch_file + + @@ -339,12 +350,19 @@ index 0000000..9e79fb0 + else: + logger.debug("Skipping versioned GraalPy patch repository on snapshot build") + patches_url = None -+ if patches_url: ++ if patches_url and patches_url != DISABLED_PATCHES_URL: ++ logger.info( ++ "Loading GraalPy post-release patch repository from %s. " ++ "This can be controlled with PIP_GRAALPY_PATCHES_URL environment variable. Set to '%s' to disable", ++ patches_url, DISABLED_PATCHES_URL) + try: + return repository_from_url_or_path(patches_url) + except RepositoryException as e: -+ logger.warning("Failed to load GraalPy patch repository from %s: %s", patches_url, e) -+ logger.warning("Falling back to internal GraalPy patch repository") ++ if patches_url == DEFAULT_PATCHES_URL and isinstance(e, RepositoryNotFound): ++ logger.info("No post-release patch repository published yet") ++ else: ++ logger.warning("Failed to load GraalPy patch repository: %s", e) ++ logger.warning("Falling back to bundled GraalPy patch repository") + try: + return LocalPatchRepository.from_path(DEFAULT_PATCHES_PATH) + except RepositoryException as e: @@ -421,6 +439,8 @@ index 0000000..9e79fb0 + "WARNING: GraalPy needs the 'patch' utility to apply compatibility patches. Please install it using your system's package manager.") + except subprocess.CalledProcessError: + logger.warning(f"Applying GraalPy patch failed for {name}. The package may still work.") ++ except Exception: ++ logger.exception(f"Failed to execute patch utility") + elif version_specs := repository.get_suggested_version_specs(name): + logger.info("We have patches to make this package work on GraalVM for some version(s).") + logger.info("If installing or running fails, consider using one of the versions that we have patches for:") diff --git a/graalpython/lib-python/3/ensurepip/_bundled/pip-24.3.1-py3-none-any.whl b/graalpython/lib-python/3/ensurepip/_bundled/pip-24.3.1-py3-none-any.whl index 11827d576c..b8e955a9b8 100644 Binary files a/graalpython/lib-python/3/ensurepip/_bundled/pip-24.3.1-py3-none-any.whl and b/graalpython/lib-python/3/ensurepip/_bundled/pip-24.3.1-py3-none-any.whl differ diff --git a/graalpython/lib-python/3/tarfile.py b/graalpython/lib-python/3/tarfile.py index 0a0f31eca0..0f60461e47 100755 --- a/graalpython/lib-python/3/tarfile.py +++ b/graalpython/lib-python/3/tarfile.py @@ -2119,12 +2119,14 @@ def gettarinfo(self, name=None, arcname=None, fileobj=None): if pwd: try: tarinfo.uname = pwd.getpwuid(tarinfo.uid)[0] - except KeyError: + # GraalPy change: catch io.UnsupportedOperation to allow this to fail on java posix backend + except (KeyError, io.UnsupportedOperation): pass if grp: try: tarinfo.gname = grp.getgrgid(tarinfo.gid)[0] - except KeyError: + # GraalPy change: catch io.UnsupportedOperation to allow this to fail on java posix backend + except (KeyError, io.UnsupportedOperation): pass if type in (CHRTYPE, BLKTYPE): diff --git a/mx.graalpython/downstream_tests.py b/mx.graalpython/downstream_tests.py index 2e4675c14c..ef29962360 100644 --- a/mx.graalpython/downstream_tests.py +++ b/mx.graalpython/downstream_tests.py @@ -153,6 +153,17 @@ def downstream_test_pyo3(graalpy): run_in_venv(venv, ['pip', 'install', 'nox']) run_in_venv(venv, ['nox', '-s', 'test-py'], cwd=src) +@downstream_test('jiter') +def downstream_test_jiter(graalpy, testdir): + run(['git', 'clone', 'https://github.com/pydantic/jiter.git', '-b', 'main'], cwd=testdir) + src = testdir / 'jiter' + venv = src / 'venv' + run([graalpy, '-m', 'venv', str(venv)]) + run_in_venv(venv, ['pip', 'install', '-r', 'crates/jiter-python/tests/requirements.txt'], cwd=src) + run_in_venv(venv, ['pip', 'install', '-e', 'crates/jiter-python', + '--config-settings=build-args=--profile dev'], cwd=src) + run_in_venv(venv, ['pytest', '-v', '--tb=short', 'crates/jiter-python/tests'], cwd=src) + run_in_venv(venv, ['python', 'crates/jiter-python/bench.py', 'jiter', 'jiter-cache', '--fast'], cwd=src) def run_downstream_test(python, project): DOWNSTREAM_TESTS[project](python) diff --git a/mx.graalpython/mx_graalpython.py b/mx.graalpython/mx_graalpython.py index 7c76ef12b2..753b569ce2 100644 --- a/mx.graalpython/mx_graalpython.py +++ b/mx.graalpython/mx_graalpython.py @@ -1925,6 +1925,7 @@ def bytecode_dsl_build_args(): '--enable-native-access=org.graalvm.shadowed.jline', '-Dpolyglot.python.PosixModuleBackend=native', '-Dpolyglot.python.Sha3ModuleBackend=native', + '-Dpolyglot.python.CompressionModulesBackend=native', ] + bytecode_dsl_build_args(), language='python', default_vm_args=[ diff --git a/mx.graalpython/suite.py b/mx.graalpython/suite.py index 5c237ab663..0e6eb52a88 100644 --- a/mx.graalpython/suite.py +++ b/mx.graalpython/suite.py @@ -9,7 +9,7 @@ "name": "graalpython", "versionConflictResolution": "latest", - "version": "26.0.0", + "version": "25.0.2", "graalpython:pythonVersion": "3.12.8", "release": False, "groupId": "org.graalvm.python", @@ -53,7 +53,7 @@ }, { "name": "tools", - "version": "a180d4f836e7c10201e90c6fc4e06817008b1af7", + "version": "c1b7966a8fdff0a23c47bf8dc2454e71bec4baf6", "subdir": True, "urls": [ {"url": "https://github.com/oracle/graal", "kind": "git"}, @@ -61,7 +61,7 @@ }, { "name": "regex", - "version": "a180d4f836e7c10201e90c6fc4e06817008b1af7", + "version": "c1b7966a8fdff0a23c47bf8dc2454e71bec4baf6", "subdir": True, "urls": [ {"url": "https://github.com/oracle/graal", "kind": "git"}, @@ -810,6 +810,7 @@ '--enable-native-access=org.graalvm.shadowed.jline', "-Dpolyglot.python.PosixModuleBackend=native", "-Dpolyglot.python.Sha3ModuleBackend=native", + "-Dpolyglot.python.CompressionModulesBackend=native", ], "dynamicBuildArgs": "libpythonvm_build_args", },