diff --git a/graalpython/AGENTS.md b/graalpython/AGENTS.md index ba87f467cf..3214a24ceb 100644 --- a/graalpython/AGENTS.md +++ b/graalpython/AGENTS.md @@ -19,6 +19,37 @@ Main implementation tree: Java (Truffle interpreter), C (CPython C-API compatibi - This subtree contains both “source of truth” code and vendored/upstream-ish imports; keep patches minimal in `lib-python/` and large C module imports. - Some large headers/databases (unicode tables) are generated; avoid editing them by hand unless you also update the generator pipeline. +## RUNNING + +- Python code can be executed with GraalPy using `mx python`, invoked just like normal `python` command. The project + *must* be first built with `mx python-jvm` or you will execute stale code. Note that `mx python-jvm` just builds, it + doesn't take arguments nor execute code. + +## TESTING + +- There are multiple kinds of tests: + - GraalPy Python tests + - Our own tests in `com.oracle.graal.python.test/src/tests/` + - Executed with `mx graalpytest test_file_name` + - New test should normally be added here, unless they need to be in Java + - CPython tests, also called tagged tests + - Tests copied from upstream CPython in `lib-python/3/tests`. Should not be modified unless specifically + requested. If modified, modifications should be marked with a `# GraalPy change` comment above the changed + part. + - Executed with `mx graalpytest --tagged test_file_name` + - Uses a "tagging" system where only a subset of tests specified in tag files is normally executed. The `--all` + flag makes it ignore the tags and execute all tests. + - JUnit tests + - In `com.oracle.graal.python.test/src` and `com.oracle.graal.python.test.integration/src` + - Used primarily for testing features exposed to Java, such as embedding, instrumentation or interop. + - The tests need to be built with `mx build` prior to execution. The `mx unittest com.example.TestName` command + can be used to run individual tests. +- The `mx graalpytest` command accepts pytest‑style test selectors (e.g., `test_mod.py::TestClass::test_method`) but is + **not** a full pytest implementation. Standard pytest command‑line flags such as `-k`, `-m`, `-v`, `--maxfail` are not + supported. +- Important: The test commands don't automatically rebuild the project. It is your reponsibility to rebuild the project + using `mx python-jvm` after making changes prior to running tests otherwise the tests will run stale code. + ## ANTI-PATTERNS - Don’t use `mxbuild/**` outputs to understand behavior; always navigate `.../src/...` trees. - C-API: never mix `PyMem_*` / `PyObject_*` allocators with platform `malloc` family. diff --git a/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/debug/PythonDebugTest.java b/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/debug/PythonDebugTest.java index e7d23a3f21..1d8b5fa3f6 100644 --- a/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/debug/PythonDebugTest.java +++ b/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/debug/PythonDebugTest.java @@ -550,6 +550,15 @@ public void testInspectJavaArray() throws Throwable { @Test public void testSourceFileURI() throws Throwable { + testSourceFileURIImpl(false); + } + + @Test + public void testSourceFileURIBytecode() throws Throwable { + testSourceFileURIImpl(true); + } + + private void testSourceFileURIImpl(boolean runFromBytecode) throws Throwable { if (System.getProperty("os.name").toLowerCase().contains("mac")) { // on the mac machines we run with symlinked directories and such, and it's annoying to // cater for that @@ -565,6 +574,11 @@ public void testSourceFileURI() throws Throwable { "sys.path.insert(0, '" + tempDir.toString() + "')\n" + "import imported\n" + "imported.sum(2, 3)\n").getBytes()); + + if (runFromBytecode) { + compileToBytecode(importedFile, importingFile); + } + Source source = Source.newBuilder("python", importingFile.toFile()).build(); try (DebuggerSession session = tester.startSession()) { Breakpoint breakpoint = Breakpoint.newBuilder(importingFile.toUri()).lineIs(4).build(); @@ -602,6 +616,16 @@ public void testSourceFileURI() throws Throwable { } } + private void compileToBytecode(Path... files) { + StringBuilder sourceCode = new StringBuilder("import py_compile\n"); + for (Path file : files) { + sourceCode.append("py_compile.compile(r\"").append(file).append("\")\n"); + } + Source compileSource = Source.newBuilder("python", sourceCode.toString(), "compile_source_uri.py").buildLiteral(); + tester.startEval(compileSource); + tester.expectDone(); + } + @Test public void testInlineEvaluationBreakpointBuiltin() throws Throwable { final Source source = Source.newBuilder("python", """ diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java index b31ebd4c60..1522e51998 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java @@ -27,7 +27,6 @@ import static com.oracle.graal.python.annotations.PythonOS.PLATFORM_WIN32; import static com.oracle.graal.python.nodes.BuiltinNames.T__SIGNAL; -import static com.oracle.graal.python.nodes.StringLiterals.J_PY_EXTENSION; import static com.oracle.graal.python.nodes.StringLiterals.T_PY_EXTENSION; import static com.oracle.graal.python.nodes.truffle.TruffleStringMigrationHelpers.isJavaString; import static com.oracle.graal.python.util.PythonUtils.ARRAY_ACCESSOR; @@ -110,7 +109,6 @@ import com.oracle.graal.python.runtime.exception.PException; import com.oracle.graal.python.runtime.object.PFactory; import com.oracle.graal.python.util.Function; -import com.oracle.graal.python.util.LazySource; import com.oracle.graal.python.util.PythonUtils; import com.oracle.graal.python.util.Supplier; import com.oracle.truffle.api.Assumption; @@ -367,6 +365,12 @@ public boolean isSingleContext() { */ private final ConcurrentHashMap sourceCache = new ConcurrentHashMap<>(); + /* + * A map from sources without content to either a Source object with content or a TruffleFile + * that can be used to construct such object. + */ + private final WeakHashMap originalSources = new WeakHashMap<>(); + @Idempotent public static PythonLanguage get(Node node) { return REFERENCE.get(node); @@ -548,54 +552,17 @@ protected CallTarget parse(ParsingRequest request) { return parse(context, source, inputType, topLevel, optimize, interactiveTerminal, argumentNames, futureFeatures); } - public static RootCallTarget callTargetFromBytecode(PythonContext context, Source source, CodeUnit code) { - boolean internal = shouldMarkSourceInternal(context); - SourceBuilder builder = null; - // The original file path should be passed as the name - String name = source.getName(); - if (name != null && !name.isEmpty()) { - builder = sourceForOriginalFile(context, code, internal, name); - if (builder == null) { - if (name.startsWith(FROZEN_FILENAME_PREFIX) && name.endsWith(FROZEN_FILENAME_SUFFIX)) { - String id = name.substring(FROZEN_FILENAME_PREFIX.length(), name.length() - FROZEN_FILENAME_SUFFIX.length()); - String fs = context.getEnv().getFileNameSeparator(); - String path = context.getStdlibHome() + fs + id.replace(".", fs) + J_PY_EXTENSION; - builder = sourceForOriginalFile(context, code, internal, path); - if (builder == null) { - path = context.getStdlibHome() + fs + id.replace(".", fs) + fs + "__init__.py"; - builder = sourceForOriginalFile(context, code, internal, path); - } - } - } - } - if (builder == null) { - builder = Source.newBuilder(source).internal(internal).content(Source.CONTENT_NONE); - } + public RootCallTarget callTargetFromBytecode(Source source, CodeUnit code) { RootNode rootNode; - LazySource lazySource = new LazySource(builder); - if (PythonOptions.ENABLE_BYTECODE_DSL_INTERPRETER) { - // TODO lazily load source in bytecode DSL interpreter too - rootNode = ((BytecodeDSLCodeUnit) code).createRootNode(context, lazySource.getSource()); + rootNode = ((BytecodeDSLCodeUnit) code).createRootNode(this, source); } else { - rootNode = PBytecodeRootNode.create(context.getLanguage(), (BytecodeCodeUnit) code, lazySource, internal); + rootNode = PBytecodeRootNode.create(this, (BytecodeCodeUnit) code, source, source.isInternal()); } return PythonUtils.getOrCreateCallTarget(rootNode); } - private static SourceBuilder sourceForOriginalFile(PythonContext context, CodeUnit code, boolean internal, String path) { - try { - TruffleFile file = context.getEnv().getPublicTruffleFile(path); - if (!file.isReadable()) { - return null; - } - return Source.newBuilder(PythonLanguage.ID, file).name(code.name.toJavaStringUncached()).internal(internal); - } catch (SecurityException | UnsupportedOperationException | InvalidPathException e) { - return null; - } - } - public RootCallTarget parse(PythonContext context, Source source, InputType type, boolean topLevel, int optimize, boolean interactiveTerminal, List argumentNames, EnumSet futureFeatures) { return parse(context, source, type, topLevel, optimize, interactiveTerminal, false, argumentNames, futureFeatures); @@ -677,7 +644,7 @@ private RootNode compileForBytecodeInterpreter(ModTy mod, Source source, int opt Compiler compiler = new Compiler(parserCallbacks); CompilationUnit cu = compiler.compile(mod, EnumSet.noneOf(Compiler.Flags.class), optimize, futureFeatures); BytecodeCodeUnit co = cu.assemble(); - return PBytecodeRootNode.create(this, co, new LazySource(source), source.isInternal(), parserCallbacks); + return PBytecodeRootNode.create(this, co, source, source.isInternal(), parserCallbacks); } private RootNode compileForBytecodeDSLInterpreter(ModTy mod, Source source, int optimize, @@ -957,7 +924,7 @@ private static Source newSource(PythonContext context, SourceBuilder srcBuilder) return srcBuilder.build(); } - private static boolean shouldMarkSourceInternal(PythonContext ctxt) { + public static boolean shouldMarkSourceInternal(PythonContext ctxt) { return !ctxt.isCoreInitialized() && !ctxt.getLanguage().getEngineOption(PythonOptions.ExposeInternalSources); } @@ -1259,6 +1226,36 @@ public Source getOrCreateSource(Function rootNodeFunction, Objec return sourceCache.computeIfAbsent(key, rootNodeFunction); } + public Source getOrCreateSourceWithContent(Source sourceWithoutContent) { + if (sourceWithoutContent.hasCharacters()) { + return sourceWithoutContent; + } + synchronized (originalSources) { + Object original = originalSources.get(sourceWithoutContent); + if (original instanceof Source originalSource) { + return originalSource; + } + if (original instanceof TruffleFile originalFile) { + Source source; + try { + source = Source.newBuilder(ID, originalFile).name(sourceWithoutContent.getName()).internal(sourceWithoutContent.isInternal()).mimeType(MIME_TYPE).build(); + } catch (IOException | SecurityException | UnsupportedOperationException | IllegalArgumentException e) { + source = sourceWithoutContent; + } + originalSources.put(sourceWithoutContent, source); + return source; + } + assert original == null; + return sourceWithoutContent; + } + } + + public void registerOriginalFile(Source sourceWithoutContent, TruffleFile originalFile) { + synchronized (originalSources) { + originalSources.put(sourceWithoutContent, originalFile); + } + } + public static PythonOS getPythonOS() { if (PythonOS.internalCurrent == PythonOS.PLATFORM_ANY) { if (ImageInfo.inImageBuildtimeCode()) { diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java index 1b667a4ec4..2267e815e0 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java @@ -507,7 +507,7 @@ private static Object doLoadBytecodeFile(Object bytecodePath, Object sourcePath, try { // get_data TruffleString strBytecodePath = PyObjectStrAsTruffleStringNode.executeUncached(bytecodePath); - TruffleFile bytecodeFile = context.getEnv().getPublicTruffleFile(strBytecodePath.toJavaStringUncached()); + TruffleFile bytecodeFile = context.getPublicTruffleFileRelaxed(strBytecodePath); byte[] bytes = bytecodeFile.readAllBytes(); // _classify_pyc if (bytes.length < 16 || !Arrays.equals(bytes, 0, 4, MAGIC_NUMBER_BYTES, 0, 4)) { @@ -560,7 +560,16 @@ private static Object doLoadBytecodeFile(Object bytecodePath, Object sourcePath, Object message = PyObjectCallMethodObjArgs.executeUncached(MESSAGE, T_FORMAT, bytecodePath, sourcePath); CallNode.executeUncached(context.lookupBuiltinModule(T__BOOTSTRAP).getAttribute(T__VERBOSE_MESSAGE), message); } - return MarshalModuleBuiltins.fromBytecodeFile(context, bytecodeFile, bytes, 16, bytes.length - 16, cacheKey); + TruffleFile sourceFile = null; + if (sourcePath != PNone.NONE) { + try { + TruffleString strSourcePath = PyObjectStrAsTruffleStringNode.executeUncached(sourcePath); + sourceFile = context.getPublicTruffleFileRelaxed(strSourcePath); + } catch (SecurityException | UnsupportedOperationException | IllegalArgumentException ignored) { + // Fall back to Marshal's empty source. + } + } + return MarshalModuleBuiltins.fromBytecodeFile(context.getLanguage(), bytecodeFile, sourceFile, bytes, 16, bytes.length - 16, cacheKey); } catch (MarshalModuleBuiltins.Marshal.MarshalError me) { throw PRaiseNode.raiseStatic(inliningTarget, me.type, me.message, me.arguments); } catch (IOException | SecurityException | UnsupportedOperationException | IllegalArgumentException e) { diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/ImpModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/ImpModuleBuiltins.java index 1d4752f59b..5893907a17 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/ImpModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/ImpModuleBuiltins.java @@ -49,12 +49,14 @@ import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___LOADER__; import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___ORIGNAME__; import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___PATH__; +import static com.oracle.graal.python.nodes.StringLiterals.J_PY_EXTENSION; import static com.oracle.graal.python.nodes.StringLiterals.T_EXT_PYD; import static com.oracle.graal.python.nodes.StringLiterals.T_EXT_SO; import static com.oracle.graal.python.nodes.StringLiterals.T_NAME; import static com.oracle.graal.python.runtime.exception.PythonErrorType.NotImplementedError; import static com.oracle.graal.python.util.PythonUtils.ARRAY_ACCESSOR; import static com.oracle.graal.python.util.PythonUtils.TS_ENCODING; +import static com.oracle.graal.python.util.PythonUtils.internString; import static com.oracle.graal.python.util.PythonUtils.toTruffleStringUncached; import static com.oracle.graal.python.util.PythonUtils.tsLiteral; @@ -117,6 +119,7 @@ import com.oracle.truffle.api.CompilerDirectives; import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; import com.oracle.truffle.api.RootCallTarget; +import com.oracle.truffle.api.TruffleFile; import com.oracle.truffle.api.TruffleSafepoint; import com.oracle.truffle.api.dsl.Bind; import com.oracle.truffle.api.dsl.Cached; @@ -473,7 +476,7 @@ static Object run(TruffleString name, Object dataObj, Object code = null; try { - code = MarshalModuleBuiltins.Marshal.load(context, bytes, size, 0); + code = MarshalModuleBuiltins.Marshal.load(context.getLanguage(), bytes, size, 0); } catch (MarshalError | NumberFormatException e) { raiseFrozenError(inliningTarget, raiseNode, FROZEN_INVALID, name); } @@ -489,8 +492,7 @@ static Object run(TruffleString name, Object dataObj, info = result.info; raiseFrozenError(inliningTarget, raiseNode, status, name); - RootCallTarget callTarget = createCallTarget(context, info); - return PFactory.createCode(context.getLanguage(), callTarget); + return createCode(context, info); } } } @@ -540,9 +542,10 @@ static Object run(TruffleString name, boolean withData, PMemoryView data = null; + PythonLanguage language = context.getLanguage(); if (withData) { - byte[] bytes = MarshalModuleBuiltins.serializeCodeUnit(inliningTarget, context, info.code); - data = PyMemoryViewFromObject.getUncached().execute(null, PFactory.createBytes(context.getLanguage(inliningTarget), bytes)); + byte[] bytes = MarshalModuleBuiltins.serializeCodeUnit(inliningTarget, language, info.code); + data = PyMemoryViewFromObject.getUncached().execute(null, PFactory.createBytes(language, bytes)); } Object[] returnValues = new Object[]{ @@ -551,7 +554,7 @@ static Object run(TruffleString name, boolean withData, info.origName == null ? PNone.NONE : info.origName }; - return PFactory.createTuple(context.getLanguage(inliningTarget), returnValues); + return PFactory.createTuple(language, returnValues); } } @@ -622,16 +625,15 @@ public static PythonModule importFrozenModuleObject(Node inliningTarget, PConstr } } - RootCallTarget callTarget = createCallTarget(core.getContext(), info); + PCode code = createCode(core.getContext(), info); PythonModule module = globals == null ? PFactory.createPythonModule(name) : globals; - PCode code = PFactory.createCode(core.getLanguage(), callTarget); if (info.isPackage) { /* Set __path__ to the empty list */ WriteAttributeToPythonObjectNode.getUncached().execute(module, T___PATH__, PFactory.createList(core.getLanguage())); } - CallDispatchers.SimpleIndirectInvokeNode.executeUncached(callTarget, PArguments.withGlobals(code, module)); + CallDispatchers.SimpleIndirectInvokeNode.executeUncached(code.getRootCallTarget(), PArguments.withGlobals(code, module)); Object origName = info.origName == null ? PNone.NONE : info.origName; WriteAttributeToPythonObjectNode.getUncached().execute(module, T___ORIGNAME__, origName); @@ -639,12 +641,42 @@ public static PythonModule importFrozenModuleObject(Node inliningTarget, PConstr return module; } - private static RootCallTarget createCallTarget(PythonContext context, FrozenInfo info) { - return (RootCallTarget) context.getLanguage().cacheCode(new PythonLanguage.CodeCacheKey(info.origName, System.identityHashCode(info.code)), () -> { - String name = PythonLanguage.FROZEN_FILENAME_PREFIX + info.name + PythonLanguage.FROZEN_FILENAME_SUFFIX; - Source source = Source.newBuilder("python", "", name).content(Source.CONTENT_NONE).build(); - return PythonLanguage.callTargetFromBytecode(context, source, info.code); - }); + private static PCode createCode(PythonContext context, FrozenInfo info) { + String moduleName = info.name.toJavaStringUncached(); + String codeName = PythonLanguage.FROZEN_FILENAME_PREFIX + moduleName + PythonLanguage.FROZEN_FILENAME_SUFFIX; + TruffleFile file = null; + String filename = codeName; + try { + String fs = context.getEnv().getFileNameSeparator(); + String basename = context.getStdlibHome() + fs + moduleName.replace(".", fs); + String path = info.isPackage ? basename + fs + "__init__.py" : basename + J_PY_EXTENSION; + file = context.getEnv().getInternalTruffleFile(path); + if (file.isReadable()) { + filename = path; + } + } catch (UnsupportedOperationException | IllegalArgumentException | SecurityException e) { + // Fallthrough + } + TruffleFile originalFile = file; + Source source = context.getLanguage().getOrCreateSource((ignored -> { + Source newSource = Source.newBuilder(PythonLanguage.ID, "", codeName) // + .content(Source.CONTENT_NONE) // + .internal(PythonLanguage.shouldMarkSourceInternal(context)) // + .mimeType(PythonLanguage.MIME_TYPE).build(); + PythonLanguage language = context.getLanguage(); + if (originalFile != null) { + language.registerOriginalFile(newSource, originalFile); + } + return newSource; + }), codeName); + RootCallTarget callTarget = (RootCallTarget) context.getLanguage().cacheCode( + new PythonLanguage.CodeCacheKey(info.origName, System.identityHashCode(info.code)), + () -> context.getLanguage().callTargetFromBytecode(source, info.code)); + /* + * Setting the original filename as the co_filename is a deviance from CPython, but it's + * more user friendly and lets us freeze more modules without it being too visible. + */ + return PFactory.createCode(context.getLanguage(), callTarget, internString(toTruffleStringUncached(filename))); } /* diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MarshalModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MarshalModuleBuiltins.java index afd63099cb..df6c6d2861 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MarshalModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MarshalModuleBuiltins.java @@ -198,7 +198,7 @@ static Object doit(VirtualFrame frame, Object value, Object file, int version, Object savedState = BoundaryCallContext.enter(frame, threadState, boundaryCallData); byte[] data; try { - data = Marshal.dump(context, value, version); + data = Marshal.dump(language, value, version); } catch (IOException e) { throw CompilerDirectives.shouldNotReachHere(e); } catch (Marshal.MarshalError me) { @@ -229,7 +229,7 @@ static Object doit(VirtualFrame frame, Object value, int version, PythonContext.PythonThreadState threadState = context.getThreadState(language); Object savedState = BoundaryCallContext.enter(frame, threadState, boundaryCallData); try { - return PFactory.createBytes(language, Marshal.dump(context, value, version)); + return PFactory.createBytes(language, Marshal.dump(language, value, version)); } catch (IOException e) { throw CompilerDirectives.shouldNotReachHere(e); } catch (Marshal.MarshalError me) { @@ -255,12 +255,13 @@ static Object doit(VirtualFrame frame, Object file, @Cached("createCallReadNode()") LookupAndCallBinaryNode callNode, @CachedLibrary(limit = "3") PythonBufferAcquireLibrary bufferLib, @Cached PRaiseNode raiseNode) { + PythonLanguage language = context.getLanguage(inliningTarget); Object buffer = callNode.executeObject(frame, file, 0); if (!bufferLib.hasBuffer(buffer)) { throw raiseNode.raise(inliningTarget, PythonBuiltinClassType.TypeError, ErrorMessages.READ_RETURNED_NOT_BYTES, buffer); } try { - return Marshal.loadFile(context, file); + return Marshal.loadFile(language, file); } catch (NumberFormatException e) { throw raiseNode.raise(inliningTarget, ValueError, ErrorMessages.BAD_MARSHAL_DATA_S, e.getMessage()); } catch (Marshal.MarshalError me) { @@ -289,7 +290,7 @@ static Object doit(VirtualFrame frame, Object buffer, if (!language.isSingleContext()) { cacheKey = language.cacheKeyForBytecode(bytes, length); } - return Marshal.load(context, bytes, length, cacheKey); + return Marshal.load(language, bytes, length, cacheKey); } catch (NumberFormatException e) { throw raiseNode.raise(inliningTarget, ValueError, ErrorMessages.BAD_MARSHAL_DATA_S, e.getMessage()); } catch (Marshal.MarshalError me) { @@ -392,15 +393,15 @@ public final Throwable fillInStackTrace() { } @TruffleBoundary - static byte[] dump(PythonContext context, Object value, int version) throws IOException, MarshalError { - Marshal outMarshal = new Marshal(context, version, context.getTrue(), context.getFalse()); + static byte[] dump(PythonLanguage language, Object value, int version) throws IOException, MarshalError { + Marshal outMarshal = new Marshal(language, version); outMarshal.writeObject(value); return outMarshal.outData.toByteArray(); } @TruffleBoundary - static Object load(PythonContext context, byte[] ary, int length, long cacheKey) throws NumberFormatException, MarshalError { - Marshal inMarshal = new Marshal(context, ary, length, cacheKey); + static Object load(PythonLanguage language, byte[] ary, int length, long cacheKey) throws NumberFormatException, MarshalError { + Marshal inMarshal = new Marshal(language, ary, length, cacheKey); Object result = inMarshal.readObject(); if (result == null) { throw new MarshalError(PythonBuiltinClassType.TypeError, ErrorMessages.BAD_MARSHAL_DATA_NULL); @@ -409,8 +410,8 @@ static Object load(PythonContext context, byte[] ary, int length, long cacheKey) } @TruffleBoundary - static Object loadFile(PythonContext context, Object file) throws NumberFormatException, MarshalError { - Marshal inMarshal = new Marshal(context, file); + static Object loadFile(PythonLanguage language, Object file) throws NumberFormatException, MarshalError { + Marshal inMarshal = new Marshal(language, file); Object result = inMarshal.readObject(); if (result == null) { throw new MarshalError(PythonBuiltinClassType.TypeError, ErrorMessages.BAD_MARSHAL_DATA_NULL); @@ -464,18 +465,17 @@ public int read(byte[] b, int off, int len) { } } - private final PythonContext context; + private final PythonLanguage language; final HashMap refMap; final ArrayList refList; final ByteArrayOutputStream outData; final DataOutput out; final DataInput in; final int version; - final PInt pyTrue; - final PInt pyFalse; int depth = 0; long cacheKey; TruffleFile bytecodeFile; + TruffleFile sourceFile; // Offset of the buffer in parent buffer in nested deserializations int baseOffset; @@ -486,11 +486,9 @@ public int read(byte[] b, int off, int len) { */ Source source = null; - Marshal(PythonContext context, int version, PInt pyTrue, PInt pyFalse) { - this.context = context; + Marshal(PythonLanguage language, int version) { + this.language = language; this.version = version; - this.pyTrue = pyTrue; - this.pyFalse = pyFalse; this.outData = new ByteArrayOutputStream(); this.out = new DataOutputStream(outData); this.refMap = new HashMap<>(); @@ -498,11 +496,9 @@ public int read(byte[] b, int off, int len) { this.refList = null; } - Marshal(PythonContext context, int version, PInt pyTrue, PInt pyFalse, DataOutput out) { - this.context = context; + Marshal(PythonLanguage language, int version, DataOutput out) { + this.language = language; this.version = version; - this.pyTrue = pyTrue; - this.pyFalse = pyFalse; this.outData = null; this.out = out; this.refMap = new HashMap<>(); @@ -510,28 +506,26 @@ public int read(byte[] b, int off, int len) { this.refList = null; } - Marshal(PythonContext context, byte[] in, int length, long cacheKey) { - this(context, SerializationUtils.createByteBufferDataInput(ByteBuffer.wrap(in, 0, length)), null, null, 0); + Marshal(PythonLanguage language, byte[] in, int length, long cacheKey) { + this(language, SerializationUtils.createByteBufferDataInput(ByteBuffer.wrap(in, 0, length)), null, null, 0); this.cacheKey = cacheKey; } - Marshal(PythonContext context, byte[] in, int length, long cacheKey, TruffleFile bytecodeFile, int baseOffset) { - this(context, SerializationUtils.createByteBufferDataInput(ByteBuffer.wrap(in, 0, length)), null, bytecodeFile, baseOffset); + Marshal(PythonLanguage language, byte[] in, int length, long cacheKey, TruffleFile bytecodeFile, int baseOffset) { + this(language, SerializationUtils.createByteBufferDataInput(ByteBuffer.wrap(in, 0, length)), null, bytecodeFile, baseOffset); this.cacheKey = cacheKey; } - Marshal(PythonContext context, Object in) { - this(context, new DataInputStream(new FileLikeInputStream(in)), null, null, 0); + Marshal(PythonLanguage language, Object in) { + this(language, new DataInputStream(new FileLikeInputStream(in)), null, null, 0); } - Marshal(PythonContext context, DataInput in, Source source, TruffleFile bytecodeFile, int baseOffset) { - this.context = context; + Marshal(PythonLanguage language, DataInput in, Source source, TruffleFile bytecodeFile, int baseOffset) { + this.language = language; this.in = in; this.source = source; this.refList = new ArrayList<>(); this.version = -1; - this.pyTrue = null; - this.pyFalse = null; this.outData = null; this.out = null; this.refMap = null; @@ -539,10 +533,6 @@ public int read(byte[] b, int off, int len) { this.baseOffset = baseOffset; } - private PythonLanguage getLanguage() { - return context.getLanguage(); - } - private void writeByte(int v) { try { out.write(v); @@ -781,9 +771,9 @@ private void writeObject(Object v) throws IOException { writeByte(TYPE_STOPITER); } else if (v == PEllipsis.INSTANCE) { writeByte(TYPE_ELLIPSIS); - } else if (v == Boolean.TRUE || v == pyTrue) { + } else if (v == Boolean.TRUE || v instanceof PInt i && i.getPythonClass() == PythonBuiltinClassType.Boolean && i.isOne()) { writeByte(TYPE_TRUE); - } else if (v == Boolean.FALSE || v == pyFalse) { + } else if (v == Boolean.FALSE || v instanceof PInt i && i.getPythonClass() == PythonBuiltinClassType.Boolean && i.isZero()) { writeByte(TYPE_FALSE); } else if (v instanceof Integer) { writeByte(TYPE_INT); @@ -1092,17 +1082,17 @@ private Object readObject(int type, AddRefAndReturn addRef) throws NumberFormatE case TYPE_BIG_INTEGER: return readBigInteger(); case TYPE_LONG: - return addRef.run(PFactory.createInt(getLanguage(), readBigInteger())); + return addRef.run(PFactory.createInt(language, readBigInteger())); case TYPE_FLOAT: return addRef.run(readDoubleString()); case TYPE_BINARY_FLOAT: return addRef.run(readDouble()); case TYPE_COMPLEX: - return addRef.run(PFactory.createComplex(getLanguage(), readDoubleString(), readDoubleString())); + return addRef.run(PFactory.createComplex(language, readDoubleString(), readDoubleString())); case TYPE_BINARY_COMPLEX: - return addRef.run(PFactory.createComplex(getLanguage(), readDouble(), readDouble())); + return addRef.run(PFactory.createComplex(language, readDouble(), readDouble())); case TYPE_STRING: - return addRef.run(PFactory.createBytes(getLanguage(), readBytes())); + return addRef.run(PFactory.createBytes(language, readBytes())); case TYPE_ASCII_INTERNED: return addRef.run(readAscii(readSize(), true)); case TYPE_ASCII: @@ -1118,24 +1108,24 @@ private Object readObject(int type, AddRefAndReturn addRef) throws NumberFormatE case TYPE_SMALL_TUPLE: int smallTupleSize = readByteSize(); Object[] smallTupleItems = new Object[smallTupleSize]; - Object smallTuple = addRef.run(PFactory.createTuple(getLanguage(), smallTupleItems)); + Object smallTuple = addRef.run(PFactory.createTuple(language, smallTupleItems)); readArray(smallTupleItems); return smallTuple; case TYPE_TUPLE: int tupleSize = readSize(); Object[] tupleItems = new Object[tupleSize]; - Object tuple = addRef.run(PFactory.createTuple(getLanguage(), tupleItems)); + Object tuple = addRef.run(PFactory.createTuple(language, tupleItems)); readArray(tupleItems); return tuple; case TYPE_LIST: int listSize = readSize(); Object[] listItems = new Object[listSize]; - Object list = addRef.run(PFactory.createList(getLanguage(), listItems)); + Object list = addRef.run(PFactory.createList(language, listItems)); readArray(listItems); return list; case TYPE_DICT: HashingStorage store = PDict.createNewStorage(0); - PDict dict = PFactory.createDict(getLanguage(), store); + PDict dict = PFactory.createDict(language, store); addRef.run(dict); while (true) { Object key = readObject(); @@ -1155,9 +1145,9 @@ private Object readObject(int type, AddRefAndReturn addRef) throws NumberFormatE HashingStorage setStore = EconomicMapStorage.create(setSz); PBaseSet set; if (type == TYPE_FROZENSET) { - set = PFactory.createFrozenSet(getLanguage(), setStore); + set = PFactory.createFrozenSet(language, setStore); } else { - set = PFactory.createSet(getLanguage(), setStore); + set = PFactory.createSet(language, setStore); } addRef.run(set); for (int i = 0; i < setSz; i++) { @@ -1517,7 +1507,7 @@ private void writeBytecodeDSLCodeUnit(BytecodeDSLCodeUnit code) throws IOExcepti * MakeFunction instruction itself carries only the integer index into this constants * array. */ - byte[] serialized = code.getSerialized(context); + byte[] serialized = code.getSerialized(language); writeBytes(serialized); writeString(code.name); writeString(code.qualname); @@ -1552,24 +1542,28 @@ private PCode readCode() { int firstLineNo = readInt(); byte[] lnoTab = readBytes(); com.oracle.graal.python.util.Supplier supplier = () -> { - String jFilename = fileName.toJavaStringUncached(); - Source subSource = Source.newBuilder(PythonLanguage.ID, "", jFilename).content(Source.CONTENT_NONE).build(); - return PythonLanguage.callTargetFromBytecode(context, subSource, code); + String jName = code.qualname.toJavaStringUncached(); + Source source = Source.newBuilder(PythonLanguage.ID, "", jName).content(Source.CONTENT_NONE).build(); + PythonLanguage language = this.language; + if (sourceFile != null) { + language.registerOriginalFile(source, sourceFile); + } + return language.callTargetFromBytecode(source, code); }; CallTarget callTarget; - if (getLanguage().isSingleContext() || cacheKey == 0) { + if (language.isSingleContext() || cacheKey == 0) { callTarget = supplier.get(); } else { - callTarget = getLanguage().cacheCode(new PythonLanguage.CodeCacheKey(fileName, cacheKey), supplier); + callTarget = language.cacheCode(new PythonLanguage.CodeCacheKey(fileName, cacheKey), supplier); } - return PFactory.createCode(getLanguage(), (RootCallTarget) callTarget, flags, firstLineNo, lnoTab, fileName); + return PFactory.createCode(language, (RootCallTarget) callTarget, flags, firstLineNo, lnoTab, fileName); } } @TruffleBoundary - public static byte[] serializeCodeUnit(Node locationForRaise, PythonContext context, CodeUnit code) { + public static byte[] serializeCodeUnit(Node locationForRaise, PythonLanguage language, CodeUnit code) { try { - Marshal marshal = new Marshal(context, CURRENT_VERSION, null, null); + Marshal marshal = new Marshal(language, CURRENT_VERSION); marshal.writeCodeUnit(code); return marshal.outData.toByteArray(); } catch (IOException e) { @@ -1582,9 +1576,9 @@ public static byte[] serializeCodeUnit(Node locationForRaise, PythonContext cont } @TruffleBoundary - public static CodeUnit deserializeCodeUnit(Node node, PythonContext context, byte[] bytes) { + public static CodeUnit deserializeCodeUnit(Node node, PythonLanguage language, byte[] bytes) { try { - Marshal marshal = new Marshal(context, bytes, bytes.length, 0); + Marshal marshal = new Marshal(language, bytes, bytes.length, 0); return marshal.readCodeUnit(); } catch (Marshal.MarshalError me) { throw PRaiseNode.raiseStatic(node, me.type, me.message, me.arguments); @@ -1611,10 +1605,10 @@ public BytecodeSupplier(byte[] serialized, TruffleFile bytecodeFile, int bytecod } @Override - public PBytecodeDSLRootNode createRootNode(PythonContext context, Source source) { + public PBytecodeDSLRootNode createRootNode(PythonLanguage language, Source source) { BytecodeRootNodes deserialized; try { - deserialized = PBytecodeDSLRootNodeGen.deserialize(context.getLanguage(), BytecodeConfig.WITH_SOURCE, + deserialized = PBytecodeDSLRootNodeGen.deserialize(language, BytecodeConfig.WITH_SOURCE, () -> SerializationUtils.createByteBufferDataInput(ByteBuffer.wrap(getBytecode())), /* * NB: Since a DSL node may reparse multiple times, we cannot reuse @@ -1622,14 +1616,14 @@ public PBytecodeDSLRootNode createRootNode(PythonContext context, Source source) * different buffer). */ (deserializerContext, buffer) -> { - Marshal marshal = new Marshal(PythonContext.get(null), buffer, source, bytecodeFile, bytecodeOffset); + Marshal marshal = new Marshal(language, buffer, source, bytecodeFile, bytecodeOffset); marshal.cacheKey = cacheKey; return marshal.readObject(); }); } catch (IOException e) { throw CompilerDirectives.shouldNotReachHere("Deserialization error."); } - if (bytecodeFile != null && bytecodeOffset >= 0 && cacheKey != 0 && !context.getOption(PythonOptions.KeepBytecodeInMemory)) { + if (bytecodeFile != null && bytecodeOffset >= 0 && cacheKey != 0 && !language.getEngineOption(PythonOptions.KeepBytecodeInMemory)) { // Free the serialized bytecode, we will fetch it from the file if needed again serialized = null; } @@ -1671,16 +1665,16 @@ private byte[] getBytecode() { } @Override - public byte[] createSerializedBytecode(PythonContext context) { + public byte[] createSerializedBytecode(PythonLanguage language) { return getBytecode(); } } public static class PBytecodeDSLSerializer implements BytecodeSerializer { - private final PythonContext pythonContext; + private final PythonLanguage language; - public PBytecodeDSLSerializer(PythonContext pythonContext) { - this.pythonContext = pythonContext; + public PBytecodeDSLSerializer(PythonLanguage language) { + this.language = language; } public void serialize(SerializerContext context, DataOutput buffer, Object object) throws IOException { @@ -1689,7 +1683,7 @@ public void serialize(SerializerContext context, DataOutput buffer, Object objec * we must also do the same here. Otherwise, the encoding may be different (e.g., a * reference for an already-emitted object). */ - new Marshal(pythonContext, CURRENT_VERSION, pythonContext.getTrue(), pythonContext.getFalse(), buffer).writeObject(object); + new Marshal(language, CURRENT_VERSION, buffer).writeObject(object); } } @@ -1702,8 +1696,9 @@ public ReparseError(String message) { } @TruffleBoundary - public static Object fromBytecodeFile(PythonContext context, TruffleFile file, byte[] bytes, int offset, int length, long cacheKey) throws IOException { - MarshalModuleBuiltins.Marshal marshal = new MarshalModuleBuiltins.Marshal(context, bytes, length + offset, cacheKey, file, 0); + public static Object fromBytecodeFile(PythonLanguage language, TruffleFile bytecodeFile, TruffleFile sourceFile, byte[] bytes, int offset, int length, long cacheKey) throws IOException { + MarshalModuleBuiltins.Marshal marshal = new MarshalModuleBuiltins.Marshal(language, bytes, length + offset, cacheKey, bytecodeFile, 0); + marshal.sourceFile = sourceFile; marshal.in.skipBytes(offset); return marshal.readObject(); } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/code/CodeNodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/code/CodeNodes.java index b6fed6522e..c7545b8d6d 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/code/CodeNodes.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/code/CodeNodes.java @@ -60,7 +60,6 @@ import com.oracle.graal.python.runtime.PythonContext; import com.oracle.graal.python.runtime.PythonOptions; import com.oracle.graal.python.runtime.object.PFactory; -import com.oracle.graal.python.util.LazySource; import com.oracle.graal.python.util.PythonUtils; import com.oracle.truffle.api.CompilerAsserts; import com.oracle.truffle.api.CompilerDirectives; @@ -95,7 +94,7 @@ public PCode execute(VirtualFrame frame, int argcount, PythonLanguage language = context.getLanguage(this); Object state = BoundaryCallContext.enter(frame, language, context, boundaryCallData); try { - return createCode(language, context, argcount, + return createCode(language, argcount, posonlyargcount, kwonlyargcount, nlocals, stacksize, flags, codedata, constants, names, varnames, freevars, cellvars, filename, name, qualname, firstlineno, linetable); @@ -105,7 +104,7 @@ public PCode execute(VirtualFrame frame, int argcount, } @TruffleBoundary - private static PCode createCode(PythonLanguage language, PythonContext context, int argCount, + private static PCode createCode(PythonLanguage language, int argCount, int positionalOnlyArgCount, int kwOnlyArgCount, int nlocals, int stacksize, int flags, byte[] codedata, Object[] constants, TruffleString[] names, @@ -141,14 +140,14 @@ private static PCode createCode(PythonLanguage language, PythonContext context, parameterNames, kwOnlyNames); } else { - ct = deserializeForBytecodeInterpreter(context, codedata, cellvars, freevars, flags); + ct = deserializeForBytecodeInterpreter(language, codedata, cellvars, freevars, flags); signature = ((PRootNode) ct.getRootNode()).getSignature(); } return PFactory.createCode(language, ct, signature, nlocals, stacksize, flags, constants, names, varnames, freevars, cellvars, filename, name, qualname, firstlineno, linetable); } - private static RootCallTarget deserializeForBytecodeInterpreter(PythonContext context, byte[] data, TruffleString[] cellvars, TruffleString[] freevars, int flags) { - CodeUnit codeUnit = MarshalModuleBuiltins.deserializeCodeUnit(null, context, data); + private static RootCallTarget deserializeForBytecodeInterpreter(PythonLanguage language, byte[] data, TruffleString[] cellvars, TruffleString[] freevars, int flags) { + CodeUnit codeUnit = MarshalModuleBuiltins.deserializeCodeUnit(null, language, data); RootNode rootNode; if (PythonOptions.ENABLE_BYTECODE_DSL_INTERPRETER) { @@ -156,7 +155,7 @@ private static RootCallTarget deserializeForBytecodeInterpreter(PythonContext co if (code.flags != flags) { code = code.withFlags(flags); } - rootNode = code.createRootNode(context, PythonUtils.createFakeSource()); + rootNode = code.createRootNode(language, PythonUtils.createFakeSource()); } else { BytecodeCodeUnit code = (BytecodeCodeUnit) codeUnit; if (cellvars != null && !Arrays.equals(code.cellvars, cellvars) || freevars != null && !Arrays.equals(code.freevars, freevars) || flags != code.flags) { @@ -168,9 +167,9 @@ private static RootCallTarget deserializeForBytecodeInterpreter(PythonContext co code.variableShouldUnbox, code.generalizeInputsKeys, code.generalizeInputsIndices, code.generalizeInputsValues, code.generalizeVarsIndices, code.generalizeVarsValues); } - rootNode = PBytecodeRootNode.create(context.getLanguage(), code, new LazySource(PythonUtils.createFakeSource()), false); + rootNode = PBytecodeRootNode.create(language, code, PythonUtils.createFakeSource(), false); if (code.isGeneratorOrCoroutine()) { - rootNode = new PBytecodeGeneratorFunctionRootNode(context.getLanguage(), rootNode.getFrameDescriptor(), (PBytecodeRootNode) rootNode, code.name); + rootNode = new PBytecodeGeneratorFunctionRootNode(language, rootNode.getFrameDescriptor(), (PBytecodeRootNode) rootNode, code.name); } } return PythonUtils.getOrCreateCallTarget(rootNode); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/code/PCode.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/code/PCode.java index 8115ecd087..1f10629bb3 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/code/PCode.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/code/PCode.java @@ -63,7 +63,6 @@ import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLRootNode; import com.oracle.graal.python.nodes.object.IsForeignObjectNode; import com.oracle.graal.python.runtime.GilNode; -import com.oracle.graal.python.runtime.PythonContext; import com.oracle.graal.python.runtime.PythonOptions; import com.oracle.graal.python.runtime.object.PFactory; import com.oracle.graal.python.runtime.sequence.storage.BoolSequenceStorage; @@ -506,7 +505,7 @@ public PCode getOrCreateChildCode(int index, BytecodeDSLCodeUnit codeUnit) { private PCode createCode(BytecodeDSLCodeUnit codeUnit) { PBytecodeDSLRootNode outerRootNode = (PBytecodeDSLRootNode) getRootNodeForExtraction(); PythonLanguage language = outerRootNode.getLanguage(); - RootCallTarget callTarget = language.createCachedCallTarget(l -> codeUnit.createRootNode(PythonContext.get(null), outerRootNode.getSource()), codeUnit); + RootCallTarget callTarget = language.createCachedCallTarget(l -> codeUnit.createRootNode(l, outerRootNode.getSource()), codeUnit); PBytecodeDSLRootNode rootNode = (PBytecodeDSLRootNode) callTarget.getRootNode(); return PFactory.createCode(language, callTarget, rootNode.getSignature(), codeUnit, getFilename()); } @@ -526,7 +525,7 @@ private PCode createCode(BytecodeCodeUnit codeUnit) { PBytecodeRootNode outerRootNode = (PBytecodeRootNode) getRootNodeForExtraction(); PythonLanguage language = outerRootNode.getLanguage(); RootCallTarget callTarget = language.createCachedCallTarget( - l -> PBytecodeRootNode.createMaybeGenerator(language, codeUnit, outerRootNode.getLazySource(), outerRootNode.isInternal()), codeUnit); + l -> PBytecodeRootNode.createMaybeGenerator(language, codeUnit, outerRootNode.getSource(), outerRootNode.isInternal()), codeUnit); RootNode rootNode = callTarget.getRootNode(); if (rootNode instanceof PBytecodeGeneratorFunctionRootNode generatorRoot) { rootNode = generatorRoot.getBytecodeRootNode(); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/RootNodeCompiler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/RootNodeCompiler.java index f56ca0a29d..1b45f71242 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/RootNodeCompiler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/RootNodeCompiler.java @@ -147,7 +147,6 @@ import com.oracle.graal.python.pegparser.sst.UnaryOpTy; import com.oracle.graal.python.pegparser.sst.WithItemTy; import com.oracle.graal.python.pegparser.tokenizer.SourceRange; -import com.oracle.graal.python.runtime.PythonContext; import com.oracle.graal.python.util.PythonUtils; import com.oracle.truffle.api.CompilerDirectives; import com.oracle.truffle.api.bytecode.BytecodeConfig; @@ -486,14 +485,14 @@ static class BytecodeSupplier extends BytecodeDSLCodeUnit.BytecodeSupplier { } @Override - public PBytecodeDSLRootNode createRootNode(PythonContext context, Source source) { + public PBytecodeDSLRootNode createRootNode(PythonLanguage language, Source source) { return nodes.getNode(0); } @Override - public byte[] createSerializedBytecode(PythonContext context) { + public byte[] createSerializedBytecode(PythonLanguage language) { try { - BytecodeSerializer serializer = new MarshalModuleBuiltins.PBytecodeDSLSerializer(context); + BytecodeSerializer serializer = new MarshalModuleBuiltins.PBytecodeDSLSerializer(language); ByteArrayOutputStream bytes = new ByteArrayOutputStream(); nodes.serialize(new DataOutputStream(bytes), serializer); return bytes.toByteArray(); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode/PBytecodeRootNode.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode/PBytecodeRootNode.java index 76108c5313..fef50c8cca 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode/PBytecodeRootNode.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode/PBytecodeRootNode.java @@ -62,8 +62,8 @@ import com.oracle.graal.python.PythonLanguage; import com.oracle.graal.python.builtins.PythonBuiltinClassType; import com.oracle.graal.python.builtins.modules.BuiltinFunctions.FormatNode; -import com.oracle.graal.python.builtins.modules.MarshalModuleBuiltins; import com.oracle.graal.python.builtins.modules.BuiltinFunctionsFactory.FormatNodeFactory.FormatNodeGen; +import com.oracle.graal.python.builtins.modules.MarshalModuleBuiltins; import com.oracle.graal.python.builtins.objects.PNone; import com.oracle.graal.python.builtins.objects.asyncio.GetAwaitableNode; import com.oracle.graal.python.builtins.objects.asyncio.GetAwaitableNodeGen; @@ -237,7 +237,6 @@ import com.oracle.graal.python.runtime.sequence.storage.LongSequenceStorage; import com.oracle.graal.python.runtime.sequence.storage.ObjectSequenceStorage; import com.oracle.graal.python.runtime.sequence.storage.SequenceStorage; -import com.oracle.graal.python.util.LazySource; import com.oracle.graal.python.util.PythonUtils; import com.oracle.truffle.api.Assumption; import com.oracle.truffle.api.CompilerAsserts; @@ -550,7 +549,7 @@ public final class PBytecodeRootNode extends PRootNode implements BytecodeOSRNod final int classcellIndex; private final BytecodeCodeUnit co; - private final LazySource lazySource; + private final Source source; private SourceSection sourceSection; // For deferred deprecation warnings private final ParserCallbacksImpl parserCallbacks; @@ -664,12 +663,12 @@ private static FrameDescriptor makeFrameDescriptor(BytecodeCodeUnit co, FrameInf } @TruffleBoundary - public static PBytecodeRootNode create(PythonLanguage language, BytecodeCodeUnit co, LazySource lazySource, boolean internal) { - return create(language, co, lazySource, internal, null); + public static PBytecodeRootNode create(PythonLanguage language, BytecodeCodeUnit co, Source source, boolean internal) { + return create(language, co, source, internal, null); } @TruffleBoundary - public static RootNode createMaybeGenerator(PythonLanguage language, BytecodeCodeUnit co, LazySource source, boolean internal) { + public static RootNode createMaybeGenerator(PythonLanguage language, BytecodeCodeUnit co, Source source, boolean internal) { PBytecodeRootNode bytecodeRootNode = PBytecodeRootNode.create(language, co, source, internal); if (co.isGeneratorOrCoroutine()) { return new PBytecodeGeneratorFunctionRootNode(language, bytecodeRootNode.getFrameDescriptor(), bytecodeRootNode, co.name); @@ -679,10 +678,10 @@ public static RootNode createMaybeGenerator(PythonLanguage language, BytecodeCod } @TruffleBoundary - public static PBytecodeRootNode create(PythonLanguage language, BytecodeCodeUnit co, LazySource lazySource, boolean internal, ParserCallbacksImpl parserCallbacks) { + public static PBytecodeRootNode create(PythonLanguage language, BytecodeCodeUnit co, Source source, boolean internal, ParserCallbacksImpl parserCallbacks) { BytecodeFrameInfo frameInfo = new BytecodeFrameInfo(); FrameDescriptor fd = makeFrameDescriptor(co, frameInfo); - PBytecodeRootNode rootNode = new PBytecodeRootNode(language, fd, co.computeSignature(), co, lazySource, internal, parserCallbacks); + PBytecodeRootNode rootNode = new PBytecodeRootNode(language, fd, co.computeSignature(), co, source, internal, parserCallbacks); PythonContext context = PythonContext.get(rootNode); if (context != null && context.getOption(PythonOptions.EagerlyMaterializeInstrumentationNodes)) { rootNode.adoptChildren(); @@ -693,14 +692,14 @@ public static PBytecodeRootNode create(PythonLanguage language, BytecodeCodeUnit } @TruffleBoundary - private PBytecodeRootNode(PythonLanguage language, FrameDescriptor fd, Signature sign, BytecodeCodeUnit co, LazySource source, boolean internal, + private PBytecodeRootNode(PythonLanguage language, FrameDescriptor fd, Signature sign, BytecodeCodeUnit co, Source source, boolean internal, ParserCallbacksImpl parserCallbacks) { super(language, fd); this.celloffset = co.varnames.length; this.freeoffset = celloffset + co.cellvars.length; this.stackoffset = freeoffset + co.freevars.length; this.bcioffset = stackoffset + co.stacksize; - this.lazySource = source; + this.source = source; this.internal = internal; this.parserCallbacks = parserCallbacks; this.signature = sign; @@ -788,12 +787,12 @@ public BytecodeCodeUnit getCodeUnit() { return co; } - public Source getSource() { - return lazySource.getSource(); + public Source getSourceWithCharacters() { + return getLanguage().getOrCreateSourceWithContent(source); } - public LazySource getLazySource() { - return lazySource; + public Source getSource() { + return source; } public byte[] getBytecode() { @@ -2920,7 +2919,7 @@ public void materializeContainedFunctionsForInstrumentation(Set PBytecodeRootNode.createMaybeGenerator(language, codeUnit, lazySource, isInternal()), + l -> PBytecodeRootNode.createMaybeGenerator(language, codeUnit, getSource(), isInternal()), codeUnit); RootNode rootNode = callTarget.getRootNode(); if (rootNode instanceof PBytecodeGeneratorFunctionRootNode) { @@ -6163,11 +6162,11 @@ public boolean frameIsVisibleToPython() { public SourceSection getSourceSection() { if (sourceSection != null) { return sourceSection; - } else if (!getSource().hasCharacters()) { - sourceSection = getSource().createUnavailableSection(); + } else if (!getSourceWithCharacters().hasCharacters()) { + sourceSection = getSourceWithCharacters().createUnavailableSection(); return sourceSection; } else { - sourceSection = co.getSourceSection(getSource()); + sourceSection = co.getSourceSection(getSourceWithCharacters()); return sourceSection; } } @@ -6231,7 +6230,7 @@ protected byte[] extractCode(Node node) { * * TODO We should revisit this when the AST interpreter is removed. */ - return MarshalModuleBuiltins.serializeCodeUnit(null, PythonContext.get(this), co); + return MarshalModuleBuiltins.serializeCodeUnit(null, getLanguage(), co); } @Override @@ -6242,7 +6241,7 @@ protected boolean isCloneUninitializedSupported() { @Override protected RootNode cloneUninitialized() { // Note: the bytecode might be quickened already, it's not practical to undo it - PBytecodeRootNode rootNode = new PBytecodeRootNode(getLanguage(), getFrameDescriptor(), getSignature(), co, lazySource, internal, parserCallbacks); + PBytecodeRootNode rootNode = new PBytecodeRootNode(getLanguage(), getFrameDescriptor(), getSignature(), co, source, internal, parserCallbacks); rootNode.variableTypes = variableTypes; return rootNode; } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode/instrumentation/InstrumentationRootImpl.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode/instrumentation/InstrumentationRootImpl.java index 4d4be9b7b3..9f4c734598 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode/instrumentation/InstrumentationRootImpl.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode/instrumentation/InstrumentationRootImpl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2022, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -66,7 +66,7 @@ public InstrumentableNode materializeInstrumentableNodes(Set WarnExperimentalFeatures = new OptionKey<>(true); - @Option(category = OptionCategory.EXPERT, usageSyntax = "true|false", help = """ + @EngineOption @Option(category = OptionCategory.EXPERT, usageSyntax = "true|false", help = """ By default GraalPy only keeps a transformed form of bytecode in memory and may need to reread bytecode files when a different form of bytecode is requested, \ such as when settrace instrumentation is enabled. This option avoids rereading bytecode files by keeping the original bytecode form in memory""") // public static final OptionKey KeepBytecodeInMemory = new OptionKey<>(false); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/util/LazySource.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/util/LazySource.java deleted file mode 100644 index 6d190ff6bb..0000000000 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/util/LazySource.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * The Universal Permissive License (UPL), Version 1.0 - * - * Subject to the condition set forth below, permission is hereby granted to any - * person obtaining a copy of this software, associated documentation and/or - * data (collectively the "Software"), free of charge and under any and all - * copyright rights in the Software, and any and all patent rights owned or - * freely licensable by each licensor hereunder covering either (i) the - * unmodified Software as contributed to or provided by such licensor, or (ii) - * the Larger Works (as defined below), to deal in both - * - * (a) the Software, and - * - * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if - * one is included with the Software each a "Larger Work" to which the Software - * is contributed by such licensors), - * - * without restriction, including without limitation the rights to copy, create - * derivative works of, display, perform, and distribute the Software and make, - * use, sell, offer for sale, import, export, have made, and have sold the - * Software and the Larger Work(s), and to sublicense the foregoing rights on - * either these or other terms. - * - * This license is subject to the following condition: - * - * The above copyright notice and either this complete permission notice or at a - * minimum a reference to the UPL must be included in all copies or substantial - * portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -package com.oracle.graal.python.util; - -import java.io.IOException; - -import com.oracle.truffle.api.source.Source; - -public final class LazySource { - private Source.SourceBuilder builder; - private Source source; - - public LazySource(Source.SourceBuilder builder) { - this.builder = builder; - } - - public LazySource(Source source) { - this.source = source; - } - - public Source getSource() { - if (source == null) { - try { - source = builder.build(); - } catch (IOException e) { - source = builder.content(Source.CONTENT_NONE).build(); - } - builder = null; - } - return source; - } -}