From 5edd93e7e83e6169324384746c75e22b8a06092b Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Thu, 24 Sep 2026 16:53:29 +0200 Subject: [PATCH 01/39] Reduce temporary bytearray allocations in PPickler --- .../builtins/modules/pickle/PPickler.java | 513 +++++++++--------- .../builtins/modules/pickle/PickleUtils.java | 76 --- 2 files changed, 246 insertions(+), 343 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java index de1e0508e8..b27219e9f2 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java @@ -49,7 +49,6 @@ import static com.oracle.graal.python.nodes.SpecialMethodNames.T___REDUCE_EX__; import static com.oracle.graal.python.nodes.SpecialMethodNames.T___REDUCE__; import static com.oracle.graal.python.nodes.StringLiterals.T_NEWLINE; -import static com.oracle.graal.python.nodes.StringLiterals.T_UTF8; import static com.oracle.graal.python.nodes.statement.AbstractImportNode.importModule; import static com.oracle.graal.python.runtime.exception.PythonErrorType.KeyError; import static com.oracle.graal.python.runtime.exception.PythonErrorType.OverflowError; @@ -71,6 +70,7 @@ import com.oracle.graal.python.builtins.objects.PNotImplemented; import com.oracle.graal.python.builtins.objects.buffer.PythonBufferAccessLibrary; import com.oracle.graal.python.builtins.objects.buffer.PythonBufferAcquireLibrary; +import com.oracle.graal.python.builtins.objects.bytes.BytesUtils; import com.oracle.graal.python.builtins.objects.bytes.PBytes; import com.oracle.graal.python.builtins.objects.common.HashingStorage; import com.oracle.graal.python.builtins.objects.common.HashingStorageNodes.HashingStorageIterator; @@ -124,6 +124,7 @@ import com.oracle.truffle.api.frame.VirtualFrame; import com.oracle.truffle.api.nodes.Node; import com.oracle.truffle.api.object.Shape; +import com.oracle.truffle.api.strings.TranscodingErrorHandler; import com.oracle.truffle.api.strings.TruffleString; public class PPickler extends PythonBuiltinObject { @@ -471,12 +472,85 @@ public static void handleError(PPickler pickler) { protected void writeASCII(PPickler pickler, TruffleString string) { assert string.getCodeRangeUncached(TS_ENCODING) == TruffleString.CodeRange.ASCII; - final byte[] bytes = PythonUtils.getAsciiBytes(string, ensureTsCopyToByteArrayNode(), ensureTsSwitchEncodingNode()); - write(pickler, bytes, bytes.length); + TruffleString encoded = ensureTsSwitchEncodingNode().execute(string, TruffleString.Encoding.US_ASCII); + int length = encoded.byteLength(TruffleString.Encoding.US_ASCII); + ensureBufferSpace(pickler, length); + ensureTsCopyToByteArrayNode().execute(encoded, 0, pickler.outputBuffer, pickler.outputLen, length, TruffleString.Encoding.US_ASCII); + pickler.outputLen += length; + } + + protected void writeUtf8(PPickler pickler, TruffleString string) { + TruffleString encoded = ensureTsSwitchEncodingNode().execute(string, TruffleString.Encoding.UTF_8, TranscodingErrorHandler.DEFAULT_KEEP_SURROGATES_IN_UTF8); + int length = encoded.byteLength(TruffleString.Encoding.UTF_8); + ensureBufferSpace(pickler, length); + ensureTsCopyToByteArrayNode().execute(encoded, 0, pickler.outputBuffer, pickler.outputLen, length, TruffleString.Encoding.UTF_8); + pickler.outputLen += length; + } + + protected void writeLongAsString(PPickler pickler, byte opcode, long value) { + TruffleString string = ensureTsFromLongNode().execute(value, TruffleString.Encoding.US_ASCII, true); + TruffleString.Encoding encoding = TruffleString.Encoding.US_ASCII; + int length = string.byteLength(encoding); + ensureBufferSpace(pickler, length + 2); + pickler.outputBuffer[pickler.outputLen++] = opcode; + ensureTsCopyToByteArrayNode().execute(string, 0, pickler.outputBuffer, pickler.outputLen, length, encoding); + pickler.outputLen += length; + pickler.outputBuffer[pickler.outputLen++] = NEW_LINE_BYTE; + } + + protected void write(PPickler pickler, byte b) { + ensureBufferSpace(pickler, 1); + pickler.outputBuffer[pickler.outputLen++] = b; + } + + protected void write(PPickler pickler, byte b1, byte b2) { + ensureBufferSpace(pickler, 2); + pickler.outputBuffer[pickler.outputLen++] = b1; + pickler.outputBuffer[pickler.outputLen++] = b2; + } + + protected void writeBuffer(PPickler pickler, Object buffer, int size, PythonBufferAccessLibrary bufferLib) { + ensureBufferSpace(pickler, size); + bufferLib.readIntoByteArray(buffer, 0, pickler.outputBuffer, pickler.outputLen, size); + pickler.outputLen += size; + } + + protected static byte intByte1(long value) { + return (byte) (value & 0xff); + } + + protected static byte intByte2(long value) { + return (byte) ((value >> 8) & 0xff); + } + + protected static byte intByte3(long value) { + return (byte) ((value >> 16) & 0xff); + } + + protected static byte intByte4(long value) { + return (byte) ((value >> 24) & 0xff); + } + + protected void writeByteOp(PPickler pickler, byte opcode, long value) { + ensureBufferSpace(pickler, 2); + pickler.outputBuffer[pickler.outputLen++] = opcode; + pickler.outputBuffer[pickler.outputLen++] = intByte1(value); + } + + protected void writeShortOp(PPickler pickler, byte opcode, long value) { + ensureBufferSpace(pickler, 3); + pickler.outputBuffer[pickler.outputLen++] = opcode; + pickler.outputBuffer[pickler.outputLen++] = intByte1(value); + pickler.outputBuffer[pickler.outputLen++] = intByte2(value); } - protected void write(PPickler pickler, byte oneByte) { - write(pickler, new byte[]{oneByte}, 1); + protected void writeIntOp(PPickler pickler, byte opcode, long value) { + ensureBufferSpace(pickler, 5); + pickler.outputBuffer[pickler.outputLen++] = opcode; + pickler.outputBuffer[pickler.outputLen++] = intByte1(value); + pickler.outputBuffer[pickler.outputLen++] = intByte2(value); + pickler.outputBuffer[pickler.outputLen++] = intByte3(value); + pickler.outputBuffer[pickler.outputLen++] = intByte4(value); } protected void write(PPickler pickler, byte[] bytes) { @@ -484,6 +558,12 @@ protected void write(PPickler pickler, byte[] bytes) { } protected void write(PPickler pickler, byte[] bytes, int dataLen) { + ensureBufferSpace(pickler, dataLen); + PythonUtils.arraycopy(bytes, 0, pickler.outputBuffer, pickler.outputLen, dataLen); + pickler.outputLen += dataLen; + } + + public void ensureBufferSpace(PPickler pickler, int dataLen) { boolean needNewFrame = pickler.isFraming() && pickler.frameStart == -1; int n = (needNewFrame) ? dataLen + PickleUtils.FRAME_HEADER_SIZE : dataLen; int required = pickler.outputLen + n; @@ -495,7 +575,7 @@ protected void write(PPickler pickler, byte[] bytes, int dataLen) { throw raise(PythonBuiltinClassType.MemoryError); } pickler.maxOutputLen = (pickler.outputLen + n) / 2 * 3; - pickler.outputBuffer = PickleUtils.resize(pickler.outputBuffer, pickler.maxOutputLen); + pickler.outputBuffer = PythonUtils.arrayCopyOf(pickler.outputBuffer, pickler.maxOutputLen); } if (needNewFrame) { int frameStart = pickler.outputLen; @@ -506,13 +586,10 @@ protected void write(PPickler pickler, byte[] bytes, int dataLen) { } pickler.outputLen += PickleUtils.FRAME_HEADER_SIZE; } - PythonUtils.arraycopy(bytes, 0, pickler.outputBuffer, pickler.outputLen, dataLen); - pickler.outputLen += dataLen; } - protected void writeBytes(VirtualFrame frame, PPickler pickler, byte[] header, int headerSize, byte[] data, int dataSize, Object payload) { + protected boolean bypassBuffer(PPickler pickler, int dataSize) { boolean bypassBuffer = dataSize >= PickleUtils.FRAME_SIZE_TARGET; - boolean framing = pickler.framing; if (bypassBuffer) { assert pickler.outputBuffer != null; @@ -521,28 +598,7 @@ protected void writeBytes(VirtualFrame frame, PPickler pickler, byte[] header, i // Disable framing temporarily pickler.framing = false; } - - write(pickler, header, headerSize); - - if (bypassBuffer && pickler.write != null) { - // Dump the output buffer to the file. - flushToFile(frame, pickler); - // Stream write the payload into the file without going through the output buffer. - Object pld = payload; - if (pld == null) { - // TODO: It would be better to use a memoryview with a linked original string if - // this is possible. - pld = PFactory.createBytes(PythonLanguage.get(this), data, dataSize); - } - getCallNode().execute(frame, pickler.write, pld); - // Reinitialize the buffer for subsequent calls to _Pickler_Write. - pickler.clearBuffer(); - } else { - write(pickler, data, dataSize); - } - - // Re-enable framing for subsequent calls to _Pickler_Write. - pickler.framing = framing; + return bypassBuffer; } protected PTuple createTuple(Object... items) { @@ -880,32 +936,19 @@ private boolean savePers(VirtualFrame frame, PPickler pickler, Object obj) { // memo methods private void memoGet(PPickler pickler, int value) { - int len; - byte[] pdata; if (pickler.isBin()) { - pdata = new byte[5]; if (value < 256) { - pdata[0] = PickleUtils.OPCODE_BINGET; - pdata[1] = (byte) (value & 0xff); - len = 2; + writeByteOp(pickler, PickleUtils.OPCODE_BINGET, value); } else if (Long.compareUnsigned(value, 0xffffffffL) <= 0) { - pdata[0] = PickleUtils.OPCODE_LONG_BINGET; - pdata[1] = (byte) (value & 0xff); - pdata[2] = (byte) ((value >> 8) & 0xff); - pdata[3] = (byte) ((value >> 16) & 0xff); - pdata[4] = (byte) ((value >> 24) & 0xff); - len = 5; + writeIntOp(pickler, PickleUtils.OPCODE_LONG_BINGET, value); } else { // unlikely throw raise(PicklingError, ErrorMessages.MEMO_ID_TOO_LARGE_FOR_S, "LONG_BINGET"); } } else { - pdata = new byte[30]; - pdata[0] = PickleUtils.OPCODE_GET; - len = PickleUtils.toAsciiBytesWithNewLine(pdata, 1, value, ensureTsFromLongNode(), ensureTsCopyToByteArrayNode()); + writeLongAsString(pickler, PickleUtils.OPCODE_GET, value); } - write(pickler, pdata, len); } private void memoPut(PPickler pickler, Object obj) { @@ -918,33 +961,19 @@ private void memoPut(PPickler pickler, Object obj) { if (pickler.proto >= 4) { write(pickler, PickleUtils.OPCODE_MEMOIZE); - return; } else { - byte[] pdata; - int len; if (!pickler.isBin()) { - pdata = new byte[30]; - pdata[0] = PickleUtils.OPCODE_PUT; - len = PickleUtils.toAsciiBytesWithNewLine(pdata, 1, idx, ensureTsFromLongNode(), ensureTsCopyToByteArrayNode()); + writeLongAsString(pickler, PickleUtils.OPCODE_PUT, idx); } else { - pdata = new byte[5]; if (idx < 256) { - pdata[0] = PickleUtils.OPCODE_BINPUT; - pdata[1] = (byte) idx; - len = 2; + writeByteOp(pickler, PickleUtils.OPCODE_BINPUT, idx); } else if (Long.compareUnsigned(idx, 0xffffffffL) <= 0) { - pdata[0] = PickleUtils.OPCODE_LONG_BINPUT; - pdata[1] = (byte) (idx & 0xff); - pdata[2] = (byte) ((idx >> 8) & 0xff); - pdata[3] = (byte) ((idx >> 16) & 0xff); - pdata[4] = (byte) ((idx >> 24) & 0xff); - len = 5; + writeIntOp(pickler, PickleUtils.OPCODE_LONG_BINPUT, idx); } else { // unlikely throw raise(PicklingError, ErrorMessages.MEMO_ID_TOO_LARGE_FOR_S, "LONG_BINPUT"); } } - write(pickler, pdata, len); } } @@ -1205,41 +1234,20 @@ private void saveBool(VirtualFrame frame, PPickler pickler, Object obj) { private void saveLong(VirtualFrame frame, PPickler pickler, Object obj) { TruffleString repr; try { - long value = asLong(frame, obj); - if (value <= 0x7fffffffL && value >= (-0x7fffffffL - 1)) { - // result fits in a signed 4-byte integer. Note: we can't use -0x80000000L in - // the above condition because some compilers (e.g., MSVC) will promote - // 0x80000000L to an unsigned type before applying the unary minus when - // sizeof(long) <= 4. The resulting value stays unsigned which is commonly not - // what we want, so MSVC happily warns us about it. However, that result would - // have been fine because we guard for sizeof(long) <= 4 which turns the - // condition true in that particular case. - byte[] pdata; - int len; - + long longValue = asLong(frame, obj); + int value = (int) longValue; + if (value == longValue) { if (pickler.isBin()) { - pdata = new byte[5]; - pdata[1] = (byte) (value & 0xff); - pdata[2] = (byte) ((value >> 8) & 0xff); - pdata[3] = (byte) ((value >> 16) & 0xff); - pdata[4] = (byte) ((value >> 24) & 0xff); - - if ((pdata[4] != 0) || (pdata[3] != 0)) { - pdata[0] = PickleUtils.OPCODE_BININT; - len = 5; - } else if (pdata[2] != 0) { - pdata[0] = PickleUtils.OPCODE_BININT2; - len = 3; + if (value >= 0 && value < (1 << 8)) { + writeByteOp(pickler, PickleUtils.OPCODE_BININT1, value); + } else if (value >= 0 && value < (1 << 16)) { + writeShortOp(pickler, PickleUtils.OPCODE_BININT2, value); } else { - pdata[0] = PickleUtils.OPCODE_BININT1; - len = 2; + writeIntOp(pickler, PickleUtils.OPCODE_BININT, value); } } else { - pdata = new byte[32]; - pdata[0] = PickleUtils.OPCODE_INT; - len = PickleUtils.toAsciiBytesWithNewLine(pdata, 1, value, ensureTsFromLongNode(), ensureTsCopyToByteArrayNode()); + writeLongAsString(pickler, PickleUtils.OPCODE_INT, longValue); } - write(pickler, pdata, len); return; } } catch (PException e) { @@ -1247,14 +1255,9 @@ private void saveLong(VirtualFrame frame, PPickler pickler, Object obj) { } if (pickler.proto >= 2) { - byte[] header = new byte[5]; // Linear-time pickling. final int sign = getSign(obj); - if (sign == 0) { - header[0] = PickleUtils.OPCODE_LONG1; - header[1] = 0; - write(pickler, header, 2); - } + assert sign != 0; // Zero was handled by the int code above final int nbits = getNumBits(obj); // How many bytes do we need? There are nbits >> 3 full bytes of data, and nbits & 7 @@ -1276,22 +1279,12 @@ private void saveLong(VirtualFrame frame, PPickler pickler, Object obj) { if (sign < 0 && nbytes > 1 && pdata[nbytes - 1] == (byte) 0xff && (pdata[nbytes - 2] & 0x80) != 0) { nbytes--; } - int size; if (nbytes < 256) { - header[0] = PickleUtils.OPCODE_LONG1; - header[1] = (byte) nbytes; - size = 2; + writeByteOp(pickler, PickleUtils.OPCODE_LONG1, nbytes); } else { - header[0] = PickleUtils.OPCODE_LONG4; - size = nbytes; - for (int i = 1; i < 5; i++) { - header[i] = (byte) (size & 0xff); - size >>= 8; - } - size = 5; + writeIntOp(pickler, PickleUtils.OPCODE_LONG4, nbytes); } - write(pickler, header, size); write(pickler, pdata, nbytes); } else { @@ -1305,10 +1298,10 @@ private void saveLong(VirtualFrame frame, PPickler pickler, Object obj) { private void saveFloat(VirtualFrame frame, PPickler pickler, Object obj, Node inliningTarget, PyFloatAsDoubleNode asDoubleNode) { final double value = asDoubleNode.execute(frame, inliningTarget, obj); if (pickler.isBin()) { - byte[] pdata = new byte[9]; - pdata[0] = PickleUtils.OPCODE_BINFLOAT; - NumericSupport.bigEndian().putDouble(pdata, 1, value); - write(pickler, pdata, 9); + write(pickler, PickleUtils.OPCODE_BINFLOAT); + ensureBufferSpace(pickler, 8); + NumericSupport.bigEndian().putDouble(pickler.outputBuffer, pickler.outputLen, value); + pickler.outputLen += 8; } else { write(pickler, PickleUtils.OPCODE_FLOAT); TruffleString repr = PickleUtils.doubleToAsciiString(value); @@ -1319,6 +1312,7 @@ private void saveFloat(VirtualFrame frame, PPickler pickler, Object obj, Node in private void saveBytes(VirtualFrame frame, PythonContext ctx, PPickler pickler, Object obj, InteropCallData interopCallData) { Object buffer = getBufferAcquireLibrary().acquireReadonly(obj, frame, interopCallData); + PythonBufferAccessLibrary bufferLib = getBufferLibrary(); try { if (pickler.proto < 3) { // Older pickle protocols do not have an opcode for pickling bytes objects. @@ -1333,107 +1327,119 @@ private void saveBytes(VirtualFrame frame, PythonContext ctx, PPickler pickler, // need to do this with newer protocols. Object reduceValue; - if (getBufferLibrary().getBufferLength(buffer) == 0) { + if (bufferLib.getBufferLength(buffer) == 0) { reduceValue = createTuple(ctx.getCore().lookupType(PythonBuiltinClassType.PBytes), createTuple()); } else { PickleState st = getGlobalState(ctx.getCore()); - final TruffleString unicodeStr = PickleUtils.decodeLatin1Strict(getBufferLibrary().getCopiedByteArray(buffer), ensureTsFromByteArrayWithCompaction()); + final TruffleString unicodeStr = PickleUtils.decodeLatin1Strict(bufferLib.getCopiedByteArray(buffer), ensureTsFromByteArrayWithCompaction()); reduceValue = createTuple(st.codecsEncode, createTuple(unicodeStr, LATIN1)); } // save_reduce() will memoize the object automatically. saveReduce(frame, ctx, pickler, reduceValue, obj); } else { - byte[] bytes = getBufferLibrary().getCopiedByteArray(buffer); - saveBytesData(frame, pickler, obj, bytes, bytes.length); + saveBytesData(frame, pickler, obj, buffer, bufferLib); } } finally { - getBufferLibrary().release(buffer, frame, interopCallData); + bufferLib.release(buffer, frame, interopCallData); } } - private void saveBytesData(VirtualFrame frame, PPickler pickler, Object obj, byte[] data, int size) { + private void saveBytesData(VirtualFrame frame, PPickler pickler, Object obj, Object buffer, PythonBufferAccessLibrary bufferLib) { assert pickler.proto >= 3; - byte[] header = new byte[9]; - int len; + int size = bufferLib.getBufferLength(buffer); + boolean wasFraming = pickler.framing; + boolean bypassBuffer = bypassBuffer(pickler, size); if (size <= 0xff) { - header[0] = PickleUtils.OPCODE_SHORT_BINBYTES; - header[1] = (byte) size; - len = 2; - } else if (Long.compareUnsigned(size, 0xffffffffL) < 0) { - header[0] = PickleUtils.OPCODE_BINBYTES; - header[1] = (byte) (size & 0xff); - header[2] = (byte) ((size >> 8) & 0xff); - header[3] = (byte) ((size >> 16) & 0xff); - header[4] = (byte) ((size >> 24) & 0xff); - len = 5; - } else if (pickler.proto >= 4) { - header[0] = PickleUtils.OPCODE_BINBYTES8; - PickleUtils.writeSize64(header, 1, size); - len = 9; + writeByteOp(pickler, PickleUtils.OPCODE_SHORT_BINBYTES, size); } else { - throw raise(OverflowError, ErrorMessages.SER_OVER_4GB); + writeIntOp(pickler, PickleUtils.OPCODE_BINBYTES, size); } - writeBytes(frame, pickler, header, len, data, size, obj); + if (bypassBuffer && pickler.write != null) { + flushToFile(frame, pickler); + getCallNode().execute(frame, pickler.write, obj); + pickler.clearBuffer(); + } else { + writeBuffer(pickler, buffer, size, bufferLib); + } + + if (bypassBuffer && wasFraming) { + pickler.framing = true; + } memoPut(pickler, obj); } private void writeUnicodeBinary(VirtualFrame frame, PPickler pickler, Object obj) { - Object encoded = null; - byte[] header = new byte[9]; - int len; - byte[] data = PickleUtils.encodeUTF8Strict(asStringStrict(obj), ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode(), ensureTsGetCodeRangeNode()); - - if (data == null) { - // Issue #8383: for strings with lone surrogates, fallback on the "surrogatepass" - // error handler. - encoded = getItem(frame, encode(frame, obj, T_UTF8, T_ERRORS_SURROGATEPASS), 0); - // Checkstyle: stop - //@formatter:off - // data = PickleUtils.encodeUTF8Strict(asStringStrict(encoded), ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode(), ensureTsGetCodeRangeNode()); - // Checkstyle: start - //@formatter:on - // TODO: [GR-39571] TruffleStrings: allow preservation of UTF-16 surrogate - data = toBytes(frame, encoded); - } - - int size = data.length; - - if (size <= 0xff && pickler.proto >= 4) { - header[0] = PickleUtils.OPCODE_SHORT_BINUNICODE; - header[1] = (byte) (size & 0xff); - len = 2; - } else if (Long.compareUnsigned(size, 0xffffffffL) <= 0) { - header[0] = PickleUtils.OPCODE_BINUNICODE; - header[1] = (byte) (size & 0xff); - header[2] = (byte) ((size >> 8) & 0xff); - header[3] = (byte) ((size >> 16) & 0xff); - header[4] = (byte) ((size >> 24) & 0xff); - len = 5; - } else if (pickler.proto >= 4) { - header[0] = PickleUtils.OPCODE_BINUNICODE8; - PickleUtils.writeSize64(header, 1, size); - len = 9; + TruffleString string = asStringStrict(obj); + TruffleString utf8 = ensureTsSwitchEncodingNode().execute(string, TruffleString.Encoding.UTF_8, TranscodingErrorHandler.DEFAULT_KEEP_SURROGATES_IN_UTF8); + int size = utf8.byteLength(TruffleString.Encoding.UTF_8); + boolean wasFraming = pickler.framing; + boolean bypassBuffer = bypassBuffer(pickler, size); + + if (bypassBuffer && pickler.write != null) { + writeIntOp(pickler, PickleUtils.OPCODE_BINUNICODE, size); + byte[] data = new byte[size]; + ensureTsCopyToByteArrayNode().execute(utf8, 0, data, 0, size, TruffleString.Encoding.UTF_8); + flushToFile(frame, pickler); + getCallNode().execute(frame, pickler.write, PFactory.createBytes(PythonLanguage.get(this), data, size)); + pickler.clearBuffer(); } else { - throw raise(OverflowError, ErrorMessages.SER_OVER_4GB); + if (size <= 0xff && pickler.proto >= 4) { + writeByteOp(pickler, PickleUtils.OPCODE_SHORT_BINUNICODE, size); + } else { + writeIntOp(pickler, PickleUtils.OPCODE_BINUNICODE, size); + } + writeUtf8(pickler, utf8); } - writeBytes(frame, pickler, header, len, data, size, encoded); + if (bypassBuffer && wasFraming) { + pickler.framing = true; + } } private void saveUnicode(VirtualFrame frame, PPickler pickler, Object obj) { if (pickler.isBin()) { writeUnicodeBinary(frame, pickler, obj); } else { - byte[] encoded = PickleUtils.rawUnicodeEscape(asStringStrict(obj), ensureTsCodePointLengthNode(), ensureTsCodePointAtIndexUTF32Node()); write(pickler, PickleUtils.OPCODE_UNICODE); - write(pickler, encoded); + writeRawUnicodeEscape(pickler, asStringStrict(obj)); writeASCII(pickler, T_NEWLINE); } memoPut(pickler, obj); } + private void writeRawUnicodeEscape(PPickler pickler, TruffleString string) { + int codePointLength = ensureTsCodePointLengthNode().execute(string, TS_ENCODING); + long escapedLength = 0; + for (int i = 0; i < codePointLength; i++) { + int ch = ensureTsCodePointAtIndexUTF32Node().execute(string, i); + escapedLength += ch >= 0x10000 ? 10 : ch >= 256 || ch == '\\' || ch == 0 || ch == '\n' || ch == '\r' || ch == 0x1a ? 6 : 1; + } + if (escapedLength > Integer.MAX_VALUE) { + throw raise(PythonBuiltinClassType.MemoryError); + } + ensureBufferSpace(pickler, (int) escapedLength); + for (int i = 0; i < codePointLength; i++) { + int ch = ensureTsCodePointAtIndexUTF32Node().execute(string, i); + if (ch >= 0x10000) { + pickler.outputBuffer[pickler.outputLen++] = '\\'; + pickler.outputBuffer[pickler.outputLen++] = 'U'; + for (int shift = 28; shift >= 0; shift -= 4) { + pickler.outputBuffer[pickler.outputLen++] = BytesUtils.HEXDIGITS[(ch >> shift) & 0xf]; + } + } else if (ch >= 256 || ch == '\\' || ch == 0 || ch == '\n' || ch == '\r' || ch == 0x1a) { + pickler.outputBuffer[pickler.outputLen++] = '\\'; + pickler.outputBuffer[pickler.outputLen++] = 'u'; + for (int shift = 12; shift >= 0; shift -= 4) { + pickler.outputBuffer[pickler.outputLen++] = BytesUtils.HEXDIGITS[(ch >> shift) & 0xf]; + } + } else { + pickler.outputBuffer[pickler.outputLen++] = (byte) ch; + } + } + } + private void batchDictExact(VirtualFrame frame, PPickler pickler, PDict dict) { final HashingStorage storage = dict.getDictStorage(); final int length = getHashingStorageLength(storage); @@ -1466,24 +1472,17 @@ private void batchDict(VirtualFrame frame, PPickler pickler, Object iterator) { } private void saveDict(VirtualFrame frame, Node inliningTarget, PyObjectCallMethodObjArgs callMethod, PPickler pickler, Object obj) { - byte[] header = new byte[3]; - int len; - if (pickler.isFast()) { fastSaveEnter(pickler, obj); } // Create an empty dict. if (pickler.isBin()) { - header[0] = PickleUtils.OPCODE_EMPTY_DICT; - len = 1; + write(pickler, PickleUtils.OPCODE_EMPTY_DICT); } else { - header[0] = PickleUtils.OPCODE_MARK; - header[1] = PickleUtils.OPCODE_DICT; - len = 2; + write(pickler, PickleUtils.OPCODE_MARK, PickleUtils.OPCODE_DICT); } - write(pickler, header, len); memoPut(pickler, obj); if (length(frame, obj) > 0) { @@ -1630,25 +1629,18 @@ private void batchList(VirtualFrame frame, PPickler pickler, Object iterator) { } private void saveList(VirtualFrame frame, PPickler pickler, Object obj) { - byte[] header = new byte[3]; - int len; - if (pickler.isFast()) { fastSaveEnter(pickler, obj); } // Create an empty list. if (pickler.isBin()) { - header[0] = PickleUtils.OPCODE_EMPTY_LIST; - len = 1; + write(pickler, PickleUtils.OPCODE_EMPTY_LIST); } else { - header[0] = PickleUtils.OPCODE_MARK; - header[1] = PickleUtils.OPCODE_LIST; - len = 2; + write(pickler, PickleUtils.OPCODE_MARK, PickleUtils.OPCODE_LIST); } - write(pickler, header, len); - len = length(frame, obj); + int len = length(frame, obj); memoPut(pickler, obj); if (len != 0) { @@ -1678,18 +1670,11 @@ private void saveTuple(VirtualFrame frame, PPickler pickler, Object obj) { int len = length(frame, obj); if (len == 0) { - byte[] pdata = new byte[2]; - if (pickler.proto != 0) { - pdata[0] = PickleUtils.OPCODE_EMPTY_TUPLE; - len = 1; + write(pickler, PickleUtils.OPCODE_EMPTY_TUPLE); } else { - pdata[0] = PickleUtils.OPCODE_MARK; - pdata[1] = PickleUtils.OPCODE_TUPLE; - len = 2; + write(pickler, PickleUtils.OPCODE_MARK, PickleUtils.OPCODE_TUPLE); } - - write(pickler, pdata, len); return; } @@ -1744,56 +1729,65 @@ private void saveTuple(VirtualFrame frame, PPickler pickler, Object obj) { memoPut(pickler, obj); } - private void saveBytearrayData(VirtualFrame frame, PPickler pickler, Object obj, byte[] data, int size) { + private void saveBytearrayData(VirtualFrame frame, PPickler pickler, Object obj, Object buffer, PythonBufferAccessLibrary bufferLib) { assert pickler.proto >= 5; - if (size < 0) { - return; - } - byte[] header = new byte[9]; + int size = bufferLib.getBufferLength(buffer); + boolean wasFraming = pickler.framing; + boolean bypassBuffer = bypassBuffer(pickler, size); + + // Our sizes are ints, but the protocol expects 8 bytes + ensureBufferSpace(pickler, 9); + writeIntOp(pickler, PickleUtils.OPCODE_BYTEARRAY8, size); + pickler.outputLen += 4; - header[0] = PickleUtils.OPCODE_BYTEARRAY8; - PickleUtils.writeSize64(header, 1, size); + if (bypassBuffer && pickler.write != null) { + flushToFile(frame, pickler); + getCallNode().execute(frame, pickler.write, obj); + pickler.clearBuffer(); + } else { + writeBuffer(pickler, buffer, size, bufferLib); + } - int len = 9; - writeBytes(frame, pickler, header, len, data, size, obj); + if (bypassBuffer && wasFraming) { + pickler.framing = true; + } memoPut(pickler, obj); } private void saveBytearray(VirtualFrame frame, Node inliningTarget, PythonContext ctx, PPickler pickler, Object obj, InteropCallData interopCallData) { Object buffer = getBufferAcquireLibrary().acquireReadonly(obj, frame, interopCallData); + PythonBufferAccessLibrary bufferLib = getBufferLibrary(); try { if (pickler.proto < 5) { // Older pickle protocols do not have an opcode for pickling bytearrays. Object reduceValue; final PythonBuiltinClass byteArrayClass = ctx.getCore().lookupType(PythonBuiltinClassType.PByteArray); - if (getBufferLibrary().getBufferLength(buffer) == 0) { + if (bufferLib.getBufferLength(buffer) == 0) { reduceValue = createTuple(byteArrayClass, createTuple()); } else { - byte[] bytes = getBufferLibrary().getCopiedByteArray(buffer); + byte[] bytes = bufferLib.getCopiedByteArray(buffer); reduceValue = createTuple(byteArrayClass, createTuple(PFactory.createBytes(ctx.getLanguage(inliningTarget), bytes))); } // save_reduce() will memoize the object automatically. saveReduce(frame, ctx, pickler, reduceValue, obj); } else { - saveBytearrayData(frame, pickler, obj, getBufferLibrary().getCopiedByteArray(buffer), length(frame, obj)); + saveBytearrayData(frame, pickler, obj, buffer, bufferLib); } } finally { - getBufferLibrary().release(buffer, frame, interopCallData); + bufferLib.release(buffer, frame, interopCallData); } } - private void savePicklebuffer(VirtualFrame frame, PPickler pickler, PPickleBuffer obj) { + private void savePickleBuffer(VirtualFrame frame, PPickler pickler, PPickleBuffer obj) { if (pickler.proto < 5) { throw raise(PicklingError, ErrorMessages.PICKLEBUFF_CANNOT_PICKLE_WITH_PROTO5); } Object buffer = obj.getView(); PythonBufferAccessLibrary bufferLib = getBufferLibrary(); - int bytesLen = bufferLib.getBufferLength(buffer); - byte[] bytes = bufferLib.getInternalOrCopiedByteArray(buffer); boolean inBand = true; if (pickler.bufferCallback != null) { @@ -1805,9 +1799,9 @@ private void savePicklebuffer(VirtualFrame frame, PPickler pickler, PPickleBuffe if (inBand) { // Write data in-band if (readOnly) { - saveBytesData(frame, pickler, obj, bytes, bytesLen); + saveBytesData(frame, pickler, obj, buffer, bufferLib); } else { - saveBytearrayData(frame, pickler, obj, bytes, bytesLen); + saveBytearrayData(frame, pickler, obj, buffer, bufferLib); } } else { // Write data out-of-band @@ -1872,14 +1866,9 @@ private void saveGlobal(VirtualFrame frame, BoundaryCallData boundaryCallData, P if (pickler.proto >= 2) { genGlobal = false; // See whether this is in the extension registry, and if so generate an EXT opcode. - PTuple extensionKey; - Object codeObj; - long code; - byte[] pdata = new byte[5]; - int n; - - extensionKey = createTuple(moduleName, globalName); - codeObj = getDictItem(frame, st.extensionRegistry, extensionKey); + + PTuple extensionKey = createTuple(moduleName, globalName); + Object codeObj = getDictItem(frame, st.extensionRegistry, extensionKey); // The object is not registered in the extension registry. This is the most likely // code path. if (codeObj == null) { @@ -1892,38 +1881,25 @@ private void saveGlobal(VirtualFrame frame, BoundaryCallData boundaryCallData, P throw raise(PicklingError, ErrorMessages.CANT_PICKLE_P_EXT_CODE_P_NOT_AN_INT, obj, codeObj); } - code = asLong(frame, codeObj); + long code = asLong(frame, codeObj); if (code <= 0 || code > 0x7fffffffL) { throw raise(PicklingError, ErrorMessages.CANT_PICKLE_P_EXT_CODE_OO_RANGE, obj, code); } // Generate an EXT opcode if (code <= 0xff) { - pdata[0] = PickleUtils.OPCODE_EXT1; - pdata[1] = (byte) code; - n = 2; + writeByteOp(pickler, PickleUtils.OPCODE_EXT1, code); } else if (code <= 0xffff) { - pdata[0] = PickleUtils.OPCODE_EXT2; - pdata[1] = (byte) (code & 0xff); - pdata[2] = (byte) ((code >> 8) & 0xff); - n = 3; + writeShortOp(pickler, PickleUtils.OPCODE_EXT2, code); } else { - pdata[0] = PickleUtils.OPCODE_EXT4; - pdata[1] = (byte) (code & 0xff); - pdata[2] = (byte) ((code >> 8) & 0xff); - pdata[3] = (byte) ((code >> 16) & 0xff); - pdata[4] = (byte) ((code >> 24) & 0xff); - n = 5; + writeIntOp(pickler, PickleUtils.OPCODE_EXT4, code); } - - write(pickler, pdata, n); } } else { genGlobal = true; } if (genGlobal) { - byte[] encoded; TruffleString lastname = dottedPath[dottedPath.length - 1]; if (parent == module) { @@ -1953,20 +1929,24 @@ private void saveGlobal(VirtualFrame frame, BoundaryCallData boundaryCallData, P // module name and the global name using UTF-8. We do so only when we are using // the pickle protocol newer than version 3. This is to ensure compatibility // with older Unpickler running on Python 2.x. - TruffleString.Encoding encoding = (pickler.proto == 3) ? TruffleString.Encoding.UTF_8 : TruffleString.Encoding.US_ASCII; - encoded = PickleUtils.encodeStrict(moduleName, ensureTsSwitchEncodingNode(), encoding, ensureTsCopyToByteArrayNode(), ensureTsGetCodeRangeNode()); - if (encoded == null) { - throw raise(PicklingError, ErrorMessages.CANT_PICKLE_MODULE_S_USING_PROTO_D, moduleName, pickler.proto); - } - write(pickler, encoded); - writeASCII(pickler, T_NEWLINE); - // Save the name of the module - encoded = PickleUtils.encodeStrict(globalName, ensureTsSwitchEncodingNode(), encoding, ensureTsCopyToByteArrayNode(), ensureTsGetCodeRangeNode()); - if (encoded == null) { - throw raise(PicklingError, ErrorMessages.CANT_PICKLE_MODULE_S_USING_PROTO_D, moduleName, pickler.proto); + if (pickler.proto >= 3) { + if (ensureTsGetCodeRangeNode().execute(moduleName, TS_ENCODING) == TruffleString.CodeRange.BROKEN || ensureTsGetCodeRangeNode().execute(globalName, + TS_ENCODING) == TruffleString.CodeRange.BROKEN) { + throw raise(PicklingError, ErrorMessages.CANT_PICKLE_MODULE_S_USING_PROTO_D, moduleName, pickler.proto); + } + writeUtf8(pickler, moduleName); + write(pickler, (byte) '\n'); + writeUtf8(pickler, globalName); + } else { + if (ensureTsGetCodeRangeNode().execute(moduleName, TS_ENCODING) != TruffleString.CodeRange.ASCII || ensureTsGetCodeRangeNode().execute(globalName, + TS_ENCODING) != TruffleString.CodeRange.ASCII) { + throw raise(PicklingError, ErrorMessages.CANT_PICKLE_MODULE_S_USING_PROTO_D, moduleName, pickler.proto); + } + writeASCII(pickler, moduleName); + write(pickler, (byte) '\n'); + writeASCII(pickler, globalName); } - write(pickler, encoded); - writeASCII(pickler, T_NEWLINE); + write(pickler, (byte) '\n'); } // Memoize the object memoPut(pickler, obj); @@ -2052,7 +2032,7 @@ void saveGeneric(VirtualFrame frame, PPickler pickler, Object objArg, int persSa saveBytearray(frame, inliningTarget, ctx, pickler, obj, interopCallData); return; } else if (obj instanceof PPickleBuffer buffer) { - savePicklebuffer(frame, pickler, buffer); + savePickleBuffer(frame, pickler, buffer); return; } @@ -2137,10 +2117,9 @@ public void dump(VirtualFrame frame, PPickler pickler, Object obj, if (pickler.proto >= 2) { assert pickler.proto <= 256; - byte[] header = new byte[]{PickleUtils.OPCODE_PROTO, (byte) pickler.proto}; try { - write(pickler, header, 2); + write(pickler, PickleUtils.OPCODE_PROTO, (byte) pickler.proto); } catch (Exception e) { handleError(pickler); throw e; diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PickleUtils.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PickleUtils.java index 276a5c3d0c..772d2abc95 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PickleUtils.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PickleUtils.java @@ -40,7 +40,6 @@ */ package com.oracle.graal.python.builtins.modules.pickle; -import static com.oracle.graal.python.builtins.modules.pickle.PPickler.BasePickleWriteNode.NEW_LINE_BYTE; import static com.oracle.graal.python.nodes.StringLiterals.T_DOT; import static com.oracle.graal.python.util.PythonUtils.TS_ENCODING; import static com.oracle.graal.python.util.PythonUtils.toTruffleStringUncached; @@ -53,8 +52,6 @@ import com.oracle.graal.python.PythonLanguage; import com.oracle.graal.python.builtins.PythonBuiltinClassType; import com.oracle.graal.python.builtins.objects.PNone; -import com.oracle.graal.python.builtins.objects.bytes.ByteArrayBuffer; -import com.oracle.graal.python.builtins.objects.bytes.BytesUtils; import com.oracle.graal.python.builtins.objects.method.PBuiltinMethod; import com.oracle.graal.python.builtins.objects.method.PMethod; import com.oracle.graal.python.builtins.objects.str.StringUtils; @@ -62,7 +59,6 @@ import com.oracle.graal.python.nodes.PRaiseNode; import com.oracle.graal.python.nodes.StringLiterals; import com.oracle.graal.python.runtime.object.PFactory; -import com.oracle.graal.python.util.PythonUtils; import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; import com.oracle.truffle.api.frame.VirtualFrame; import com.oracle.truffle.api.nodes.ExplodeLoop; @@ -229,28 +225,6 @@ public static void writeSize64(byte[] out, int start, int value) { } } - public static byte[] resize(byte[] src, int newSize) { - byte[] newBytes = new byte[newSize]; - if (src != null) { - PythonUtils.arraycopy(src, 0, newBytes, 0, Math.min(newSize, src.length)); - } - return newBytes; - } - - private static int arrayCopyWithNewLine(byte[] dst, int offset, byte[] src) { - PythonUtils.arraycopy(src, 0, dst, offset, src.length); - int len = src.length + 2; - dst[len - 1] = NEW_LINE_BYTE; - return len; - } - - public static int toAsciiBytesWithNewLine(byte[] dst, int offset, long value, TruffleString.FromLongNode fromLongNode, TruffleString.CopyToByteArrayNode copyToByteArrayNode) { - TruffleString s = fromLongNode.execute(value, TruffleString.Encoding.US_ASCII, true); - byte[] buf = new byte[s.byteLength(TruffleString.Encoding.US_ASCII)]; - copyToByteArrayNode.execute(s, 0, buf, 0, buf.length, TruffleString.Encoding.US_ASCII); - return arrayCopyWithNewLine(dst, offset, buf); - } - public static int getStringSize(byte[] bytes) { int len = bytes.length > 0 && bytes[bytes.length - 1] == '\n' ? bytes.length - 1 : bytes.length; return len > 0 && bytes[len - 1] == 0 ? len - 1 : len; @@ -295,23 +269,6 @@ public static double asciiBytesToDouble(Node node, byte[] bytes, PythonBuiltinCl } } - public static byte[] encodeUTF8Strict(TruffleString str, TruffleString.SwitchEncodingNode switchEncodingNode, TruffleString.CopyToByteArrayNode copyToByteArrayNode, - TruffleString.GetCodeRangeNode getCodeRangeNode) { - return encodeStrict(str, switchEncodingNode, TruffleString.Encoding.UTF_8, copyToByteArrayNode, getCodeRangeNode); - } - - public static byte[] encodeStrict(TruffleString ts, TruffleString.SwitchEncodingNode switchEncodingNode, TruffleString.Encoding encoding, TruffleString.CopyToByteArrayNode copyToByteArrayNode, - TruffleString.GetCodeRangeNode getCodeRangeNode) { - // TODO: [GR-39571] TruffleStrings: allow preservation of UTF-16 surrogate - if (getCodeRangeNode.execute(ts, TS_ENCODING) == TruffleString.CodeRange.BROKEN) { - return null; - } - TruffleString s = switchEncodingNode.execute(ts, encoding); - byte[] buf = new byte[s.byteLength(encoding)]; - copyToByteArrayNode.execute(s, 0, buf, 0, buf.length, encoding); - return buf; - } - public static TruffleString decodeUTF8Strict(byte[] data, int len, TruffleString.FromByteArrayNode fromByteArrayNode, TruffleString.SwitchEncodingNode switchEncodingNode) { return decodeStrict(data, len, fromByteArrayNode, TruffleString.Encoding.UTF_8, switchEncodingNode); } @@ -326,39 +283,6 @@ private static TruffleString decodeStrict(byte[] data, int len, TruffleString.Fr return switchEncodingNode.execute(ret, TS_ENCODING); } - public static byte[] rawUnicodeEscape(TruffleString unicode, TruffleString.CodePointLengthNode codePointLengthNode, TruffleString.CodePointAtIndexUTF32Node codePointAtIndexNode) { - int len = codePointLengthNode.execute(unicode, TS_ENCODING); - ByteArrayBuffer buffer = new ByteArrayBuffer(len); - for (int i = 0; i < len; i++) { - final int ch = codePointAtIndexNode.execute(unicode, i); - if (ch >= 0x10000) { - // Map 32-bit characters to \Uxxxxxxxx - buffer.append('\\'); - buffer.append('U'); - buffer.append((char) BytesUtils.HEXDIGITS[(ch >> 28) & 0xf]); - buffer.append((char) BytesUtils.HEXDIGITS[(ch >> 24) & 0xf]); - buffer.append((char) BytesUtils.HEXDIGITS[(ch >> 20) & 0xf]); - buffer.append((char) BytesUtils.HEXDIGITS[(ch >> 16) & 0xf]); - buffer.append((char) BytesUtils.HEXDIGITS[(ch >> 12) & 0xf]); - buffer.append((char) BytesUtils.HEXDIGITS[(ch >> 8) & 0xf]); - buffer.append((char) BytesUtils.HEXDIGITS[(ch >> 4) & 0xf]); - buffer.append((char) BytesUtils.HEXDIGITS[ch & 15]); - } else if (ch >= 256 || ch == '\\' || ch == 0 || ch == '\n' || ch == '\r' || ch == 0x1a) { - // Map 16-bit characters, \\ and \n to \Uxxxx - buffer.append('\\'); - buffer.append('u'); - buffer.append((char) BytesUtils.HEXDIGITS[(ch >> 12) & 0xf]); - buffer.append((char) BytesUtils.HEXDIGITS[(ch >> 8) & 0xf]); - buffer.append((char) BytesUtils.HEXDIGITS[(ch >> 4) & 0xf]); - buffer.append((char) BytesUtils.HEXDIGITS[ch & 15]); - } else { - // Copy everything else as-is - buffer.append((char) ch); - } - } - return buffer.getByteArray(); - } - public static Pair initMethodRef(VirtualFrame frame, Node inliningTarget, PyObjectLookupAttr lookup, Object receiver, TruffleString identifier) { final Object func = lookup.execute(frame, inliningTarget, receiver, identifier); if (func == PNone.NO_VALUE) { From 34df922e10dd50867b3bd8b80ad1830a2203a536 Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Fri, 25 Sep 2026 09:34:01 +0200 Subject: [PATCH 02/39] Lazily initialize fastMemo table --- .../python/builtins/modules/pickle/PPickler.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java index b27219e9f2..372811a2af 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java @@ -163,7 +163,7 @@ public class PPickler extends PythonBuiltinObject { private int fastNesting; // Indicate whether Pickler should fix the name of globals for Python 2.x. private boolean fixImports; - private final Map fastMemo = createFastMemoTable(); + private Map fastMemo; // Callback for out-of-band buffers, or NULL private Object bufferCallback; @@ -261,18 +261,28 @@ public void setPersFuncSelf(Object persFuncSelf) { this.persFuncSelf = persFuncSelf; } + @TruffleBoundary + private void ensureFastMemo() { + if (fastMemo == null) { + fastMemo = createFastMemoTable(); + } + } + @TruffleBoundary private boolean fastMemoContains(Object object) { + ensureFastMemo(); return this.fastMemo.containsKey(object); } @TruffleBoundary private void fastMemoPut(Object object) { + ensureFastMemo(); this.fastMemo.put(object, true); } @TruffleBoundary private void fastMemoRemove(Object object) { + ensureFastMemo(); this.fastMemo.remove(object); } From 58067cbc9f5ae52add7656401845c2da5e06e74a Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Thu, 24 Sep 2026 23:46:29 +0200 Subject: [PATCH 03/39] Simplify pickling builtins --- .../builtins/modules/pickle/PPickler.java | 296 +++++++++--------- .../oracle/graal/python/nodes/PGuards.java | 4 + 2 files changed, 154 insertions(+), 146 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java index 372811a2af..0db29d7729 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java @@ -71,7 +71,9 @@ import com.oracle.graal.python.builtins.objects.buffer.PythonBufferAccessLibrary; import com.oracle.graal.python.builtins.objects.buffer.PythonBufferAcquireLibrary; import com.oracle.graal.python.builtins.objects.bytes.BytesUtils; +import com.oracle.graal.python.builtins.objects.bytes.PByteArray; import com.oracle.graal.python.builtins.objects.bytes.PBytes; +import com.oracle.graal.python.builtins.objects.cext.PythonNativeClass; import com.oracle.graal.python.builtins.objects.common.HashingStorage; import com.oracle.graal.python.builtins.objects.common.HashingStorageNodes.HashingStorageIterator; import com.oracle.graal.python.builtins.objects.common.HashingStorageNodes.HashingStorageIteratorKey; @@ -80,24 +82,28 @@ import com.oracle.graal.python.builtins.objects.common.HashingStorageNodes.HashingStorageLen; import com.oracle.graal.python.builtins.objects.dict.PDict; import com.oracle.graal.python.builtins.objects.ellipsis.PEllipsis; +import com.oracle.graal.python.builtins.objects.floats.PFloat; +import com.oracle.graal.python.builtins.objects.function.PFunction; import com.oracle.graal.python.builtins.objects.ints.IntNodes; import com.oracle.graal.python.builtins.objects.ints.IntNodesFactory; +import com.oracle.graal.python.builtins.objects.ints.PInt; +import com.oracle.graal.python.builtins.objects.list.PList; import com.oracle.graal.python.builtins.objects.object.PythonBuiltinObject; import com.oracle.graal.python.builtins.objects.set.PFrozenSet; import com.oracle.graal.python.builtins.objects.set.PSet; +import com.oracle.graal.python.builtins.objects.str.PString; import com.oracle.graal.python.builtins.objects.tuple.PTuple; import com.oracle.graal.python.builtins.objects.type.PythonBuiltinClass; +import com.oracle.graal.python.builtins.objects.type.PythonManagedClass; import com.oracle.graal.python.builtins.objects.type.TypeNodes; import com.oracle.graal.python.lib.IteratorExhausted; import com.oracle.graal.python.lib.PyCallableCheckNode; -import com.oracle.graal.python.lib.PyFloatAsDoubleNode; import com.oracle.graal.python.lib.PyLongAsLongNode; import com.oracle.graal.python.lib.PyObjectCallMethodObjArgs; import com.oracle.graal.python.lib.PyObjectGetIter; import com.oracle.graal.python.lib.PyObjectIsTrueNode; import com.oracle.graal.python.lib.PyObjectLookupAttr; import com.oracle.graal.python.lib.PyObjectReprAsTruffleStringNode; -import com.oracle.graal.python.lib.PyObjectSizeNode; import com.oracle.graal.python.lib.PyObjectStrAsObjectNode; import com.oracle.graal.python.lib.PyTupleCheckNode; import com.oracle.graal.python.nodes.ErrorMessages; @@ -108,13 +114,13 @@ import com.oracle.graal.python.nodes.classes.IsSubtypeNode; import com.oracle.graal.python.nodes.object.IsNode; import com.oracle.graal.python.runtime.IndirectCallData.BoundaryCallData; -import com.oracle.graal.python.runtime.IndirectCallData.InteropCallData; import com.oracle.graal.python.runtime.PythonContext; import com.oracle.graal.python.runtime.exception.PException; import com.oracle.graal.python.runtime.object.PFactory; import com.oracle.graal.python.runtime.sequence.storage.SequenceStorage; import com.oracle.graal.python.util.Consumer; import com.oracle.graal.python.util.NumericSupport; +import com.oracle.graal.python.util.OverflowException; import com.oracle.graal.python.util.PythonUtils; import com.oracle.truffle.api.CompilerDirectives; import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; @@ -900,11 +906,6 @@ private void saveListIterator(VirtualFrame frame, PPickler pickler, Object itera (Object item) -> save(frame, pickler, item, 0)); } - private void saveFrozenSetIterator(VirtualFrame frame, PPickler pickler, Object iterator) { - saveIterator(frame, pickler, iterator, PickleUtils.NO_OPCODE, - (Object item) -> save(frame, pickler, item, 0)); - } - private void saveDictIterator(VirtualFrame frame, PPickler pickler, Object iterator) { saveIterator(frame, pickler, iterator, PickleUtils.OPCODE_SETITEM, (Object item) -> { @@ -1225,45 +1226,54 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, } } - private void saveNone(PPickler pickler, @SuppressWarnings("unused") Object obj) { + private void saveNone(PPickler pickler) { write(pickler, PickleUtils.OPCODE_NONE); } - private void saveBool(VirtualFrame frame, PPickler pickler, Object obj) { + private void saveBool(PPickler pickler, boolean value) { if (pickler.proto >= 2) { - write(pickler, isTrue(frame, obj) ? PickleUtils.OPCODE_NEWTRUE : PickleUtils.OPCODE_NEWFALSE); + write(pickler, value ? PickleUtils.OPCODE_NEWTRUE : PickleUtils.OPCODE_NEWFALSE); } else { // These aren't opcodes -- they're ways to pickle bools before protocol 2 so that // unpicklers written before bools were introduced unpickle them as ints, but // unpicklers after can recognize that bools were intended. Note that protocol 2 // added direct ways to pickle bools. - writeASCII(pickler, isTrue(frame, obj) ? PickleUtils.T_PROTO_LE2_TRUE : PickleUtils.T_PROTO_LE2_FALSE); + writeASCII(pickler, value ? PickleUtils.T_PROTO_LE2_TRUE : PickleUtils.T_PROTO_LE2_FALSE); } } - private void saveLong(VirtualFrame frame, PPickler pickler, Object obj) { - TruffleString repr; - try { - long longValue = asLong(frame, obj); - int value = (int) longValue; - if (value == longValue) { - if (pickler.isBin()) { - if (value >= 0 && value < (1 << 8)) { - writeByteOp(pickler, PickleUtils.OPCODE_BININT1, value); - } else if (value >= 0 && value < (1 << 16)) { - writeShortOp(pickler, PickleUtils.OPCODE_BININT2, value); - } else { - writeIntOp(pickler, PickleUtils.OPCODE_BININT, value); - } - } else { - writeLongAsString(pickler, PickleUtils.OPCODE_INT, longValue); - } - return; + private void saveLong(PPickler pickler, int value) { + if (pickler.isBin()) { + if (value >= 0 && value < (1 << 8)) { + writeByteOp(pickler, PickleUtils.OPCODE_BININT1, value); + } else if (value >= 0 && value < (1 << 16)) { + writeShortOp(pickler, PickleUtils.OPCODE_BININT2, value); + } else { + writeIntOp(pickler, PickleUtils.OPCODE_BININT, value); } - } catch (PException e) { - e.expectCached(OverflowError, ensureErrProfile()); + } else { + writeLongAsString(pickler, PickleUtils.OPCODE_INT, value); + } + } + + private void saveLong(VirtualFrame frame, PPickler pickler, long value) { + int intValue = (int) value; + if (intValue == value) { + saveLong(pickler, intValue); + } else { + saveLongLarge(frame, pickler, value); } + } + + private void saveLong(VirtualFrame frame, PPickler pickler, PInt value) { + try { + saveLong(frame, pickler, value.longValueExact()); + } catch (OverflowException e) { + saveLongLarge(frame, pickler, value); + } + } + private void saveLongLarge(VirtualFrame frame, PPickler pickler, Object obj) { if (pickler.proto >= 2) { // Linear-time pickling. final int sign = getSign(obj); @@ -1298,15 +1308,14 @@ private void saveLong(VirtualFrame frame, PPickler pickler, Object obj) { write(pickler, pdata, nbytes); } else { - repr = repr(frame, obj); + TruffleString repr = repr(frame, obj); write(pickler, PickleUtils.OPCODE_LONG); writeASCII(pickler, asStringStrict(repr)); writeASCII(pickler, T_L_NEW_LINE); } } - private void saveFloat(VirtualFrame frame, PPickler pickler, Object obj, Node inliningTarget, PyFloatAsDoubleNode asDoubleNode) { - final double value = asDoubleNode.execute(frame, inliningTarget, obj); + private void saveFloat(PPickler pickler, double value) { if (pickler.isBin()) { write(pickler, PickleUtils.OPCODE_BINFLOAT); ensureBufferSpace(pickler, 8); @@ -1320,8 +1329,8 @@ private void saveFloat(VirtualFrame frame, PPickler pickler, Object obj, Node in } } - private void saveBytes(VirtualFrame frame, PythonContext ctx, PPickler pickler, Object obj, InteropCallData interopCallData) { - Object buffer = getBufferAcquireLibrary().acquireReadonly(obj, frame, interopCallData); + private void saveBytes(VirtualFrame frame, PythonContext ctx, PPickler pickler, PBytes obj) { + Object buffer = getBufferAcquireLibrary().acquireReadonly(obj); PythonBufferAccessLibrary bufferLib = getBufferLibrary(); try { if (pickler.proto < 3) { @@ -1350,7 +1359,7 @@ private void saveBytes(VirtualFrame frame, PythonContext ctx, PPickler pickler, saveBytesData(frame, pickler, obj, buffer, bufferLib); } } finally { - bufferLib.release(buffer, frame, interopCallData); + bufferLib.release(buffer); } } @@ -1380,8 +1389,7 @@ private void saveBytesData(VirtualFrame frame, PPickler pickler, Object obj, Obj memoPut(pickler, obj); } - private void writeUnicodeBinary(VirtualFrame frame, PPickler pickler, Object obj) { - TruffleString string = asStringStrict(obj); + private void writeUnicodeBinary(VirtualFrame frame, PPickler pickler, TruffleString string) { TruffleString utf8 = ensureTsSwitchEncodingNode().execute(string, TruffleString.Encoding.UTF_8, TranscodingErrorHandler.DEFAULT_KEEP_SURROGATES_IN_UTF8); int size = utf8.byteLength(TruffleString.Encoding.UTF_8); boolean wasFraming = pickler.framing; @@ -1408,12 +1416,20 @@ private void writeUnicodeBinary(VirtualFrame frame, PPickler pickler, Object obj } } - private void saveUnicode(VirtualFrame frame, PPickler pickler, Object obj) { + private void saveUnicode(VirtualFrame frame, PPickler pickler, TruffleString obj) { + saveUnicode(frame, pickler, obj, obj); + } + + private void saveUnicode(VirtualFrame frame, PPickler pickler, PString obj) { + saveUnicode(frame, pickler, obj, asStringStrict(obj)); + } + + private void saveUnicode(VirtualFrame frame, PPickler pickler, Object obj, TruffleString string) { if (pickler.isBin()) { - writeUnicodeBinary(frame, pickler, obj); + writeUnicodeBinary(frame, pickler, string); } else { write(pickler, PickleUtils.OPCODE_UNICODE); - writeRawUnicodeEscape(pickler, asStringStrict(obj)); + writeRawUnicodeEscape(pickler, string); writeASCII(pickler, T_NEWLINE); } memoPut(pickler, obj); @@ -1481,7 +1497,7 @@ private void batchDict(VirtualFrame frame, PPickler pickler, Object iterator) { } } - private void saveDict(VirtualFrame frame, Node inliningTarget, PyObjectCallMethodObjArgs callMethod, PPickler pickler, Object obj) { + private void saveDict(VirtualFrame frame, Node inliningTarget, PyObjectCallMethodObjArgs callMethod, PPickler pickler, PDict obj) { if (pickler.isFast()) { fastSaveEnter(pickler, obj); } @@ -1495,9 +1511,9 @@ private void saveDict(VirtualFrame frame, Node inliningTarget, PyObjectCallMetho memoPut(pickler, obj); - if (length(frame, obj) > 0) { - if (PGuards.isDict(obj) && pickler.proto > 0) { - batchDictExact(frame, pickler, (PDict) obj); + if (getHashingStorageLength(obj.getDictStorage()) > 0) { + if (pickler.proto > 0) { + batchDictExact(frame, pickler, obj); } else { final Object items = callMethod.execute(frame, inliningTarget, obj, T_ITEMS); batchDict(frame, pickler, getIter(frame, items)); @@ -1513,33 +1529,7 @@ private void batchSetExact(VirtualFrame frame, PPickler pickler, PSet set) { saveSetHashingStorageBatched(frame, pickler, set.getDictStorage()); } - private void batchSet(VirtualFrame frame, PPickler pickler, Object obj) { - Object iterator = getIter(frame, obj); - int setSize = length(frame, obj); - int i; - do { - i = 0; - write(pickler, PickleUtils.OPCODE_MARK); - while (true) { - Object item; - try { - item = getNextItem(frame, iterator); - } catch (IteratorExhausted e) { - break; - } - save(frame, pickler, item, 0); - if (++i == PickleUtils.BATCHSIZE) { - break; - } - } - write(pickler, PickleUtils.OPCODE_ADDITEMS); - if (length(frame, obj) != setSize) { - throw raise(PythonBuiltinClassType.RuntimeError, ErrorMessages.CHANGED_SIZE_DURING_ITERATION, "set"); - } - } while (i == PickleUtils.BATCHSIZE); - } - - private void saveSet(VirtualFrame frame, PythonContext ctx, PPickler pickler, Object obj) { + private void saveSet(VirtualFrame frame, PythonContext ctx, PPickler pickler, PSet obj) { Object reduceValue; if (pickler.proto < 4) { Object items = createList(frame, obj); @@ -1558,14 +1548,10 @@ private void saveSet(VirtualFrame frame, PythonContext ctx, PPickler pickler, Ob } // Write in batches of BATCHSIZE. - if (obj instanceof PSet set) { - batchSetExact(frame, pickler, set); - } else { - batchSet(frame, pickler, obj); - } + batchSetExact(frame, pickler, obj); } - private void saveFrozenset(VirtualFrame frame, PythonContext ctx, PPickler pickler, Object obj) { + private void saveFrozenset(VirtualFrame frame, PythonContext ctx, PPickler pickler, PFrozenSet obj) { if (pickler.isFast()) { fastSaveEnter(pickler, obj); } @@ -1579,12 +1565,7 @@ private void saveFrozenset(VirtualFrame frame, PythonContext ctx, PPickler pickl write(pickler, PickleUtils.OPCODE_MARK); - if (obj instanceof PFrozenSet set) { - saveSetHashingStorage(frame, pickler, set.getDictStorage()); - } else { - final Object iterator = getIter(frame, obj); - saveFrozenSetIterator(frame, pickler, iterator); - } + saveSetHashingStorage(frame, pickler, obj.getDictStorage()); // If the object is already in the memo, this means it is recursive. In this case, throw // away everything we put on the stack, and fetch the object back from the memo. @@ -1599,10 +1580,10 @@ private void saveFrozenset(VirtualFrame frame, PythonContext ctx, PPickler pickl memoPut(pickler, obj); } - private void batchListExact(VirtualFrame frame, PPickler pickler, Object obj) { - final SequenceStorage storage = getSequenceStorage(obj); + private void batchListExact(VirtualFrame frame, PPickler pickler, PList obj) { + final SequenceStorage storage = obj.getSequenceStorage(); Object item; - if (length(frame, obj) == 1) { + if (storage.length() == 1) { item = getItem(frame, storage, 0); save(frame, pickler, item, 0); write(pickler, PickleUtils.OPCODE_APPEND); @@ -1615,7 +1596,7 @@ private void batchListExact(VirtualFrame frame, PPickler pickler, Object obj) { do { thisBatch = 0; write(pickler, PickleUtils.OPCODE_MARK); - while (total < length(frame, obj)) { + while (total < storage.length()) { item = getItem(frame, storage, total); save(frame, pickler, item, 0); total++; @@ -1624,7 +1605,7 @@ private void batchListExact(VirtualFrame frame, PPickler pickler, Object obj) { } } write(pickler, PickleUtils.OPCODE_APPENDS); - } while (total < length(frame, obj)); + } while (total < storage.length()); } private void batchList(VirtualFrame frame, PPickler pickler, Object iterator) { @@ -1638,7 +1619,7 @@ private void batchList(VirtualFrame frame, PPickler pickler, Object iterator) { } } - private void saveList(VirtualFrame frame, PPickler pickler, Object obj) { + private void saveList(VirtualFrame frame, PPickler pickler, PList obj) { if (pickler.isFast()) { fastSaveEnter(pickler, obj); } @@ -1650,12 +1631,11 @@ private void saveList(VirtualFrame frame, PPickler pickler, Object obj) { write(pickler, PickleUtils.OPCODE_MARK, PickleUtils.OPCODE_LIST); } - int len = length(frame, obj); + int len = obj.getSequenceStorage().length(); memoPut(pickler, obj); if (len != 0) { - // Materialize the list elements. - if (PGuards.isList(obj) && pickler.proto > 0) { + if (pickler.proto > 0) { batchListExact(frame, pickler, obj); } else { batchList(frame, pickler, getIter(frame, obj)); @@ -1667,17 +1647,30 @@ private void saveList(VirtualFrame frame, PPickler pickler, Object obj) { } } - private void storeTupleElements(VirtualFrame frame, PPickler pickler, Object obj, int len) { + private void storeTupleElements(VirtualFrame frame, PPickler pickler, Object obj, SequenceStorage storage, int len) { // A helper for save_tuple. Push the len elements in tuple t on the stack - assert PyObjectSizeNode.executeUncached(obj) == len; - for (int i = 0; i < len; i++) { - Object element = getItem(frame, obj, i); - save(frame, pickler, element, 0); + if (storage == null) { + for (int i = 0; i < len; i++) { + save(frame, pickler, getItem(frame, obj, i), 0); + } + } else { + assert storage.length() == len; + for (int i = 0; i < len; i++) { + save(frame, pickler, getItem(frame, storage, i), 0); + } } } - private void saveTuple(VirtualFrame frame, PPickler pickler, Object obj) { - int len = length(frame, obj); + private void saveTuple(VirtualFrame frame, PPickler pickler, PTuple obj) { + SequenceStorage storage = obj.getSequenceStorage(); + saveTuple(frame, pickler, obj, storage, storage.length()); + } + + private void saveNativeTuple(VirtualFrame frame, PPickler pickler, Object obj) { + saveTuple(frame, pickler, obj, null, length(frame, obj)); + } + + private void saveTuple(VirtualFrame frame, PPickler pickler, Object obj, SequenceStorage storage, int len) { if (len == 0) { if (pickler.proto != 0) { @@ -1693,7 +1686,7 @@ private void saveTuple(VirtualFrame frame, PPickler pickler, Object obj) { // the stack, and fetch its value from the memo. if (len <= 3 && pickler.proto >= 2) { // Use TUPLE{1,2,3} opcodes. - storeTupleElements(frame, pickler, obj, len); + storeTupleElements(frame, pickler, obj, storage, len); int memoIndex = pickler.memo.get(obj); if (memoIndex != -1) { @@ -1715,7 +1708,7 @@ private void saveTuple(VirtualFrame frame, PPickler pickler, Object obj) { // proto < 2 and len > 0, or proto >= 2 and len > 3. Generate MARK e1 e2 ... TUPLE write(pickler, PickleUtils.OPCODE_MARK); - storeTupleElements(frame, pickler, obj, len); + storeTupleElements(frame, pickler, obj, storage, len); int memoIndex = pickler.memo.get(obj); if (memoIndex != -1) { @@ -1765,8 +1758,8 @@ private void saveBytearrayData(VirtualFrame frame, PPickler pickler, Object obj, memoPut(pickler, obj); } - private void saveBytearray(VirtualFrame frame, Node inliningTarget, PythonContext ctx, PPickler pickler, Object obj, InteropCallData interopCallData) { - Object buffer = getBufferAcquireLibrary().acquireReadonly(obj, frame, interopCallData); + private void saveBytearray(VirtualFrame frame, Node inliningTarget, PythonContext ctx, PPickler pickler, PByteArray obj) { + Object buffer = getBufferAcquireLibrary().acquireReadonly(obj); PythonBufferAccessLibrary bufferLib = getBufferLibrary(); try { if (pickler.proto < 5) { @@ -1787,7 +1780,7 @@ private void saveBytearray(VirtualFrame frame, Node inliningTarget, PythonContex saveBytearrayData(frame, pickler, obj, buffer, bufferLib); } } finally { - bufferLib.release(buffer, frame, interopCallData); + bufferLib.release(buffer); } } @@ -1827,16 +1820,15 @@ private void saveSingletonType(VirtualFrame frame, PythonContext ctx, PPickler p saveReduce(frame, ctx, pickler, reduceValue, obj); } - private void saveType(VirtualFrame frame, BoundaryCallData boundaryCallData, PythonContext ctx, PPickler pickler, Object obj) { - if (isBuiltinClass(obj, PythonBuiltinClassType.PNone)) { + private void saveType(VirtualFrame frame, BoundaryCallData boundaryCallData, PythonContext ctx, PPickler pickler, PythonManagedClass obj) { + if (obj instanceof PythonBuiltinClass cls && cls.getType() == PythonBuiltinClassType.PNone) { saveSingletonType(frame, ctx, pickler, obj, PNone.NONE); - } else if (isBuiltinClass(obj, PythonBuiltinClassType.PEllipsis)) { + } else if (obj instanceof PythonBuiltinClass cls && cls.getType() == PythonBuiltinClassType.PEllipsis) { saveSingletonType(frame, ctx, pickler, obj, PEllipsis.INSTANCE); - } else if (isBuiltinClass(obj, PythonBuiltinClassType.PNotImplemented)) { + } else if (obj instanceof PythonBuiltinClass cls && cls.getType() == PythonBuiltinClassType.PNotImplemented) { saveSingletonType(frame, ctx, pickler, obj, PNotImplemented.NOT_IMPLEMENTED); } else { - Object type = (obj instanceof PythonBuiltinClassType) ? ctx.getCore().lookupType((PythonBuiltinClassType) obj) : obj; - saveGlobal(frame, boundaryCallData, ctx, pickler, type, null); + saveGlobal(frame, boundaryCallData, ctx, pickler, obj, null); } } @@ -1967,8 +1959,6 @@ private void saveGlobal(VirtualFrame frame, BoundaryCallData boundaryCallData, P void saveGeneric(VirtualFrame frame, PPickler pickler, Object objArg, int persSave, @Bind Node inliningTarget, @Cached("createFor($node)") BoundaryCallData boundaryCallData, - @Cached("createFor($node)") InteropCallData interopCallData, - @Cached PyFloatAsDoubleNode asDoubleNode, @Cached CallNode callNode, @Cached PyObjectCallMethodObjArgs callMethod) { Object obj = objArg; @@ -1982,23 +1972,27 @@ void saveGeneric(VirtualFrame frame, PPickler pickler, Object objArg, int persSa } } - final Object type = getClass(obj); - // The old cPickle had an optimization that used switch-case statement dispatching on - // the first letter of the type name. This has was removed since benchmarks shown that - // this optimization was actually slowing things down. - // Atom types; these aren't memoized, so don't check the memo. if (obj == PNone.NONE) { - saveNone(pickler, obj); + saveNone(pickler); return; - } else if (isBuiltinClass(type, PythonBuiltinClassType.Boolean)) { - saveBool(frame, pickler, obj); + } else if (obj instanceof Boolean value) { + saveBool(pickler, value); return; - } else if (isBuiltinClass(type, PythonBuiltinClassType.PInt)) { - saveLong(frame, pickler, obj); + } else if (obj instanceof Integer value) { + saveLong(pickler, value); return; - } else if (isBuiltinClass(type, PythonBuiltinClassType.PFloat)) { - saveFloat(frame, pickler, obj, inliningTarget, asDoubleNode); + } else if (obj instanceof Long value) { + saveLong(frame, pickler, value); + return; + } else if (obj instanceof PInt value && PGuards.isBuiltinPInt(value)) { + saveLong(frame, pickler, value); + return; + } else if (obj instanceof Double value) { + saveFloat(pickler, value); + return; + } else if (obj instanceof PFloat value && PGuards.isBuiltinPFloat(value)) { + saveFloat(pickler, value.getValue()); return; } @@ -2017,29 +2011,35 @@ void saveGeneric(VirtualFrame frame, PPickler pickler, Object objArg, int persSa PythonContext ctx = PythonContext.get(this); - if (isBuiltinClass(type, PythonBuiltinClassType.PBytes)) { - saveBytes(frame, ctx, pickler, obj, interopCallData); + if (obj instanceof PBytes bytes && PGuards.isBuiltinBytes(bytes)) { + saveBytes(frame, ctx, pickler, bytes); + return; + } else if (obj instanceof TruffleString string) { + saveUnicode(frame, pickler, string); return; - } else if (isBuiltinClass(type, PythonBuiltinClassType.PString)) { - saveUnicode(frame, pickler, obj); + } else if (obj instanceof PString string && PGuards.isBuiltinPString(string)) { + saveUnicode(frame, pickler, string); return; - } else if (isBuiltinClass(type, PythonBuiltinClassType.PDict)) { - saveDict(frame, inliningTarget, callMethod, pickler, obj); + } else if (obj instanceof PDict dict && PGuards.isBuiltinDict(dict)) { + saveDict(frame, inliningTarget, callMethod, pickler, dict); return; - } else if (isBuiltinClass(type, PythonBuiltinClassType.PSet)) { - saveSet(frame, ctx, pickler, obj); + } else if (obj instanceof PSet set && PGuards.isBuiltinSet(set)) { + saveSet(frame, ctx, pickler, set); return; - } else if (isBuiltinClass(type, PythonBuiltinClassType.PFrozenSet)) { - saveFrozenset(frame, ctx, pickler, obj); + } else if (obj instanceof PFrozenSet frozenSet && PGuards.isBuiltinFrozenSet(frozenSet)) { + saveFrozenset(frame, ctx, pickler, frozenSet); return; - } else if (isBuiltinClass(type, PythonBuiltinClassType.PList)) { - saveList(frame, pickler, obj); + } else if (obj instanceof PList list && PGuards.isBuiltinList(list)) { + saveList(frame, pickler, list); return; - } else if (isBuiltinClass(type, PythonBuiltinClassType.PTuple)) { - saveTuple(frame, pickler, obj); + } else if (obj instanceof PTuple tuple && PGuards.isBuiltinTuple(tuple)) { + saveTuple(frame, pickler, tuple); return; - } else if (isBuiltinClass(type, PythonBuiltinClassType.PByteArray)) { - saveBytearray(frame, inliningTarget, ctx, pickler, obj, interopCallData); + } else if (PGuards.isNativeObject(obj) && isBuiltinClass(getClass(obj), PythonBuiltinClassType.PTuple)) { + saveNativeTuple(frame, pickler, obj); + return; + } else if (obj instanceof PByteArray byteArray && PGuards.isBuiltinByteArray(byteArray)) { + saveBytearray(frame, inliningTarget, ctx, pickler, byteArray); return; } else if (obj instanceof PPickleBuffer buffer) { savePickleBuffer(frame, pickler, buffer); @@ -2056,10 +2056,13 @@ void saveGeneric(VirtualFrame frame, PPickler pickler, Object objArg, int persSa } } - if (isBuiltinClass(type, PythonBuiltinClassType.PythonClass)) { - saveType(frame, boundaryCallData, ctx, pickler, obj); + if (obj instanceof PythonManagedClass clazz && clazz.getPythonClass() == PythonBuiltinClassType.PythonClass) { + saveType(frame, boundaryCallData, ctx, pickler, clazz); + return; + } else if (obj instanceof PythonNativeClass && isBuiltinClass(getClass(obj), PythonBuiltinClassType.PythonClass)) { + saveGlobal(frame, boundaryCallData, ctx, pickler, obj, null); return; - } else if (isBuiltinClass(type, PythonBuiltinClassType.PFunction)) { + } else if (obj instanceof PFunction) { saveGlobal(frame, boundaryCallData, ctx, pickler, obj, null); return; } @@ -2067,6 +2070,7 @@ void saveGeneric(VirtualFrame frame, PPickler pickler, Object objArg, int persSa // Get a reduction callable, and call it. This may come from self.dispatch_table, // copyreg.dispatch_table, the object's __reduce_ex__ method, or the object's __reduce__ // method. + Object type = getClass(obj); Object reduceFunc = null; if (pickler.dispatchTable == null) { PickleState state = getGlobalState(ctx.getCore()); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/PGuards.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/PGuards.java index 2f2a5daa95..f86db3fc36 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/PGuards.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/PGuards.java @@ -309,6 +309,10 @@ public static boolean isBuiltinPInt(PInt obj) { return obj.getPythonClass() == PythonBuiltinClassType.PInt; } + public static boolean isBuiltinPFloat(PFloat obj) { + return obj.getPythonClass() == PythonBuiltinClassType.PFloat; + } + public static boolean isPString(Object obj) { return obj instanceof PString; } From ce1f410d4dee3c396d7fb0ab489f3ff2da437c52 Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Fri, 25 Sep 2026 09:28:16 +0200 Subject: [PATCH 04/39] Profile version and fast mode when pickling --- .../builtins/modules/pickle/PPickler.java | 296 +++++++++--------- 1 file changed, 153 insertions(+), 143 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java index 0db29d7729..a569141a3f 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java @@ -130,6 +130,8 @@ import com.oracle.truffle.api.frame.VirtualFrame; import com.oracle.truffle.api.nodes.Node; import com.oracle.truffle.api.object.Shape; +import com.oracle.truffle.api.profiles.InlinedConditionProfile; +import com.oracle.truffle.api.profiles.InlinedIntValueProfile; import com.oracle.truffle.api.strings.TranscodingErrorHandler; import com.oracle.truffle.api.strings.TruffleString; @@ -767,6 +769,10 @@ private PythonBufferAccessLibrary getBufferLibrary() { public abstract void execute(VirtualFrame frame, PPickler pickler, Object obj, int persSave); + private static boolean isBin(int proto) { + return proto > 0; + } + // collections private void saveHashingStorageBatched(VirtualFrame frame, PPickler pickler, HashingStorage storage, byte opcodeBatch, TruffleString name, boolean saveValues) { @@ -919,7 +925,7 @@ private void saveDictIterator(VirtualFrame frame, PPickler pickler, Object itera // save methods - private boolean savePers(VirtualFrame frame, PPickler pickler, Object obj) { + private boolean savePers(VirtualFrame frame, PPickler pickler, int proto, Object obj) { Object pid; if (pickler.persFuncSelf == null) { pid = getCallNode().execute(frame, pickler.persFunc, obj); @@ -927,7 +933,7 @@ private boolean savePers(VirtualFrame frame, PPickler pickler, Object obj) { pid = getCallNode().execute(frame, pickler.persFunc, pickler.persFuncSelf, obj); } if (pid != PNone.NONE) { - if (pickler.isBin()) { + if (isBin(proto)) { save(frame, pickler, pid, 1); write(pickler, PickleUtils.OPCODE_BINPERSID); } else { @@ -946,8 +952,8 @@ private boolean savePers(VirtualFrame frame, PPickler pickler, Object obj) { } // memo methods - private void memoGet(PPickler pickler, int value) { - if (pickler.isBin()) { + private void memoGet(PPickler pickler, int proto, int value) { + if (isBin(proto)) { if (value < 256) { writeByteOp(pickler, PickleUtils.OPCODE_BINGET, value); } else if (Long.compareUnsigned(value, 0xffffffffL) <= 0) { @@ -962,18 +968,18 @@ private void memoGet(PPickler pickler, int value) { } - private void memoPut(PPickler pickler, Object obj) { - if (pickler.isFast()) { + private void memoPut(PPickler pickler, int proto, boolean fastMode, Object obj) { + if (fastMode) { return; } int idx = pickler.memo.size(); pickler.memo.set(obj, idx); - if (pickler.proto >= 4) { + if (proto >= 4) { write(pickler, PickleUtils.OPCODE_MEMOIZE); } else { - if (!pickler.isBin()) { + if (!isBin(proto)) { writeLongAsString(pickler, PickleUtils.OPCODE_PUT, idx); } else { if (idx < 256) { @@ -989,9 +995,9 @@ private void memoPut(PPickler pickler, Object obj) { } // save methods - private void handleReduce(VirtualFrame frame, BoundaryCallData boundaryCallData, PythonContext ctx, PPickler pickler, Object obj, Object reduceValue) { + private void handleReduce(VirtualFrame frame, BoundaryCallData boundaryCallData, PythonContext ctx, PPickler pickler, int proto, boolean fastMode, Object obj, Object reduceValue) { if (PGuards.isString(reduceValue)) { - saveGlobal(frame, boundaryCallData, ctx, pickler, obj, reduceValue); + saveGlobal(frame, boundaryCallData, ctx, pickler, proto, fastMode, obj, reduceValue); return; } @@ -999,10 +1005,10 @@ private void handleReduce(VirtualFrame frame, BoundaryCallData boundaryCallData, throw raise(PicklingError, ErrorMessages.S_MUST_RETURN_S_OR_S, T___REDUCE__, "string", "tuple"); } - saveReduce(frame, ctx, pickler, reduceValue, obj); + saveReduce(frame, ctx, pickler, proto, fastMode, reduceValue, obj); } - private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, Object arguments, Object obj) { + private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, int proto, boolean fastMode, Object arguments, Object obj) { // We're saving obj, and args is the 2-thru-5 tuple returned by the appropriate // __reduce__ method for obj. @@ -1053,7 +1059,7 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, throw raise(PicklingError, ErrorMessages.S_ELEM_REDUCE_MUST_BE_S_NOT_P, "sixth", "a function", stateSetter); } - if (pickler.proto >= 2) { + if (proto >= 2) { Object name = lookupAttribute(frame, callable, T___NAME__); if (name != PNone.NO_VALUE) { final TruffleString stringName = asString(name); @@ -1092,7 +1098,7 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, throw raise(PicklingError, ErrorMessages.S_ITEM_FROM_S_MUST_BE_S_NOT_P, "third", "NEWOBJ_EX", "a dict", kwargs); } - if (pickler.proto >= 4) { + if (proto >= 4) { save(frame, pickler, cls, 0); save(frame, pickler, args, 0); save(frame, pickler, kwargs, 0); @@ -1190,18 +1196,18 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, int memoIndex = pickler.memo.get(obj); if (memoIndex != -1) { write(pickler, PickleUtils.OPCODE_POP); - memoGet(pickler, memoIndex); + memoGet(pickler, proto, memoIndex); } else { - memoPut(pickler, obj); + memoPut(pickler, proto, fastMode, obj); } } if (listitems != null) { - batchList(frame, pickler, listitems); + batchList(frame, pickler, proto, listitems); } if (dictitems != null) { - batchDict(frame, pickler, dictitems); + batchDict(frame, pickler, proto, dictitems); } if (state != null) { @@ -1230,8 +1236,8 @@ private void saveNone(PPickler pickler) { write(pickler, PickleUtils.OPCODE_NONE); } - private void saveBool(PPickler pickler, boolean value) { - if (pickler.proto >= 2) { + private void saveBool(PPickler pickler, int proto, boolean value) { + if (proto >= 2) { write(pickler, value ? PickleUtils.OPCODE_NEWTRUE : PickleUtils.OPCODE_NEWFALSE); } else { // These aren't opcodes -- they're ways to pickle bools before protocol 2 so that @@ -1242,8 +1248,8 @@ private void saveBool(PPickler pickler, boolean value) { } } - private void saveLong(PPickler pickler, int value) { - if (pickler.isBin()) { + private void saveLong(PPickler pickler, int proto, int value) { + if (isBin(proto)) { if (value >= 0 && value < (1 << 8)) { writeByteOp(pickler, PickleUtils.OPCODE_BININT1, value); } else if (value >= 0 && value < (1 << 16)) { @@ -1256,25 +1262,25 @@ private void saveLong(PPickler pickler, int value) { } } - private void saveLong(VirtualFrame frame, PPickler pickler, long value) { + private void saveLong(VirtualFrame frame, PPickler pickler, int proto, long value) { int intValue = (int) value; if (intValue == value) { - saveLong(pickler, intValue); + saveLong(pickler, proto, intValue); } else { - saveLongLarge(frame, pickler, value); + saveLongLarge(frame, pickler, proto, value); } } - private void saveLong(VirtualFrame frame, PPickler pickler, PInt value) { + private void saveLong(VirtualFrame frame, PPickler pickler, int proto, PInt value) { try { - saveLong(frame, pickler, value.longValueExact()); + saveLong(frame, pickler, proto, value.longValueExact()); } catch (OverflowException e) { - saveLongLarge(frame, pickler, value); + saveLongLarge(frame, pickler, proto, value); } } - private void saveLongLarge(VirtualFrame frame, PPickler pickler, Object obj) { - if (pickler.proto >= 2) { + private void saveLongLarge(VirtualFrame frame, PPickler pickler, int proto, Object obj) { + if (proto >= 2) { // Linear-time pickling. final int sign = getSign(obj); assert sign != 0; // Zero was handled by the int code above @@ -1315,8 +1321,8 @@ private void saveLongLarge(VirtualFrame frame, PPickler pickler, Object obj) { } } - private void saveFloat(PPickler pickler, double value) { - if (pickler.isBin()) { + private void saveFloat(PPickler pickler, int proto, double value) { + if (isBin(proto)) { write(pickler, PickleUtils.OPCODE_BINFLOAT); ensureBufferSpace(pickler, 8); NumericSupport.bigEndian().putDouble(pickler.outputBuffer, pickler.outputLen, value); @@ -1329,11 +1335,11 @@ private void saveFloat(PPickler pickler, double value) { } } - private void saveBytes(VirtualFrame frame, PythonContext ctx, PPickler pickler, PBytes obj) { + private void saveBytes(VirtualFrame frame, PythonContext ctx, PPickler pickler, int proto, boolean fastMode, PBytes obj) { Object buffer = getBufferAcquireLibrary().acquireReadonly(obj); PythonBufferAccessLibrary bufferLib = getBufferLibrary(); try { - if (pickler.proto < 3) { + if (proto < 3) { // Older pickle protocols do not have an opcode for pickling bytes objects. // Therefore, we need to fake the copy protocol (i.e., the __reduce__ method) to // permit bytes object unpickling. Here we use a hack to be compatible with @@ -1354,17 +1360,17 @@ private void saveBytes(VirtualFrame frame, PythonContext ctx, PPickler pickler, reduceValue = createTuple(st.codecsEncode, createTuple(unicodeStr, LATIN1)); } // save_reduce() will memoize the object automatically. - saveReduce(frame, ctx, pickler, reduceValue, obj); + saveReduce(frame, ctx, pickler, proto, fastMode, reduceValue, obj); } else { - saveBytesData(frame, pickler, obj, buffer, bufferLib); + saveBytesData(frame, pickler, proto, fastMode, obj, buffer, bufferLib); } } finally { bufferLib.release(buffer); } } - private void saveBytesData(VirtualFrame frame, PPickler pickler, Object obj, Object buffer, PythonBufferAccessLibrary bufferLib) { - assert pickler.proto >= 3; + private void saveBytesData(VirtualFrame frame, PPickler pickler, int proto, boolean fastMode, Object obj, Object buffer, PythonBufferAccessLibrary bufferLib) { + assert proto >= 3; int size = bufferLib.getBufferLength(buffer); boolean wasFraming = pickler.framing; boolean bypassBuffer = bypassBuffer(pickler, size); @@ -1386,10 +1392,10 @@ private void saveBytesData(VirtualFrame frame, PPickler pickler, Object obj, Obj if (bypassBuffer && wasFraming) { pickler.framing = true; } - memoPut(pickler, obj); + memoPut(pickler, proto, fastMode, obj); } - private void writeUnicodeBinary(VirtualFrame frame, PPickler pickler, TruffleString string) { + private void writeUnicodeBinary(VirtualFrame frame, PPickler pickler, int proto, TruffleString string) { TruffleString utf8 = ensureTsSwitchEncodingNode().execute(string, TruffleString.Encoding.UTF_8, TranscodingErrorHandler.DEFAULT_KEEP_SURROGATES_IN_UTF8); int size = utf8.byteLength(TruffleString.Encoding.UTF_8); boolean wasFraming = pickler.framing; @@ -1403,7 +1409,7 @@ private void writeUnicodeBinary(VirtualFrame frame, PPickler pickler, TruffleStr getCallNode().execute(frame, pickler.write, PFactory.createBytes(PythonLanguage.get(this), data, size)); pickler.clearBuffer(); } else { - if (size <= 0xff && pickler.proto >= 4) { + if (size <= 0xff && proto >= 4) { writeByteOp(pickler, PickleUtils.OPCODE_SHORT_BINUNICODE, size); } else { writeIntOp(pickler, PickleUtils.OPCODE_BINUNICODE, size); @@ -1416,23 +1422,23 @@ private void writeUnicodeBinary(VirtualFrame frame, PPickler pickler, TruffleStr } } - private void saveUnicode(VirtualFrame frame, PPickler pickler, TruffleString obj) { - saveUnicode(frame, pickler, obj, obj); + private void saveUnicode(VirtualFrame frame, PPickler pickler, int proto, boolean fastMode, TruffleString obj) { + saveUnicode(frame, pickler, proto, fastMode, obj, obj); } - private void saveUnicode(VirtualFrame frame, PPickler pickler, PString obj) { - saveUnicode(frame, pickler, obj, asStringStrict(obj)); + private void saveUnicode(VirtualFrame frame, PPickler pickler, int proto, boolean fastMode, PString obj) { + saveUnicode(frame, pickler, proto, fastMode, obj, asStringStrict(obj)); } - private void saveUnicode(VirtualFrame frame, PPickler pickler, Object obj, TruffleString string) { - if (pickler.isBin()) { - writeUnicodeBinary(frame, pickler, string); + private void saveUnicode(VirtualFrame frame, PPickler pickler, int proto, boolean fastMode, Object obj, TruffleString string) { + if (isBin(proto)) { + writeUnicodeBinary(frame, pickler, proto, string); } else { write(pickler, PickleUtils.OPCODE_UNICODE); writeRawUnicodeEscape(pickler, string); writeASCII(pickler, T_NEWLINE); } - memoPut(pickler, obj); + memoPut(pickler, proto, fastMode, obj); } private void writeRawUnicodeEscape(PPickler pickler, TruffleString string) { @@ -1486,9 +1492,9 @@ private void batchDictExact(VirtualFrame frame, PPickler pickler, PDict dict) { } } - private void batchDict(VirtualFrame frame, PPickler pickler, Object iterator) { + private void batchDict(VirtualFrame frame, PPickler pickler, int proto, Object iterator) { assert iterator != null; - if (pickler.proto == 0) { + if (proto == 0) { // SETITEMS isn't available; do one at a time. saveDictIterator(frame, pickler, iterator); } else { @@ -1497,30 +1503,30 @@ private void batchDict(VirtualFrame frame, PPickler pickler, Object iterator) { } } - private void saveDict(VirtualFrame frame, Node inliningTarget, PyObjectCallMethodObjArgs callMethod, PPickler pickler, PDict obj) { - if (pickler.isFast()) { + private void saveDict(VirtualFrame frame, Node inliningTarget, PyObjectCallMethodObjArgs callMethod, PPickler pickler, int proto, boolean fastMode, PDict obj) { + if (fastMode) { fastSaveEnter(pickler, obj); } // Create an empty dict. - if (pickler.isBin()) { + if (isBin(proto)) { write(pickler, PickleUtils.OPCODE_EMPTY_DICT); } else { write(pickler, PickleUtils.OPCODE_MARK, PickleUtils.OPCODE_DICT); } - memoPut(pickler, obj); + memoPut(pickler, proto, fastMode, obj); if (getHashingStorageLength(obj.getDictStorage()) > 0) { - if (pickler.proto > 0) { + if (proto > 0) { batchDictExact(frame, pickler, obj); } else { final Object items = callMethod.execute(frame, inliningTarget, obj, T_ITEMS); - batchDict(frame, pickler, getIter(frame, items)); + batchDict(frame, pickler, proto, getIter(frame, items)); } } - if (pickler.isFast()) { + if (fastMode) { fastSaveLeave(pickler, obj); } } @@ -1529,17 +1535,17 @@ private void batchSetExact(VirtualFrame frame, PPickler pickler, PSet set) { saveSetHashingStorageBatched(frame, pickler, set.getDictStorage()); } - private void saveSet(VirtualFrame frame, PythonContext ctx, PPickler pickler, PSet obj) { + private void saveSet(VirtualFrame frame, PythonContext ctx, PPickler pickler, int proto, boolean fastMode, PSet obj) { Object reduceValue; - if (pickler.proto < 4) { + if (proto < 4) { Object items = createList(frame, obj); reduceValue = createTuple(ctx.getCore().lookupType(PythonBuiltinClassType.PSet), createTuple(items)); - saveReduce(frame, ctx, pickler, reduceValue, obj); + saveReduce(frame, ctx, pickler, proto, fastMode, reduceValue, obj); return; } write(pickler, PickleUtils.OPCODE_EMPTY_SET); - memoPut(pickler, obj); + memoPut(pickler, proto, fastMode, obj); final int setSize = length(frame, obj); if (setSize == 0) { @@ -1551,15 +1557,15 @@ private void saveSet(VirtualFrame frame, PythonContext ctx, PPickler pickler, PS batchSetExact(frame, pickler, obj); } - private void saveFrozenset(VirtualFrame frame, PythonContext ctx, PPickler pickler, PFrozenSet obj) { - if (pickler.isFast()) { + private void saveFrozenset(VirtualFrame frame, PythonContext ctx, PPickler pickler, int proto, boolean fastMode, PFrozenSet obj) { + if (fastMode) { fastSaveEnter(pickler, obj); } - if (pickler.proto < 4) { + if (proto < 4) { Object items = createList(frame, obj); Object reduceValue = createTuple(ctx.getCore().lookupType(PythonBuiltinClassType.PFrozenSet), createTuple(items)); - saveReduce(frame, ctx, pickler, reduceValue, obj); + saveReduce(frame, ctx, pickler, proto, fastMode, reduceValue, obj); return; } @@ -1572,12 +1578,12 @@ private void saveFrozenset(VirtualFrame frame, PythonContext ctx, PPickler pickl int memoIndex = pickler.memo.get(obj); if (memoIndex != -1) { write(pickler, PickleUtils.OPCODE_POP_MARK); - memoGet(pickler, memoIndex); + memoGet(pickler, proto, memoIndex); return; } write(pickler, PickleUtils.OPCODE_FROZENSET); - memoPut(pickler, obj); + memoPut(pickler, proto, fastMode, obj); } private void batchListExact(VirtualFrame frame, PPickler pickler, PList obj) { @@ -1608,9 +1614,9 @@ private void batchListExact(VirtualFrame frame, PPickler pickler, PList obj) { } while (total < storage.length()); } - private void batchList(VirtualFrame frame, PPickler pickler, Object iterator) { + private void batchList(VirtualFrame frame, PPickler pickler, int proto, Object iterator) { assert iterator != null; - if (pickler.proto == 0) { + if (proto == 0) { // APPENDS isn't available; do one at a time. saveListIterator(frame, pickler, iterator); } else { @@ -1619,30 +1625,30 @@ private void batchList(VirtualFrame frame, PPickler pickler, Object iterator) { } } - private void saveList(VirtualFrame frame, PPickler pickler, PList obj) { - if (pickler.isFast()) { + private void saveList(VirtualFrame frame, PPickler pickler, int proto, boolean fastMode, PList obj) { + if (fastMode) { fastSaveEnter(pickler, obj); } // Create an empty list. - if (pickler.isBin()) { + if (isBin(proto)) { write(pickler, PickleUtils.OPCODE_EMPTY_LIST); } else { write(pickler, PickleUtils.OPCODE_MARK, PickleUtils.OPCODE_LIST); } int len = obj.getSequenceStorage().length(); - memoPut(pickler, obj); + memoPut(pickler, proto, fastMode, obj); if (len != 0) { - if (pickler.proto > 0) { + if (proto > 0) { batchListExact(frame, pickler, obj); } else { - batchList(frame, pickler, getIter(frame, obj)); + batchList(frame, pickler, proto, getIter(frame, obj)); } } - if (pickler.isFast()) { + if (fastMode) { fastSaveLeave(pickler, obj); } } @@ -1661,19 +1667,19 @@ private void storeTupleElements(VirtualFrame frame, PPickler pickler, Object obj } } - private void saveTuple(VirtualFrame frame, PPickler pickler, PTuple obj) { + private void saveTuple(VirtualFrame frame, PPickler pickler, int proto, boolean fastMode, PTuple obj) { SequenceStorage storage = obj.getSequenceStorage(); - saveTuple(frame, pickler, obj, storage, storage.length()); + saveTuple(frame, pickler, proto, fastMode, obj, storage, storage.length()); } - private void saveNativeTuple(VirtualFrame frame, PPickler pickler, Object obj) { - saveTuple(frame, pickler, obj, null, length(frame, obj)); + private void saveNativeTuple(VirtualFrame frame, PPickler pickler, int proto, boolean fastMode, Object obj) { + saveTuple(frame, pickler, proto, fastMode, obj, null, length(frame, obj)); } - private void saveTuple(VirtualFrame frame, PPickler pickler, Object obj, SequenceStorage storage, int len) { + private void saveTuple(VirtualFrame frame, PPickler pickler, int proto, boolean fastMode, Object obj, SequenceStorage storage, int len) { if (len == 0) { - if (pickler.proto != 0) { + if (proto != 0) { write(pickler, PickleUtils.OPCODE_EMPTY_TUPLE); } else { write(pickler, PickleUtils.OPCODE_MARK, PickleUtils.OPCODE_TUPLE); @@ -1684,7 +1690,7 @@ private void saveTuple(VirtualFrame frame, PPickler pickler, Object obj, Sequenc // The tuple isn't in the memo now. If it shows up there after saving the tuple // elements, the tuple must be recursive, in which case we'll pop everything we put on // the stack, and fetch its value from the memo. - if (len <= 3 && pickler.proto >= 2) { + if (len <= 3 && proto >= 2) { // Use TUPLE{1,2,3} opcodes. storeTupleElements(frame, pickler, obj, storage, len); @@ -1695,14 +1701,14 @@ private void saveTuple(VirtualFrame frame, PPickler pickler, Object obj, Sequenc write(pickler, PickleUtils.OPCODE_POP); } // fetch from memo - memoGet(pickler, memoIndex); + memoGet(pickler, proto, memoIndex); return; } else { // Not recursive. write(pickler, TUPLE_LEN_2_OPCODE[len]); } - memoPut(pickler, obj); + memoPut(pickler, proto, fastMode, obj); return; } @@ -1713,7 +1719,7 @@ private void saveTuple(VirtualFrame frame, PPickler pickler, Object obj, Sequenc int memoIndex = pickler.memo.get(obj); if (memoIndex != -1) { // pop the stack stuff we pushed - if (pickler.isBin()) { + if (isBin(proto)) { write(pickler, PickleUtils.OPCODE_POP_MARK); } else { // Note that we pop one more than len, to remove the MARK too. @@ -1722,18 +1728,18 @@ private void saveTuple(VirtualFrame frame, PPickler pickler, Object obj, Sequenc } } // fetch from memo - memoGet(pickler, memoIndex); + memoGet(pickler, proto, memoIndex); return; } else { // Not recursive. write(pickler, PickleUtils.OPCODE_TUPLE); } - memoPut(pickler, obj); + memoPut(pickler, proto, fastMode, obj); } - private void saveBytearrayData(VirtualFrame frame, PPickler pickler, Object obj, Object buffer, PythonBufferAccessLibrary bufferLib) { - assert pickler.proto >= 5; + private void saveBytearrayData(VirtualFrame frame, PPickler pickler, int proto, boolean fastMode, Object obj, Object buffer, PythonBufferAccessLibrary bufferLib) { + assert proto >= 5; int size = bufferLib.getBufferLength(buffer); boolean wasFraming = pickler.framing; @@ -1755,14 +1761,14 @@ private void saveBytearrayData(VirtualFrame frame, PPickler pickler, Object obj, if (bypassBuffer && wasFraming) { pickler.framing = true; } - memoPut(pickler, obj); + memoPut(pickler, proto, fastMode, obj); } - private void saveBytearray(VirtualFrame frame, Node inliningTarget, PythonContext ctx, PPickler pickler, PByteArray obj) { + private void saveBytearray(VirtualFrame frame, Node inliningTarget, PythonContext ctx, PPickler pickler, int proto, boolean fastMode, PByteArray obj) { Object buffer = getBufferAcquireLibrary().acquireReadonly(obj); PythonBufferAccessLibrary bufferLib = getBufferLibrary(); try { - if (pickler.proto < 5) { + if (proto < 5) { // Older pickle protocols do not have an opcode for pickling bytearrays. Object reduceValue; @@ -1775,17 +1781,17 @@ private void saveBytearray(VirtualFrame frame, Node inliningTarget, PythonContex } // save_reduce() will memoize the object automatically. - saveReduce(frame, ctx, pickler, reduceValue, obj); + saveReduce(frame, ctx, pickler, proto, fastMode, reduceValue, obj); } else { - saveBytearrayData(frame, pickler, obj, buffer, bufferLib); + saveBytearrayData(frame, pickler, proto, fastMode, obj, buffer, bufferLib); } } finally { bufferLib.release(buffer); } } - private void savePickleBuffer(VirtualFrame frame, PPickler pickler, PPickleBuffer obj) { - if (pickler.proto < 5) { + private void savePickleBuffer(VirtualFrame frame, PPickler pickler, int proto, boolean fastMode, PPickleBuffer obj) { + if (proto < 5) { throw raise(PicklingError, ErrorMessages.PICKLEBUFF_CANNOT_PICKLE_WITH_PROTO5); } @@ -1802,9 +1808,9 @@ private void savePickleBuffer(VirtualFrame frame, PPickler pickler, PPickleBuffe if (inBand) { // Write data in-band if (readOnly) { - saveBytesData(frame, pickler, obj, buffer, bufferLib); + saveBytesData(frame, pickler, proto, fastMode, obj, buffer, bufferLib); } else { - saveBytearrayData(frame, pickler, obj, buffer, bufferLib); + saveBytearrayData(frame, pickler, proto, fastMode, obj, buffer, bufferLib); } } else { // Write data out-of-band @@ -1815,24 +1821,24 @@ private void savePickleBuffer(VirtualFrame frame, PPickler pickler, PPickleBuffe } } - private void saveSingletonType(VirtualFrame frame, PythonContext ctx, PPickler pickler, Object obj, Object singleton) { + private void saveSingletonType(VirtualFrame frame, PythonContext ctx, PPickler pickler, int proto, boolean fastMode, Object obj, Object singleton) { Object reduceValue = createTuple(ctx.getCore().lookupType(PythonBuiltinClassType.PythonClass), createTuple(singleton)); - saveReduce(frame, ctx, pickler, reduceValue, obj); + saveReduce(frame, ctx, pickler, proto, fastMode, reduceValue, obj); } - private void saveType(VirtualFrame frame, BoundaryCallData boundaryCallData, PythonContext ctx, PPickler pickler, PythonManagedClass obj) { + private void saveType(VirtualFrame frame, BoundaryCallData boundaryCallData, PythonContext ctx, PPickler pickler, int proto, boolean fastMode, PythonManagedClass obj) { if (obj instanceof PythonBuiltinClass cls && cls.getType() == PythonBuiltinClassType.PNone) { - saveSingletonType(frame, ctx, pickler, obj, PNone.NONE); + saveSingletonType(frame, ctx, pickler, proto, fastMode, obj, PNone.NONE); } else if (obj instanceof PythonBuiltinClass cls && cls.getType() == PythonBuiltinClassType.PEllipsis) { - saveSingletonType(frame, ctx, pickler, obj, PEllipsis.INSTANCE); + saveSingletonType(frame, ctx, pickler, proto, fastMode, obj, PEllipsis.INSTANCE); } else if (obj instanceof PythonBuiltinClass cls && cls.getType() == PythonBuiltinClassType.PNotImplemented) { - saveSingletonType(frame, ctx, pickler, obj, PNotImplemented.NOT_IMPLEMENTED); + saveSingletonType(frame, ctx, pickler, proto, fastMode, obj, PNotImplemented.NOT_IMPLEMENTED); } else { - saveGlobal(frame, boundaryCallData, ctx, pickler, obj, null); + saveGlobal(frame, boundaryCallData, ctx, pickler, proto, fastMode, obj, null); } } - private void saveGlobal(VirtualFrame frame, BoundaryCallData boundaryCallData, PythonContext ctx, PPickler pickler, Object obj, Object name) { + private void saveGlobal(VirtualFrame frame, BoundaryCallData boundaryCallData, PythonContext ctx, PPickler pickler, int proto, boolean fastMode, Object obj, Object name) { PickleState st = getGlobalState(ctx.getCore()); Object gName; if (name != null) { @@ -1865,7 +1871,7 @@ private void saveGlobal(VirtualFrame frame, BoundaryCallData boundaryCallData, P } boolean genGlobal; - if (pickler.proto >= 2) { + if (proto >= 2) { genGlobal = false; // See whether this is in the extension registry, and if so generate an EXT opcode. @@ -1908,20 +1914,20 @@ private void saveGlobal(VirtualFrame frame, BoundaryCallData boundaryCallData, P globalName = lastname; } - if (pickler.proto >= 4) { + if (proto >= 4) { save(frame, pickler, moduleName, 0); save(frame, pickler, globalName, 0); write(pickler, PickleUtils.OPCODE_STACK_GLOBAL); } else if (parent != module) { Object reduceValue = createTuple(st.getattr, createTuple(parent, lastname)); - saveReduce(frame, ctx, pickler, reduceValue, null); + saveReduce(frame, ctx, pickler, proto, fastMode, reduceValue, null); } else { // Generate a normal global opcode if we are using a pickle protocol < 4, or if // the object is not registered in the extension registry write(pickler, PickleUtils.OPCODE_GLOBAL); // For protocol < 3 and if the user didn't request against doing so, we convert // module names to the old 2.x module names. - if (pickler.proto < 3 && pickler.fixImports) { + if (proto < 3 && pickler.fixImports) { final Pair to2Mapping = get3to2Mapping(frame, ctx.getCore(), moduleName, globalName); moduleName = to2Mapping.getLeft(); globalName = to2Mapping.getRight(); @@ -1931,10 +1937,10 @@ private void saveGlobal(VirtualFrame frame, BoundaryCallData boundaryCallData, P // module name and the global name using UTF-8. We do so only when we are using // the pickle protocol newer than version 3. This is to ensure compatibility // with older Unpickler running on Python 2.x. - if (pickler.proto >= 3) { + if (proto >= 3) { if (ensureTsGetCodeRangeNode().execute(moduleName, TS_ENCODING) == TruffleString.CodeRange.BROKEN || ensureTsGetCodeRangeNode().execute(globalName, TS_ENCODING) == TruffleString.CodeRange.BROKEN) { - throw raise(PicklingError, ErrorMessages.CANT_PICKLE_MODULE_S_USING_PROTO_D, moduleName, pickler.proto); + throw raise(PicklingError, ErrorMessages.CANT_PICKLE_MODULE_S_USING_PROTO_D, moduleName, proto); } writeUtf8(pickler, moduleName); write(pickler, (byte) '\n'); @@ -1942,7 +1948,7 @@ private void saveGlobal(VirtualFrame frame, BoundaryCallData boundaryCallData, P } else { if (ensureTsGetCodeRangeNode().execute(moduleName, TS_ENCODING) != TruffleString.CodeRange.ASCII || ensureTsGetCodeRangeNode().execute(globalName, TS_ENCODING) != TruffleString.CodeRange.ASCII) { - throw raise(PicklingError, ErrorMessages.CANT_PICKLE_MODULE_S_USING_PROTO_D, moduleName, pickler.proto); + throw raise(PicklingError, ErrorMessages.CANT_PICKLE_MODULE_S_USING_PROTO_D, moduleName, proto); } writeASCII(pickler, moduleName); write(pickler, (byte) '\n'); @@ -1951,7 +1957,7 @@ private void saveGlobal(VirtualFrame frame, BoundaryCallData boundaryCallData, P write(pickler, (byte) '\n'); } // Memoize the object - memoPut(pickler, obj); + memoPut(pickler, proto, fastMode, obj); } } @@ -1960,14 +1966,18 @@ void saveGeneric(VirtualFrame frame, PPickler pickler, Object objArg, int persSa @Bind Node inliningTarget, @Cached("createFor($node)") BoundaryCallData boundaryCallData, @Cached CallNode callNode, - @Cached PyObjectCallMethodObjArgs callMethod) { + @Cached PyObjectCallMethodObjArgs callMethod, + @Cached InlinedIntValueProfile protoProfile, + @Cached InlinedConditionProfile fastModeProfile) { Object obj = objArg; + int proto = protoProfile.profile(inliningTarget, pickler.getProto()); + boolean fastMode = fastModeProfile.profile(inliningTarget, pickler.isFast()); opcodeBoundary(frame, pickler); // The extra pers_save argument is necessary to avoid calling save_pers() // on its returned object. if (persSave == 0 && pickler.persFunc != null) { - if (savePers(frame, pickler, obj)) { + if (savePers(frame, pickler, proto, obj)) { return; } } @@ -1977,22 +1987,22 @@ void saveGeneric(VirtualFrame frame, PPickler pickler, Object objArg, int persSa saveNone(pickler); return; } else if (obj instanceof Boolean value) { - saveBool(pickler, value); + saveBool(pickler, proto, value); return; } else if (obj instanceof Integer value) { - saveLong(pickler, value); + saveLong(pickler, proto, value); return; } else if (obj instanceof Long value) { - saveLong(frame, pickler, value); + saveLong(frame, pickler, proto, value); return; } else if (obj instanceof PInt value && PGuards.isBuiltinPInt(value)) { - saveLong(frame, pickler, value); + saveLong(frame, pickler, proto, value); return; } else if (obj instanceof Double value) { - saveFloat(pickler, value); + saveFloat(pickler, proto, value); return; } else if (obj instanceof PFloat value && PGuards.isBuiltinPFloat(value)) { - saveFloat(pickler, value.getValue()); + saveFloat(pickler, proto, value.getValue()); return; } @@ -2005,44 +2015,44 @@ void saveGeneric(VirtualFrame frame, PPickler pickler, Object objArg, int persSa // instead of pickling the object once again. int memoIndex = pickler.memo.get(obj); if (memoIndex != -1) { - memoGet(pickler, memoIndex); + memoGet(pickler, proto, memoIndex); return; } PythonContext ctx = PythonContext.get(this); if (obj instanceof PBytes bytes && PGuards.isBuiltinBytes(bytes)) { - saveBytes(frame, ctx, pickler, bytes); + saveBytes(frame, ctx, pickler, proto, fastMode, bytes); return; } else if (obj instanceof TruffleString string) { - saveUnicode(frame, pickler, string); + saveUnicode(frame, pickler, proto, fastMode, string); return; } else if (obj instanceof PString string && PGuards.isBuiltinPString(string)) { - saveUnicode(frame, pickler, string); + saveUnicode(frame, pickler, proto, fastMode, string); return; } else if (obj instanceof PDict dict && PGuards.isBuiltinDict(dict)) { - saveDict(frame, inliningTarget, callMethod, pickler, dict); + saveDict(frame, inliningTarget, callMethod, pickler, proto, fastMode, dict); return; } else if (obj instanceof PSet set && PGuards.isBuiltinSet(set)) { - saveSet(frame, ctx, pickler, set); + saveSet(frame, ctx, pickler, proto, fastMode, set); return; } else if (obj instanceof PFrozenSet frozenSet && PGuards.isBuiltinFrozenSet(frozenSet)) { - saveFrozenset(frame, ctx, pickler, frozenSet); + saveFrozenset(frame, ctx, pickler, proto, fastMode, frozenSet); return; } else if (obj instanceof PList list && PGuards.isBuiltinList(list)) { - saveList(frame, pickler, list); + saveList(frame, pickler, proto, fastMode, list); return; } else if (obj instanceof PTuple tuple && PGuards.isBuiltinTuple(tuple)) { - saveTuple(frame, pickler, tuple); + saveTuple(frame, pickler, proto, fastMode, tuple); return; } else if (PGuards.isNativeObject(obj) && isBuiltinClass(getClass(obj), PythonBuiltinClassType.PTuple)) { - saveNativeTuple(frame, pickler, obj); + saveNativeTuple(frame, pickler, proto, fastMode, obj); return; } else if (obj instanceof PByteArray byteArray && PGuards.isBuiltinByteArray(byteArray)) { - saveBytearray(frame, inliningTarget, ctx, pickler, byteArray); + saveBytearray(frame, inliningTarget, ctx, pickler, proto, fastMode, byteArray); return; } else if (obj instanceof PPickleBuffer buffer) { - savePickleBuffer(frame, pickler, buffer); + savePickleBuffer(frame, pickler, proto, fastMode, buffer); return; } @@ -2051,19 +2061,19 @@ void saveGeneric(VirtualFrame frame, PPickler pickler, Object objArg, int persSa if (pickler.reducerOverride != null) { Object reduceValue = callNode.execute(frame, pickler.reducerOverride, obj); if (reduceValue != PNotImplemented.NOT_IMPLEMENTED) { - handleReduce(frame, boundaryCallData, ctx, pickler, obj, reduceValue); + handleReduce(frame, boundaryCallData, ctx, pickler, proto, fastMode, obj, reduceValue); return; } } if (obj instanceof PythonManagedClass clazz && clazz.getPythonClass() == PythonBuiltinClassType.PythonClass) { - saveType(frame, boundaryCallData, ctx, pickler, clazz); + saveType(frame, boundaryCallData, ctx, pickler, proto, fastMode, clazz); return; } else if (obj instanceof PythonNativeClass && isBuiltinClass(getClass(obj), PythonBuiltinClassType.PythonClass)) { - saveGlobal(frame, boundaryCallData, ctx, pickler, obj, null); + saveGlobal(frame, boundaryCallData, ctx, pickler, proto, fastMode, obj, null); return; } else if (obj instanceof PFunction) { - saveGlobal(frame, boundaryCallData, ctx, pickler, obj, null); + saveGlobal(frame, boundaryCallData, ctx, pickler, proto, fastMode, obj, null); return; } @@ -2087,7 +2097,7 @@ void saveGeneric(VirtualFrame frame, PPickler pickler, Object objArg, int persSa if (reduceFunc != null) { reduceValue = callNode.execute(frame, reduceFunc, obj); } else if (isSubType(type, PythonBuiltinClassType.PythonClass)) { - saveGlobal(frame, boundaryCallData, ctx, pickler, obj, null); + saveGlobal(frame, boundaryCallData, ctx, pickler, proto, fastMode, obj, null); return; } else { // XXX: If the __reduce__ method is defined, __reduce_ex__ is automatically defined @@ -2099,7 +2109,7 @@ void saveGeneric(VirtualFrame frame, PPickler pickler, Object objArg, int persSa // Check for a __reduce_ex__ method. reduceFunc = lookupAttribute(frame, obj, T___REDUCE_EX__); if (reduceFunc != PNone.NO_VALUE) { - reduceValue = callNode.execute(frame, reduceFunc, pickler.proto); + reduceValue = callNode.execute(frame, reduceFunc, proto); } else { // Check for a __reduce__ method. reduceFunc = lookupAttribute(frame, obj, T___REDUCE__); @@ -2111,7 +2121,7 @@ void saveGeneric(VirtualFrame frame, PPickler pickler, Object objArg, int persSa } } - handleReduce(frame, boundaryCallData, ctx, pickler, obj, reduceValue); + handleReduce(frame, boundaryCallData, ctx, pickler, proto, fastMode, obj, reduceValue); } } From 964bd5ed2e54fabaef8860faefc70fca75226551 Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Fri, 25 Sep 2026 11:51:34 +0200 Subject: [PATCH 05/39] Add a recursion limit to PPickler --- .../builtins/modules/pickle/PPickler.java | 69 ++++++++++++++----- 1 file changed, 53 insertions(+), 16 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java index a569141a3f..b17836e755 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java @@ -113,6 +113,7 @@ import com.oracle.graal.python.nodes.call.CallNode; import com.oracle.graal.python.nodes.classes.IsSubtypeNode; import com.oracle.graal.python.nodes.object.IsNode; +import com.oracle.graal.python.runtime.ExecutionContext.BoundaryCallContext; import com.oracle.graal.python.runtime.IndirectCallData.BoundaryCallData; import com.oracle.graal.python.runtime.PythonContext; import com.oracle.graal.python.runtime.exception.PException; @@ -658,7 +659,11 @@ public void flush(VirtualFrame frame, PPickler pickler, } public abstract static class SaveNode extends BasePickleWriteNode { + private static final int MAX_RECURSION_DEPTH = 3; + + private final int depth; @Child private SaveNode recursiveSaveNode; + @Child private BoundaryCallData boundaryCallData; @Child private PyLongAsLongNode pyLongAsLongNode; @Child private PyObjectStrAsObjectNode pyObjectStrAsObjectNode; @Child private PyObjectIsTrueNode isTrueNode; @@ -671,6 +676,18 @@ public abstract static class SaveNode extends BasePickleWriteNode { @Child private HashingStorageLen hashingStorageLenNode; @Child private PyTupleCheckNode.CachedNode tupleCheckNode; + protected SaveNode(int depth) { + this.depth = depth; + } + + private BoundaryCallData getBoundaryCallData() { + if (boundaryCallData == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + boundaryCallData = insert(BoundaryCallData.createFor(this)); + } + return boundaryCallData; + } + private int getHashingStorageLength(HashingStorage storage) { if (hashingStorageLenNode == null) { CompilerDirectives.transferToInterpreterAndInvalidate(); @@ -688,11 +705,32 @@ private boolean isTuple(Object object) { } private void save(VirtualFrame frame, PPickler pickler, Object obj, int persSave) { + if (depth < 0) { + // This should never be reached for compilation, but native-image can't prove it + CompilerDirectives.transferToInterpreterAndInvalidate(); + execute(frame, pickler, obj, persSave); + return; + } if (recursiveSaveNode == null) { CompilerDirectives.transferToInterpreterAndInvalidate(); - recursiveSaveNode = insert(PPicklerFactory.SaveNodeGen.create()); + recursiveSaveNode = insert(PPicklerFactory.SaveNodeGen.create(depth < MAX_RECURSION_DEPTH ? depth + 1 : -1)); + } + if (depth < MAX_RECURSION_DEPTH) { + recursiveSaveNode.execute(frame, pickler, obj, persSave); + } else { + BoundaryCallData callData = getBoundaryCallData(); + Object savedState = BoundaryCallContext.enter(frame, callData); + try { + saveBoundary(pickler, obj, persSave); + } finally { + BoundaryCallContext.exit(frame, callData, savedState); + } } - recursiveSaveNode.execute(frame, pickler, obj, persSave); + } + + @TruffleBoundary + private void saveBoundary(PPickler pickler, Object obj, int persSave) { + recursiveSaveNode.execute(null, pickler, obj, persSave); } private long asLong(VirtualFrame frame, Object object) { @@ -995,9 +1033,9 @@ private void memoPut(PPickler pickler, int proto, boolean fastMode, Object obj) } // save methods - private void handleReduce(VirtualFrame frame, BoundaryCallData boundaryCallData, PythonContext ctx, PPickler pickler, int proto, boolean fastMode, Object obj, Object reduceValue) { + private void handleReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, int proto, boolean fastMode, Object obj, Object reduceValue) { if (PGuards.isString(reduceValue)) { - saveGlobal(frame, boundaryCallData, ctx, pickler, proto, fastMode, obj, reduceValue); + saveGlobal(frame, ctx, pickler, proto, fastMode, obj, reduceValue); return; } @@ -1826,7 +1864,7 @@ private void saveSingletonType(VirtualFrame frame, PythonContext ctx, PPickler p saveReduce(frame, ctx, pickler, proto, fastMode, reduceValue, obj); } - private void saveType(VirtualFrame frame, BoundaryCallData boundaryCallData, PythonContext ctx, PPickler pickler, int proto, boolean fastMode, PythonManagedClass obj) { + private void saveType(VirtualFrame frame, PythonContext ctx, PPickler pickler, int proto, boolean fastMode, PythonManagedClass obj) { if (obj instanceof PythonBuiltinClass cls && cls.getType() == PythonBuiltinClassType.PNone) { saveSingletonType(frame, ctx, pickler, proto, fastMode, obj, PNone.NONE); } else if (obj instanceof PythonBuiltinClass cls && cls.getType() == PythonBuiltinClassType.PEllipsis) { @@ -1834,11 +1872,11 @@ private void saveType(VirtualFrame frame, BoundaryCallData boundaryCallData, Pyt } else if (obj instanceof PythonBuiltinClass cls && cls.getType() == PythonBuiltinClassType.PNotImplemented) { saveSingletonType(frame, ctx, pickler, proto, fastMode, obj, PNotImplemented.NOT_IMPLEMENTED); } else { - saveGlobal(frame, boundaryCallData, ctx, pickler, proto, fastMode, obj, null); + saveGlobal(frame, ctx, pickler, proto, fastMode, obj, null); } } - private void saveGlobal(VirtualFrame frame, BoundaryCallData boundaryCallData, PythonContext ctx, PPickler pickler, int proto, boolean fastMode, Object obj, Object name) { + private void saveGlobal(VirtualFrame frame, PythonContext ctx, PPickler pickler, int proto, boolean fastMode, Object obj, Object name) { PickleState st = getGlobalState(ctx.getCore()); Object gName; if (name != null) { @@ -1856,7 +1894,7 @@ private void saveGlobal(VirtualFrame frame, BoundaryCallData boundaryCallData, P Object module; try { - module = importModule(frame, boundaryCallData, moduleName); + module = importModule(frame, getBoundaryCallData(), moduleName); } catch (PException e) { throw raise(PicklingError, ErrorMessages.CANT_PICKLE_P_IMPORT_OF_MODULE_S_FAILED, obj, moduleName); } @@ -1964,7 +2002,6 @@ private void saveGlobal(VirtualFrame frame, BoundaryCallData boundaryCallData, P @Specialization void saveGeneric(VirtualFrame frame, PPickler pickler, Object objArg, int persSave, @Bind Node inliningTarget, - @Cached("createFor($node)") BoundaryCallData boundaryCallData, @Cached CallNode callNode, @Cached PyObjectCallMethodObjArgs callMethod, @Cached InlinedIntValueProfile protoProfile, @@ -2061,19 +2098,19 @@ void saveGeneric(VirtualFrame frame, PPickler pickler, Object objArg, int persSa if (pickler.reducerOverride != null) { Object reduceValue = callNode.execute(frame, pickler.reducerOverride, obj); if (reduceValue != PNotImplemented.NOT_IMPLEMENTED) { - handleReduce(frame, boundaryCallData, ctx, pickler, proto, fastMode, obj, reduceValue); + handleReduce(frame, ctx, pickler, proto, fastMode, obj, reduceValue); return; } } if (obj instanceof PythonManagedClass clazz && clazz.getPythonClass() == PythonBuiltinClassType.PythonClass) { - saveType(frame, boundaryCallData, ctx, pickler, proto, fastMode, clazz); + saveType(frame, ctx, pickler, proto, fastMode, clazz); return; } else if (obj instanceof PythonNativeClass && isBuiltinClass(getClass(obj), PythonBuiltinClassType.PythonClass)) { - saveGlobal(frame, boundaryCallData, ctx, pickler, proto, fastMode, obj, null); + saveGlobal(frame, ctx, pickler, proto, fastMode, obj, null); return; } else if (obj instanceof PFunction) { - saveGlobal(frame, boundaryCallData, ctx, pickler, proto, fastMode, obj, null); + saveGlobal(frame, ctx, pickler, proto, fastMode, obj, null); return; } @@ -2097,7 +2134,7 @@ void saveGeneric(VirtualFrame frame, PPickler pickler, Object objArg, int persSa if (reduceFunc != null) { reduceValue = callNode.execute(frame, reduceFunc, obj); } else if (isSubType(type, PythonBuiltinClassType.PythonClass)) { - saveGlobal(frame, boundaryCallData, ctx, pickler, proto, fastMode, obj, null); + saveGlobal(frame, ctx, pickler, proto, fastMode, obj, null); return; } else { // XXX: If the __reduce__ method is defined, __reduce_ex__ is automatically defined @@ -2121,7 +2158,7 @@ void saveGeneric(VirtualFrame frame, PPickler pickler, Object objArg, int persSa } } - handleReduce(frame, boundaryCallData, ctx, pickler, proto, fastMode, obj, reduceValue); + handleReduce(frame, ctx, pickler, proto, fastMode, obj, reduceValue); } } @@ -2130,7 +2167,7 @@ public abstract static class DumpNode extends BasePickleWriteNode { @Specialization public void dump(VirtualFrame frame, PPickler pickler, Object obj, - @Cached SaveNode saveNode) { + @Cached("create(0)") SaveNode saveNode) { try { final Object tmp = getLookupAttrNode().executeCached(frame, pickler, REDUCE_OVERRIDE); if (tmp != PNone.NO_VALUE) { From 1011631ac95883157242b98cd1d727023c629d53 Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Fri, 25 Sep 2026 13:38:14 +0200 Subject: [PATCH 06/39] Add more profiling to PPickler --- .../builtins/modules/pickle/PPickler.java | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java index b17836e755..6b2b64986d 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java @@ -124,6 +124,7 @@ import com.oracle.graal.python.util.OverflowException; import com.oracle.graal.python.util.PythonUtils; import com.oracle.truffle.api.CompilerDirectives; +import com.oracle.truffle.api.CompilerDirectives.CompilationFinal; import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; import com.oracle.truffle.api.dsl.Bind; import com.oracle.truffle.api.dsl.Cached; @@ -661,6 +662,33 @@ public void flush(VirtualFrame frame, PPickler pickler, public abstract static class SaveNode extends BasePickleWriteNode { private static final int MAX_RECURSION_DEPTH = 3; + private static final int SEEN_NONE = 1 << 0; + private static final int SEEN_BOOLEAN = 1 << 1; + private static final int SEEN_INTEGER = 1 << 2; + private static final int SEEN_LONG = 1 << 3; + private static final int SEEN_PINT = 1 << 4; + private static final int SEEN_DOUBLE = 1 << 5; + private static final int SEEN_PFLOAT = 1 << 6; + private static final int SEEN_BUILTIN_CLASS_TYPE = 1 << 7; + private static final int SEEN_BYTES = 1 << 8; + private static final int SEEN_TRUFFLE_STRING = 1 << 9; + private static final int SEEN_PSTRING = 1 << 10; + private static final int SEEN_DICT = 1 << 11; + private static final int SEEN_SET = 1 << 12; + private static final int SEEN_FROZENSET = 1 << 13; + private static final int SEEN_LIST = 1 << 14; + private static final int SEEN_TUPLE = 1 << 15; + private static final int SEEN_NATIVE_TUPLE = 1 << 16; + private static final int SEEN_BYTE_ARRAY = 1 << 17; + private static final int SEEN_PICKLE_BUFFER = 1 << 18; + private static final int SEEN_MANAGED_CLASS = 1 << 19; + private static final int SEEN_NATIVE_CLASS = 1 << 20; + private static final int SEEN_FUNCTION = 1 << 21; + private static final int SEEN_TYPE = 1 << 22; + private static final int SEEN_OBJECT = 1 << 23; + + @CompilationFinal private int seenTypes; + private final int depth; @Child private SaveNode recursiveSaveNode; @Child private BoundaryCallData boundaryCallData; @@ -680,6 +708,13 @@ protected SaveNode(int depth) { this.depth = depth; } + private void profileSeen(int seen) { + if ((seenTypes & seen) == 0) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + seenTypes |= seen; + } + } + private BoundaryCallData getBoundaryCallData() { if (boundaryCallData == null) { CompilerDirectives.transferToInterpreterAndInvalidate(); @@ -2021,30 +2056,38 @@ void saveGeneric(VirtualFrame frame, PPickler pickler, Object objArg, int persSa // Atom types; these aren't memoized, so don't check the memo. if (obj == PNone.NONE) { + profileSeen(SEEN_NONE); saveNone(pickler); return; } else if (obj instanceof Boolean value) { + profileSeen(SEEN_BOOLEAN); saveBool(pickler, proto, value); return; } else if (obj instanceof Integer value) { + profileSeen(SEEN_INTEGER); saveLong(pickler, proto, value); return; } else if (obj instanceof Long value) { + profileSeen(SEEN_LONG); saveLong(frame, pickler, proto, value); return; } else if (obj instanceof PInt value && PGuards.isBuiltinPInt(value)) { + profileSeen(SEEN_PINT); saveLong(frame, pickler, proto, value); return; } else if (obj instanceof Double value) { + profileSeen(SEEN_DOUBLE); saveFloat(pickler, proto, value); return; } else if (obj instanceof PFloat value && PGuards.isBuiltinPFloat(value)) { + profileSeen(SEEN_PFLOAT); saveFloat(pickler, proto, value.getValue()); return; } // to avoid misses in memo for the class: if (obj instanceof PythonBuiltinClassType classType) { + profileSeen(SEEN_BUILTIN_CLASS_TYPE); obj = PythonContext.get(this).getCore().lookupType(classType); } @@ -2059,36 +2102,47 @@ void saveGeneric(VirtualFrame frame, PPickler pickler, Object objArg, int persSa PythonContext ctx = PythonContext.get(this); if (obj instanceof PBytes bytes && PGuards.isBuiltinBytes(bytes)) { + profileSeen(SEEN_BYTES); saveBytes(frame, ctx, pickler, proto, fastMode, bytes); return; } else if (obj instanceof TruffleString string) { + profileSeen(SEEN_TRUFFLE_STRING); saveUnicode(frame, pickler, proto, fastMode, string); return; } else if (obj instanceof PString string && PGuards.isBuiltinPString(string)) { + profileSeen(SEEN_PSTRING); saveUnicode(frame, pickler, proto, fastMode, string); return; } else if (obj instanceof PDict dict && PGuards.isBuiltinDict(dict)) { + profileSeen(SEEN_DICT); saveDict(frame, inliningTarget, callMethod, pickler, proto, fastMode, dict); return; } else if (obj instanceof PSet set && PGuards.isBuiltinSet(set)) { + profileSeen(SEEN_SET); saveSet(frame, ctx, pickler, proto, fastMode, set); return; } else if (obj instanceof PFrozenSet frozenSet && PGuards.isBuiltinFrozenSet(frozenSet)) { + profileSeen(SEEN_FROZENSET); saveFrozenset(frame, ctx, pickler, proto, fastMode, frozenSet); return; } else if (obj instanceof PList list && PGuards.isBuiltinList(list)) { + profileSeen(SEEN_LIST); saveList(frame, pickler, proto, fastMode, list); return; } else if (obj instanceof PTuple tuple && PGuards.isBuiltinTuple(tuple)) { + profileSeen(SEEN_TUPLE); saveTuple(frame, pickler, proto, fastMode, tuple); return; } else if (PGuards.isNativeObject(obj) && isBuiltinClass(getClass(obj), PythonBuiltinClassType.PTuple)) { + profileSeen(SEEN_NATIVE_TUPLE); saveNativeTuple(frame, pickler, proto, fastMode, obj); return; } else if (obj instanceof PByteArray byteArray && PGuards.isBuiltinByteArray(byteArray)) { + profileSeen(SEEN_BYTE_ARRAY); saveBytearray(frame, inliningTarget, ctx, pickler, proto, fastMode, byteArray); return; } else if (obj instanceof PPickleBuffer buffer) { + profileSeen(SEEN_PICKLE_BUFFER); savePickleBuffer(frame, pickler, proto, fastMode, buffer); return; } @@ -2104,16 +2158,21 @@ void saveGeneric(VirtualFrame frame, PPickler pickler, Object objArg, int persSa } if (obj instanceof PythonManagedClass clazz && clazz.getPythonClass() == PythonBuiltinClassType.PythonClass) { + profileSeen(SEEN_MANAGED_CLASS); saveType(frame, ctx, pickler, proto, fastMode, clazz); return; } else if (obj instanceof PythonNativeClass && isBuiltinClass(getClass(obj), PythonBuiltinClassType.PythonClass)) { + profileSeen(SEEN_NATIVE_CLASS); saveGlobal(frame, ctx, pickler, proto, fastMode, obj, null); return; } else if (obj instanceof PFunction) { + profileSeen(SEEN_FUNCTION); saveGlobal(frame, ctx, pickler, proto, fastMode, obj, null); return; } + profileSeen(SEEN_OBJECT); + // Get a reduction callable, and call it. This may come from self.dispatch_table, // copyreg.dispatch_table, the object's __reduce_ex__ method, or the object's __reduce__ // method. @@ -2134,6 +2193,7 @@ void saveGeneric(VirtualFrame frame, PPickler pickler, Object objArg, int persSa if (reduceFunc != null) { reduceValue = callNode.execute(frame, reduceFunc, obj); } else if (isSubType(type, PythonBuiltinClassType.PythonClass)) { + profileSeen(SEEN_TYPE); saveGlobal(frame, ctx, pickler, proto, fastMode, obj, null); return; } else { From f1044d3caf42abb3fa9864742e75b631144d8c4f Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Fri, 25 Sep 2026 15:22:04 +0200 Subject: [PATCH 07/39] Add fast path for pickling sequences of primitives --- .../builtins/modules/pickle/PPickler.java | 132 +++++++++++++++++- .../builtins/modules/pickle/PicklerNodes.java | 8 +- 2 files changed, 133 insertions(+), 7 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java index 6b2b64986d..2733245bfb 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java @@ -118,6 +118,9 @@ import com.oracle.graal.python.runtime.PythonContext; import com.oracle.graal.python.runtime.exception.PException; import com.oracle.graal.python.runtime.object.PFactory; +import com.oracle.graal.python.runtime.sequence.storage.DoubleSequenceStorage; +import com.oracle.graal.python.runtime.sequence.storage.IntSequenceStorage; +import com.oracle.graal.python.runtime.sequence.storage.LongSequenceStorage; import com.oracle.graal.python.runtime.sequence.storage.SequenceStorage; import com.oracle.graal.python.util.Consumer; import com.oracle.graal.python.util.NumericSupport; @@ -686,6 +689,10 @@ public abstract static class SaveNode extends BasePickleWriteNode { private static final int SEEN_FUNCTION = 1 << 21; private static final int SEEN_TYPE = 1 << 22; private static final int SEEN_OBJECT = 1 << 23; + private static final int SEEN_INT_STORAGE = 1 << 24; + private static final int SEEN_LONG_STORAGE = 1 << 25; + private static final int SEEN_DOUBLE_STORAGE = 1 << 26; + private static final int SEEN_GENERIC_STORAGE = 1 << 27; @CompilationFinal private int seenTypes; @@ -1661,9 +1668,25 @@ private void saveFrozenset(VirtualFrame frame, PythonContext ctx, PPickler pickl private void batchListExact(VirtualFrame frame, PPickler pickler, PList obj) { final SequenceStorage storage = obj.getSequenceStorage(); + if (pickler.persFunc == null) { + if (storage instanceof IntSequenceStorage intStorage) { + profileSeen(SEEN_INT_STORAGE); + batchIntList(frame, pickler, intStorage); + return; + } else if (storage instanceof LongSequenceStorage longStorage) { + profileSeen(SEEN_LONG_STORAGE); + batchLongList(frame, pickler, longStorage); + return; + } else if (storage instanceof DoubleSequenceStorage doubleStorage) { + profileSeen(SEEN_DOUBLE_STORAGE); + batchDoubleList(frame, pickler, doubleStorage); + return; + } + } + profileSeen(SEEN_GENERIC_STORAGE); Object item; if (storage.length() == 1) { - item = getItem(frame, storage, 0); + item = getItem(storage, 0); save(frame, pickler, item, 0); write(pickler, PickleUtils.OPCODE_APPEND); return; @@ -1676,7 +1699,7 @@ private void batchListExact(VirtualFrame frame, PPickler pickler, PList obj) { thisBatch = 0; write(pickler, PickleUtils.OPCODE_MARK); while (total < storage.length()) { - item = getItem(frame, storage, total); + item = getItem(storage, total); save(frame, pickler, item, 0); total++; if (++thisBatch == PickleUtils.BATCHSIZE) { @@ -1687,6 +1710,69 @@ private void batchListExact(VirtualFrame frame, PPickler pickler, PList obj) { } while (total < storage.length()); } + private void batchIntList(VirtualFrame frame, PPickler pickler, IntSequenceStorage storage) { + int proto = pickler.getProto(); + if (storage.length() == 1) { + opcodeBoundary(frame, pickler); + saveLong(pickler, proto, storage.getIntItemNormalized(0)); + write(pickler, PickleUtils.OPCODE_APPEND); + return; + } + + int length = storage.length(); + for (int batchStart = 0; batchStart < length; batchStart += PickleUtils.BATCHSIZE) { + int batchEnd = batchStart + Math.min(PickleUtils.BATCHSIZE, length - batchStart); + write(pickler, PickleUtils.OPCODE_MARK); + for (int i = batchStart; i < batchEnd; i++) { + opcodeBoundary(frame, pickler); + saveLong(pickler, proto, storage.getIntItemNormalized(i)); + } + write(pickler, PickleUtils.OPCODE_APPENDS); + } + } + + private void batchLongList(VirtualFrame frame, PPickler pickler, LongSequenceStorage storage) { + int proto = pickler.getProto(); + if (storage.length() == 1) { + opcodeBoundary(frame, pickler); + saveLong(frame, pickler, proto, storage.getLongItemNormalized(0)); + write(pickler, PickleUtils.OPCODE_APPEND); + return; + } + + int length = storage.length(); + for (int batchStart = 0; batchStart < length; batchStart += PickleUtils.BATCHSIZE) { + int batchEnd = batchStart + Math.min(PickleUtils.BATCHSIZE, length - batchStart); + write(pickler, PickleUtils.OPCODE_MARK); + for (int i = batchStart; i < batchEnd; i++) { + opcodeBoundary(frame, pickler); + saveLong(frame, pickler, proto, storage.getLongItemNormalized(i)); + } + write(pickler, PickleUtils.OPCODE_APPENDS); + } + } + + private void batchDoubleList(VirtualFrame frame, PPickler pickler, DoubleSequenceStorage storage) { + int proto = pickler.getProto(); + if (storage.length() == 1) { + opcodeBoundary(frame, pickler); + saveFloat(pickler, proto, storage.getDoubleItemNormalized(0)); + write(pickler, PickleUtils.OPCODE_APPEND); + return; + } + + int length = storage.length(); + for (int batchStart = 0; batchStart < length; batchStart += PickleUtils.BATCHSIZE) { + int batchEnd = batchStart + Math.min(PickleUtils.BATCHSIZE, length - batchStart); + write(pickler, PickleUtils.OPCODE_MARK); + for (int i = batchStart; i < batchEnd; i++) { + opcodeBoundary(frame, pickler); + saveFloat(pickler, proto, storage.getDoubleItemNormalized(i)); + } + write(pickler, PickleUtils.OPCODE_APPENDS); + } + } + private void batchList(VirtualFrame frame, PPickler pickler, int proto, Object iterator) { assert iterator != null; if (proto == 0) { @@ -1734,12 +1820,52 @@ private void storeTupleElements(VirtualFrame frame, PPickler pickler, Object obj } } else { assert storage.length() == len; + if (pickler.persFunc == null) { + if (storage instanceof IntSequenceStorage intStorage) { + profileSeen(SEEN_INT_STORAGE); + storeIntTupleElements(frame, pickler, intStorage); + return; + } else if (storage instanceof LongSequenceStorage longStorage) { + profileSeen(SEEN_LONG_STORAGE); + storeLongTupleElements(frame, pickler, longStorage); + return; + } else if (storage instanceof DoubleSequenceStorage doubleStorage) { + profileSeen(SEEN_DOUBLE_STORAGE); + storeDoubleTupleElements(frame, pickler, doubleStorage); + return; + } + } + profileSeen(SEEN_GENERIC_STORAGE); for (int i = 0; i < len; i++) { - save(frame, pickler, getItem(frame, storage, i), 0); + save(frame, pickler, getItem(storage, i), 0); } } } + private void storeIntTupleElements(VirtualFrame frame, PPickler pickler, IntSequenceStorage storage) { + int proto = pickler.getProto(); + for (int i = 0; i < storage.length(); i++) { + opcodeBoundary(frame, pickler); + saveLong(pickler, proto, storage.getIntItemNormalized(i)); + } + } + + private void storeLongTupleElements(VirtualFrame frame, PPickler pickler, LongSequenceStorage storage) { + int proto = pickler.getProto(); + for (int i = 0; i < storage.length(); i++) { + opcodeBoundary(frame, pickler); + saveLong(frame, pickler, proto, storage.getLongItemNormalized(i)); + } + } + + private void storeDoubleTupleElements(VirtualFrame frame, PPickler pickler, DoubleSequenceStorage storage) { + int proto = pickler.getProto(); + for (int i = 0; i < storage.length(); i++) { + opcodeBoundary(frame, pickler); + saveFloat(pickler, proto, storage.getDoubleItemNormalized(i)); + } + } + private void saveTuple(VirtualFrame frame, PPickler pickler, int proto, boolean fastMode, PTuple obj) { SequenceStorage storage = obj.getSequenceStorage(); saveTuple(frame, pickler, proto, fastMode, obj, storage, storage.length()); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java index 8ac0b1339e..419ec3a8b6 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java @@ -135,7 +135,7 @@ abstract static class BasePickleNode extends Node { @Child private HashingStorageSetItem setHashingStorageItemNode; @Child private PyObjectSetItem pyObjectSetItemNode; @Child private CachedHashingStorageGetItem getHashingStorageItemNode; - @Child private SequenceStorageNodes.GetItemNode getSeqStorageItemNode; + @Child private SequenceStorageNodes.GetItemScalarNode getSeqStorageItemNode; @Child private PyNumberAsSizeNode asSizeNode; @Child private CastToTruffleStringNode castToTruffleStringNode; @Child private SequenceNodes.GetSequenceStorageNode getSequenceStorageNode; @@ -556,12 +556,12 @@ public Object getNextItem(VirtualFrame frame, Object iterator) { return getNextNode.executeCached(frame, iterator); } - public Object getItem(VirtualFrame frame, SequenceStorage storage, int i) { + public Object getItem(SequenceStorage storage, int i) { if (getSeqStorageItemNode == null) { CompilerDirectives.transferToInterpreterAndInvalidate(); - getSeqStorageItemNode = insert(SequenceStorageNodes.GetItemNode.create()); + getSeqStorageItemNode = insert(SequenceStorageNodes.GetItemScalarNode.create()); } - return getSeqStorageItemNode.execute(frame, storage, i); + return getSeqStorageItemNode.executeCached(storage, i); } public Object getItem(VirtualFrame frame, Object obj, Object slice) { From ba662f23aea95192447a2e288595b760da57b3dc Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Tue, 29 Sep 2026 11:31:27 +0200 Subject: [PATCH 08/39] Add more pickler optimizations --- .../builtins/modules/pickle/PPickler.java | 29 ++++++++++++------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java index 2733245bfb..75d1e8b08e 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java @@ -504,6 +504,10 @@ protected void writeASCII(PPickler pickler, TruffleString string) { protected void writeUtf8(PPickler pickler, TruffleString string) { TruffleString encoded = ensureTsSwitchEncodingNode().execute(string, TruffleString.Encoding.UTF_8, TranscodingErrorHandler.DEFAULT_KEEP_SURROGATES_IN_UTF8); + writeUtf8Converted(pickler, encoded); + } + + public void writeUtf8Converted(PPickler pickler, TruffleString encoded) { int length = encoded.byteLength(TruffleString.Encoding.UTF_8); ensureBufferSpace(pickler, length); ensureTsCopyToByteArrayNode().execute(encoded, 0, pickler.outputBuffer, pickler.outputLen, length, TruffleString.Encoding.UTF_8); @@ -588,25 +592,20 @@ protected void write(PPickler pickler, byte[] bytes, int dataLen) { public void ensureBufferSpace(PPickler pickler, int dataLen) { boolean needNewFrame = pickler.isFraming() && pickler.frameStart == -1; - int n = (needNewFrame) ? dataLen + PickleUtils.FRAME_HEADER_SIZE : dataLen; + int n = CompilerDirectives.injectBranchProbability(CompilerDirectives.SLOWPATH_PROBABILITY, needNewFrame) ? dataLen + PickleUtils.FRAME_HEADER_SIZE : dataLen; int required = pickler.outputLen + n; - if (required > pickler.maxOutputLen) { + if (CompilerDirectives.injectBranchProbability(CompilerDirectives.SLOWPATH_PROBABILITY, required > pickler.maxOutputLen)) { // Make place in buffer for the pickle chunk // TODO: when GR-24978 is completed we should use PY_SSIZE_T_MAX - if (pickler.outputLen >= Integer.MAX_VALUE / 2 - n) { + if (CompilerDirectives.injectBranchProbability(CompilerDirectives.SLOWPATH_PROBABILITY, pickler.outputLen >= Integer.MAX_VALUE / 2 - n)) { throw raise(PythonBuiltinClassType.MemoryError); } pickler.maxOutputLen = (pickler.outputLen + n) / 2 * 3; pickler.outputBuffer = PythonUtils.arrayCopyOf(pickler.outputBuffer, pickler.maxOutputLen); } - if (needNewFrame) { - int frameStart = pickler.outputLen; - pickler.frameStart = frameStart; - for (int i = 0; i < PickleUtils.FRAME_HEADER_SIZE; i++) { - // Write an invalid value, for debugging - pickler.outputBuffer[frameStart + i] = (byte) 0xFE; - } + if (CompilerDirectives.injectBranchProbability(CompilerDirectives.SLOWPATH_PROBABILITY, needNewFrame)) { + pickler.frameStart = pickler.outputLen; pickler.outputLen += PickleUtils.FRAME_HEADER_SIZE; } } @@ -693,6 +692,10 @@ public abstract static class SaveNode extends BasePickleWriteNode { private static final int SEEN_LONG_STORAGE = 1 << 25; private static final int SEEN_DOUBLE_STORAGE = 1 << 26; private static final int SEEN_GENERIC_STORAGE = 1 << 27; + private static final int SEEN_CUSTOM_DISPATCH = 1 << 28; + private static final int SEEN_REGISTERED_REDUCER = 1 << 29; + private static final int SEEN_REDUCE_EX = 1 << 30; + private static final int SEEN_REDUCE = 1 << 31; @CompilationFinal private int seenTypes; @@ -1494,7 +1497,7 @@ private void writeUnicodeBinary(VirtualFrame frame, PPickler pickler, int proto, } else { writeIntOp(pickler, PickleUtils.OPCODE_BINUNICODE, size); } - writeUtf8(pickler, utf8); + writeUtf8Converted(pickler, utf8); } if (bypassBuffer && wasFraming) { @@ -2308,6 +2311,7 @@ void saveGeneric(VirtualFrame frame, PPickler pickler, Object objArg, int persSa PickleState state = getGlobalState(ctx.getCore()); reduceFunc = getDictItem(frame, state.dispatchTable, type); } else { + profileSeen(SEEN_CUSTOM_DISPATCH); try { reduceFunc = getItem(frame, pickler.dispatchTable, type); } catch (PException ex) { @@ -2317,6 +2321,7 @@ void saveGeneric(VirtualFrame frame, PPickler pickler, Object objArg, int persSa Object reduceValue; if (reduceFunc != null) { + profileSeen(SEEN_REGISTERED_REDUCER); reduceValue = callNode.execute(frame, reduceFunc, obj); } else if (isSubType(type, PythonBuiltinClassType.PythonClass)) { profileSeen(SEEN_TYPE); @@ -2332,11 +2337,13 @@ void saveGeneric(VirtualFrame frame, PPickler pickler, Object objArg, int persSa // Check for a __reduce_ex__ method. reduceFunc = lookupAttribute(frame, obj, T___REDUCE_EX__); if (reduceFunc != PNone.NO_VALUE) { + profileSeen(SEEN_REDUCE_EX); reduceValue = callNode.execute(frame, reduceFunc, proto); } else { // Check for a __reduce__ method. reduceFunc = lookupAttribute(frame, obj, T___REDUCE__); if (reduceFunc != PNone.NO_VALUE) { + profileSeen(SEEN_REDUCE); reduceValue = callNode.execute(frame, reduceFunc); } else { throw raise(PicklingError, ErrorMessages.CANNOT_PICKLE_P_P, type, obj); From b60b36c563c3261217ab483bb18d0ec31310946a Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Tue, 29 Sep 2026 14:58:29 +0200 Subject: [PATCH 09/39] Push down helper nodes into PPickler.SaveNode --- .../builtins/modules/pickle/PPickler.java | 513 ++++++++++++++++-- .../builtins/modules/pickle/PicklerNodes.java | 110 ---- 2 files changed, 474 insertions(+), 149 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java index 75d1e8b08e..296cb5cecf 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java @@ -40,6 +40,13 @@ */ package com.oracle.graal.python.builtins.modules.pickle; +import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.T_CP_REVERSE_IMPORT_MAPPING; +import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.T_CP_REVERSE_NAME_MAPPING; +import static com.oracle.graal.python.builtins.objects.PNone.NO_VALUE; +import static com.oracle.graal.python.nodes.BuiltinNames.T___MAIN__; +import static com.oracle.graal.python.nodes.ErrorMessages.MUST_BE_STR_NOT_P; +import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___CLASS__; +import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___MODULE__; import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___NAME__; import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___NEWOBJ_EX__; import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___NEWOBJ__; @@ -48,8 +55,10 @@ import static com.oracle.graal.python.nodes.SpecialMethodNames.T___NEW__; import static com.oracle.graal.python.nodes.SpecialMethodNames.T___REDUCE_EX__; import static com.oracle.graal.python.nodes.SpecialMethodNames.T___REDUCE__; +import static com.oracle.graal.python.nodes.StringLiterals.T_DOT; import static com.oracle.graal.python.nodes.StringLiterals.T_NEWLINE; import static com.oracle.graal.python.nodes.statement.AbstractImportNode.importModule; +import static com.oracle.graal.python.runtime.exception.PythonErrorType.AttributeError; import static com.oracle.graal.python.runtime.exception.PythonErrorType.KeyError; import static com.oracle.graal.python.runtime.exception.PythonErrorType.OverflowError; import static com.oracle.graal.python.runtime.exception.PythonErrorType.PicklingError; @@ -65,6 +74,7 @@ import org.graalvm.collections.Pair; import com.oracle.graal.python.PythonLanguage; +import com.oracle.graal.python.builtins.Python3Core; import com.oracle.graal.python.builtins.PythonBuiltinClassType; import com.oracle.graal.python.builtins.objects.PNone; import com.oracle.graal.python.builtins.objects.PNotImplemented; @@ -75,15 +85,19 @@ import com.oracle.graal.python.builtins.objects.bytes.PBytes; import com.oracle.graal.python.builtins.objects.cext.PythonNativeClass; import com.oracle.graal.python.builtins.objects.common.HashingStorage; +import com.oracle.graal.python.builtins.objects.common.HashingStorageNodes.CachedHashingStorageGetItem; +import com.oracle.graal.python.builtins.objects.common.HashingStorageNodes.HashingStorageGetIterator; import com.oracle.graal.python.builtins.objects.common.HashingStorageNodes.HashingStorageIterator; import com.oracle.graal.python.builtins.objects.common.HashingStorageNodes.HashingStorageIteratorKey; import com.oracle.graal.python.builtins.objects.common.HashingStorageNodes.HashingStorageIteratorNext; import com.oracle.graal.python.builtins.objects.common.HashingStorageNodes.HashingStorageIteratorValue; import com.oracle.graal.python.builtins.objects.common.HashingStorageNodes.HashingStorageLen; +import com.oracle.graal.python.builtins.objects.common.SequenceStorageNodes; import com.oracle.graal.python.builtins.objects.dict.PDict; import com.oracle.graal.python.builtins.objects.ellipsis.PEllipsis; import com.oracle.graal.python.builtins.objects.floats.PFloat; import com.oracle.graal.python.builtins.objects.function.PFunction; +import com.oracle.graal.python.builtins.objects.function.PKeyword; import com.oracle.graal.python.builtins.objects.ints.IntNodes; import com.oracle.graal.python.builtins.objects.ints.IntNodesFactory; import com.oracle.graal.python.builtins.objects.ints.PInt; @@ -92,27 +106,41 @@ import com.oracle.graal.python.builtins.objects.set.PFrozenSet; import com.oracle.graal.python.builtins.objects.set.PSet; import com.oracle.graal.python.builtins.objects.str.PString; +import com.oracle.graal.python.builtins.objects.str.StringUtils; import com.oracle.graal.python.builtins.objects.tuple.PTuple; import com.oracle.graal.python.builtins.objects.type.PythonBuiltinClass; import com.oracle.graal.python.builtins.objects.type.PythonManagedClass; import com.oracle.graal.python.builtins.objects.type.TypeNodes; import com.oracle.graal.python.lib.IteratorExhausted; import com.oracle.graal.python.lib.PyCallableCheckNode; +import com.oracle.graal.python.lib.PyIterCheckNode; +import com.oracle.graal.python.lib.PyIterNextNode; import com.oracle.graal.python.lib.PyLongAsLongNode; import com.oracle.graal.python.lib.PyObjectCallMethodObjArgs; +import com.oracle.graal.python.lib.PyObjectGetItem; import com.oracle.graal.python.lib.PyObjectGetIter; import com.oracle.graal.python.lib.PyObjectIsTrueNode; import com.oracle.graal.python.lib.PyObjectLookupAttr; import com.oracle.graal.python.lib.PyObjectReprAsTruffleStringNode; +import com.oracle.graal.python.lib.PyObjectSizeNode; import com.oracle.graal.python.lib.PyObjectStrAsObjectNode; import com.oracle.graal.python.lib.PyTupleCheckNode; import com.oracle.graal.python.nodes.ErrorMessages; +import com.oracle.graal.python.nodes.HiddenAttr; import com.oracle.graal.python.nodes.PGuards; import com.oracle.graal.python.nodes.PRaiseNode; +import com.oracle.graal.python.nodes.argument.keywords.ExpandKeywordStarargsNode; +import com.oracle.graal.python.nodes.argument.keywords.ExpandKeywordStarargsNodeGen; +import com.oracle.graal.python.nodes.argument.positional.ExecutePositionalStarargsNode; import com.oracle.graal.python.nodes.builtins.ListNodes; import com.oracle.graal.python.nodes.call.CallNode; import com.oracle.graal.python.nodes.classes.IsSubtypeNode; +import com.oracle.graal.python.nodes.object.BuiltinClassProfiles.InlineIsBuiltinClassProfile; +import com.oracle.graal.python.nodes.object.BuiltinClassProfiles.IsBuiltinObjectProfile; +import com.oracle.graal.python.nodes.object.GetClassNode; import com.oracle.graal.python.nodes.object.IsNode; +import com.oracle.graal.python.nodes.util.CannotCastException; +import com.oracle.graal.python.nodes.util.CastToTruffleStringNode; import com.oracle.graal.python.runtime.ExecutionContext.BoundaryCallContext; import com.oracle.graal.python.runtime.IndirectCallData.BoundaryCallData; import com.oracle.graal.python.runtime.PythonContext; @@ -131,10 +159,13 @@ import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; import com.oracle.truffle.api.dsl.Bind; import com.oracle.truffle.api.dsl.Cached; +import com.oracle.truffle.api.dsl.GenerateInline; import com.oracle.truffle.api.dsl.Specialization; +import com.oracle.truffle.api.frame.Frame; import com.oracle.truffle.api.frame.VirtualFrame; import com.oracle.truffle.api.nodes.Node; import com.oracle.truffle.api.object.Shape; +import com.oracle.truffle.api.profiles.BranchProfile; import com.oracle.truffle.api.profiles.InlinedConditionProfile; import com.oracle.truffle.api.profiles.InlinedIntValueProfile; import com.oracle.truffle.api.strings.TranscodingErrorHandler; @@ -387,12 +418,15 @@ public PBytes getString(PythonLanguage language) { } // inner nodes - public abstract static class BasePickleWriteNode extends PicklerNodes.BasePickleNode { + public abstract static class SaveNode extends Node { + private static final int MAX_RECURSION_DEPTH = 3; + private static final TruffleString T_LOCALS = tsLiteral(""); + private static final TruffleString T_SYS_MODULES = tsLiteral("sys.modules"); + static final byte[] TUPLE_LEN_2_OPCODE = new byte[]{PickleUtils.OPCODE_EMPTY_TUPLE, PickleUtils.OPCODE_TUPLE1, PickleUtils.OPCODE_TUPLE2, PickleUtils.OPCODE_TUPLE3}; static final byte NEW_LINE_BYTE = '\n'; static final TruffleString DICT_ITEMS = tsLiteral("dict items"); static final TruffleString LATIN1 = tsLiteral("latin1"); - static final TruffleString REDUCE_OVERRIDE = tsLiteral("reducer_override"); static final TruffleString T_L_NEW_LINE = tsLiteral("L\n"); static final TruffleString T_SET = tsLiteral("set"); static final TruffleString T_DICTIONARY = tsLiteral("dictionary"); @@ -405,6 +439,413 @@ public abstract static class BasePickleWriteNode extends PicklerNodes.BasePickle @Child private TypeNodes.IsTypeNode isTypeNode; @Child private FlushToFileNode flushToFileNode; @Child private CallNode callNode; + @SuppressWarnings("FieldMayBeFinal") @Child private PyObjectGetItem getItemNode = PyObjectGetItem.create(); + @SuppressWarnings("FieldMayBeFinal") @Child private PyIterNextNode getNextNode = PyIterNextNode.create(); + @Child private CastToTruffleStringNode toStringNode = CastToTruffleStringNode.create(); + @Child private HiddenAttr.ReadNode readHiddenAttributeNode; + @Child private IsBuiltinObjectProfile errProfile; + @Child private InlineIsBuiltinClassProfile isBuiltinClassProfile; + @Child private CachedHashingStorageGetItem getHashingStorageItemNode; + @Child private SequenceStorageNodes.GetItemScalarNode getSeqStorageItemNode; + @Child private PyObjectSizeNode sizeNode; + @Child private PyIterCheckNode isIteratorObjectNode; + @Child private GetClassNode getClassNode; + @Child private PyObjectLookupAttr lookupAttrNode; + @Child private ExecutePositionalStarargsNode getArgsNode; + @Child private ExpandKeywordStarargsNode getKwArgsNode; + @Child private HashingStorageGetIterator getHashingStorageIteratorNode; + @Child private HashingStorageIteratorNext hashingStorageItNext; + @Child private HashingStorageIteratorKey hashingStorageItKey; + @Child private HashingStorageIteratorValue hashingStorageItValue; + @Child private TruffleString.FromByteArrayWithCompactionUTF32Node tsFromByteArrayWithCompactionNode; + @Child private TruffleString.CodePointLengthNode tsCodePointLengthNode; + @Child private TruffleString.CodePointAtIndexUTF32Node tsCodePointAtIndexUTF32Node; + @Child private TruffleString.FromLongNode tsFromLongNode; + @Child private TruffleString.IndexOfStringNode tsIndexOfStringNode; + @Child private TruffleString.SubstringNode tsSubstringNode; + @Child private TruffleString.EqualNode tsEqualNode; + @Child private TruffleString.CopyToByteArrayNode tsCopyToByteArrayNode; + @Child private TruffleString.GetCodeRangeNode tsGetCodeRangeNode; + @Child private TruffleString.SwitchEncodingNode tsSwitchEncodingNode; + private final BranchProfile errorProfile = BranchProfile.create(); + + protected PException raise(PythonBuiltinClassType type, TruffleString string) { + errorProfile.enter(); + return PRaiseNode.raiseStatic(this, type, string); + } + + protected PException raise(PythonBuiltinClassType exceptionType) { + errorProfile.enter(); + return PRaiseNode.raiseStatic(this, exceptionType); + } + + protected final PException raise(PythonBuiltinClassType type, TruffleString format, Object... arguments) { + errorProfile.enter(); + return PRaiseNode.raiseStatic(this, type, format, arguments); + } + + protected TruffleString.FromByteArrayWithCompactionUTF32Node ensureTsFromByteArrayWithCompaction() { + if (tsFromByteArrayWithCompactionNode == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + tsFromByteArrayWithCompactionNode = insert(TruffleString.FromByteArrayWithCompactionUTF32Node.create()); + } + return tsFromByteArrayWithCompactionNode; + } + + protected TruffleString.CodePointLengthNode ensureTsCodePointLengthNode() { + if (tsCodePointLengthNode == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + tsCodePointLengthNode = insert(TruffleString.CodePointLengthNode.create()); + } + return tsCodePointLengthNode; + } + + protected TruffleString.CodePointAtIndexUTF32Node ensureTsCodePointAtIndexUTF32Node() { + if (tsCodePointAtIndexUTF32Node == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + tsCodePointAtIndexUTF32Node = insert(TruffleString.CodePointAtIndexUTF32Node.create()); + } + return tsCodePointAtIndexUTF32Node; + } + + protected TruffleString.FromLongNode ensureTsFromLongNode() { + if (tsFromLongNode == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + tsFromLongNode = insert(TruffleString.FromLongNode.create()); + } + return tsFromLongNode; + } + + protected TruffleString.IndexOfStringNode ensureTsIndexOfStringNode() { + if (tsIndexOfStringNode == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + tsIndexOfStringNode = insert(TruffleString.IndexOfStringNode.create()); + } + return tsIndexOfStringNode; + } + + protected TruffleString.SubstringNode ensureTsSubstringNode() { + if (tsSubstringNode == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + tsSubstringNode = insert(TruffleString.SubstringNode.create()); + } + return tsSubstringNode; + } + + protected TruffleString.EqualNode ensureTsEqualNode() { + if (tsEqualNode == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + tsEqualNode = insert(TruffleString.EqualNode.create()); + } + return tsEqualNode; + } + + protected TruffleString.CopyToByteArrayNode ensureTsCopyToByteArrayNode() { + if (tsCopyToByteArrayNode == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + tsCopyToByteArrayNode = insert(TruffleString.CopyToByteArrayNode.create()); + } + return tsCopyToByteArrayNode; + } + + protected TruffleString.GetCodeRangeNode ensureTsGetCodeRangeNode() { + if (tsGetCodeRangeNode == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + tsGetCodeRangeNode = insert(TruffleString.GetCodeRangeNode.create()); + } + return tsGetCodeRangeNode; + } + + protected TruffleString.SwitchEncodingNode ensureTsSwitchEncodingNode() { + if (tsSwitchEncodingNode == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + tsSwitchEncodingNode = insert(TruffleString.SwitchEncodingNode.create()); + } + return tsSwitchEncodingNode; + } + + protected HashingStorageIterator getHashingStorageIterator(HashingStorage storage) { + if (getHashingStorageIteratorNode == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + getHashingStorageIteratorNode = insert(HashingStorageGetIterator.create()); + } + return getHashingStorageIteratorNode.executeCached(storage); + } + + protected HashingStorageIteratorNext ensureHashingStorageIteratorNext() { + if (hashingStorageItNext == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + hashingStorageItNext = insert(HashingStorageIteratorNext.create()); + } + return hashingStorageItNext; + } + + protected HashingStorageIteratorKey ensureHashingStorageIteratorKey() { + if (hashingStorageItKey == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + hashingStorageItKey = insert(HashingStorageIteratorKey.create()); + } + return hashingStorageItKey; + } + + protected HashingStorageIteratorValue ensureHashingStorageIteratorValue() { + if (hashingStorageItValue == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + hashingStorageItValue = insert(HashingStorageIteratorValue.create()); + } + return hashingStorageItValue; + } + + protected int length(VirtualFrame frame, Object object) { + if (sizeNode == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + sizeNode = insert(PyObjectSizeNode.create()); + } + return sizeNode.executeCached(frame, object); + } + + protected boolean isIterator(Object iter) { + if (isIteratorObjectNode == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + isIteratorObjectNode = insert(PyIterCheckNode.create()); + } + return isIteratorObjectNode.executeCached(iter); + } + + protected CallNode ensureCallNode() { + if (callNode == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + callNode = insert(CallNode.create()); + } + return callNode; + } + + protected Object callStarArgsAndKwArgs(VirtualFrame frame, Object method, Object args, Object kwargs) { + if (getArgsNode == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + getArgsNode = insert(ExecutePositionalStarargsNode.create()); + } + PKeyword[] keywords; + if (kwargs == null) { + keywords = PKeyword.EMPTY_KEYWORDS; + } else { + if (getKwArgsNode == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + getKwArgsNode = insert(ExpandKeywordStarargsNodeGen.create()); + } + keywords = getKwArgsNode.executeCached(kwargs); + } + return ensureCallNode().execute(frame, method, getArgsNode.executeWith(frame, args), keywords); + } + + protected PyObjectLookupAttr getLookupAttrNode() { + if (lookupAttrNode == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + lookupAttrNode = insert(PyObjectLookupAttr.create()); + } + return lookupAttrNode; + } + + protected Object lookupAttribute(Frame frame, Object receiver, TruffleString name) { + return getLookupAttrNode().executeCached(frame, receiver, name); + } + + protected Object lookupAttributeStrict(Frame frame, Object receiver, TruffleString name) { + Object attr = lookupAttribute(frame, receiver, name); + if (attr == NO_VALUE) { + throw raise(TypeError, ErrorMessages.OBJ_P_HAS_NO_ATTR_S, attr, name); + } + return attr; + } + + protected Object getClass(Object object) { + if (getClassNode == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + getClassNode = insert(GetClassNode.create()); + } + return getClassNode.executeCached(object); + } + + protected Object getClass(Frame frame, Object object) { + Object cls = getLookupAttrNode().executeCached(frame, object, T___CLASS__); + return cls == NO_VALUE ? getClass(object) : cls; + } + + protected HiddenAttr.ReadNode ensureReadHiddenAttrNode() { + if (readHiddenAttributeNode == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + readHiddenAttributeNode = insert(HiddenAttr.ReadNode.create()); + } + return readHiddenAttributeNode; + } + + public PickleState getGlobalState(Python3Core core) { + Object state = ensureReadHiddenAttrNode().executeCached(core.lookupType(PythonBuiltinClassType.Pickler), HiddenAttr.PICKLE_STATE, NO_VALUE); + assert state instanceof PickleState; + return (PickleState) state; + } + + protected Object getDictItem(VirtualFrame frame, PDict dict, Object key) { + if (getHashingStorageItemNode == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + getHashingStorageItemNode = insert(CachedHashingStorageGetItem.create()); + } + return getHashingStorageItemNode.execute(frame, dict.getDictStorage(), key); + } + + protected IsBuiltinObjectProfile ensureErrProfile() { + if (errProfile == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + errProfile = insert(IsBuiltinObjectProfile.create()); + } + return errProfile; + } + + protected boolean isBuiltinClass(Object cls, PythonBuiltinClassType type) { + if (isBuiltinClassProfile == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + isBuiltinClassProfile = insert(InlineIsBuiltinClassProfile.create()); + } + return isBuiltinClassProfile.profileClassCached(cls, type); + } + + public Object getNextItem(VirtualFrame frame, Object iterator) { + return getNextNode.executeCached(frame, iterator); + } + + public Object getItem(SequenceStorage storage, int index) { + if (getSeqStorageItemNode == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + getSeqStorageItemNode = insert(SequenceStorageNodes.GetItemScalarNode.create()); + } + return getSeqStorageItemNode.executeCached(storage, index); + } + + public Object getItem(VirtualFrame frame, Object obj, Object key) { + return getItemNode.executeCached(frame, obj, key); + } + + public Object getItem(VirtualFrame frame, Object obj, int size, int pos, Object defaultValue) { + return size > pos ? getItemNode.executeCached(frame, obj, pos) : defaultValue; + } + + public TruffleString asString(Object value) { + if (value instanceof TruffleString string) { + return string; + } + try { + return toStringNode.executeCached(value); + } catch (CannotCastException e) { + return null; + } + } + + public TruffleString asStringStrict(Object value) { + if (value instanceof TruffleString string) { + return string; + } + try { + return toStringNode.executeCached(value); + } catch (CannotCastException e) { + throw raise(TypeError, MUST_BE_STR_NOT_P, value); + } + } + + public static Pair getDeepAttribute(VirtualFrame frame, PyObjectLookupAttr lookup, Object obj, TruffleString[] names) { + Object parent = null; + Object object = obj; + for (TruffleString name : names) { + parent = object; + object = lookup.executeCached(frame, parent, name); + if (object == NO_VALUE) { + return null; + } + } + return Pair.create(object, parent); + } + + public TruffleString[] getDottedPath(Object obj, TruffleString name) { + TruffleString[] dottedPath = StringUtils.split(name, T_DOT, ensureTsCodePointLengthNode(), ensureTsIndexOfStringNode(), ensureTsSubstringNode(), ensureTsEqualNode()); + assert dottedPath.length > 0; + for (TruffleString subPath : dottedPath) { + if (ensureTsEqualNode().execute(subPath, T_LOCALS, TS_ENCODING)) { + if (obj == null) { + throw raise(AttributeError, ErrorMessages.CANT_PICKLE_LOCAL_OBJ_S, name); + } else { + throw raise(AttributeError, ErrorMessages.CANT_PICKLE_ATTR_S_OF_P, name, obj); + } + } + } + return dottedPath; + } + + protected Pair get3to2Mapping(VirtualFrame frame, Python3Core core, TruffleString moduleName, TruffleString globalName) { + PickleState state = getGlobalState(core); + return getMapping(frame, state.nameMapping3To2, state.importMapping3To2, T_CP_REVERSE_NAME_MAPPING, T_CP_REVERSE_IMPORT_MAPPING, moduleName, globalName); + } + + private Pair getMapping(VirtualFrame frame, PDict nameMapping, PDict importMapping, + TruffleString nameMappingLabel, TruffleString importMappingLabel, TruffleString moduleName, TruffleString globalName) { + Object key = PFactory.createTuple(PythonLanguage.get(this), new Object[]{moduleName, globalName}); + Object item = getDictItem(frame, nameMapping, key); + if (item != null) { + if (!(item instanceof PTuple) || length(frame, item) != 2) { + throw raise(PythonBuiltinClassType.RuntimeError, ErrorMessages.S_SHOULD_BE_S_NOT_P, nameMappingLabel, "2-tuples", item); + } + Object mappedModuleName = getItem(frame, item, 0); + Object mappedGlobalName = getItem(frame, item, 1); + if (!PGuards.isString(mappedModuleName) || !PGuards.isString(mappedGlobalName)) { + throw raise(PythonBuiltinClassType.RuntimeError, ErrorMessages.S_SHOULD_BE_S_NOT_P_P, nameMappingLabel, "str", mappedModuleName, mappedGlobalName); + } + return Pair.create(asString(mappedModuleName), asString(mappedGlobalName)); + } + item = getDictItem(frame, importMapping, moduleName); + if (item != null) { + if (!PGuards.isString(item)) { + throw raise(PythonBuiltinClassType.RuntimeError, ErrorMessages.S_SHOULD_BE_S_NOT_P, importMappingLabel, "strings", item); + } + return Pair.create(asString(item), globalName); + } + return Pair.create(moduleName, globalName); + } + + private static boolean checkModule(VirtualFrame frame, PyObjectLookupAttr lookup, Object moduleName, Object module, Object global, TruffleString[] dottedPath) { + if (module == PNone.NONE) { + return false; + } + assert !(moduleName instanceof String) : "moduleName shouldn't be j.l.String"; + if (moduleName instanceof TruffleString && ((TruffleString) moduleName).equalsUncached(T___MAIN__, TS_ENCODING)) { + return false; + } + Pair pair = getDeepAttribute(frame, lookup, module, dottedPath); + return pair != null && pair.getLeft() == global; + } + + public TruffleString whichModule(VirtualFrame frame, PythonContext context, Object global, TruffleString[] dottedPath) { + Object moduleName = lookupAttribute(frame, global, T___MODULE__); + if (moduleName != NO_VALUE && moduleName != PNone.NONE) { + return asStringStrict(moduleName); + } + + PDict sysModules = context.getSysModules(); + if (sysModules == null) { + throw raise(PythonBuiltinClassType.RuntimeError, ErrorMessages.UNABLE_TO_GET_S, T_SYS_MODULES); + } + + HashingStorage storage = sysModules.getDictStorage(); + HashingStorageIterator it = getHashingStorageIterator(storage); + HashingStorageIteratorNext nextNode = ensureHashingStorageIteratorNext(); + HashingStorageIteratorKey getKeyNode = ensureHashingStorageIteratorKey(); + HashingStorageIteratorValue getValueNode = ensureHashingStorageIteratorValue(); + while (nextNode.executeCached(storage, it)) { + Object value = getValueNode.executeCached(storage, it); + Object key = getKeyNode.executeCached(storage, it); + if (checkModule(frame, getLookupAttrNode(), moduleName, value, global, dottedPath)) { + return asStringStrict(key); + } + } + return T___MAIN__; + } protected boolean isType(Object obj) { if (isTypeNode == null) { @@ -462,14 +903,6 @@ protected boolean isSubType(Object clsA, Object clsB) { return isSubTypeNode.execute(clsA, clsB); } - protected CallNode getCallNode() { - if (callNode == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - callNode = insert(CallNode.create()); - } - return callNode; - } - public void opcodeBoundary(VirtualFrame frame, PPickler pickler) { if (!pickler.isFraming() || pickler.frameStart == -1) { return; @@ -646,23 +1079,6 @@ protected static void fastSaveLeave(PPickler pickler, Object obj) { pickler.fastMemoRemove(obj); } } - } - - public abstract static class FlushToFileNode extends BasePickleWriteNode { - public abstract void execute(VirtualFrame frame, PPickler pickler); - - @Specialization - public void flush(VirtualFrame frame, PPickler pickler, - @Bind PythonLanguage language) { - assert pickler.write != null; - // This will commit the frame first - PBytes output = pickler.getString(language); - call(frame, pickler.write, output); - } - } - - public abstract static class SaveNode extends BasePickleWriteNode { - private static final int MAX_RECURSION_DEPTH = 3; private static final int SEEN_NONE = 1 << 0; private static final int SEEN_BOOLEAN = 1 << 1; @@ -1011,9 +1427,9 @@ private void saveDictIterator(VirtualFrame frame, PPickler pickler, Object itera private boolean savePers(VirtualFrame frame, PPickler pickler, int proto, Object obj) { Object pid; if (pickler.persFuncSelf == null) { - pid = getCallNode().execute(frame, pickler.persFunc, obj); + pid = ensureCallNode().execute(frame, pickler.persFunc, obj); } else { - pid = getCallNode().execute(frame, pickler.persFunc, pickler.persFuncSelf, obj); + pid = ensureCallNode().execute(frame, pickler.persFunc, pickler.persFuncSelf, obj); } if (pid != PNone.NONE) { if (isBin(proto)) { @@ -1466,7 +1882,7 @@ private void saveBytesData(VirtualFrame frame, PPickler pickler, int proto, bool if (bypassBuffer && pickler.write != null) { flushToFile(frame, pickler); - getCallNode().execute(frame, pickler.write, obj); + ensureCallNode().execute(frame, pickler.write, obj); pickler.clearBuffer(); } else { writeBuffer(pickler, buffer, size, bufferLib); @@ -1489,7 +1905,7 @@ private void writeUnicodeBinary(VirtualFrame frame, PPickler pickler, int proto, byte[] data = new byte[size]; ensureTsCopyToByteArrayNode().execute(utf8, 0, data, 0, size, TruffleString.Encoding.UTF_8); flushToFile(frame, pickler); - getCallNode().execute(frame, pickler.write, PFactory.createBytes(PythonLanguage.get(this), data, size)); + ensureCallNode().execute(frame, pickler.write, PFactory.createBytes(PythonLanguage.get(this), data, size)); pickler.clearBuffer(); } else { if (size <= 0xff && proto >= 4) { @@ -1954,7 +2370,7 @@ private void saveBytearrayData(VirtualFrame frame, PPickler pickler, int proto, if (bypassBuffer && pickler.write != null) { flushToFile(frame, pickler); - getCallNode().execute(frame, pickler.write, obj); + ensureCallNode().execute(frame, pickler.write, obj); pickler.clearBuffer(); } else { writeBuffer(pickler, buffer, size, bufferLib); @@ -2002,7 +2418,7 @@ private void savePickleBuffer(VirtualFrame frame, PPickler pickler, int proto, b boolean inBand = true; if (pickler.bufferCallback != null) { - Object ret = getCallNode().execute(frame, pickler.bufferCallback, obj); + Object ret = ensureCallNode().execute(frame, pickler.bufferCallback, obj); inBand = isTrue(frame, ret); } @@ -2355,14 +2771,33 @@ void saveGeneric(VirtualFrame frame, PPickler pickler, Object objArg, int persSa } } - public abstract static class DumpNode extends BasePickleWriteNode { + @GenerateInline(false) + public abstract static class FlushToFileNode extends Node { + public abstract void execute(VirtualFrame frame, PPickler pickler); + + @Specialization + public void flush(VirtualFrame frame, PPickler pickler, + @Bind PythonLanguage language, + @Cached CallNode callNode) { + assert pickler.write != null; + // This will commit the frame first + PBytes output = pickler.getString(language); + callNode.execute(frame, pickler.write, output); + } + } + + @GenerateInline(false) + public abstract static class DumpNode extends Node { + private static final TruffleString REDUCE_OVERRIDE = tsLiteral("reducer_override"); + public abstract void execute(VirtualFrame frame, PPickler pickler, Object obj); @Specialization public void dump(VirtualFrame frame, PPickler pickler, Object obj, - @Cached("create(0)") SaveNode saveNode) { + @Cached("create(0)") SaveNode saveNode, + @Cached PyObjectLookupAttr lookupAttrNode) { try { - final Object tmp = getLookupAttrNode().executeCached(frame, pickler, REDUCE_OVERRIDE); + final Object tmp = lookupAttrNode.execute(frame, this, pickler, REDUCE_OVERRIDE); if (tmp != PNone.NO_VALUE) { pickler.reducerOverride = tmp; } else { @@ -2373,9 +2808,9 @@ public void dump(VirtualFrame frame, PPickler pickler, Object obj, assert pickler.proto <= 256; try { - write(pickler, PickleUtils.OPCODE_PROTO, (byte) pickler.proto); + saveNode.write(pickler, PickleUtils.OPCODE_PROTO, (byte) pickler.proto); } catch (Exception e) { - handleError(pickler); + SaveNode.handleError(pickler); throw e; } @@ -2385,7 +2820,7 @@ public void dump(VirtualFrame frame, PPickler pickler, Object obj, } saveNode.execute(frame, pickler, obj, 0); - write(pickler, PickleUtils.OPCODE_STOP); + saveNode.write(pickler, PickleUtils.OPCODE_STOP); pickler.commitFrame(); } finally { pickler.framing = false; diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java index 419ec3a8b6..7c94c9425b 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java @@ -42,13 +42,8 @@ import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.T_CP_IMPORT_MAPPING; import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.T_CP_NAME_MAPPING; -import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.T_CP_REVERSE_IMPORT_MAPPING; -import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.T_CP_REVERSE_NAME_MAPPING; import static com.oracle.graal.python.builtins.objects.PNone.NO_VALUE; -import static com.oracle.graal.python.nodes.BuiltinNames.T___MAIN__; -import static com.oracle.graal.python.nodes.ErrorMessages.MUST_BE_STR_NOT_P; import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___CLASS__; -import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___MODULE__; import static com.oracle.graal.python.nodes.StringLiterals.T_DOT; import static com.oracle.graal.python.nodes.StringLiterals.T_UTF8; import static com.oracle.graal.python.nodes.statement.AbstractImportNode.importModule; @@ -64,7 +59,6 @@ import com.oracle.graal.python.builtins.PythonBuiltinClassType; import com.oracle.graal.python.builtins.modules.CodecsModuleBuiltins; import com.oracle.graal.python.builtins.modules.CodecsModuleBuiltinsFactory; -import com.oracle.graal.python.builtins.objects.PNone; import com.oracle.graal.python.builtins.objects.bytes.BytesNodes; import com.oracle.graal.python.builtins.objects.common.HashingStorage; import com.oracle.graal.python.builtins.objects.common.HashingStorageNodes.CachedHashingStorageGetItem; @@ -103,7 +97,6 @@ import com.oracle.graal.python.nodes.util.CannotCastException; import com.oracle.graal.python.nodes.util.CastToTruffleStringNode; import com.oracle.graal.python.runtime.IndirectCallData.BoundaryCallData; -import com.oracle.graal.python.runtime.PythonContext; import com.oracle.graal.python.runtime.exception.PException; import com.oracle.graal.python.runtime.object.PFactory; import com.oracle.graal.python.runtime.sequence.storage.SequenceStorage; @@ -118,7 +111,6 @@ public final class PicklerNodes { abstract static class BasePickleNode extends Node { private static final TruffleString T_LOCALS = tsLiteral(""); - static final TruffleString T_SYS_MODULES = tsLiteral("sys.modules"); public static final TruffleString T_CODEC_RAW_UNICODE_ESCAPE = tsLiteral("raw_unicode_escape"); public static final TruffleString T_CODEC_BYTES = tsLiteral("bytes"); public static final TruffleString T_CODEC_ASCII = tsLiteral("ascii"); @@ -155,13 +147,9 @@ abstract static class BasePickleNode extends Node { @Child private TruffleString.FromByteArrayNode tsFromByteArrayNode; @Child private TruffleString.FromByteArrayWithCompactionUTF32Node tsFromByteArrayWithCompactionNode; @Child private TruffleString.CodePointLengthNode tsCodePointLengthNode; - @Child private TruffleString.CodePointAtIndexUTF32Node tsCodePointAtIndexUTF32Node; - @Child private TruffleString.FromLongNode tsFromLongNode; @Child private TruffleString.IndexOfStringNode tsIndexOfStringNode; @Child private TruffleString.SubstringNode tsSubstringNode; @Child private TruffleString.EqualNode tsEqualNode; - @Child private TruffleString.CopyToByteArrayNode tsCopyToByteArrayNode; - @Child private TruffleString.GetCodeRangeNode tsGetCodeRangeNode; @Child private TruffleString.SwitchEncodingNode tsSwitchEncodingNode; @Child private HashingStorageGetIterator getHashingStorageIteratorNode; @Child private HashingStorageIteratorNext hashingStorageItNext; @@ -208,22 +196,6 @@ protected TruffleString.CodePointLengthNode ensureTsCodePointLengthNode() { return tsCodePointLengthNode; } - protected TruffleString.CodePointAtIndexUTF32Node ensureTsCodePointAtIndexUTF32Node() { - if (tsCodePointAtIndexUTF32Node == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - tsCodePointAtIndexUTF32Node = insert(TruffleString.CodePointAtIndexUTF32Node.create()); - } - return tsCodePointAtIndexUTF32Node; - } - - protected TruffleString.FromLongNode ensureTsFromLongNode() { - if (tsFromLongNode == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - tsFromLongNode = insert(TruffleString.FromLongNode.create()); - } - return tsFromLongNode; - } - protected TruffleString.IndexOfStringNode ensureTsIndexOfStringNode() { if (tsIndexOfStringNode == null) { CompilerDirectives.transferToInterpreterAndInvalidate(); @@ -248,22 +220,6 @@ protected TruffleString.EqualNode ensureTsEqualNode() { return tsEqualNode; } - protected TruffleString.CopyToByteArrayNode ensureTsCopyToByteArrayNode() { - if (tsCopyToByteArrayNode == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - tsCopyToByteArrayNode = insert(TruffleString.CopyToByteArrayNode.create()); - } - return tsCopyToByteArrayNode; - } - - protected TruffleString.GetCodeRangeNode ensureTsGetCodeRangeNode() { - if (tsGetCodeRangeNode == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - tsGetCodeRangeNode = insert(TruffleString.GetCodeRangeNode.create()); - } - return tsGetCodeRangeNode; - } - protected TruffleString.SwitchEncodingNode ensureTsSwitchEncodingNode() { if (tsSwitchEncodingNode == null) { CompilerDirectives.transferToInterpreterAndInvalidate(); @@ -587,18 +543,6 @@ public TruffleString asString(Object value) { } } - public TruffleString asStringStrict(Object value) { - if (value instanceof TruffleString) { - return (TruffleString) value; - } else { - try { - return toStringNode.executeCached(value); - } catch (CannotCastException e) { - throw raise(PythonBuiltinClassType.TypeError, MUST_BE_STR_NOT_P, value); - } - } - } - public static Pair getDeepAttribute(VirtualFrame frame, PyObjectLookupAttr lookup, Object obj, TruffleString[] names) { Object parent = null; Object object = obj; @@ -644,11 +588,6 @@ public Object getattribute(VirtualFrame frame, Object obj, TruffleString name, b } } - protected Pair get3to2Mapping(VirtualFrame frame, Python3Core core, TruffleString moduleName, TruffleString globalName) { - PickleState state = getGlobalState(core); - return getMapping(frame, state.nameMapping3To2, state.importMapping3To2, T_CP_REVERSE_NAME_MAPPING, T_CP_REVERSE_IMPORT_MAPPING, moduleName, globalName); - } - protected Pair get2To3Mapping(VirtualFrame frame, Python3Core core, TruffleString moduleName, TruffleString globalName) { PickleState state = getGlobalState(core); return getMapping(frame, state.nameMapping2To3, state.importMapping2To3, T_CP_NAME_MAPPING, T_CP_IMPORT_MAPPING, moduleName, globalName); @@ -702,54 +641,5 @@ public Object findClass(VirtualFrame frame, BoundaryCallData boundaryCallData, P Object module = importModule(frame, boundaryCallData, mName); return getattribute(frame, module, gName, self.getProto() >= 4); } - - private static boolean checkModule(VirtualFrame frame, PyObjectLookupAttr lookup, Object moduleName, Object module, Object global, TruffleString[] dottedPath) { - if (module == PNone.NONE) { - return false; - } - - assert !(moduleName instanceof String) : "moduleName shouldn't be j.l.String"; - - if (moduleName instanceof TruffleString && ((TruffleString) moduleName).equalsUncached(T___MAIN__, TS_ENCODING)) { - return false; - } - final Pair pair = getDeepAttribute(frame, lookup, module, dottedPath); - if (pair == null) { - return false; - } - return pair.getLeft() == global; - } - - public TruffleString whichModule(VirtualFrame frame, PythonContext context, Object global, TruffleString[] dottedPath) { - Object moduleName = lookupAttribute(frame, global, T___MODULE__); - if (moduleName != NO_VALUE) { - // In some rare cases (e.g., bound methods of extension types), __module__ can be - // None. If it is so, then search sys.modules for the module of global - if (moduleName != PNone.NONE) { - return asStringStrict(moduleName); - } - } - - // Fallback on walking sys.modules - final PDict sysModules = context.getSysModules(); - if (sysModules == null) { - throw raise(PythonBuiltinClassType.RuntimeError, ErrorMessages.UNABLE_TO_GET_S, T_SYS_MODULES); - } - - final HashingStorage storage = sysModules.getDictStorage(); - HashingStorageIterator it = getHashingStorageIterator(storage); - HashingStorageIteratorNext nextNode = ensureHashingStorageIteratorNext(); - HashingStorageIteratorKey getKeyNode = ensureHashingStorageIteratorKey(); - HashingStorageIteratorValue getValueNode = ensureHashingStorageIteratorValue(); - while (nextNode.executeCached(storage, it)) { - Object value = getValueNode.executeCached(storage, it); - Object key = getKeyNode.executeCached(storage, it); - if (checkModule(frame, getLookupAttrNode(), moduleName, value, global, dottedPath)) { - return asStringStrict(key); - } - } - // If no module is found, use __main__ - return T___MAIN__; - } } } From afc6dd2e34445f7358b71396d6af0514aedaa13a Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Tue, 29 Sep 2026 15:21:39 +0200 Subject: [PATCH 10/39] Make pickler write helpers static --- .../builtins/modules/pickle/PPickler.java | 307 +++++++++--------- 1 file changed, 159 insertions(+), 148 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java index 296cb5cecf..dddf0773fb 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java @@ -856,11 +856,15 @@ protected boolean isType(Object obj) { } protected void flushToFile(VirtualFrame frame, PPickler pickler) { + ensureFlushToFileNode().execute(frame, pickler); + } + + protected FlushToFileNode ensureFlushToFileNode() { if (flushToFileNode == null) { CompilerDirectives.transferToInterpreterAndInvalidate(); flushToFileNode = insert(PPicklerFactory.FlushToFileNodeGen.create()); } - flushToFileNode.execute(frame, pickler); + return flushToFileNode; } protected int getSign(Object value) { @@ -926,51 +930,54 @@ public static void handleError(PPickler pickler) { pickler.reducerOverride = null; } - protected void writeASCII(PPickler pickler, TruffleString string) { + protected static void writeASCII(Node node, PPickler pickler, TruffleString string, TruffleString.SwitchEncodingNode switchEncodingNode, + TruffleString.CopyToByteArrayNode copyToByteArrayNode) { assert string.getCodeRangeUncached(TS_ENCODING) == TruffleString.CodeRange.ASCII; - TruffleString encoded = ensureTsSwitchEncodingNode().execute(string, TruffleString.Encoding.US_ASCII); + TruffleString encoded = switchEncodingNode.execute(string, TruffleString.Encoding.US_ASCII); int length = encoded.byteLength(TruffleString.Encoding.US_ASCII); - ensureBufferSpace(pickler, length); - ensureTsCopyToByteArrayNode().execute(encoded, 0, pickler.outputBuffer, pickler.outputLen, length, TruffleString.Encoding.US_ASCII); + ensureBufferSpace(node, pickler, length); + copyToByteArrayNode.execute(encoded, 0, pickler.outputBuffer, pickler.outputLen, length, TruffleString.Encoding.US_ASCII); pickler.outputLen += length; } - protected void writeUtf8(PPickler pickler, TruffleString string) { - TruffleString encoded = ensureTsSwitchEncodingNode().execute(string, TruffleString.Encoding.UTF_8, TranscodingErrorHandler.DEFAULT_KEEP_SURROGATES_IN_UTF8); - writeUtf8Converted(pickler, encoded); + protected static void writeUtf8(Node node, PPickler pickler, TruffleString string, TruffleString.SwitchEncodingNode switchEncodingNode, + TruffleString.CopyToByteArrayNode copyToByteArrayNode) { + TruffleString encoded = switchEncodingNode.execute(string, TruffleString.Encoding.UTF_8, TranscodingErrorHandler.DEFAULT_KEEP_SURROGATES_IN_UTF8); + writeUtf8Converted(node, pickler, encoded, copyToByteArrayNode); } - public void writeUtf8Converted(PPickler pickler, TruffleString encoded) { + public static void writeUtf8Converted(Node node, PPickler pickler, TruffleString encoded, TruffleString.CopyToByteArrayNode copyToByteArrayNode) { int length = encoded.byteLength(TruffleString.Encoding.UTF_8); - ensureBufferSpace(pickler, length); - ensureTsCopyToByteArrayNode().execute(encoded, 0, pickler.outputBuffer, pickler.outputLen, length, TruffleString.Encoding.UTF_8); + ensureBufferSpace(node, pickler, length); + copyToByteArrayNode.execute(encoded, 0, pickler.outputBuffer, pickler.outputLen, length, TruffleString.Encoding.UTF_8); pickler.outputLen += length; } - protected void writeLongAsString(PPickler pickler, byte opcode, long value) { - TruffleString string = ensureTsFromLongNode().execute(value, TruffleString.Encoding.US_ASCII, true); + protected static void writeLongAsString(Node node, PPickler pickler, byte opcode, long value, TruffleString.FromLongNode fromLongNode, + TruffleString.CopyToByteArrayNode copyToByteArrayNode) { + TruffleString string = fromLongNode.execute(value, TruffleString.Encoding.US_ASCII, true); TruffleString.Encoding encoding = TruffleString.Encoding.US_ASCII; int length = string.byteLength(encoding); - ensureBufferSpace(pickler, length + 2); + ensureBufferSpace(node, pickler, length + 2); pickler.outputBuffer[pickler.outputLen++] = opcode; - ensureTsCopyToByteArrayNode().execute(string, 0, pickler.outputBuffer, pickler.outputLen, length, encoding); + copyToByteArrayNode.execute(string, 0, pickler.outputBuffer, pickler.outputLen, length, encoding); pickler.outputLen += length; pickler.outputBuffer[pickler.outputLen++] = NEW_LINE_BYTE; } - protected void write(PPickler pickler, byte b) { - ensureBufferSpace(pickler, 1); + protected static void write(Node node, PPickler pickler, byte b) { + ensureBufferSpace(node, pickler, 1); pickler.outputBuffer[pickler.outputLen++] = b; } - protected void write(PPickler pickler, byte b1, byte b2) { - ensureBufferSpace(pickler, 2); + protected static void write(Node node, PPickler pickler, byte b1, byte b2) { + ensureBufferSpace(node, pickler, 2); pickler.outputBuffer[pickler.outputLen++] = b1; pickler.outputBuffer[pickler.outputLen++] = b2; } - protected void writeBuffer(PPickler pickler, Object buffer, int size, PythonBufferAccessLibrary bufferLib) { - ensureBufferSpace(pickler, size); + protected static void writeBuffer(Node node, PPickler pickler, Object buffer, int size, PythonBufferAccessLibrary bufferLib) { + ensureBufferSpace(node, pickler, size); bufferLib.readIntoByteArray(buffer, 0, pickler.outputBuffer, pickler.outputLen, size); pickler.outputLen += size; } @@ -991,21 +998,21 @@ protected static byte intByte4(long value) { return (byte) ((value >> 24) & 0xff); } - protected void writeByteOp(PPickler pickler, byte opcode, long value) { - ensureBufferSpace(pickler, 2); + protected static void writeByteOp(Node node, PPickler pickler, byte opcode, long value) { + ensureBufferSpace(node, pickler, 2); pickler.outputBuffer[pickler.outputLen++] = opcode; pickler.outputBuffer[pickler.outputLen++] = intByte1(value); } - protected void writeShortOp(PPickler pickler, byte opcode, long value) { - ensureBufferSpace(pickler, 3); + protected static void writeShortOp(Node node, PPickler pickler, byte opcode, long value) { + ensureBufferSpace(node, pickler, 3); pickler.outputBuffer[pickler.outputLen++] = opcode; pickler.outputBuffer[pickler.outputLen++] = intByte1(value); pickler.outputBuffer[pickler.outputLen++] = intByte2(value); } - protected void writeIntOp(PPickler pickler, byte opcode, long value) { - ensureBufferSpace(pickler, 5); + protected static void writeIntOp(Node node, PPickler pickler, byte opcode, long value) { + ensureBufferSpace(node, pickler, 5); pickler.outputBuffer[pickler.outputLen++] = opcode; pickler.outputBuffer[pickler.outputLen++] = intByte1(value); pickler.outputBuffer[pickler.outputLen++] = intByte2(value); @@ -1013,17 +1020,17 @@ protected void writeIntOp(PPickler pickler, byte opcode, long value) { pickler.outputBuffer[pickler.outputLen++] = intByte4(value); } - protected void write(PPickler pickler, byte[] bytes) { - write(pickler, bytes, bytes.length); + protected static void write(Node node, PPickler pickler, byte[] bytes) { + write(node, pickler, bytes, bytes.length); } - protected void write(PPickler pickler, byte[] bytes, int dataLen) { - ensureBufferSpace(pickler, dataLen); + protected static void write(Node node, PPickler pickler, byte[] bytes, int dataLen) { + ensureBufferSpace(node, pickler, dataLen); PythonUtils.arraycopy(bytes, 0, pickler.outputBuffer, pickler.outputLen, dataLen); pickler.outputLen += dataLen; } - public void ensureBufferSpace(PPickler pickler, int dataLen) { + public static void ensureBufferSpace(Node node, PPickler pickler, int dataLen) { boolean needNewFrame = pickler.isFraming() && pickler.frameStart == -1; int n = CompilerDirectives.injectBranchProbability(CompilerDirectives.SLOWPATH_PROBABILITY, needNewFrame) ? dataLen + PickleUtils.FRAME_HEADER_SIZE : dataLen; int required = pickler.outputLen + n; @@ -1032,7 +1039,8 @@ public void ensureBufferSpace(PPickler pickler, int dataLen) { // Make place in buffer for the pickle chunk // TODO: when GR-24978 is completed we should use PY_SSIZE_T_MAX if (CompilerDirectives.injectBranchProbability(CompilerDirectives.SLOWPATH_PROBABILITY, pickler.outputLen >= Integer.MAX_VALUE / 2 - n)) { - throw raise(PythonBuiltinClassType.MemoryError); + CompilerDirectives.transferToInterpreter(); + throw PRaiseNode.raiseStatic(node, PythonBuiltinClassType.MemoryError); } pickler.maxOutputLen = (pickler.outputLen + n) / 2 * 3; pickler.outputBuffer = PythonUtils.arrayCopyOf(pickler.outputBuffer, pickler.maxOutputLen); @@ -1043,7 +1051,7 @@ public void ensureBufferSpace(PPickler pickler, int dataLen) { } } - protected boolean bypassBuffer(PPickler pickler, int dataSize) { + protected static boolean bypassBuffer(PPickler pickler, int dataSize) { boolean bypassBuffer = dataSize >= PickleUtils.FRAME_SIZE_TARGET; if (bypassBuffer) { @@ -1286,7 +1294,7 @@ private void saveHashingStorageBatched(VirtualFrame frame, PPickler pickler, Has int i; do { i = 0; - write(pickler, PickleUtils.OPCODE_MARK); + write(this, pickler, PickleUtils.OPCODE_MARK); while (nextNode.executeCached(storage, it)) { save(frame, pickler, getKeyNode.executeCached(storage, it), 0); if (saveValues) { @@ -1296,7 +1304,7 @@ private void saveHashingStorageBatched(VirtualFrame frame, PPickler pickler, Has break; } } - write(pickler, opcodeBatch); + write(this, pickler, opcodeBatch); if (getHashingStorageLength(storage) != initialSize) { throw raise(PythonBuiltinClassType.RuntimeError, ErrorMessages.CHANGED_SIZE_DURING_ITERATION, name); } @@ -1342,12 +1350,12 @@ private void saveIteratorBatchedUnrolled(VirtualFrame frame, PPickler pickler, O } catch (IteratorExhausted e) { // Only one item to write saveItem.accept(firstItem); - write(pickler, opcodeOneItem); + write(this, pickler, opcodeOneItem); break; } // More than one item to write // Pump out MARK, items, APPENDS. - write(pickler, PickleUtils.OPCODE_MARK); + write(this, pickler, PickleUtils.OPCODE_MARK); saveItem.accept(firstItem); n = 1; while (true) { @@ -1366,7 +1374,7 @@ private void saveIteratorBatchedUnrolled(VirtualFrame frame, PPickler pickler, O break; } } - write(pickler, opcodeMultipleItems); + write(this, pickler, opcodeMultipleItems); } while (n == PickleUtils.BATCHSIZE); } @@ -1401,7 +1409,7 @@ private void saveIterator(VirtualFrame frame, PPickler pickler, Object iterator, } itemConsumer.accept(item); if (opcode != PickleUtils.NO_OPCODE) { - write(pickler, opcode); + write(this, pickler, opcode); } } } @@ -1434,16 +1442,16 @@ private boolean savePers(VirtualFrame frame, PPickler pickler, int proto, Object if (pid != PNone.NONE) { if (isBin(proto)) { save(frame, pickler, pid, 1); - write(pickler, PickleUtils.OPCODE_BINPERSID); + write(this, pickler, PickleUtils.OPCODE_BINPERSID); } else { final TruffleString pidStr = asString(convertToStr(frame, pid)); // XXX: Should it check whether the pid contains embedded newlines? if (!PythonUtils.isAscii(pidStr, ensureTsGetCodeRangeNode())) { throw raise(PythonBuiltinClassType.PicklingError, ErrorMessages.PIDS_MUST_BE_ASCII_STRS); } - write(pickler, PickleUtils.OPCODE_PERSID); - writeASCII(pickler, pidStr); - writeASCII(pickler, T_NEWLINE); + write(this, pickler, PickleUtils.OPCODE_PERSID); + writeASCII(this, pickler, pidStr, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); + writeASCII(this, pickler, T_NEWLINE, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); } return true; } @@ -1454,15 +1462,15 @@ private boolean savePers(VirtualFrame frame, PPickler pickler, int proto, Object private void memoGet(PPickler pickler, int proto, int value) { if (isBin(proto)) { if (value < 256) { - writeByteOp(pickler, PickleUtils.OPCODE_BINGET, value); + writeByteOp(this, pickler, PickleUtils.OPCODE_BINGET, value); } else if (Long.compareUnsigned(value, 0xffffffffL) <= 0) { - writeIntOp(pickler, PickleUtils.OPCODE_LONG_BINGET, value); + writeIntOp(this, pickler, PickleUtils.OPCODE_LONG_BINGET, value); } else { // unlikely throw raise(PicklingError, ErrorMessages.MEMO_ID_TOO_LARGE_FOR_S, "LONG_BINGET"); } } else { - writeLongAsString(pickler, PickleUtils.OPCODE_GET, value); + writeLongAsString(this, pickler, PickleUtils.OPCODE_GET, value, ensureTsFromLongNode(), ensureTsCopyToByteArrayNode()); } } @@ -1476,15 +1484,15 @@ private void memoPut(PPickler pickler, int proto, boolean fastMode, Object obj) pickler.memo.set(obj, idx); if (proto >= 4) { - write(pickler, PickleUtils.OPCODE_MEMOIZE); + write(this, pickler, PickleUtils.OPCODE_MEMOIZE); } else { if (!isBin(proto)) { - writeLongAsString(pickler, PickleUtils.OPCODE_PUT, idx); + writeLongAsString(this, pickler, PickleUtils.OPCODE_PUT, idx, ensureTsFromLongNode(), ensureTsCopyToByteArrayNode()); } else { if (idx < 256) { - writeByteOp(pickler, PickleUtils.OPCODE_BINPUT, idx); + writeByteOp(this, pickler, PickleUtils.OPCODE_BINPUT, idx); } else if (Long.compareUnsigned(idx, 0xffffffffL) <= 0) { - writeIntOp(pickler, PickleUtils.OPCODE_LONG_BINPUT, idx); + writeIntOp(this, pickler, PickleUtils.OPCODE_LONG_BINPUT, idx); } else { // unlikely throw raise(PicklingError, ErrorMessages.MEMO_ID_TOO_LARGE_FOR_S, "LONG_BINPUT"); @@ -1601,7 +1609,7 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, save(frame, pickler, cls, 0); save(frame, pickler, args, 0); save(frame, pickler, kwargs, 0); - write(pickler, PickleUtils.OPCODE_NEWOBJ_EX); + write(this, pickler, PickleUtils.OPCODE_NEWOBJ_EX); } else { PickleState st = getGlobalState(ctx.getCore()); final int argsSize = length(frame, args); @@ -1620,7 +1628,7 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, save(frame, pickler, callable, 0); save(frame, pickler, PFactory.createEmptyTuple(PythonLanguage.get(this)), 0); - write(pickler, PickleUtils.OPCODE_REDUCE); + write(this, pickler, PickleUtils.OPCODE_REDUCE); } } else if (useNewobj) { Object cls; @@ -1677,12 +1685,12 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, save(frame, pickler, cls, 0); newargtup = getItem(frame, argtup, PFactory.createIntSlice(PythonLanguage.get(this), 1, argtupSize, 1)); save(frame, pickler, newargtup, 0); - write(pickler, PickleUtils.OPCODE_NEWOBJ); + write(this, pickler, PickleUtils.OPCODE_NEWOBJ); } else { // Not using NEWOBJ save(frame, pickler, callable, 0); save(frame, pickler, argtup, 0); - write(pickler, PickleUtils.OPCODE_REDUCE); + write(this, pickler, PickleUtils.OPCODE_REDUCE); } // obj can be NULL when save_reduce() is used directly. A NULL obj means the caller do @@ -1694,7 +1702,7 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, // memo int memoIndex = pickler.memo.get(obj); if (memoIndex != -1) { - write(pickler, PickleUtils.OPCODE_POP); + write(this, pickler, PickleUtils.OPCODE_POP); memoGet(pickler, proto, memoIndex); } else { memoPut(pickler, proto, fastMode, obj); @@ -1712,7 +1720,7 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, if (state != null) { if (stateSetter == null) { save(frame, pickler, state, 0); - write(pickler, PickleUtils.OPCODE_BUILD); + write(this, pickler, PickleUtils.OPCODE_BUILD); } else { // If a state_setter is specified, call it instead of load_build to update obj's // with its previous state. The first 4 save/write instructions push @@ -1724,40 +1732,40 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, save(frame, pickler, stateSetter, 0); save(frame, pickler, obj, 0); save(frame, pickler, state, 0); - write(pickler, PickleUtils.OPCODE_TUPLE2); - write(pickler, PickleUtils.OPCODE_REDUCE); - write(pickler, PickleUtils.OPCODE_POP); + write(this, pickler, PickleUtils.OPCODE_TUPLE2); + write(this, pickler, PickleUtils.OPCODE_REDUCE); + write(this, pickler, PickleUtils.OPCODE_POP); } } } private void saveNone(PPickler pickler) { - write(pickler, PickleUtils.OPCODE_NONE); + write(this, pickler, PickleUtils.OPCODE_NONE); } private void saveBool(PPickler pickler, int proto, boolean value) { if (proto >= 2) { - write(pickler, value ? PickleUtils.OPCODE_NEWTRUE : PickleUtils.OPCODE_NEWFALSE); + write(this, pickler, value ? PickleUtils.OPCODE_NEWTRUE : PickleUtils.OPCODE_NEWFALSE); } else { // These aren't opcodes -- they're ways to pickle bools before protocol 2 so that // unpicklers written before bools were introduced unpickle them as ints, but // unpicklers after can recognize that bools were intended. Note that protocol 2 // added direct ways to pickle bools. - writeASCII(pickler, value ? PickleUtils.T_PROTO_LE2_TRUE : PickleUtils.T_PROTO_LE2_FALSE); + writeASCII(this, pickler, value ? PickleUtils.T_PROTO_LE2_TRUE : PickleUtils.T_PROTO_LE2_FALSE, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); } } private void saveLong(PPickler pickler, int proto, int value) { if (isBin(proto)) { if (value >= 0 && value < (1 << 8)) { - writeByteOp(pickler, PickleUtils.OPCODE_BININT1, value); + writeByteOp(this, pickler, PickleUtils.OPCODE_BININT1, value); } else if (value >= 0 && value < (1 << 16)) { - writeShortOp(pickler, PickleUtils.OPCODE_BININT2, value); + writeShortOp(this, pickler, PickleUtils.OPCODE_BININT2, value); } else { - writeIntOp(pickler, PickleUtils.OPCODE_BININT, value); + writeIntOp(this, pickler, PickleUtils.OPCODE_BININT, value); } } else { - writeLongAsString(pickler, PickleUtils.OPCODE_INT, value); + writeLongAsString(this, pickler, PickleUtils.OPCODE_INT, value, ensureTsFromLongNode(), ensureTsCopyToByteArrayNode()); } } @@ -1805,32 +1813,32 @@ private void saveLongLarge(VirtualFrame frame, PPickler pickler, int proto, Obje nbytes--; } if (nbytes < 256) { - writeByteOp(pickler, PickleUtils.OPCODE_LONG1, nbytes); + writeByteOp(this, pickler, PickleUtils.OPCODE_LONG1, nbytes); } else { - writeIntOp(pickler, PickleUtils.OPCODE_LONG4, nbytes); + writeIntOp(this, pickler, PickleUtils.OPCODE_LONG4, nbytes); } - write(pickler, pdata, nbytes); + write(this, pickler, pdata, nbytes); } else { TruffleString repr = repr(frame, obj); - write(pickler, PickleUtils.OPCODE_LONG); - writeASCII(pickler, asStringStrict(repr)); - writeASCII(pickler, T_L_NEW_LINE); + write(this, pickler, PickleUtils.OPCODE_LONG); + writeASCII(this, pickler, asStringStrict(repr), ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); + writeASCII(this, pickler, T_L_NEW_LINE, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); } } private void saveFloat(PPickler pickler, int proto, double value) { if (isBin(proto)) { - write(pickler, PickleUtils.OPCODE_BINFLOAT); - ensureBufferSpace(pickler, 8); + write(this, pickler, PickleUtils.OPCODE_BINFLOAT); + ensureBufferSpace(this, pickler, 8); NumericSupport.bigEndian().putDouble(pickler.outputBuffer, pickler.outputLen, value); pickler.outputLen += 8; } else { - write(pickler, PickleUtils.OPCODE_FLOAT); + write(this, pickler, PickleUtils.OPCODE_FLOAT); TruffleString repr = PickleUtils.doubleToAsciiString(value); - writeASCII(pickler, repr); - writeASCII(pickler, T_NEWLINE); + writeASCII(this, pickler, repr, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); + writeASCII(this, pickler, T_NEWLINE, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); } } @@ -1875,9 +1883,9 @@ private void saveBytesData(VirtualFrame frame, PPickler pickler, int proto, bool boolean bypassBuffer = bypassBuffer(pickler, size); if (size <= 0xff) { - writeByteOp(pickler, PickleUtils.OPCODE_SHORT_BINBYTES, size); + writeByteOp(this, pickler, PickleUtils.OPCODE_SHORT_BINBYTES, size); } else { - writeIntOp(pickler, PickleUtils.OPCODE_BINBYTES, size); + writeIntOp(this, pickler, PickleUtils.OPCODE_BINBYTES, size); } if (bypassBuffer && pickler.write != null) { @@ -1885,7 +1893,7 @@ private void saveBytesData(VirtualFrame frame, PPickler pickler, int proto, bool ensureCallNode().execute(frame, pickler.write, obj); pickler.clearBuffer(); } else { - writeBuffer(pickler, buffer, size, bufferLib); + writeBuffer(this, pickler, buffer, size, bufferLib); } if (bypassBuffer && wasFraming) { @@ -1894,26 +1902,28 @@ private void saveBytesData(VirtualFrame frame, PPickler pickler, int proto, bool memoPut(pickler, proto, fastMode, obj); } - private void writeUnicodeBinary(VirtualFrame frame, PPickler pickler, int proto, TruffleString string) { - TruffleString utf8 = ensureTsSwitchEncodingNode().execute(string, TruffleString.Encoding.UTF_8, TranscodingErrorHandler.DEFAULT_KEEP_SURROGATES_IN_UTF8); + private static void writeUnicodeBinary(Node node, VirtualFrame frame, PPickler pickler, int proto, TruffleString string, + TruffleString.SwitchEncodingNode switchEncodingNode, TruffleString.CopyToByteArrayNode copyToByteArrayNode, + FlushToFileNode flushToFileNode, CallNode callNode) { + TruffleString utf8 = switchEncodingNode.execute(string, TruffleString.Encoding.UTF_8, TranscodingErrorHandler.DEFAULT_KEEP_SURROGATES_IN_UTF8); int size = utf8.byteLength(TruffleString.Encoding.UTF_8); boolean wasFraming = pickler.framing; boolean bypassBuffer = bypassBuffer(pickler, size); if (bypassBuffer && pickler.write != null) { - writeIntOp(pickler, PickleUtils.OPCODE_BINUNICODE, size); + writeIntOp(node, pickler, PickleUtils.OPCODE_BINUNICODE, size); byte[] data = new byte[size]; - ensureTsCopyToByteArrayNode().execute(utf8, 0, data, 0, size, TruffleString.Encoding.UTF_8); - flushToFile(frame, pickler); - ensureCallNode().execute(frame, pickler.write, PFactory.createBytes(PythonLanguage.get(this), data, size)); + copyToByteArrayNode.execute(utf8, 0, data, 0, size, TruffleString.Encoding.UTF_8); + flushToFileNode.execute(frame, pickler); + callNode.execute(frame, pickler.write, PFactory.createBytes(PythonLanguage.get(node), data, size)); pickler.clearBuffer(); } else { if (size <= 0xff && proto >= 4) { - writeByteOp(pickler, PickleUtils.OPCODE_SHORT_BINUNICODE, size); + writeByteOp(node, pickler, PickleUtils.OPCODE_SHORT_BINUNICODE, size); } else { - writeIntOp(pickler, PickleUtils.OPCODE_BINUNICODE, size); + writeIntOp(node, pickler, PickleUtils.OPCODE_BINUNICODE, size); } - writeUtf8Converted(pickler, utf8); + writeUtf8Converted(node, pickler, utf8, copyToByteArrayNode); } if (bypassBuffer && wasFraming) { @@ -1931,28 +1941,29 @@ private void saveUnicode(VirtualFrame frame, PPickler pickler, int proto, boolea private void saveUnicode(VirtualFrame frame, PPickler pickler, int proto, boolean fastMode, Object obj, TruffleString string) { if (isBin(proto)) { - writeUnicodeBinary(frame, pickler, proto, string); + writeUnicodeBinary(this, frame, pickler, proto, string, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode(), ensureFlushToFileNode(), ensureCallNode()); } else { - write(pickler, PickleUtils.OPCODE_UNICODE); - writeRawUnicodeEscape(pickler, string); - writeASCII(pickler, T_NEWLINE); + write(this, pickler, PickleUtils.OPCODE_UNICODE); + writeRawUnicodeEscape(this, pickler, string, ensureTsCodePointLengthNode(), ensureTsCodePointAtIndexUTF32Node()); + writeASCII(this, pickler, T_NEWLINE, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); } memoPut(pickler, proto, fastMode, obj); } - private void writeRawUnicodeEscape(PPickler pickler, TruffleString string) { - int codePointLength = ensureTsCodePointLengthNode().execute(string, TS_ENCODING); + private static void writeRawUnicodeEscape(Node node, PPickler pickler, TruffleString string, + TruffleString.CodePointLengthNode codePointLengthNode, TruffleString.CodePointAtIndexUTF32Node codePointAtIndexNode) { + int codePointLength = codePointLengthNode.execute(string, TS_ENCODING); long escapedLength = 0; for (int i = 0; i < codePointLength; i++) { - int ch = ensureTsCodePointAtIndexUTF32Node().execute(string, i); + int ch = codePointAtIndexNode.execute(string, i); escapedLength += ch >= 0x10000 ? 10 : ch >= 256 || ch == '\\' || ch == 0 || ch == '\n' || ch == '\r' || ch == 0x1a ? 6 : 1; } if (escapedLength > Integer.MAX_VALUE) { - throw raise(PythonBuiltinClassType.MemoryError); + throw PRaiseNode.raiseStatic(node, PythonBuiltinClassType.MemoryError); } - ensureBufferSpace(pickler, (int) escapedLength); + ensureBufferSpace(node, pickler, (int) escapedLength); for (int i = 0; i < codePointLength; i++) { - int ch = ensureTsCodePointAtIndexUTF32Node().execute(string, i); + int ch = codePointAtIndexNode.execute(string, i); if (ch >= 0x10000) { pickler.outputBuffer[pickler.outputLen++] = '\\'; pickler.outputBuffer[pickler.outputLen++] = 'U'; @@ -1984,7 +1995,7 @@ private void batchDictExact(VirtualFrame frame, PPickler pickler, PDict dict) { HashingStorageIteratorValue getValueNode = ensureHashingStorageIteratorValue(); save(frame, pickler, getKeyNode.executeCached(storage, it), 0); save(frame, pickler, getValueNode.executeCached(storage, it), 0); - write(pickler, PickleUtils.OPCODE_SETITEM); + write(this, pickler, PickleUtils.OPCODE_SETITEM); } else { // Write in batches of BATCHSIZE. saveDictHashingStorageBatched(frame, pickler, storage); @@ -2009,9 +2020,9 @@ private void saveDict(VirtualFrame frame, Node inliningTarget, PyObjectCallMetho // Create an empty dict. if (isBin(proto)) { - write(pickler, PickleUtils.OPCODE_EMPTY_DICT); + write(this, pickler, PickleUtils.OPCODE_EMPTY_DICT); } else { - write(pickler, PickleUtils.OPCODE_MARK, PickleUtils.OPCODE_DICT); + write(this, pickler, PickleUtils.OPCODE_MARK, PickleUtils.OPCODE_DICT); } memoPut(pickler, proto, fastMode, obj); @@ -2043,7 +2054,7 @@ private void saveSet(VirtualFrame frame, PythonContext ctx, PPickler pickler, in return; } - write(pickler, PickleUtils.OPCODE_EMPTY_SET); + write(this, pickler, PickleUtils.OPCODE_EMPTY_SET); memoPut(pickler, proto, fastMode, obj); final int setSize = length(frame, obj); @@ -2068,7 +2079,7 @@ private void saveFrozenset(VirtualFrame frame, PythonContext ctx, PPickler pickl return; } - write(pickler, PickleUtils.OPCODE_MARK); + write(this, pickler, PickleUtils.OPCODE_MARK); saveSetHashingStorage(frame, pickler, obj.getDictStorage()); @@ -2076,12 +2087,12 @@ private void saveFrozenset(VirtualFrame frame, PythonContext ctx, PPickler pickl // away everything we put on the stack, and fetch the object back from the memo. int memoIndex = pickler.memo.get(obj); if (memoIndex != -1) { - write(pickler, PickleUtils.OPCODE_POP_MARK); + write(this, pickler, PickleUtils.OPCODE_POP_MARK); memoGet(pickler, proto, memoIndex); return; } - write(pickler, PickleUtils.OPCODE_FROZENSET); + write(this, pickler, PickleUtils.OPCODE_FROZENSET); memoPut(pickler, proto, fastMode, obj); } @@ -2107,7 +2118,7 @@ private void batchListExact(VirtualFrame frame, PPickler pickler, PList obj) { if (storage.length() == 1) { item = getItem(storage, 0); save(frame, pickler, item, 0); - write(pickler, PickleUtils.OPCODE_APPEND); + write(this, pickler, PickleUtils.OPCODE_APPEND); return; } @@ -2116,7 +2127,7 @@ private void batchListExact(VirtualFrame frame, PPickler pickler, PList obj) { int thisBatch; do { thisBatch = 0; - write(pickler, PickleUtils.OPCODE_MARK); + write(this, pickler, PickleUtils.OPCODE_MARK); while (total < storage.length()) { item = getItem(storage, total); save(frame, pickler, item, 0); @@ -2125,7 +2136,7 @@ private void batchListExact(VirtualFrame frame, PPickler pickler, PList obj) { break; } } - write(pickler, PickleUtils.OPCODE_APPENDS); + write(this, pickler, PickleUtils.OPCODE_APPENDS); } while (total < storage.length()); } @@ -2134,19 +2145,19 @@ private void batchIntList(VirtualFrame frame, PPickler pickler, IntSequenceStora if (storage.length() == 1) { opcodeBoundary(frame, pickler); saveLong(pickler, proto, storage.getIntItemNormalized(0)); - write(pickler, PickleUtils.OPCODE_APPEND); + write(this, pickler, PickleUtils.OPCODE_APPEND); return; } int length = storage.length(); for (int batchStart = 0; batchStart < length; batchStart += PickleUtils.BATCHSIZE) { int batchEnd = batchStart + Math.min(PickleUtils.BATCHSIZE, length - batchStart); - write(pickler, PickleUtils.OPCODE_MARK); + write(this, pickler, PickleUtils.OPCODE_MARK); for (int i = batchStart; i < batchEnd; i++) { opcodeBoundary(frame, pickler); saveLong(pickler, proto, storage.getIntItemNormalized(i)); } - write(pickler, PickleUtils.OPCODE_APPENDS); + write(this, pickler, PickleUtils.OPCODE_APPENDS); } } @@ -2155,19 +2166,19 @@ private void batchLongList(VirtualFrame frame, PPickler pickler, LongSequenceSto if (storage.length() == 1) { opcodeBoundary(frame, pickler); saveLong(frame, pickler, proto, storage.getLongItemNormalized(0)); - write(pickler, PickleUtils.OPCODE_APPEND); + write(this, pickler, PickleUtils.OPCODE_APPEND); return; } int length = storage.length(); for (int batchStart = 0; batchStart < length; batchStart += PickleUtils.BATCHSIZE) { int batchEnd = batchStart + Math.min(PickleUtils.BATCHSIZE, length - batchStart); - write(pickler, PickleUtils.OPCODE_MARK); + write(this, pickler, PickleUtils.OPCODE_MARK); for (int i = batchStart; i < batchEnd; i++) { opcodeBoundary(frame, pickler); saveLong(frame, pickler, proto, storage.getLongItemNormalized(i)); } - write(pickler, PickleUtils.OPCODE_APPENDS); + write(this, pickler, PickleUtils.OPCODE_APPENDS); } } @@ -2176,19 +2187,19 @@ private void batchDoubleList(VirtualFrame frame, PPickler pickler, DoubleSequenc if (storage.length() == 1) { opcodeBoundary(frame, pickler); saveFloat(pickler, proto, storage.getDoubleItemNormalized(0)); - write(pickler, PickleUtils.OPCODE_APPEND); + write(this, pickler, PickleUtils.OPCODE_APPEND); return; } int length = storage.length(); for (int batchStart = 0; batchStart < length; batchStart += PickleUtils.BATCHSIZE) { int batchEnd = batchStart + Math.min(PickleUtils.BATCHSIZE, length - batchStart); - write(pickler, PickleUtils.OPCODE_MARK); + write(this, pickler, PickleUtils.OPCODE_MARK); for (int i = batchStart; i < batchEnd; i++) { opcodeBoundary(frame, pickler); saveFloat(pickler, proto, storage.getDoubleItemNormalized(i)); } - write(pickler, PickleUtils.OPCODE_APPENDS); + write(this, pickler, PickleUtils.OPCODE_APPENDS); } } @@ -2210,9 +2221,9 @@ private void saveList(VirtualFrame frame, PPickler pickler, int proto, boolean f // Create an empty list. if (isBin(proto)) { - write(pickler, PickleUtils.OPCODE_EMPTY_LIST); + write(this, pickler, PickleUtils.OPCODE_EMPTY_LIST); } else { - write(pickler, PickleUtils.OPCODE_MARK, PickleUtils.OPCODE_LIST); + write(this, pickler, PickleUtils.OPCODE_MARK, PickleUtils.OPCODE_LIST); } int len = obj.getSequenceStorage().length(); @@ -2298,9 +2309,9 @@ private void saveTuple(VirtualFrame frame, PPickler pickler, int proto, boolean if (len == 0) { if (proto != 0) { - write(pickler, PickleUtils.OPCODE_EMPTY_TUPLE); + write(this, pickler, PickleUtils.OPCODE_EMPTY_TUPLE); } else { - write(pickler, PickleUtils.OPCODE_MARK, PickleUtils.OPCODE_TUPLE); + write(this, pickler, PickleUtils.OPCODE_MARK, PickleUtils.OPCODE_TUPLE); } return; } @@ -2316,14 +2327,14 @@ private void saveTuple(VirtualFrame frame, PPickler pickler, int proto, boolean if (memoIndex != -1) { // pop the len elements for (int i = 0; i < len; i++) { - write(pickler, PickleUtils.OPCODE_POP); + write(this, pickler, PickleUtils.OPCODE_POP); } // fetch from memo memoGet(pickler, proto, memoIndex); return; } else { // Not recursive. - write(pickler, TUPLE_LEN_2_OPCODE[len]); + write(this, pickler, TUPLE_LEN_2_OPCODE[len]); } memoPut(pickler, proto, fastMode, obj); @@ -2331,18 +2342,18 @@ private void saveTuple(VirtualFrame frame, PPickler pickler, int proto, boolean } // proto < 2 and len > 0, or proto >= 2 and len > 3. Generate MARK e1 e2 ... TUPLE - write(pickler, PickleUtils.OPCODE_MARK); + write(this, pickler, PickleUtils.OPCODE_MARK); storeTupleElements(frame, pickler, obj, storage, len); int memoIndex = pickler.memo.get(obj); if (memoIndex != -1) { // pop the stack stuff we pushed if (isBin(proto)) { - write(pickler, PickleUtils.OPCODE_POP_MARK); + write(this, pickler, PickleUtils.OPCODE_POP_MARK); } else { // Note that we pop one more than len, to remove the MARK too. for (int i = 0; i <= len; i++) { - write(pickler, PickleUtils.OPCODE_POP); + write(this, pickler, PickleUtils.OPCODE_POP); } } // fetch from memo @@ -2350,7 +2361,7 @@ private void saveTuple(VirtualFrame frame, PPickler pickler, int proto, boolean return; } else { // Not recursive. - write(pickler, PickleUtils.OPCODE_TUPLE); + write(this, pickler, PickleUtils.OPCODE_TUPLE); } memoPut(pickler, proto, fastMode, obj); @@ -2364,8 +2375,8 @@ private void saveBytearrayData(VirtualFrame frame, PPickler pickler, int proto, boolean bypassBuffer = bypassBuffer(pickler, size); // Our sizes are ints, but the protocol expects 8 bytes - ensureBufferSpace(pickler, 9); - writeIntOp(pickler, PickleUtils.OPCODE_BYTEARRAY8, size); + ensureBufferSpace(this, pickler, 9); + writeIntOp(this, pickler, PickleUtils.OPCODE_BYTEARRAY8, size); pickler.outputLen += 4; if (bypassBuffer && pickler.write != null) { @@ -2373,7 +2384,7 @@ private void saveBytearrayData(VirtualFrame frame, PPickler pickler, int proto, ensureCallNode().execute(frame, pickler.write, obj); pickler.clearBuffer(); } else { - writeBuffer(pickler, buffer, size, bufferLib); + writeBuffer(this, pickler, buffer, size, bufferLib); } if (bypassBuffer && wasFraming) { @@ -2432,9 +2443,9 @@ private void savePickleBuffer(VirtualFrame frame, PPickler pickler, int proto, b } } else { // Write data out-of-band - write(pickler, PickleUtils.OPCODE_NEXT_BUFFER); + write(this, pickler, PickleUtils.OPCODE_NEXT_BUFFER); if (readOnly) { - write(pickler, PickleUtils.OPCODE_READONLY_BUFFER); + write(this, pickler, PickleUtils.OPCODE_READONLY_BUFFER); } } } @@ -2514,11 +2525,11 @@ private void saveGlobal(VirtualFrame frame, PythonContext ctx, PPickler pickler, // Generate an EXT opcode if (code <= 0xff) { - writeByteOp(pickler, PickleUtils.OPCODE_EXT1, code); + writeByteOp(this, pickler, PickleUtils.OPCODE_EXT1, code); } else if (code <= 0xffff) { - writeShortOp(pickler, PickleUtils.OPCODE_EXT2, code); + writeShortOp(this, pickler, PickleUtils.OPCODE_EXT2, code); } else { - writeIntOp(pickler, PickleUtils.OPCODE_EXT4, code); + writeIntOp(this, pickler, PickleUtils.OPCODE_EXT4, code); } } } else { @@ -2535,14 +2546,14 @@ private void saveGlobal(VirtualFrame frame, PythonContext ctx, PPickler pickler, if (proto >= 4) { save(frame, pickler, moduleName, 0); save(frame, pickler, globalName, 0); - write(pickler, PickleUtils.OPCODE_STACK_GLOBAL); + write(this, pickler, PickleUtils.OPCODE_STACK_GLOBAL); } else if (parent != module) { Object reduceValue = createTuple(st.getattr, createTuple(parent, lastname)); saveReduce(frame, ctx, pickler, proto, fastMode, reduceValue, null); } else { // Generate a normal global opcode if we are using a pickle protocol < 4, or if // the object is not registered in the extension registry - write(pickler, PickleUtils.OPCODE_GLOBAL); + write(this, pickler, PickleUtils.OPCODE_GLOBAL); // For protocol < 3 and if the user didn't request against doing so, we convert // module names to the old 2.x module names. if (proto < 3 && pickler.fixImports) { @@ -2560,19 +2571,19 @@ private void saveGlobal(VirtualFrame frame, PythonContext ctx, PPickler pickler, TS_ENCODING) == TruffleString.CodeRange.BROKEN) { throw raise(PicklingError, ErrorMessages.CANT_PICKLE_MODULE_S_USING_PROTO_D, moduleName, proto); } - writeUtf8(pickler, moduleName); - write(pickler, (byte) '\n'); - writeUtf8(pickler, globalName); + writeUtf8(this, pickler, moduleName, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); + write(this, pickler, (byte) '\n'); + writeUtf8(this, pickler, globalName, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); } else { if (ensureTsGetCodeRangeNode().execute(moduleName, TS_ENCODING) != TruffleString.CodeRange.ASCII || ensureTsGetCodeRangeNode().execute(globalName, TS_ENCODING) != TruffleString.CodeRange.ASCII) { throw raise(PicklingError, ErrorMessages.CANT_PICKLE_MODULE_S_USING_PROTO_D, moduleName, proto); } - writeASCII(pickler, moduleName); - write(pickler, (byte) '\n'); - writeASCII(pickler, globalName); + writeASCII(this, pickler, moduleName, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); + write(this, pickler, (byte) '\n'); + writeASCII(this, pickler, globalName, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); } - write(pickler, (byte) '\n'); + write(this, pickler, (byte) '\n'); } // Memoize the object memoPut(pickler, proto, fastMode, obj); @@ -2808,7 +2819,7 @@ public void dump(VirtualFrame frame, PPickler pickler, Object obj, assert pickler.proto <= 256; try { - saveNode.write(pickler, PickleUtils.OPCODE_PROTO, (byte) pickler.proto); + SaveNode.write(this, pickler, PickleUtils.OPCODE_PROTO, (byte) pickler.proto); } catch (Exception e) { SaveNode.handleError(pickler); throw e; @@ -2820,7 +2831,7 @@ public void dump(VirtualFrame frame, PPickler pickler, Object obj, } saveNode.execute(frame, pickler, obj, 0); - saveNode.write(pickler, PickleUtils.OPCODE_STOP); + SaveNode.write(this, pickler, PickleUtils.OPCODE_STOP); pickler.commitFrame(); } finally { pickler.framing = false; From a9f6108fd17d02cd2e5d90b46d027dda3e1dd074 Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Wed, 30 Sep 2026 11:57:04 +0200 Subject: [PATCH 11/39] Avoid graph size explosion --- .../builtins/modules/pickle/PPickler.java | 109 +++++++++++++----- 1 file changed, 82 insertions(+), 27 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java index dddf0773fb..2b7adb2f3d 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java @@ -1120,11 +1120,19 @@ protected static void fastSaveLeave(PPickler pickler, Object obj) { private static final int SEEN_REGISTERED_REDUCER = 1 << 29; private static final int SEEN_REDUCE_EX = 1 << 30; private static final int SEEN_REDUCE = 1 << 31; + private static final long SEEN_NEWOBJ_EX = 1L << 32; + private static final long SEEN_NEWOBJ = 1L << 33; + private static final long SEEN_PLAIN_REDUCE = 1L << 34; + private static final long SEEN_REDUCE_LISTITEMS = 1L << 35; + private static final long SEEN_REDUCE_DICTITEMS = 1L << 36; + private static final long SEEN_REDUCE_STATE = 1L << 37; - @CompilationFinal private int seenTypes; + @CompilationFinal private long seenTypes; private final int depth; - @Child private SaveNode recursiveSaveNode; + @Child private SaveNode recursiveSaveNode1; + @Child private SaveNode recursiveSaveNode2; + @Child private SaveNode recursiveSaveNode3; @Child private BoundaryCallData boundaryCallData; @Child private PyLongAsLongNode pyLongAsLongNode; @Child private PyObjectStrAsObjectNode pyObjectStrAsObjectNode; @@ -1142,7 +1150,7 @@ protected SaveNode(int depth) { this.depth = depth; } - private void profileSeen(int seen) { + private void profileSeen(long seen) { if ((seenTypes & seen) == 0) { CompilerDirectives.transferToInterpreterAndInvalidate(); seenTypes |= seen; @@ -1175,31 +1183,72 @@ private boolean isTuple(Object object) { private void save(VirtualFrame frame, PPickler pickler, Object obj, int persSave) { if (depth < 0) { - // This should never be reached for compilation, but native-image can't prove it + saveDepthExceeded(pickler, obj, persSave); + return; + } + if (recursiveSaveNode1 == null) { CompilerDirectives.transferToInterpreterAndInvalidate(); - execute(frame, pickler, obj, persSave); + recursiveSaveNode1 = insert(PPicklerFactory.SaveNodeGen.create(depth < MAX_RECURSION_DEPTH ? depth + 1 : -1)); + } + if (depth < MAX_RECURSION_DEPTH) { + recursiveSaveNode1.execute(frame, pickler, obj, persSave); + } else { + saveSetupBoundary(frame, recursiveSaveNode1, pickler, obj, persSave); + } + } + + private void save2(VirtualFrame frame, PPickler pickler, Object obj, int persSave) { + if (depth < 0) { + saveDepthExceeded(pickler, obj, persSave); return; } - if (recursiveSaveNode == null) { + if (recursiveSaveNode2 == null) { CompilerDirectives.transferToInterpreterAndInvalidate(); - recursiveSaveNode = insert(PPicklerFactory.SaveNodeGen.create(depth < MAX_RECURSION_DEPTH ? depth + 1 : -1)); + recursiveSaveNode2 = insert(PPicklerFactory.SaveNodeGen.create(depth < MAX_RECURSION_DEPTH ? depth + 1 : -1)); } if (depth < MAX_RECURSION_DEPTH) { - recursiveSaveNode.execute(frame, pickler, obj, persSave); + recursiveSaveNode2.execute(frame, pickler, obj, persSave); } else { - BoundaryCallData callData = getBoundaryCallData(); - Object savedState = BoundaryCallContext.enter(frame, callData); - try { - saveBoundary(pickler, obj, persSave); - } finally { - BoundaryCallContext.exit(frame, callData, savedState); - } + saveSetupBoundary(frame, recursiveSaveNode2, pickler, obj, persSave); + } + } + + private void save3(VirtualFrame frame, PPickler pickler, Object obj, int persSave) { + if (depth < 0) { + saveDepthExceeded(pickler, obj, persSave); + return; + } + if (recursiveSaveNode3 == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + recursiveSaveNode3 = insert(PPicklerFactory.SaveNodeGen.create(depth < MAX_RECURSION_DEPTH ? depth + 1 : -1)); + } + if (depth < MAX_RECURSION_DEPTH) { + recursiveSaveNode3.execute(frame, pickler, obj, persSave); + } else { + saveSetupBoundary(frame, recursiveSaveNode3, pickler, obj, persSave); } } + // This should never be reached for compilation, but native-image can't prove it at build time @TruffleBoundary - private void saveBoundary(PPickler pickler, Object obj, int persSave) { - recursiveSaveNode.execute(null, pickler, obj, persSave); + private void saveDepthExceeded(PPickler pickler, Object obj, int persSave) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + execute(null, pickler, obj, persSave); + } + + private void saveSetupBoundary(VirtualFrame frame, SaveNode saveNode, PPickler pickler, Object obj, int persSave) { + BoundaryCallData callData = getBoundaryCallData(); + Object savedState = BoundaryCallContext.enter(frame, callData); + try { + saveBoundary(saveNode, pickler, obj, persSave); + } finally { + BoundaryCallContext.exit(frame, callData, savedState); + } + } + + @TruffleBoundary + private void saveBoundary(SaveNode saveNode, PPickler pickler, Object obj, int persSave) { + saveNode.execute(null, pickler, obj, persSave); } private long asLong(VirtualFrame frame, Object object) { @@ -1298,7 +1347,7 @@ private void saveHashingStorageBatched(VirtualFrame frame, PPickler pickler, Has while (nextNode.executeCached(storage, it)) { save(frame, pickler, getKeyNode.executeCached(storage, it), 0); if (saveValues) { - save(frame, pickler, getValueNode.executeCached(storage, it), 0); + save2(frame, pickler, getValueNode.executeCached(storage, it), 0); } if (++i == PickleUtils.BATCHSIZE) { break; @@ -1388,7 +1437,7 @@ private void saveDictIteratorBatchUnrolled(VirtualFrame frame, PPickler pickler, }, (Object item) -> { save(frame, pickler, getItem(frame, item, 0), 0); - save(frame, pickler, getItem(frame, item, 1), 0); + save2(frame, pickler, getItem(frame, item, 1), 0); }); } @@ -1426,7 +1475,7 @@ private void saveDictIterator(VirtualFrame frame, PPickler pickler, Object itera throw raise(TypeError, ErrorMessages.MUST_S_ITER_RETURN_2TUPLE, DICT_ITEMS); } save(frame, pickler, getItem(frame, item, 0), 0); - save(frame, pickler, getItem(frame, item, 1), 0); + save2(frame, pickler, getItem(frame, item, 1), 0); }); } @@ -1582,6 +1631,7 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, final int argtupSize = length(frame, argtup); if (useNewobjEx) { + profileSeen(SEEN_NEWOBJ_EX); Object cls; Object args; Object kwargs; @@ -1607,8 +1657,8 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, if (proto >= 4) { save(frame, pickler, cls, 0); - save(frame, pickler, args, 0); - save(frame, pickler, kwargs, 0); + save2(frame, pickler, args, 0); + save3(frame, pickler, kwargs, 0); write(this, pickler, PickleUtils.OPCODE_NEWOBJ_EX); } else { PickleState st = getGlobalState(ctx.getCore()); @@ -1627,10 +1677,11 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, callable = callStarArgsAndKwArgs(frame, st.partial, newargs, kwargs); save(frame, pickler, callable, 0); - save(frame, pickler, PFactory.createEmptyTuple(PythonLanguage.get(this)), 0); + save2(frame, pickler, PFactory.createEmptyTuple(PythonLanguage.get(this)), 0); write(this, pickler, PickleUtils.OPCODE_REDUCE); } } else if (useNewobj) { + profileSeen(SEEN_NEWOBJ); Object cls; Object newargtup; Object objClass; @@ -1684,12 +1735,13 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, // Save the class and its __new__ arguments save(frame, pickler, cls, 0); newargtup = getItem(frame, argtup, PFactory.createIntSlice(PythonLanguage.get(this), 1, argtupSize, 1)); - save(frame, pickler, newargtup, 0); + save2(frame, pickler, newargtup, 0); write(this, pickler, PickleUtils.OPCODE_NEWOBJ); } else { + profileSeen(SEEN_PLAIN_REDUCE); // Not using NEWOBJ save(frame, pickler, callable, 0); - save(frame, pickler, argtup, 0); + save2(frame, pickler, argtup, 0); write(this, pickler, PickleUtils.OPCODE_REDUCE); } @@ -1710,14 +1762,17 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, } if (listitems != null) { + profileSeen(SEEN_REDUCE_LISTITEMS); batchList(frame, pickler, proto, listitems); } if (dictitems != null) { + profileSeen(SEEN_REDUCE_DICTITEMS); batchDict(frame, pickler, proto, dictitems); } if (state != null) { + profileSeen(SEEN_REDUCE_STATE); if (stateSetter == null) { save(frame, pickler, state, 0); write(this, pickler, PickleUtils.OPCODE_BUILD); @@ -1730,8 +1785,8 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, // whole operation has to be stack-transparent. Thus, we finally pop the call's // output from the stack. save(frame, pickler, stateSetter, 0); - save(frame, pickler, obj, 0); - save(frame, pickler, state, 0); + save2(frame, pickler, obj, 0); + save3(frame, pickler, state, 0); write(this, pickler, PickleUtils.OPCODE_TUPLE2); write(this, pickler, PickleUtils.OPCODE_REDUCE); write(this, pickler, PickleUtils.OPCODE_POP); From ad3897172fb43f45cac42da551e287be21bb3106 Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Wed, 30 Sep 2026 14:43:31 +0200 Subject: [PATCH 12/39] Get rid of MemoIterator --- .../builtins/modules/pickle/MemoTable.java | 62 +++++++------------ .../pickle/PicklerMemoProxyBuiltins.java | 14 ++--- 2 files changed, 30 insertions(+), 46 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/MemoTable.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/MemoTable.java index 4393f54f42..37e813fd87 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/MemoTable.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/MemoTable.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -54,39 +54,6 @@ public final class MemoTable { private static final int OCCUPANCY_EXPONENT = 1; // 2^X relation between capacity and size private static final int CAPACITY_INC_EXPONENT = 2; // 2^X increase in capacity when resizing - public static final class MemoIterator { - - private int index; - private final Object[] keys; - private final int[] values; - - public MemoIterator(MemoTable table) { - this.keys = table.keys; - this.values = table.values; - this.index = -1; - } - - public boolean advance() { - while (true) { - index++; - if (index >= keys.length) { - return false; - } - if (keys[index] != null) { - return true; - } - } - } - - public Object key() { - return keys[index]; - } - - public int value() { - return values[index]; - } - } - private Object[] keys; private int[] values; private int mask; @@ -176,11 +143,20 @@ private void resize() { throw PRaiseNode.raiseStatic(null, PicklingError, ErrorMessages.STRUCT_SIZE_TOO_LONG); } - MemoIterator iterator = iterator(); // captures the current contents + Object[] oldKeys = keys; + int[] oldValues = values; initArrays(newLength); - while (iterator.advance()) { - setInternal(iterator.key(), iterator.value()); + for (int i = 0; i < oldKeys.length; i++) { + Object key = oldKeys[i]; + if (key != null) { + int index = getIndex(key); + while (keys[index] != null) { + index = (index + 1) & mask; + } + keys[index] = key; + values[index] = oldValues[i]; + } } } @@ -192,7 +168,15 @@ public void set(Object key, int value) { } } - public MemoIterator iterator() { - return new MemoIterator(this); + int capacity() { + return keys.length; + } + + Object getKeyAt(int index) { + return keys[index]; + } + + int getValueAt(int index) { + return values[index]; } } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerMemoProxyBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerMemoProxyBuiltins.java index 9b5ad2622f..5d8ea60e57 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerMemoProxyBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerMemoProxyBuiltins.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -53,7 +53,6 @@ import com.oracle.graal.python.builtins.CoreFunctions; import com.oracle.graal.python.builtins.PythonBuiltinClassType; import com.oracle.graal.python.builtins.PythonBuiltins; -import com.oracle.graal.python.builtins.modules.pickle.MemoTable.MemoIterator; import com.oracle.graal.python.builtins.objects.PNone; import com.oracle.graal.python.builtins.objects.dict.PDict; import com.oracle.graal.python.builtins.objects.tuple.PTuple; @@ -106,11 +105,12 @@ Object clear(PPicklerMemoProxy self) { public static PDict picklerMemoCopyImpl(MemoTable memoTable) { PythonLanguage language = PythonLanguage.get(null); LinkedHashMap copy = new LinkedHashMap<>(); - MemoIterator iterator = memoTable.iterator(); - while (iterator.advance()) { - copy.put(System.identityHashCode(iterator.key()), - PFactory.createTuple(language, new Object[]{iterator.value(), iterator.key()})); - + for (int i = 0; i < memoTable.capacity(); i++) { + Object key = memoTable.getKeyAt(i); + if (key != null) { + copy.put(System.identityHashCode(key), + PFactory.createTuple(language, new Object[]{memoTable.getValueAt(i), key})); + } } return PFactory.createDictFromMapGeneric(language, copy); } From d867056e40ca3896ecf4fc1befcff2ad840cf3ff Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Wed, 30 Sep 2026 14:49:50 +0200 Subject: [PATCH 13/39] Profile buffer bypass --- .../graal/python/builtins/modules/pickle/PPickler.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java index 2b7adb2f3d..0e6062be4e 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java @@ -912,7 +912,7 @@ public void opcodeBoundary(VirtualFrame frame, PPickler pickler) { return; } int frameLen = pickler.outputLen - pickler.frameStart - PickleUtils.FRAME_HEADER_SIZE; - if (frameLen >= PickleUtils.FRAME_SIZE_TARGET) { + if (CompilerDirectives.injectBranchProbability(CompilerDirectives.SLOWPATH_PROBABILITY, frameLen >= PickleUtils.FRAME_SIZE_TARGET)) { pickler.commitFrame(); // Flush the content of the committed frame to the underlying // file and reuse the pickler buffer for the next frame so as @@ -1051,10 +1051,11 @@ public static void ensureBufferSpace(Node node, PPickler pickler, int dataLen) { } } - protected static boolean bypassBuffer(PPickler pickler, int dataSize) { + protected boolean bypassBuffer(PPickler pickler, int dataSize) { boolean bypassBuffer = dataSize >= PickleUtils.FRAME_SIZE_TARGET; if (bypassBuffer) { + profileSeen(SEEN_BYPASS_BUFFER); assert pickler.outputBuffer != null; // Commit the previous frame. pickler.commitFrame(); @@ -1126,6 +1127,7 @@ protected static void fastSaveLeave(PPickler pickler, Object obj) { private static final long SEEN_REDUCE_LISTITEMS = 1L << 35; private static final long SEEN_REDUCE_DICTITEMS = 1L << 36; private static final long SEEN_REDUCE_STATE = 1L << 37; + private static final long SEEN_BYPASS_BUFFER = 1L << 38; @CompilationFinal private long seenTypes; @@ -1957,7 +1959,7 @@ private void saveBytesData(VirtualFrame frame, PPickler pickler, int proto, bool memoPut(pickler, proto, fastMode, obj); } - private static void writeUnicodeBinary(Node node, VirtualFrame frame, PPickler pickler, int proto, TruffleString string, + private void writeUnicodeBinary(Node node, VirtualFrame frame, PPickler pickler, int proto, TruffleString string, TruffleString.SwitchEncodingNode switchEncodingNode, TruffleString.CopyToByteArrayNode copyToByteArrayNode, FlushToFileNode flushToFileNode, CallNode callNode) { TruffleString utf8 = switchEncodingNode.execute(string, TruffleString.Encoding.UTF_8, TranscodingErrorHandler.DEFAULT_KEEP_SURROGATES_IN_UTF8); From c31f65db5409cb65be8265f2f4b32b16b0aa61af Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Wed, 30 Sep 2026 15:33:36 +0200 Subject: [PATCH 14/39] Keep size estimates for buffer and memo table --- .../builtins/modules/pickle/MemoTable.java | 14 +++- .../builtins/modules/pickle/PPickler.java | 12 ++- .../modules/pickle/PickleModuleBuiltins.java | 74 +++++++++++++++++-- .../graal/python/runtime/object/PFactory.java | 4 +- 4 files changed, 89 insertions(+), 15 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/MemoTable.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/MemoTable.java index 37e813fd87..182d4cdd95 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/MemoTable.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/MemoTable.java @@ -42,6 +42,8 @@ import static com.oracle.graal.python.runtime.exception.PythonErrorType.PicklingError; +import java.util.Arrays; + import com.oracle.graal.python.nodes.ErrorMessages; import com.oracle.graal.python.nodes.PRaiseNode; import com.oracle.graal.python.util.PythonUtils; @@ -50,7 +52,7 @@ public final class MemoTable { - private static final int INITIAL_CAPACITY = 8; + static final int INITIAL_CAPACITY = 8; private static final int OCCUPANCY_EXPONENT = 1; // 2^X relation between capacity and size private static final int CAPACITY_INC_EXPONENT = 2; // 2^X increase in capacity when resizing @@ -60,7 +62,12 @@ public final class MemoTable { private int size; public MemoTable() { - initArrays(INITIAL_CAPACITY); + this(INITIAL_CAPACITY); + } + + public MemoTable(int initialCapacity) { + assert initialCapacity >= INITIAL_CAPACITY && Integer.bitCount(initialCapacity) == 1; + initArrays(initialCapacity); } private MemoTable(MemoTable map) { @@ -79,7 +86,8 @@ public int size() { } public void clear() { - initArrays(INITIAL_CAPACITY); + Arrays.fill(keys, null); + Arrays.fill(values, 0); this.size = 0; } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java index 0e6062be4e..f8ea57e6d9 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java @@ -212,6 +212,10 @@ public class PPickler extends PythonBuiltinObject { private Object bufferCallback; public PPickler(Object cls, Shape instanceShape) { + this(cls, instanceShape, PickleUtils.WRITE_BUF_SIZE, MemoTable.INITIAL_CAPACITY); + } + + public PPickler(Object cls, Shape instanceShape, int outputCapacity, int memoCapacity) { super(cls, instanceShape); persFunc = null; dispatchTable = null; @@ -224,11 +228,11 @@ public PPickler(Object cls, Shape instanceShape) { fast = 0; fastNesting = 0; fixImports = false; - maxOutputLen = PickleUtils.WRITE_BUF_SIZE; + maxOutputLen = outputCapacity; outputLen = 0; reducerOverride = null; - memo = new MemoTable(); + memo = new MemoTable(memoCapacity); outputBuffer = new byte[maxOutputLen]; } @@ -285,6 +289,10 @@ public MemoTable getMemo() { return memo; } + public int getMaxOutputLen() { + return maxOutputLen; + } + public void setMemo(MemoTable memo) { this.memo = memo; } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PickleModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PickleModuleBuiltins.java index 9b8518028a..6cc08bb72c 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PickleModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PickleModuleBuiltins.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -63,6 +63,8 @@ import com.oracle.graal.python.nodes.function.builtins.clinic.ArgumentClinicProvider; import com.oracle.graal.python.runtime.IndirectCallData.InteropCallData; import com.oracle.graal.python.runtime.object.PFactory; +import com.oracle.truffle.api.CompilerDirectives; +import com.oracle.truffle.api.CompilerDirectives.CompilationFinal; import com.oracle.truffle.api.dsl.Bind; import com.oracle.truffle.api.dsl.Cached; import com.oracle.truffle.api.dsl.GenerateNodeFactory; @@ -80,6 +82,59 @@ protected List> getNodeFa return PickleModuleBuiltinsFactory.getFactories(); } + /** Learns the capacities reached by successful calls at one builtin node. */ + private static final class PicklerSizeProfile { + private static final int SHIFT = 3; + + @CompilationFinal private long outputEstimate = (long) PickleUtils.WRITE_BUF_SIZE << SHIFT; + @CompilationFinal private long memoEstimate = (long) MemoTable.INITIAL_CAPACITY << SHIFT; + + int outputCapacity() { + return (int) (outputEstimate >> SHIFT); + } + + int memoCapacity() { + int estimate = (int) (memoEstimate >> SHIFT); + return estimate <= MemoTable.INITIAL_CAPACITY ? MemoTable.INITIAL_CAPACITY : Integer.highestOneBit(estimate - 1) << 1; + } + + void report(PPickler pickler) { + if (CompilerDirectives.inInterpreter()) { + if (pickler.getMaxOutputLen() > outputCapacity()) { + outputEstimate = update(outputEstimate, pickler.getMaxOutputLen()); + } + int memoCapacity = pickler.getMemo().capacity(); + if (memoCapacity > (memoEstimate >> SHIFT)) { + memoEstimate = update(memoEstimate, memoCapacity); + } + } + } + + private static long update(long shiftedEstimate, int capacity) { + long estimate = shiftedEstimate >> SHIFT; + return shiftedEstimate + capacity - estimate; + } + } + + abstract static class ProfiledPickleDumpNode extends PythonClinicBuiltinNode { + @CompilationFinal private PicklerSizeProfile sizes = new PicklerSizeProfile(); + + final PPickler createPickler(PythonLanguage language) { + return PFactory.createPickler(language, sizes.outputCapacity(), sizes.memoCapacity()); + } + + final void reportSizes(PPickler pickler) { + sizes.report(pickler); + } + + @Override + public Node copy() { + ProfiledPickleDumpNode copy = (ProfiledPickleDumpNode) super.copy(); + copy.sizes = new PicklerSizeProfile(); + return copy; + } + } + @Override public void postInitialize(Python3Core core) { super.postInitialize(core); @@ -95,26 +150,27 @@ public void postInitialize(Python3Core core) { @ArgumentClinic(name = "protocol", conversion = ArgumentClinic.ClinicConversion.Int, defaultValue = J_DEFAULT_PICKLE_PROTOCOL, useDefaultForNone = true) @ArgumentClinic(name = "fix_imports", conversion = ArgumentClinic.ClinicConversion.Boolean, defaultValue = "true") @GenerateNodeFactory - abstract static class PickleDumpNode extends PythonClinicBuiltinNode { + abstract static class PickleDumpNode extends ProfiledPickleDumpNode { @Override protected ArgumentClinicProvider getArgumentClinic() { return PickleModuleBuiltinsClinicProviders.PickleDumpNodeClinicProviderGen.INSTANCE; } @Specialization - static Object dump(VirtualFrame frame, @SuppressWarnings("unused") PythonModule self, Object obj, Object file, int protocol, boolean fixImports, Object bufferCallback, + Object dump(VirtualFrame frame, @SuppressWarnings("unused") PythonModule self, Object obj, Object file, int protocol, boolean fixImports, Object bufferCallback, @Bind Node inliningTarget, @Bind PythonLanguage language, @Cached PPickler.DumpNode dumpNode, @Cached PPickler.FlushToFileNode flushToFileNode, @Cached PyObjectLookupAttr lookup, @Cached PRaiseNode raiseNode) { - PPickler pickler = PFactory.createPickler(language); + PPickler pickler = createPickler(language); pickler.setProtocol(inliningTarget, raiseNode, protocol, fixImports); pickler.setOutputStream(frame, inliningTarget, raiseNode, lookup, file); pickler.setBufferCallback(inliningTarget, raiseNode, bufferCallback); dumpNode.execute(frame, pickler, obj); flushToFileNode.execute(frame, pickler); + reportSizes(pickler); return PNone.NONE; } } @@ -125,23 +181,25 @@ static Object dump(VirtualFrame frame, @SuppressWarnings("unused") PythonModule @ArgumentClinic(name = "protocol", conversion = ArgumentClinic.ClinicConversion.Int, defaultValue = J_DEFAULT_PICKLE_PROTOCOL, useDefaultForNone = true) @ArgumentClinic(name = "fix_imports", conversion = ArgumentClinic.ClinicConversion.Boolean, defaultValue = "true") @GenerateNodeFactory - abstract static class PickleDumpsNode extends PythonClinicBuiltinNode { + abstract static class PickleDumpsNode extends ProfiledPickleDumpNode { @Override protected ArgumentClinicProvider getArgumentClinic() { return PickleModuleBuiltinsClinicProviders.PickleDumpsNodeClinicProviderGen.INSTANCE; } @Specialization - static Object dump(VirtualFrame frame, @SuppressWarnings("unused") PythonModule self, Object obj, int protocol, boolean fixImports, Object bufferCallback, + Object dump(VirtualFrame frame, @SuppressWarnings("unused") PythonModule self, Object obj, int protocol, boolean fixImports, Object bufferCallback, @Bind Node inliningTarget, @Bind PythonLanguage language, @Cached PPickler.DumpNode dumpNode, @Cached PRaiseNode raiseNode) { - PPickler pickler = PFactory.createPickler(language); + PPickler pickler = createPickler(language); pickler.setProtocol(inliningTarget, raiseNode, protocol, fixImports); pickler.setBufferCallback(inliningTarget, raiseNode, bufferCallback); dumpNode.execute(frame, pickler, obj); - return pickler.getString(language); + Object result = pickler.getString(language); + reportSizes(pickler); + return result; } } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/object/PFactory.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/object/PFactory.java index 35e58a8b29..989b5a5c52 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/object/PFactory.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/object/PFactory.java @@ -1579,8 +1579,8 @@ public static PPickleBuffer createPickleBuffer(Object view, Object cls, Shape sh return new PPickleBuffer(cls, shape, view); } - public static PPickler createPickler(PythonLanguage language) { - return createPickler(PythonBuiltinClassType.Pickler, PythonBuiltinClassType.Pickler.getInstanceShape(language)); + public static PPickler createPickler(PythonLanguage language, int outputCapacity, int memoCapacity) { + return new PPickler(PythonBuiltinClassType.Pickler, PythonBuiltinClassType.Pickler.getInstanceShape(language), outputCapacity, memoCapacity); } public static PPickler createPickler(Object cls, Shape shape) { From bc0e552e5d6c07052e44a09a3667940cee661c69 Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Wed, 30 Sep 2026 16:40:23 +0200 Subject: [PATCH 15/39] Use more specific nodes when working with collections --- .../builtins/modules/pickle/PPickler.java | 157 +++++++++--------- 1 file changed, 83 insertions(+), 74 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java index f8ea57e6d9..5fc03d2831 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java @@ -133,8 +133,8 @@ import com.oracle.graal.python.nodes.argument.keywords.ExpandKeywordStarargsNodeGen; import com.oracle.graal.python.nodes.argument.positional.ExecutePositionalStarargsNode; import com.oracle.graal.python.nodes.builtins.ListNodes; +import com.oracle.graal.python.nodes.builtins.TupleNodes.GetTupleStorage; import com.oracle.graal.python.nodes.call.CallNode; -import com.oracle.graal.python.nodes.classes.IsSubtypeNode; import com.oracle.graal.python.nodes.object.BuiltinClassProfiles.InlineIsBuiltinClassProfile; import com.oracle.graal.python.nodes.object.BuiltinClassProfiles.IsBuiltinObjectProfile; import com.oracle.graal.python.nodes.object.GetClassNode; @@ -426,6 +426,18 @@ public PBytes getString(PythonLanguage language) { } // inner nodes + @GenerateInline(false) + public abstract static class TupleStorageNode extends Node { + public abstract SequenceStorage execute(Object tuple); + + @Specialization + static SequenceStorage get(Object tuple, + @Bind Node inliningTarget, + @Cached GetTupleStorage getTupleStorage) { + return getTupleStorage.execute(inliningTarget, tuple); + } + } + public abstract static class SaveNode extends Node { private static final int MAX_RECURSION_DEPTH = 3; private static final TruffleString T_LOCALS = tsLiteral(""); @@ -443,7 +455,6 @@ public abstract static class SaveNode extends Node { @Child private IntNodes.PyLongNumBits pyLongNumBits; @Child private IntNodes.PyLongAsByteArray pyLongAsByteArray; @Child private ListNodes.ConstructListNode constructListNode; - @Child private IsSubtypeNode isSubTypeNode; @Child private TypeNodes.IsTypeNode isTypeNode; @Child private FlushToFileNode flushToFileNode; @Child private CallNode callNode; @@ -455,6 +466,7 @@ public abstract static class SaveNode extends Node { @Child private InlineIsBuiltinClassProfile isBuiltinClassProfile; @Child private CachedHashingStorageGetItem getHashingStorageItemNode; @Child private SequenceStorageNodes.GetItemScalarNode getSeqStorageItemNode; + @Child private TupleStorageNode getTupleStorageNode; @Child private PyObjectSizeNode sizeNode; @Child private PyIterCheckNode isIteratorObjectNode; @Child private GetClassNode getClassNode; @@ -729,12 +741,16 @@ public Object getItem(SequenceStorage storage, int index) { return getSeqStorageItemNode.executeCached(storage, index); } - public Object getItem(VirtualFrame frame, Object obj, Object key) { - return getItemNode.executeCached(frame, obj, key); + protected SequenceStorage getTupleStorage(Object tuple) { + if (getTupleStorageNode == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + getTupleStorageNode = insert(PPicklerFactory.TupleStorageNodeGen.create()); + } + return getTupleStorageNode.execute(tuple); } - public Object getItem(VirtualFrame frame, Object obj, int size, int pos, Object defaultValue) { - return size > pos ? getItemNode.executeCached(frame, obj, pos) : defaultValue; + public Object getItem(VirtualFrame frame, Object obj, Object key) { + return getItemNode.executeCached(frame, obj, key); } public TruffleString asString(Object value) { @@ -797,11 +813,12 @@ private Pair getMapping(VirtualFrame frame, PDict Object key = PFactory.createTuple(PythonLanguage.get(this), new Object[]{moduleName, globalName}); Object item = getDictItem(frame, nameMapping, key); if (item != null) { - if (!(item instanceof PTuple) || length(frame, item) != 2) { + if (!(item instanceof PTuple) || ((PTuple) item).getSequenceStorage().length() != 2) { throw raise(PythonBuiltinClassType.RuntimeError, ErrorMessages.S_SHOULD_BE_S_NOT_P, nameMappingLabel, "2-tuples", item); } - Object mappedModuleName = getItem(frame, item, 0); - Object mappedGlobalName = getItem(frame, item, 1); + SequenceStorage storage = ((PTuple) item).getSequenceStorage(); + Object mappedModuleName = getItem(storage, 0); + Object mappedGlobalName = getItem(storage, 1); if (!PGuards.isString(mappedModuleName) || !PGuards.isString(mappedGlobalName)) { throw raise(PythonBuiltinClassType.RuntimeError, ErrorMessages.S_SHOULD_BE_S_NOT_P_P, nameMappingLabel, "str", mappedModuleName, mappedGlobalName); } @@ -907,14 +924,6 @@ protected Object createList(VirtualFrame frame, Object iterable) { return constructListNode.execute(frame, iterable); } - protected boolean isSubType(Object clsA, Object clsB) { - if (isSubTypeNode == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - isSubTypeNode = insert(IsSubtypeNode.create()); - } - return isSubTypeNode.execute(clsA, clsB); - } - public void opcodeBoundary(VirtualFrame frame, PPickler pickler) { if (!pickler.isFraming() || pickler.frameStart == -1) { return; @@ -1441,13 +1450,14 @@ private void saveIteratorBatchedUnrolled(VirtualFrame frame, PPickler pickler, O private void saveDictIteratorBatchUnrolled(VirtualFrame frame, PPickler pickler, Object iterator) { saveIteratorBatchedUnrolled(frame, pickler, iterator, PickleUtils.OPCODE_SETITEM, PickleUtils.OPCODE_SETITEMS, (Object item) -> { - if (!isTuple(item) || length(frame, item) != 2) { + if (!isTuple(item) || getTupleStorage(item).length() != 2) { throw raise(TypeError, ErrorMessages.MUST_S_ITER_RETURN_2TUPLE, DICT_ITEMS); } }, (Object item) -> { - save(frame, pickler, getItem(frame, item, 0), 0); - save2(frame, pickler, getItem(frame, item, 1), 0); + SequenceStorage storage = getTupleStorage(item); + save(frame, pickler, getItem(storage, 0), 0); + save2(frame, pickler, getItem(storage, 1), 0); }); } @@ -1481,11 +1491,12 @@ private void saveListIterator(VirtualFrame frame, PPickler pickler, Object itera private void saveDictIterator(VirtualFrame frame, PPickler pickler, Object iterator) { saveIterator(frame, pickler, iterator, PickleUtils.OPCODE_SETITEM, (Object item) -> { - if (!isTuple(item) || length(frame, item) != 2) { + if (!isTuple(item) || getTupleStorage(item).length() != 2) { throw raise(TypeError, ErrorMessages.MUST_S_ITER_RETURN_2TUPLE, DICT_ITEMS); } - save(frame, pickler, getItem(frame, item, 0), 0); - save2(frame, pickler, getItem(frame, item, 1), 0); + SequenceStorage storage = getTupleStorage(item); + save(frame, pickler, getItem(storage, 0), 0); + save2(frame, pickler, getItem(storage, 1), 0); }); } @@ -1583,17 +1594,18 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, if (!isTuple(arguments)) { throw raise(PythonBuiltinClassType.SystemError, ErrorMessages.BAD_ARG_TO_INTERNAL_FUNC); } - int size = length(frame, arguments); + SequenceStorage argumentsStorage = getTupleStorage(arguments); + int size = argumentsStorage.length(); if (size < 2 || size > 6) { throw raise(PicklingError, ErrorMessages.TUPLE_RET_BY_REDUCE_2_6); } - Object callable = getItem(frame, arguments, 0); - Object argtup = getItem(frame, arguments, 1); - Object state = getItem(frame, arguments, size, 2, null); - Object listitems = getItem(frame, arguments, size, 3, PNone.NONE); - Object dictitems = getItem(frame, arguments, size, 4, PNone.NONE); - Object stateSetter = getItem(frame, arguments, size, 5, PNone.NONE); + Object callable = getItem(argumentsStorage, 0); + Object argtup = getItem(argumentsStorage, 1); + Object state = size > 2 ? getItem(argumentsStorage, 2) : null; + Object listitems = size > 3 ? getItem(argumentsStorage, 3) : PNone.NONE; + Object dictitems = size > 4 ? getItem(argumentsStorage, 4) : PNone.NONE; + Object stateSetter = size > 5 ? getItem(argumentsStorage, 5) : PNone.NONE; if (!isCallable(callable)) { throw raise(PicklingError, ErrorMessages.S_ITEM_REDUCE_MUST_BE_S, "first", "callable"); @@ -1638,7 +1650,8 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, } } - final int argtupSize = length(frame, argtup); + final SequenceStorage argtupStorage = getTupleStorage(argtup); + final int argtupSize = argtupStorage.length(); if (useNewobjEx) { profileSeen(SEEN_NEWOBJ_EX); @@ -1650,17 +1663,17 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, throw raise(PicklingError, ErrorMessages.LEN_OF_S_MUST_BE_D_NOT_D, "NEWOBJ_EX", 3, argtupSize); } - cls = getItem(frame, argtup, 0); + cls = getItem(argtupStorage, 0); if (!isType(cls)) { throw raise(PicklingError, ErrorMessages.S_ITEM_FROM_S_MUST_BE_S_NOT_P, "first", "NEWOBJ_EX", "a class", cls); } - args = getItem(frame, argtup, 1); + args = getItem(argtupStorage, 1); if (!isTuple(args)) { throw raise(PicklingError, ErrorMessages.S_ITEM_FROM_S_MUST_BE_S_NOT_P, "second", "NEWOBJ_EX", "a tuple", args); } - kwargs = getItem(frame, argtup, 2); + kwargs = getItem(argtupStorage, 2); if (!(kwargs instanceof PDict)) { throw raise(PicklingError, ErrorMessages.S_ITEM_FROM_S_MUST_BE_S_NOT_P, "third", "NEWOBJ_EX", "a dict", kwargs); } @@ -1672,7 +1685,8 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, write(this, pickler, PickleUtils.OPCODE_NEWOBJ_EX); } else { PickleState st = getGlobalState(ctx.getCore()); - final int argsSize = length(frame, args); + final SequenceStorage argsStorage = getTupleStorage(args); + final int argsSize = argsStorage.length(); Object[] newargs = new Object[argsSize + 2]; Object clsNew; int i; @@ -1681,7 +1695,7 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, newargs[0] = clsNew; newargs[1] = cls; for (i = 0; i < argsSize; i++) { - Object item = getItem(frame, args, i); + Object item = getItem(argsStorage, i); newargs[i + 2] = item; } callable = callStarArgsAndKwArgs(frame, st.partial, newargs, kwargs); @@ -1702,7 +1716,7 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, throw raise(PicklingError, ErrorMessages.IS_EMPTY, "__newobj__ arglist"); } - cls = getItem(frame, argtup, 0); + cls = getItem(argtupStorage, 0); if (!isType(cls)) { throw raise(PicklingError, ErrorMessages.ARGS_0_FROM_S_ARGS_S, "__newobj__", "is not a type"); } @@ -1744,7 +1758,11 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, // Save the class and its __new__ arguments save(frame, pickler, cls, 0); - newargtup = getItem(frame, argtup, PFactory.createIntSlice(PythonLanguage.get(this), 1, argtupSize, 1)); + Object[] newargs = new Object[argtupSize - 1]; + for (int i = 1; i < argtupSize; i++) { + newargs[i - 1] = getItem(argtupStorage, i); + } + newargtup = createTuple(newargs); save2(frame, pickler, newargtup, 0); write(this, pickler, PickleUtils.OPCODE_NEWOBJ); } else { @@ -2106,10 +2124,6 @@ private void saveDict(VirtualFrame frame, Node inliningTarget, PyObjectCallMetho } } - private void batchSetExact(VirtualFrame frame, PPickler pickler, PSet set) { - saveSetHashingStorageBatched(frame, pickler, set.getDictStorage()); - } - private void saveSet(VirtualFrame frame, PythonContext ctx, PPickler pickler, int proto, boolean fastMode, PSet obj) { Object reduceValue; if (proto < 4) { @@ -2122,14 +2136,14 @@ private void saveSet(VirtualFrame frame, PythonContext ctx, PPickler pickler, in write(this, pickler, PickleUtils.OPCODE_EMPTY_SET); memoPut(pickler, proto, fastMode, obj); - final int setSize = length(frame, obj); - if (setSize == 0) { + HashingStorage storage = obj.getDictStorage(); + if (getHashingStorageLength(storage) == 0) { // nothing to do return; } // Write in batches of BATCHSIZE. - batchSetExact(frame, pickler, obj); + saveSetHashingStorageBatched(frame, pickler, storage); } private void saveFrozenset(VirtualFrame frame, PythonContext ctx, PPickler pickler, int proto, boolean fastMode, PFrozenSet obj) { @@ -2307,34 +2321,28 @@ private void saveList(VirtualFrame frame, PPickler pickler, int proto, boolean f } } - private void storeTupleElements(VirtualFrame frame, PPickler pickler, Object obj, SequenceStorage storage, int len) { + private void storeTupleElements(VirtualFrame frame, PPickler pickler, SequenceStorage storage, int len) { // A helper for save_tuple. Push the len elements in tuple t on the stack - if (storage == null) { - for (int i = 0; i < len; i++) { - save(frame, pickler, getItem(frame, obj, i), 0); - } - } else { - assert storage.length() == len; - if (pickler.persFunc == null) { - if (storage instanceof IntSequenceStorage intStorage) { - profileSeen(SEEN_INT_STORAGE); - storeIntTupleElements(frame, pickler, intStorage); - return; - } else if (storage instanceof LongSequenceStorage longStorage) { - profileSeen(SEEN_LONG_STORAGE); - storeLongTupleElements(frame, pickler, longStorage); - return; - } else if (storage instanceof DoubleSequenceStorage doubleStorage) { - profileSeen(SEEN_DOUBLE_STORAGE); - storeDoubleTupleElements(frame, pickler, doubleStorage); - return; - } - } - profileSeen(SEEN_GENERIC_STORAGE); - for (int i = 0; i < len; i++) { - save(frame, pickler, getItem(storage, i), 0); + assert storage.length() == len; + if (pickler.persFunc == null) { + if (storage instanceof IntSequenceStorage intStorage) { + profileSeen(SEEN_INT_STORAGE); + storeIntTupleElements(frame, pickler, intStorage); + return; + } else if (storage instanceof LongSequenceStorage longStorage) { + profileSeen(SEEN_LONG_STORAGE); + storeLongTupleElements(frame, pickler, longStorage); + return; + } else if (storage instanceof DoubleSequenceStorage doubleStorage) { + profileSeen(SEEN_DOUBLE_STORAGE); + storeDoubleTupleElements(frame, pickler, doubleStorage); + return; } } + profileSeen(SEEN_GENERIC_STORAGE); + for (int i = 0; i < len; i++) { + save(frame, pickler, getItem(storage, i), 0); + } } private void storeIntTupleElements(VirtualFrame frame, PPickler pickler, IntSequenceStorage storage) { @@ -2367,7 +2375,8 @@ private void saveTuple(VirtualFrame frame, PPickler pickler, int proto, boolean } private void saveNativeTuple(VirtualFrame frame, PPickler pickler, int proto, boolean fastMode, Object obj) { - saveTuple(frame, pickler, proto, fastMode, obj, null, length(frame, obj)); + SequenceStorage storage = getTupleStorage(obj); + saveTuple(frame, pickler, proto, fastMode, obj, storage, storage.length()); } private void saveTuple(VirtualFrame frame, PPickler pickler, int proto, boolean fastMode, Object obj, SequenceStorage storage, int len) { @@ -2386,7 +2395,7 @@ private void saveTuple(VirtualFrame frame, PPickler pickler, int proto, boolean // the stack, and fetch its value from the memo. if (len <= 3 && proto >= 2) { // Use TUPLE{1,2,3} opcodes. - storeTupleElements(frame, pickler, obj, storage, len); + storeTupleElements(frame, pickler, storage, len); int memoIndex = pickler.memo.get(obj); if (memoIndex != -1) { @@ -2408,7 +2417,7 @@ private void saveTuple(VirtualFrame frame, PPickler pickler, int proto, boolean // proto < 2 and len > 0, or proto >= 2 and len > 3. Generate MARK e1 e2 ... TUPLE write(this, pickler, PickleUtils.OPCODE_MARK); - storeTupleElements(frame, pickler, obj, storage, len); + storeTupleElements(frame, pickler, storage, len); int memoIndex = pickler.memo.get(obj); if (memoIndex != -1) { @@ -2815,7 +2824,7 @@ void saveGeneric(VirtualFrame frame, PPickler pickler, Object objArg, int persSa if (reduceFunc != null) { profileSeen(SEEN_REGISTERED_REDUCER); reduceValue = callNode.execute(frame, reduceFunc, obj); - } else if (isSubType(type, PythonBuiltinClassType.PythonClass)) { + } else if (isType(obj)) { profileSeen(SEEN_TYPE); saveGlobal(frame, ctx, pickler, proto, fastMode, obj, null); return; From 2d7de94d69c66165998ce7fd7c68eb2212436637 Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Wed, 30 Sep 2026 16:41:39 +0200 Subject: [PATCH 16/39] Fix emitting OPCODE_BYTEARRAY8 --- .../graal/python/builtins/modules/pickle/PPickler.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java index 5fc03d2831..12c54f5b4f 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java @@ -2450,8 +2450,9 @@ private void saveBytearrayData(VirtualFrame frame, PPickler pickler, int proto, // Our sizes are ints, but the protocol expects 8 bytes ensureBufferSpace(this, pickler, 9); - writeIntOp(this, pickler, PickleUtils.OPCODE_BYTEARRAY8, size); - pickler.outputLen += 4; + pickler.outputBuffer[pickler.outputLen++] = PickleUtils.OPCODE_BYTEARRAY8; + PickleUtils.writeSize64(pickler.outputBuffer, pickler.outputLen, size); + pickler.outputLen += 8; if (bypassBuffer && pickler.write != null) { flushToFile(frame, pickler); From 2600c46f2b1b4ed734c27f25f52605771b9b418a Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Thu, 1 Oct 2026 10:22:41 +0200 Subject: [PATCH 17/39] Reintroduce BasePickleNode again --- .../builtins/modules/pickle/PPickler.java | 366 +----------------- .../builtins/modules/pickle/PicklerNodes.java | 14 +- 2 files changed, 19 insertions(+), 361 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java index 12c54f5b4f..80329180c5 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java @@ -40,12 +40,9 @@ */ package com.oracle.graal.python.builtins.modules.pickle; -import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.T_CP_REVERSE_IMPORT_MAPPING; -import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.T_CP_REVERSE_NAME_MAPPING; import static com.oracle.graal.python.builtins.objects.PNone.NO_VALUE; import static com.oracle.graal.python.nodes.BuiltinNames.T___MAIN__; import static com.oracle.graal.python.nodes.ErrorMessages.MUST_BE_STR_NOT_P; -import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___CLASS__; import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___MODULE__; import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___NAME__; import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___NEWOBJ_EX__; @@ -55,10 +52,8 @@ import static com.oracle.graal.python.nodes.SpecialMethodNames.T___NEW__; import static com.oracle.graal.python.nodes.SpecialMethodNames.T___REDUCE_EX__; import static com.oracle.graal.python.nodes.SpecialMethodNames.T___REDUCE__; -import static com.oracle.graal.python.nodes.StringLiterals.T_DOT; import static com.oracle.graal.python.nodes.StringLiterals.T_NEWLINE; import static com.oracle.graal.python.nodes.statement.AbstractImportNode.importModule; -import static com.oracle.graal.python.runtime.exception.PythonErrorType.AttributeError; import static com.oracle.graal.python.runtime.exception.PythonErrorType.KeyError; import static com.oracle.graal.python.runtime.exception.PythonErrorType.OverflowError; import static com.oracle.graal.python.runtime.exception.PythonErrorType.PicklingError; @@ -74,7 +69,6 @@ import org.graalvm.collections.Pair; import com.oracle.graal.python.PythonLanguage; -import com.oracle.graal.python.builtins.Python3Core; import com.oracle.graal.python.builtins.PythonBuiltinClassType; import com.oracle.graal.python.builtins.objects.PNone; import com.oracle.graal.python.builtins.objects.PNotImplemented; @@ -85,19 +79,15 @@ import com.oracle.graal.python.builtins.objects.bytes.PBytes; import com.oracle.graal.python.builtins.objects.cext.PythonNativeClass; import com.oracle.graal.python.builtins.objects.common.HashingStorage; -import com.oracle.graal.python.builtins.objects.common.HashingStorageNodes.CachedHashingStorageGetItem; -import com.oracle.graal.python.builtins.objects.common.HashingStorageNodes.HashingStorageGetIterator; import com.oracle.graal.python.builtins.objects.common.HashingStorageNodes.HashingStorageIterator; import com.oracle.graal.python.builtins.objects.common.HashingStorageNodes.HashingStorageIteratorKey; import com.oracle.graal.python.builtins.objects.common.HashingStorageNodes.HashingStorageIteratorNext; import com.oracle.graal.python.builtins.objects.common.HashingStorageNodes.HashingStorageIteratorValue; import com.oracle.graal.python.builtins.objects.common.HashingStorageNodes.HashingStorageLen; -import com.oracle.graal.python.builtins.objects.common.SequenceStorageNodes; import com.oracle.graal.python.builtins.objects.dict.PDict; import com.oracle.graal.python.builtins.objects.ellipsis.PEllipsis; import com.oracle.graal.python.builtins.objects.floats.PFloat; import com.oracle.graal.python.builtins.objects.function.PFunction; -import com.oracle.graal.python.builtins.objects.function.PKeyword; import com.oracle.graal.python.builtins.objects.ints.IntNodes; import com.oracle.graal.python.builtins.objects.ints.IntNodesFactory; import com.oracle.graal.python.builtins.objects.ints.PInt; @@ -106,41 +96,27 @@ import com.oracle.graal.python.builtins.objects.set.PFrozenSet; import com.oracle.graal.python.builtins.objects.set.PSet; import com.oracle.graal.python.builtins.objects.str.PString; -import com.oracle.graal.python.builtins.objects.str.StringUtils; import com.oracle.graal.python.builtins.objects.tuple.PTuple; import com.oracle.graal.python.builtins.objects.type.PythonBuiltinClass; import com.oracle.graal.python.builtins.objects.type.PythonManagedClass; import com.oracle.graal.python.builtins.objects.type.TypeNodes; import com.oracle.graal.python.lib.IteratorExhausted; import com.oracle.graal.python.lib.PyCallableCheckNode; -import com.oracle.graal.python.lib.PyIterCheckNode; -import com.oracle.graal.python.lib.PyIterNextNode; import com.oracle.graal.python.lib.PyLongAsLongNode; import com.oracle.graal.python.lib.PyObjectCallMethodObjArgs; -import com.oracle.graal.python.lib.PyObjectGetItem; import com.oracle.graal.python.lib.PyObjectGetIter; import com.oracle.graal.python.lib.PyObjectIsTrueNode; import com.oracle.graal.python.lib.PyObjectLookupAttr; import com.oracle.graal.python.lib.PyObjectReprAsTruffleStringNode; -import com.oracle.graal.python.lib.PyObjectSizeNode; import com.oracle.graal.python.lib.PyObjectStrAsObjectNode; import com.oracle.graal.python.lib.PyTupleCheckNode; import com.oracle.graal.python.nodes.ErrorMessages; -import com.oracle.graal.python.nodes.HiddenAttr; import com.oracle.graal.python.nodes.PGuards; import com.oracle.graal.python.nodes.PRaiseNode; -import com.oracle.graal.python.nodes.argument.keywords.ExpandKeywordStarargsNode; -import com.oracle.graal.python.nodes.argument.keywords.ExpandKeywordStarargsNodeGen; -import com.oracle.graal.python.nodes.argument.positional.ExecutePositionalStarargsNode; import com.oracle.graal.python.nodes.builtins.ListNodes; import com.oracle.graal.python.nodes.builtins.TupleNodes.GetTupleStorage; import com.oracle.graal.python.nodes.call.CallNode; -import com.oracle.graal.python.nodes.object.BuiltinClassProfiles.InlineIsBuiltinClassProfile; -import com.oracle.graal.python.nodes.object.BuiltinClassProfiles.IsBuiltinObjectProfile; -import com.oracle.graal.python.nodes.object.GetClassNode; import com.oracle.graal.python.nodes.object.IsNode; -import com.oracle.graal.python.nodes.util.CannotCastException; -import com.oracle.graal.python.nodes.util.CastToTruffleStringNode; import com.oracle.graal.python.runtime.ExecutionContext.BoundaryCallContext; import com.oracle.graal.python.runtime.IndirectCallData.BoundaryCallData; import com.oracle.graal.python.runtime.PythonContext; @@ -161,11 +137,9 @@ import com.oracle.truffle.api.dsl.Cached; import com.oracle.truffle.api.dsl.GenerateInline; import com.oracle.truffle.api.dsl.Specialization; -import com.oracle.truffle.api.frame.Frame; import com.oracle.truffle.api.frame.VirtualFrame; import com.oracle.truffle.api.nodes.Node; import com.oracle.truffle.api.object.Shape; -import com.oracle.truffle.api.profiles.BranchProfile; import com.oracle.truffle.api.profiles.InlinedConditionProfile; import com.oracle.truffle.api.profiles.InlinedIntValueProfile; import com.oracle.truffle.api.strings.TranscodingErrorHandler; @@ -438,9 +412,8 @@ static SequenceStorage get(Object tuple, } } - public abstract static class SaveNode extends Node { + public abstract static class SaveNode extends PicklerNodes.BasePickleNode { private static final int MAX_RECURSION_DEPTH = 3; - private static final TruffleString T_LOCALS = tsLiteral(""); private static final TruffleString T_SYS_MODULES = tsLiteral("sys.modules"); static final byte[] TUPLE_LEN_2_OPCODE = new byte[]{PickleUtils.OPCODE_EMPTY_TUPLE, PickleUtils.OPCODE_TUPLE1, PickleUtils.OPCODE_TUPLE2, PickleUtils.OPCODE_TUPLE3}; @@ -457,67 +430,18 @@ public abstract static class SaveNode extends Node { @Child private ListNodes.ConstructListNode constructListNode; @Child private TypeNodes.IsTypeNode isTypeNode; @Child private FlushToFileNode flushToFileNode; - @Child private CallNode callNode; - @SuppressWarnings("FieldMayBeFinal") @Child private PyObjectGetItem getItemNode = PyObjectGetItem.create(); - @SuppressWarnings("FieldMayBeFinal") @Child private PyIterNextNode getNextNode = PyIterNextNode.create(); - @Child private CastToTruffleStringNode toStringNode = CastToTruffleStringNode.create(); - @Child private HiddenAttr.ReadNode readHiddenAttributeNode; - @Child private IsBuiltinObjectProfile errProfile; - @Child private InlineIsBuiltinClassProfile isBuiltinClassProfile; - @Child private CachedHashingStorageGetItem getHashingStorageItemNode; - @Child private SequenceStorageNodes.GetItemScalarNode getSeqStorageItemNode; @Child private TupleStorageNode getTupleStorageNode; - @Child private PyObjectSizeNode sizeNode; - @Child private PyIterCheckNode isIteratorObjectNode; - @Child private GetClassNode getClassNode; - @Child private PyObjectLookupAttr lookupAttrNode; - @Child private ExecutePositionalStarargsNode getArgsNode; - @Child private ExpandKeywordStarargsNode getKwArgsNode; - @Child private HashingStorageGetIterator getHashingStorageIteratorNode; - @Child private HashingStorageIteratorNext hashingStorageItNext; - @Child private HashingStorageIteratorKey hashingStorageItKey; - @Child private HashingStorageIteratorValue hashingStorageItValue; - @Child private TruffleString.FromByteArrayWithCompactionUTF32Node tsFromByteArrayWithCompactionNode; - @Child private TruffleString.CodePointLengthNode tsCodePointLengthNode; @Child private TruffleString.CodePointAtIndexUTF32Node tsCodePointAtIndexUTF32Node; @Child private TruffleString.FromLongNode tsFromLongNode; - @Child private TruffleString.IndexOfStringNode tsIndexOfStringNode; - @Child private TruffleString.SubstringNode tsSubstringNode; - @Child private TruffleString.EqualNode tsEqualNode; @Child private TruffleString.CopyToByteArrayNode tsCopyToByteArrayNode; @Child private TruffleString.GetCodeRangeNode tsGetCodeRangeNode; - @Child private TruffleString.SwitchEncodingNode tsSwitchEncodingNode; - private final BranchProfile errorProfile = BranchProfile.create(); - protected PException raise(PythonBuiltinClassType type, TruffleString string) { - errorProfile.enter(); - return PRaiseNode.raiseStatic(this, type, string); - } - - protected PException raise(PythonBuiltinClassType exceptionType) { - errorProfile.enter(); - return PRaiseNode.raiseStatic(this, exceptionType); - } - - protected final PException raise(PythonBuiltinClassType type, TruffleString format, Object... arguments) { - errorProfile.enter(); - return PRaiseNode.raiseStatic(this, type, format, arguments); - } - - protected TruffleString.FromByteArrayWithCompactionUTF32Node ensureTsFromByteArrayWithCompaction() { - if (tsFromByteArrayWithCompactionNode == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - tsFromByteArrayWithCompactionNode = insert(TruffleString.FromByteArrayWithCompactionUTF32Node.create()); - } - return tsFromByteArrayWithCompactionNode; - } - - protected TruffleString.CodePointLengthNode ensureTsCodePointLengthNode() { - if (tsCodePointLengthNode == null) { + protected SequenceStorage getTupleStorage(Object tuple) { + if (getTupleStorageNode == null) { CompilerDirectives.transferToInterpreterAndInvalidate(); - tsCodePointLengthNode = insert(TruffleString.CodePointLengthNode.create()); + getTupleStorageNode = insert(PPicklerFactory.TupleStorageNodeGen.create()); } - return tsCodePointLengthNode; + return getTupleStorageNode.execute(tuple); } protected TruffleString.CodePointAtIndexUTF32Node ensureTsCodePointAtIndexUTF32Node() { @@ -536,30 +460,6 @@ protected TruffleString.FromLongNode ensureTsFromLongNode() { return tsFromLongNode; } - protected TruffleString.IndexOfStringNode ensureTsIndexOfStringNode() { - if (tsIndexOfStringNode == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - tsIndexOfStringNode = insert(TruffleString.IndexOfStringNode.create()); - } - return tsIndexOfStringNode; - } - - protected TruffleString.SubstringNode ensureTsSubstringNode() { - if (tsSubstringNode == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - tsSubstringNode = insert(TruffleString.SubstringNode.create()); - } - return tsSubstringNode; - } - - protected TruffleString.EqualNode ensureTsEqualNode() { - if (tsEqualNode == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - tsEqualNode = insert(TruffleString.EqualNode.create()); - } - return tsEqualNode; - } - protected TruffleString.CopyToByteArrayNode ensureTsCopyToByteArrayNode() { if (tsCopyToByteArrayNode == null) { CompilerDirectives.transferToInterpreterAndInvalidate(); @@ -576,262 +476,12 @@ protected TruffleString.GetCodeRangeNode ensureTsGetCodeRangeNode() { return tsGetCodeRangeNode; } - protected TruffleString.SwitchEncodingNode ensureTsSwitchEncodingNode() { - if (tsSwitchEncodingNode == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - tsSwitchEncodingNode = insert(TruffleString.SwitchEncodingNode.create()); - } - return tsSwitchEncodingNode; - } - - protected HashingStorageIterator getHashingStorageIterator(HashingStorage storage) { - if (getHashingStorageIteratorNode == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - getHashingStorageIteratorNode = insert(HashingStorageGetIterator.create()); - } - return getHashingStorageIteratorNode.executeCached(storage); - } - - protected HashingStorageIteratorNext ensureHashingStorageIteratorNext() { - if (hashingStorageItNext == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - hashingStorageItNext = insert(HashingStorageIteratorNext.create()); - } - return hashingStorageItNext; - } - - protected HashingStorageIteratorKey ensureHashingStorageIteratorKey() { - if (hashingStorageItKey == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - hashingStorageItKey = insert(HashingStorageIteratorKey.create()); - } - return hashingStorageItKey; - } - - protected HashingStorageIteratorValue ensureHashingStorageIteratorValue() { - if (hashingStorageItValue == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - hashingStorageItValue = insert(HashingStorageIteratorValue.create()); - } - return hashingStorageItValue; - } - - protected int length(VirtualFrame frame, Object object) { - if (sizeNode == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - sizeNode = insert(PyObjectSizeNode.create()); - } - return sizeNode.executeCached(frame, object); - } - - protected boolean isIterator(Object iter) { - if (isIteratorObjectNode == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - isIteratorObjectNode = insert(PyIterCheckNode.create()); - } - return isIteratorObjectNode.executeCached(iter); - } - - protected CallNode ensureCallNode() { - if (callNode == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - callNode = insert(CallNode.create()); - } - return callNode; - } - - protected Object callStarArgsAndKwArgs(VirtualFrame frame, Object method, Object args, Object kwargs) { - if (getArgsNode == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - getArgsNode = insert(ExecutePositionalStarargsNode.create()); - } - PKeyword[] keywords; - if (kwargs == null) { - keywords = PKeyword.EMPTY_KEYWORDS; - } else { - if (getKwArgsNode == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - getKwArgsNode = insert(ExpandKeywordStarargsNodeGen.create()); - } - keywords = getKwArgsNode.executeCached(kwargs); - } - return ensureCallNode().execute(frame, method, getArgsNode.executeWith(frame, args), keywords); - } - - protected PyObjectLookupAttr getLookupAttrNode() { - if (lookupAttrNode == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - lookupAttrNode = insert(PyObjectLookupAttr.create()); - } - return lookupAttrNode; - } - - protected Object lookupAttribute(Frame frame, Object receiver, TruffleString name) { - return getLookupAttrNode().executeCached(frame, receiver, name); - } - - protected Object lookupAttributeStrict(Frame frame, Object receiver, TruffleString name) { - Object attr = lookupAttribute(frame, receiver, name); - if (attr == NO_VALUE) { - throw raise(TypeError, ErrorMessages.OBJ_P_HAS_NO_ATTR_S, attr, name); - } - return attr; - } - - protected Object getClass(Object object) { - if (getClassNode == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - getClassNode = insert(GetClassNode.create()); - } - return getClassNode.executeCached(object); - } - - protected Object getClass(Frame frame, Object object) { - Object cls = getLookupAttrNode().executeCached(frame, object, T___CLASS__); - return cls == NO_VALUE ? getClass(object) : cls; - } - - protected HiddenAttr.ReadNode ensureReadHiddenAttrNode() { - if (readHiddenAttributeNode == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - readHiddenAttributeNode = insert(HiddenAttr.ReadNode.create()); - } - return readHiddenAttributeNode; - } - - public PickleState getGlobalState(Python3Core core) { - Object state = ensureReadHiddenAttrNode().executeCached(core.lookupType(PythonBuiltinClassType.Pickler), HiddenAttr.PICKLE_STATE, NO_VALUE); - assert state instanceof PickleState; - return (PickleState) state; - } - - protected Object getDictItem(VirtualFrame frame, PDict dict, Object key) { - if (getHashingStorageItemNode == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - getHashingStorageItemNode = insert(CachedHashingStorageGetItem.create()); - } - return getHashingStorageItemNode.execute(frame, dict.getDictStorage(), key); - } - - protected IsBuiltinObjectProfile ensureErrProfile() { - if (errProfile == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - errProfile = insert(IsBuiltinObjectProfile.create()); - } - return errProfile; - } - - protected boolean isBuiltinClass(Object cls, PythonBuiltinClassType type) { - if (isBuiltinClassProfile == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - isBuiltinClassProfile = insert(InlineIsBuiltinClassProfile.create()); - } - return isBuiltinClassProfile.profileClassCached(cls, type); - } - - public Object getNextItem(VirtualFrame frame, Object iterator) { - return getNextNode.executeCached(frame, iterator); - } - - public Object getItem(SequenceStorage storage, int index) { - if (getSeqStorageItemNode == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - getSeqStorageItemNode = insert(SequenceStorageNodes.GetItemScalarNode.create()); - } - return getSeqStorageItemNode.executeCached(storage, index); - } - - protected SequenceStorage getTupleStorage(Object tuple) { - if (getTupleStorageNode == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - getTupleStorageNode = insert(PPicklerFactory.TupleStorageNodeGen.create()); - } - return getTupleStorageNode.execute(tuple); - } - - public Object getItem(VirtualFrame frame, Object obj, Object key) { - return getItemNode.executeCached(frame, obj, key); - } - - public TruffleString asString(Object value) { - if (value instanceof TruffleString string) { - return string; - } - try { - return toStringNode.executeCached(value); - } catch (CannotCastException e) { - return null; - } - } - public TruffleString asStringStrict(Object value) { - if (value instanceof TruffleString string) { - return string; - } - try { - return toStringNode.executeCached(value); - } catch (CannotCastException e) { + TruffleString string = asString(value); + if (string == null) { throw raise(TypeError, MUST_BE_STR_NOT_P, value); } - } - - public static Pair getDeepAttribute(VirtualFrame frame, PyObjectLookupAttr lookup, Object obj, TruffleString[] names) { - Object parent = null; - Object object = obj; - for (TruffleString name : names) { - parent = object; - object = lookup.executeCached(frame, parent, name); - if (object == NO_VALUE) { - return null; - } - } - return Pair.create(object, parent); - } - - public TruffleString[] getDottedPath(Object obj, TruffleString name) { - TruffleString[] dottedPath = StringUtils.split(name, T_DOT, ensureTsCodePointLengthNode(), ensureTsIndexOfStringNode(), ensureTsSubstringNode(), ensureTsEqualNode()); - assert dottedPath.length > 0; - for (TruffleString subPath : dottedPath) { - if (ensureTsEqualNode().execute(subPath, T_LOCALS, TS_ENCODING)) { - if (obj == null) { - throw raise(AttributeError, ErrorMessages.CANT_PICKLE_LOCAL_OBJ_S, name); - } else { - throw raise(AttributeError, ErrorMessages.CANT_PICKLE_ATTR_S_OF_P, name, obj); - } - } - } - return dottedPath; - } - - protected Pair get3to2Mapping(VirtualFrame frame, Python3Core core, TruffleString moduleName, TruffleString globalName) { - PickleState state = getGlobalState(core); - return getMapping(frame, state.nameMapping3To2, state.importMapping3To2, T_CP_REVERSE_NAME_MAPPING, T_CP_REVERSE_IMPORT_MAPPING, moduleName, globalName); - } - - private Pair getMapping(VirtualFrame frame, PDict nameMapping, PDict importMapping, - TruffleString nameMappingLabel, TruffleString importMappingLabel, TruffleString moduleName, TruffleString globalName) { - Object key = PFactory.createTuple(PythonLanguage.get(this), new Object[]{moduleName, globalName}); - Object item = getDictItem(frame, nameMapping, key); - if (item != null) { - if (!(item instanceof PTuple) || ((PTuple) item).getSequenceStorage().length() != 2) { - throw raise(PythonBuiltinClassType.RuntimeError, ErrorMessages.S_SHOULD_BE_S_NOT_P, nameMappingLabel, "2-tuples", item); - } - SequenceStorage storage = ((PTuple) item).getSequenceStorage(); - Object mappedModuleName = getItem(storage, 0); - Object mappedGlobalName = getItem(storage, 1); - if (!PGuards.isString(mappedModuleName) || !PGuards.isString(mappedGlobalName)) { - throw raise(PythonBuiltinClassType.RuntimeError, ErrorMessages.S_SHOULD_BE_S_NOT_P_P, nameMappingLabel, "str", mappedModuleName, mappedGlobalName); - } - return Pair.create(asString(mappedModuleName), asString(mappedGlobalName)); - } - item = getDictItem(frame, importMapping, moduleName); - if (item != null) { - if (!PGuards.isString(item)) { - throw raise(PythonBuiltinClassType.RuntimeError, ErrorMessages.S_SHOULD_BE_S_NOT_P, importMappingLabel, "strings", item); - } - return Pair.create(asString(item), globalName); - } - return Pair.create(moduleName, globalName); + return string; } private static boolean checkModule(VirtualFrame frame, PyObjectLookupAttr lookup, Object moduleName, Object module, Object global, TruffleString[] dottedPath) { diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java index 7c94c9425b..203d7ea33b 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java @@ -42,6 +42,8 @@ import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.T_CP_IMPORT_MAPPING; import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.T_CP_NAME_MAPPING; +import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.T_CP_REVERSE_IMPORT_MAPPING; +import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.T_CP_REVERSE_NAME_MAPPING; import static com.oracle.graal.python.builtins.objects.PNone.NO_VALUE; import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___CLASS__; import static com.oracle.graal.python.nodes.StringLiterals.T_DOT; @@ -588,6 +590,11 @@ public Object getattribute(VirtualFrame frame, Object obj, TruffleString name, b } } + protected Pair get3to2Mapping(VirtualFrame frame, Python3Core core, TruffleString moduleName, TruffleString globalName) { + PickleState state = getGlobalState(core); + return getMapping(frame, state.nameMapping3To2, state.importMapping3To2, T_CP_REVERSE_NAME_MAPPING, T_CP_REVERSE_IMPORT_MAPPING, moduleName, globalName); + } + protected Pair get2To3Mapping(VirtualFrame frame, Python3Core core, TruffleString moduleName, TruffleString globalName) { PickleState state = getGlobalState(core); return getMapping(frame, state.nameMapping2To3, state.importMapping2To3, T_CP_NAME_MAPPING, T_CP_IMPORT_MAPPING, moduleName, globalName); @@ -598,11 +605,12 @@ private Pair getMapping(VirtualFrame frame, PDict Object key = PFactory.createTuple(PythonLanguage.get(this), new Object[]{moduleName, globalName}); Object item = getDictItem(frame, nameMapping, key); if (item != null) { - if (!(item instanceof PTuple) || length(frame, item) != 2) { + if (!(item instanceof PTuple) || ((PTuple) item).getSequenceStorage().length() != 2) { throw raise(PythonBuiltinClassType.RuntimeError, ErrorMessages.S_SHOULD_BE_S_NOT_P, nameMappingLabel, "2-tuples", item); } - Object mappedModuleName = getItem(frame, item, 0); - Object mappedGlobalName = getItem(frame, item, 1); + SequenceStorage storage = ((PTuple) item).getSequenceStorage(); + Object mappedModuleName = getItem(storage, 0); + Object mappedGlobalName = getItem(storage, 1); if (!PGuards.isString(mappedModuleName) || !PGuards.isString(mappedGlobalName)) { throw raise(PythonBuiltinClassType.RuntimeError, ErrorMessages.S_SHOULD_BE_S_NOT_P_P, nameMappingLabel, "str", mappedModuleName, mappedGlobalName); } From c275c1d6923628af5c4712fe32aaa9580c23facb Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Thu, 1 Oct 2026 10:29:22 +0200 Subject: [PATCH 18/39] Organize SaveNode fields --- .../builtins/modules/pickle/PPickler.java | 126 +++++++++--------- 1 file changed, 63 insertions(+), 63 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java index 80329180c5..d7b1883fb5 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java @@ -424,6 +424,53 @@ public abstract static class SaveNode extends PicklerNodes.BasePickleNode { static final TruffleString T_SET = tsLiteral("set"); static final TruffleString T_DICTIONARY = tsLiteral("dictionary"); + private static final long SEEN_NONE = 1; + private static final long SEEN_BOOLEAN = 1 << 1; + private static final long SEEN_INTEGER = 1 << 2; + private static final long SEEN_LONG = 1 << 3; + private static final long SEEN_PINT = 1 << 4; + private static final long SEEN_DOUBLE = 1 << 5; + private static final long SEEN_PFLOAT = 1 << 6; + private static final long SEEN_BUILTIN_CLASS_TYPE = 1 << 7; + private static final long SEEN_BYTES = 1 << 8; + private static final long SEEN_TRUFFLE_STRING = 1 << 9; + private static final long SEEN_PSTRING = 1 << 10; + private static final long SEEN_DICT = 1 << 11; + private static final long SEEN_SET = 1 << 12; + private static final long SEEN_FROZENSET = 1 << 13; + private static final long SEEN_LIST = 1 << 14; + private static final long SEEN_TUPLE = 1 << 15; + private static final long SEEN_NATIVE_TUPLE = 1 << 16; + private static final long SEEN_BYTE_ARRAY = 1 << 17; + private static final long SEEN_PICKLE_BUFFER = 1 << 18; + private static final long SEEN_MANAGED_CLASS = 1 << 19; + private static final long SEEN_NATIVE_CLASS = 1 << 20; + private static final long SEEN_FUNCTION = 1 << 21; + private static final long SEEN_TYPE = 1 << 22; + private static final long SEEN_OBJECT = 1 << 23; + private static final long SEEN_INT_STORAGE = 1 << 24; + private static final long SEEN_LONG_STORAGE = 1 << 25; + private static final long SEEN_DOUBLE_STORAGE = 1 << 26; + private static final long SEEN_GENERIC_STORAGE = 1 << 27; + private static final long SEEN_CUSTOM_DISPATCH = 1 << 28; + private static final long SEEN_REGISTERED_REDUCER = 1 << 29; + private static final long SEEN_REDUCE_EX = 1 << 30; + private static final long SEEN_REDUCE = 1L << 31; + private static final long SEEN_NEWOBJ_EX = 1L << 32; + private static final long SEEN_NEWOBJ = 1L << 33; + private static final long SEEN_PLAIN_REDUCE = 1L << 34; + private static final long SEEN_REDUCE_LISTITEMS = 1L << 35; + private static final long SEEN_REDUCE_DICTITEMS = 1L << 36; + private static final long SEEN_REDUCE_STATE = 1L << 37; + private static final long SEEN_BYPASS_BUFFER = 1L << 38; + + @CompilationFinal private long seenTypes; + + private final int depth; + @Child private SaveNode recursiveSaveNode1; + @Child private SaveNode recursiveSaveNode2; + @Child private SaveNode recursiveSaveNode3; + @Child private IntNodes.PyLongSign pyLongSign; @Child private IntNodes.PyLongNumBits pyLongNumBits; @Child private IntNodes.PyLongAsByteArray pyLongAsByteArray; @@ -435,6 +482,22 @@ public abstract static class SaveNode extends PicklerNodes.BasePickleNode { @Child private TruffleString.FromLongNode tsFromLongNode; @Child private TruffleString.CopyToByteArrayNode tsCopyToByteArrayNode; @Child private TruffleString.GetCodeRangeNode tsGetCodeRangeNode; + @Child private BoundaryCallData boundaryCallData; + @Child private PyLongAsLongNode pyLongAsLongNode; + @Child private PyObjectStrAsObjectNode pyObjectStrAsObjectNode; + @Child private PyObjectIsTrueNode isTrueNode; + @Child private IsNode isNode; + @Child private PythonBufferAcquireLibrary bufferAcquireLibrary; + @Child private PythonBufferAccessLibrary bufferLibrary; + @Child private PyCallableCheckNode callableCheckNode; + @Child private PyObjectReprAsTruffleStringNode reprNode; + @Child private PyObjectGetIter getIterNode; + @Child private HashingStorageLen hashingStorageLenNode; + @Child private PyTupleCheckNode.CachedNode tupleCheckNode; + + protected SaveNode(int depth) { + this.depth = depth; + } protected SequenceStorage getTupleStorage(Object tuple) { if (getTupleStorageNode == null) { @@ -756,69 +819,6 @@ protected static void fastSaveLeave(PPickler pickler, Object obj) { } } - private static final int SEEN_NONE = 1 << 0; - private static final int SEEN_BOOLEAN = 1 << 1; - private static final int SEEN_INTEGER = 1 << 2; - private static final int SEEN_LONG = 1 << 3; - private static final int SEEN_PINT = 1 << 4; - private static final int SEEN_DOUBLE = 1 << 5; - private static final int SEEN_PFLOAT = 1 << 6; - private static final int SEEN_BUILTIN_CLASS_TYPE = 1 << 7; - private static final int SEEN_BYTES = 1 << 8; - private static final int SEEN_TRUFFLE_STRING = 1 << 9; - private static final int SEEN_PSTRING = 1 << 10; - private static final int SEEN_DICT = 1 << 11; - private static final int SEEN_SET = 1 << 12; - private static final int SEEN_FROZENSET = 1 << 13; - private static final int SEEN_LIST = 1 << 14; - private static final int SEEN_TUPLE = 1 << 15; - private static final int SEEN_NATIVE_TUPLE = 1 << 16; - private static final int SEEN_BYTE_ARRAY = 1 << 17; - private static final int SEEN_PICKLE_BUFFER = 1 << 18; - private static final int SEEN_MANAGED_CLASS = 1 << 19; - private static final int SEEN_NATIVE_CLASS = 1 << 20; - private static final int SEEN_FUNCTION = 1 << 21; - private static final int SEEN_TYPE = 1 << 22; - private static final int SEEN_OBJECT = 1 << 23; - private static final int SEEN_INT_STORAGE = 1 << 24; - private static final int SEEN_LONG_STORAGE = 1 << 25; - private static final int SEEN_DOUBLE_STORAGE = 1 << 26; - private static final int SEEN_GENERIC_STORAGE = 1 << 27; - private static final int SEEN_CUSTOM_DISPATCH = 1 << 28; - private static final int SEEN_REGISTERED_REDUCER = 1 << 29; - private static final int SEEN_REDUCE_EX = 1 << 30; - private static final int SEEN_REDUCE = 1 << 31; - private static final long SEEN_NEWOBJ_EX = 1L << 32; - private static final long SEEN_NEWOBJ = 1L << 33; - private static final long SEEN_PLAIN_REDUCE = 1L << 34; - private static final long SEEN_REDUCE_LISTITEMS = 1L << 35; - private static final long SEEN_REDUCE_DICTITEMS = 1L << 36; - private static final long SEEN_REDUCE_STATE = 1L << 37; - private static final long SEEN_BYPASS_BUFFER = 1L << 38; - - @CompilationFinal private long seenTypes; - - private final int depth; - @Child private SaveNode recursiveSaveNode1; - @Child private SaveNode recursiveSaveNode2; - @Child private SaveNode recursiveSaveNode3; - @Child private BoundaryCallData boundaryCallData; - @Child private PyLongAsLongNode pyLongAsLongNode; - @Child private PyObjectStrAsObjectNode pyObjectStrAsObjectNode; - @Child private PyObjectIsTrueNode isTrueNode; - @Child private IsNode isNode; - @Child private PythonBufferAcquireLibrary bufferAcquireLibrary; - @Child private PythonBufferAccessLibrary bufferLibrary; - @Child private PyCallableCheckNode callableCheckNode; - @Child private PyObjectReprAsTruffleStringNode reprNode; - @Child private PyObjectGetIter getIterNode; - @Child private HashingStorageLen hashingStorageLenNode; - @Child private PyTupleCheckNode.CachedNode tupleCheckNode; - - protected SaveNode(int depth) { - this.depth = depth; - } - private void profileSeen(long seen) { if ((seenTypes & seen) == 0) { CompilerDirectives.transferToInterpreterAndInvalidate(); From 7848db81fbc20eb7865721e7b8814b17438a51a7 Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Thu, 1 Oct 2026 10:39:26 +0200 Subject: [PATCH 19/39] Move write helpers to PPickler --- .../builtins/modules/pickle/PPickler.java | 548 +++++++++--------- 1 file changed, 276 insertions(+), 272 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java index d7b1883fb5..646f5f925a 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java @@ -146,6 +146,7 @@ import com.oracle.truffle.api.strings.TruffleString; public class PPickler extends PythonBuiltinObject { + private static final byte NEW_LINE_BYTE = '\n'; // Memo table, keep track of the seen objects to support self-referential objects pickling private MemoTable memo; // persistent_id() method, can be NULL @@ -399,6 +400,168 @@ public PBytes getString(PythonLanguage language) { return PFactory.createBytes(language, outputBuffer, outputLen); } + private void writeASCII(Node node, TruffleString string, TruffleString.SwitchEncodingNode switchEncodingNode, + TruffleString.CopyToByteArrayNode copyToByteArrayNode) { + assert string.getCodeRangeUncached(TS_ENCODING) == TruffleString.CodeRange.ASCII; + TruffleString encoded = switchEncodingNode.execute(string, TruffleString.Encoding.US_ASCII); + int length = encoded.byteLength(TruffleString.Encoding.US_ASCII); + ensureBufferSpace(node, length); + copyToByteArrayNode.execute(encoded, 0, outputBuffer, outputLen, length, TruffleString.Encoding.US_ASCII); + outputLen += length; + } + + private void writeUtf8(Node node, TruffleString string, TruffleString.SwitchEncodingNode switchEncodingNode, + TruffleString.CopyToByteArrayNode copyToByteArrayNode) { + TruffleString encoded = switchEncodingNode.execute(string, TruffleString.Encoding.UTF_8, TranscodingErrorHandler.DEFAULT_KEEP_SURROGATES_IN_UTF8); + writeUtf8Converted(node, encoded, copyToByteArrayNode); + } + + private void writeUtf8Converted(Node node, TruffleString encoded, TruffleString.CopyToByteArrayNode copyToByteArrayNode) { + int length = encoded.byteLength(TruffleString.Encoding.UTF_8); + ensureBufferSpace(node, length); + copyToByteArrayNode.execute(encoded, 0, outputBuffer, outputLen, length, TruffleString.Encoding.UTF_8); + outputLen += length; + } + + private void writeLongAsString(Node node, byte opcode, long value, TruffleString.FromLongNode fromLongNode, + TruffleString.CopyToByteArrayNode copyToByteArrayNode) { + TruffleString string = fromLongNode.execute(value, TruffleString.Encoding.US_ASCII, true); + TruffleString.Encoding encoding = TruffleString.Encoding.US_ASCII; + int length = string.byteLength(encoding); + ensureBufferSpace(node, length + 2); + outputBuffer[outputLen++] = opcode; + copyToByteArrayNode.execute(string, 0, outputBuffer, outputLen, length, encoding); + outputLen += length; + outputBuffer[outputLen++] = NEW_LINE_BYTE; + } + + private void write(Node node, byte b) { + ensureBufferSpace(node, 1); + outputBuffer[outputLen++] = b; + } + + private void write(Node node, byte b1, byte b2) { + ensureBufferSpace(node, 2); + outputBuffer[outputLen++] = b1; + outputBuffer[outputLen++] = b2; + } + + private void writeBuffer(Node node, Object buffer, int size, PythonBufferAccessLibrary bufferLib) { + ensureBufferSpace(node, size); + bufferLib.readIntoByteArray(buffer, 0, outputBuffer, outputLen, size); + outputLen += size; + } + + private static byte intByte1(long value) { + return (byte) (value & 0xff); + } + + private static byte intByte2(long value) { + return (byte) ((value >> 8) & 0xff); + } + + private static byte intByte3(long value) { + return (byte) ((value >> 16) & 0xff); + } + + private static byte intByte4(long value) { + return (byte) ((value >> 24) & 0xff); + } + + private void writeByteOp(Node node, byte opcode, long value) { + ensureBufferSpace(node, 2); + outputBuffer[outputLen++] = opcode; + outputBuffer[outputLen++] = intByte1(value); + } + + private void writeShortOp(Node node, byte opcode, long value) { + ensureBufferSpace(node, 3); + outputBuffer[outputLen++] = opcode; + outputBuffer[outputLen++] = intByte1(value); + outputBuffer[outputLen++] = intByte2(value); + } + + private void writeIntOp(Node node, byte opcode, long value) { + ensureBufferSpace(node, 5); + outputBuffer[outputLen++] = opcode; + outputBuffer[outputLen++] = intByte1(value); + outputBuffer[outputLen++] = intByte2(value); + outputBuffer[outputLen++] = intByte3(value); + outputBuffer[outputLen++] = intByte4(value); + } + + private void write(Node node, byte[] bytes, int dataLen) { + ensureBufferSpace(node, dataLen); + PythonUtils.arraycopy(bytes, 0, outputBuffer, outputLen, dataLen); + outputLen += dataLen; + } + + private void ensureBufferSpace(Node node, int dataLen) { + boolean needNewFrame = isFraming() && frameStart == -1; + int n = CompilerDirectives.injectBranchProbability(CompilerDirectives.SLOWPATH_PROBABILITY, needNewFrame) ? dataLen + PickleUtils.FRAME_HEADER_SIZE : dataLen; + int required = outputLen + n; + + if (CompilerDirectives.injectBranchProbability(CompilerDirectives.SLOWPATH_PROBABILITY, required > maxOutputLen)) { + // Make place in buffer for the pickle chunk + // TODO: when GR-24978 is completed we should use PY_SSIZE_T_MAX + if (CompilerDirectives.injectBranchProbability(CompilerDirectives.SLOWPATH_PROBABILITY, outputLen >= Integer.MAX_VALUE / 2 - n)) { + CompilerDirectives.transferToInterpreter(); + throw PRaiseNode.raiseStatic(node, PythonBuiltinClassType.MemoryError); + } + maxOutputLen = (outputLen + n) / 2 * 3; + outputBuffer = PythonUtils.arrayCopyOf(outputBuffer, maxOutputLen); + } + if (CompilerDirectives.injectBranchProbability(CompilerDirectives.SLOWPATH_PROBABILITY, needNewFrame)) { + frameStart = outputLen; + outputLen += PickleUtils.FRAME_HEADER_SIZE; + } + } + + private boolean bypassBuffer(int dataSize) { + boolean bypassBuffer = dataSize >= PickleUtils.FRAME_SIZE_TARGET; + + if (bypassBuffer) { + assert outputBuffer != null; + // Commit the previous frame. + commitFrame(); + // Disable framing temporarily + framing = false; + } + return bypassBuffer; + } + + private void writeRawUnicodeEscape(Node node, TruffleString string, + TruffleString.CodePointLengthNode codePointLengthNode, TruffleString.CodePointAtIndexUTF32Node codePointAtIndexNode) { + int codePointLength = codePointLengthNode.execute(string, TS_ENCODING); + long escapedLength = 0; + for (int i = 0; i < codePointLength; i++) { + int ch = codePointAtIndexNode.execute(string, i); + escapedLength += ch >= 0x10000 ? 10 : ch >= 256 || ch == '\\' || ch == 0 || ch == '\n' || ch == '\r' || ch == 0x1a ? 6 : 1; + } + if (escapedLength > Integer.MAX_VALUE) { + throw PRaiseNode.raiseStatic(node, PythonBuiltinClassType.MemoryError); + } + ensureBufferSpace(node, (int) escapedLength); + for (int i = 0; i < codePointLength; i++) { + int ch = codePointAtIndexNode.execute(string, i); + if (ch >= 0x10000) { + outputBuffer[outputLen++] = '\\'; + outputBuffer[outputLen++] = 'U'; + for (int shift = 28; shift >= 0; shift -= 4) { + outputBuffer[outputLen++] = BytesUtils.HEXDIGITS[(ch >> shift) & 0xf]; + } + } else if (ch >= 256 || ch == '\\' || ch == 0 || ch == '\n' || ch == '\r' || ch == 0x1a) { + outputBuffer[outputLen++] = '\\'; + outputBuffer[outputLen++] = 'u'; + for (int shift = 12; shift >= 0; shift -= 4) { + outputBuffer[outputLen++] = BytesUtils.HEXDIGITS[(ch >> shift) & 0xf]; + } + } else { + outputBuffer[outputLen++] = (byte) ch; + } + } + } + // inner nodes @GenerateInline(false) public abstract static class TupleStorageNode extends Node { @@ -417,7 +580,6 @@ public abstract static class SaveNode extends PicklerNodes.BasePickleNode { private static final TruffleString T_SYS_MODULES = tsLiteral("sys.modules"); static final byte[] TUPLE_LEN_2_OPCODE = new byte[]{PickleUtils.OPCODE_EMPTY_TUPLE, PickleUtils.OPCODE_TUPLE1, PickleUtils.OPCODE_TUPLE2, PickleUtils.OPCODE_TUPLE3}; - static final byte NEW_LINE_BYTE = '\n'; static final TruffleString DICT_ITEMS = tsLiteral("dict items"); static final TruffleString LATIN1 = tsLiteral("latin1"); static final TruffleString T_L_NEW_LINE = tsLiteral("L\n"); @@ -660,141 +822,6 @@ public static void handleError(PPickler pickler) { pickler.reducerOverride = null; } - protected static void writeASCII(Node node, PPickler pickler, TruffleString string, TruffleString.SwitchEncodingNode switchEncodingNode, - TruffleString.CopyToByteArrayNode copyToByteArrayNode) { - assert string.getCodeRangeUncached(TS_ENCODING) == TruffleString.CodeRange.ASCII; - TruffleString encoded = switchEncodingNode.execute(string, TruffleString.Encoding.US_ASCII); - int length = encoded.byteLength(TruffleString.Encoding.US_ASCII); - ensureBufferSpace(node, pickler, length); - copyToByteArrayNode.execute(encoded, 0, pickler.outputBuffer, pickler.outputLen, length, TruffleString.Encoding.US_ASCII); - pickler.outputLen += length; - } - - protected static void writeUtf8(Node node, PPickler pickler, TruffleString string, TruffleString.SwitchEncodingNode switchEncodingNode, - TruffleString.CopyToByteArrayNode copyToByteArrayNode) { - TruffleString encoded = switchEncodingNode.execute(string, TruffleString.Encoding.UTF_8, TranscodingErrorHandler.DEFAULT_KEEP_SURROGATES_IN_UTF8); - writeUtf8Converted(node, pickler, encoded, copyToByteArrayNode); - } - - public static void writeUtf8Converted(Node node, PPickler pickler, TruffleString encoded, TruffleString.CopyToByteArrayNode copyToByteArrayNode) { - int length = encoded.byteLength(TruffleString.Encoding.UTF_8); - ensureBufferSpace(node, pickler, length); - copyToByteArrayNode.execute(encoded, 0, pickler.outputBuffer, pickler.outputLen, length, TruffleString.Encoding.UTF_8); - pickler.outputLen += length; - } - - protected static void writeLongAsString(Node node, PPickler pickler, byte opcode, long value, TruffleString.FromLongNode fromLongNode, - TruffleString.CopyToByteArrayNode copyToByteArrayNode) { - TruffleString string = fromLongNode.execute(value, TruffleString.Encoding.US_ASCII, true); - TruffleString.Encoding encoding = TruffleString.Encoding.US_ASCII; - int length = string.byteLength(encoding); - ensureBufferSpace(node, pickler, length + 2); - pickler.outputBuffer[pickler.outputLen++] = opcode; - copyToByteArrayNode.execute(string, 0, pickler.outputBuffer, pickler.outputLen, length, encoding); - pickler.outputLen += length; - pickler.outputBuffer[pickler.outputLen++] = NEW_LINE_BYTE; - } - - protected static void write(Node node, PPickler pickler, byte b) { - ensureBufferSpace(node, pickler, 1); - pickler.outputBuffer[pickler.outputLen++] = b; - } - - protected static void write(Node node, PPickler pickler, byte b1, byte b2) { - ensureBufferSpace(node, pickler, 2); - pickler.outputBuffer[pickler.outputLen++] = b1; - pickler.outputBuffer[pickler.outputLen++] = b2; - } - - protected static void writeBuffer(Node node, PPickler pickler, Object buffer, int size, PythonBufferAccessLibrary bufferLib) { - ensureBufferSpace(node, pickler, size); - bufferLib.readIntoByteArray(buffer, 0, pickler.outputBuffer, pickler.outputLen, size); - pickler.outputLen += size; - } - - protected static byte intByte1(long value) { - return (byte) (value & 0xff); - } - - protected static byte intByte2(long value) { - return (byte) ((value >> 8) & 0xff); - } - - protected static byte intByte3(long value) { - return (byte) ((value >> 16) & 0xff); - } - - protected static byte intByte4(long value) { - return (byte) ((value >> 24) & 0xff); - } - - protected static void writeByteOp(Node node, PPickler pickler, byte opcode, long value) { - ensureBufferSpace(node, pickler, 2); - pickler.outputBuffer[pickler.outputLen++] = opcode; - pickler.outputBuffer[pickler.outputLen++] = intByte1(value); - } - - protected static void writeShortOp(Node node, PPickler pickler, byte opcode, long value) { - ensureBufferSpace(node, pickler, 3); - pickler.outputBuffer[pickler.outputLen++] = opcode; - pickler.outputBuffer[pickler.outputLen++] = intByte1(value); - pickler.outputBuffer[pickler.outputLen++] = intByte2(value); - } - - protected static void writeIntOp(Node node, PPickler pickler, byte opcode, long value) { - ensureBufferSpace(node, pickler, 5); - pickler.outputBuffer[pickler.outputLen++] = opcode; - pickler.outputBuffer[pickler.outputLen++] = intByte1(value); - pickler.outputBuffer[pickler.outputLen++] = intByte2(value); - pickler.outputBuffer[pickler.outputLen++] = intByte3(value); - pickler.outputBuffer[pickler.outputLen++] = intByte4(value); - } - - protected static void write(Node node, PPickler pickler, byte[] bytes) { - write(node, pickler, bytes, bytes.length); - } - - protected static void write(Node node, PPickler pickler, byte[] bytes, int dataLen) { - ensureBufferSpace(node, pickler, dataLen); - PythonUtils.arraycopy(bytes, 0, pickler.outputBuffer, pickler.outputLen, dataLen); - pickler.outputLen += dataLen; - } - - public static void ensureBufferSpace(Node node, PPickler pickler, int dataLen) { - boolean needNewFrame = pickler.isFraming() && pickler.frameStart == -1; - int n = CompilerDirectives.injectBranchProbability(CompilerDirectives.SLOWPATH_PROBABILITY, needNewFrame) ? dataLen + PickleUtils.FRAME_HEADER_SIZE : dataLen; - int required = pickler.outputLen + n; - - if (CompilerDirectives.injectBranchProbability(CompilerDirectives.SLOWPATH_PROBABILITY, required > pickler.maxOutputLen)) { - // Make place in buffer for the pickle chunk - // TODO: when GR-24978 is completed we should use PY_SSIZE_T_MAX - if (CompilerDirectives.injectBranchProbability(CompilerDirectives.SLOWPATH_PROBABILITY, pickler.outputLen >= Integer.MAX_VALUE / 2 - n)) { - CompilerDirectives.transferToInterpreter(); - throw PRaiseNode.raiseStatic(node, PythonBuiltinClassType.MemoryError); - } - pickler.maxOutputLen = (pickler.outputLen + n) / 2 * 3; - pickler.outputBuffer = PythonUtils.arrayCopyOf(pickler.outputBuffer, pickler.maxOutputLen); - } - if (CompilerDirectives.injectBranchProbability(CompilerDirectives.SLOWPATH_PROBABILITY, needNewFrame)) { - pickler.frameStart = pickler.outputLen; - pickler.outputLen += PickleUtils.FRAME_HEADER_SIZE; - } - } - - protected boolean bypassBuffer(PPickler pickler, int dataSize) { - boolean bypassBuffer = dataSize >= PickleUtils.FRAME_SIZE_TARGET; - - if (bypassBuffer) { - profileSeen(SEEN_BYPASS_BUFFER); - assert pickler.outputBuffer != null; - // Commit the previous frame. - pickler.commitFrame(); - // Disable framing temporarily - pickler.framing = false; - } - return bypassBuffer; - } - protected PTuple createTuple(Object... items) { if (items.length == 0) { return PFactory.createEmptyTuple(PythonLanguage.get(this)); @@ -1012,7 +1039,7 @@ private void saveHashingStorageBatched(VirtualFrame frame, PPickler pickler, Has int i; do { i = 0; - write(this, pickler, PickleUtils.OPCODE_MARK); + pickler.write(this, PickleUtils.OPCODE_MARK); while (nextNode.executeCached(storage, it)) { save(frame, pickler, getKeyNode.executeCached(storage, it), 0); if (saveValues) { @@ -1022,7 +1049,7 @@ private void saveHashingStorageBatched(VirtualFrame frame, PPickler pickler, Has break; } } - write(this, pickler, opcodeBatch); + pickler.write(this, opcodeBatch); if (getHashingStorageLength(storage) != initialSize) { throw raise(PythonBuiltinClassType.RuntimeError, ErrorMessages.CHANGED_SIZE_DURING_ITERATION, name); } @@ -1068,12 +1095,12 @@ private void saveIteratorBatchedUnrolled(VirtualFrame frame, PPickler pickler, O } catch (IteratorExhausted e) { // Only one item to write saveItem.accept(firstItem); - write(this, pickler, opcodeOneItem); + pickler.write(this, opcodeOneItem); break; } // More than one item to write // Pump out MARK, items, APPENDS. - write(this, pickler, PickleUtils.OPCODE_MARK); + pickler.write(this, PickleUtils.OPCODE_MARK); saveItem.accept(firstItem); n = 1; while (true) { @@ -1092,7 +1119,7 @@ private void saveIteratorBatchedUnrolled(VirtualFrame frame, PPickler pickler, O break; } } - write(this, pickler, opcodeMultipleItems); + pickler.write(this, opcodeMultipleItems); } while (n == PickleUtils.BATCHSIZE); } @@ -1128,7 +1155,7 @@ private void saveIterator(VirtualFrame frame, PPickler pickler, Object iterator, } itemConsumer.accept(item); if (opcode != PickleUtils.NO_OPCODE) { - write(this, pickler, opcode); + pickler.write(this, opcode); } } } @@ -1162,16 +1189,16 @@ private boolean savePers(VirtualFrame frame, PPickler pickler, int proto, Object if (pid != PNone.NONE) { if (isBin(proto)) { save(frame, pickler, pid, 1); - write(this, pickler, PickleUtils.OPCODE_BINPERSID); + pickler.write(this, PickleUtils.OPCODE_BINPERSID); } else { final TruffleString pidStr = asString(convertToStr(frame, pid)); // XXX: Should it check whether the pid contains embedded newlines? if (!PythonUtils.isAscii(pidStr, ensureTsGetCodeRangeNode())) { throw raise(PythonBuiltinClassType.PicklingError, ErrorMessages.PIDS_MUST_BE_ASCII_STRS); } - write(this, pickler, PickleUtils.OPCODE_PERSID); - writeASCII(this, pickler, pidStr, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); - writeASCII(this, pickler, T_NEWLINE, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); + pickler.write(this, PickleUtils.OPCODE_PERSID); + pickler.writeASCII(this, pidStr, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); + pickler.writeASCII(this, T_NEWLINE, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); } return true; } @@ -1182,15 +1209,15 @@ private boolean savePers(VirtualFrame frame, PPickler pickler, int proto, Object private void memoGet(PPickler pickler, int proto, int value) { if (isBin(proto)) { if (value < 256) { - writeByteOp(this, pickler, PickleUtils.OPCODE_BINGET, value); + pickler.writeByteOp(this, PickleUtils.OPCODE_BINGET, value); } else if (Long.compareUnsigned(value, 0xffffffffL) <= 0) { - writeIntOp(this, pickler, PickleUtils.OPCODE_LONG_BINGET, value); + pickler.writeIntOp(this, PickleUtils.OPCODE_LONG_BINGET, value); } else { // unlikely throw raise(PicklingError, ErrorMessages.MEMO_ID_TOO_LARGE_FOR_S, "LONG_BINGET"); } } else { - writeLongAsString(this, pickler, PickleUtils.OPCODE_GET, value, ensureTsFromLongNode(), ensureTsCopyToByteArrayNode()); + pickler.writeLongAsString(this, PickleUtils.OPCODE_GET, value, ensureTsFromLongNode(), ensureTsCopyToByteArrayNode()); } } @@ -1204,15 +1231,15 @@ private void memoPut(PPickler pickler, int proto, boolean fastMode, Object obj) pickler.memo.set(obj, idx); if (proto >= 4) { - write(this, pickler, PickleUtils.OPCODE_MEMOIZE); + pickler.write(this, PickleUtils.OPCODE_MEMOIZE); } else { if (!isBin(proto)) { - writeLongAsString(this, pickler, PickleUtils.OPCODE_PUT, idx, ensureTsFromLongNode(), ensureTsCopyToByteArrayNode()); + pickler.writeLongAsString(this, PickleUtils.OPCODE_PUT, idx, ensureTsFromLongNode(), ensureTsCopyToByteArrayNode()); } else { if (idx < 256) { - writeByteOp(this, pickler, PickleUtils.OPCODE_BINPUT, idx); + pickler.writeByteOp(this, PickleUtils.OPCODE_BINPUT, idx); } else if (Long.compareUnsigned(idx, 0xffffffffL) <= 0) { - writeIntOp(this, pickler, PickleUtils.OPCODE_LONG_BINPUT, idx); + pickler.writeIntOp(this, PickleUtils.OPCODE_LONG_BINPUT, idx); } else { // unlikely throw raise(PicklingError, ErrorMessages.MEMO_ID_TOO_LARGE_FOR_S, "LONG_BINPUT"); @@ -1332,7 +1359,7 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, save(frame, pickler, cls, 0); save2(frame, pickler, args, 0); save3(frame, pickler, kwargs, 0); - write(this, pickler, PickleUtils.OPCODE_NEWOBJ_EX); + pickler.write(this, PickleUtils.OPCODE_NEWOBJ_EX); } else { PickleState st = getGlobalState(ctx.getCore()); final SequenceStorage argsStorage = getTupleStorage(args); @@ -1352,7 +1379,7 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, save(frame, pickler, callable, 0); save2(frame, pickler, PFactory.createEmptyTuple(PythonLanguage.get(this)), 0); - write(this, pickler, PickleUtils.OPCODE_REDUCE); + pickler.write(this, PickleUtils.OPCODE_REDUCE); } } else if (useNewobj) { profileSeen(SEEN_NEWOBJ); @@ -1414,13 +1441,13 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, } newargtup = createTuple(newargs); save2(frame, pickler, newargtup, 0); - write(this, pickler, PickleUtils.OPCODE_NEWOBJ); + pickler.write(this, PickleUtils.OPCODE_NEWOBJ); } else { profileSeen(SEEN_PLAIN_REDUCE); // Not using NEWOBJ save(frame, pickler, callable, 0); save2(frame, pickler, argtup, 0); - write(this, pickler, PickleUtils.OPCODE_REDUCE); + pickler.write(this, PickleUtils.OPCODE_REDUCE); } // obj can be NULL when save_reduce() is used directly. A NULL obj means the caller do @@ -1432,7 +1459,7 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, // memo int memoIndex = pickler.memo.get(obj); if (memoIndex != -1) { - write(this, pickler, PickleUtils.OPCODE_POP); + pickler.write(this, PickleUtils.OPCODE_POP); memoGet(pickler, proto, memoIndex); } else { memoPut(pickler, proto, fastMode, obj); @@ -1453,7 +1480,7 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, profileSeen(SEEN_REDUCE_STATE); if (stateSetter == null) { save(frame, pickler, state, 0); - write(this, pickler, PickleUtils.OPCODE_BUILD); + pickler.write(this, PickleUtils.OPCODE_BUILD); } else { // If a state_setter is specified, call it instead of load_build to update obj's // with its previous state. The first 4 save/write instructions push @@ -1465,40 +1492,40 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, save(frame, pickler, stateSetter, 0); save2(frame, pickler, obj, 0); save3(frame, pickler, state, 0); - write(this, pickler, PickleUtils.OPCODE_TUPLE2); - write(this, pickler, PickleUtils.OPCODE_REDUCE); - write(this, pickler, PickleUtils.OPCODE_POP); + pickler.write(this, PickleUtils.OPCODE_TUPLE2); + pickler.write(this, PickleUtils.OPCODE_REDUCE); + pickler.write(this, PickleUtils.OPCODE_POP); } } } private void saveNone(PPickler pickler) { - write(this, pickler, PickleUtils.OPCODE_NONE); + pickler.write(this, PickleUtils.OPCODE_NONE); } private void saveBool(PPickler pickler, int proto, boolean value) { if (proto >= 2) { - write(this, pickler, value ? PickleUtils.OPCODE_NEWTRUE : PickleUtils.OPCODE_NEWFALSE); + pickler.write(this, value ? PickleUtils.OPCODE_NEWTRUE : PickleUtils.OPCODE_NEWFALSE); } else { // These aren't opcodes -- they're ways to pickle bools before protocol 2 so that // unpicklers written before bools were introduced unpickle them as ints, but // unpicklers after can recognize that bools were intended. Note that protocol 2 // added direct ways to pickle bools. - writeASCII(this, pickler, value ? PickleUtils.T_PROTO_LE2_TRUE : PickleUtils.T_PROTO_LE2_FALSE, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); + pickler.writeASCII(this, value ? PickleUtils.T_PROTO_LE2_TRUE : PickleUtils.T_PROTO_LE2_FALSE, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); } } private void saveLong(PPickler pickler, int proto, int value) { if (isBin(proto)) { if (value >= 0 && value < (1 << 8)) { - writeByteOp(this, pickler, PickleUtils.OPCODE_BININT1, value); + pickler.writeByteOp(this, PickleUtils.OPCODE_BININT1, value); } else if (value >= 0 && value < (1 << 16)) { - writeShortOp(this, pickler, PickleUtils.OPCODE_BININT2, value); + pickler.writeShortOp(this, PickleUtils.OPCODE_BININT2, value); } else { - writeIntOp(this, pickler, PickleUtils.OPCODE_BININT, value); + pickler.writeIntOp(this, PickleUtils.OPCODE_BININT, value); } } else { - writeLongAsString(this, pickler, PickleUtils.OPCODE_INT, value, ensureTsFromLongNode(), ensureTsCopyToByteArrayNode()); + pickler.writeLongAsString(this, PickleUtils.OPCODE_INT, value, ensureTsFromLongNode(), ensureTsCopyToByteArrayNode()); } } @@ -1546,32 +1573,32 @@ private void saveLongLarge(VirtualFrame frame, PPickler pickler, int proto, Obje nbytes--; } if (nbytes < 256) { - writeByteOp(this, pickler, PickleUtils.OPCODE_LONG1, nbytes); + pickler.writeByteOp(this, PickleUtils.OPCODE_LONG1, nbytes); } else { - writeIntOp(this, pickler, PickleUtils.OPCODE_LONG4, nbytes); + pickler.writeIntOp(this, PickleUtils.OPCODE_LONG4, nbytes); } - write(this, pickler, pdata, nbytes); + pickler.write(this, pdata, nbytes); } else { TruffleString repr = repr(frame, obj); - write(this, pickler, PickleUtils.OPCODE_LONG); - writeASCII(this, pickler, asStringStrict(repr), ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); - writeASCII(this, pickler, T_L_NEW_LINE, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); + pickler.write(this, PickleUtils.OPCODE_LONG); + pickler.writeASCII(this, asStringStrict(repr), ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); + pickler.writeASCII(this, T_L_NEW_LINE, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); } } private void saveFloat(PPickler pickler, int proto, double value) { if (isBin(proto)) { - write(this, pickler, PickleUtils.OPCODE_BINFLOAT); - ensureBufferSpace(this, pickler, 8); + pickler.write(this, PickleUtils.OPCODE_BINFLOAT); + pickler.ensureBufferSpace(this, 8); NumericSupport.bigEndian().putDouble(pickler.outputBuffer, pickler.outputLen, value); pickler.outputLen += 8; } else { - write(this, pickler, PickleUtils.OPCODE_FLOAT); + pickler.write(this, PickleUtils.OPCODE_FLOAT); TruffleString repr = PickleUtils.doubleToAsciiString(value); - writeASCII(this, pickler, repr, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); - writeASCII(this, pickler, T_NEWLINE, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); + pickler.writeASCII(this, repr, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); + pickler.writeASCII(this, T_NEWLINE, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); } } @@ -1613,12 +1640,15 @@ private void saveBytesData(VirtualFrame frame, PPickler pickler, int proto, bool assert proto >= 3; int size = bufferLib.getBufferLength(buffer); boolean wasFraming = pickler.framing; - boolean bypassBuffer = bypassBuffer(pickler, size); + boolean bypassBuffer = pickler.bypassBuffer(size); + if (bypassBuffer) { + profileSeen(SEEN_BYPASS_BUFFER); + } if (size <= 0xff) { - writeByteOp(this, pickler, PickleUtils.OPCODE_SHORT_BINBYTES, size); + pickler.writeByteOp(this, PickleUtils.OPCODE_SHORT_BINBYTES, size); } else { - writeIntOp(this, pickler, PickleUtils.OPCODE_BINBYTES, size); + pickler.writeIntOp(this, PickleUtils.OPCODE_BINBYTES, size); } if (bypassBuffer && pickler.write != null) { @@ -1626,7 +1656,7 @@ private void saveBytesData(VirtualFrame frame, PPickler pickler, int proto, bool ensureCallNode().execute(frame, pickler.write, obj); pickler.clearBuffer(); } else { - writeBuffer(this, pickler, buffer, size, bufferLib); + pickler.writeBuffer(this, buffer, size, bufferLib); } if (bypassBuffer && wasFraming) { @@ -1641,10 +1671,13 @@ private void writeUnicodeBinary(Node node, VirtualFrame frame, PPickler pickler, TruffleString utf8 = switchEncodingNode.execute(string, TruffleString.Encoding.UTF_8, TranscodingErrorHandler.DEFAULT_KEEP_SURROGATES_IN_UTF8); int size = utf8.byteLength(TruffleString.Encoding.UTF_8); boolean wasFraming = pickler.framing; - boolean bypassBuffer = bypassBuffer(pickler, size); + boolean bypassBuffer = pickler.bypassBuffer(size); + if (bypassBuffer) { + profileSeen(SEEN_BYPASS_BUFFER); + } if (bypassBuffer && pickler.write != null) { - writeIntOp(node, pickler, PickleUtils.OPCODE_BINUNICODE, size); + pickler.writeIntOp(node, PickleUtils.OPCODE_BINUNICODE, size); byte[] data = new byte[size]; copyToByteArrayNode.execute(utf8, 0, data, 0, size, TruffleString.Encoding.UTF_8); flushToFileNode.execute(frame, pickler); @@ -1652,11 +1685,11 @@ private void writeUnicodeBinary(Node node, VirtualFrame frame, PPickler pickler, pickler.clearBuffer(); } else { if (size <= 0xff && proto >= 4) { - writeByteOp(node, pickler, PickleUtils.OPCODE_SHORT_BINUNICODE, size); + pickler.writeByteOp(node, PickleUtils.OPCODE_SHORT_BINUNICODE, size); } else { - writeIntOp(node, pickler, PickleUtils.OPCODE_BINUNICODE, size); + pickler.writeIntOp(node, PickleUtils.OPCODE_BINUNICODE, size); } - writeUtf8Converted(node, pickler, utf8, copyToByteArrayNode); + pickler.writeUtf8Converted(node, utf8, copyToByteArrayNode); } if (bypassBuffer && wasFraming) { @@ -1676,45 +1709,13 @@ private void saveUnicode(VirtualFrame frame, PPickler pickler, int proto, boolea if (isBin(proto)) { writeUnicodeBinary(this, frame, pickler, proto, string, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode(), ensureFlushToFileNode(), ensureCallNode()); } else { - write(this, pickler, PickleUtils.OPCODE_UNICODE); - writeRawUnicodeEscape(this, pickler, string, ensureTsCodePointLengthNode(), ensureTsCodePointAtIndexUTF32Node()); - writeASCII(this, pickler, T_NEWLINE, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); + pickler.write(this, PickleUtils.OPCODE_UNICODE); + pickler.writeRawUnicodeEscape(this, string, ensureTsCodePointLengthNode(), ensureTsCodePointAtIndexUTF32Node()); + pickler.writeASCII(this, T_NEWLINE, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); } memoPut(pickler, proto, fastMode, obj); } - private static void writeRawUnicodeEscape(Node node, PPickler pickler, TruffleString string, - TruffleString.CodePointLengthNode codePointLengthNode, TruffleString.CodePointAtIndexUTF32Node codePointAtIndexNode) { - int codePointLength = codePointLengthNode.execute(string, TS_ENCODING); - long escapedLength = 0; - for (int i = 0; i < codePointLength; i++) { - int ch = codePointAtIndexNode.execute(string, i); - escapedLength += ch >= 0x10000 ? 10 : ch >= 256 || ch == '\\' || ch == 0 || ch == '\n' || ch == '\r' || ch == 0x1a ? 6 : 1; - } - if (escapedLength > Integer.MAX_VALUE) { - throw PRaiseNode.raiseStatic(node, PythonBuiltinClassType.MemoryError); - } - ensureBufferSpace(node, pickler, (int) escapedLength); - for (int i = 0; i < codePointLength; i++) { - int ch = codePointAtIndexNode.execute(string, i); - if (ch >= 0x10000) { - pickler.outputBuffer[pickler.outputLen++] = '\\'; - pickler.outputBuffer[pickler.outputLen++] = 'U'; - for (int shift = 28; shift >= 0; shift -= 4) { - pickler.outputBuffer[pickler.outputLen++] = BytesUtils.HEXDIGITS[(ch >> shift) & 0xf]; - } - } else if (ch >= 256 || ch == '\\' || ch == 0 || ch == '\n' || ch == '\r' || ch == 0x1a) { - pickler.outputBuffer[pickler.outputLen++] = '\\'; - pickler.outputBuffer[pickler.outputLen++] = 'u'; - for (int shift = 12; shift >= 0; shift -= 4) { - pickler.outputBuffer[pickler.outputLen++] = BytesUtils.HEXDIGITS[(ch >> shift) & 0xf]; - } - } else { - pickler.outputBuffer[pickler.outputLen++] = (byte) ch; - } - } - } - private void batchDictExact(VirtualFrame frame, PPickler pickler, PDict dict) { final HashingStorage storage = dict.getDictStorage(); final int length = getHashingStorageLength(storage); @@ -1728,7 +1729,7 @@ private void batchDictExact(VirtualFrame frame, PPickler pickler, PDict dict) { HashingStorageIteratorValue getValueNode = ensureHashingStorageIteratorValue(); save(frame, pickler, getKeyNode.executeCached(storage, it), 0); save(frame, pickler, getValueNode.executeCached(storage, it), 0); - write(this, pickler, PickleUtils.OPCODE_SETITEM); + pickler.write(this, PickleUtils.OPCODE_SETITEM); } else { // Write in batches of BATCHSIZE. saveDictHashingStorageBatched(frame, pickler, storage); @@ -1753,9 +1754,9 @@ private void saveDict(VirtualFrame frame, Node inliningTarget, PyObjectCallMetho // Create an empty dict. if (isBin(proto)) { - write(this, pickler, PickleUtils.OPCODE_EMPTY_DICT); + pickler.write(this, PickleUtils.OPCODE_EMPTY_DICT); } else { - write(this, pickler, PickleUtils.OPCODE_MARK, PickleUtils.OPCODE_DICT); + pickler.write(this, PickleUtils.OPCODE_MARK, PickleUtils.OPCODE_DICT); } memoPut(pickler, proto, fastMode, obj); @@ -1783,7 +1784,7 @@ private void saveSet(VirtualFrame frame, PythonContext ctx, PPickler pickler, in return; } - write(this, pickler, PickleUtils.OPCODE_EMPTY_SET); + pickler.write(this, PickleUtils.OPCODE_EMPTY_SET); memoPut(pickler, proto, fastMode, obj); HashingStorage storage = obj.getDictStorage(); @@ -1808,7 +1809,7 @@ private void saveFrozenset(VirtualFrame frame, PythonContext ctx, PPickler pickl return; } - write(this, pickler, PickleUtils.OPCODE_MARK); + pickler.write(this, PickleUtils.OPCODE_MARK); saveSetHashingStorage(frame, pickler, obj.getDictStorage()); @@ -1816,12 +1817,12 @@ private void saveFrozenset(VirtualFrame frame, PythonContext ctx, PPickler pickl // away everything we put on the stack, and fetch the object back from the memo. int memoIndex = pickler.memo.get(obj); if (memoIndex != -1) { - write(this, pickler, PickleUtils.OPCODE_POP_MARK); + pickler.write(this, PickleUtils.OPCODE_POP_MARK); memoGet(pickler, proto, memoIndex); return; } - write(this, pickler, PickleUtils.OPCODE_FROZENSET); + pickler.write(this, PickleUtils.OPCODE_FROZENSET); memoPut(pickler, proto, fastMode, obj); } @@ -1847,7 +1848,7 @@ private void batchListExact(VirtualFrame frame, PPickler pickler, PList obj) { if (storage.length() == 1) { item = getItem(storage, 0); save(frame, pickler, item, 0); - write(this, pickler, PickleUtils.OPCODE_APPEND); + pickler.write(this, PickleUtils.OPCODE_APPEND); return; } @@ -1856,7 +1857,7 @@ private void batchListExact(VirtualFrame frame, PPickler pickler, PList obj) { int thisBatch; do { thisBatch = 0; - write(this, pickler, PickleUtils.OPCODE_MARK); + pickler.write(this, PickleUtils.OPCODE_MARK); while (total < storage.length()) { item = getItem(storage, total); save(frame, pickler, item, 0); @@ -1865,7 +1866,7 @@ private void batchListExact(VirtualFrame frame, PPickler pickler, PList obj) { break; } } - write(this, pickler, PickleUtils.OPCODE_APPENDS); + pickler.write(this, PickleUtils.OPCODE_APPENDS); } while (total < storage.length()); } @@ -1874,19 +1875,19 @@ private void batchIntList(VirtualFrame frame, PPickler pickler, IntSequenceStora if (storage.length() == 1) { opcodeBoundary(frame, pickler); saveLong(pickler, proto, storage.getIntItemNormalized(0)); - write(this, pickler, PickleUtils.OPCODE_APPEND); + pickler.write(this, PickleUtils.OPCODE_APPEND); return; } int length = storage.length(); for (int batchStart = 0; batchStart < length; batchStart += PickleUtils.BATCHSIZE) { int batchEnd = batchStart + Math.min(PickleUtils.BATCHSIZE, length - batchStart); - write(this, pickler, PickleUtils.OPCODE_MARK); + pickler.write(this, PickleUtils.OPCODE_MARK); for (int i = batchStart; i < batchEnd; i++) { opcodeBoundary(frame, pickler); saveLong(pickler, proto, storage.getIntItemNormalized(i)); } - write(this, pickler, PickleUtils.OPCODE_APPENDS); + pickler.write(this, PickleUtils.OPCODE_APPENDS); } } @@ -1895,19 +1896,19 @@ private void batchLongList(VirtualFrame frame, PPickler pickler, LongSequenceSto if (storage.length() == 1) { opcodeBoundary(frame, pickler); saveLong(frame, pickler, proto, storage.getLongItemNormalized(0)); - write(this, pickler, PickleUtils.OPCODE_APPEND); + pickler.write(this, PickleUtils.OPCODE_APPEND); return; } int length = storage.length(); for (int batchStart = 0; batchStart < length; batchStart += PickleUtils.BATCHSIZE) { int batchEnd = batchStart + Math.min(PickleUtils.BATCHSIZE, length - batchStart); - write(this, pickler, PickleUtils.OPCODE_MARK); + pickler.write(this, PickleUtils.OPCODE_MARK); for (int i = batchStart; i < batchEnd; i++) { opcodeBoundary(frame, pickler); saveLong(frame, pickler, proto, storage.getLongItemNormalized(i)); } - write(this, pickler, PickleUtils.OPCODE_APPENDS); + pickler.write(this, PickleUtils.OPCODE_APPENDS); } } @@ -1916,19 +1917,19 @@ private void batchDoubleList(VirtualFrame frame, PPickler pickler, DoubleSequenc if (storage.length() == 1) { opcodeBoundary(frame, pickler); saveFloat(pickler, proto, storage.getDoubleItemNormalized(0)); - write(this, pickler, PickleUtils.OPCODE_APPEND); + pickler.write(this, PickleUtils.OPCODE_APPEND); return; } int length = storage.length(); for (int batchStart = 0; batchStart < length; batchStart += PickleUtils.BATCHSIZE) { int batchEnd = batchStart + Math.min(PickleUtils.BATCHSIZE, length - batchStart); - write(this, pickler, PickleUtils.OPCODE_MARK); + pickler.write(this, PickleUtils.OPCODE_MARK); for (int i = batchStart; i < batchEnd; i++) { opcodeBoundary(frame, pickler); saveFloat(pickler, proto, storage.getDoubleItemNormalized(i)); } - write(this, pickler, PickleUtils.OPCODE_APPENDS); + pickler.write(this, PickleUtils.OPCODE_APPENDS); } } @@ -1950,9 +1951,9 @@ private void saveList(VirtualFrame frame, PPickler pickler, int proto, boolean f // Create an empty list. if (isBin(proto)) { - write(this, pickler, PickleUtils.OPCODE_EMPTY_LIST); + pickler.write(this, PickleUtils.OPCODE_EMPTY_LIST); } else { - write(this, pickler, PickleUtils.OPCODE_MARK, PickleUtils.OPCODE_LIST); + pickler.write(this, PickleUtils.OPCODE_MARK, PickleUtils.OPCODE_LIST); } int len = obj.getSequenceStorage().length(); @@ -2033,9 +2034,9 @@ private void saveTuple(VirtualFrame frame, PPickler pickler, int proto, boolean if (len == 0) { if (proto != 0) { - write(this, pickler, PickleUtils.OPCODE_EMPTY_TUPLE); + pickler.write(this, PickleUtils.OPCODE_EMPTY_TUPLE); } else { - write(this, pickler, PickleUtils.OPCODE_MARK, PickleUtils.OPCODE_TUPLE); + pickler.write(this, PickleUtils.OPCODE_MARK, PickleUtils.OPCODE_TUPLE); } return; } @@ -2051,14 +2052,14 @@ private void saveTuple(VirtualFrame frame, PPickler pickler, int proto, boolean if (memoIndex != -1) { // pop the len elements for (int i = 0; i < len; i++) { - write(this, pickler, PickleUtils.OPCODE_POP); + pickler.write(this, PickleUtils.OPCODE_POP); } // fetch from memo memoGet(pickler, proto, memoIndex); return; } else { // Not recursive. - write(this, pickler, TUPLE_LEN_2_OPCODE[len]); + pickler.write(this, TUPLE_LEN_2_OPCODE[len]); } memoPut(pickler, proto, fastMode, obj); @@ -2066,18 +2067,18 @@ private void saveTuple(VirtualFrame frame, PPickler pickler, int proto, boolean } // proto < 2 and len > 0, or proto >= 2 and len > 3. Generate MARK e1 e2 ... TUPLE - write(this, pickler, PickleUtils.OPCODE_MARK); + pickler.write(this, PickleUtils.OPCODE_MARK); storeTupleElements(frame, pickler, storage, len); int memoIndex = pickler.memo.get(obj); if (memoIndex != -1) { // pop the stack stuff we pushed if (isBin(proto)) { - write(this, pickler, PickleUtils.OPCODE_POP_MARK); + pickler.write(this, PickleUtils.OPCODE_POP_MARK); } else { // Note that we pop one more than len, to remove the MARK too. for (int i = 0; i <= len; i++) { - write(this, pickler, PickleUtils.OPCODE_POP); + pickler.write(this, PickleUtils.OPCODE_POP); } } // fetch from memo @@ -2085,7 +2086,7 @@ private void saveTuple(VirtualFrame frame, PPickler pickler, int proto, boolean return; } else { // Not recursive. - write(this, pickler, PickleUtils.OPCODE_TUPLE); + pickler.write(this, PickleUtils.OPCODE_TUPLE); } memoPut(pickler, proto, fastMode, obj); @@ -2096,10 +2097,13 @@ private void saveBytearrayData(VirtualFrame frame, PPickler pickler, int proto, int size = bufferLib.getBufferLength(buffer); boolean wasFraming = pickler.framing; - boolean bypassBuffer = bypassBuffer(pickler, size); + boolean bypassBuffer = pickler.bypassBuffer(size); + if (bypassBuffer) { + profileSeen(SEEN_BYPASS_BUFFER); + } // Our sizes are ints, but the protocol expects 8 bytes - ensureBufferSpace(this, pickler, 9); + pickler.ensureBufferSpace(this, 9); pickler.outputBuffer[pickler.outputLen++] = PickleUtils.OPCODE_BYTEARRAY8; PickleUtils.writeSize64(pickler.outputBuffer, pickler.outputLen, size); pickler.outputLen += 8; @@ -2109,7 +2113,7 @@ private void saveBytearrayData(VirtualFrame frame, PPickler pickler, int proto, ensureCallNode().execute(frame, pickler.write, obj); pickler.clearBuffer(); } else { - writeBuffer(this, pickler, buffer, size, bufferLib); + pickler.writeBuffer(this, buffer, size, bufferLib); } if (bypassBuffer && wasFraming) { @@ -2168,9 +2172,9 @@ private void savePickleBuffer(VirtualFrame frame, PPickler pickler, int proto, b } } else { // Write data out-of-band - write(this, pickler, PickleUtils.OPCODE_NEXT_BUFFER); + pickler.write(this, PickleUtils.OPCODE_NEXT_BUFFER); if (readOnly) { - write(this, pickler, PickleUtils.OPCODE_READONLY_BUFFER); + pickler.write(this, PickleUtils.OPCODE_READONLY_BUFFER); } } } @@ -2250,11 +2254,11 @@ private void saveGlobal(VirtualFrame frame, PythonContext ctx, PPickler pickler, // Generate an EXT opcode if (code <= 0xff) { - writeByteOp(this, pickler, PickleUtils.OPCODE_EXT1, code); + pickler.writeByteOp(this, PickleUtils.OPCODE_EXT1, code); } else if (code <= 0xffff) { - writeShortOp(this, pickler, PickleUtils.OPCODE_EXT2, code); + pickler.writeShortOp(this, PickleUtils.OPCODE_EXT2, code); } else { - writeIntOp(this, pickler, PickleUtils.OPCODE_EXT4, code); + pickler.writeIntOp(this, PickleUtils.OPCODE_EXT4, code); } } } else { @@ -2271,14 +2275,14 @@ private void saveGlobal(VirtualFrame frame, PythonContext ctx, PPickler pickler, if (proto >= 4) { save(frame, pickler, moduleName, 0); save(frame, pickler, globalName, 0); - write(this, pickler, PickleUtils.OPCODE_STACK_GLOBAL); + pickler.write(this, PickleUtils.OPCODE_STACK_GLOBAL); } else if (parent != module) { Object reduceValue = createTuple(st.getattr, createTuple(parent, lastname)); saveReduce(frame, ctx, pickler, proto, fastMode, reduceValue, null); } else { // Generate a normal global opcode if we are using a pickle protocol < 4, or if // the object is not registered in the extension registry - write(this, pickler, PickleUtils.OPCODE_GLOBAL); + pickler.write(this, PickleUtils.OPCODE_GLOBAL); // For protocol < 3 and if the user didn't request against doing so, we convert // module names to the old 2.x module names. if (proto < 3 && pickler.fixImports) { @@ -2296,19 +2300,19 @@ private void saveGlobal(VirtualFrame frame, PythonContext ctx, PPickler pickler, TS_ENCODING) == TruffleString.CodeRange.BROKEN) { throw raise(PicklingError, ErrorMessages.CANT_PICKLE_MODULE_S_USING_PROTO_D, moduleName, proto); } - writeUtf8(this, pickler, moduleName, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); - write(this, pickler, (byte) '\n'); - writeUtf8(this, pickler, globalName, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); + pickler.writeUtf8(this, moduleName, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); + pickler.write(this, (byte) '\n'); + pickler.writeUtf8(this, globalName, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); } else { if (ensureTsGetCodeRangeNode().execute(moduleName, TS_ENCODING) != TruffleString.CodeRange.ASCII || ensureTsGetCodeRangeNode().execute(globalName, TS_ENCODING) != TruffleString.CodeRange.ASCII) { throw raise(PicklingError, ErrorMessages.CANT_PICKLE_MODULE_S_USING_PROTO_D, moduleName, proto); } - writeASCII(this, pickler, moduleName, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); - write(this, pickler, (byte) '\n'); - writeASCII(this, pickler, globalName, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); + pickler.writeASCII(this, moduleName, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); + pickler.write(this, (byte) '\n'); + pickler.writeASCII(this, globalName, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); } - write(this, pickler, (byte) '\n'); + pickler.write(this, (byte) '\n'); } // Memoize the object memoPut(pickler, proto, fastMode, obj); @@ -2544,7 +2548,7 @@ public void dump(VirtualFrame frame, PPickler pickler, Object obj, assert pickler.proto <= 256; try { - SaveNode.write(this, pickler, PickleUtils.OPCODE_PROTO, (byte) pickler.proto); + pickler.write(this, PickleUtils.OPCODE_PROTO, (byte) pickler.proto); } catch (Exception e) { SaveNode.handleError(pickler); throw e; @@ -2556,7 +2560,7 @@ public void dump(VirtualFrame frame, PPickler pickler, Object obj, } saveNode.execute(frame, pickler, obj, 0); - SaveNode.write(this, pickler, PickleUtils.OPCODE_STOP); + pickler.write(this, PickleUtils.OPCODE_STOP); pickler.commitFrame(); } finally { pickler.framing = false; From 7bfa0b69dc73edc654a5496c409ad4d57a83dd14 Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Thu, 1 Oct 2026 13:22:31 +0200 Subject: [PATCH 20/39] Get rid of PPickler.maxOutputLen --- .../builtins/modules/pickle/PPickler.java | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java index 646f5f925a..ce6beb5fba 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java @@ -164,8 +164,6 @@ public class PPickler extends PythonBuiltinObject { private byte[] outputBuffer; // Length of output_buffer private int outputLen; - // Allocation size of output_buffer - private int maxOutputLen; // Pickle protocol number, >= 0 private int proto; // true if proto > 0 @@ -203,12 +201,11 @@ public PPickler(Object cls, Shape instanceShape, int outputCapacity, int memoCap fast = 0; fastNesting = 0; fixImports = false; - maxOutputLen = outputCapacity; outputLen = 0; reducerOverride = null; memo = new MemoTable(memoCapacity); - outputBuffer = new byte[maxOutputLen]; + outputBuffer = new byte[outputCapacity]; } @TruffleBoundary @@ -265,7 +262,7 @@ public MemoTable getMemo() { } public int getMaxOutputLen() { - return maxOutputLen; + return outputBuffer.length; } public void setMemo(MemoTable memo) { @@ -350,8 +347,7 @@ public void initInternals(VirtualFrame frame, Node inliningTarget, PyObjectLooku this.outputLen = 0; if (this.outputBuffer == null) { - this.maxOutputLen = PickleUtils.WRITE_BUF_SIZE; - this.outputBuffer = new byte[this.maxOutputLen]; + this.outputBuffer = new byte[PickleUtils.WRITE_BUF_SIZE]; } this.fast = 0; @@ -372,7 +368,7 @@ public void clearMemo() { } public void clearBuffer() { - this.outputBuffer = new byte[this.maxOutputLen]; + this.outputBuffer = new byte[this.outputBuffer.length]; this.outputLen = 0; this.frameStart = -1; } @@ -501,15 +497,15 @@ private void ensureBufferSpace(Node node, int dataLen) { int n = CompilerDirectives.injectBranchProbability(CompilerDirectives.SLOWPATH_PROBABILITY, needNewFrame) ? dataLen + PickleUtils.FRAME_HEADER_SIZE : dataLen; int required = outputLen + n; - if (CompilerDirectives.injectBranchProbability(CompilerDirectives.SLOWPATH_PROBABILITY, required > maxOutputLen)) { + if (CompilerDirectives.injectBranchProbability(CompilerDirectives.SLOWPATH_PROBABILITY, required > outputBuffer.length)) { // Make place in buffer for the pickle chunk // TODO: when GR-24978 is completed we should use PY_SSIZE_T_MAX if (CompilerDirectives.injectBranchProbability(CompilerDirectives.SLOWPATH_PROBABILITY, outputLen >= Integer.MAX_VALUE / 2 - n)) { CompilerDirectives.transferToInterpreter(); throw PRaiseNode.raiseStatic(node, PythonBuiltinClassType.MemoryError); } - maxOutputLen = (outputLen + n) / 2 * 3; - outputBuffer = PythonUtils.arrayCopyOf(outputBuffer, maxOutputLen); + int newCapacity = (required) / 2 * 3; + outputBuffer = PythonUtils.arrayCopyOf(outputBuffer, newCapacity); } if (CompilerDirectives.injectBranchProbability(CompilerDirectives.SLOWPATH_PROBABILITY, needNewFrame)) { frameStart = outputLen; From 5f78bd2280cad5722fa03191de0711da250fdf12 Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Thu, 1 Oct 2026 14:16:05 +0200 Subject: [PATCH 21/39] Get rid of ByteArrayView --- .../modules/pickle/ByteArrayView.java | 105 ------------------ .../builtins/modules/pickle/PPickler.java | 8 +- .../builtins/modules/pickle/PUnpickler.java | 101 ++++++++--------- .../builtins/modules/pickle/PickleUtils.java | 17 ++- .../builtins/modules/pickle/PicklerNodes.java | 5 +- .../builtins/objects/ints/IntNodes.java | 26 +++-- 6 files changed, 80 insertions(+), 182 deletions(-) delete mode 100644 graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/ByteArrayView.java diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/ByteArrayView.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/ByteArrayView.java deleted file mode 100644 index 34dd8b26cf..0000000000 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/ByteArrayView.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * The Universal Permissive License (UPL), Version 1.0 - * - * Subject to the condition set forth below, permission is hereby granted to any - * person obtaining a copy of this software, associated documentation and/or - * data (collectively the "Software"), free of charge and under any and all - * copyright rights in the Software, and any and all patent rights owned or - * freely licensable by each licensor hereunder covering either (i) the - * unmodified Software as contributed to or provided by such licensor, or (ii) - * the Larger Works (as defined below), to deal in both - * - * (a) the Software, and - * - * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if - * one is included with the Software each a "Larger Work" to which the Software - * is contributed by such licensors), - * - * without restriction, including without limitation the rights to copy, create - * derivative works of, display, perform, and distribute the Software and make, - * use, sell, offer for sale, import, export, have made, and have sold the - * Software and the Larger Work(s), and to sublicense the foregoing rights on - * either these or other terms. - * - * This license is subject to the following condition: - * - * The above copyright notice and either this complete permission notice or at a - * minimum a reference to the UPL must be included in all copies or substantial - * portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -package com.oracle.graal.python.builtins.modules.pickle; - -import com.oracle.graal.python.util.PythonUtils; - -public class ByteArrayView { - private final byte[] bytes; - private int offset; - - public ByteArrayView(byte[] bytes) { - this(bytes, 0); - } - - public ByteArrayView(byte[] bytes, int offset) { - this.bytes = bytes; - this.offset = offset; - validateOffset(0); - } - - private void validateOffset(int n) { - assert offset + n >= 0 && offset + n <= bytes.length; - } - - public void add(int n) { - validateOffset(n); - this.offset += n; - } - - public void sub(int n) { - this.add(-n); - } - - public byte[] getBytes(int len) { - assert len >= 0 && (len + offset) >= 0 && (len + offset) <= bytes.length; - if (offset == 0 && len == bytes.length) { - return bytes; - } - byte[] arr = new byte[len]; - PythonUtils.arraycopy(this.bytes, this.offset, arr, 0, len); - return arr; - } - - public int getOffset() { - return offset; - } - - public byte get(int i) { - return this.bytes[this.offset + i]; - } - - public int getUnsigned(int i) { - return get(i) & 0xff; - } - - public void put(int i, byte value) { - this.bytes[this.offset + i] = value; - } - - public void writeSize64(int value) { - PickleUtils.writeSize64(bytes, offset, value); - } - - public void memmove(int off, int num) { - PythonUtils.arraycopy(this.bytes, this.offset + off, this.bytes, this.offset, num); - } -} diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java index ce6beb5fba..73114ca26d 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java @@ -379,13 +379,11 @@ public void commitFrame() { } int frameLen = outputLen - frameStart - PickleUtils.FRAME_HEADER_SIZE; - ByteArrayView qdata = new ByteArrayView(outputBuffer, frameStart); if (frameLen >= PickleUtils.FRAME_SIZE_MIN) { - qdata.put(0, PickleUtils.OPCODE_FRAME); - qdata.add(1); - qdata.writeSize64(frameLen); + outputBuffer[frameStart] = PickleUtils.OPCODE_FRAME; + PickleUtils.writeSize64(outputBuffer, frameStart + 1, frameLen); } else { - qdata.memmove(PickleUtils.FRAME_HEADER_SIZE, frameLen); + PythonUtils.arraycopy(outputBuffer, frameStart + PickleUtils.FRAME_HEADER_SIZE, outputBuffer, frameStart, frameLen); outputLen -= PickleUtils.FRAME_HEADER_SIZE; } frameStart = -1; diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PUnpickler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PUnpickler.java index 85ab4c5bba..0b437491b9 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PUnpickler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PUnpickler.java @@ -534,19 +534,20 @@ protected int readFromFile(VirtualFrame frame, PUnpickler self, int n) { } protected byte read(VirtualFrame frame, PUnpickler self) { - return read(frame, self, 1).get(0); + int offset = read(frame, self, 1); + return self.inputBuffer[offset]; } - protected ByteArrayView read(VirtualFrame frame, PUnpickler self, int n) { + protected int read(VirtualFrame frame, PUnpickler self, int n) { if (n <= self.inputLen - self.nextReadIdx) { - ByteArrayView bytesView = new ByteArrayView(self.inputBuffer, self.nextReadIdx); + int offset = self.nextReadIdx; self.nextReadIdx += n; - return bytesView; + return offset; } return readImpl(frame, self, n); } - private ByteArrayView readImpl(VirtualFrame frame, PUnpickler self, int n) { + private int readImpl(VirtualFrame frame, PUnpickler self, int n) { int numRead; // TODO: when GR-24978 is completed we should use PY_SSIZE_T_MAX if (self.nextReadIdx > Integer.MAX_VALUE - 1) { @@ -563,7 +564,7 @@ private ByteArrayView readImpl(VirtualFrame frame, PUnpickler self, int n) { throw badReadLine(); } self.nextReadIdx = n; - return new ByteArrayView(self.inputBuffer); + return 0; } protected int readInto(VirtualFrame frame, PUnpickler self, byte[] buffer) { @@ -722,12 +723,12 @@ protected boolean isTuple(Object object) { return tupleCheck.execute(object); } - protected Object longFromBytes(byte[] data, boolean littleEndian) { + protected Object longFromBytes(byte[] data, int offset, int length, boolean littleEndian) { if (pyLongFromByteArray == null) { CompilerDirectives.transferToInterpreterAndInvalidate(); pyLongFromByteArray = insert(IntNodesFactory.PyLongFromByteArrayNodeGen.create()); } - return pyLongFromByteArray.executeCached(data, littleEndian, true); + return pyLongFromByteArray.executeCached(data, offset, length, littleEndian, true); } protected void setAttribute(VirtualFrame frame, Object object, Object key, Object value) { @@ -809,11 +810,11 @@ void loadNone(PUnpickler self) { pDataPush(self, PNone.NONE); } - private static long calcBinInt(ByteArrayView s, int nBytes) { + private static long calcBinInt(byte[] s, int offset, int nBytes) { long x = 0; for (int i = 0; i < nBytes; i++) { - x |= (long) s.getUnsigned(i) << (8 * i); + x |= (long) (s[offset + i] & 0xff) << (8 * i); } // Unlike BININT1 and BININT2, BININT (more accurately BININT4) is signed, so on a box @@ -826,7 +827,7 @@ private static long calcBinInt(ByteArrayView s, int nBytes) { return x; } - private int calcBinSize(ByteArrayView s, int nbytes) { + private int calcBinSize(byte[] s, int offset, int nbytes) { int i; int x = 0; int n = nbytes; @@ -836,7 +837,7 @@ private int calcBinSize(ByteArrayView s, int nbytes) { // Check for integer overflow. BINBYTES8 and BINUNICODE8 opcodes have 64-bit size // that can't be represented on 32-bit platform. for (i = Integer.BYTES; i < n; i++) { - if (s.get(i) != 0) { + if (s[offset + i] != 0) { throw raise(PythonBuiltinClassType.OverflowError); } } @@ -844,7 +845,7 @@ private int calcBinSize(ByteArrayView s, int nbytes) { } for (i = 0; i < n; i++) { - x |= s.getUnsigned(i) << (8 * i); + x |= (s[offset + i] & 0xff) << (8 * i); } // TODO: GR-24978 check for PY_SSIZE_T_MAX (see: _cpickle.c:calc_binsize) @@ -855,24 +856,24 @@ private int calcBinSize(ByteArrayView s, int nbytes) { return x; } - private void loadBinIntX(PUnpickler self, ByteArrayView s, int size) { - long x = calcBinInt(s, size); + private void loadBinIntX(PUnpickler self, int offset, int size) { + long x = calcBinInt(self.inputBuffer, offset, size); pDataPush(self, x); } private void loadBinInt(VirtualFrame frame, PUnpickler self) { - final ByteArrayView s = read(frame, self, 4); - loadBinIntX(self, s, 4); + int offset = read(frame, self, 4); + loadBinIntX(self, offset, 4); } private void loadBinInt1(VirtualFrame frame, PUnpickler self) { - final ByteArrayView s = read(frame, self, 1); - loadBinIntX(self, s, 1); + int offset = read(frame, self, 1); + loadBinIntX(self, offset, 1); } private void loadBinInt2(VirtualFrame frame, PUnpickler self) { - final ByteArrayView s = read(frame, self, 2); - loadBinIntX(self, s, 2); + int offset = read(frame, self, 2); + loadBinIntX(self, offset, 2); } private void loadInt(VirtualFrame frame, PUnpickler self) { @@ -922,8 +923,8 @@ private void loadLong(VirtualFrame frame, PUnpickler self) { private void loadCountedLong(VirtualFrame frame, PUnpickler self, int n) { assert n == 1 || n == 4; int size = n; - final ByteArrayView nbytes = read(frame, self, size); - size = (int) calcBinInt(nbytes, size); + int sizeOffset = read(frame, self, size); + size = (int) calcBinInt(self.inputBuffer, sizeOffset, size); Object value; @@ -935,8 +936,8 @@ private void loadCountedLong(VirtualFrame frame, PUnpickler self, int n) { value = 0L; } else { // Read the raw little-endian bytes and convert. - final ByteArrayView pdata = read(frame, self, size); - value = longFromBytes(pdata.getBytes(size), true); + int dataOffset = read(frame, self, size); + value = longFromBytes(self.inputBuffer, dataOffset, size, true); } pDataPush(self, value); } @@ -956,15 +957,15 @@ private void loadFloat(VirtualFrame frame, PUnpickler self) { private void loadBinFloat(VirtualFrame frame, PUnpickler self) { Object value; - ByteArrayView s = read(frame, self, 8); + int offset = read(frame, self, 8); - value = NumericSupport.bigEndian().getDouble(s.getBytes(Double.BYTES), 0); + value = NumericSupport.bigEndian().getDouble(self.inputBuffer, offset); pDataPush(self, value); } private void loadCountedBinBytes(VirtualFrame frame, PUnpickler self, int nbytes) { - final ByteArrayView s = read(frame, self, nbytes); - int size = calcBinSize(s, nbytes); + int offset = read(frame, self, nbytes); + int size = calcBinSize(self.inputBuffer, offset, nbytes); if (size < 0) { throw raise(PythonBuiltinClassType.OverflowError, ErrorMessages.S_EXCEEDS_MAX_SIZE_N_BYTES, "BINBYTES", Integer.MAX_VALUE); } @@ -977,8 +978,8 @@ private void loadCountedBinBytes(VirtualFrame frame, PUnpickler self, int nbytes } private void loadCountedByteArray(VirtualFrame frame, PUnpickler self) { - final ByteArrayView s = read(frame, self, 8); - int size = calcBinSize(s, 8); + int offset = read(frame, self, 8); + int size = calcBinSize(self.inputBuffer, offset, 8); if (size < 0) { throw raise(PythonBuiltinClassType.OverflowError, ErrorMessages.S_EXCEEDS_MAX_SIZE_N_BYTES, "BYTEARRAY8", Integer.MAX_VALUE); } @@ -1025,18 +1026,18 @@ private void loadReadOnlyBuffer(VirtualFrame frame, PUnpickler self) { private void loadCountedBinString(VirtualFrame frame, PUnpickler self, int nbytes) { Object obj; - ByteArrayView s = read(frame, self, nbytes); + int offset = read(frame, self, nbytes); - int size = calcBinSize(s, nbytes); + int size = calcBinSize(self.inputBuffer, offset, nbytes); if (size < 0) { throw raise(PythonBuiltinClassType.UnpicklingError, ErrorMessages.S_EXCEEDS_MAX_SIZE_N_BYTES, "BINSTRING", Integer.MAX_VALUE); } - s = read(frame, self, size); + offset = read(frame, self, size); // Convert Python 2.x strings to bytes if the *encoding* given to the Unpickler was // 'bytes'. Otherwise, convert them to unicode. - final PBytes bytes = PFactory.createBytes(PythonLanguage.get(this), s.getBytes(size), size); + final PBytes bytes = PFactory.createBytes(PythonLanguage.get(this), PythonUtils.arrayCopyOfRange(self.inputBuffer, offset, offset + size)); if (ensureTsEqualNode().execute(self.encoding, T_CODEC_BYTES, TS_ENCODING)) { obj = bytes; } else { @@ -1095,16 +1096,16 @@ private void loadUnicode(VirtualFrame frame, PUnpickler self) { private void loadBinCountedUnicode(VirtualFrame frame, PUnpickler self, int nbytes) { Object str; - ByteArrayView s = read(frame, self, nbytes); + int offset = read(frame, self, nbytes); - int size = calcBinSize(s, nbytes); + int size = calcBinSize(self.inputBuffer, offset, nbytes); if (size < 0) { throw raise(PythonBuiltinClassType.OverflowError, ErrorMessages.S_EXCEEDS_MAX_SIZE_N_BYTES, "BINUNICODE", Integer.MAX_VALUE); } - s = read(frame, self, size); + offset = read(frame, self, size); - str = decodeUTF8(frame, s, size, T_ERRORS_SURROGATEPASS); + str = decodeUTF8(frame, self.inputBuffer, offset, size, T_ERRORS_SURROGATEPASS); pDataPush(self, str); } @@ -1499,8 +1500,8 @@ private void loadBinGet(VirtualFrame frame, PUnpickler self) { } private void loadLongBinGet(VirtualFrame frame, PUnpickler self) { - ByteArrayView s = read(frame, self, 4); - int idx = calcBinSize(s, 4); + int offset = read(frame, self, 4); + int idx = calcBinSize(self.inputBuffer, offset, 4); Object value = self.memoGet(idx); if (value == null) { @@ -1559,13 +1560,13 @@ private void loadBinPut(VirtualFrame frame, PUnpickler self) { } private void loadLongBinPut(VirtualFrame frame, PUnpickler self) { - ByteArrayView s = read(frame, self, 4); + int offset = read(frame, self, 4); if (self.stack.size <= self.stack.fence) { throw pDataStackRaiseUnderflow(self); } Object value = self.stack.data[self.stack.size - 1]; - int idx = calcBinSize(s, 4); + int idx = calcBinSize(self.inputBuffer, offset, 4); if (idx < 0) { throw raise(PythonBuiltinClassType.ValueError, ErrorMessages.NEG_S_ARG, "LONG_BINPUT"); } @@ -1717,9 +1718,9 @@ private void loadReduce(VirtualFrame frame, PUnpickler self) { } private void loadProto(VirtualFrame frame, PUnpickler self) { - final ByteArrayView s = read(frame, self, 1); + int offset = read(frame, self, 1); - int i = s.getUnsigned(0); + int i = self.inputBuffer[offset] & 0xff; if (i <= PICKLE_PROTOCOL_HIGHEST) { self.proto = i; return; @@ -1729,14 +1730,14 @@ private void loadProto(VirtualFrame frame, PUnpickler self) { } private void loadFrame(VirtualFrame frame, PUnpickler self) { - ByteArrayView s = read(frame, self, 8); + int offset = read(frame, self, 8); - int frameLen = calcBinSize(s, 8); + int frameLen = calcBinSize(self.inputBuffer, offset, 8); if (frameLen < 0) { throw raise(PythonBuiltinClassType.OverflowError, ErrorMessages.S_EXCEEDS_MAX_SIZE_N_BYTES, "FRAME", Integer.MAX_VALUE); } - s = read(frame, self, frameLen); + read(frame, self, frameLen); // Rewind to start of frame self.nextReadIdx -= frameLen; @@ -1745,9 +1746,9 @@ private void loadFrame(VirtualFrame frame, PUnpickler self) { private void loadExtension(VirtualFrame frame, BoundaryCallData boundaryCallData, PythonContext ctx, PUnpickler self, int nbytes) { assert (nbytes == 1 || nbytes == 2 || nbytes == 4); // the nbytes bytes after the opcode - ByteArrayView codebytes = read(frame, self, nbytes); + int codeOffset = read(frame, self, nbytes); // calc_binint returns long - long code = calcBinInt(codebytes, nbytes); + long code = calcBinInt(self.inputBuffer, codeOffset, nbytes); if (code <= 0) { // note that 0 is forbidden diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PickleUtils.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PickleUtils.java index 772d2abc95..193840a77b 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PickleUtils.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PickleUtils.java @@ -61,7 +61,6 @@ import com.oracle.graal.python.runtime.object.PFactory; import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; import com.oracle.truffle.api.frame.VirtualFrame; -import com.oracle.truffle.api.nodes.ExplodeLoop; import com.oracle.truffle.api.nodes.Node; import com.oracle.truffle.api.strings.TruffleString; @@ -214,15 +213,15 @@ public final class PickleUtils { public static final TruffleString T_PROTO_LE2_TRUE = tsLiteral("I01\n"); public static final TruffleString T_PROTO_LE2_FALSE = tsLiteral("I00\n"); - @ExplodeLoop public static void writeSize64(byte[] out, int start, int value) { - final int sizeofSizeT = Integer.BYTES; - for (int i = 0; i < sizeofSizeT; i++) { - out[start + i] = (byte) ((value >> (8 * i)) & 0xff); - } - for (int i = sizeofSizeT; i < 8; i++) { - out[start + i] = 0; - } + out[start] = (byte) value; + out[start + 1] = (byte) (value >> 8); + out[start + 2] = (byte) (value >> 16); + out[start + 3] = (byte) (value >> 24); + out[start + 4] = 0; + out[start + 5] = 0; + out[start + 6] = 0; + out[start + 7] = 0; } public static int getStringSize(byte[] bytes) { diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java index 203d7ea33b..2c1292c6cf 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java @@ -102,6 +102,7 @@ import com.oracle.graal.python.runtime.exception.PException; import com.oracle.graal.python.runtime.object.PFactory; import com.oracle.graal.python.runtime.sequence.storage.SequenceStorage; +import com.oracle.graal.python.util.PythonUtils; import com.oracle.truffle.api.CompilerDirectives; import com.oracle.truffle.api.frame.Frame; import com.oracle.truffle.api.frame.VirtualFrame; @@ -326,8 +327,8 @@ protected Object decodeASCII(VirtualFrame frame, byte[] bytes, int len, TruffleS return decode(frame, PFactory.createBytes(PythonLanguage.get(this), bytes, len), T_CODEC_ASCII, errors); } - protected Object decodeUTF8(VirtualFrame frame, ByteArrayView bytes, int len, TruffleString errors) { - return decode(frame, PFactory.createBytes(PythonLanguage.get(this), bytes.getBytes(len), len), T_UTF8, errors); + protected Object decodeUTF8(VirtualFrame frame, byte[] bytes, int offset, int len, TruffleString errors) { + return decode(frame, PFactory.createBytes(PythonLanguage.get(this), PythonUtils.arrayCopyOfRange(bytes, offset, offset + len)), T_UTF8, errors); } protected Object decode(VirtualFrame frame, Object value, TruffleString encoding) { diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/ints/IntNodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/ints/IntNodes.java index 4cd53ffff2..fd4f0cdae8 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/ints/IntNodes.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/ints/IntNodes.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -181,10 +181,14 @@ static byte[] doPInt(Node inliningTarget, PInt value, int size, boolean bigEndia @GenerateInline(inlineByDefault = true) @GenerateUncached public abstract static class PyLongFromByteArray extends Node { - public abstract Object execute(Node inliningTarget, byte[] data, boolean littleEndian, boolean signed); + public abstract Object execute(Node inliningTarget, byte[] data, int offset, int length, boolean littleEndian, boolean signed); - public final Object executeCached(byte[] data, boolean littleEndian, boolean signed) { - return execute(this, data, littleEndian, signed); + public final Object execute(Node inliningTarget, byte[] data, boolean littleEndian, boolean signed) { + return execute(inliningTarget, data, 0, data.length, littleEndian, signed); + } + + public final Object executeCached(byte[] data, int offset, int length, boolean littleEndian, boolean signed) { + return execute(this, data, offset, length, littleEndian, signed); } public static Object executeUncached(byte[] data, boolean littleEndian, boolean signed) { @@ -192,7 +196,7 @@ public static Object executeUncached(byte[] data, boolean littleEndian, boolean } @Specialization - static Object doOther(Node inliningTarget, byte[] data, boolean littleEndian, boolean signed, + static Object doOther(Node inliningTarget, byte[] data, int offset, int length, boolean littleEndian, boolean signed, @Cached InlinedBranchProfile fastPath1, @Cached InlinedBranchProfile fastPath2, @Cached InlinedBranchProfile fastPath4, @@ -201,28 +205,28 @@ static Object doOther(Node inliningTarget, byte[] data, boolean littleEndian, bo @Cached PRaiseNode raiseNode) { NumericSupport support = littleEndian ? NumericSupport.littleEndian() : NumericSupport.bigEndian(); if (signed) { - switch (data.length) { + switch (length) { case 1 -> { fastPath1.enter(inliningTarget); - return (int) support.getByte(data, 0); + return (int) support.getByte(data, offset); } case 2 -> { fastPath2.enter(inliningTarget); - return (int) support.getShort(data, 0); + return (int) support.getShort(data, offset); } case 4 -> { fastPath4.enter(inliningTarget); - return support.getInt(data, 0); + return support.getInt(data, offset); } case 8 -> { fastPath8.enter(inliningTarget); - return support.getLong(data, 0); + return support.getLong(data, offset); } } } generic.enter(inliningTarget); try { - BigInteger integer = support.getBigInteger(data, signed); + BigInteger integer = support.getBigInteger(data, offset, length, signed); if (PInt.bigIntegerFitsInLong(integer)) { long longValue = PInt.longValue(integer); return PInt.isIntRange(longValue) ? (int) longValue : longValue; From eb14674fa644f8492ee58bc62ef7056382ef8817 Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Thu, 1 Oct 2026 14:21:54 +0200 Subject: [PATCH 22/39] Add unpickler fast path for decoding UTF-8 --- .../builtins/modules/pickle/PicklerNodes.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java index 2c1292c6cf..70c443df35 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java @@ -154,6 +154,7 @@ abstract static class BasePickleNode extends Node { @Child private TruffleString.SubstringNode tsSubstringNode; @Child private TruffleString.EqualNode tsEqualNode; @Child private TruffleString.SwitchEncodingNode tsSwitchEncodingNode; + @Child private TruffleString.IsValidNode tsIsValidNode; @Child private HashingStorageGetIterator getHashingStorageIteratorNode; @Child private HashingStorageIteratorNext hashingStorageItNext; @Child private HashingStorageIteratorKey hashingStorageItKey; @@ -231,6 +232,14 @@ protected TruffleString.SwitchEncodingNode ensureTsSwitchEncodingNode() { return tsSwitchEncodingNode; } + protected TruffleString.IsValidNode ensureTsIsValidNode() { + if (tsIsValidNode == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + tsIsValidNode = insert(TruffleString.IsValidNode.create()); + } + return tsIsValidNode; + } + protected byte[] toBytes(VirtualFrame frame, Object obj) { if (toBytesNode == null) { CompilerDirectives.transferToInterpreterAndInvalidate(); @@ -328,6 +337,10 @@ protected Object decodeASCII(VirtualFrame frame, byte[] bytes, int len, TruffleS } protected Object decodeUTF8(VirtualFrame frame, byte[] bytes, int offset, int len, TruffleString errors) { + TruffleString utf8 = ensureTsFromByteArray().execute(bytes, offset, len, TruffleString.Encoding.UTF_8, true); + if (ensureTsIsValidNode().execute(utf8, TruffleString.Encoding.UTF_8)) { + return ensureTsSwitchEncodingNode().execute(utf8, TS_ENCODING); + } return decode(frame, PFactory.createBytes(PythonLanguage.get(this), PythonUtils.arrayCopyOfRange(bytes, offset, offset + len)), T_UTF8, errors); } From 9e2d6e5c632f500489909c4d3a23be3d9b3b1c81 Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Thu, 1 Oct 2026 14:27:55 +0200 Subject: [PATCH 23/39] Add profiling for seen unpickler opcodes --- .../builtins/modules/pickle/PUnpickler.java | 157 ++++++++++++++++++ 1 file changed, 157 insertions(+) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PUnpickler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PUnpickler.java index 0b437491b9..2fa8f5f11a 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PUnpickler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PUnpickler.java @@ -183,6 +183,7 @@ import com.oracle.graal.python.util.NumericSupport; import com.oracle.graal.python.util.PythonUtils; import com.oracle.truffle.api.CompilerDirectives; +import com.oracle.truffle.api.CompilerDirectives.CompilationFinal; import com.oracle.truffle.api.dsl.Bind; import com.oracle.truffle.api.dsl.Cached; import com.oracle.truffle.api.dsl.Specialization; @@ -684,6 +685,78 @@ Object find(VirtualFrame frame, PUnpickler unpickler, TruffleString module, Truf } public abstract static class LoadNode extends BasePickleReadNode { + private static final int SEEN_NONE = 0; + private static final int SEEN_BININT = 1; + private static final int SEEN_BININT1 = 2; + private static final int SEEN_BININT2 = 3; + private static final int SEEN_INT = 4; + private static final int SEEN_LONG = 5; + private static final int SEEN_LONG1 = 6; + private static final int SEEN_LONG4 = 7; + private static final int SEEN_FLOAT = 8; + private static final int SEEN_BINFLOAT = 9; + private static final int SEEN_SHORT_BINBYTES = 10; + private static final int SEEN_BINBYTES = 11; + private static final int SEEN_BINBYTES8 = 12; + private static final int SEEN_BYTEARRAY8 = 13; + private static final int SEEN_NEXT_BUFFER = 14; + private static final int SEEN_READONLY_BUFFER = 15; + private static final int SEEN_SHORT_BINSTRING = 16; + private static final int SEEN_BINSTRING = 17; + private static final int SEEN_STRING = 18; + private static final int SEEN_UNICODE = 19; + private static final int SEEN_SHORT_BINUNICODE = 20; + private static final int SEEN_BINUNICODE = 21; + private static final int SEEN_BINUNICODE8 = 22; + private static final int SEEN_EMPTY_TUPLE = 23; + private static final int SEEN_TUPLE1 = 24; + private static final int SEEN_TUPLE2 = 25; + private static final int SEEN_TUPLE3 = 26; + private static final int SEEN_TUPLE = 27; + private static final int SEEN_EMPTY_LIST = 28; + private static final int SEEN_LIST = 29; + private static final int SEEN_EMPTY_DICT = 30; + private static final int SEEN_DICT = 31; + private static final int SEEN_EMPTY_SET = 32; + private static final int SEEN_ADDITEMS = 33; + private static final int SEEN_FROZENSET = 34; + private static final int SEEN_OBJ = 35; + private static final int SEEN_INST = 36; + private static final int SEEN_NEWOBJ = 37; + private static final int SEEN_NEWOBJ_EX = 38; + private static final int SEEN_GLOBAL = 39; + private static final int SEEN_STACK_GLOBAL = 40; + private static final int SEEN_APPEND = 41; + private static final int SEEN_APPENDS = 42; + private static final int SEEN_BUILD = 43; + private static final int SEEN_DUP = 44; + private static final int SEEN_BINGET = 45; + private static final int SEEN_LONG_BINGET = 46; + private static final int SEEN_GET = 47; + private static final int SEEN_MARK = 48; + private static final int SEEN_BINPUT = 49; + private static final int SEEN_LONG_BINPUT = 50; + private static final int SEEN_PUT = 51; + private static final int SEEN_MEMOIZE = 52; + private static final int SEEN_POP = 53; + private static final int SEEN_POP_MARK = 54; + private static final int SEEN_SETITEM = 55; + private static final int SEEN_SETITEMS = 56; + private static final int SEEN_PERSID = 57; + private static final int SEEN_BINPERSID = 58; + private static final int SEEN_REDUCE = 59; + private static final int SEEN_PROTO = 60; + private static final int SEEN_FRAME = 61; + private static final int SEEN_EXT1 = 62; + private static final int SEEN_EXT2 = 63; + private static final int SEEN_EXT4 = 64; + private static final int SEEN_NEWTRUE = 65; + private static final int SEEN_NEWFALSE = 66; + private static final int SEEN_STOP = 67; + + @CompilationFinal private long seenOpcodes0; + @CompilationFinal private long seenOpcodes1; + @Child private PData.PDataPushNode pDataPushNode; @Child private PData.PDataPopNode pDataPopNode; @Child private PData.PDataPopTupleNode pDataPopTupleNode; @@ -699,6 +772,22 @@ public abstract static class LoadNode extends BasePickleReadNode { public abstract Object execute(VirtualFrame frame, PUnpickler self); + private void profileSeen(int seen) { + if (seen < Long.SIZE) { + long mask = 1L << seen; + if ((seenOpcodes0 & mask) == 0) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + seenOpcodes0 |= mask; + } + } else { + long mask = 1L << (seen - Long.SIZE); + if ((seenOpcodes1 & mask) == 0) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + seenOpcodes1 |= mask; + } + } + } + protected HashingStorageCopy ensureHashingStorageCopy() { if (hashCopy == null) { CompilerDirectives.transferToInterpreterAndInvalidate(); @@ -1829,207 +1918,275 @@ public Object load(VirtualFrame frame, PUnpickler self, } switch (s) { case OPCODE_NONE: + profileSeen(SEEN_NONE); loadNone(self); continue; case OPCODE_BININT: + profileSeen(SEEN_BININT); loadBinInt(frame, self); continue; case OPCODE_BININT1: + profileSeen(SEEN_BININT1); loadBinInt1(frame, self); continue; case OPCODE_BININT2: + profileSeen(SEEN_BININT2); loadBinInt2(frame, self); continue; case OPCODE_INT: + profileSeen(SEEN_INT); loadInt(frame, self); continue; case OPCODE_LONG: + profileSeen(SEEN_LONG); loadLong(frame, self); continue; case OPCODE_LONG1: + profileSeen(SEEN_LONG1); loadCountedLong(frame, self, 1); continue; case OPCODE_LONG4: + profileSeen(SEEN_LONG4); loadCountedLong(frame, self, 4); continue; case OPCODE_FLOAT: + profileSeen(SEEN_FLOAT); loadFloat(frame, self); continue; case OPCODE_BINFLOAT: + profileSeen(SEEN_BINFLOAT); loadBinFloat(frame, self); continue; case OPCODE_SHORT_BINBYTES: + profileSeen(SEEN_SHORT_BINBYTES); loadCountedBinBytes(frame, self, 1); continue; case OPCODE_BINBYTES: + profileSeen(SEEN_BINBYTES); loadCountedBinBytes(frame, self, 4); continue; case OPCODE_BINBYTES8: + profileSeen(SEEN_BINBYTES8); loadCountedBinBytes(frame, self, 8); continue; case OPCODE_BYTEARRAY8: + profileSeen(SEEN_BYTEARRAY8); loadCountedByteArray(frame, self); continue; case OPCODE_NEXT_BUFFER: + profileSeen(SEEN_NEXT_BUFFER); loadNextBuffer(frame, self); continue; case OPCODE_READONLY_BUFFER: + profileSeen(SEEN_READONLY_BUFFER); loadReadOnlyBuffer(frame, self); continue; case OPCODE_SHORT_BINSTRING: + profileSeen(SEEN_SHORT_BINSTRING); loadCountedBinString(frame, self, 1); continue; case OPCODE_BINSTRING: + profileSeen(SEEN_BINSTRING); loadCountedBinString(frame, self, 4); continue; case OPCODE_STRING: + profileSeen(SEEN_STRING); loadString(frame, self); continue; case OPCODE_UNICODE: + profileSeen(SEEN_UNICODE); loadUnicode(frame, self); continue; case OPCODE_SHORT_BINUNICODE: + profileSeen(SEEN_SHORT_BINUNICODE); loadBinCountedUnicode(frame, self, 1); continue; case OPCODE_BINUNICODE: + profileSeen(SEEN_BINUNICODE); loadBinCountedUnicode(frame, self, 4); continue; case OPCODE_BINUNICODE8: + profileSeen(SEEN_BINUNICODE8); loadBinCountedUnicode(frame, self, 8); continue; case OPCODE_EMPTY_TUPLE: + profileSeen(SEEN_EMPTY_TUPLE); loadCountedTuple(self, 0); continue; case OPCODE_TUPLE1: + profileSeen(SEEN_TUPLE1); loadCountedTuple(self, 1); continue; case OPCODE_TUPLE2: + profileSeen(SEEN_TUPLE2); loadCountedTuple(self, 2); continue; case OPCODE_TUPLE3: + profileSeen(SEEN_TUPLE3); loadCountedTuple(self, 3); continue; case OPCODE_TUPLE: + profileSeen(SEEN_TUPLE); loadTuple(self); continue; case OPCODE_EMPTY_LIST: + profileSeen(SEEN_EMPTY_LIST); loadEmptyList(self); continue; case OPCODE_LIST: + profileSeen(SEEN_LIST); loadList(self); continue; case OPCODE_EMPTY_DICT: + profileSeen(SEEN_EMPTY_DICT); loadEmptyDict(self); continue; case OPCODE_DICT: + profileSeen(SEEN_DICT); loadDict(frame, self); continue; case OPCODE_EMPTY_SET: + profileSeen(SEEN_EMPTY_SET); loadEmptySet(self); continue; case OPCODE_ADDITEMS: + profileSeen(SEEN_ADDITEMS); loadAddItems(frame, self); continue; case OPCODE_FROZENSET: + profileSeen(SEEN_FROZENSET); loadFrozenSet(frame, self); continue; case OPCODE_OBJ: + profileSeen(SEEN_OBJ); loadObj(frame, inliningTarget, self, callMethod); continue; case OPCODE_INST: + profileSeen(SEEN_INST); loadInst(frame, inliningTarget, boundaryCallData, ctx, self, callMethod); continue; case OPCODE_NEWOBJ: + profileSeen(SEEN_NEWOBJ); loadNewObj(frame, inliningTarget, self, getSlots, callNew, expandArgs); continue; case OPCODE_NEWOBJ_EX: + profileSeen(SEEN_NEWOBJ_EX); loadNewObjEx(frame, inliningTarget, self, getSlots, callNew, expandArgs, expandKwargs); continue; case OPCODE_GLOBAL: + profileSeen(SEEN_GLOBAL); loadGlobal(frame, boundaryCallData, ctx, self); continue; case OPCODE_STACK_GLOBAL: + profileSeen(SEEN_STACK_GLOBAL); loadStackGlobal(frame, boundaryCallData, ctx, self); continue; case OPCODE_APPEND: + profileSeen(SEEN_APPEND); loadAppend(frame, self); continue; case OPCODE_APPENDS: + profileSeen(SEEN_APPENDS); loadAppends(frame, self); continue; case OPCODE_BUILD: + profileSeen(SEEN_BUILD); loadBuild(frame, self); continue; case OPCODE_DUP: + profileSeen(SEEN_DUP); loadDup(self); continue; case OPCODE_BINGET: + profileSeen(SEEN_BINGET); loadBinGet(frame, self); continue; case OPCODE_LONG_BINGET: + profileSeen(SEEN_LONG_BINGET); loadLongBinGet(frame, self); continue; case OPCODE_GET: + profileSeen(SEEN_GET); loadGet(frame, self); continue; case OPCODE_MARK: + profileSeen(SEEN_MARK); loadMark(self); continue; case OPCODE_BINPUT: + profileSeen(SEEN_BINPUT); loadBinPut(frame, self); continue; case OPCODE_LONG_BINPUT: + profileSeen(SEEN_LONG_BINPUT); loadLongBinPut(frame, self); continue; case OPCODE_PUT: + profileSeen(SEEN_PUT); loadPut(frame, self); continue; case OPCODE_MEMOIZE: + profileSeen(SEEN_MEMOIZE); loadMemoize(self); continue; case OPCODE_POP: + profileSeen(SEEN_POP); loadPop(self); continue; case OPCODE_POP_MARK: + profileSeen(SEEN_POP_MARK); loadPopMark(self); continue; case OPCODE_SETITEM: + profileSeen(SEEN_SETITEM); loadSetItem(frame, self); continue; case OPCODE_SETITEMS: + profileSeen(SEEN_SETITEMS); loadSetItems(frame, self); continue; case OPCODE_PERSID: + profileSeen(SEEN_PERSID); loadPersId(frame, self); continue; case OPCODE_BINPERSID: + profileSeen(SEEN_BINPERSID); loadBinPersId(frame, self); continue; case OPCODE_REDUCE: + profileSeen(SEEN_REDUCE); loadReduce(frame, self); continue; case OPCODE_PROTO: + profileSeen(SEEN_PROTO); loadProto(frame, self); continue; case OPCODE_FRAME: + profileSeen(SEEN_FRAME); loadFrame(frame, self); continue; case OPCODE_EXT1: + profileSeen(SEEN_EXT1); loadExtension(frame, boundaryCallData, ctx, self, 1); continue; case OPCODE_EXT2: + profileSeen(SEEN_EXT2); loadExtension(frame, boundaryCallData, ctx, self, 2); continue; case OPCODE_EXT4: + profileSeen(SEEN_EXT4); loadExtension(frame, boundaryCallData, ctx, self, 4); continue; case OPCODE_NEWTRUE: + profileSeen(SEEN_NEWTRUE); loadBool(self, true); continue; case OPCODE_NEWFALSE: + profileSeen(SEEN_NEWFALSE); loadBool(self, false); continue; case OPCODE_STOP: + profileSeen(SEEN_STOP); break; default: if (0x20 <= s && s <= 0x7e && s != '\'' && s != '\\') { From 2102b34e2fb030166a4cfcacc8d19fc19eae9f38 Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Thu, 1 Oct 2026 14:37:34 +0200 Subject: [PATCH 24/39] Pre-size dictionaries when unpickling --- .../graal/python/builtins/modules/pickle/PUnpickler.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PUnpickler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PUnpickler.java index 2fa8f5f11a..32f19e5e26 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PUnpickler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PUnpickler.java @@ -1239,7 +1239,7 @@ private void loadDict(VirtualFrame frame, PUnpickler self) { throw raise(PythonBuiltinClassType.UnpicklingError, ErrorMessages.ODD_NR_ITEMS_FOR_S, "DICT"); } - HashingStorage storage = EmptyStorage.INSTANCE; + HashingStorage storage = PDict.createNewStorage((j - i) / 2); for (k = i + 1; k < j; k += 2) { key = self.stack.data[k - 1]; value = self.stack.data[k]; @@ -1736,6 +1736,9 @@ private void doSetItems(VirtualFrame frame, PUnpickler self, int x) { Object dict = self.stack.data[x - 1]; final boolean isBuiltinDict = dict instanceof PDict pDict && PGuards.isBuiltinDict(pDict); + if (isBuiltinDict && ((PDict) dict).getDictStorage() == EmptyStorage.INSTANCE) { + ((PDict) dict).setDictStorage(PDict.createNewStorage((len - x) / 2)); + } for (i = x + 1; i < len; i += 2) { key = self.stack.data[i - 1]; value = self.stack.data[i]; From 2212a831a8b2e7e24b2cbff8352eb2adffacbc73 Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Thu, 1 Oct 2026 15:38:06 +0200 Subject: [PATCH 25/39] Add a few branch probabilities --- .../graal/python/builtins/modules/pickle/PUnpickler.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PUnpickler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PUnpickler.java index 32f19e5e26..ed13b21d3e 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PUnpickler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PUnpickler.java @@ -362,7 +362,7 @@ public Object memoGet(int idx) { } public void memoPut(int idx, Object value) { - if (idx >= memo.length) { + if (CompilerDirectives.injectBranchProbability(CompilerDirectives.SLOWPATH_PROBABILITY, idx >= memo.length)) { resizeMemoList(idx * 2); assert idx < memo.length; } @@ -540,7 +540,7 @@ protected byte read(VirtualFrame frame, PUnpickler self) { } protected int read(VirtualFrame frame, PUnpickler self, int n) { - if (n <= self.inputLen - self.nextReadIdx) { + if (CompilerDirectives.injectBranchProbability(CompilerDirectives.FASTPATH_PROBABILITY, n <= self.inputLen - self.nextReadIdx)) { int offset = self.nextReadIdx; self.nextReadIdx += n; return offset; From 16e83bf34bdf03185d37ba3666cd88d5919b6a6d Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Thu, 1 Oct 2026 15:44:09 +0200 Subject: [PATCH 26/39] Avoid copying buffer in pickle.loads --- .../python/builtins/modules/pickle/PickleModuleBuiltins.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PickleModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PickleModuleBuiltins.java index 6cc08bb72c..f58154dafd 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PickleModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PickleModuleBuiltins.java @@ -256,8 +256,8 @@ static Object loads(VirtualFrame frame, @SuppressWarnings("unused") PythonModule @Cached PyObjectGetIter getIter) { try { PUnpickler unpickler = PFactory.createUnpickler(language); - byte[] data = bufferLib.getCopiedByteArray(buffer); - unpickler.setStringInput(data, data.length); + byte[] data = bufferLib.getInternalOrCopiedByteArray(buffer); + unpickler.setStringInput(data, bufferLib.getBufferLength(buffer)); unpickler.setInputEncoding(encoding, errors); unpickler.setBuffers(frame, inliningTarget, getIter, buffers); unpickler.setFixImports(fixImports); From 1a5c6500cda87a9f6010ffe6f0693d62a21ab440 Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Thu, 1 Oct 2026 19:05:17 +0200 Subject: [PATCH 27/39] Profile file reads in unpickle --- .../graal/python/builtins/modules/pickle/PUnpickler.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PUnpickler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PUnpickler.java index ed13b21d3e..f534ad21d8 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PUnpickler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PUnpickler.java @@ -406,6 +406,8 @@ public abstract static class BasePickleReadNode extends PicklerNodes.BasePickleN @Child private TruffleString.ParseIntNode tsParseIntNode; + @CompilationFinal private boolean seenFileRead; + // we only need the reference, doesn't matter that the object may not yet be fully // constructed @SuppressWarnings("this-escape") private InteropCallData interopCallData = InteropCallData.createFor(this); @@ -545,6 +547,10 @@ protected int read(VirtualFrame frame, PUnpickler self, int n) { self.nextReadIdx += n; return offset; } + if (!seenFileRead) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + seenFileRead = true; + } return readImpl(frame, self, n); } From 3047bf54509bb294c09b9672755ddf8879943184 Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Thu, 1 Oct 2026 21:03:39 +0200 Subject: [PATCH 28/39] Add fast paths for pickling/unpickling large longs --- .../builtins/modules/pickle/PPickler.java | 100 ++++++------------ .../builtins/objects/ints/IntNodes.java | 28 +++++ 2 files changed, 63 insertions(+), 65 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java index 73114ca26d..77df6fbd65 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java @@ -55,7 +55,6 @@ import static com.oracle.graal.python.nodes.StringLiterals.T_NEWLINE; import static com.oracle.graal.python.nodes.statement.AbstractImportNode.importModule; import static com.oracle.graal.python.runtime.exception.PythonErrorType.KeyError; -import static com.oracle.graal.python.runtime.exception.PythonErrorType.OverflowError; import static com.oracle.graal.python.runtime.exception.PythonErrorType.PicklingError; import static com.oracle.graal.python.runtime.exception.PythonErrorType.TypeError; import static com.oracle.graal.python.runtime.exception.PythonErrorType.ValueError; @@ -88,8 +87,6 @@ import com.oracle.graal.python.builtins.objects.ellipsis.PEllipsis; import com.oracle.graal.python.builtins.objects.floats.PFloat; import com.oracle.graal.python.builtins.objects.function.PFunction; -import com.oracle.graal.python.builtins.objects.ints.IntNodes; -import com.oracle.graal.python.builtins.objects.ints.IntNodesFactory; import com.oracle.graal.python.builtins.objects.ints.PInt; import com.oracle.graal.python.builtins.objects.list.PList; import com.oracle.graal.python.builtins.objects.object.PythonBuiltinObject; @@ -627,9 +624,6 @@ public abstract static class SaveNode extends PicklerNodes.BasePickleNode { @Child private SaveNode recursiveSaveNode2; @Child private SaveNode recursiveSaveNode3; - @Child private IntNodes.PyLongSign pyLongSign; - @Child private IntNodes.PyLongNumBits pyLongNumBits; - @Child private IntNodes.PyLongAsByteArray pyLongAsByteArray; @Child private ListNodes.ConstructListNode constructListNode; @Child private TypeNodes.IsTypeNode isTypeNode; @Child private FlushToFileNode flushToFileNode; @@ -761,30 +755,6 @@ protected FlushToFileNode ensureFlushToFileNode() { return flushToFileNode; } - protected int getSign(Object value) { - if (pyLongSign == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - pyLongSign = insert(IntNodesFactory.PyLongSignNodeGen.create()); - } - return pyLongSign.execute(value); - } - - protected int getNumBits(Object value) { - if (pyLongNumBits == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - pyLongNumBits = insert(IntNodesFactory.PyLongNumBitsNodeGen.create()); - } - return pyLongNumBits.execute(value); - } - - protected byte[] longAsBytes(Object value, int size, boolean bigEndian) { - if (pyLongAsByteArray == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - pyLongAsByteArray = insert(IntNodesFactory.PyLongAsByteArrayNodeGen.create()); - } - return pyLongAsByteArray.executeCached(value, size, bigEndian); - } - protected Object createList(VirtualFrame frame, Object iterable) { if (constructListNode == null) { CompilerDirectives.transferToInterpreterAndInvalidate(); @@ -1523,61 +1493,61 @@ private void saveLong(PPickler pickler, int proto, int value) { } } - private void saveLong(VirtualFrame frame, PPickler pickler, int proto, long value) { + private void saveLong(PPickler pickler, int proto, long value) { int intValue = (int) value; if (intValue == value) { saveLong(pickler, proto, intValue); } else { - saveLongLarge(frame, pickler, proto, value); + saveLongLarge(pickler, proto, value); } } private void saveLong(VirtualFrame frame, PPickler pickler, int proto, PInt value) { try { - saveLong(frame, pickler, proto, value.longValueExact()); + saveLong(pickler, proto, value.longValueExact()); } catch (OverflowException e) { saveLongLarge(frame, pickler, proto, value); } } - private void saveLongLarge(VirtualFrame frame, PPickler pickler, int proto, Object obj) { + private void saveLongLarge(PPickler pickler, int proto, long value) { if (proto >= 2) { - // Linear-time pickling. - final int sign = getSign(obj); - assert sign != 0; // Zero was handled by the int code above - - final int nbits = getNumBits(obj); - // How many bytes do we need? There are nbits >> 3 full bytes of data, and nbits & 7 - // leftover bits. If there are any leftover bits, then we clearly need another byte. - // What's not so obvious is that we *probably* need another byte even if there - // aren't any leftovers: the most-significant bit of the most-significant byte acts - // like a sign bit, and it's usually got a sense opposite of the one we need. The - // exception is ints of the form -(2**(8*j-1)) for j > 0. Such an int is its own - // 256's-complement, so has the right sign bit even without the extra byte. That's a - // pain to check for in advance, though, so we always grab an extra byte at the - // start, and cut it back later if possible. + // Count the significant bits excluding the sign extension, then add a sign bit. + int nbits = Long.SIZE - Long.numberOfLeadingZeros(value < 0 ? ~value : value); int nbytes = (nbits >> 3) + 1; - if (Long.compareUnsigned(nbytes, 0x7fffffffL) > 0) { - throw raise(OverflowError, ErrorMessages.S_TO_LARGE_TO_PICKLE, "int"); - } - byte[] pdata = longAsBytes(obj, nbytes, false); - // If the int is negative, this may be a byte more than needed. This is so iff the - // MSB is all redundant sign bits. - if (sign < 0 && nbytes > 1 && pdata[nbytes - 1] == (byte) 0xff && (pdata[nbytes - 2] & 0x80) != 0) { - nbytes--; + pickler.writeByteOp(this, PickleUtils.OPCODE_LONG1, nbytes); + pickler.ensureBufferSpace(this, nbytes); + for (int i = 0; i < nbytes; i++) { + pickler.outputBuffer[pickler.outputLen++] = (byte) value; + value >>= Byte.SIZE; } + } else { + TruffleString repr = ensureTsFromLongNode().execute(value, TS_ENCODING, true); + pickler.write(this, PickleUtils.OPCODE_LONG); + pickler.writeASCII(this, repr, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); + pickler.writeASCII(this, T_L_NEW_LINE, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); + } + } + + private void saveLongLarge(VirtualFrame frame, PPickler pickler, int proto, PInt value) { + if (proto >= 2) { + // BigInteger already provides the minimal signed two's-complement representation. + byte[] data = value.toByteArray(); + int nbytes = data.length; if (nbytes < 256) { pickler.writeByteOp(this, PickleUtils.OPCODE_LONG1, nbytes); } else { pickler.writeIntOp(this, PickleUtils.OPCODE_LONG4, nbytes); } - - pickler.write(this, pdata, nbytes); - + pickler.ensureBufferSpace(this, nbytes); + // BigInteger's bytes are big endian; pickle requires little endian. + for (int i = nbytes - 1; i >= 0; i--) { + pickler.outputBuffer[pickler.outputLen++] = data[i]; + } } else { - TruffleString repr = repr(frame, obj); + TruffleString repr = repr(frame, value); pickler.write(this, PickleUtils.OPCODE_LONG); - pickler.writeASCII(this, asStringStrict(repr), ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); + pickler.writeASCII(this, repr, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); pickler.writeASCII(this, T_L_NEW_LINE, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); } } @@ -1889,7 +1859,7 @@ private void batchLongList(VirtualFrame frame, PPickler pickler, LongSequenceSto int proto = pickler.getProto(); if (storage.length() == 1) { opcodeBoundary(frame, pickler); - saveLong(frame, pickler, proto, storage.getLongItemNormalized(0)); + saveLong(pickler, proto, storage.getLongItemNormalized(0)); pickler.write(this, PickleUtils.OPCODE_APPEND); return; } @@ -1900,7 +1870,7 @@ private void batchLongList(VirtualFrame frame, PPickler pickler, LongSequenceSto pickler.write(this, PickleUtils.OPCODE_MARK); for (int i = batchStart; i < batchEnd; i++) { opcodeBoundary(frame, pickler); - saveLong(frame, pickler, proto, storage.getLongItemNormalized(i)); + saveLong(pickler, proto, storage.getLongItemNormalized(i)); } pickler.write(this, PickleUtils.OPCODE_APPENDS); } @@ -2002,7 +1972,7 @@ private void storeLongTupleElements(VirtualFrame frame, PPickler pickler, LongSe int proto = pickler.getProto(); for (int i = 0; i < storage.length(); i++) { opcodeBoundary(frame, pickler); - saveLong(frame, pickler, proto, storage.getLongItemNormalized(i)); + saveLong(pickler, proto, storage.getLongItemNormalized(i)); } } @@ -2348,7 +2318,7 @@ void saveGeneric(VirtualFrame frame, PPickler pickler, Object objArg, int persSa return; } else if (obj instanceof Long value) { profileSeen(SEEN_LONG); - saveLong(frame, pickler, proto, value); + saveLong(pickler, proto, value); return; } else if (obj instanceof PInt value && PGuards.isBuiltinPInt(value)) { profileSeen(SEEN_PINT); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/ints/IntNodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/ints/IntNodes.java index fd4f0cdae8..5767768938 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/ints/IntNodes.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/ints/IntNodes.java @@ -201,6 +201,9 @@ static Object doOther(Node inliningTarget, byte[] data, int offset, int length, @Cached InlinedBranchProfile fastPath2, @Cached InlinedBranchProfile fastPath4, @Cached InlinedBranchProfile fastPath8, + @Cached InlinedBranchProfile variablePath, + @Cached InlinedBranchProfile fastPathVariable, + @Cached InlinedBranchProfile overlong, @Cached InlinedBranchProfile generic, @Cached PRaiseNode raiseNode) { NumericSupport support = littleEndian ? NumericSupport.littleEndian() : NumericSupport.bigEndian(); @@ -224,6 +227,31 @@ static Object doOther(Node inliningTarget, byte[] data, int offset, int length, } } } + variablePath.enter(inliningTarget); + int remaining = length; + int index = littleEndian ? offset + length - 1 : offset; + int step = littleEndian ? -1 : 1; + boolean negative = signed && length > 0 && data[index] < 0; + if (remaining > Long.BYTES) { + overlong.enter(inliningTarget); + // Ignore redundant sign or zero extension beyond the width of a long. + byte padding = (byte) (negative ? -1 : 0); + while (remaining > Long.BYTES && data[index] == padding) { + index += step; + remaining--; + } + } + // For eight bytes, the remaining sign bit must agree with the original sign. + // This also excludes unsigned values greater than Long.MAX_VALUE. + if (remaining < Long.BYTES || (remaining == Long.BYTES && (data[index] < 0) == negative)) { + fastPathVariable.enter(inliningTarget); + long value = negative ? -1L : 0L; + for (int i = 0; i < remaining; i++) { + value = (value << Byte.SIZE) | (data[index] & 0xffL); + index += step; + } + return value; + } generic.enter(inliningTarget); try { BigInteger integer = support.getBigInteger(data, offset, length, signed); From 538816c84f61dfb0c6d3fdf478ddfdf327243b0b Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Thu, 1 Oct 2026 22:28:27 +0200 Subject: [PATCH 29/39] Avoid boundary in getDottedPath --- .../builtins/modules/pickle/PicklerNodes.java | 46 +++++++++++++++---- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java index 70c443df35..58eefbaac4 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java @@ -46,7 +46,6 @@ import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.T_CP_REVERSE_NAME_MAPPING; import static com.oracle.graal.python.builtins.objects.PNone.NO_VALUE; import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___CLASS__; -import static com.oracle.graal.python.nodes.StringLiterals.T_DOT; import static com.oracle.graal.python.nodes.StringLiterals.T_UTF8; import static com.oracle.graal.python.nodes.statement.AbstractImportNode.importModule; import static com.oracle.graal.python.runtime.exception.PythonErrorType.AttributeError; @@ -74,7 +73,6 @@ import com.oracle.graal.python.builtins.objects.common.SequenceStorageNodes; import com.oracle.graal.python.builtins.objects.dict.PDict; import com.oracle.graal.python.builtins.objects.function.PKeyword; -import com.oracle.graal.python.builtins.objects.str.StringUtils; import com.oracle.graal.python.builtins.objects.tuple.PTuple; import com.oracle.graal.python.lib.PyIterCheckNode; import com.oracle.graal.python.lib.PyIterNextNode; @@ -150,7 +148,7 @@ abstract static class BasePickleNode extends Node { @Child private TruffleString.FromByteArrayNode tsFromByteArrayNode; @Child private TruffleString.FromByteArrayWithCompactionUTF32Node tsFromByteArrayWithCompactionNode; @Child private TruffleString.CodePointLengthNode tsCodePointLengthNode; - @Child private TruffleString.IndexOfStringNode tsIndexOfStringNode; + @Child private TruffleString.IndexOfCodePointNode tsIndexOfCodePointNode; @Child private TruffleString.SubstringNode tsSubstringNode; @Child private TruffleString.EqualNode tsEqualNode; @Child private TruffleString.SwitchEncodingNode tsSwitchEncodingNode; @@ -200,12 +198,12 @@ protected TruffleString.CodePointLengthNode ensureTsCodePointLengthNode() { return tsCodePointLengthNode; } - protected TruffleString.IndexOfStringNode ensureTsIndexOfStringNode() { - if (tsIndexOfStringNode == null) { + protected TruffleString.IndexOfCodePointNode ensureTsIndexOfCodePointNode() { + if (tsIndexOfCodePointNode == null) { CompilerDirectives.transferToInterpreterAndInvalidate(); - tsIndexOfStringNode = insert(TruffleString.IndexOfStringNode.create()); + tsIndexOfCodePointNode = insert(TruffleString.IndexOfCodePointNode.create()); } - return tsIndexOfStringNode; + return tsIndexOfCodePointNode; } protected TruffleString.SubstringNode ensureTsSubstringNode() { @@ -574,9 +572,39 @@ public static Pair getDeepAttribute(VirtualFrame frame, PyObject return Pair.create(object, parent); } + private TruffleString[] splitDottedPath(TruffleString name) { + int length = ensureTsCodePointLengthNode().execute(name, TS_ENCODING); + TruffleString.IndexOfCodePointNode indexOf = ensureTsIndexOfCodePointNode(); + int firstDot = length == 0 ? -1 : indexOf.execute(name, '.', 0, length, TS_ENCODING); + if (firstDot < 0) { + return new TruffleString[]{name}; + } + + // Count the components so that we can allocate the result without growing or copying it. + int parts = 2; + int start = firstDot + 1; + while (start < length) { + int dot = indexOf.execute(name, '.', start, length, TS_ENCODING); + if (dot < 0) { + break; + } + parts++; + start = dot + 1; + } + TruffleString[] dottedPath = new TruffleString[parts]; + TruffleString.SubstringNode substring = ensureTsSubstringNode(); + start = 0; + int part = 0; + for (int dot = firstDot; dot >= 0; dot = start < length ? indexOf.execute(name, '.', start, length, TS_ENCODING) : -1) { + dottedPath[part++] = substring.execute(name, start, dot - start, TS_ENCODING, false); + start = dot + 1; + } + dottedPath[part] = substring.execute(name, start, length - start, TS_ENCODING, false); + return dottedPath; + } + public TruffleString[] getDottedPath(Object obj, TruffleString name) { - TruffleString[] dottedPath = StringUtils.split(name, T_DOT, ensureTsCodePointLengthNode(), ensureTsIndexOfStringNode(), ensureTsSubstringNode(), ensureTsEqualNode()); - assert dottedPath.length > 0; + TruffleString[] dottedPath = splitDottedPath(name); for (int i = 0; i < dottedPath.length; i++) { if (ensureTsEqualNode().execute(dottedPath[i], T_LOCALS, TS_ENCODING)) { if (obj == null) { From d0c6a40826ad3273d05dc5be9e89bf6418be18b7 Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Thu, 1 Oct 2026 22:54:32 +0200 Subject: [PATCH 30/39] Cache unpickled names --- .../builtins/modules/pickle/PPickler.java | 6 +-- .../builtins/modules/pickle/PicklerNodes.java | 48 +++++++++++++++++-- 2 files changed, 47 insertions(+), 7 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java index 77df6fbd65..74a87d4cce 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java @@ -697,7 +697,7 @@ public TruffleString asStringStrict(Object value) { return string; } - private static boolean checkModule(VirtualFrame frame, PyObjectLookupAttr lookup, Object moduleName, Object module, Object global, TruffleString[] dottedPath) { + private static boolean checkModule(VirtualFrame frame, PicklerNodes.LookupGlobalAttributeNode lookup, Object moduleName, Object module, Object global, TruffleString[] dottedPath) { if (module == PNone.NONE) { return false; } @@ -728,7 +728,7 @@ public TruffleString whichModule(VirtualFrame frame, PythonContext context, Obje while (nextNode.executeCached(storage, it)) { Object value = getValueNode.executeCached(storage, it); Object key = getKeyNode.executeCached(storage, it); - if (checkModule(frame, getLookupAttrNode(), moduleName, value, global, dottedPath)) { + if (checkModule(frame, getLookupGlobalAttrNode(), moduleName, value, global, dottedPath)) { return asStringStrict(key); } } @@ -2182,7 +2182,7 @@ private void saveGlobal(VirtualFrame frame, PythonContext ctx, PPickler pickler, } catch (PException e) { throw raise(PicklingError, ErrorMessages.CANT_PICKLE_P_IMPORT_OF_MODULE_S_FAILED, obj, moduleName); } - final Pair pair = getDeepAttribute(frame, getLookupAttrNode(), module, dottedPath); + final Pair pair = getDeepAttribute(frame, getLookupGlobalAttrNode(), module, dottedPath); if (pair == null) { throw raise(PicklingError, ErrorMessages.CANT_PICKLE_P_ATTR_LOOKUP_FAIL_S_S, obj, globalName, moduleName); } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java index 58eefbaac4..3c35ea9f85 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java @@ -102,6 +102,11 @@ import com.oracle.graal.python.runtime.sequence.storage.SequenceStorage; import com.oracle.graal.python.util.PythonUtils; import com.oracle.truffle.api.CompilerDirectives; +import com.oracle.truffle.api.dsl.Cached; +import com.oracle.truffle.api.dsl.Cached.Exclusive; +import com.oracle.truffle.api.dsl.GenerateInline; +import com.oracle.truffle.api.dsl.ImportStatic; +import com.oracle.truffle.api.dsl.Specialization; import com.oracle.truffle.api.frame.Frame; import com.oracle.truffle.api.frame.VirtualFrame; import com.oracle.truffle.api.nodes.LoopNode; @@ -110,6 +115,28 @@ import com.oracle.truffle.api.strings.TruffleString; public final class PicklerNodes { + @GenerateInline(false) + @ImportStatic(PythonUtils.class) + abstract static class LookupGlobalAttributeNode extends Node { + abstract Object execute(VirtualFrame frame, Object receiver, TruffleString name); + + // Unpickling decodes fresh strings. Give attribute lookup a stable name identity without + // caching the attribute value or interning arbitrary pickle contents. + @Specialization(guards = "equal.execute(name, cachedName, TS_ENCODING)", limit = "3") + static Object cached(VirtualFrame frame, Object receiver, TruffleString name, + @Cached("name") TruffleString cachedName, + @Cached TruffleString.EqualNode equal, + @Exclusive @Cached(inline = false) PyObjectLookupAttr lookup) { + return lookup.executeCached(frame, receiver, cachedName); + } + + @Specialization(replaces = "cached") + static Object generic(VirtualFrame frame, Object receiver, TruffleString name, + @Exclusive @Cached(inline = false) PyObjectLookupAttr lookup) { + return lookup.executeCached(frame, receiver, name); + } + } + abstract static class BasePickleNode extends Node { private static final TruffleString T_LOCALS = tsLiteral(""); public static final TruffleString T_CODEC_RAW_UNICODE_ESCAPE = tsLiteral("raw_unicode_escape"); @@ -143,6 +170,7 @@ abstract static class BasePickleNode extends Node { @Child private GetClassNode getClassNode; @Child private PyObjectSizeNode sizeNode; @Child private PyObjectLookupAttr lookupAttrNode; + @Child private LookupGlobalAttributeNode lookupGlobalAttrNode; @Child private BytesNodes.ToBytesNode toBytesNode; @Child private PyObjectReprAsTruffleStringNode reprNode; @Child private TruffleString.FromByteArrayNode tsFromByteArrayNode; @@ -398,6 +426,14 @@ protected PyObjectLookupAttr getLookupAttrNode() { return lookupAttrNode; } + protected LookupGlobalAttributeNode getLookupGlobalAttrNode() { + if (lookupGlobalAttrNode == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + lookupGlobalAttrNode = insert(PicklerNodesFactory.LookupGlobalAttributeNodeGen.create()); + } + return lookupGlobalAttrNode; + } + protected Object lookupAttribute(Frame frame, Object receiver, TruffleString name) { return getLookupAttrNode().executeCached(frame, receiver, name); } @@ -557,12 +593,12 @@ public TruffleString asString(Object value) { } } - public static Pair getDeepAttribute(VirtualFrame frame, PyObjectLookupAttr lookup, Object obj, TruffleString[] names) { + public static Pair getDeepAttribute(VirtualFrame frame, LookupGlobalAttributeNode lookup, Object obj, TruffleString[] names) { Object parent = null; Object object = obj; for (int i = 0; i < names.length; i++) { parent = object; - object = lookup.executeCached(frame, parent, names[i]); + object = lookup.execute(frame, parent, names[i]); if (object == NO_VALUE) { LoopNode.reportLoopCount(lookup, i); return null; @@ -621,14 +657,18 @@ public TruffleString[] getDottedPath(Object obj, TruffleString name) { public Object getattribute(VirtualFrame frame, Object obj, TruffleString name, boolean allowQualname) { if (allowQualname) { TruffleString[] dottedPath = getDottedPath(obj, name); - Pair result = getDeepAttribute(frame, getLookupAttrNode(), obj, dottedPath); + Pair result = getDeepAttribute(frame, getLookupGlobalAttrNode(), obj, dottedPath); if (result != null) { return result.getLeft(); } else { throw raise(AttributeError, ErrorMessages.CANT_GET_ATTRIBUTE_S_ON_S, name, getReprNode().execute(frame, null, obj)); } } else { - return lookupAttributeStrict(frame, obj, name); + Object attr = getLookupGlobalAttrNode().execute(frame, obj, name); + if (attr == NO_VALUE) { + throw raise(TypeError, ErrorMessages.OBJ_P_HAS_NO_ATTR_S, attr, name); + } + return attr; } } From 2934d6d06f051082ece0dedf588e7ea0c2697148 Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Thu, 1 Oct 2026 23:16:07 +0200 Subject: [PATCH 31/39] Use ImportName in unpickler --- .../builtins/modules/pickle/PUnpickler.java | 35 +++++++++---------- .../builtins/modules/pickle/PicklerNodes.java | 34 +++++++++++++----- 2 files changed, 42 insertions(+), 27 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PUnpickler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PUnpickler.java index f534ad21d8..bbec803c47 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PUnpickler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PUnpickler.java @@ -174,7 +174,6 @@ import com.oracle.graal.python.nodes.PRaiseNode; import com.oracle.graal.python.nodes.argument.keywords.ExpandKeywordStarargsNode; import com.oracle.graal.python.nodes.argument.positional.ExecutePositionalStarargsNode; -import com.oracle.graal.python.runtime.IndirectCallData.BoundaryCallData; import com.oracle.graal.python.runtime.IndirectCallData.InteropCallData; import com.oracle.graal.python.runtime.PythonContext; import com.oracle.graal.python.runtime.exception.PException; @@ -684,9 +683,8 @@ public abstract static class FindClassNode extends BasePickleReadNode { public abstract Object execute(VirtualFrame frame, PUnpickler unpickler, TruffleString module, TruffleString name); @Specialization - Object find(VirtualFrame frame, PUnpickler unpickler, TruffleString module, TruffleString name, - @Cached("createFor($node)") BoundaryCallData boundaryCallData) { - return findClass(frame, boundaryCallData, PythonContext.get(this).getCore(), unpickler, module, name); + Object find(VirtualFrame frame, PUnpickler unpickler, TruffleString module, TruffleString name) { + return findClass(frame, PythonContext.get(this).getCore(), unpickler, module, name); } } @@ -1331,7 +1329,7 @@ private void loadObj(VirtualFrame frame, Node inliningTarget, PUnpickler self, P pDataPush(self, obj); } - private void loadInst(VirtualFrame frame, Node inliningTarget, BoundaryCallData boundaryCallData, PythonContext ctx, PUnpickler self, PyObjectCallMethodObjArgs callMethod) { + private void loadInst(VirtualFrame frame, Node inliningTarget, PythonContext ctx, PUnpickler self, PyObjectCallMethodObjArgs callMethod) { Object cls = null; Object obj = null; int i = marker(self); @@ -1351,7 +1349,7 @@ private void loadInst(VirtualFrame frame, Node inliningTarget, BoundaryCallData throw badReadLine(); } Object className = decodeASCII(frame, s, s.length - 1, T_ERRORS_STRICT); - cls = findClass(frame, boundaryCallData, ctx.getCore(), self, moduleName, className); + cls = findClass(frame, ctx.getCore(), self, moduleName, className); } assert cls != null; @@ -1451,7 +1449,7 @@ private void doAppend(VirtualFrame frame, PUnpickler self, int x) { } } - private void loadGlobal(VirtualFrame frame, BoundaryCallData boundaryCallData, PythonContext ctx, PUnpickler self) { + private void loadGlobal(VirtualFrame frame, PythonContext ctx, PUnpickler self) { Object global = null; TruffleString globalName; byte[] s = readLine(frame, self); @@ -1467,14 +1465,14 @@ private void loadGlobal(VirtualFrame frame, BoundaryCallData boundaryCallData, P } globalName = PickleUtils.decodeUTF8Strict(s, s.length - 1, ensureTsFromByteArray(), ensureTsSwitchEncodingNode()); if (globalName != null) { - global = findClass(frame, boundaryCallData, ctx.getCore(), self, moduleName, globalName); + global = findClass(frame, ctx.getCore(), self, moduleName, globalName); } } pDataPush(self, global); } - private void loadStackGlobal(VirtualFrame frame, BoundaryCallData boundaryCallData, PythonContext ctx, PUnpickler self) { + private void loadStackGlobal(VirtualFrame frame, PythonContext ctx, PUnpickler self) { Object globalName = null; Object moduleName = null; try { @@ -1486,7 +1484,7 @@ private void loadStackGlobal(VirtualFrame frame, BoundaryCallData boundaryCallDa if (!PGuards.isString(moduleName) || !PGuards.isString(globalName)) { throw raise(PythonBuiltinClassType.UnpicklingError, ErrorMessages.S_REQ_STR, "STACK_GLOBAL"); } - Object global = findClass(frame, boundaryCallData, ctx.getCore(), self, moduleName, globalName); + Object global = findClass(frame, ctx.getCore(), self, moduleName, globalName); pDataPush(self, global); } @@ -1841,7 +1839,7 @@ private void loadFrame(VirtualFrame frame, PUnpickler self) { self.nextReadIdx -= frameLen; } - private void loadExtension(VirtualFrame frame, BoundaryCallData boundaryCallData, PythonContext ctx, PUnpickler self, int nbytes) { + private void loadExtension(VirtualFrame frame, PythonContext ctx, PUnpickler self, int nbytes) { assert (nbytes == 1 || nbytes == 2 || nbytes == 4); // the nbytes bytes after the opcode int codeOffset = read(frame, self, nbytes); @@ -1887,7 +1885,7 @@ private void loadExtension(VirtualFrame frame, BoundaryCallData boundaryCallData } // Load the object. - obj = findClass(frame, boundaryCallData, ctx.getCore(), self, moduleName, className); + obj = findClass(frame, ctx.getCore(), self, moduleName, className); // Cache code -> obj. setDictItem(frame, st.extensionCache, code, obj); @@ -1901,7 +1899,6 @@ private void loadBool(PUnpickler self, boolean bool) { @Specialization public Object load(VirtualFrame frame, PUnpickler self, @Bind Node inliningTarget, - @Cached("createFor($node)") BoundaryCallData boundaryCallData, @Cached GetCachedTpSlotsNode getSlots, @Cached CallSlotTpNewNode callNew, @Cached ExecutePositionalStarargsNode expandArgs, @@ -2072,7 +2069,7 @@ public Object load(VirtualFrame frame, PUnpickler self, continue; case OPCODE_INST: profileSeen(SEEN_INST); - loadInst(frame, inliningTarget, boundaryCallData, ctx, self, callMethod); + loadInst(frame, inliningTarget, ctx, self, callMethod); continue; case OPCODE_NEWOBJ: profileSeen(SEEN_NEWOBJ); @@ -2084,11 +2081,11 @@ public Object load(VirtualFrame frame, PUnpickler self, continue; case OPCODE_GLOBAL: profileSeen(SEEN_GLOBAL); - loadGlobal(frame, boundaryCallData, ctx, self); + loadGlobal(frame, ctx, self); continue; case OPCODE_STACK_GLOBAL: profileSeen(SEEN_STACK_GLOBAL); - loadStackGlobal(frame, boundaryCallData, ctx, self); + loadStackGlobal(frame, ctx, self); continue; case OPCODE_APPEND: profileSeen(SEEN_APPEND); @@ -2176,15 +2173,15 @@ public Object load(VirtualFrame frame, PUnpickler self, continue; case OPCODE_EXT1: profileSeen(SEEN_EXT1); - loadExtension(frame, boundaryCallData, ctx, self, 1); + loadExtension(frame, ctx, self, 1); continue; case OPCODE_EXT2: profileSeen(SEEN_EXT2); - loadExtension(frame, boundaryCallData, ctx, self, 2); + loadExtension(frame, ctx, self, 2); continue; case OPCODE_EXT4: profileSeen(SEEN_EXT4); - loadExtension(frame, boundaryCallData, ctx, self, 4); + loadExtension(frame, ctx, self, 4); continue; case OPCODE_NEWTRUE: profileSeen(SEEN_NEWTRUE); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java index 3c35ea9f85..93a2fded58 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java @@ -47,8 +47,8 @@ import static com.oracle.graal.python.builtins.objects.PNone.NO_VALUE; import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___CLASS__; import static com.oracle.graal.python.nodes.StringLiterals.T_UTF8; -import static com.oracle.graal.python.nodes.statement.AbstractImportNode.importModule; import static com.oracle.graal.python.runtime.exception.PythonErrorType.AttributeError; +import static com.oracle.graal.python.runtime.exception.PythonErrorType.KeyError; import static com.oracle.graal.python.runtime.exception.PythonErrorType.TypeError; import static com.oracle.graal.python.util.PythonUtils.TS_ENCODING; import static com.oracle.graal.python.util.PythonUtils.tsLiteral; @@ -60,6 +60,7 @@ import com.oracle.graal.python.builtins.PythonBuiltinClassType; import com.oracle.graal.python.builtins.modules.CodecsModuleBuiltins; import com.oracle.graal.python.builtins.modules.CodecsModuleBuiltinsFactory; +import com.oracle.graal.python.builtins.objects.PNone; import com.oracle.graal.python.builtins.objects.bytes.BytesNodes; import com.oracle.graal.python.builtins.objects.common.HashingStorage; import com.oracle.graal.python.builtins.objects.common.HashingStorageNodes.CachedHashingStorageGetItem; @@ -73,6 +74,7 @@ import com.oracle.graal.python.builtins.objects.common.SequenceStorageNodes; import com.oracle.graal.python.builtins.objects.dict.PDict; import com.oracle.graal.python.builtins.objects.function.PKeyword; +import com.oracle.graal.python.builtins.objects.module.PythonModule; import com.oracle.graal.python.builtins.objects.tuple.PTuple; import com.oracle.graal.python.lib.PyIterCheckNode; import com.oracle.graal.python.lib.PyIterNextNode; @@ -94,9 +96,11 @@ import com.oracle.graal.python.nodes.object.BuiltinClassProfiles.InlineIsBuiltinClassProfile; import com.oracle.graal.python.nodes.object.BuiltinClassProfiles.IsBuiltinObjectProfile; import com.oracle.graal.python.nodes.object.GetClassNode; +import com.oracle.graal.python.nodes.statement.AbstractImportNode.ImportName; +import com.oracle.graal.python.nodes.statement.AbstractImportNodeFactory.ImportNameNodeGen; import com.oracle.graal.python.nodes.util.CannotCastException; import com.oracle.graal.python.nodes.util.CastToTruffleStringNode; -import com.oracle.graal.python.runtime.IndirectCallData.BoundaryCallData; +import com.oracle.graal.python.runtime.PythonContext; import com.oracle.graal.python.runtime.exception.PException; import com.oracle.graal.python.runtime.object.PFactory; import com.oracle.graal.python.runtime.sequence.storage.SequenceStorage; @@ -160,6 +164,7 @@ abstract static class BasePickleNode extends Node { @Child private CastToTruffleStringNode castToTruffleStringNode; @Child private SequenceNodes.GetSequenceStorageNode getSequenceStorageNode; @Child private CallNode callNode; + @Child private ImportName importNameNode; @Child private ExecutePositionalStarargsNode getArgsNode; @Child private ExpandKeywordStarargsNode getKwArgsNode; @Child private PyLongFromUnicodeObject pyLongFromUnicodeObject; @@ -711,11 +716,11 @@ private Pair getMapping(VirtualFrame frame, PDict return Pair.create(moduleName, globalName); } - public Object findClass(VirtualFrame frame, BoundaryCallData boundaryCallData, Python3Core core, PUnpickler self, Object moduleName, Object globalName) { - return findClass(frame, boundaryCallData, core, self, castToString(moduleName), castToString(globalName)); + public Object findClass(VirtualFrame frame, Python3Core core, PUnpickler self, Object moduleName, Object globalName) { + return findClass(frame, core, self, castToString(moduleName), castToString(globalName)); } - public Object findClass(VirtualFrame frame, BoundaryCallData boundaryCallData, Python3Core core, PUnpickler self, TruffleString moduleName, TruffleString globalName) { + public Object findClass(VirtualFrame frame, Python3Core core, PUnpickler self, TruffleString moduleName, TruffleString globalName) { // Try to map the old names used in Python 2.x to the new ones used in Python 3.x. We do // this only with old pickle protocols and when the user has not disabled the feature. TruffleString mName = moduleName; @@ -726,9 +731,22 @@ public Object findClass(VirtualFrame frame, BoundaryCallData boundaryCallData, P gName = to3Mapping.getRight(); } - // we don't use PyImport_GetModule here, because it can return partially-initialised - // modules, which then cause the getattribute to fail. - Object module = importModule(frame, boundaryCallData, mName); + if (importNameNode == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + importNameNode = insert(ImportNameNodeGen.create()); + } + // ImportName checks initialization for cached modules and honors custom import hooks. + // With an empty fromlist, a dotted import returns the top-level package, so look up + // the requested module by its full name after importing for side effects. + importNameNode.execute(frame, PythonContext.get(this), core.getBuiltins(), mName, PNone.NONE, PythonUtils.EMPTY_TRUFFLESTRING_ARRAY, 0); + Object module = getDictItem(frame, core.getSysModules(), mName); + if (module == null) { + errorProfile.enter(); + throw PRaiseNode.raiseStatic(this, KeyError, new Object[]{mName}); + } + if (!(module instanceof PythonModule)) { + throw raise(PythonBuiltinClassType.NotImplementedError, ErrorMessages.PUTTING_NON_MODULE_OBJECTS_IN_SYS_MODULES_IS_NOT_SUPPORTED); + } return getattribute(frame, module, gName, self.getProto() >= 4); } } From 427c4fb120c4f5f796ea211a253ba6be579c1a66 Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Thu, 1 Oct 2026 23:36:30 +0200 Subject: [PATCH 32/39] Push members down from BasePickleNode --- .../builtins/modules/pickle/PPickler.java | 14 +-- .../builtins/modules/pickle/PUnpickler.java | 90 ++++++++++++++++- .../builtins/modules/pickle/PicklerNodes.java | 96 +------------------ 3 files changed, 99 insertions(+), 101 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java index 74a87d4cce..3ef61a98cc 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java @@ -40,6 +40,8 @@ */ package com.oracle.graal.python.builtins.modules.pickle; +import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.T_CP_REVERSE_IMPORT_MAPPING; +import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.T_CP_REVERSE_NAME_MAPPING; import static com.oracle.graal.python.builtins.objects.PNone.NO_VALUE; import static com.oracle.graal.python.nodes.BuiltinNames.T___MAIN__; import static com.oracle.graal.python.nodes.ErrorMessages.MUST_BE_STR_NOT_P; @@ -68,6 +70,7 @@ import org.graalvm.collections.Pair; import com.oracle.graal.python.PythonLanguage; +import com.oracle.graal.python.builtins.Python3Core; import com.oracle.graal.python.builtins.PythonBuiltinClassType; import com.oracle.graal.python.builtins.objects.PNone; import com.oracle.graal.python.builtins.objects.PNotImplemented; @@ -481,12 +484,6 @@ private void writeIntOp(Node node, byte opcode, long value) { outputBuffer[outputLen++] = intByte4(value); } - private void write(Node node, byte[] bytes, int dataLen) { - ensureBufferSpace(node, dataLen); - PythonUtils.arraycopy(bytes, 0, outputBuffer, outputLen, dataLen); - outputLen += dataLen; - } - private void ensureBufferSpace(Node node, int dataLen) { boolean needNewFrame = isFraming() && frameStart == -1; int n = CompilerDirectives.injectBranchProbability(CompilerDirectives.SLOWPATH_PROBABILITY, needNewFrame) ? dataLen + PickleUtils.FRAME_HEADER_SIZE : dataLen; @@ -2160,6 +2157,11 @@ private void saveType(VirtualFrame frame, PythonContext ctx, PPickler pickler, i } } + protected Pair get3to2Mapping(VirtualFrame frame, Python3Core core, TruffleString moduleName, TruffleString globalName) { + PickleState state = getGlobalState(core); + return getMapping(frame, state.nameMapping3To2, state.importMapping3To2, T_CP_REVERSE_NAME_MAPPING, T_CP_REVERSE_IMPORT_MAPPING, moduleName, globalName); + } + private void saveGlobal(VirtualFrame frame, PythonContext ctx, PPickler pickler, int proto, boolean fastMode, Object obj, Object name) { PickleState st = getGlobalState(ctx.getCore()); Object gName; diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PUnpickler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PUnpickler.java index bbec803c47..56f7e032aa 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PUnpickler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PUnpickler.java @@ -111,12 +111,15 @@ import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.PICKLE_PROTOCOL_HIGHEST; import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.PREFETCH; import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.READ_WHOLE_LINE; +import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.T_CP_IMPORT_MAPPING; +import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.T_CP_NAME_MAPPING; import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.T_METHOD_PEEK; import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.T_METHOD_PERSISTENT_LOAD; import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.T_METHOD_READ; import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.T_METHOD_READINTO; import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.T_METHOD_READLINE; import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.getValidIntString; +import static com.oracle.graal.python.builtins.objects.PNone.NO_VALUE; import static com.oracle.graal.python.nodes.BuiltinNames.T_ADD; import static com.oracle.graal.python.nodes.BuiltinNames.T_APPEND; import static com.oracle.graal.python.nodes.BuiltinNames.T_EXTEND; @@ -126,11 +129,15 @@ import static com.oracle.graal.python.nodes.SpecialMethodNames.T___SETSTATE__; import static com.oracle.graal.python.nodes.StringLiterals.T_ASCII_UPPERCASE; import static com.oracle.graal.python.nodes.StringLiterals.T_STRICT; +import static com.oracle.graal.python.runtime.exception.PythonErrorType.AttributeError; +import static com.oracle.graal.python.runtime.exception.PythonErrorType.KeyError; +import static com.oracle.graal.python.runtime.exception.PythonErrorType.TypeError; import static com.oracle.graal.python.util.PythonUtils.TS_ENCODING; import org.graalvm.collections.Pair; import com.oracle.graal.python.PythonLanguage; +import com.oracle.graal.python.builtins.Python3Core; import com.oracle.graal.python.builtins.PythonBuiltinClassType; import com.oracle.graal.python.builtins.objects.PNone; import com.oracle.graal.python.builtins.objects.buffer.BufferFlags; @@ -155,6 +162,7 @@ import com.oracle.graal.python.builtins.objects.list.PList; import com.oracle.graal.python.builtins.objects.memoryview.MemoryViewBuiltins; import com.oracle.graal.python.builtins.objects.memoryview.MemoryViewBuiltinsFactory; +import com.oracle.graal.python.builtins.objects.module.PythonModule; import com.oracle.graal.python.builtins.objects.object.PythonBuiltinObject; import com.oracle.graal.python.builtins.objects.set.PSet; import com.oracle.graal.python.builtins.objects.tuple.PTuple; @@ -166,6 +174,7 @@ import com.oracle.graal.python.lib.PyObjectCallMethodObjArgs; import com.oracle.graal.python.lib.PyObjectGetIter; import com.oracle.graal.python.lib.PyObjectLookupAttr; +import com.oracle.graal.python.lib.PyObjectReprAsTruffleStringNode; import com.oracle.graal.python.lib.PyObjectSetAttrO; import com.oracle.graal.python.lib.PyObjectSetItem; import com.oracle.graal.python.lib.PyTupleCheckNode; @@ -174,6 +183,9 @@ import com.oracle.graal.python.nodes.PRaiseNode; import com.oracle.graal.python.nodes.argument.keywords.ExpandKeywordStarargsNode; import com.oracle.graal.python.nodes.argument.positional.ExecutePositionalStarargsNode; +import com.oracle.graal.python.nodes.statement.AbstractImportNode.ImportName; +import com.oracle.graal.python.nodes.statement.AbstractImportNodeFactory.ImportNameNodeGen; +import com.oracle.graal.python.nodes.util.CastToTruffleStringNode; import com.oracle.graal.python.runtime.IndirectCallData.InteropCallData; import com.oracle.graal.python.runtime.PythonContext; import com.oracle.graal.python.runtime.exception.PException; @@ -396,6 +408,9 @@ public void clearMemo() { // inner nodes public abstract static class BasePickleReadNode extends PicklerNodes.BasePickleNode { + @Child private CastToTruffleStringNode castToTruffleStringNode; + @Child private ImportName importNameNode; + @Child private PyObjectReprAsTruffleStringNode reprNode; @Child private PyMemoryViewFromObject memoryViewNode; @Child private MemoryViewBuiltins.ToReadonlyNode toReadonlyNode; @Child private PythonBufferAcquireLibrary bufferAcquireLibrary; @@ -409,7 +424,80 @@ public abstract static class BasePickleReadNode extends PicklerNodes.BasePickleN // we only need the reference, doesn't matter that the object may not yet be fully // constructed - @SuppressWarnings("this-escape") private InteropCallData interopCallData = InteropCallData.createFor(this); + @SuppressWarnings("this-escape") private final InteropCallData interopCallData = InteropCallData.createFor(this); + + protected TruffleString castToString(Object value) { + if (castToTruffleStringNode == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + castToTruffleStringNode = insert(CastToTruffleStringNode.create()); + } + return castToTruffleStringNode.executeCached(value); + } + + private PyObjectReprAsTruffleStringNode getReprNode() { + if (reprNode == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + reprNode = insert(PyObjectReprAsTruffleStringNode.create()); + } + return reprNode; + } + + public Object getattribute(VirtualFrame frame, Object obj, TruffleString name, boolean allowQualname) { + if (allowQualname) { + TruffleString[] dottedPath = getDottedPath(obj, name); + Pair result = getDeepAttribute(frame, getLookupGlobalAttrNode(), obj, dottedPath); + if (result != null) { + return result.getLeft(); + } else { + throw raise(AttributeError, ErrorMessages.CANT_GET_ATTRIBUTE_S_ON_S, name, getReprNode().execute(frame, null, obj)); + } + } else { + Object attr = getLookupGlobalAttrNode().execute(frame, obj, name); + if (attr == NO_VALUE) { + throw raise(TypeError, ErrorMessages.OBJ_P_HAS_NO_ATTR_S, attr, name); + } + return attr; + } + } + + protected Pair get2To3Mapping(VirtualFrame frame, Python3Core core, TruffleString moduleName, TruffleString globalName) { + PickleState state = getGlobalState(core); + return getMapping(frame, state.nameMapping2To3, state.importMapping2To3, T_CP_NAME_MAPPING, T_CP_IMPORT_MAPPING, moduleName, globalName); + } + + public Object findClass(VirtualFrame frame, Python3Core core, PUnpickler self, Object moduleName, Object globalName) { + return findClass(frame, core, self, castToString(moduleName), castToString(globalName)); + } + + public Object findClass(VirtualFrame frame, Python3Core core, PUnpickler self, TruffleString moduleName, TruffleString globalName) { + // Try to map the old names used in Python 2.x to the new ones used in Python 3.x. We do + // this only with old pickle protocols and when the user has not disabled the feature. + TruffleString mName = moduleName; + TruffleString gName = globalName; + if (self.getProto() < 3 && self.isFixImports()) { + final Pair to3Mapping = get2To3Mapping(frame, core, mName, gName); + mName = to3Mapping.getLeft(); + gName = to3Mapping.getRight(); + } + + if (importNameNode == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + importNameNode = insert(ImportNameNodeGen.create()); + } + // ImportName checks initialization for cached modules and honors custom import hooks. + // With an empty fromlist, a dotted import returns the top-level package, so look up + // the requested module by its full name after importing for side effects. + importNameNode.execute(frame, PythonContext.get(this), core.getBuiltins(), mName, PNone.NONE, PythonUtils.EMPTY_TRUFFLESTRING_ARRAY, 0); + Object module = getDictItem(frame, core.getSysModules(), mName); + if (module == null) { + errorProfile.enter(); + throw PRaiseNode.raiseStatic(this, KeyError, new Object[]{mName}); + } + if (!(module instanceof PythonModule)) { + throw raise(PythonBuiltinClassType.NotImplementedError, ErrorMessages.PUTTING_NON_MODULE_OBJECTS_IN_SYS_MODULES_IS_NOT_SUPPORTED); + } + return getattribute(frame, module, gName, self.getProto() >= 4); + } protected TruffleString.ParseLongNode ensureTsParseLongNode() { if (tsParseLongNode == null) { diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java index 93a2fded58..157db2b791 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java @@ -40,15 +40,10 @@ */ package com.oracle.graal.python.builtins.modules.pickle; -import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.T_CP_IMPORT_MAPPING; -import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.T_CP_NAME_MAPPING; -import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.T_CP_REVERSE_IMPORT_MAPPING; -import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.T_CP_REVERSE_NAME_MAPPING; import static com.oracle.graal.python.builtins.objects.PNone.NO_VALUE; import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___CLASS__; import static com.oracle.graal.python.nodes.StringLiterals.T_UTF8; import static com.oracle.graal.python.runtime.exception.PythonErrorType.AttributeError; -import static com.oracle.graal.python.runtime.exception.PythonErrorType.KeyError; import static com.oracle.graal.python.runtime.exception.PythonErrorType.TypeError; import static com.oracle.graal.python.util.PythonUtils.TS_ENCODING; import static com.oracle.graal.python.util.PythonUtils.tsLiteral; @@ -60,7 +55,6 @@ import com.oracle.graal.python.builtins.PythonBuiltinClassType; import com.oracle.graal.python.builtins.modules.CodecsModuleBuiltins; import com.oracle.graal.python.builtins.modules.CodecsModuleBuiltinsFactory; -import com.oracle.graal.python.builtins.objects.PNone; import com.oracle.graal.python.builtins.objects.bytes.BytesNodes; import com.oracle.graal.python.builtins.objects.common.HashingStorage; import com.oracle.graal.python.builtins.objects.common.HashingStorageNodes.CachedHashingStorageGetItem; @@ -74,7 +68,6 @@ import com.oracle.graal.python.builtins.objects.common.SequenceStorageNodes; import com.oracle.graal.python.builtins.objects.dict.PDict; import com.oracle.graal.python.builtins.objects.function.PKeyword; -import com.oracle.graal.python.builtins.objects.module.PythonModule; import com.oracle.graal.python.builtins.objects.tuple.PTuple; import com.oracle.graal.python.lib.PyIterCheckNode; import com.oracle.graal.python.lib.PyIterNextNode; @@ -82,7 +75,6 @@ import com.oracle.graal.python.lib.PyNumberAsSizeNode; import com.oracle.graal.python.lib.PyObjectGetItem; import com.oracle.graal.python.lib.PyObjectLookupAttr; -import com.oracle.graal.python.lib.PyObjectReprAsTruffleStringNode; import com.oracle.graal.python.lib.PyObjectSetItem; import com.oracle.graal.python.lib.PyObjectSizeNode; import com.oracle.graal.python.nodes.ErrorMessages; @@ -96,11 +88,8 @@ import com.oracle.graal.python.nodes.object.BuiltinClassProfiles.InlineIsBuiltinClassProfile; import com.oracle.graal.python.nodes.object.BuiltinClassProfiles.IsBuiltinObjectProfile; import com.oracle.graal.python.nodes.object.GetClassNode; -import com.oracle.graal.python.nodes.statement.AbstractImportNode.ImportName; -import com.oracle.graal.python.nodes.statement.AbstractImportNodeFactory.ImportNameNodeGen; import com.oracle.graal.python.nodes.util.CannotCastException; import com.oracle.graal.python.nodes.util.CastToTruffleStringNode; -import com.oracle.graal.python.runtime.PythonContext; import com.oracle.graal.python.runtime.exception.PException; import com.oracle.graal.python.runtime.object.PFactory; import com.oracle.graal.python.runtime.sequence.storage.SequenceStorage; @@ -161,10 +150,8 @@ abstract static class BasePickleNode extends Node { @Child private CachedHashingStorageGetItem getHashingStorageItemNode; @Child private SequenceStorageNodes.GetItemScalarNode getSeqStorageItemNode; @Child private PyNumberAsSizeNode asSizeNode; - @Child private CastToTruffleStringNode castToTruffleStringNode; @Child private SequenceNodes.GetSequenceStorageNode getSequenceStorageNode; @Child private CallNode callNode; - @Child private ImportName importNameNode; @Child private ExecutePositionalStarargsNode getArgsNode; @Child private ExpandKeywordStarargsNode getKwArgsNode; @Child private PyLongFromUnicodeObject pyLongFromUnicodeObject; @@ -177,7 +164,6 @@ abstract static class BasePickleNode extends Node { @Child private PyObjectLookupAttr lookupAttrNode; @Child private LookupGlobalAttributeNode lookupGlobalAttrNode; @Child private BytesNodes.ToBytesNode toBytesNode; - @Child private PyObjectReprAsTruffleStringNode reprNode; @Child private TruffleString.FromByteArrayNode tsFromByteArrayNode; @Child private TruffleString.FromByteArrayWithCompactionUTF32Node tsFromByteArrayWithCompactionNode; @Child private TruffleString.CodePointLengthNode tsCodePointLengthNode; @@ -190,7 +176,7 @@ abstract static class BasePickleNode extends Node { @Child private HashingStorageIteratorNext hashingStorageItNext; @Child private HashingStorageIteratorKey hashingStorageItKey; @Child private HashingStorageIteratorValue hashingStorageItValue; - private final BranchProfile errorProfile = BranchProfile.create(); + protected final BranchProfile errorProfile = BranchProfile.create(); protected PException raise(PythonBuiltinClassType type, TruffleString string) { errorProfile.enter(); @@ -279,14 +265,6 @@ protected byte[] toBytes(VirtualFrame frame, Object obj) { return toBytesNode.execute(frame, obj); } - private PyObjectReprAsTruffleStringNode getReprNode() { - if (reprNode == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - reprNode = insert(PyObjectReprAsTruffleStringNode.create()); - } - return reprNode; - } - protected HashingStorageIterator getHashingStorageIterator(HashingStorage s) { if (getHashingStorageIteratorNode == null) { CompilerDirectives.transferToInterpreterAndInvalidate(); @@ -488,14 +466,6 @@ protected SequenceStorage getSequenceStorage(Object iterator) { return getSequenceStorageNode.executeCached(iterator); } - protected TruffleString castToString(Object value) { - if (castToTruffleStringNode == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - castToTruffleStringNode = insert(CastToTruffleStringNode.create()); - } - return castToTruffleStringNode.executeCached(value); - } - public int asSizeExact(VirtualFrame frame, Object pyNumber) { if (asSizeNode == null) { CompilerDirectives.transferToInterpreterAndInvalidate(); @@ -659,35 +629,7 @@ public TruffleString[] getDottedPath(Object obj, TruffleString name) { return dottedPath; } - public Object getattribute(VirtualFrame frame, Object obj, TruffleString name, boolean allowQualname) { - if (allowQualname) { - TruffleString[] dottedPath = getDottedPath(obj, name); - Pair result = getDeepAttribute(frame, getLookupGlobalAttrNode(), obj, dottedPath); - if (result != null) { - return result.getLeft(); - } else { - throw raise(AttributeError, ErrorMessages.CANT_GET_ATTRIBUTE_S_ON_S, name, getReprNode().execute(frame, null, obj)); - } - } else { - Object attr = getLookupGlobalAttrNode().execute(frame, obj, name); - if (attr == NO_VALUE) { - throw raise(TypeError, ErrorMessages.OBJ_P_HAS_NO_ATTR_S, attr, name); - } - return attr; - } - } - - protected Pair get3to2Mapping(VirtualFrame frame, Python3Core core, TruffleString moduleName, TruffleString globalName) { - PickleState state = getGlobalState(core); - return getMapping(frame, state.nameMapping3To2, state.importMapping3To2, T_CP_REVERSE_NAME_MAPPING, T_CP_REVERSE_IMPORT_MAPPING, moduleName, globalName); - } - - protected Pair get2To3Mapping(VirtualFrame frame, Python3Core core, TruffleString moduleName, TruffleString globalName) { - PickleState state = getGlobalState(core); - return getMapping(frame, state.nameMapping2To3, state.importMapping2To3, T_CP_NAME_MAPPING, T_CP_IMPORT_MAPPING, moduleName, globalName); - } - - private Pair getMapping(VirtualFrame frame, PDict nameMapping, PDict importMapping, + protected Pair getMapping(VirtualFrame frame, PDict nameMapping, PDict importMapping, TruffleString nameMappingLabel, TruffleString importMappingLabel, TruffleString moduleName, TruffleString globalName) { Object key = PFactory.createTuple(PythonLanguage.get(this), new Object[]{moduleName, globalName}); Object item = getDictItem(frame, nameMapping, key); @@ -715,39 +657,5 @@ private Pair getMapping(VirtualFrame frame, PDict return Pair.create(moduleName, globalName); } - - public Object findClass(VirtualFrame frame, Python3Core core, PUnpickler self, Object moduleName, Object globalName) { - return findClass(frame, core, self, castToString(moduleName), castToString(globalName)); - } - - public Object findClass(VirtualFrame frame, Python3Core core, PUnpickler self, TruffleString moduleName, TruffleString globalName) { - // Try to map the old names used in Python 2.x to the new ones used in Python 3.x. We do - // this only with old pickle protocols and when the user has not disabled the feature. - TruffleString mName = moduleName; - TruffleString gName = globalName; - if (self.getProto() < 3 && self.isFixImports()) { - final Pair to3Mapping = get2To3Mapping(frame, core, mName, gName); - mName = to3Mapping.getLeft(); - gName = to3Mapping.getRight(); - } - - if (importNameNode == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - importNameNode = insert(ImportNameNodeGen.create()); - } - // ImportName checks initialization for cached modules and honors custom import hooks. - // With an empty fromlist, a dotted import returns the top-level package, so look up - // the requested module by its full name after importing for side effects. - importNameNode.execute(frame, PythonContext.get(this), core.getBuiltins(), mName, PNone.NONE, PythonUtils.EMPTY_TRUFFLESTRING_ARRAY, 0); - Object module = getDictItem(frame, core.getSysModules(), mName); - if (module == null) { - errorProfile.enter(); - throw PRaiseNode.raiseStatic(this, KeyError, new Object[]{mName}); - } - if (!(module instanceof PythonModule)) { - throw raise(PythonBuiltinClassType.NotImplementedError, ErrorMessages.PUTTING_NON_MODULE_OBJECTS_IN_SYS_MODULES_IS_NOT_SUPPORTED); - } - return getattribute(frame, module, gName, self.getProto() >= 4); - } } } From a75550eaaee41c3bfa4e556040ee1d186bd9ca1c Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Fri, 2 Oct 2026 10:34:07 +0200 Subject: [PATCH 33/39] Use the same import mechanism in pickler and unpickler --- .../builtins/modules/pickle/PPickler.java | 3 +- .../builtins/modules/pickle/PUnpickler.java | 22 +--------- .../builtins/modules/pickle/PicklerNodes.java | 40 +++++++++++++++++++ 3 files changed, 42 insertions(+), 23 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java index 3ef61a98cc..5ae2f5cfc7 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java @@ -55,7 +55,6 @@ import static com.oracle.graal.python.nodes.SpecialMethodNames.T___REDUCE_EX__; import static com.oracle.graal.python.nodes.SpecialMethodNames.T___REDUCE__; import static com.oracle.graal.python.nodes.StringLiterals.T_NEWLINE; -import static com.oracle.graal.python.nodes.statement.AbstractImportNode.importModule; import static com.oracle.graal.python.runtime.exception.PythonErrorType.KeyError; import static com.oracle.graal.python.runtime.exception.PythonErrorType.PicklingError; import static com.oracle.graal.python.runtime.exception.PythonErrorType.TypeError; @@ -2180,7 +2179,7 @@ private void saveGlobal(VirtualFrame frame, PythonContext ctx, PPickler pickler, Object module; try { - module = importModule(frame, getBoundaryCallData(), moduleName); + module = importModule(frame, ctx, moduleName); } catch (PException e) { throw raise(PicklingError, ErrorMessages.CANT_PICKLE_P_IMPORT_OF_MODULE_S_FAILED, obj, moduleName); } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PUnpickler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PUnpickler.java index 56f7e032aa..fa5874818d 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PUnpickler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PUnpickler.java @@ -130,7 +130,6 @@ import static com.oracle.graal.python.nodes.StringLiterals.T_ASCII_UPPERCASE; import static com.oracle.graal.python.nodes.StringLiterals.T_STRICT; import static com.oracle.graal.python.runtime.exception.PythonErrorType.AttributeError; -import static com.oracle.graal.python.runtime.exception.PythonErrorType.KeyError; import static com.oracle.graal.python.runtime.exception.PythonErrorType.TypeError; import static com.oracle.graal.python.util.PythonUtils.TS_ENCODING; @@ -162,7 +161,6 @@ import com.oracle.graal.python.builtins.objects.list.PList; import com.oracle.graal.python.builtins.objects.memoryview.MemoryViewBuiltins; import com.oracle.graal.python.builtins.objects.memoryview.MemoryViewBuiltinsFactory; -import com.oracle.graal.python.builtins.objects.module.PythonModule; import com.oracle.graal.python.builtins.objects.object.PythonBuiltinObject; import com.oracle.graal.python.builtins.objects.set.PSet; import com.oracle.graal.python.builtins.objects.tuple.PTuple; @@ -183,8 +181,6 @@ import com.oracle.graal.python.nodes.PRaiseNode; import com.oracle.graal.python.nodes.argument.keywords.ExpandKeywordStarargsNode; import com.oracle.graal.python.nodes.argument.positional.ExecutePositionalStarargsNode; -import com.oracle.graal.python.nodes.statement.AbstractImportNode.ImportName; -import com.oracle.graal.python.nodes.statement.AbstractImportNodeFactory.ImportNameNodeGen; import com.oracle.graal.python.nodes.util.CastToTruffleStringNode; import com.oracle.graal.python.runtime.IndirectCallData.InteropCallData; import com.oracle.graal.python.runtime.PythonContext; @@ -409,7 +405,6 @@ public void clearMemo() { // inner nodes public abstract static class BasePickleReadNode extends PicklerNodes.BasePickleNode { @Child private CastToTruffleStringNode castToTruffleStringNode; - @Child private ImportName importNameNode; @Child private PyObjectReprAsTruffleStringNode reprNode; @Child private PyMemoryViewFromObject memoryViewNode; @Child private MemoryViewBuiltins.ToReadonlyNode toReadonlyNode; @@ -480,22 +475,7 @@ public Object findClass(VirtualFrame frame, Python3Core core, PUnpickler self, T gName = to3Mapping.getRight(); } - if (importNameNode == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - importNameNode = insert(ImportNameNodeGen.create()); - } - // ImportName checks initialization for cached modules and honors custom import hooks. - // With an empty fromlist, a dotted import returns the top-level package, so look up - // the requested module by its full name after importing for side effects. - importNameNode.execute(frame, PythonContext.get(this), core.getBuiltins(), mName, PNone.NONE, PythonUtils.EMPTY_TRUFFLESTRING_ARRAY, 0); - Object module = getDictItem(frame, core.getSysModules(), mName); - if (module == null) { - errorProfile.enter(); - throw PRaiseNode.raiseStatic(this, KeyError, new Object[]{mName}); - } - if (!(module instanceof PythonModule)) { - throw raise(PythonBuiltinClassType.NotImplementedError, ErrorMessages.PUTTING_NON_MODULE_OBJECTS_IN_SYS_MODULES_IS_NOT_SUPPORTED); - } + Object module = importModule(frame, PythonContext.get(this), mName); return getattribute(frame, module, gName, self.getProto() >= 4); } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java index 157db2b791..92461803a3 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java @@ -44,6 +44,7 @@ import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___CLASS__; import static com.oracle.graal.python.nodes.StringLiterals.T_UTF8; import static com.oracle.graal.python.runtime.exception.PythonErrorType.AttributeError; +import static com.oracle.graal.python.runtime.exception.PythonErrorType.KeyError; import static com.oracle.graal.python.runtime.exception.PythonErrorType.TypeError; import static com.oracle.graal.python.util.PythonUtils.TS_ENCODING; import static com.oracle.graal.python.util.PythonUtils.tsLiteral; @@ -55,6 +56,7 @@ import com.oracle.graal.python.builtins.PythonBuiltinClassType; import com.oracle.graal.python.builtins.modules.CodecsModuleBuiltins; import com.oracle.graal.python.builtins.modules.CodecsModuleBuiltinsFactory; +import com.oracle.graal.python.builtins.objects.PNone; import com.oracle.graal.python.builtins.objects.bytes.BytesNodes; import com.oracle.graal.python.builtins.objects.common.HashingStorage; import com.oracle.graal.python.builtins.objects.common.HashingStorageNodes.CachedHashingStorageGetItem; @@ -68,6 +70,7 @@ import com.oracle.graal.python.builtins.objects.common.SequenceStorageNodes; import com.oracle.graal.python.builtins.objects.dict.PDict; import com.oracle.graal.python.builtins.objects.function.PKeyword; +import com.oracle.graal.python.builtins.objects.module.PythonModule; import com.oracle.graal.python.builtins.objects.tuple.PTuple; import com.oracle.graal.python.lib.PyIterCheckNode; import com.oracle.graal.python.lib.PyIterNextNode; @@ -88,13 +91,16 @@ import com.oracle.graal.python.nodes.object.BuiltinClassProfiles.InlineIsBuiltinClassProfile; import com.oracle.graal.python.nodes.object.BuiltinClassProfiles.IsBuiltinObjectProfile; import com.oracle.graal.python.nodes.object.GetClassNode; +import com.oracle.graal.python.nodes.statement.AbstractImportNode.ImportName; import com.oracle.graal.python.nodes.util.CannotCastException; import com.oracle.graal.python.nodes.util.CastToTruffleStringNode; +import com.oracle.graal.python.runtime.PythonContext; import com.oracle.graal.python.runtime.exception.PException; import com.oracle.graal.python.runtime.object.PFactory; import com.oracle.graal.python.runtime.sequence.storage.SequenceStorage; import com.oracle.graal.python.util.PythonUtils; import com.oracle.truffle.api.CompilerDirectives; +import com.oracle.truffle.api.dsl.Bind; import com.oracle.truffle.api.dsl.Cached; import com.oracle.truffle.api.dsl.Cached.Exclusive; import com.oracle.truffle.api.dsl.GenerateInline; @@ -108,6 +114,31 @@ import com.oracle.truffle.api.strings.TruffleString; public final class PicklerNodes { + @GenerateInline(false) + abstract static class ImportModuleNode extends Node { + abstract PythonModule execute(VirtualFrame frame, PythonContext context, TruffleString name); + + @Specialization + static PythonModule doImport(VirtualFrame frame, PythonContext context, TruffleString name, + @Bind Node inliningTarget, + @Cached ImportName importNameNode, + @Cached CachedHashingStorageGetItem getItemNode, + @Cached PRaiseNode raiseNode) { + // ImportName checks initialization for cached modules and honors custom import hooks. + // With an empty fromlist, a dotted import returns the top-level package, so look up + // the requested module by its full name after importing for side effects. + importNameNode.execute(frame, context, context.getBuiltins(), name, PNone.NONE, PythonUtils.EMPTY_TRUFFLESTRING_ARRAY, 0); + Object module = getItemNode.execute(frame, context.getSysModules().getDictStorage(), name); + if (module == null) { + throw raiseNode.raise(inliningTarget, KeyError, new Object[]{name}); + } + if (module instanceof PythonModule pythonModule) { + return pythonModule; + } + throw raiseNode.raise(inliningTarget, PythonBuiltinClassType.NotImplementedError, ErrorMessages.PUTTING_NON_MODULE_OBJECTS_IN_SYS_MODULES_IS_NOT_SUPPORTED); + } + } + @GenerateInline(false) @ImportStatic(PythonUtils.class) abstract static class LookupGlobalAttributeNode extends Node { @@ -163,6 +194,7 @@ abstract static class BasePickleNode extends Node { @Child private PyObjectSizeNode sizeNode; @Child private PyObjectLookupAttr lookupAttrNode; @Child private LookupGlobalAttributeNode lookupGlobalAttrNode; + @Child private ImportModuleNode importModuleNode; @Child private BytesNodes.ToBytesNode toBytesNode; @Child private TruffleString.FromByteArrayNode tsFromByteArrayNode; @Child private TruffleString.FromByteArrayWithCompactionUTF32Node tsFromByteArrayWithCompactionNode; @@ -273,6 +305,14 @@ protected HashingStorageIterator getHashingStorageIterator(HashingStorage s) { return getHashingStorageIteratorNode.executeCached(s); } + protected PythonModule importModule(VirtualFrame frame, PythonContext context, TruffleString name) { + if (importModuleNode == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + importModuleNode = insert(PicklerNodesFactory.ImportModuleNodeGen.create()); + } + return importModuleNode.execute(frame, context, name); + } + protected HashingStorageIteratorNext ensureHashingStorageIteratorNext() { if (hashingStorageItNext == null) { CompilerDirectives.transferToInterpreterAndInvalidate(); From 29988e99e9f763a838883539ec3b678298c9cf0a Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Fri, 2 Oct 2026 11:48:09 +0200 Subject: [PATCH 34/39] Improve pickler graph size --- .../builtins/modules/pickle/PPickler.java | 78 +++++++------------ .../builtins/modules/pickle/PickleUtils.java | 10 +-- 2 files changed, 29 insertions(+), 59 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java index 5ae2f5cfc7..21cc0543c9 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java @@ -445,42 +445,28 @@ private void writeBuffer(Node node, Object buffer, int size, PythonBufferAccessL outputLen += size; } - private static byte intByte1(long value) { - return (byte) (value & 0xff); - } - - private static byte intByte2(long value) { - return (byte) ((value >> 8) & 0xff); - } - - private static byte intByte3(long value) { - return (byte) ((value >> 16) & 0xff); - } - - private static byte intByte4(long value) { - return (byte) ((value >> 24) & 0xff); - } - private void writeByteOp(Node node, byte opcode, long value) { ensureBufferSpace(node, 2); - outputBuffer[outputLen++] = opcode; - outputBuffer[outputLen++] = intByte1(value); + int start = outputLen; + outputBuffer[start] = opcode; + outputBuffer[start + 1] = (byte) value; + outputLen = start + 2; } private void writeShortOp(Node node, byte opcode, long value) { ensureBufferSpace(node, 3); - outputBuffer[outputLen++] = opcode; - outputBuffer[outputLen++] = intByte1(value); - outputBuffer[outputLen++] = intByte2(value); + int start = outputLen; + outputBuffer[start] = opcode; + NumericSupport.littleEndian().putShort(outputBuffer, start + 1, (short) value); + outputLen = start + 3; } private void writeIntOp(Node node, byte opcode, long value) { ensureBufferSpace(node, 5); - outputBuffer[outputLen++] = opcode; - outputBuffer[outputLen++] = intByte1(value); - outputBuffer[outputLen++] = intByte2(value); - outputBuffer[outputLen++] = intByte3(value); - outputBuffer[outputLen++] = intByte4(value); + int start = outputLen; + outputBuffer[start] = opcode; + NumericSupport.littleEndian().putInt(outputBuffer, start + 1, (int) value); + outputLen = start + 5; } private void ensureBufferSpace(Node node, int dataLen) { @@ -504,19 +490,6 @@ private void ensureBufferSpace(Node node, int dataLen) { } } - private boolean bypassBuffer(int dataSize) { - boolean bypassBuffer = dataSize >= PickleUtils.FRAME_SIZE_TARGET; - - if (bypassBuffer) { - assert outputBuffer != null; - // Commit the previous frame. - commitFrame(); - // Disable framing temporarily - framing = false; - } - return bypassBuffer; - } - private void writeRawUnicodeEscape(Node node, TruffleString string, TruffleString.CodePointLengthNode codePointLengthNode, TruffleString.CodePointAtIndexUTF32Node codePointAtIndexNode) { int codePointLength = codePointLengthNode.execute(string, TS_ENCODING); @@ -813,6 +786,18 @@ private void profileSeen(long seen) { } } + private boolean bypassBuffer(PPickler pickler, int dataSize) { + boolean bypassBuffer = dataSize >= PickleUtils.FRAME_SIZE_TARGET; + if (bypassBuffer) { + profileSeen(SEEN_BYPASS_BUFFER); + assert pickler.outputBuffer != null; + // Commit the previous frame before disabling framing temporarily. + pickler.commitFrame(); + pickler.framing = false; + } + return bypassBuffer; + } + private BoundaryCallData getBoundaryCallData() { if (boundaryCallData == null) { CompilerDirectives.transferToInterpreterAndInvalidate(); @@ -1600,10 +1585,7 @@ private void saveBytesData(VirtualFrame frame, PPickler pickler, int proto, bool assert proto >= 3; int size = bufferLib.getBufferLength(buffer); boolean wasFraming = pickler.framing; - boolean bypassBuffer = pickler.bypassBuffer(size); - if (bypassBuffer) { - profileSeen(SEEN_BYPASS_BUFFER); - } + boolean bypassBuffer = bypassBuffer(pickler, size); if (size <= 0xff) { pickler.writeByteOp(this, PickleUtils.OPCODE_SHORT_BINBYTES, size); @@ -1631,10 +1613,7 @@ private void writeUnicodeBinary(Node node, VirtualFrame frame, PPickler pickler, TruffleString utf8 = switchEncodingNode.execute(string, TruffleString.Encoding.UTF_8, TranscodingErrorHandler.DEFAULT_KEEP_SURROGATES_IN_UTF8); int size = utf8.byteLength(TruffleString.Encoding.UTF_8); boolean wasFraming = pickler.framing; - boolean bypassBuffer = pickler.bypassBuffer(size); - if (bypassBuffer) { - profileSeen(SEEN_BYPASS_BUFFER); - } + boolean bypassBuffer = bypassBuffer(pickler, size); if (bypassBuffer && pickler.write != null) { pickler.writeIntOp(node, PickleUtils.OPCODE_BINUNICODE, size); @@ -2057,10 +2036,7 @@ private void saveBytearrayData(VirtualFrame frame, PPickler pickler, int proto, int size = bufferLib.getBufferLength(buffer); boolean wasFraming = pickler.framing; - boolean bypassBuffer = pickler.bypassBuffer(size); - if (bypassBuffer) { - profileSeen(SEEN_BYPASS_BUFFER); - } + boolean bypassBuffer = bypassBuffer(pickler, size); // Our sizes are ints, but the protocol expects 8 bytes pickler.ensureBufferSpace(this, 9); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PickleUtils.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PickleUtils.java index 193840a77b..feb9b8f210 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PickleUtils.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PickleUtils.java @@ -59,6 +59,7 @@ import com.oracle.graal.python.nodes.PRaiseNode; import com.oracle.graal.python.nodes.StringLiterals; import com.oracle.graal.python.runtime.object.PFactory; +import com.oracle.graal.python.util.NumericSupport; import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; import com.oracle.truffle.api.frame.VirtualFrame; import com.oracle.truffle.api.nodes.Node; @@ -214,14 +215,7 @@ public final class PickleUtils { public static final TruffleString T_PROTO_LE2_FALSE = tsLiteral("I00\n"); public static void writeSize64(byte[] out, int start, int value) { - out[start] = (byte) value; - out[start + 1] = (byte) (value >> 8); - out[start + 2] = (byte) (value >> 16); - out[start + 3] = (byte) (value >> 24); - out[start + 4] = 0; - out[start + 5] = 0; - out[start + 6] = 0; - out[start + 7] = 0; + NumericSupport.littleEndian().putLong(out, start, Integer.toUnsignedLong(value)); } public static int getStringSize(byte[] bytes) { From 90dc553c5a12353244db7d72b6be50147d3ab4ce Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Fri, 2 Oct 2026 13:31:10 +0200 Subject: [PATCH 35/39] Use separate recursive SaveNodes --- .../builtins/modules/pickle/PPickler.java | 262 ++++++++++++++---- 1 file changed, 204 insertions(+), 58 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java index 21cc0543c9..968f29e056 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java @@ -589,9 +589,22 @@ public abstract static class SaveNode extends PicklerNodes.BasePickleNode { @CompilationFinal private long seenTypes; private final int depth; - @Child private SaveNode recursiveSaveNode1; - @Child private SaveNode recursiveSaveNode2; - @Child private SaveNode recursiveSaveNode3; + // Keep recursive profiles separate for each role in the pickle stream. + @Child private SaveNode dictKeySaveNode; + @Child private SaveNode dictValueSaveNode; + @Child private SaveNode setElementSaveNode; + @Child private SaveNode listElementSaveNode; + @Child private SaveNode tupleElementSaveNode; + @Child private SaveNode persistentIdSaveNode; + @Child private SaveNode reduceClassSaveNode; + @Child private SaveNode reduceCallableSaveNode; + @Child private SaveNode reduceArgsSaveNode; + @Child private SaveNode reduceKwargsSaveNode; + @Child private SaveNode reduceStateSaveNode; + @Child private SaveNode reduceStateSetterSaveNode; + @Child private SaveNode reduceObjectSaveNode; + @Child private SaveNode globalModuleNameSaveNode; + @Child private SaveNode globalNameSaveNode; @Child private ListNodes.ConstructListNode constructListNode; @Child private TypeNodes.IsTypeNode isTypeNode; @@ -822,51 +835,181 @@ private boolean isTuple(Object object) { return tupleCheckNode.execute(object); } - private void save(VirtualFrame frame, PPickler pickler, Object obj, int persSave) { + private void saveDictKey(VirtualFrame frame, PPickler pickler, Object obj) { if (depth < 0) { - saveDepthExceeded(pickler, obj, persSave); + saveDepthExceeded(pickler, obj, 0); return; } - if (recursiveSaveNode1 == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - recursiveSaveNode1 = insert(PPicklerFactory.SaveNodeGen.create(depth < MAX_RECURSION_DEPTH ? depth + 1 : -1)); + if (dictKeySaveNode == null) { + dictKeySaveNode = createRecursiveSaveNode(); } - if (depth < MAX_RECURSION_DEPTH) { - recursiveSaveNode1.execute(frame, pickler, obj, persSave); - } else { - saveSetupBoundary(frame, recursiveSaveNode1, pickler, obj, persSave); + saveRecursive(frame, dictKeySaveNode, pickler, obj, 0); + } + + private void saveDictValue(VirtualFrame frame, PPickler pickler, Object obj) { + if (depth < 0) { + saveDepthExceeded(pickler, obj, 0); + return; + } + if (dictValueSaveNode == null) { + dictValueSaveNode = createRecursiveSaveNode(); } + saveRecursive(frame, dictValueSaveNode, pickler, obj, 0); } - private void save2(VirtualFrame frame, PPickler pickler, Object obj, int persSave) { + private void saveSetElement(VirtualFrame frame, PPickler pickler, Object obj) { if (depth < 0) { - saveDepthExceeded(pickler, obj, persSave); + saveDepthExceeded(pickler, obj, 0); return; } - if (recursiveSaveNode2 == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - recursiveSaveNode2 = insert(PPicklerFactory.SaveNodeGen.create(depth < MAX_RECURSION_DEPTH ? depth + 1 : -1)); + if (setElementSaveNode == null) { + setElementSaveNode = createRecursiveSaveNode(); } - if (depth < MAX_RECURSION_DEPTH) { - recursiveSaveNode2.execute(frame, pickler, obj, persSave); - } else { - saveSetupBoundary(frame, recursiveSaveNode2, pickler, obj, persSave); + saveRecursive(frame, setElementSaveNode, pickler, obj, 0); + } + + private void saveListElement(VirtualFrame frame, PPickler pickler, Object obj) { + if (depth < 0) { + saveDepthExceeded(pickler, obj, 0); + return; + } + if (listElementSaveNode == null) { + listElementSaveNode = createRecursiveSaveNode(); } + saveRecursive(frame, listElementSaveNode, pickler, obj, 0); } - private void save3(VirtualFrame frame, PPickler pickler, Object obj, int persSave) { + private void saveTupleElement(VirtualFrame frame, PPickler pickler, Object obj) { if (depth < 0) { - saveDepthExceeded(pickler, obj, persSave); + saveDepthExceeded(pickler, obj, 0); return; } - if (recursiveSaveNode3 == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - recursiveSaveNode3 = insert(PPicklerFactory.SaveNodeGen.create(depth < MAX_RECURSION_DEPTH ? depth + 1 : -1)); + if (tupleElementSaveNode == null) { + tupleElementSaveNode = createRecursiveSaveNode(); + } + saveRecursive(frame, tupleElementSaveNode, pickler, obj, 0); + } + + private void savePersistentId(VirtualFrame frame, PPickler pickler, Object obj) { + if (depth < 0) { + saveDepthExceeded(pickler, obj, 1); + return; + } + if (persistentIdSaveNode == null) { + persistentIdSaveNode = createRecursiveSaveNode(); } + saveRecursive(frame, persistentIdSaveNode, pickler, obj, 1); + } + + private void saveReduceClass(VirtualFrame frame, PPickler pickler, Object obj) { + if (depth < 0) { + saveDepthExceeded(pickler, obj, 0); + return; + } + if (reduceClassSaveNode == null) { + reduceClassSaveNode = createRecursiveSaveNode(); + } + saveRecursive(frame, reduceClassSaveNode, pickler, obj, 0); + } + + private void saveReduceCallable(VirtualFrame frame, PPickler pickler, Object obj) { + if (depth < 0) { + saveDepthExceeded(pickler, obj, 0); + return; + } + if (reduceCallableSaveNode == null) { + reduceCallableSaveNode = createRecursiveSaveNode(); + } + saveRecursive(frame, reduceCallableSaveNode, pickler, obj, 0); + } + + private void saveReduceArgs(VirtualFrame frame, PPickler pickler, Object obj) { + if (depth < 0) { + saveDepthExceeded(pickler, obj, 0); + return; + } + if (reduceArgsSaveNode == null) { + reduceArgsSaveNode = createRecursiveSaveNode(); + } + saveRecursive(frame, reduceArgsSaveNode, pickler, obj, 0); + } + + private void saveReduceKwargs(VirtualFrame frame, PPickler pickler, Object obj) { + if (depth < 0) { + saveDepthExceeded(pickler, obj, 0); + return; + } + if (reduceKwargsSaveNode == null) { + reduceKwargsSaveNode = createRecursiveSaveNode(); + } + saveRecursive(frame, reduceKwargsSaveNode, pickler, obj, 0); + } + + private void saveReduceState(VirtualFrame frame, PPickler pickler, Object obj) { + if (depth < 0) { + saveDepthExceeded(pickler, obj, 0); + return; + } + if (reduceStateSaveNode == null) { + reduceStateSaveNode = createRecursiveSaveNode(); + } + saveRecursive(frame, reduceStateSaveNode, pickler, obj, 0); + } + + private void saveReduceStateSetter(VirtualFrame frame, PPickler pickler, Object obj) { + if (depth < 0) { + saveDepthExceeded(pickler, obj, 0); + return; + } + if (reduceStateSetterSaveNode == null) { + reduceStateSetterSaveNode = createRecursiveSaveNode(); + } + saveRecursive(frame, reduceStateSetterSaveNode, pickler, obj, 0); + } + + private void saveReduceObject(VirtualFrame frame, PPickler pickler, Object obj) { + if (depth < 0) { + saveDepthExceeded(pickler, obj, 0); + return; + } + if (reduceObjectSaveNode == null) { + reduceObjectSaveNode = createRecursiveSaveNode(); + } + saveRecursive(frame, reduceObjectSaveNode, pickler, obj, 0); + } + + private void saveGlobalModuleName(VirtualFrame frame, PPickler pickler, Object obj) { + if (depth < 0) { + saveDepthExceeded(pickler, obj, 0); + return; + } + if (globalModuleNameSaveNode == null) { + globalModuleNameSaveNode = createRecursiveSaveNode(); + } + saveRecursive(frame, globalModuleNameSaveNode, pickler, obj, 0); + } + + private void saveGlobalName(VirtualFrame frame, PPickler pickler, Object obj) { + if (depth < 0) { + saveDepthExceeded(pickler, obj, 0); + return; + } + if (globalNameSaveNode == null) { + globalNameSaveNode = createRecursiveSaveNode(); + } + saveRecursive(frame, globalNameSaveNode, pickler, obj, 0); + } + + private SaveNode createRecursiveSaveNode() { + CompilerDirectives.transferToInterpreterAndInvalidate(); + return insert(PPicklerFactory.SaveNodeGen.create(depth < MAX_RECURSION_DEPTH ? depth + 1 : -1)); + } + + private void saveRecursive(VirtualFrame frame, SaveNode saveNode, PPickler pickler, Object obj, int persSave) { if (depth < MAX_RECURSION_DEPTH) { - recursiveSaveNode3.execute(frame, pickler, obj, persSave); + saveNode.execute(frame, pickler, obj, persSave); } else { - saveSetupBoundary(frame, recursiveSaveNode3, pickler, obj, persSave); + saveSetupBoundary(frame, saveNode, pickler, obj, persSave); } } @@ -986,9 +1129,12 @@ private void saveHashingStorageBatched(VirtualFrame frame, PPickler pickler, Has i = 0; pickler.write(this, PickleUtils.OPCODE_MARK); while (nextNode.executeCached(storage, it)) { - save(frame, pickler, getKeyNode.executeCached(storage, it), 0); + Object key = getKeyNode.executeCached(storage, it); if (saveValues) { - save2(frame, pickler, getValueNode.executeCached(storage, it), 0); + saveDictKey(frame, pickler, key); + saveDictValue(frame, pickler, getValueNode.executeCached(storage, it)); + } else { + saveSetElement(frame, pickler, key); } if (++i == PickleUtils.BATCHSIZE) { break; @@ -1015,7 +1161,7 @@ private void saveSetHashingStorage(VirtualFrame frame, PPickler pickler, Hashing HashingStorageIteratorKey getKeyNode = ensureHashingStorageIteratorKey(); while (next.executeCached(storage, it)) { Object item = getKeyNode.executeCached(storage, it); - save(frame, pickler, item, 0); + saveSetElement(frame, pickler, item); } } @@ -1078,8 +1224,8 @@ private void saveDictIteratorBatchUnrolled(VirtualFrame frame, PPickler pickler, }, (Object item) -> { SequenceStorage storage = getTupleStorage(item); - save(frame, pickler, getItem(storage, 0), 0); - save2(frame, pickler, getItem(storage, 1), 0); + saveDictKey(frame, pickler, getItem(storage, 0)); + saveDictValue(frame, pickler, getItem(storage, 1)); }); } @@ -1087,7 +1233,7 @@ private void saveListIteratorBatchUnrolled(VirtualFrame frame, PPickler pickler, saveIteratorBatchedUnrolled(frame, pickler, iterator, PickleUtils.OPCODE_APPEND, PickleUtils.OPCODE_APPENDS, (Object item) -> { }, - (Object item) -> save(frame, pickler, item, 0)); + (Object item) -> saveListElement(frame, pickler, item)); } private void saveIterator(VirtualFrame frame, PPickler pickler, Object iterator, byte opcode, Consumer itemConsumer) { @@ -1107,7 +1253,7 @@ private void saveIterator(VirtualFrame frame, PPickler pickler, Object iterator, private void saveListIterator(VirtualFrame frame, PPickler pickler, Object iterator) { saveIterator(frame, pickler, iterator, PickleUtils.OPCODE_APPEND, - (Object item) -> save(frame, pickler, item, 0)); + (Object item) -> saveListElement(frame, pickler, item)); } private void saveDictIterator(VirtualFrame frame, PPickler pickler, Object iterator) { @@ -1117,8 +1263,8 @@ private void saveDictIterator(VirtualFrame frame, PPickler pickler, Object itera throw raise(TypeError, ErrorMessages.MUST_S_ITER_RETURN_2TUPLE, DICT_ITEMS); } SequenceStorage storage = getTupleStorage(item); - save(frame, pickler, getItem(storage, 0), 0); - save2(frame, pickler, getItem(storage, 1), 0); + saveDictKey(frame, pickler, getItem(storage, 0)); + saveDictValue(frame, pickler, getItem(storage, 1)); }); } @@ -1133,7 +1279,7 @@ private boolean savePers(VirtualFrame frame, PPickler pickler, int proto, Object } if (pid != PNone.NONE) { if (isBin(proto)) { - save(frame, pickler, pid, 1); + savePersistentId(frame, pickler, pid); pickler.write(this, PickleUtils.OPCODE_BINPERSID); } else { final TruffleString pidStr = asString(convertToStr(frame, pid)); @@ -1301,9 +1447,9 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, } if (proto >= 4) { - save(frame, pickler, cls, 0); - save2(frame, pickler, args, 0); - save3(frame, pickler, kwargs, 0); + saveReduceClass(frame, pickler, cls); + saveReduceArgs(frame, pickler, args); + saveReduceKwargs(frame, pickler, kwargs); pickler.write(this, PickleUtils.OPCODE_NEWOBJ_EX); } else { PickleState st = getGlobalState(ctx.getCore()); @@ -1322,8 +1468,8 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, } callable = callStarArgsAndKwArgs(frame, st.partial, newargs, kwargs); - save(frame, pickler, callable, 0); - save2(frame, pickler, PFactory.createEmptyTuple(PythonLanguage.get(this)), 0); + saveReduceCallable(frame, pickler, callable); + saveReduceArgs(frame, pickler, PFactory.createEmptyTuple(PythonLanguage.get(this))); pickler.write(this, PickleUtils.OPCODE_REDUCE); } } else if (useNewobj) { @@ -1379,19 +1525,19 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, // function. // Save the class and its __new__ arguments - save(frame, pickler, cls, 0); + saveReduceClass(frame, pickler, cls); Object[] newargs = new Object[argtupSize - 1]; for (int i = 1; i < argtupSize; i++) { newargs[i - 1] = getItem(argtupStorage, i); } newargtup = createTuple(newargs); - save2(frame, pickler, newargtup, 0); + saveReduceArgs(frame, pickler, newargtup); pickler.write(this, PickleUtils.OPCODE_NEWOBJ); } else { profileSeen(SEEN_PLAIN_REDUCE); // Not using NEWOBJ - save(frame, pickler, callable, 0); - save2(frame, pickler, argtup, 0); + saveReduceCallable(frame, pickler, callable); + saveReduceArgs(frame, pickler, argtup); pickler.write(this, PickleUtils.OPCODE_REDUCE); } @@ -1424,7 +1570,7 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, if (state != null) { profileSeen(SEEN_REDUCE_STATE); if (stateSetter == null) { - save(frame, pickler, state, 0); + saveReduceState(frame, pickler, state); pickler.write(this, PickleUtils.OPCODE_BUILD); } else { // If a state_setter is specified, call it instead of load_build to update obj's @@ -1434,9 +1580,9 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, // Finally, because state-updating routines only do in-place modification, the // whole operation has to be stack-transparent. Thus, we finally pop the call's // output from the stack. - save(frame, pickler, stateSetter, 0); - save2(frame, pickler, obj, 0); - save3(frame, pickler, state, 0); + saveReduceStateSetter(frame, pickler, stateSetter); + saveReduceObject(frame, pickler, obj); + saveReduceState(frame, pickler, state); pickler.write(this, PickleUtils.OPCODE_TUPLE2); pickler.write(this, PickleUtils.OPCODE_REDUCE); pickler.write(this, PickleUtils.OPCODE_POP); @@ -1666,8 +1812,8 @@ private void batchDictExact(VirtualFrame frame, PPickler pickler, PDict dict) { } HashingStorageIteratorKey getKeyNode = ensureHashingStorageIteratorKey(); HashingStorageIteratorValue getValueNode = ensureHashingStorageIteratorValue(); - save(frame, pickler, getKeyNode.executeCached(storage, it), 0); - save(frame, pickler, getValueNode.executeCached(storage, it), 0); + saveDictKey(frame, pickler, getKeyNode.executeCached(storage, it)); + saveDictValue(frame, pickler, getValueNode.executeCached(storage, it)); pickler.write(this, PickleUtils.OPCODE_SETITEM); } else { // Write in batches of BATCHSIZE. @@ -1786,7 +1932,7 @@ private void batchListExact(VirtualFrame frame, PPickler pickler, PList obj) { Object item; if (storage.length() == 1) { item = getItem(storage, 0); - save(frame, pickler, item, 0); + saveListElement(frame, pickler, item); pickler.write(this, PickleUtils.OPCODE_APPEND); return; } @@ -1799,7 +1945,7 @@ private void batchListExact(VirtualFrame frame, PPickler pickler, PList obj) { pickler.write(this, PickleUtils.OPCODE_MARK); while (total < storage.length()) { item = getItem(storage, total); - save(frame, pickler, item, 0); + saveListElement(frame, pickler, item); total++; if (++thisBatch == PickleUtils.BATCHSIZE) { break; @@ -1931,7 +2077,7 @@ private void storeTupleElements(VirtualFrame frame, PPickler pickler, SequenceSt } profileSeen(SEEN_GENERIC_STORAGE); for (int i = 0; i < len; i++) { - save(frame, pickler, getItem(storage, i), 0); + saveTupleElement(frame, pickler, getItem(storage, i)); } } @@ -2214,8 +2360,8 @@ private void saveGlobal(VirtualFrame frame, PythonContext ctx, PPickler pickler, } if (proto >= 4) { - save(frame, pickler, moduleName, 0); - save(frame, pickler, globalName, 0); + saveGlobalModuleName(frame, pickler, moduleName); + saveGlobalName(frame, pickler, globalName); pickler.write(this, PickleUtils.OPCODE_STACK_GLOBAL); } else if (parent != module) { Object reduceValue = createTuple(st.getattr, createTuple(parent, lastname)); From dff33e71d943f4c22fafe8dd79f0a2ac38dfa58c Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Thu, 8 Oct 2026 22:43:02 +0200 Subject: [PATCH 36/39] Report loop counts in pickle code --- .../python/builtins/modules/pickle/PPickler.java | 12 ++++++++++++ .../python/builtins/modules/pickle/PUnpickler.java | 2 ++ .../python/builtins/modules/pickle/PicklerNodes.java | 2 ++ 3 files changed, 16 insertions(+) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java index 968f29e056..a8223a7ab6 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java @@ -137,6 +137,7 @@ import com.oracle.truffle.api.dsl.GenerateInline; import com.oracle.truffle.api.dsl.Specialization; import com.oracle.truffle.api.frame.VirtualFrame; +import com.oracle.truffle.api.nodes.LoopNode; import com.oracle.truffle.api.nodes.Node; import com.oracle.truffle.api.object.Shape; import com.oracle.truffle.api.profiles.InlinedConditionProfile; @@ -1953,6 +1954,7 @@ private void batchListExact(VirtualFrame frame, PPickler pickler, PList obj) { } pickler.write(this, PickleUtils.OPCODE_APPENDS); } while (total < storage.length()); + LoopNode.reportLoopCount(this, total); } private void batchIntList(VirtualFrame frame, PPickler pickler, IntSequenceStorage storage) { @@ -1972,8 +1974,10 @@ private void batchIntList(VirtualFrame frame, PPickler pickler, IntSequenceStora opcodeBoundary(frame, pickler); saveLong(pickler, proto, storage.getIntItemNormalized(i)); } + LoopNode.reportLoopCount(this, batchEnd - batchStart); pickler.write(this, PickleUtils.OPCODE_APPENDS); } + LoopNode.reportLoopCount(this, length / PickleUtils.BATCHSIZE + (length % PickleUtils.BATCHSIZE == 0 ? 0 : 1)); } private void batchLongList(VirtualFrame frame, PPickler pickler, LongSequenceStorage storage) { @@ -1993,8 +1997,10 @@ private void batchLongList(VirtualFrame frame, PPickler pickler, LongSequenceSto opcodeBoundary(frame, pickler); saveLong(pickler, proto, storage.getLongItemNormalized(i)); } + LoopNode.reportLoopCount(this, batchEnd - batchStart); pickler.write(this, PickleUtils.OPCODE_APPENDS); } + LoopNode.reportLoopCount(this, length / PickleUtils.BATCHSIZE + (length % PickleUtils.BATCHSIZE == 0 ? 0 : 1)); } private void batchDoubleList(VirtualFrame frame, PPickler pickler, DoubleSequenceStorage storage) { @@ -2014,8 +2020,10 @@ private void batchDoubleList(VirtualFrame frame, PPickler pickler, DoubleSequenc opcodeBoundary(frame, pickler); saveFloat(pickler, proto, storage.getDoubleItemNormalized(i)); } + LoopNode.reportLoopCount(this, batchEnd - batchStart); pickler.write(this, PickleUtils.OPCODE_APPENDS); } + LoopNode.reportLoopCount(this, length / PickleUtils.BATCHSIZE + (length % PickleUtils.BATCHSIZE == 0 ? 0 : 1)); } private void batchList(VirtualFrame frame, PPickler pickler, int proto, Object iterator) { @@ -2079,6 +2087,7 @@ private void storeTupleElements(VirtualFrame frame, PPickler pickler, SequenceSt for (int i = 0; i < len; i++) { saveTupleElement(frame, pickler, getItem(storage, i)); } + LoopNode.reportLoopCount(this, len); } private void storeIntTupleElements(VirtualFrame frame, PPickler pickler, IntSequenceStorage storage) { @@ -2087,6 +2096,7 @@ private void storeIntTupleElements(VirtualFrame frame, PPickler pickler, IntSequ opcodeBoundary(frame, pickler); saveLong(pickler, proto, storage.getIntItemNormalized(i)); } + LoopNode.reportLoopCount(this, storage.length()); } private void storeLongTupleElements(VirtualFrame frame, PPickler pickler, LongSequenceStorage storage) { @@ -2095,6 +2105,7 @@ private void storeLongTupleElements(VirtualFrame frame, PPickler pickler, LongSe opcodeBoundary(frame, pickler); saveLong(pickler, proto, storage.getLongItemNormalized(i)); } + LoopNode.reportLoopCount(this, storage.length()); } private void storeDoubleTupleElements(VirtualFrame frame, PPickler pickler, DoubleSequenceStorage storage) { @@ -2103,6 +2114,7 @@ private void storeDoubleTupleElements(VirtualFrame frame, PPickler pickler, Doub opcodeBoundary(frame, pickler); saveFloat(pickler, proto, storage.getDoubleItemNormalized(i)); } + LoopNode.reportLoopCount(this, storage.length()); } private void saveTuple(VirtualFrame frame, PPickler pickler, int proto, boolean fastMode, PTuple obj) { diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PUnpickler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PUnpickler.java index fa5874818d..b70dc9e32c 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PUnpickler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PUnpickler.java @@ -1318,6 +1318,7 @@ private void loadDict(VirtualFrame frame, PUnpickler self) { storage = setHashingStorageItem(frame, storage, key, value); } + LoopNode.reportLoopCount(this, (j - i) / 2); self.stack.clear(i); pDataPush(self, PFactory.createDict(PythonLanguage.get(this), storage)); } @@ -1821,6 +1822,7 @@ private void doSetItems(VirtualFrame frame, PUnpickler self, int x) { } } + LoopNode.reportLoopCount(this, (len - x) / 2); self.stack.clear(x); } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java index 92461803a3..3815619a8d 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java @@ -642,6 +642,7 @@ private TruffleString[] splitDottedPath(TruffleString name) { parts++; start = dot + 1; } + LoopNode.reportLoopCount(this, parts - 2 + (start < length ? 1 : 0)); TruffleString[] dottedPath = new TruffleString[parts]; TruffleString.SubstringNode substring = ensureTsSubstringNode(); start = 0; @@ -650,6 +651,7 @@ private TruffleString[] splitDottedPath(TruffleString name) { dottedPath[part++] = substring.execute(name, start, dot - start, TS_ENCODING, false); start = dot + 1; } + LoopNode.reportLoopCount(this, part); dottedPath[part] = substring.execute(name, start, length - start, TS_ENCODING, false); return dottedPath; } From c99ab1c55ff112feab98af7e6925fdb31603530f Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Mon, 12 Oct 2026 12:21:05 +0200 Subject: [PATCH 37/39] Avoid out-of-bounds access on list mutation in pickler --- .../builtins/modules/pickle/PPickler.java | 70 +++++++++++++------ 1 file changed, 49 insertions(+), 21 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java index a8223a7ab6..635f3a8569 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java @@ -746,9 +746,9 @@ protected Object createList(VirtualFrame frame, Object iterable) { return constructListNode.execute(frame, iterable); } - public void opcodeBoundary(VirtualFrame frame, PPickler pickler) { + public boolean opcodeBoundary(VirtualFrame frame, PPickler pickler) { if (!pickler.isFraming() || pickler.frameStart == -1) { - return; + return false; } int frameLen = pickler.outputLen - pickler.frameStart - PickleUtils.FRAME_HEADER_SIZE; if (CompilerDirectives.injectBranchProbability(CompilerDirectives.SLOWPATH_PROBABILITY, frameLen >= PickleUtils.FRAME_SIZE_TARGET)) { @@ -760,8 +760,10 @@ public void opcodeBoundary(VirtualFrame frame, PPickler pickler) { if (pickler.write != null) { flushToFile(frame, pickler); pickler.clearBuffer(); + return true; } } + return false; } public static void handleError(PPickler pickler) { @@ -1913,7 +1915,7 @@ private void saveFrozenset(VirtualFrame frame, PythonContext ctx, PPickler pickl } private void batchListExact(VirtualFrame frame, PPickler pickler, PList obj) { - final SequenceStorage storage = obj.getSequenceStorage(); + SequenceStorage storage = obj.getSequenceStorage(); if (pickler.persFunc == null) { if (storage instanceof IntSequenceStorage intStorage) { profileSeen(SEEN_INT_STORAGE); @@ -1953,77 +1955,103 @@ private void batchListExact(VirtualFrame frame, PPickler pickler, PList obj) { } } pickler.write(this, PickleUtils.OPCODE_APPENDS); + // It could have been mutated as a side effect of the reduce calls or the file write + storage = obj.getSequenceStorage(); } while (total < storage.length()); LoopNode.reportLoopCount(this, total); } private void batchIntList(VirtualFrame frame, PPickler pickler, IntSequenceStorage storage) { + opcodeBoundary(frame, pickler); int proto = pickler.getProto(); - if (storage.length() == 1) { - opcodeBoundary(frame, pickler); + int length = storage.length(); + + if (length == 1) { saveLong(pickler, proto, storage.getIntItemNormalized(0)); pickler.write(this, PickleUtils.OPCODE_APPEND); return; } - int length = storage.length(); for (int batchStart = 0; batchStart < length; batchStart += PickleUtils.BATCHSIZE) { int batchEnd = batchStart + Math.min(PickleUtils.BATCHSIZE, length - batchStart); pickler.write(this, PickleUtils.OPCODE_MARK); for (int i = batchStart; i < batchEnd; i++) { - opcodeBoundary(frame, pickler); + boolean didFlush = opcodeBoundary(frame, pickler); + // To make sure we don't go out-of-bounds if the file write mutated the list + if (CompilerDirectives.injectBranchProbability(CompilerDirectives.SLOWPATH_PROBABILITY, didFlush && length != storage.length())) { + length = storage.length(); + batchEnd = batchStart + Math.min(PickleUtils.BATCHSIZE, length - batchStart); + if (i >= batchEnd) { + break; + } + } saveLong(pickler, proto, storage.getIntItemNormalized(i)); } - LoopNode.reportLoopCount(this, batchEnd - batchStart); pickler.write(this, PickleUtils.OPCODE_APPENDS); } - LoopNode.reportLoopCount(this, length / PickleUtils.BATCHSIZE + (length % PickleUtils.BATCHSIZE == 0 ? 0 : 1)); + LoopNode.reportLoopCount(this, length); } private void batchLongList(VirtualFrame frame, PPickler pickler, LongSequenceStorage storage) { + opcodeBoundary(frame, pickler); int proto = pickler.getProto(); - if (storage.length() == 1) { - opcodeBoundary(frame, pickler); + int length = storage.length(); + + if (length == 1) { saveLong(pickler, proto, storage.getLongItemNormalized(0)); pickler.write(this, PickleUtils.OPCODE_APPEND); return; } - int length = storage.length(); for (int batchStart = 0; batchStart < length; batchStart += PickleUtils.BATCHSIZE) { int batchEnd = batchStart + Math.min(PickleUtils.BATCHSIZE, length - batchStart); pickler.write(this, PickleUtils.OPCODE_MARK); for (int i = batchStart; i < batchEnd; i++) { - opcodeBoundary(frame, pickler); + boolean didFlush = opcodeBoundary(frame, pickler); + // To make sure we don't go out-of-bounds if the file write mutated the list + if (CompilerDirectives.injectBranchProbability(CompilerDirectives.SLOWPATH_PROBABILITY, didFlush && length != storage.length())) { + length = storage.length(); + batchEnd = batchStart + Math.min(PickleUtils.BATCHSIZE, length - batchStart); + if (i >= batchEnd) { + break; + } + } saveLong(pickler, proto, storage.getLongItemNormalized(i)); } - LoopNode.reportLoopCount(this, batchEnd - batchStart); pickler.write(this, PickleUtils.OPCODE_APPENDS); } - LoopNode.reportLoopCount(this, length / PickleUtils.BATCHSIZE + (length % PickleUtils.BATCHSIZE == 0 ? 0 : 1)); + LoopNode.reportLoopCount(this, length); } private void batchDoubleList(VirtualFrame frame, PPickler pickler, DoubleSequenceStorage storage) { + opcodeBoundary(frame, pickler); int proto = pickler.getProto(); - if (storage.length() == 1) { - opcodeBoundary(frame, pickler); + int length = storage.length(); + + if (length == 1) { saveFloat(pickler, proto, storage.getDoubleItemNormalized(0)); pickler.write(this, PickleUtils.OPCODE_APPEND); return; } - int length = storage.length(); for (int batchStart = 0; batchStart < length; batchStart += PickleUtils.BATCHSIZE) { int batchEnd = batchStart + Math.min(PickleUtils.BATCHSIZE, length - batchStart); pickler.write(this, PickleUtils.OPCODE_MARK); for (int i = batchStart; i < batchEnd; i++) { - opcodeBoundary(frame, pickler); + boolean didFlush = opcodeBoundary(frame, pickler); + // To make sure we don't go out-of-bounds if the file write mutated the list + if (CompilerDirectives.injectBranchProbability(CompilerDirectives.SLOWPATH_PROBABILITY, didFlush && length != storage.length())) { + length = storage.length(); + batchEnd = batchStart + Math.min(PickleUtils.BATCHSIZE, length - batchStart); + if (i >= batchEnd) { + break; + } + } saveFloat(pickler, proto, storage.getDoubleItemNormalized(i)); } - LoopNode.reportLoopCount(this, batchEnd - batchStart); pickler.write(this, PickleUtils.OPCODE_APPENDS); } - LoopNode.reportLoopCount(this, length / PickleUtils.BATCHSIZE + (length % PickleUtils.BATCHSIZE == 0 ? 0 : 1)); + LoopNode.reportLoopCount(this, length); } private void batchList(VirtualFrame frame, PPickler pickler, int proto, Object iterator) { From 50ce7eebbe624842bf83da960ee502716ee2b0c1 Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Tue, 13 Oct 2026 14:30:52 +0200 Subject: [PATCH 38/39] Downsize oversized output bytes --- .../python/builtins/modules/pickle/PPickler.java | 12 +++++++----- .../modules/pickle/PickleModuleBuiltins.java | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java index 635f3a8569..68d31392b7 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java @@ -368,7 +368,6 @@ public void clearMemo() { } public void clearBuffer() { - this.outputBuffer = new byte[this.outputBuffer.length]; this.outputLen = 0; this.frameStart = -1; } @@ -389,9 +388,12 @@ public void commitFrame() { frameStart = -1; } - public PBytes getString(PythonLanguage language) { - commitFrame(); - return PFactory.createBytes(language, outputBuffer, outputLen); + public PBytes getString(PythonLanguage language, boolean copy) { + if (copy || outputBuffer.length > outputLen * 5 / 4) { + return PFactory.createBytes(language, PythonUtils.arrayCopyOf(outputBuffer, outputLen)); + } else { + return PFactory.createBytes(language, outputBuffer, outputLen); + } } private void writeASCII(Node node, TruffleString string, TruffleString.SwitchEncodingNode switchEncodingNode, @@ -2648,7 +2650,7 @@ public void flush(VirtualFrame frame, PPickler pickler, @Cached CallNode callNode) { assert pickler.write != null; // This will commit the frame first - PBytes output = pickler.getString(language); + PBytes output = pickler.getString(language, true); callNode.execute(frame, pickler.write, output); } } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PickleModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PickleModuleBuiltins.java index f58154dafd..bc20d0115d 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PickleModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PickleModuleBuiltins.java @@ -197,7 +197,7 @@ Object dump(VirtualFrame frame, @SuppressWarnings("unused") PythonModule self, O pickler.setProtocol(inliningTarget, raiseNode, protocol, fixImports); pickler.setBufferCallback(inliningTarget, raiseNode, bufferCallback); dumpNode.execute(frame, pickler, obj); - Object result = pickler.getString(language); + Object result = pickler.getString(language, false); reportSizes(pickler); return result; } From a1b186e482c55074c7e54dcb82a309c11de746f0 Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Tue, 13 Oct 2026 14:35:44 +0200 Subject: [PATCH 39/39] Cap the max pickle buffer/memo size estimate at 15 MiB --- .../python/builtins/modules/pickle/PickleModuleBuiltins.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PickleModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PickleModuleBuiltins.java index bc20d0115d..539038ee45 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PickleModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PickleModuleBuiltins.java @@ -85,6 +85,7 @@ protected List> getNodeFa /** Learns the capacities reached by successful calls at one builtin node. */ private static final class PicklerSizeProfile { private static final int SHIFT = 3; + private static final long MAX_ESTIMATE = (16L * 1024 * 1024) << SHIFT; @CompilationFinal private long outputEstimate = (long) PickleUtils.WRITE_BUF_SIZE << SHIFT; @CompilationFinal private long memoEstimate = (long) MemoTable.INITIAL_CAPACITY << SHIFT; @@ -112,7 +113,7 @@ void report(PPickler pickler) { private static long update(long shiftedEstimate, int capacity) { long estimate = shiftedEstimate >> SHIFT; - return shiftedEstimate + capacity - estimate; + return Math.min(shiftedEstimate + capacity - estimate, MAX_ESTIMATE); } }