diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/ByteArrayView.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/ByteArrayView.java deleted file mode 100644 index 34dd8b26cf..0000000000 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/ByteArrayView.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * The Universal Permissive License (UPL), Version 1.0 - * - * Subject to the condition set forth below, permission is hereby granted to any - * person obtaining a copy of this software, associated documentation and/or - * data (collectively the "Software"), free of charge and under any and all - * copyright rights in the Software, and any and all patent rights owned or - * freely licensable by each licensor hereunder covering either (i) the - * unmodified Software as contributed to or provided by such licensor, or (ii) - * the Larger Works (as defined below), to deal in both - * - * (a) the Software, and - * - * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if - * one is included with the Software each a "Larger Work" to which the Software - * is contributed by such licensors), - * - * without restriction, including without limitation the rights to copy, create - * derivative works of, display, perform, and distribute the Software and make, - * use, sell, offer for sale, import, export, have made, and have sold the - * Software and the Larger Work(s), and to sublicense the foregoing rights on - * either these or other terms. - * - * This license is subject to the following condition: - * - * The above copyright notice and either this complete permission notice or at a - * minimum a reference to the UPL must be included in all copies or substantial - * portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -package com.oracle.graal.python.builtins.modules.pickle; - -import com.oracle.graal.python.util.PythonUtils; - -public class ByteArrayView { - private final byte[] bytes; - private int offset; - - public ByteArrayView(byte[] bytes) { - this(bytes, 0); - } - - public ByteArrayView(byte[] bytes, int offset) { - this.bytes = bytes; - this.offset = offset; - validateOffset(0); - } - - private void validateOffset(int n) { - assert offset + n >= 0 && offset + n <= bytes.length; - } - - public void add(int n) { - validateOffset(n); - this.offset += n; - } - - public void sub(int n) { - this.add(-n); - } - - public byte[] getBytes(int len) { - assert len >= 0 && (len + offset) >= 0 && (len + offset) <= bytes.length; - if (offset == 0 && len == bytes.length) { - return bytes; - } - byte[] arr = new byte[len]; - PythonUtils.arraycopy(this.bytes, this.offset, arr, 0, len); - return arr; - } - - public int getOffset() { - return offset; - } - - public byte get(int i) { - return this.bytes[this.offset + i]; - } - - public int getUnsigned(int i) { - return get(i) & 0xff; - } - - public void put(int i, byte value) { - this.bytes[this.offset + i] = value; - } - - public void writeSize64(int value) { - PickleUtils.writeSize64(bytes, offset, value); - } - - public void memmove(int off, int num) { - PythonUtils.arraycopy(this.bytes, this.offset + off, this.bytes, this.offset, num); - } -} diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/MemoTable.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/MemoTable.java index 4393f54f42..182d4cdd95 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/MemoTable.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/MemoTable.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -42,6 +42,8 @@ import static com.oracle.graal.python.runtime.exception.PythonErrorType.PicklingError; +import java.util.Arrays; + import com.oracle.graal.python.nodes.ErrorMessages; import com.oracle.graal.python.nodes.PRaiseNode; import com.oracle.graal.python.util.PythonUtils; @@ -50,50 +52,22 @@ public final class MemoTable { - private static final int INITIAL_CAPACITY = 8; + static final int INITIAL_CAPACITY = 8; private static final int OCCUPANCY_EXPONENT = 1; // 2^X relation between capacity and size private static final int CAPACITY_INC_EXPONENT = 2; // 2^X increase in capacity when resizing - public static final class MemoIterator { - - private int index; - private final Object[] keys; - private final int[] values; - - public MemoIterator(MemoTable table) { - this.keys = table.keys; - this.values = table.values; - this.index = -1; - } - - public boolean advance() { - while (true) { - index++; - if (index >= keys.length) { - return false; - } - if (keys[index] != null) { - return true; - } - } - } - - public Object key() { - return keys[index]; - } - - public int value() { - return values[index]; - } - } - private Object[] keys; private int[] values; private int mask; private int size; public MemoTable() { - initArrays(INITIAL_CAPACITY); + this(INITIAL_CAPACITY); + } + + public MemoTable(int initialCapacity) { + assert initialCapacity >= INITIAL_CAPACITY && Integer.bitCount(initialCapacity) == 1; + initArrays(initialCapacity); } private MemoTable(MemoTable map) { @@ -112,7 +86,8 @@ public int size() { } public void clear() { - initArrays(INITIAL_CAPACITY); + Arrays.fill(keys, null); + Arrays.fill(values, 0); this.size = 0; } @@ -176,11 +151,20 @@ private void resize() { throw PRaiseNode.raiseStatic(null, PicklingError, ErrorMessages.STRUCT_SIZE_TOO_LONG); } - MemoIterator iterator = iterator(); // captures the current contents + Object[] oldKeys = keys; + int[] oldValues = values; initArrays(newLength); - while (iterator.advance()) { - setInternal(iterator.key(), iterator.value()); + for (int i = 0; i < oldKeys.length; i++) { + Object key = oldKeys[i]; + if (key != null) { + int index = getIndex(key); + while (keys[index] != null) { + index = (index + 1) & mask; + } + keys[index] = key; + values[index] = oldValues[i]; + } } } @@ -192,7 +176,15 @@ public void set(Object key, int value) { } } - public MemoIterator iterator() { - return new MemoIterator(this); + int capacity() { + return keys.length; + } + + Object getKeyAt(int index) { + return keys[index]; + } + + int getValueAt(int index) { + return values[index]; } } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java index de1e0508e8..68d31392b7 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PPickler.java @@ -40,6 +40,12 @@ */ package com.oracle.graal.python.builtins.modules.pickle; +import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.T_CP_REVERSE_IMPORT_MAPPING; +import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.T_CP_REVERSE_NAME_MAPPING; +import static com.oracle.graal.python.builtins.objects.PNone.NO_VALUE; +import static com.oracle.graal.python.nodes.BuiltinNames.T___MAIN__; +import static com.oracle.graal.python.nodes.ErrorMessages.MUST_BE_STR_NOT_P; +import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___MODULE__; import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___NAME__; import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___NEWOBJ_EX__; import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___NEWOBJ__; @@ -49,10 +55,7 @@ import static com.oracle.graal.python.nodes.SpecialMethodNames.T___REDUCE_EX__; import static com.oracle.graal.python.nodes.SpecialMethodNames.T___REDUCE__; import static com.oracle.graal.python.nodes.StringLiterals.T_NEWLINE; -import static com.oracle.graal.python.nodes.StringLiterals.T_UTF8; -import static com.oracle.graal.python.nodes.statement.AbstractImportNode.importModule; import static com.oracle.graal.python.runtime.exception.PythonErrorType.KeyError; -import static com.oracle.graal.python.runtime.exception.PythonErrorType.OverflowError; import static com.oracle.graal.python.runtime.exception.PythonErrorType.PicklingError; import static com.oracle.graal.python.runtime.exception.PythonErrorType.TypeError; import static com.oracle.graal.python.runtime.exception.PythonErrorType.ValueError; @@ -66,12 +69,16 @@ import org.graalvm.collections.Pair; import com.oracle.graal.python.PythonLanguage; +import com.oracle.graal.python.builtins.Python3Core; import com.oracle.graal.python.builtins.PythonBuiltinClassType; import com.oracle.graal.python.builtins.objects.PNone; import com.oracle.graal.python.builtins.objects.PNotImplemented; import com.oracle.graal.python.builtins.objects.buffer.PythonBufferAccessLibrary; import com.oracle.graal.python.builtins.objects.buffer.PythonBufferAcquireLibrary; +import com.oracle.graal.python.builtins.objects.bytes.BytesUtils; +import com.oracle.graal.python.builtins.objects.bytes.PByteArray; import com.oracle.graal.python.builtins.objects.bytes.PBytes; +import com.oracle.graal.python.builtins.objects.cext.PythonNativeClass; import com.oracle.graal.python.builtins.objects.common.HashingStorage; import com.oracle.graal.python.builtins.objects.common.HashingStorageNodes.HashingStorageIterator; import com.oracle.graal.python.builtins.objects.common.HashingStorageNodes.HashingStorageIteratorKey; @@ -80,53 +87,66 @@ import com.oracle.graal.python.builtins.objects.common.HashingStorageNodes.HashingStorageLen; import com.oracle.graal.python.builtins.objects.dict.PDict; import com.oracle.graal.python.builtins.objects.ellipsis.PEllipsis; -import com.oracle.graal.python.builtins.objects.ints.IntNodes; -import com.oracle.graal.python.builtins.objects.ints.IntNodesFactory; +import com.oracle.graal.python.builtins.objects.floats.PFloat; +import com.oracle.graal.python.builtins.objects.function.PFunction; +import com.oracle.graal.python.builtins.objects.ints.PInt; +import com.oracle.graal.python.builtins.objects.list.PList; import com.oracle.graal.python.builtins.objects.object.PythonBuiltinObject; import com.oracle.graal.python.builtins.objects.set.PFrozenSet; import com.oracle.graal.python.builtins.objects.set.PSet; +import com.oracle.graal.python.builtins.objects.str.PString; import com.oracle.graal.python.builtins.objects.tuple.PTuple; import com.oracle.graal.python.builtins.objects.type.PythonBuiltinClass; +import com.oracle.graal.python.builtins.objects.type.PythonManagedClass; import com.oracle.graal.python.builtins.objects.type.TypeNodes; import com.oracle.graal.python.lib.IteratorExhausted; import com.oracle.graal.python.lib.PyCallableCheckNode; -import com.oracle.graal.python.lib.PyFloatAsDoubleNode; import com.oracle.graal.python.lib.PyLongAsLongNode; import com.oracle.graal.python.lib.PyObjectCallMethodObjArgs; import com.oracle.graal.python.lib.PyObjectGetIter; import com.oracle.graal.python.lib.PyObjectIsTrueNode; import com.oracle.graal.python.lib.PyObjectLookupAttr; import com.oracle.graal.python.lib.PyObjectReprAsTruffleStringNode; -import com.oracle.graal.python.lib.PyObjectSizeNode; import com.oracle.graal.python.lib.PyObjectStrAsObjectNode; import com.oracle.graal.python.lib.PyTupleCheckNode; import com.oracle.graal.python.nodes.ErrorMessages; import com.oracle.graal.python.nodes.PGuards; import com.oracle.graal.python.nodes.PRaiseNode; import com.oracle.graal.python.nodes.builtins.ListNodes; +import com.oracle.graal.python.nodes.builtins.TupleNodes.GetTupleStorage; import com.oracle.graal.python.nodes.call.CallNode; -import com.oracle.graal.python.nodes.classes.IsSubtypeNode; import com.oracle.graal.python.nodes.object.IsNode; +import com.oracle.graal.python.runtime.ExecutionContext.BoundaryCallContext; import com.oracle.graal.python.runtime.IndirectCallData.BoundaryCallData; -import com.oracle.graal.python.runtime.IndirectCallData.InteropCallData; import com.oracle.graal.python.runtime.PythonContext; import com.oracle.graal.python.runtime.exception.PException; import com.oracle.graal.python.runtime.object.PFactory; +import com.oracle.graal.python.runtime.sequence.storage.DoubleSequenceStorage; +import com.oracle.graal.python.runtime.sequence.storage.IntSequenceStorage; +import com.oracle.graal.python.runtime.sequence.storage.LongSequenceStorage; import com.oracle.graal.python.runtime.sequence.storage.SequenceStorage; import com.oracle.graal.python.util.Consumer; import com.oracle.graal.python.util.NumericSupport; +import com.oracle.graal.python.util.OverflowException; import com.oracle.graal.python.util.PythonUtils; import com.oracle.truffle.api.CompilerDirectives; +import com.oracle.truffle.api.CompilerDirectives.CompilationFinal; import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; import com.oracle.truffle.api.dsl.Bind; import com.oracle.truffle.api.dsl.Cached; +import com.oracle.truffle.api.dsl.GenerateInline; import com.oracle.truffle.api.dsl.Specialization; import com.oracle.truffle.api.frame.VirtualFrame; +import com.oracle.truffle.api.nodes.LoopNode; import com.oracle.truffle.api.nodes.Node; import com.oracle.truffle.api.object.Shape; +import com.oracle.truffle.api.profiles.InlinedConditionProfile; +import com.oracle.truffle.api.profiles.InlinedIntValueProfile; +import com.oracle.truffle.api.strings.TranscodingErrorHandler; import com.oracle.truffle.api.strings.TruffleString; public class PPickler extends PythonBuiltinObject { + private static final byte NEW_LINE_BYTE = '\n'; // Memo table, keep track of the seen objects to support self-referential objects pickling private MemoTable memo; // persistent_id() method, can be NULL @@ -144,8 +164,6 @@ public class PPickler extends PythonBuiltinObject { private byte[] outputBuffer; // Length of output_buffer private int outputLen; - // Allocation size of output_buffer - private int maxOutputLen; // Pickle protocol number, >= 0 private int proto; // true if proto > 0 @@ -162,11 +180,15 @@ public class PPickler extends PythonBuiltinObject { private int fastNesting; // Indicate whether Pickler should fix the name of globals for Python 2.x. private boolean fixImports; - private final Map fastMemo = createFastMemoTable(); + private Map fastMemo; // Callback for out-of-band buffers, or NULL private Object bufferCallback; public PPickler(Object cls, Shape instanceShape) { + this(cls, instanceShape, PickleUtils.WRITE_BUF_SIZE, MemoTable.INITIAL_CAPACITY); + } + + public PPickler(Object cls, Shape instanceShape, int outputCapacity, int memoCapacity) { super(cls, instanceShape); persFunc = null; dispatchTable = null; @@ -179,12 +201,11 @@ public PPickler(Object cls, Shape instanceShape) { fast = 0; fastNesting = 0; fixImports = false; - maxOutputLen = PickleUtils.WRITE_BUF_SIZE; outputLen = 0; reducerOverride = null; - memo = new MemoTable(); - outputBuffer = new byte[maxOutputLen]; + memo = new MemoTable(memoCapacity); + outputBuffer = new byte[outputCapacity]; } @TruffleBoundary @@ -240,6 +261,10 @@ public MemoTable getMemo() { return memo; } + public int getMaxOutputLen() { + return outputBuffer.length; + } + public void setMemo(MemoTable memo) { this.memo = memo; } @@ -260,18 +285,28 @@ public void setPersFuncSelf(Object persFuncSelf) { this.persFuncSelf = persFuncSelf; } + @TruffleBoundary + private void ensureFastMemo() { + if (fastMemo == null) { + fastMemo = createFastMemoTable(); + } + } + @TruffleBoundary private boolean fastMemoContains(Object object) { + ensureFastMemo(); return this.fastMemo.containsKey(object); } @TruffleBoundary private void fastMemoPut(Object object) { + ensureFastMemo(); this.fastMemo.put(object, true); } @TruffleBoundary private void fastMemoRemove(Object object) { + ensureFastMemo(); this.fastMemo.remove(object); } @@ -312,8 +347,7 @@ public void initInternals(VirtualFrame frame, Node inliningTarget, PyObjectLooku this.outputLen = 0; if (this.outputBuffer == null) { - this.maxOutputLen = PickleUtils.WRITE_BUF_SIZE; - this.outputBuffer = new byte[this.maxOutputLen]; + this.outputBuffer = new byte[PickleUtils.WRITE_BUF_SIZE]; } this.fast = 0; @@ -334,7 +368,6 @@ public void clearMemo() { } public void clearBuffer() { - this.outputBuffer = new byte[this.maxOutputLen]; this.outputLen = 0; this.frameStart = -1; } @@ -345,113 +378,382 @@ public void commitFrame() { } int frameLen = outputLen - frameStart - PickleUtils.FRAME_HEADER_SIZE; - ByteArrayView qdata = new ByteArrayView(outputBuffer, frameStart); if (frameLen >= PickleUtils.FRAME_SIZE_MIN) { - qdata.put(0, PickleUtils.OPCODE_FRAME); - qdata.add(1); - qdata.writeSize64(frameLen); + outputBuffer[frameStart] = PickleUtils.OPCODE_FRAME; + PickleUtils.writeSize64(outputBuffer, frameStart + 1, frameLen); } else { - qdata.memmove(PickleUtils.FRAME_HEADER_SIZE, frameLen); + PythonUtils.arraycopy(outputBuffer, frameStart + PickleUtils.FRAME_HEADER_SIZE, outputBuffer, frameStart, frameLen); outputLen -= PickleUtils.FRAME_HEADER_SIZE; } frameStart = -1; } - public PBytes getString(PythonLanguage language) { - commitFrame(); - return PFactory.createBytes(language, outputBuffer, outputLen); + public PBytes getString(PythonLanguage language, boolean copy) { + if (copy || outputBuffer.length > outputLen * 5 / 4) { + return PFactory.createBytes(language, PythonUtils.arrayCopyOf(outputBuffer, outputLen)); + } else { + return PFactory.createBytes(language, outputBuffer, outputLen); + } + } + + private void writeASCII(Node node, TruffleString string, TruffleString.SwitchEncodingNode switchEncodingNode, + TruffleString.CopyToByteArrayNode copyToByteArrayNode) { + assert string.getCodeRangeUncached(TS_ENCODING) == TruffleString.CodeRange.ASCII; + TruffleString encoded = switchEncodingNode.execute(string, TruffleString.Encoding.US_ASCII); + int length = encoded.byteLength(TruffleString.Encoding.US_ASCII); + ensureBufferSpace(node, length); + copyToByteArrayNode.execute(encoded, 0, outputBuffer, outputLen, length, TruffleString.Encoding.US_ASCII); + outputLen += length; + } + + private void writeUtf8(Node node, TruffleString string, TruffleString.SwitchEncodingNode switchEncodingNode, + TruffleString.CopyToByteArrayNode copyToByteArrayNode) { + TruffleString encoded = switchEncodingNode.execute(string, TruffleString.Encoding.UTF_8, TranscodingErrorHandler.DEFAULT_KEEP_SURROGATES_IN_UTF8); + writeUtf8Converted(node, encoded, copyToByteArrayNode); + } + + private void writeUtf8Converted(Node node, TruffleString encoded, TruffleString.CopyToByteArrayNode copyToByteArrayNode) { + int length = encoded.byteLength(TruffleString.Encoding.UTF_8); + ensureBufferSpace(node, length); + copyToByteArrayNode.execute(encoded, 0, outputBuffer, outputLen, length, TruffleString.Encoding.UTF_8); + outputLen += length; + } + + private void writeLongAsString(Node node, byte opcode, long value, TruffleString.FromLongNode fromLongNode, + TruffleString.CopyToByteArrayNode copyToByteArrayNode) { + TruffleString string = fromLongNode.execute(value, TruffleString.Encoding.US_ASCII, true); + TruffleString.Encoding encoding = TruffleString.Encoding.US_ASCII; + int length = string.byteLength(encoding); + ensureBufferSpace(node, length + 2); + outputBuffer[outputLen++] = opcode; + copyToByteArrayNode.execute(string, 0, outputBuffer, outputLen, length, encoding); + outputLen += length; + outputBuffer[outputLen++] = NEW_LINE_BYTE; + } + + private void write(Node node, byte b) { + ensureBufferSpace(node, 1); + outputBuffer[outputLen++] = b; + } + + private void write(Node node, byte b1, byte b2) { + ensureBufferSpace(node, 2); + outputBuffer[outputLen++] = b1; + outputBuffer[outputLen++] = b2; + } + + private void writeBuffer(Node node, Object buffer, int size, PythonBufferAccessLibrary bufferLib) { + ensureBufferSpace(node, size); + bufferLib.readIntoByteArray(buffer, 0, outputBuffer, outputLen, size); + outputLen += size; + } + + private void writeByteOp(Node node, byte opcode, long value) { + ensureBufferSpace(node, 2); + int start = outputLen; + outputBuffer[start] = opcode; + outputBuffer[start + 1] = (byte) value; + outputLen = start + 2; + } + + private void writeShortOp(Node node, byte opcode, long value) { + ensureBufferSpace(node, 3); + int start = outputLen; + outputBuffer[start] = opcode; + NumericSupport.littleEndian().putShort(outputBuffer, start + 1, (short) value); + outputLen = start + 3; + } + + private void writeIntOp(Node node, byte opcode, long value) { + ensureBufferSpace(node, 5); + int start = outputLen; + outputBuffer[start] = opcode; + NumericSupport.littleEndian().putInt(outputBuffer, start + 1, (int) value); + outputLen = start + 5; + } + + private void ensureBufferSpace(Node node, int dataLen) { + boolean needNewFrame = isFraming() && frameStart == -1; + int n = CompilerDirectives.injectBranchProbability(CompilerDirectives.SLOWPATH_PROBABILITY, needNewFrame) ? dataLen + PickleUtils.FRAME_HEADER_SIZE : dataLen; + int required = outputLen + n; + + if (CompilerDirectives.injectBranchProbability(CompilerDirectives.SLOWPATH_PROBABILITY, required > outputBuffer.length)) { + // Make place in buffer for the pickle chunk + // TODO: when GR-24978 is completed we should use PY_SSIZE_T_MAX + if (CompilerDirectives.injectBranchProbability(CompilerDirectives.SLOWPATH_PROBABILITY, outputLen >= Integer.MAX_VALUE / 2 - n)) { + CompilerDirectives.transferToInterpreter(); + throw PRaiseNode.raiseStatic(node, PythonBuiltinClassType.MemoryError); + } + int newCapacity = (required) / 2 * 3; + outputBuffer = PythonUtils.arrayCopyOf(outputBuffer, newCapacity); + } + if (CompilerDirectives.injectBranchProbability(CompilerDirectives.SLOWPATH_PROBABILITY, needNewFrame)) { + frameStart = outputLen; + outputLen += PickleUtils.FRAME_HEADER_SIZE; + } + } + + private void writeRawUnicodeEscape(Node node, TruffleString string, + TruffleString.CodePointLengthNode codePointLengthNode, TruffleString.CodePointAtIndexUTF32Node codePointAtIndexNode) { + int codePointLength = codePointLengthNode.execute(string, TS_ENCODING); + long escapedLength = 0; + for (int i = 0; i < codePointLength; i++) { + int ch = codePointAtIndexNode.execute(string, i); + escapedLength += ch >= 0x10000 ? 10 : ch >= 256 || ch == '\\' || ch == 0 || ch == '\n' || ch == '\r' || ch == 0x1a ? 6 : 1; + } + if (escapedLength > Integer.MAX_VALUE) { + throw PRaiseNode.raiseStatic(node, PythonBuiltinClassType.MemoryError); + } + ensureBufferSpace(node, (int) escapedLength); + for (int i = 0; i < codePointLength; i++) { + int ch = codePointAtIndexNode.execute(string, i); + if (ch >= 0x10000) { + outputBuffer[outputLen++] = '\\'; + outputBuffer[outputLen++] = 'U'; + for (int shift = 28; shift >= 0; shift -= 4) { + outputBuffer[outputLen++] = BytesUtils.HEXDIGITS[(ch >> shift) & 0xf]; + } + } else if (ch >= 256 || ch == '\\' || ch == 0 || ch == '\n' || ch == '\r' || ch == 0x1a) { + outputBuffer[outputLen++] = '\\'; + outputBuffer[outputLen++] = 'u'; + for (int shift = 12; shift >= 0; shift -= 4) { + outputBuffer[outputLen++] = BytesUtils.HEXDIGITS[(ch >> shift) & 0xf]; + } + } else { + outputBuffer[outputLen++] = (byte) ch; + } + } } // inner nodes - public abstract static class BasePickleWriteNode extends PicklerNodes.BasePickleNode { + @GenerateInline(false) + public abstract static class TupleStorageNode extends Node { + public abstract SequenceStorage execute(Object tuple); + + @Specialization + static SequenceStorage get(Object tuple, + @Bind Node inliningTarget, + @Cached GetTupleStorage getTupleStorage) { + return getTupleStorage.execute(inliningTarget, tuple); + } + } + + public abstract static class SaveNode extends PicklerNodes.BasePickleNode { + private static final int MAX_RECURSION_DEPTH = 3; + private static final TruffleString T_SYS_MODULES = tsLiteral("sys.modules"); + static final byte[] TUPLE_LEN_2_OPCODE = new byte[]{PickleUtils.OPCODE_EMPTY_TUPLE, PickleUtils.OPCODE_TUPLE1, PickleUtils.OPCODE_TUPLE2, PickleUtils.OPCODE_TUPLE3}; - static final byte NEW_LINE_BYTE = '\n'; static final TruffleString DICT_ITEMS = tsLiteral("dict items"); static final TruffleString LATIN1 = tsLiteral("latin1"); - static final TruffleString REDUCE_OVERRIDE = tsLiteral("reducer_override"); static final TruffleString T_L_NEW_LINE = tsLiteral("L\n"); static final TruffleString T_SET = tsLiteral("set"); static final TruffleString T_DICTIONARY = tsLiteral("dictionary"); - @Child private IntNodes.PyLongSign pyLongSign; - @Child private IntNodes.PyLongNumBits pyLongNumBits; - @Child private IntNodes.PyLongAsByteArray pyLongAsByteArray; + private static final long SEEN_NONE = 1; + private static final long SEEN_BOOLEAN = 1 << 1; + private static final long SEEN_INTEGER = 1 << 2; + private static final long SEEN_LONG = 1 << 3; + private static final long SEEN_PINT = 1 << 4; + private static final long SEEN_DOUBLE = 1 << 5; + private static final long SEEN_PFLOAT = 1 << 6; + private static final long SEEN_BUILTIN_CLASS_TYPE = 1 << 7; + private static final long SEEN_BYTES = 1 << 8; + private static final long SEEN_TRUFFLE_STRING = 1 << 9; + private static final long SEEN_PSTRING = 1 << 10; + private static final long SEEN_DICT = 1 << 11; + private static final long SEEN_SET = 1 << 12; + private static final long SEEN_FROZENSET = 1 << 13; + private static final long SEEN_LIST = 1 << 14; + private static final long SEEN_TUPLE = 1 << 15; + private static final long SEEN_NATIVE_TUPLE = 1 << 16; + private static final long SEEN_BYTE_ARRAY = 1 << 17; + private static final long SEEN_PICKLE_BUFFER = 1 << 18; + private static final long SEEN_MANAGED_CLASS = 1 << 19; + private static final long SEEN_NATIVE_CLASS = 1 << 20; + private static final long SEEN_FUNCTION = 1 << 21; + private static final long SEEN_TYPE = 1 << 22; + private static final long SEEN_OBJECT = 1 << 23; + private static final long SEEN_INT_STORAGE = 1 << 24; + private static final long SEEN_LONG_STORAGE = 1 << 25; + private static final long SEEN_DOUBLE_STORAGE = 1 << 26; + private static final long SEEN_GENERIC_STORAGE = 1 << 27; + private static final long SEEN_CUSTOM_DISPATCH = 1 << 28; + private static final long SEEN_REGISTERED_REDUCER = 1 << 29; + private static final long SEEN_REDUCE_EX = 1 << 30; + private static final long SEEN_REDUCE = 1L << 31; + private static final long SEEN_NEWOBJ_EX = 1L << 32; + private static final long SEEN_NEWOBJ = 1L << 33; + private static final long SEEN_PLAIN_REDUCE = 1L << 34; + private static final long SEEN_REDUCE_LISTITEMS = 1L << 35; + private static final long SEEN_REDUCE_DICTITEMS = 1L << 36; + private static final long SEEN_REDUCE_STATE = 1L << 37; + private static final long SEEN_BYPASS_BUFFER = 1L << 38; + + @CompilationFinal private long seenTypes; + + private final int depth; + // Keep recursive profiles separate for each role in the pickle stream. + @Child private SaveNode dictKeySaveNode; + @Child private SaveNode dictValueSaveNode; + @Child private SaveNode setElementSaveNode; + @Child private SaveNode listElementSaveNode; + @Child private SaveNode tupleElementSaveNode; + @Child private SaveNode persistentIdSaveNode; + @Child private SaveNode reduceClassSaveNode; + @Child private SaveNode reduceCallableSaveNode; + @Child private SaveNode reduceArgsSaveNode; + @Child private SaveNode reduceKwargsSaveNode; + @Child private SaveNode reduceStateSaveNode; + @Child private SaveNode reduceStateSetterSaveNode; + @Child private SaveNode reduceObjectSaveNode; + @Child private SaveNode globalModuleNameSaveNode; + @Child private SaveNode globalNameSaveNode; + @Child private ListNodes.ConstructListNode constructListNode; - @Child private IsSubtypeNode isSubTypeNode; @Child private TypeNodes.IsTypeNode isTypeNode; @Child private FlushToFileNode flushToFileNode; - @Child private CallNode callNode; + @Child private TupleStorageNode getTupleStorageNode; + @Child private TruffleString.CodePointAtIndexUTF32Node tsCodePointAtIndexUTF32Node; + @Child private TruffleString.FromLongNode tsFromLongNode; + @Child private TruffleString.CopyToByteArrayNode tsCopyToByteArrayNode; + @Child private TruffleString.GetCodeRangeNode tsGetCodeRangeNode; + @Child private BoundaryCallData boundaryCallData; + @Child private PyLongAsLongNode pyLongAsLongNode; + @Child private PyObjectStrAsObjectNode pyObjectStrAsObjectNode; + @Child private PyObjectIsTrueNode isTrueNode; + @Child private IsNode isNode; + @Child private PythonBufferAcquireLibrary bufferAcquireLibrary; + @Child private PythonBufferAccessLibrary bufferLibrary; + @Child private PyCallableCheckNode callableCheckNode; + @Child private PyObjectReprAsTruffleStringNode reprNode; + @Child private PyObjectGetIter getIterNode; + @Child private HashingStorageLen hashingStorageLenNode; + @Child private PyTupleCheckNode.CachedNode tupleCheckNode; - protected boolean isType(Object obj) { - if (isTypeNode == null) { + protected SaveNode(int depth) { + this.depth = depth; + } + + protected SequenceStorage getTupleStorage(Object tuple) { + if (getTupleStorageNode == null) { CompilerDirectives.transferToInterpreterAndInvalidate(); - isTypeNode = insert(TypeNodes.IsTypeNode.create()); + getTupleStorageNode = insert(PPicklerFactory.TupleStorageNodeGen.create()); } - return isTypeNode.executeCached(obj); + return getTupleStorageNode.execute(tuple); } - protected void flushToFile(VirtualFrame frame, PPickler pickler) { - if (flushToFileNode == null) { + protected TruffleString.CodePointAtIndexUTF32Node ensureTsCodePointAtIndexUTF32Node() { + if (tsCodePointAtIndexUTF32Node == null) { CompilerDirectives.transferToInterpreterAndInvalidate(); - flushToFileNode = insert(PPicklerFactory.FlushToFileNodeGen.create()); + tsCodePointAtIndexUTF32Node = insert(TruffleString.CodePointAtIndexUTF32Node.create()); } - flushToFileNode.execute(frame, pickler); + return tsCodePointAtIndexUTF32Node; } - protected int getSign(Object value) { - if (pyLongSign == null) { + protected TruffleString.FromLongNode ensureTsFromLongNode() { + if (tsFromLongNode == null) { CompilerDirectives.transferToInterpreterAndInvalidate(); - pyLongSign = insert(IntNodesFactory.PyLongSignNodeGen.create()); + tsFromLongNode = insert(TruffleString.FromLongNode.create()); } - return pyLongSign.execute(value); + return tsFromLongNode; } - protected int getNumBits(Object value) { - if (pyLongNumBits == null) { + protected TruffleString.CopyToByteArrayNode ensureTsCopyToByteArrayNode() { + if (tsCopyToByteArrayNode == null) { CompilerDirectives.transferToInterpreterAndInvalidate(); - pyLongNumBits = insert(IntNodesFactory.PyLongNumBitsNodeGen.create()); + tsCopyToByteArrayNode = insert(TruffleString.CopyToByteArrayNode.create()); } - return pyLongNumBits.execute(value); + return tsCopyToByteArrayNode; } - protected byte[] longAsBytes(Object value, int size, boolean bigEndian) { - if (pyLongAsByteArray == null) { + protected TruffleString.GetCodeRangeNode ensureTsGetCodeRangeNode() { + if (tsGetCodeRangeNode == null) { CompilerDirectives.transferToInterpreterAndInvalidate(); - pyLongAsByteArray = insert(IntNodesFactory.PyLongAsByteArrayNodeGen.create()); + tsGetCodeRangeNode = insert(TruffleString.GetCodeRangeNode.create()); } - return pyLongAsByteArray.executeCached(value, size, bigEndian); + return tsGetCodeRangeNode; } - protected Object createList(VirtualFrame frame, Object iterable) { - if (constructListNode == null) { + public TruffleString asStringStrict(Object value) { + TruffleString string = asString(value); + if (string == null) { + throw raise(TypeError, MUST_BE_STR_NOT_P, value); + } + return string; + } + + private static boolean checkModule(VirtualFrame frame, PicklerNodes.LookupGlobalAttributeNode lookup, Object moduleName, Object module, Object global, TruffleString[] dottedPath) { + if (module == PNone.NONE) { + return false; + } + assert !(moduleName instanceof String) : "moduleName shouldn't be j.l.String"; + if (moduleName instanceof TruffleString && ((TruffleString) moduleName).equalsUncached(T___MAIN__, TS_ENCODING)) { + return false; + } + Pair pair = getDeepAttribute(frame, lookup, module, dottedPath); + return pair != null && pair.getLeft() == global; + } + + public TruffleString whichModule(VirtualFrame frame, PythonContext context, Object global, TruffleString[] dottedPath) { + Object moduleName = lookupAttribute(frame, global, T___MODULE__); + if (moduleName != NO_VALUE && moduleName != PNone.NONE) { + return asStringStrict(moduleName); + } + + PDict sysModules = context.getSysModules(); + if (sysModules == null) { + throw raise(PythonBuiltinClassType.RuntimeError, ErrorMessages.UNABLE_TO_GET_S, T_SYS_MODULES); + } + + HashingStorage storage = sysModules.getDictStorage(); + HashingStorageIterator it = getHashingStorageIterator(storage); + HashingStorageIteratorNext nextNode = ensureHashingStorageIteratorNext(); + HashingStorageIteratorKey getKeyNode = ensureHashingStorageIteratorKey(); + HashingStorageIteratorValue getValueNode = ensureHashingStorageIteratorValue(); + while (nextNode.executeCached(storage, it)) { + Object value = getValueNode.executeCached(storage, it); + Object key = getKeyNode.executeCached(storage, it); + if (checkModule(frame, getLookupGlobalAttrNode(), moduleName, value, global, dottedPath)) { + return asStringStrict(key); + } + } + return T___MAIN__; + } + + protected boolean isType(Object obj) { + if (isTypeNode == null) { CompilerDirectives.transferToInterpreterAndInvalidate(); - constructListNode = insert(ListNodes.ConstructListNode.create()); + isTypeNode = insert(TypeNodes.IsTypeNode.create()); } - return constructListNode.execute(frame, iterable); + return isTypeNode.executeCached(obj); } - protected boolean isSubType(Object clsA, Object clsB) { - if (isSubTypeNode == null) { + protected void flushToFile(VirtualFrame frame, PPickler pickler) { + ensureFlushToFileNode().execute(frame, pickler); + } + + protected FlushToFileNode ensureFlushToFileNode() { + if (flushToFileNode == null) { CompilerDirectives.transferToInterpreterAndInvalidate(); - isSubTypeNode = insert(IsSubtypeNode.create()); + flushToFileNode = insert(PPicklerFactory.FlushToFileNodeGen.create()); } - return isSubTypeNode.execute(clsA, clsB); + return flushToFileNode; } - protected CallNode getCallNode() { - if (callNode == null) { + protected Object createList(VirtualFrame frame, Object iterable) { + if (constructListNode == null) { CompilerDirectives.transferToInterpreterAndInvalidate(); - callNode = insert(CallNode.create()); + constructListNode = insert(ListNodes.ConstructListNode.create()); } - return callNode; + return constructListNode.execute(frame, iterable); } - public void opcodeBoundary(VirtualFrame frame, PPickler pickler) { + public boolean opcodeBoundary(VirtualFrame frame, PPickler pickler) { if (!pickler.isFraming() || pickler.frameStart == -1) { - return; + return false; } int frameLen = pickler.outputLen - pickler.frameStart - PickleUtils.FRAME_HEADER_SIZE; - if (frameLen >= PickleUtils.FRAME_SIZE_TARGET) { + if (CompilerDirectives.injectBranchProbability(CompilerDirectives.SLOWPATH_PROBABILITY, frameLen >= PickleUtils.FRAME_SIZE_TARGET)) { pickler.commitFrame(); // Flush the content of the committed frame to the underlying // file and reuse the pickler buffer for the next frame so as @@ -460,8 +762,10 @@ public void opcodeBoundary(VirtualFrame frame, PPickler pickler) { if (pickler.write != null) { flushToFile(frame, pickler); pickler.clearBuffer(); + return true; } } + return false; } public static void handleError(PPickler pickler) { @@ -469,82 +773,6 @@ public static void handleError(PPickler pickler) { pickler.reducerOverride = null; } - protected void writeASCII(PPickler pickler, TruffleString string) { - assert string.getCodeRangeUncached(TS_ENCODING) == TruffleString.CodeRange.ASCII; - final byte[] bytes = PythonUtils.getAsciiBytes(string, ensureTsCopyToByteArrayNode(), ensureTsSwitchEncodingNode()); - write(pickler, bytes, bytes.length); - } - - protected void write(PPickler pickler, byte oneByte) { - write(pickler, new byte[]{oneByte}, 1); - } - - protected void write(PPickler pickler, byte[] bytes) { - write(pickler, bytes, bytes.length); - } - - protected void write(PPickler pickler, byte[] bytes, int dataLen) { - boolean needNewFrame = pickler.isFraming() && pickler.frameStart == -1; - int n = (needNewFrame) ? dataLen + PickleUtils.FRAME_HEADER_SIZE : dataLen; - int required = pickler.outputLen + n; - - if (required > pickler.maxOutputLen) { - // Make place in buffer for the pickle chunk - // TODO: when GR-24978 is completed we should use PY_SSIZE_T_MAX - if (pickler.outputLen >= Integer.MAX_VALUE / 2 - n) { - throw raise(PythonBuiltinClassType.MemoryError); - } - pickler.maxOutputLen = (pickler.outputLen + n) / 2 * 3; - pickler.outputBuffer = PickleUtils.resize(pickler.outputBuffer, pickler.maxOutputLen); - } - if (needNewFrame) { - int frameStart = pickler.outputLen; - pickler.frameStart = frameStart; - for (int i = 0; i < PickleUtils.FRAME_HEADER_SIZE; i++) { - // Write an invalid value, for debugging - pickler.outputBuffer[frameStart + i] = (byte) 0xFE; - } - pickler.outputLen += PickleUtils.FRAME_HEADER_SIZE; - } - PythonUtils.arraycopy(bytes, 0, pickler.outputBuffer, pickler.outputLen, dataLen); - pickler.outputLen += dataLen; - } - - protected void writeBytes(VirtualFrame frame, PPickler pickler, byte[] header, int headerSize, byte[] data, int dataSize, Object payload) { - boolean bypassBuffer = dataSize >= PickleUtils.FRAME_SIZE_TARGET; - boolean framing = pickler.framing; - - if (bypassBuffer) { - assert pickler.outputBuffer != null; - // Commit the previous frame. - pickler.commitFrame(); - // Disable framing temporarily - pickler.framing = false; - } - - write(pickler, header, headerSize); - - if (bypassBuffer && pickler.write != null) { - // Dump the output buffer to the file. - flushToFile(frame, pickler); - // Stream write the payload into the file without going through the output buffer. - Object pld = payload; - if (pld == null) { - // TODO: It would be better to use a memoryview with a linked original string if - // this is possible. - pld = PFactory.createBytes(PythonLanguage.get(this), data, dataSize); - } - getCallNode().execute(frame, pickler.write, pld); - // Reinitialize the buffer for subsequent calls to _Pickler_Write. - pickler.clearBuffer(); - } else { - write(pickler, data, dataSize); - } - - // Re-enable framing for subsequent calls to _Pickler_Write. - pickler.framing = framing; - } - protected PTuple createTuple(Object... items) { if (items.length == 0) { return PFactory.createEmptyTuple(PythonLanguage.get(this)); @@ -568,34 +796,33 @@ protected static void fastSaveLeave(PPickler pickler, Object obj) { pickler.fastMemoRemove(obj); } } - } - public abstract static class FlushToFileNode extends BasePickleWriteNode { - public abstract void execute(VirtualFrame frame, PPickler pickler); + private void profileSeen(long seen) { + if ((seenTypes & seen) == 0) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + seenTypes |= seen; + } + } - @Specialization - public void flush(VirtualFrame frame, PPickler pickler, - @Bind PythonLanguage language) { - assert pickler.write != null; - // This will commit the frame first - PBytes output = pickler.getString(language); - call(frame, pickler.write, output); + private boolean bypassBuffer(PPickler pickler, int dataSize) { + boolean bypassBuffer = dataSize >= PickleUtils.FRAME_SIZE_TARGET; + if (bypassBuffer) { + profileSeen(SEEN_BYPASS_BUFFER); + assert pickler.outputBuffer != null; + // Commit the previous frame before disabling framing temporarily. + pickler.commitFrame(); + pickler.framing = false; + } + return bypassBuffer; } - } - public abstract static class SaveNode extends BasePickleWriteNode { - @Child private SaveNode recursiveSaveNode; - @Child private PyLongAsLongNode pyLongAsLongNode; - @Child private PyObjectStrAsObjectNode pyObjectStrAsObjectNode; - @Child private PyObjectIsTrueNode isTrueNode; - @Child private IsNode isNode; - @Child private PythonBufferAcquireLibrary bufferAcquireLibrary; - @Child private PythonBufferAccessLibrary bufferLibrary; - @Child private PyCallableCheckNode callableCheckNode; - @Child private PyObjectReprAsTruffleStringNode reprNode; - @Child private PyObjectGetIter getIterNode; - @Child private HashingStorageLen hashingStorageLenNode; - @Child private PyTupleCheckNode.CachedNode tupleCheckNode; + private BoundaryCallData getBoundaryCallData() { + if (boundaryCallData == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + boundaryCallData = insert(BoundaryCallData.createFor(this)); + } + return boundaryCallData; + } private int getHashingStorageLength(HashingStorage storage) { if (hashingStorageLenNode == null) { @@ -613,12 +840,204 @@ private boolean isTuple(Object object) { return tupleCheckNode.execute(object); } - private void save(VirtualFrame frame, PPickler pickler, Object obj, int persSave) { - if (recursiveSaveNode == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - recursiveSaveNode = insert(PPicklerFactory.SaveNodeGen.create()); + private void saveDictKey(VirtualFrame frame, PPickler pickler, Object obj) { + if (depth < 0) { + saveDepthExceeded(pickler, obj, 0); + return; + } + if (dictKeySaveNode == null) { + dictKeySaveNode = createRecursiveSaveNode(); + } + saveRecursive(frame, dictKeySaveNode, pickler, obj, 0); + } + + private void saveDictValue(VirtualFrame frame, PPickler pickler, Object obj) { + if (depth < 0) { + saveDepthExceeded(pickler, obj, 0); + return; + } + if (dictValueSaveNode == null) { + dictValueSaveNode = createRecursiveSaveNode(); + } + saveRecursive(frame, dictValueSaveNode, pickler, obj, 0); + } + + private void saveSetElement(VirtualFrame frame, PPickler pickler, Object obj) { + if (depth < 0) { + saveDepthExceeded(pickler, obj, 0); + return; + } + if (setElementSaveNode == null) { + setElementSaveNode = createRecursiveSaveNode(); + } + saveRecursive(frame, setElementSaveNode, pickler, obj, 0); + } + + private void saveListElement(VirtualFrame frame, PPickler pickler, Object obj) { + if (depth < 0) { + saveDepthExceeded(pickler, obj, 0); + return; + } + if (listElementSaveNode == null) { + listElementSaveNode = createRecursiveSaveNode(); + } + saveRecursive(frame, listElementSaveNode, pickler, obj, 0); + } + + private void saveTupleElement(VirtualFrame frame, PPickler pickler, Object obj) { + if (depth < 0) { + saveDepthExceeded(pickler, obj, 0); + return; + } + if (tupleElementSaveNode == null) { + tupleElementSaveNode = createRecursiveSaveNode(); + } + saveRecursive(frame, tupleElementSaveNode, pickler, obj, 0); + } + + private void savePersistentId(VirtualFrame frame, PPickler pickler, Object obj) { + if (depth < 0) { + saveDepthExceeded(pickler, obj, 1); + return; + } + if (persistentIdSaveNode == null) { + persistentIdSaveNode = createRecursiveSaveNode(); + } + saveRecursive(frame, persistentIdSaveNode, pickler, obj, 1); + } + + private void saveReduceClass(VirtualFrame frame, PPickler pickler, Object obj) { + if (depth < 0) { + saveDepthExceeded(pickler, obj, 0); + return; + } + if (reduceClassSaveNode == null) { + reduceClassSaveNode = createRecursiveSaveNode(); + } + saveRecursive(frame, reduceClassSaveNode, pickler, obj, 0); + } + + private void saveReduceCallable(VirtualFrame frame, PPickler pickler, Object obj) { + if (depth < 0) { + saveDepthExceeded(pickler, obj, 0); + return; + } + if (reduceCallableSaveNode == null) { + reduceCallableSaveNode = createRecursiveSaveNode(); + } + saveRecursive(frame, reduceCallableSaveNode, pickler, obj, 0); + } + + private void saveReduceArgs(VirtualFrame frame, PPickler pickler, Object obj) { + if (depth < 0) { + saveDepthExceeded(pickler, obj, 0); + return; + } + if (reduceArgsSaveNode == null) { + reduceArgsSaveNode = createRecursiveSaveNode(); + } + saveRecursive(frame, reduceArgsSaveNode, pickler, obj, 0); + } + + private void saveReduceKwargs(VirtualFrame frame, PPickler pickler, Object obj) { + if (depth < 0) { + saveDepthExceeded(pickler, obj, 0); + return; + } + if (reduceKwargsSaveNode == null) { + reduceKwargsSaveNode = createRecursiveSaveNode(); } - recursiveSaveNode.execute(frame, pickler, obj, persSave); + saveRecursive(frame, reduceKwargsSaveNode, pickler, obj, 0); + } + + private void saveReduceState(VirtualFrame frame, PPickler pickler, Object obj) { + if (depth < 0) { + saveDepthExceeded(pickler, obj, 0); + return; + } + if (reduceStateSaveNode == null) { + reduceStateSaveNode = createRecursiveSaveNode(); + } + saveRecursive(frame, reduceStateSaveNode, pickler, obj, 0); + } + + private void saveReduceStateSetter(VirtualFrame frame, PPickler pickler, Object obj) { + if (depth < 0) { + saveDepthExceeded(pickler, obj, 0); + return; + } + if (reduceStateSetterSaveNode == null) { + reduceStateSetterSaveNode = createRecursiveSaveNode(); + } + saveRecursive(frame, reduceStateSetterSaveNode, pickler, obj, 0); + } + + private void saveReduceObject(VirtualFrame frame, PPickler pickler, Object obj) { + if (depth < 0) { + saveDepthExceeded(pickler, obj, 0); + return; + } + if (reduceObjectSaveNode == null) { + reduceObjectSaveNode = createRecursiveSaveNode(); + } + saveRecursive(frame, reduceObjectSaveNode, pickler, obj, 0); + } + + private void saveGlobalModuleName(VirtualFrame frame, PPickler pickler, Object obj) { + if (depth < 0) { + saveDepthExceeded(pickler, obj, 0); + return; + } + if (globalModuleNameSaveNode == null) { + globalModuleNameSaveNode = createRecursiveSaveNode(); + } + saveRecursive(frame, globalModuleNameSaveNode, pickler, obj, 0); + } + + private void saveGlobalName(VirtualFrame frame, PPickler pickler, Object obj) { + if (depth < 0) { + saveDepthExceeded(pickler, obj, 0); + return; + } + if (globalNameSaveNode == null) { + globalNameSaveNode = createRecursiveSaveNode(); + } + saveRecursive(frame, globalNameSaveNode, pickler, obj, 0); + } + + private SaveNode createRecursiveSaveNode() { + CompilerDirectives.transferToInterpreterAndInvalidate(); + return insert(PPicklerFactory.SaveNodeGen.create(depth < MAX_RECURSION_DEPTH ? depth + 1 : -1)); + } + + private void saveRecursive(VirtualFrame frame, SaveNode saveNode, PPickler pickler, Object obj, int persSave) { + if (depth < MAX_RECURSION_DEPTH) { + saveNode.execute(frame, pickler, obj, persSave); + } else { + saveSetupBoundary(frame, saveNode, pickler, obj, persSave); + } + } + + // This should never be reached for compilation, but native-image can't prove it at build time + @TruffleBoundary + private void saveDepthExceeded(PPickler pickler, Object obj, int persSave) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + execute(null, pickler, obj, persSave); + } + + private void saveSetupBoundary(VirtualFrame frame, SaveNode saveNode, PPickler pickler, Object obj, int persSave) { + BoundaryCallData callData = getBoundaryCallData(); + Object savedState = BoundaryCallContext.enter(frame, callData); + try { + saveBoundary(saveNode, pickler, obj, persSave); + } finally { + BoundaryCallContext.exit(frame, callData, savedState); + } + } + + @TruffleBoundary + private void saveBoundary(SaveNode saveNode, PPickler pickler, Object obj, int persSave) { + saveNode.execute(null, pickler, obj, persSave); } private long asLong(VirtualFrame frame, Object object) { @@ -695,6 +1114,10 @@ private PythonBufferAccessLibrary getBufferLibrary() { public abstract void execute(VirtualFrame frame, PPickler pickler, Object obj, int persSave); + private static boolean isBin(int proto) { + return proto > 0; + } + // collections private void saveHashingStorageBatched(VirtualFrame frame, PPickler pickler, HashingStorage storage, byte opcodeBatch, TruffleString name, boolean saveValues) { @@ -709,17 +1132,20 @@ private void saveHashingStorageBatched(VirtualFrame frame, PPickler pickler, Has int i; do { i = 0; - write(pickler, PickleUtils.OPCODE_MARK); + pickler.write(this, PickleUtils.OPCODE_MARK); while (nextNode.executeCached(storage, it)) { - save(frame, pickler, getKeyNode.executeCached(storage, it), 0); + Object key = getKeyNode.executeCached(storage, it); if (saveValues) { - save(frame, pickler, getValueNode.executeCached(storage, it), 0); + saveDictKey(frame, pickler, key); + saveDictValue(frame, pickler, getValueNode.executeCached(storage, it)); + } else { + saveSetElement(frame, pickler, key); } if (++i == PickleUtils.BATCHSIZE) { break; } } - write(pickler, opcodeBatch); + pickler.write(this, opcodeBatch); if (getHashingStorageLength(storage) != initialSize) { throw raise(PythonBuiltinClassType.RuntimeError, ErrorMessages.CHANGED_SIZE_DURING_ITERATION, name); } @@ -740,7 +1166,7 @@ private void saveSetHashingStorage(VirtualFrame frame, PPickler pickler, Hashing HashingStorageIteratorKey getKeyNode = ensureHashingStorageIteratorKey(); while (next.executeCached(storage, it)) { Object item = getKeyNode.executeCached(storage, it); - save(frame, pickler, item, 0); + saveSetElement(frame, pickler, item); } } @@ -765,12 +1191,12 @@ private void saveIteratorBatchedUnrolled(VirtualFrame frame, PPickler pickler, O } catch (IteratorExhausted e) { // Only one item to write saveItem.accept(firstItem); - write(pickler, opcodeOneItem); + pickler.write(this, opcodeOneItem); break; } // More than one item to write // Pump out MARK, items, APPENDS. - write(pickler, PickleUtils.OPCODE_MARK); + pickler.write(this, PickleUtils.OPCODE_MARK); saveItem.accept(firstItem); n = 1; while (true) { @@ -789,7 +1215,7 @@ private void saveIteratorBatchedUnrolled(VirtualFrame frame, PPickler pickler, O break; } } - write(pickler, opcodeMultipleItems); + pickler.write(this, opcodeMultipleItems); } while (n == PickleUtils.BATCHSIZE); } @@ -797,13 +1223,14 @@ private void saveIteratorBatchedUnrolled(VirtualFrame frame, PPickler pickler, O private void saveDictIteratorBatchUnrolled(VirtualFrame frame, PPickler pickler, Object iterator) { saveIteratorBatchedUnrolled(frame, pickler, iterator, PickleUtils.OPCODE_SETITEM, PickleUtils.OPCODE_SETITEMS, (Object item) -> { - if (!isTuple(item) || length(frame, item) != 2) { + if (!isTuple(item) || getTupleStorage(item).length() != 2) { throw raise(TypeError, ErrorMessages.MUST_S_ITER_RETURN_2TUPLE, DICT_ITEMS); } }, (Object item) -> { - save(frame, pickler, getItem(frame, item, 0), 0); - save(frame, pickler, getItem(frame, item, 1), 0); + SequenceStorage storage = getTupleStorage(item); + saveDictKey(frame, pickler, getItem(storage, 0)); + saveDictValue(frame, pickler, getItem(storage, 1)); }); } @@ -811,7 +1238,7 @@ private void saveListIteratorBatchUnrolled(VirtualFrame frame, PPickler pickler, saveIteratorBatchedUnrolled(frame, pickler, iterator, PickleUtils.OPCODE_APPEND, PickleUtils.OPCODE_APPENDS, (Object item) -> { }, - (Object item) -> save(frame, pickler, item, 0)); + (Object item) -> saveListElement(frame, pickler, item)); } private void saveIterator(VirtualFrame frame, PPickler pickler, Object iterator, byte opcode, Consumer itemConsumer) { @@ -824,54 +1251,50 @@ private void saveIterator(VirtualFrame frame, PPickler pickler, Object iterator, } itemConsumer.accept(item); if (opcode != PickleUtils.NO_OPCODE) { - write(pickler, opcode); + pickler.write(this, opcode); } } } private void saveListIterator(VirtualFrame frame, PPickler pickler, Object iterator) { saveIterator(frame, pickler, iterator, PickleUtils.OPCODE_APPEND, - (Object item) -> save(frame, pickler, item, 0)); - } - - private void saveFrozenSetIterator(VirtualFrame frame, PPickler pickler, Object iterator) { - saveIterator(frame, pickler, iterator, PickleUtils.NO_OPCODE, - (Object item) -> save(frame, pickler, item, 0)); + (Object item) -> saveListElement(frame, pickler, item)); } private void saveDictIterator(VirtualFrame frame, PPickler pickler, Object iterator) { saveIterator(frame, pickler, iterator, PickleUtils.OPCODE_SETITEM, (Object item) -> { - if (!isTuple(item) || length(frame, item) != 2) { + if (!isTuple(item) || getTupleStorage(item).length() != 2) { throw raise(TypeError, ErrorMessages.MUST_S_ITER_RETURN_2TUPLE, DICT_ITEMS); } - save(frame, pickler, getItem(frame, item, 0), 0); - save(frame, pickler, getItem(frame, item, 1), 0); + SequenceStorage storage = getTupleStorage(item); + saveDictKey(frame, pickler, getItem(storage, 0)); + saveDictValue(frame, pickler, getItem(storage, 1)); }); } // save methods - private boolean savePers(VirtualFrame frame, PPickler pickler, Object obj) { + private boolean savePers(VirtualFrame frame, PPickler pickler, int proto, Object obj) { Object pid; if (pickler.persFuncSelf == null) { - pid = getCallNode().execute(frame, pickler.persFunc, obj); + pid = ensureCallNode().execute(frame, pickler.persFunc, obj); } else { - pid = getCallNode().execute(frame, pickler.persFunc, pickler.persFuncSelf, obj); + pid = ensureCallNode().execute(frame, pickler.persFunc, pickler.persFuncSelf, obj); } if (pid != PNone.NONE) { - if (pickler.isBin()) { - save(frame, pickler, pid, 1); - write(pickler, PickleUtils.OPCODE_BINPERSID); + if (isBin(proto)) { + savePersistentId(frame, pickler, pid); + pickler.write(this, PickleUtils.OPCODE_BINPERSID); } else { final TruffleString pidStr = asString(convertToStr(frame, pid)); // XXX: Should it check whether the pid contains embedded newlines? if (!PythonUtils.isAscii(pidStr, ensureTsGetCodeRangeNode())) { throw raise(PythonBuiltinClassType.PicklingError, ErrorMessages.PIDS_MUST_BE_ASCII_STRS); } - write(pickler, PickleUtils.OPCODE_PERSID); - writeASCII(pickler, pidStr); - writeASCII(pickler, T_NEWLINE); + pickler.write(this, PickleUtils.OPCODE_PERSID); + pickler.writeASCII(this, pidStr, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); + pickler.writeASCII(this, T_NEWLINE, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); } return true; } @@ -879,79 +1302,52 @@ private boolean savePers(VirtualFrame frame, PPickler pickler, Object obj) { } // memo methods - private void memoGet(PPickler pickler, int value) { - int len; - byte[] pdata; - if (pickler.isBin()) { - pdata = new byte[5]; + private void memoGet(PPickler pickler, int proto, int value) { + if (isBin(proto)) { if (value < 256) { - pdata[0] = PickleUtils.OPCODE_BINGET; - pdata[1] = (byte) (value & 0xff); - len = 2; + pickler.writeByteOp(this, PickleUtils.OPCODE_BINGET, value); } else if (Long.compareUnsigned(value, 0xffffffffL) <= 0) { - pdata[0] = PickleUtils.OPCODE_LONG_BINGET; - pdata[1] = (byte) (value & 0xff); - pdata[2] = (byte) ((value >> 8) & 0xff); - pdata[3] = (byte) ((value >> 16) & 0xff); - pdata[4] = (byte) ((value >> 24) & 0xff); - len = 5; + pickler.writeIntOp(this, PickleUtils.OPCODE_LONG_BINGET, value); } else { // unlikely throw raise(PicklingError, ErrorMessages.MEMO_ID_TOO_LARGE_FOR_S, "LONG_BINGET"); } } else { - pdata = new byte[30]; - pdata[0] = PickleUtils.OPCODE_GET; - len = PickleUtils.toAsciiBytesWithNewLine(pdata, 1, value, ensureTsFromLongNode(), ensureTsCopyToByteArrayNode()); + pickler.writeLongAsString(this, PickleUtils.OPCODE_GET, value, ensureTsFromLongNode(), ensureTsCopyToByteArrayNode()); } - write(pickler, pdata, len); } - private void memoPut(PPickler pickler, Object obj) { - if (pickler.isFast()) { + private void memoPut(PPickler pickler, int proto, boolean fastMode, Object obj) { + if (fastMode) { return; } int idx = pickler.memo.size(); pickler.memo.set(obj, idx); - if (pickler.proto >= 4) { - write(pickler, PickleUtils.OPCODE_MEMOIZE); - return; + if (proto >= 4) { + pickler.write(this, PickleUtils.OPCODE_MEMOIZE); } else { - byte[] pdata; - int len; - if (!pickler.isBin()) { - pdata = new byte[30]; - pdata[0] = PickleUtils.OPCODE_PUT; - len = PickleUtils.toAsciiBytesWithNewLine(pdata, 1, idx, ensureTsFromLongNode(), ensureTsCopyToByteArrayNode()); + if (!isBin(proto)) { + pickler.writeLongAsString(this, PickleUtils.OPCODE_PUT, idx, ensureTsFromLongNode(), ensureTsCopyToByteArrayNode()); } else { - pdata = new byte[5]; if (idx < 256) { - pdata[0] = PickleUtils.OPCODE_BINPUT; - pdata[1] = (byte) idx; - len = 2; + pickler.writeByteOp(this, PickleUtils.OPCODE_BINPUT, idx); } else if (Long.compareUnsigned(idx, 0xffffffffL) <= 0) { - pdata[0] = PickleUtils.OPCODE_LONG_BINPUT; - pdata[1] = (byte) (idx & 0xff); - pdata[2] = (byte) ((idx >> 8) & 0xff); - pdata[3] = (byte) ((idx >> 16) & 0xff); - pdata[4] = (byte) ((idx >> 24) & 0xff); - len = 5; + pickler.writeIntOp(this, PickleUtils.OPCODE_LONG_BINPUT, idx); } else { // unlikely throw raise(PicklingError, ErrorMessages.MEMO_ID_TOO_LARGE_FOR_S, "LONG_BINPUT"); } } - write(pickler, pdata, len); } } // save methods - private void handleReduce(VirtualFrame frame, BoundaryCallData boundaryCallData, PythonContext ctx, PPickler pickler, Object obj, Object reduceValue) { + private void handleReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, int proto, boolean fastMode, Object obj, Object reduceValue) { if (PGuards.isString(reduceValue)) { - saveGlobal(frame, boundaryCallData, ctx, pickler, obj, reduceValue); + saveGlobal(frame, ctx, pickler, proto, fastMode, obj, reduceValue); return; } @@ -959,10 +1355,10 @@ private void handleReduce(VirtualFrame frame, BoundaryCallData boundaryCallData, throw raise(PicklingError, ErrorMessages.S_MUST_RETURN_S_OR_S, T___REDUCE__, "string", "tuple"); } - saveReduce(frame, ctx, pickler, reduceValue, obj); + saveReduce(frame, ctx, pickler, proto, fastMode, reduceValue, obj); } - private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, Object arguments, Object obj) { + private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, int proto, boolean fastMode, Object arguments, Object obj) { // We're saving obj, and args is the 2-thru-5 tuple returned by the appropriate // __reduce__ method for obj. @@ -971,17 +1367,18 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, if (!isTuple(arguments)) { throw raise(PythonBuiltinClassType.SystemError, ErrorMessages.BAD_ARG_TO_INTERNAL_FUNC); } - int size = length(frame, arguments); + SequenceStorage argumentsStorage = getTupleStorage(arguments); + int size = argumentsStorage.length(); if (size < 2 || size > 6) { throw raise(PicklingError, ErrorMessages.TUPLE_RET_BY_REDUCE_2_6); } - Object callable = getItem(frame, arguments, 0); - Object argtup = getItem(frame, arguments, 1); - Object state = getItem(frame, arguments, size, 2, null); - Object listitems = getItem(frame, arguments, size, 3, PNone.NONE); - Object dictitems = getItem(frame, arguments, size, 4, PNone.NONE); - Object stateSetter = getItem(frame, arguments, size, 5, PNone.NONE); + Object callable = getItem(argumentsStorage, 0); + Object argtup = getItem(argumentsStorage, 1); + Object state = size > 2 ? getItem(argumentsStorage, 2) : null; + Object listitems = size > 3 ? getItem(argumentsStorage, 3) : PNone.NONE; + Object dictitems = size > 4 ? getItem(argumentsStorage, 4) : PNone.NONE; + Object stateSetter = size > 5 ? getItem(argumentsStorage, 5) : PNone.NONE; if (!isCallable(callable)) { throw raise(PicklingError, ErrorMessages.S_ITEM_REDUCE_MUST_BE_S, "first", "callable"); @@ -1013,7 +1410,7 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, throw raise(PicklingError, ErrorMessages.S_ELEM_REDUCE_MUST_BE_S_NOT_P, "sixth", "a function", stateSetter); } - if (pickler.proto >= 2) { + if (proto >= 2) { Object name = lookupAttribute(frame, callable, T___NAME__); if (name != PNone.NO_VALUE) { final TruffleString stringName = asString(name); @@ -1026,9 +1423,11 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, } } - final int argtupSize = length(frame, argtup); + final SequenceStorage argtupStorage = getTupleStorage(argtup); + final int argtupSize = argtupStorage.length(); if (useNewobjEx) { + profileSeen(SEEN_NEWOBJ_EX); Object cls; Object args; Object kwargs; @@ -1037,29 +1436,30 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, throw raise(PicklingError, ErrorMessages.LEN_OF_S_MUST_BE_D_NOT_D, "NEWOBJ_EX", 3, argtupSize); } - cls = getItem(frame, argtup, 0); + cls = getItem(argtupStorage, 0); if (!isType(cls)) { throw raise(PicklingError, ErrorMessages.S_ITEM_FROM_S_MUST_BE_S_NOT_P, "first", "NEWOBJ_EX", "a class", cls); } - args = getItem(frame, argtup, 1); + args = getItem(argtupStorage, 1); if (!isTuple(args)) { throw raise(PicklingError, ErrorMessages.S_ITEM_FROM_S_MUST_BE_S_NOT_P, "second", "NEWOBJ_EX", "a tuple", args); } - kwargs = getItem(frame, argtup, 2); + kwargs = getItem(argtupStorage, 2); if (!(kwargs instanceof PDict)) { throw raise(PicklingError, ErrorMessages.S_ITEM_FROM_S_MUST_BE_S_NOT_P, "third", "NEWOBJ_EX", "a dict", kwargs); } - if (pickler.proto >= 4) { - save(frame, pickler, cls, 0); - save(frame, pickler, args, 0); - save(frame, pickler, kwargs, 0); - write(pickler, PickleUtils.OPCODE_NEWOBJ_EX); + if (proto >= 4) { + saveReduceClass(frame, pickler, cls); + saveReduceArgs(frame, pickler, args); + saveReduceKwargs(frame, pickler, kwargs); + pickler.write(this, PickleUtils.OPCODE_NEWOBJ_EX); } else { PickleState st = getGlobalState(ctx.getCore()); - final int argsSize = length(frame, args); + final SequenceStorage argsStorage = getTupleStorage(args); + final int argsSize = argsStorage.length(); Object[] newargs = new Object[argsSize + 2]; Object clsNew; int i; @@ -1068,16 +1468,17 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, newargs[0] = clsNew; newargs[1] = cls; for (i = 0; i < argsSize; i++) { - Object item = getItem(frame, args, i); + Object item = getItem(argsStorage, i); newargs[i + 2] = item; } callable = callStarArgsAndKwArgs(frame, st.partial, newargs, kwargs); - save(frame, pickler, callable, 0); - save(frame, pickler, PFactory.createEmptyTuple(PythonLanguage.get(this)), 0); - write(pickler, PickleUtils.OPCODE_REDUCE); + saveReduceCallable(frame, pickler, callable); + saveReduceArgs(frame, pickler, PFactory.createEmptyTuple(PythonLanguage.get(this))); + pickler.write(this, PickleUtils.OPCODE_REDUCE); } } else if (useNewobj) { + profileSeen(SEEN_NEWOBJ); Object cls; Object newargtup; Object objClass; @@ -1088,7 +1489,7 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, throw raise(PicklingError, ErrorMessages.IS_EMPTY, "__newobj__ arglist"); } - cls = getItem(frame, argtup, 0); + cls = getItem(argtupStorage, 0); if (!isType(cls)) { throw raise(PicklingError, ErrorMessages.ARGS_0_FROM_S_ARGS_S, "__newobj__", "is not a type"); } @@ -1129,15 +1530,20 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, // function. // Save the class and its __new__ arguments - save(frame, pickler, cls, 0); - newargtup = getItem(frame, argtup, PFactory.createIntSlice(PythonLanguage.get(this), 1, argtupSize, 1)); - save(frame, pickler, newargtup, 0); - write(pickler, PickleUtils.OPCODE_NEWOBJ); + saveReduceClass(frame, pickler, cls); + Object[] newargs = new Object[argtupSize - 1]; + for (int i = 1; i < argtupSize; i++) { + newargs[i - 1] = getItem(argtupStorage, i); + } + newargtup = createTuple(newargs); + saveReduceArgs(frame, pickler, newargtup); + pickler.write(this, PickleUtils.OPCODE_NEWOBJ); } else { + profileSeen(SEEN_PLAIN_REDUCE); // Not using NEWOBJ - save(frame, pickler, callable, 0); - save(frame, pickler, argtup, 0); - write(pickler, PickleUtils.OPCODE_REDUCE); + saveReduceCallable(frame, pickler, callable); + saveReduceArgs(frame, pickler, argtup); + pickler.write(this, PickleUtils.OPCODE_REDUCE); } // obj can be NULL when save_reduce() is used directly. A NULL obj means the caller do @@ -1149,25 +1555,28 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, // memo int memoIndex = pickler.memo.get(obj); if (memoIndex != -1) { - write(pickler, PickleUtils.OPCODE_POP); - memoGet(pickler, memoIndex); + pickler.write(this, PickleUtils.OPCODE_POP); + memoGet(pickler, proto, memoIndex); } else { - memoPut(pickler, obj); + memoPut(pickler, proto, fastMode, obj); } } if (listitems != null) { - batchList(frame, pickler, listitems); + profileSeen(SEEN_REDUCE_LISTITEMS); + batchList(frame, pickler, proto, listitems); } if (dictitems != null) { - batchDict(frame, pickler, dictitems); + profileSeen(SEEN_REDUCE_DICTITEMS); + batchDict(frame, pickler, proto, dictitems); } if (state != null) { + profileSeen(SEEN_REDUCE_STATE); if (stateSetter == null) { - save(frame, pickler, state, 0); - write(pickler, PickleUtils.OPCODE_BUILD); + saveReduceState(frame, pickler, state); + pickler.write(this, PickleUtils.OPCODE_BUILD); } else { // If a state_setter is specified, call it instead of load_build to update obj's // with its previous state. The first 4 save/write instructions push @@ -1176,151 +1585,124 @@ private void saveReduce(VirtualFrame frame, PythonContext ctx, PPickler pickler, // Finally, because state-updating routines only do in-place modification, the // whole operation has to be stack-transparent. Thus, we finally pop the call's // output from the stack. - save(frame, pickler, stateSetter, 0); - save(frame, pickler, obj, 0); - save(frame, pickler, state, 0); - write(pickler, PickleUtils.OPCODE_TUPLE2); - write(pickler, PickleUtils.OPCODE_REDUCE); - write(pickler, PickleUtils.OPCODE_POP); + saveReduceStateSetter(frame, pickler, stateSetter); + saveReduceObject(frame, pickler, obj); + saveReduceState(frame, pickler, state); + pickler.write(this, PickleUtils.OPCODE_TUPLE2); + pickler.write(this, PickleUtils.OPCODE_REDUCE); + pickler.write(this, PickleUtils.OPCODE_POP); } } } - private void saveNone(PPickler pickler, @SuppressWarnings("unused") Object obj) { - write(pickler, PickleUtils.OPCODE_NONE); + private void saveNone(PPickler pickler) { + pickler.write(this, PickleUtils.OPCODE_NONE); } - private void saveBool(VirtualFrame frame, PPickler pickler, Object obj) { - if (pickler.proto >= 2) { - write(pickler, isTrue(frame, obj) ? PickleUtils.OPCODE_NEWTRUE : PickleUtils.OPCODE_NEWFALSE); + private void saveBool(PPickler pickler, int proto, boolean value) { + if (proto >= 2) { + pickler.write(this, value ? PickleUtils.OPCODE_NEWTRUE : PickleUtils.OPCODE_NEWFALSE); } else { // These aren't opcodes -- they're ways to pickle bools before protocol 2 so that // unpicklers written before bools were introduced unpickle them as ints, but // unpicklers after can recognize that bools were intended. Note that protocol 2 // added direct ways to pickle bools. - writeASCII(pickler, isTrue(frame, obj) ? PickleUtils.T_PROTO_LE2_TRUE : PickleUtils.T_PROTO_LE2_FALSE); + pickler.writeASCII(this, value ? PickleUtils.T_PROTO_LE2_TRUE : PickleUtils.T_PROTO_LE2_FALSE, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); } } - private void saveLong(VirtualFrame frame, PPickler pickler, Object obj) { - TruffleString repr; - try { - long value = asLong(frame, obj); - if (value <= 0x7fffffffL && value >= (-0x7fffffffL - 1)) { - // result fits in a signed 4-byte integer. Note: we can't use -0x80000000L in - // the above condition because some compilers (e.g., MSVC) will promote - // 0x80000000L to an unsigned type before applying the unary minus when - // sizeof(long) <= 4. The resulting value stays unsigned which is commonly not - // what we want, so MSVC happily warns us about it. However, that result would - // have been fine because we guard for sizeof(long) <= 4 which turns the - // condition true in that particular case. - byte[] pdata; - int len; - - if (pickler.isBin()) { - pdata = new byte[5]; - pdata[1] = (byte) (value & 0xff); - pdata[2] = (byte) ((value >> 8) & 0xff); - pdata[3] = (byte) ((value >> 16) & 0xff); - pdata[4] = (byte) ((value >> 24) & 0xff); - - if ((pdata[4] != 0) || (pdata[3] != 0)) { - pdata[0] = PickleUtils.OPCODE_BININT; - len = 5; - } else if (pdata[2] != 0) { - pdata[0] = PickleUtils.OPCODE_BININT2; - len = 3; - } else { - pdata[0] = PickleUtils.OPCODE_BININT1; - len = 2; - } - } else { - pdata = new byte[32]; - pdata[0] = PickleUtils.OPCODE_INT; - len = PickleUtils.toAsciiBytesWithNewLine(pdata, 1, value, ensureTsFromLongNode(), ensureTsCopyToByteArrayNode()); - } - write(pickler, pdata, len); - return; + private void saveLong(PPickler pickler, int proto, int value) { + if (isBin(proto)) { + if (value >= 0 && value < (1 << 8)) { + pickler.writeByteOp(this, PickleUtils.OPCODE_BININT1, value); + } else if (value >= 0 && value < (1 << 16)) { + pickler.writeShortOp(this, PickleUtils.OPCODE_BININT2, value); + } else { + pickler.writeIntOp(this, PickleUtils.OPCODE_BININT, value); } - } catch (PException e) { - e.expectCached(OverflowError, ensureErrProfile()); + } else { + pickler.writeLongAsString(this, PickleUtils.OPCODE_INT, value, ensureTsFromLongNode(), ensureTsCopyToByteArrayNode()); } + } - if (pickler.proto >= 2) { - byte[] header = new byte[5]; - // Linear-time pickling. - final int sign = getSign(obj); - if (sign == 0) { - header[0] = PickleUtils.OPCODE_LONG1; - header[1] = 0; - write(pickler, header, 2); - } + private void saveLong(PPickler pickler, int proto, long value) { + int intValue = (int) value; + if (intValue == value) { + saveLong(pickler, proto, intValue); + } else { + saveLongLarge(pickler, proto, value); + } + } + + private void saveLong(VirtualFrame frame, PPickler pickler, int proto, PInt value) { + try { + saveLong(pickler, proto, value.longValueExact()); + } catch (OverflowException e) { + saveLongLarge(frame, pickler, proto, value); + } + } - final int nbits = getNumBits(obj); - // How many bytes do we need? There are nbits >> 3 full bytes of data, and nbits & 7 - // leftover bits. If there are any leftover bits, then we clearly need another byte. - // What's not so obvious is that we *probably* need another byte even if there - // aren't any leftovers: the most-significant bit of the most-significant byte acts - // like a sign bit, and it's usually got a sense opposite of the one we need. The - // exception is ints of the form -(2**(8*j-1)) for j > 0. Such an int is its own - // 256's-complement, so has the right sign bit even without the extra byte. That's a - // pain to check for in advance, though, so we always grab an extra byte at the - // start, and cut it back later if possible. + private void saveLongLarge(PPickler pickler, int proto, long value) { + if (proto >= 2) { + // Count the significant bits excluding the sign extension, then add a sign bit. + int nbits = Long.SIZE - Long.numberOfLeadingZeros(value < 0 ? ~value : value); int nbytes = (nbits >> 3) + 1; - if (Long.compareUnsigned(nbytes, 0x7fffffffL) > 0) { - throw raise(OverflowError, ErrorMessages.S_TO_LARGE_TO_PICKLE, "int"); - } - byte[] pdata = longAsBytes(obj, nbytes, false); - // If the int is negative, this may be a byte more than needed. This is so iff the - // MSB is all redundant sign bits. - if (sign < 0 && nbytes > 1 && pdata[nbytes - 1] == (byte) 0xff && (pdata[nbytes - 2] & 0x80) != 0) { - nbytes--; + pickler.writeByteOp(this, PickleUtils.OPCODE_LONG1, nbytes); + pickler.ensureBufferSpace(this, nbytes); + for (int i = 0; i < nbytes; i++) { + pickler.outputBuffer[pickler.outputLen++] = (byte) value; + value >>= Byte.SIZE; } - int size; + } else { + TruffleString repr = ensureTsFromLongNode().execute(value, TS_ENCODING, true); + pickler.write(this, PickleUtils.OPCODE_LONG); + pickler.writeASCII(this, repr, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); + pickler.writeASCII(this, T_L_NEW_LINE, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); + } + } + + private void saveLongLarge(VirtualFrame frame, PPickler pickler, int proto, PInt value) { + if (proto >= 2) { + // BigInteger already provides the minimal signed two's-complement representation. + byte[] data = value.toByteArray(); + int nbytes = data.length; if (nbytes < 256) { - header[0] = PickleUtils.OPCODE_LONG1; - header[1] = (byte) nbytes; - size = 2; + pickler.writeByteOp(this, PickleUtils.OPCODE_LONG1, nbytes); } else { - header[0] = PickleUtils.OPCODE_LONG4; - size = nbytes; - for (int i = 1; i < 5; i++) { - header[i] = (byte) (size & 0xff); - size >>= 8; - } - size = 5; + pickler.writeIntOp(this, PickleUtils.OPCODE_LONG4, nbytes); + } + pickler.ensureBufferSpace(this, nbytes); + // BigInteger's bytes are big endian; pickle requires little endian. + for (int i = nbytes - 1; i >= 0; i--) { + pickler.outputBuffer[pickler.outputLen++] = data[i]; } - - write(pickler, header, size); - write(pickler, pdata, nbytes); - } else { - repr = repr(frame, obj); - write(pickler, PickleUtils.OPCODE_LONG); - writeASCII(pickler, asStringStrict(repr)); - writeASCII(pickler, T_L_NEW_LINE); + TruffleString repr = repr(frame, value); + pickler.write(this, PickleUtils.OPCODE_LONG); + pickler.writeASCII(this, repr, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); + pickler.writeASCII(this, T_L_NEW_LINE, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); } } - private void saveFloat(VirtualFrame frame, PPickler pickler, Object obj, Node inliningTarget, PyFloatAsDoubleNode asDoubleNode) { - final double value = asDoubleNode.execute(frame, inliningTarget, obj); - if (pickler.isBin()) { - byte[] pdata = new byte[9]; - pdata[0] = PickleUtils.OPCODE_BINFLOAT; - NumericSupport.bigEndian().putDouble(pdata, 1, value); - write(pickler, pdata, 9); + private void saveFloat(PPickler pickler, int proto, double value) { + if (isBin(proto)) { + pickler.write(this, PickleUtils.OPCODE_BINFLOAT); + pickler.ensureBufferSpace(this, 8); + NumericSupport.bigEndian().putDouble(pickler.outputBuffer, pickler.outputLen, value); + pickler.outputLen += 8; } else { - write(pickler, PickleUtils.OPCODE_FLOAT); + pickler.write(this, PickleUtils.OPCODE_FLOAT); TruffleString repr = PickleUtils.doubleToAsciiString(value); - writeASCII(pickler, repr); - writeASCII(pickler, T_NEWLINE); + pickler.writeASCII(this, repr, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); + pickler.writeASCII(this, T_NEWLINE, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); } } - private void saveBytes(VirtualFrame frame, PythonContext ctx, PPickler pickler, Object obj, InteropCallData interopCallData) { - Object buffer = getBufferAcquireLibrary().acquireReadonly(obj, frame, interopCallData); + private void saveBytes(VirtualFrame frame, PythonContext ctx, PPickler pickler, int proto, boolean fastMode, PBytes obj) { + Object buffer = getBufferAcquireLibrary().acquireReadonly(obj); + PythonBufferAccessLibrary bufferLib = getBufferLibrary(); try { - if (pickler.proto < 3) { + if (proto < 3) { // Older pickle protocols do not have an opcode for pickling bytes objects. // Therefore, we need to fake the copy protocol (i.e., the __reduce__ method) to // permit bytes object unpickling. Here we use a hack to be compatible with @@ -1333,105 +1715,95 @@ private void saveBytes(VirtualFrame frame, PythonContext ctx, PPickler pickler, // need to do this with newer protocols. Object reduceValue; - if (getBufferLibrary().getBufferLength(buffer) == 0) { + if (bufferLib.getBufferLength(buffer) == 0) { reduceValue = createTuple(ctx.getCore().lookupType(PythonBuiltinClassType.PBytes), createTuple()); } else { PickleState st = getGlobalState(ctx.getCore()); - final TruffleString unicodeStr = PickleUtils.decodeLatin1Strict(getBufferLibrary().getCopiedByteArray(buffer), ensureTsFromByteArrayWithCompaction()); + final TruffleString unicodeStr = PickleUtils.decodeLatin1Strict(bufferLib.getCopiedByteArray(buffer), ensureTsFromByteArrayWithCompaction()); reduceValue = createTuple(st.codecsEncode, createTuple(unicodeStr, LATIN1)); } // save_reduce() will memoize the object automatically. - saveReduce(frame, ctx, pickler, reduceValue, obj); + saveReduce(frame, ctx, pickler, proto, fastMode, reduceValue, obj); } else { - byte[] bytes = getBufferLibrary().getCopiedByteArray(buffer); - saveBytesData(frame, pickler, obj, bytes, bytes.length); + saveBytesData(frame, pickler, proto, fastMode, obj, buffer, bufferLib); } } finally { - getBufferLibrary().release(buffer, frame, interopCallData); + bufferLib.release(buffer); } } - private void saveBytesData(VirtualFrame frame, PPickler pickler, Object obj, byte[] data, int size) { - assert pickler.proto >= 3; - byte[] header = new byte[9]; - int len; + private void saveBytesData(VirtualFrame frame, PPickler pickler, int proto, boolean fastMode, Object obj, Object buffer, PythonBufferAccessLibrary bufferLib) { + assert proto >= 3; + int size = bufferLib.getBufferLength(buffer); + boolean wasFraming = pickler.framing; + boolean bypassBuffer = bypassBuffer(pickler, size); if (size <= 0xff) { - header[0] = PickleUtils.OPCODE_SHORT_BINBYTES; - header[1] = (byte) size; - len = 2; - } else if (Long.compareUnsigned(size, 0xffffffffL) < 0) { - header[0] = PickleUtils.OPCODE_BINBYTES; - header[1] = (byte) (size & 0xff); - header[2] = (byte) ((size >> 8) & 0xff); - header[3] = (byte) ((size >> 16) & 0xff); - header[4] = (byte) ((size >> 24) & 0xff); - len = 5; - } else if (pickler.proto >= 4) { - header[0] = PickleUtils.OPCODE_BINBYTES8; - PickleUtils.writeSize64(header, 1, size); - len = 9; + pickler.writeByteOp(this, PickleUtils.OPCODE_SHORT_BINBYTES, size); } else { - throw raise(OverflowError, ErrorMessages.SER_OVER_4GB); - } - - writeBytes(frame, pickler, header, len, data, size, obj); - memoPut(pickler, obj); - } - - private void writeUnicodeBinary(VirtualFrame frame, PPickler pickler, Object obj) { - Object encoded = null; - byte[] header = new byte[9]; - int len; - byte[] data = PickleUtils.encodeUTF8Strict(asStringStrict(obj), ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode(), ensureTsGetCodeRangeNode()); - - if (data == null) { - // Issue #8383: for strings with lone surrogates, fallback on the "surrogatepass" - // error handler. - encoded = getItem(frame, encode(frame, obj, T_UTF8, T_ERRORS_SURROGATEPASS), 0); - // Checkstyle: stop - //@formatter:off - // data = PickleUtils.encodeUTF8Strict(asStringStrict(encoded), ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode(), ensureTsGetCodeRangeNode()); - // Checkstyle: start - //@formatter:on - // TODO: [GR-39571] TruffleStrings: allow preservation of UTF-16 surrogate - data = toBytes(frame, encoded); - } - - int size = data.length; - - if (size <= 0xff && pickler.proto >= 4) { - header[0] = PickleUtils.OPCODE_SHORT_BINUNICODE; - header[1] = (byte) (size & 0xff); - len = 2; - } else if (Long.compareUnsigned(size, 0xffffffffL) <= 0) { - header[0] = PickleUtils.OPCODE_BINUNICODE; - header[1] = (byte) (size & 0xff); - header[2] = (byte) ((size >> 8) & 0xff); - header[3] = (byte) ((size >> 16) & 0xff); - header[4] = (byte) ((size >> 24) & 0xff); - len = 5; - } else if (pickler.proto >= 4) { - header[0] = PickleUtils.OPCODE_BINUNICODE8; - PickleUtils.writeSize64(header, 1, size); - len = 9; + pickler.writeIntOp(this, PickleUtils.OPCODE_BINBYTES, size); + } + + if (bypassBuffer && pickler.write != null) { + flushToFile(frame, pickler); + ensureCallNode().execute(frame, pickler.write, obj); + pickler.clearBuffer(); + } else { + pickler.writeBuffer(this, buffer, size, bufferLib); + } + + if (bypassBuffer && wasFraming) { + pickler.framing = true; + } + memoPut(pickler, proto, fastMode, obj); + } + + private void writeUnicodeBinary(Node node, VirtualFrame frame, PPickler pickler, int proto, TruffleString string, + TruffleString.SwitchEncodingNode switchEncodingNode, TruffleString.CopyToByteArrayNode copyToByteArrayNode, + FlushToFileNode flushToFileNode, CallNode callNode) { + TruffleString utf8 = switchEncodingNode.execute(string, TruffleString.Encoding.UTF_8, TranscodingErrorHandler.DEFAULT_KEEP_SURROGATES_IN_UTF8); + int size = utf8.byteLength(TruffleString.Encoding.UTF_8); + boolean wasFraming = pickler.framing; + boolean bypassBuffer = bypassBuffer(pickler, size); + + if (bypassBuffer && pickler.write != null) { + pickler.writeIntOp(node, PickleUtils.OPCODE_BINUNICODE, size); + byte[] data = new byte[size]; + copyToByteArrayNode.execute(utf8, 0, data, 0, size, TruffleString.Encoding.UTF_8); + flushToFileNode.execute(frame, pickler); + callNode.execute(frame, pickler.write, PFactory.createBytes(PythonLanguage.get(node), data, size)); + pickler.clearBuffer(); } else { - throw raise(OverflowError, ErrorMessages.SER_OVER_4GB); + if (size <= 0xff && proto >= 4) { + pickler.writeByteOp(node, PickleUtils.OPCODE_SHORT_BINUNICODE, size); + } else { + pickler.writeIntOp(node, PickleUtils.OPCODE_BINUNICODE, size); + } + pickler.writeUtf8Converted(node, utf8, copyToByteArrayNode); + } + + if (bypassBuffer && wasFraming) { + pickler.framing = true; } + } + + private void saveUnicode(VirtualFrame frame, PPickler pickler, int proto, boolean fastMode, TruffleString obj) { + saveUnicode(frame, pickler, proto, fastMode, obj, obj); + } - writeBytes(frame, pickler, header, len, data, size, encoded); + private void saveUnicode(VirtualFrame frame, PPickler pickler, int proto, boolean fastMode, PString obj) { + saveUnicode(frame, pickler, proto, fastMode, obj, asStringStrict(obj)); } - private void saveUnicode(VirtualFrame frame, PPickler pickler, Object obj) { - if (pickler.isBin()) { - writeUnicodeBinary(frame, pickler, obj); + private void saveUnicode(VirtualFrame frame, PPickler pickler, int proto, boolean fastMode, Object obj, TruffleString string) { + if (isBin(proto)) { + writeUnicodeBinary(this, frame, pickler, proto, string, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode(), ensureFlushToFileNode(), ensureCallNode()); } else { - byte[] encoded = PickleUtils.rawUnicodeEscape(asStringStrict(obj), ensureTsCodePointLengthNode(), ensureTsCodePointAtIndexUTF32Node()); - write(pickler, PickleUtils.OPCODE_UNICODE); - write(pickler, encoded); - writeASCII(pickler, T_NEWLINE); + pickler.write(this, PickleUtils.OPCODE_UNICODE); + pickler.writeRawUnicodeEscape(this, string, ensureTsCodePointLengthNode(), ensureTsCodePointAtIndexUTF32Node()); + pickler.writeASCII(this, T_NEWLINE, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); } - memoPut(pickler, obj); + memoPut(pickler, proto, fastMode, obj); } private void batchDictExact(VirtualFrame frame, PPickler pickler, PDict dict) { @@ -1445,18 +1817,18 @@ private void batchDictExact(VirtualFrame frame, PPickler pickler, PDict dict) { } HashingStorageIteratorKey getKeyNode = ensureHashingStorageIteratorKey(); HashingStorageIteratorValue getValueNode = ensureHashingStorageIteratorValue(); - save(frame, pickler, getKeyNode.executeCached(storage, it), 0); - save(frame, pickler, getValueNode.executeCached(storage, it), 0); - write(pickler, PickleUtils.OPCODE_SETITEM); + saveDictKey(frame, pickler, getKeyNode.executeCached(storage, it)); + saveDictValue(frame, pickler, getValueNode.executeCached(storage, it)); + pickler.write(this, PickleUtils.OPCODE_SETITEM); } else { // Write in batches of BATCHSIZE. saveDictHashingStorageBatched(frame, pickler, storage); } } - private void batchDict(VirtualFrame frame, PPickler pickler, Object iterator) { + private void batchDict(VirtualFrame frame, PPickler pickler, int proto, Object iterator) { assert iterator != null; - if (pickler.proto == 0) { + if (proto == 0) { // SETITEMS isn't available; do one at a time. saveDictIterator(frame, pickler, iterator); } else { @@ -1465,138 +1837,108 @@ private void batchDict(VirtualFrame frame, PPickler pickler, Object iterator) { } } - private void saveDict(VirtualFrame frame, Node inliningTarget, PyObjectCallMethodObjArgs callMethod, PPickler pickler, Object obj) { - byte[] header = new byte[3]; - int len; - - if (pickler.isFast()) { + private void saveDict(VirtualFrame frame, Node inliningTarget, PyObjectCallMethodObjArgs callMethod, PPickler pickler, int proto, boolean fastMode, PDict obj) { + if (fastMode) { fastSaveEnter(pickler, obj); } // Create an empty dict. - if (pickler.isBin()) { - header[0] = PickleUtils.OPCODE_EMPTY_DICT; - len = 1; + if (isBin(proto)) { + pickler.write(this, PickleUtils.OPCODE_EMPTY_DICT); } else { - header[0] = PickleUtils.OPCODE_MARK; - header[1] = PickleUtils.OPCODE_DICT; - len = 2; + pickler.write(this, PickleUtils.OPCODE_MARK, PickleUtils.OPCODE_DICT); } - write(pickler, header, len); - memoPut(pickler, obj); + memoPut(pickler, proto, fastMode, obj); - if (length(frame, obj) > 0) { - if (PGuards.isDict(obj) && pickler.proto > 0) { - batchDictExact(frame, pickler, (PDict) obj); + if (getHashingStorageLength(obj.getDictStorage()) > 0) { + if (proto > 0) { + batchDictExact(frame, pickler, obj); } else { final Object items = callMethod.execute(frame, inliningTarget, obj, T_ITEMS); - batchDict(frame, pickler, getIter(frame, items)); + batchDict(frame, pickler, proto, getIter(frame, items)); } } - if (pickler.isFast()) { + if (fastMode) { fastSaveLeave(pickler, obj); } } - private void batchSetExact(VirtualFrame frame, PPickler pickler, PSet set) { - saveSetHashingStorageBatched(frame, pickler, set.getDictStorage()); - } - - private void batchSet(VirtualFrame frame, PPickler pickler, Object obj) { - Object iterator = getIter(frame, obj); - int setSize = length(frame, obj); - int i; - do { - i = 0; - write(pickler, PickleUtils.OPCODE_MARK); - while (true) { - Object item; - try { - item = getNextItem(frame, iterator); - } catch (IteratorExhausted e) { - break; - } - save(frame, pickler, item, 0); - if (++i == PickleUtils.BATCHSIZE) { - break; - } - } - write(pickler, PickleUtils.OPCODE_ADDITEMS); - if (length(frame, obj) != setSize) { - throw raise(PythonBuiltinClassType.RuntimeError, ErrorMessages.CHANGED_SIZE_DURING_ITERATION, "set"); - } - } while (i == PickleUtils.BATCHSIZE); - } - - private void saveSet(VirtualFrame frame, PythonContext ctx, PPickler pickler, Object obj) { + private void saveSet(VirtualFrame frame, PythonContext ctx, PPickler pickler, int proto, boolean fastMode, PSet obj) { Object reduceValue; - if (pickler.proto < 4) { + if (proto < 4) { Object items = createList(frame, obj); reduceValue = createTuple(ctx.getCore().lookupType(PythonBuiltinClassType.PSet), createTuple(items)); - saveReduce(frame, ctx, pickler, reduceValue, obj); + saveReduce(frame, ctx, pickler, proto, fastMode, reduceValue, obj); return; } - write(pickler, PickleUtils.OPCODE_EMPTY_SET); - memoPut(pickler, obj); + pickler.write(this, PickleUtils.OPCODE_EMPTY_SET); + memoPut(pickler, proto, fastMode, obj); - final int setSize = length(frame, obj); - if (setSize == 0) { + HashingStorage storage = obj.getDictStorage(); + if (getHashingStorageLength(storage) == 0) { // nothing to do return; } // Write in batches of BATCHSIZE. - if (obj instanceof PSet set) { - batchSetExact(frame, pickler, set); - } else { - batchSet(frame, pickler, obj); - } + saveSetHashingStorageBatched(frame, pickler, storage); } - private void saveFrozenset(VirtualFrame frame, PythonContext ctx, PPickler pickler, Object obj) { - if (pickler.isFast()) { + private void saveFrozenset(VirtualFrame frame, PythonContext ctx, PPickler pickler, int proto, boolean fastMode, PFrozenSet obj) { + if (fastMode) { fastSaveEnter(pickler, obj); } - if (pickler.proto < 4) { + if (proto < 4) { Object items = createList(frame, obj); Object reduceValue = createTuple(ctx.getCore().lookupType(PythonBuiltinClassType.PFrozenSet), createTuple(items)); - saveReduce(frame, ctx, pickler, reduceValue, obj); + saveReduce(frame, ctx, pickler, proto, fastMode, reduceValue, obj); return; } - write(pickler, PickleUtils.OPCODE_MARK); + pickler.write(this, PickleUtils.OPCODE_MARK); - if (obj instanceof PFrozenSet set) { - saveSetHashingStorage(frame, pickler, set.getDictStorage()); - } else { - final Object iterator = getIter(frame, obj); - saveFrozenSetIterator(frame, pickler, iterator); - } + saveSetHashingStorage(frame, pickler, obj.getDictStorage()); // If the object is already in the memo, this means it is recursive. In this case, throw // away everything we put on the stack, and fetch the object back from the memo. int memoIndex = pickler.memo.get(obj); if (memoIndex != -1) { - write(pickler, PickleUtils.OPCODE_POP_MARK); - memoGet(pickler, memoIndex); + pickler.write(this, PickleUtils.OPCODE_POP_MARK); + memoGet(pickler, proto, memoIndex); return; } - write(pickler, PickleUtils.OPCODE_FROZENSET); - memoPut(pickler, obj); + pickler.write(this, PickleUtils.OPCODE_FROZENSET); + memoPut(pickler, proto, fastMode, obj); } - private void batchListExact(VirtualFrame frame, PPickler pickler, Object obj) { - final SequenceStorage storage = getSequenceStorage(obj); + private void batchListExact(VirtualFrame frame, PPickler pickler, PList obj) { + SequenceStorage storage = obj.getSequenceStorage(); + if (pickler.persFunc == null) { + if (storage instanceof IntSequenceStorage intStorage) { + profileSeen(SEEN_INT_STORAGE); + batchIntList(frame, pickler, intStorage); + return; + } else if (storage instanceof LongSequenceStorage longStorage) { + profileSeen(SEEN_LONG_STORAGE); + batchLongList(frame, pickler, longStorage); + return; + } else if (storage instanceof DoubleSequenceStorage doubleStorage) { + profileSeen(SEEN_DOUBLE_STORAGE); + batchDoubleList(frame, pickler, doubleStorage); + return; + } + } + profileSeen(SEEN_GENERIC_STORAGE); Object item; - if (length(frame, obj) == 1) { - item = getItem(frame, storage, 0); - save(frame, pickler, item, 0); - write(pickler, PickleUtils.OPCODE_APPEND); + if (storage.length() == 1) { + item = getItem(storage, 0); + saveListElement(frame, pickler, item); + pickler.write(this, PickleUtils.OPCODE_APPEND); return; } @@ -1605,22 +1947,118 @@ private void batchListExact(VirtualFrame frame, PPickler pickler, Object obj) { int thisBatch; do { thisBatch = 0; - write(pickler, PickleUtils.OPCODE_MARK); - while (total < length(frame, obj)) { - item = getItem(frame, storage, total); - save(frame, pickler, item, 0); + pickler.write(this, PickleUtils.OPCODE_MARK); + while (total < storage.length()) { + item = getItem(storage, total); + saveListElement(frame, pickler, item); total++; if (++thisBatch == PickleUtils.BATCHSIZE) { break; } } - write(pickler, PickleUtils.OPCODE_APPENDS); - } while (total < length(frame, obj)); + pickler.write(this, PickleUtils.OPCODE_APPENDS); + // It could have been mutated as a side effect of the reduce calls or the file write + storage = obj.getSequenceStorage(); + } while (total < storage.length()); + LoopNode.reportLoopCount(this, total); + } + + private void batchIntList(VirtualFrame frame, PPickler pickler, IntSequenceStorage storage) { + opcodeBoundary(frame, pickler); + int proto = pickler.getProto(); + int length = storage.length(); + + if (length == 1) { + saveLong(pickler, proto, storage.getIntItemNormalized(0)); + pickler.write(this, PickleUtils.OPCODE_APPEND); + return; + } + + for (int batchStart = 0; batchStart < length; batchStart += PickleUtils.BATCHSIZE) { + int batchEnd = batchStart + Math.min(PickleUtils.BATCHSIZE, length - batchStart); + pickler.write(this, PickleUtils.OPCODE_MARK); + for (int i = batchStart; i < batchEnd; i++) { + boolean didFlush = opcodeBoundary(frame, pickler); + // To make sure we don't go out-of-bounds if the file write mutated the list + if (CompilerDirectives.injectBranchProbability(CompilerDirectives.SLOWPATH_PROBABILITY, didFlush && length != storage.length())) { + length = storage.length(); + batchEnd = batchStart + Math.min(PickleUtils.BATCHSIZE, length - batchStart); + if (i >= batchEnd) { + break; + } + } + saveLong(pickler, proto, storage.getIntItemNormalized(i)); + } + pickler.write(this, PickleUtils.OPCODE_APPENDS); + } + LoopNode.reportLoopCount(this, length); + } + + private void batchLongList(VirtualFrame frame, PPickler pickler, LongSequenceStorage storage) { + opcodeBoundary(frame, pickler); + int proto = pickler.getProto(); + int length = storage.length(); + + if (length == 1) { + saveLong(pickler, proto, storage.getLongItemNormalized(0)); + pickler.write(this, PickleUtils.OPCODE_APPEND); + return; + } + + for (int batchStart = 0; batchStart < length; batchStart += PickleUtils.BATCHSIZE) { + int batchEnd = batchStart + Math.min(PickleUtils.BATCHSIZE, length - batchStart); + pickler.write(this, PickleUtils.OPCODE_MARK); + for (int i = batchStart; i < batchEnd; i++) { + boolean didFlush = opcodeBoundary(frame, pickler); + // To make sure we don't go out-of-bounds if the file write mutated the list + if (CompilerDirectives.injectBranchProbability(CompilerDirectives.SLOWPATH_PROBABILITY, didFlush && length != storage.length())) { + length = storage.length(); + batchEnd = batchStart + Math.min(PickleUtils.BATCHSIZE, length - batchStart); + if (i >= batchEnd) { + break; + } + } + saveLong(pickler, proto, storage.getLongItemNormalized(i)); + } + pickler.write(this, PickleUtils.OPCODE_APPENDS); + } + LoopNode.reportLoopCount(this, length); } - private void batchList(VirtualFrame frame, PPickler pickler, Object iterator) { + private void batchDoubleList(VirtualFrame frame, PPickler pickler, DoubleSequenceStorage storage) { + opcodeBoundary(frame, pickler); + int proto = pickler.getProto(); + int length = storage.length(); + + if (length == 1) { + saveFloat(pickler, proto, storage.getDoubleItemNormalized(0)); + pickler.write(this, PickleUtils.OPCODE_APPEND); + return; + } + + for (int batchStart = 0; batchStart < length; batchStart += PickleUtils.BATCHSIZE) { + int batchEnd = batchStart + Math.min(PickleUtils.BATCHSIZE, length - batchStart); + pickler.write(this, PickleUtils.OPCODE_MARK); + for (int i = batchStart; i < batchEnd; i++) { + boolean didFlush = opcodeBoundary(frame, pickler); + // To make sure we don't go out-of-bounds if the file write mutated the list + if (CompilerDirectives.injectBranchProbability(CompilerDirectives.SLOWPATH_PROBABILITY, didFlush && length != storage.length())) { + length = storage.length(); + batchEnd = batchStart + Math.min(PickleUtils.BATCHSIZE, length - batchStart); + if (i >= batchEnd) { + break; + } + } + saveFloat(pickler, proto, storage.getDoubleItemNormalized(i)); + } + pickler.write(this, PickleUtils.OPCODE_APPENDS); + } + LoopNode.reportLoopCount(this, length); + } + + private void batchList(VirtualFrame frame, PPickler pickler, int proto, Object iterator) { assert iterator != null; - if (pickler.proto == 0) { + if (proto == 0) { // APPENDS isn't available; do one at a time. saveListIterator(frame, pickler, iterator); } else { @@ -1629,175 +2067,222 @@ private void batchList(VirtualFrame frame, PPickler pickler, Object iterator) { } } - private void saveList(VirtualFrame frame, PPickler pickler, Object obj) { - byte[] header = new byte[3]; - int len; - - if (pickler.isFast()) { + private void saveList(VirtualFrame frame, PPickler pickler, int proto, boolean fastMode, PList obj) { + if (fastMode) { fastSaveEnter(pickler, obj); } // Create an empty list. - if (pickler.isBin()) { - header[0] = PickleUtils.OPCODE_EMPTY_LIST; - len = 1; + if (isBin(proto)) { + pickler.write(this, PickleUtils.OPCODE_EMPTY_LIST); } else { - header[0] = PickleUtils.OPCODE_MARK; - header[1] = PickleUtils.OPCODE_LIST; - len = 2; + pickler.write(this, PickleUtils.OPCODE_MARK, PickleUtils.OPCODE_LIST); } - write(pickler, header, len); - len = length(frame, obj); - memoPut(pickler, obj); + int len = obj.getSequenceStorage().length(); + memoPut(pickler, proto, fastMode, obj); if (len != 0) { - // Materialize the list elements. - if (PGuards.isList(obj) && pickler.proto > 0) { + if (proto > 0) { batchListExact(frame, pickler, obj); } else { - batchList(frame, pickler, getIter(frame, obj)); + batchList(frame, pickler, proto, getIter(frame, obj)); } } - if (pickler.isFast()) { + if (fastMode) { fastSaveLeave(pickler, obj); } } - private void storeTupleElements(VirtualFrame frame, PPickler pickler, Object obj, int len) { + private void storeTupleElements(VirtualFrame frame, PPickler pickler, SequenceStorage storage, int len) { // A helper for save_tuple. Push the len elements in tuple t on the stack - assert PyObjectSizeNode.executeUncached(obj) == len; + assert storage.length() == len; + if (pickler.persFunc == null) { + if (storage instanceof IntSequenceStorage intStorage) { + profileSeen(SEEN_INT_STORAGE); + storeIntTupleElements(frame, pickler, intStorage); + return; + } else if (storage instanceof LongSequenceStorage longStorage) { + profileSeen(SEEN_LONG_STORAGE); + storeLongTupleElements(frame, pickler, longStorage); + return; + } else if (storage instanceof DoubleSequenceStorage doubleStorage) { + profileSeen(SEEN_DOUBLE_STORAGE); + storeDoubleTupleElements(frame, pickler, doubleStorage); + return; + } + } + profileSeen(SEEN_GENERIC_STORAGE); for (int i = 0; i < len; i++) { - Object element = getItem(frame, obj, i); - save(frame, pickler, element, 0); + saveTupleElement(frame, pickler, getItem(storage, i)); } + LoopNode.reportLoopCount(this, len); } - private void saveTuple(VirtualFrame frame, PPickler pickler, Object obj) { - int len = length(frame, obj); + private void storeIntTupleElements(VirtualFrame frame, PPickler pickler, IntSequenceStorage storage) { + int proto = pickler.getProto(); + for (int i = 0; i < storage.length(); i++) { + opcodeBoundary(frame, pickler); + saveLong(pickler, proto, storage.getIntItemNormalized(i)); + } + LoopNode.reportLoopCount(this, storage.length()); + } - if (len == 0) { - byte[] pdata = new byte[2]; + private void storeLongTupleElements(VirtualFrame frame, PPickler pickler, LongSequenceStorage storage) { + int proto = pickler.getProto(); + for (int i = 0; i < storage.length(); i++) { + opcodeBoundary(frame, pickler); + saveLong(pickler, proto, storage.getLongItemNormalized(i)); + } + LoopNode.reportLoopCount(this, storage.length()); + } + + private void storeDoubleTupleElements(VirtualFrame frame, PPickler pickler, DoubleSequenceStorage storage) { + int proto = pickler.getProto(); + for (int i = 0; i < storage.length(); i++) { + opcodeBoundary(frame, pickler); + saveFloat(pickler, proto, storage.getDoubleItemNormalized(i)); + } + LoopNode.reportLoopCount(this, storage.length()); + } - if (pickler.proto != 0) { - pdata[0] = PickleUtils.OPCODE_EMPTY_TUPLE; - len = 1; + private void saveTuple(VirtualFrame frame, PPickler pickler, int proto, boolean fastMode, PTuple obj) { + SequenceStorage storage = obj.getSequenceStorage(); + saveTuple(frame, pickler, proto, fastMode, obj, storage, storage.length()); + } + + private void saveNativeTuple(VirtualFrame frame, PPickler pickler, int proto, boolean fastMode, Object obj) { + SequenceStorage storage = getTupleStorage(obj); + saveTuple(frame, pickler, proto, fastMode, obj, storage, storage.length()); + } + + private void saveTuple(VirtualFrame frame, PPickler pickler, int proto, boolean fastMode, Object obj, SequenceStorage storage, int len) { + + if (len == 0) { + if (proto != 0) { + pickler.write(this, PickleUtils.OPCODE_EMPTY_TUPLE); } else { - pdata[0] = PickleUtils.OPCODE_MARK; - pdata[1] = PickleUtils.OPCODE_TUPLE; - len = 2; + pickler.write(this, PickleUtils.OPCODE_MARK, PickleUtils.OPCODE_TUPLE); } - - write(pickler, pdata, len); return; } // The tuple isn't in the memo now. If it shows up there after saving the tuple // elements, the tuple must be recursive, in which case we'll pop everything we put on // the stack, and fetch its value from the memo. - if (len <= 3 && pickler.proto >= 2) { + if (len <= 3 && proto >= 2) { // Use TUPLE{1,2,3} opcodes. - storeTupleElements(frame, pickler, obj, len); + storeTupleElements(frame, pickler, storage, len); int memoIndex = pickler.memo.get(obj); if (memoIndex != -1) { // pop the len elements for (int i = 0; i < len; i++) { - write(pickler, PickleUtils.OPCODE_POP); + pickler.write(this, PickleUtils.OPCODE_POP); } // fetch from memo - memoGet(pickler, memoIndex); + memoGet(pickler, proto, memoIndex); return; } else { // Not recursive. - write(pickler, TUPLE_LEN_2_OPCODE[len]); + pickler.write(this, TUPLE_LEN_2_OPCODE[len]); } - memoPut(pickler, obj); + memoPut(pickler, proto, fastMode, obj); return; } // proto < 2 and len > 0, or proto >= 2 and len > 3. Generate MARK e1 e2 ... TUPLE - write(pickler, PickleUtils.OPCODE_MARK); - storeTupleElements(frame, pickler, obj, len); + pickler.write(this, PickleUtils.OPCODE_MARK); + storeTupleElements(frame, pickler, storage, len); int memoIndex = pickler.memo.get(obj); if (memoIndex != -1) { // pop the stack stuff we pushed - if (pickler.isBin()) { - write(pickler, PickleUtils.OPCODE_POP_MARK); + if (isBin(proto)) { + pickler.write(this, PickleUtils.OPCODE_POP_MARK); } else { // Note that we pop one more than len, to remove the MARK too. for (int i = 0; i <= len; i++) { - write(pickler, PickleUtils.OPCODE_POP); + pickler.write(this, PickleUtils.OPCODE_POP); } } // fetch from memo - memoGet(pickler, memoIndex); + memoGet(pickler, proto, memoIndex); return; } else { // Not recursive. - write(pickler, PickleUtils.OPCODE_TUPLE); + pickler.write(this, PickleUtils.OPCODE_TUPLE); } - memoPut(pickler, obj); + memoPut(pickler, proto, fastMode, obj); } - private void saveBytearrayData(VirtualFrame frame, PPickler pickler, Object obj, byte[] data, int size) { - assert pickler.proto >= 5; - if (size < 0) { - return; - } + private void saveBytearrayData(VirtualFrame frame, PPickler pickler, int proto, boolean fastMode, Object obj, Object buffer, PythonBufferAccessLibrary bufferLib) { + assert proto >= 5; - byte[] header = new byte[9]; + int size = bufferLib.getBufferLength(buffer); + boolean wasFraming = pickler.framing; + boolean bypassBuffer = bypassBuffer(pickler, size); - header[0] = PickleUtils.OPCODE_BYTEARRAY8; - PickleUtils.writeSize64(header, 1, size); + // Our sizes are ints, but the protocol expects 8 bytes + pickler.ensureBufferSpace(this, 9); + pickler.outputBuffer[pickler.outputLen++] = PickleUtils.OPCODE_BYTEARRAY8; + PickleUtils.writeSize64(pickler.outputBuffer, pickler.outputLen, size); + pickler.outputLen += 8; - int len = 9; - writeBytes(frame, pickler, header, len, data, size, obj); - memoPut(pickler, obj); + if (bypassBuffer && pickler.write != null) { + flushToFile(frame, pickler); + ensureCallNode().execute(frame, pickler.write, obj); + pickler.clearBuffer(); + } else { + pickler.writeBuffer(this, buffer, size, bufferLib); + } + + if (bypassBuffer && wasFraming) { + pickler.framing = true; + } + memoPut(pickler, proto, fastMode, obj); } - private void saveBytearray(VirtualFrame frame, Node inliningTarget, PythonContext ctx, PPickler pickler, Object obj, InteropCallData interopCallData) { - Object buffer = getBufferAcquireLibrary().acquireReadonly(obj, frame, interopCallData); + private void saveBytearray(VirtualFrame frame, Node inliningTarget, PythonContext ctx, PPickler pickler, int proto, boolean fastMode, PByteArray obj) { + Object buffer = getBufferAcquireLibrary().acquireReadonly(obj); + PythonBufferAccessLibrary bufferLib = getBufferLibrary(); try { - if (pickler.proto < 5) { + if (proto < 5) { // Older pickle protocols do not have an opcode for pickling bytearrays. Object reduceValue; final PythonBuiltinClass byteArrayClass = ctx.getCore().lookupType(PythonBuiltinClassType.PByteArray); - if (getBufferLibrary().getBufferLength(buffer) == 0) { + if (bufferLib.getBufferLength(buffer) == 0) { reduceValue = createTuple(byteArrayClass, createTuple()); } else { - byte[] bytes = getBufferLibrary().getCopiedByteArray(buffer); + byte[] bytes = bufferLib.getCopiedByteArray(buffer); reduceValue = createTuple(byteArrayClass, createTuple(PFactory.createBytes(ctx.getLanguage(inliningTarget), bytes))); } // save_reduce() will memoize the object automatically. - saveReduce(frame, ctx, pickler, reduceValue, obj); + saveReduce(frame, ctx, pickler, proto, fastMode, reduceValue, obj); } else { - saveBytearrayData(frame, pickler, obj, getBufferLibrary().getCopiedByteArray(buffer), length(frame, obj)); + saveBytearrayData(frame, pickler, proto, fastMode, obj, buffer, bufferLib); } } finally { - getBufferLibrary().release(buffer, frame, interopCallData); + bufferLib.release(buffer); } } - private void savePicklebuffer(VirtualFrame frame, PPickler pickler, PPickleBuffer obj) { - if (pickler.proto < 5) { + private void savePickleBuffer(VirtualFrame frame, PPickler pickler, int proto, boolean fastMode, PPickleBuffer obj) { + if (proto < 5) { throw raise(PicklingError, ErrorMessages.PICKLEBUFF_CANNOT_PICKLE_WITH_PROTO5); } Object buffer = obj.getView(); PythonBufferAccessLibrary bufferLib = getBufferLibrary(); - int bytesLen = bufferLib.getBufferLength(buffer); - byte[] bytes = bufferLib.getInternalOrCopiedByteArray(buffer); boolean inBand = true; if (pickler.bufferCallback != null) { - Object ret = getCallNode().execute(frame, pickler.bufferCallback, obj); + Object ret = ensureCallNode().execute(frame, pickler.bufferCallback, obj); inBand = isTrue(frame, ret); } @@ -1805,38 +2290,42 @@ private void savePicklebuffer(VirtualFrame frame, PPickler pickler, PPickleBuffe if (inBand) { // Write data in-band if (readOnly) { - saveBytesData(frame, pickler, obj, bytes, bytesLen); + saveBytesData(frame, pickler, proto, fastMode, obj, buffer, bufferLib); } else { - saveBytearrayData(frame, pickler, obj, bytes, bytesLen); + saveBytearrayData(frame, pickler, proto, fastMode, obj, buffer, bufferLib); } } else { // Write data out-of-band - write(pickler, PickleUtils.OPCODE_NEXT_BUFFER); + pickler.write(this, PickleUtils.OPCODE_NEXT_BUFFER); if (readOnly) { - write(pickler, PickleUtils.OPCODE_READONLY_BUFFER); + pickler.write(this, PickleUtils.OPCODE_READONLY_BUFFER); } } } - private void saveSingletonType(VirtualFrame frame, PythonContext ctx, PPickler pickler, Object obj, Object singleton) { + private void saveSingletonType(VirtualFrame frame, PythonContext ctx, PPickler pickler, int proto, boolean fastMode, Object obj, Object singleton) { Object reduceValue = createTuple(ctx.getCore().lookupType(PythonBuiltinClassType.PythonClass), createTuple(singleton)); - saveReduce(frame, ctx, pickler, reduceValue, obj); + saveReduce(frame, ctx, pickler, proto, fastMode, reduceValue, obj); } - private void saveType(VirtualFrame frame, BoundaryCallData boundaryCallData, PythonContext ctx, PPickler pickler, Object obj) { - if (isBuiltinClass(obj, PythonBuiltinClassType.PNone)) { - saveSingletonType(frame, ctx, pickler, obj, PNone.NONE); - } else if (isBuiltinClass(obj, PythonBuiltinClassType.PEllipsis)) { - saveSingletonType(frame, ctx, pickler, obj, PEllipsis.INSTANCE); - } else if (isBuiltinClass(obj, PythonBuiltinClassType.PNotImplemented)) { - saveSingletonType(frame, ctx, pickler, obj, PNotImplemented.NOT_IMPLEMENTED); + private void saveType(VirtualFrame frame, PythonContext ctx, PPickler pickler, int proto, boolean fastMode, PythonManagedClass obj) { + if (obj instanceof PythonBuiltinClass cls && cls.getType() == PythonBuiltinClassType.PNone) { + saveSingletonType(frame, ctx, pickler, proto, fastMode, obj, PNone.NONE); + } else if (obj instanceof PythonBuiltinClass cls && cls.getType() == PythonBuiltinClassType.PEllipsis) { + saveSingletonType(frame, ctx, pickler, proto, fastMode, obj, PEllipsis.INSTANCE); + } else if (obj instanceof PythonBuiltinClass cls && cls.getType() == PythonBuiltinClassType.PNotImplemented) { + saveSingletonType(frame, ctx, pickler, proto, fastMode, obj, PNotImplemented.NOT_IMPLEMENTED); } else { - Object type = (obj instanceof PythonBuiltinClassType) ? ctx.getCore().lookupType((PythonBuiltinClassType) obj) : obj; - saveGlobal(frame, boundaryCallData, ctx, pickler, type, null); + saveGlobal(frame, ctx, pickler, proto, fastMode, obj, null); } } - private void saveGlobal(VirtualFrame frame, BoundaryCallData boundaryCallData, PythonContext ctx, PPickler pickler, Object obj, Object name) { + protected Pair get3to2Mapping(VirtualFrame frame, Python3Core core, TruffleString moduleName, TruffleString globalName) { + PickleState state = getGlobalState(core); + return getMapping(frame, state.nameMapping3To2, state.importMapping3To2, T_CP_REVERSE_NAME_MAPPING, T_CP_REVERSE_IMPORT_MAPPING, moduleName, globalName); + } + + private void saveGlobal(VirtualFrame frame, PythonContext ctx, PPickler pickler, int proto, boolean fastMode, Object obj, Object name) { PickleState st = getGlobalState(ctx.getCore()); Object gName; if (name != null) { @@ -1854,11 +2343,11 @@ private void saveGlobal(VirtualFrame frame, BoundaryCallData boundaryCallData, P Object module; try { - module = importModule(frame, boundaryCallData, moduleName); + module = importModule(frame, ctx, moduleName); } catch (PException e) { throw raise(PicklingError, ErrorMessages.CANT_PICKLE_P_IMPORT_OF_MODULE_S_FAILED, obj, moduleName); } - final Pair pair = getDeepAttribute(frame, getLookupAttrNode(), module, dottedPath); + final Pair pair = getDeepAttribute(frame, getLookupGlobalAttrNode(), module, dottedPath); if (pair == null) { throw raise(PicklingError, ErrorMessages.CANT_PICKLE_P_ATTR_LOOKUP_FAIL_S_S, obj, globalName, moduleName); } @@ -1869,17 +2358,12 @@ private void saveGlobal(VirtualFrame frame, BoundaryCallData boundaryCallData, P } boolean genGlobal; - if (pickler.proto >= 2) { + if (proto >= 2) { genGlobal = false; // See whether this is in the extension registry, and if so generate an EXT opcode. - PTuple extensionKey; - Object codeObj; - long code; - byte[] pdata = new byte[5]; - int n; - - extensionKey = createTuple(moduleName, globalName); - codeObj = getDictItem(frame, st.extensionRegistry, extensionKey); + + PTuple extensionKey = createTuple(moduleName, globalName); + Object codeObj = getDictItem(frame, st.extensionRegistry, extensionKey); // The object is not registered in the extension registry. This is the most likely // code path. if (codeObj == null) { @@ -1892,58 +2376,45 @@ private void saveGlobal(VirtualFrame frame, BoundaryCallData boundaryCallData, P throw raise(PicklingError, ErrorMessages.CANT_PICKLE_P_EXT_CODE_P_NOT_AN_INT, obj, codeObj); } - code = asLong(frame, codeObj); + long code = asLong(frame, codeObj); if (code <= 0 || code > 0x7fffffffL) { throw raise(PicklingError, ErrorMessages.CANT_PICKLE_P_EXT_CODE_OO_RANGE, obj, code); } // Generate an EXT opcode if (code <= 0xff) { - pdata[0] = PickleUtils.OPCODE_EXT1; - pdata[1] = (byte) code; - n = 2; + pickler.writeByteOp(this, PickleUtils.OPCODE_EXT1, code); } else if (code <= 0xffff) { - pdata[0] = PickleUtils.OPCODE_EXT2; - pdata[1] = (byte) (code & 0xff); - pdata[2] = (byte) ((code >> 8) & 0xff); - n = 3; + pickler.writeShortOp(this, PickleUtils.OPCODE_EXT2, code); } else { - pdata[0] = PickleUtils.OPCODE_EXT4; - pdata[1] = (byte) (code & 0xff); - pdata[2] = (byte) ((code >> 8) & 0xff); - pdata[3] = (byte) ((code >> 16) & 0xff); - pdata[4] = (byte) ((code >> 24) & 0xff); - n = 5; + pickler.writeIntOp(this, PickleUtils.OPCODE_EXT4, code); } - - write(pickler, pdata, n); } } else { genGlobal = true; } if (genGlobal) { - byte[] encoded; TruffleString lastname = dottedPath[dottedPath.length - 1]; if (parent == module) { globalName = lastname; } - if (pickler.proto >= 4) { - save(frame, pickler, moduleName, 0); - save(frame, pickler, globalName, 0); - write(pickler, PickleUtils.OPCODE_STACK_GLOBAL); + if (proto >= 4) { + saveGlobalModuleName(frame, pickler, moduleName); + saveGlobalName(frame, pickler, globalName); + pickler.write(this, PickleUtils.OPCODE_STACK_GLOBAL); } else if (parent != module) { Object reduceValue = createTuple(st.getattr, createTuple(parent, lastname)); - saveReduce(frame, ctx, pickler, reduceValue, null); + saveReduce(frame, ctx, pickler, proto, fastMode, reduceValue, null); } else { // Generate a normal global opcode if we are using a pickle protocol < 4, or if // the object is not registered in the extension registry - write(pickler, PickleUtils.OPCODE_GLOBAL); + pickler.write(this, PickleUtils.OPCODE_GLOBAL); // For protocol < 3 and if the user didn't request against doing so, we convert // module names to the old 2.x module names. - if (pickler.proto < 3 && pickler.fixImports) { + if (proto < 3 && pickler.fixImports) { final Pair to2Mapping = get3to2Mapping(frame, ctx.getCore(), moduleName, globalName); moduleName = to2Mapping.getLeft(); globalName = to2Mapping.getRight(); @@ -1953,67 +2424,84 @@ private void saveGlobal(VirtualFrame frame, BoundaryCallData boundaryCallData, P // module name and the global name using UTF-8. We do so only when we are using // the pickle protocol newer than version 3. This is to ensure compatibility // with older Unpickler running on Python 2.x. - TruffleString.Encoding encoding = (pickler.proto == 3) ? TruffleString.Encoding.UTF_8 : TruffleString.Encoding.US_ASCII; - encoded = PickleUtils.encodeStrict(moduleName, ensureTsSwitchEncodingNode(), encoding, ensureTsCopyToByteArrayNode(), ensureTsGetCodeRangeNode()); - if (encoded == null) { - throw raise(PicklingError, ErrorMessages.CANT_PICKLE_MODULE_S_USING_PROTO_D, moduleName, pickler.proto); - } - write(pickler, encoded); - writeASCII(pickler, T_NEWLINE); - // Save the name of the module - encoded = PickleUtils.encodeStrict(globalName, ensureTsSwitchEncodingNode(), encoding, ensureTsCopyToByteArrayNode(), ensureTsGetCodeRangeNode()); - if (encoded == null) { - throw raise(PicklingError, ErrorMessages.CANT_PICKLE_MODULE_S_USING_PROTO_D, moduleName, pickler.proto); + if (proto >= 3) { + if (ensureTsGetCodeRangeNode().execute(moduleName, TS_ENCODING) == TruffleString.CodeRange.BROKEN || ensureTsGetCodeRangeNode().execute(globalName, + TS_ENCODING) == TruffleString.CodeRange.BROKEN) { + throw raise(PicklingError, ErrorMessages.CANT_PICKLE_MODULE_S_USING_PROTO_D, moduleName, proto); + } + pickler.writeUtf8(this, moduleName, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); + pickler.write(this, (byte) '\n'); + pickler.writeUtf8(this, globalName, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); + } else { + if (ensureTsGetCodeRangeNode().execute(moduleName, TS_ENCODING) != TruffleString.CodeRange.ASCII || ensureTsGetCodeRangeNode().execute(globalName, + TS_ENCODING) != TruffleString.CodeRange.ASCII) { + throw raise(PicklingError, ErrorMessages.CANT_PICKLE_MODULE_S_USING_PROTO_D, moduleName, proto); + } + pickler.writeASCII(this, moduleName, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); + pickler.write(this, (byte) '\n'); + pickler.writeASCII(this, globalName, ensureTsSwitchEncodingNode(), ensureTsCopyToByteArrayNode()); } - write(pickler, encoded); - writeASCII(pickler, T_NEWLINE); + pickler.write(this, (byte) '\n'); } // Memoize the object - memoPut(pickler, obj); + memoPut(pickler, proto, fastMode, obj); } } @Specialization void saveGeneric(VirtualFrame frame, PPickler pickler, Object objArg, int persSave, @Bind Node inliningTarget, - @Cached("createFor($node)") BoundaryCallData boundaryCallData, - @Cached("createFor($node)") InteropCallData interopCallData, - @Cached PyFloatAsDoubleNode asDoubleNode, @Cached CallNode callNode, - @Cached PyObjectCallMethodObjArgs callMethod) { + @Cached PyObjectCallMethodObjArgs callMethod, + @Cached InlinedIntValueProfile protoProfile, + @Cached InlinedConditionProfile fastModeProfile) { Object obj = objArg; + int proto = protoProfile.profile(inliningTarget, pickler.getProto()); + boolean fastMode = fastModeProfile.profile(inliningTarget, pickler.isFast()); opcodeBoundary(frame, pickler); // The extra pers_save argument is necessary to avoid calling save_pers() // on its returned object. if (persSave == 0 && pickler.persFunc != null) { - if (savePers(frame, pickler, obj)) { + if (savePers(frame, pickler, proto, obj)) { return; } } - final Object type = getClass(obj); - // The old cPickle had an optimization that used switch-case statement dispatching on - // the first letter of the type name. This has was removed since benchmarks shown that - // this optimization was actually slowing things down. - // Atom types; these aren't memoized, so don't check the memo. if (obj == PNone.NONE) { - saveNone(pickler, obj); + profileSeen(SEEN_NONE); + saveNone(pickler); + return; + } else if (obj instanceof Boolean value) { + profileSeen(SEEN_BOOLEAN); + saveBool(pickler, proto, value); return; - } else if (isBuiltinClass(type, PythonBuiltinClassType.Boolean)) { - saveBool(frame, pickler, obj); + } else if (obj instanceof Integer value) { + profileSeen(SEEN_INTEGER); + saveLong(pickler, proto, value); return; - } else if (isBuiltinClass(type, PythonBuiltinClassType.PInt)) { - saveLong(frame, pickler, obj); + } else if (obj instanceof Long value) { + profileSeen(SEEN_LONG); + saveLong(pickler, proto, value); return; - } else if (isBuiltinClass(type, PythonBuiltinClassType.PFloat)) { - saveFloat(frame, pickler, obj, inliningTarget, asDoubleNode); + } else if (obj instanceof PInt value && PGuards.isBuiltinPInt(value)) { + profileSeen(SEEN_PINT); + saveLong(frame, pickler, proto, value); + return; + } else if (obj instanceof Double value) { + profileSeen(SEEN_DOUBLE); + saveFloat(pickler, proto, value); + return; + } else if (obj instanceof PFloat value && PGuards.isBuiltinPFloat(value)) { + profileSeen(SEEN_PFLOAT); + saveFloat(pickler, proto, value.getValue()); return; } // to avoid misses in memo for the class: if (obj instanceof PythonBuiltinClassType classType) { + profileSeen(SEEN_BUILTIN_CLASS_TYPE); obj = PythonContext.get(this).getCore().lookupType(classType); } @@ -2021,38 +2509,55 @@ void saveGeneric(VirtualFrame frame, PPickler pickler, Object objArg, int persSa // instead of pickling the object once again. int memoIndex = pickler.memo.get(obj); if (memoIndex != -1) { - memoGet(pickler, memoIndex); + memoGet(pickler, proto, memoIndex); return; } PythonContext ctx = PythonContext.get(this); - if (isBuiltinClass(type, PythonBuiltinClassType.PBytes)) { - saveBytes(frame, ctx, pickler, obj, interopCallData); + if (obj instanceof PBytes bytes && PGuards.isBuiltinBytes(bytes)) { + profileSeen(SEEN_BYTES); + saveBytes(frame, ctx, pickler, proto, fastMode, bytes); + return; + } else if (obj instanceof TruffleString string) { + profileSeen(SEEN_TRUFFLE_STRING); + saveUnicode(frame, pickler, proto, fastMode, string); return; - } else if (isBuiltinClass(type, PythonBuiltinClassType.PString)) { - saveUnicode(frame, pickler, obj); + } else if (obj instanceof PString string && PGuards.isBuiltinPString(string)) { + profileSeen(SEEN_PSTRING); + saveUnicode(frame, pickler, proto, fastMode, string); return; - } else if (isBuiltinClass(type, PythonBuiltinClassType.PDict)) { - saveDict(frame, inliningTarget, callMethod, pickler, obj); + } else if (obj instanceof PDict dict && PGuards.isBuiltinDict(dict)) { + profileSeen(SEEN_DICT); + saveDict(frame, inliningTarget, callMethod, pickler, proto, fastMode, dict); return; - } else if (isBuiltinClass(type, PythonBuiltinClassType.PSet)) { - saveSet(frame, ctx, pickler, obj); + } else if (obj instanceof PSet set && PGuards.isBuiltinSet(set)) { + profileSeen(SEEN_SET); + saveSet(frame, ctx, pickler, proto, fastMode, set); return; - } else if (isBuiltinClass(type, PythonBuiltinClassType.PFrozenSet)) { - saveFrozenset(frame, ctx, pickler, obj); + } else if (obj instanceof PFrozenSet frozenSet && PGuards.isBuiltinFrozenSet(frozenSet)) { + profileSeen(SEEN_FROZENSET); + saveFrozenset(frame, ctx, pickler, proto, fastMode, frozenSet); return; - } else if (isBuiltinClass(type, PythonBuiltinClassType.PList)) { - saveList(frame, pickler, obj); + } else if (obj instanceof PList list && PGuards.isBuiltinList(list)) { + profileSeen(SEEN_LIST); + saveList(frame, pickler, proto, fastMode, list); return; - } else if (isBuiltinClass(type, PythonBuiltinClassType.PTuple)) { - saveTuple(frame, pickler, obj); + } else if (obj instanceof PTuple tuple && PGuards.isBuiltinTuple(tuple)) { + profileSeen(SEEN_TUPLE); + saveTuple(frame, pickler, proto, fastMode, tuple); return; - } else if (isBuiltinClass(type, PythonBuiltinClassType.PByteArray)) { - saveBytearray(frame, inliningTarget, ctx, pickler, obj, interopCallData); + } else if (PGuards.isNativeObject(obj) && isBuiltinClass(getClass(obj), PythonBuiltinClassType.PTuple)) { + profileSeen(SEEN_NATIVE_TUPLE); + saveNativeTuple(frame, pickler, proto, fastMode, obj); + return; + } else if (obj instanceof PByteArray byteArray && PGuards.isBuiltinByteArray(byteArray)) { + profileSeen(SEEN_BYTE_ARRAY); + saveBytearray(frame, inliningTarget, ctx, pickler, proto, fastMode, byteArray); return; } else if (obj instanceof PPickleBuffer buffer) { - savePicklebuffer(frame, pickler, buffer); + profileSeen(SEEN_PICKLE_BUFFER); + savePickleBuffer(frame, pickler, proto, fastMode, buffer); return; } @@ -2061,27 +2566,37 @@ void saveGeneric(VirtualFrame frame, PPickler pickler, Object objArg, int persSa if (pickler.reducerOverride != null) { Object reduceValue = callNode.execute(frame, pickler.reducerOverride, obj); if (reduceValue != PNotImplemented.NOT_IMPLEMENTED) { - handleReduce(frame, boundaryCallData, ctx, pickler, obj, reduceValue); + handleReduce(frame, ctx, pickler, proto, fastMode, obj, reduceValue); return; } } - if (isBuiltinClass(type, PythonBuiltinClassType.PythonClass)) { - saveType(frame, boundaryCallData, ctx, pickler, obj); + if (obj instanceof PythonManagedClass clazz && clazz.getPythonClass() == PythonBuiltinClassType.PythonClass) { + profileSeen(SEEN_MANAGED_CLASS); + saveType(frame, ctx, pickler, proto, fastMode, clazz); + return; + } else if (obj instanceof PythonNativeClass && isBuiltinClass(getClass(obj), PythonBuiltinClassType.PythonClass)) { + profileSeen(SEEN_NATIVE_CLASS); + saveGlobal(frame, ctx, pickler, proto, fastMode, obj, null); return; - } else if (isBuiltinClass(type, PythonBuiltinClassType.PFunction)) { - saveGlobal(frame, boundaryCallData, ctx, pickler, obj, null); + } else if (obj instanceof PFunction) { + profileSeen(SEEN_FUNCTION); + saveGlobal(frame, ctx, pickler, proto, fastMode, obj, null); return; } + profileSeen(SEEN_OBJECT); + // Get a reduction callable, and call it. This may come from self.dispatch_table, // copyreg.dispatch_table, the object's __reduce_ex__ method, or the object's __reduce__ // method. + Object type = getClass(obj); Object reduceFunc = null; if (pickler.dispatchTable == null) { PickleState state = getGlobalState(ctx.getCore()); reduceFunc = getDictItem(frame, state.dispatchTable, type); } else { + profileSeen(SEEN_CUSTOM_DISPATCH); try { reduceFunc = getItem(frame, pickler.dispatchTable, type); } catch (PException ex) { @@ -2091,9 +2606,11 @@ void saveGeneric(VirtualFrame frame, PPickler pickler, Object objArg, int persSa Object reduceValue; if (reduceFunc != null) { + profileSeen(SEEN_REGISTERED_REDUCER); reduceValue = callNode.execute(frame, reduceFunc, obj); - } else if (isSubType(type, PythonBuiltinClassType.PythonClass)) { - saveGlobal(frame, boundaryCallData, ctx, pickler, obj, null); + } else if (isType(obj)) { + profileSeen(SEEN_TYPE); + saveGlobal(frame, ctx, pickler, proto, fastMode, obj, null); return; } else { // XXX: If the __reduce__ method is defined, __reduce_ex__ is automatically defined @@ -2105,11 +2622,13 @@ void saveGeneric(VirtualFrame frame, PPickler pickler, Object objArg, int persSa // Check for a __reduce_ex__ method. reduceFunc = lookupAttribute(frame, obj, T___REDUCE_EX__); if (reduceFunc != PNone.NO_VALUE) { - reduceValue = callNode.execute(frame, reduceFunc, pickler.proto); + profileSeen(SEEN_REDUCE_EX); + reduceValue = callNode.execute(frame, reduceFunc, proto); } else { // Check for a __reduce__ method. reduceFunc = lookupAttribute(frame, obj, T___REDUCE__); if (reduceFunc != PNone.NO_VALUE) { + profileSeen(SEEN_REDUCE); reduceValue = callNode.execute(frame, reduceFunc); } else { throw raise(PicklingError, ErrorMessages.CANNOT_PICKLE_P_P, type, obj); @@ -2117,18 +2636,37 @@ void saveGeneric(VirtualFrame frame, PPickler pickler, Object objArg, int persSa } } - handleReduce(frame, boundaryCallData, ctx, pickler, obj, reduceValue); + handleReduce(frame, ctx, pickler, proto, fastMode, obj, reduceValue); } } - public abstract static class DumpNode extends BasePickleWriteNode { + @GenerateInline(false) + public abstract static class FlushToFileNode extends Node { + public abstract void execute(VirtualFrame frame, PPickler pickler); + + @Specialization + public void flush(VirtualFrame frame, PPickler pickler, + @Bind PythonLanguage language, + @Cached CallNode callNode) { + assert pickler.write != null; + // This will commit the frame first + PBytes output = pickler.getString(language, true); + callNode.execute(frame, pickler.write, output); + } + } + + @GenerateInline(false) + public abstract static class DumpNode extends Node { + private static final TruffleString REDUCE_OVERRIDE = tsLiteral("reducer_override"); + public abstract void execute(VirtualFrame frame, PPickler pickler, Object obj); @Specialization public void dump(VirtualFrame frame, PPickler pickler, Object obj, - @Cached SaveNode saveNode) { + @Cached("create(0)") SaveNode saveNode, + @Cached PyObjectLookupAttr lookupAttrNode) { try { - final Object tmp = getLookupAttrNode().executeCached(frame, pickler, REDUCE_OVERRIDE); + final Object tmp = lookupAttrNode.execute(frame, this, pickler, REDUCE_OVERRIDE); if (tmp != PNone.NO_VALUE) { pickler.reducerOverride = tmp; } else { @@ -2137,12 +2675,11 @@ public void dump(VirtualFrame frame, PPickler pickler, Object obj, if (pickler.proto >= 2) { assert pickler.proto <= 256; - byte[] header = new byte[]{PickleUtils.OPCODE_PROTO, (byte) pickler.proto}; try { - write(pickler, header, 2); + pickler.write(this, PickleUtils.OPCODE_PROTO, (byte) pickler.proto); } catch (Exception e) { - handleError(pickler); + SaveNode.handleError(pickler); throw e; } @@ -2152,7 +2689,7 @@ public void dump(VirtualFrame frame, PPickler pickler, Object obj, } saveNode.execute(frame, pickler, obj, 0); - write(pickler, PickleUtils.OPCODE_STOP); + pickler.write(this, PickleUtils.OPCODE_STOP); pickler.commitFrame(); } finally { pickler.framing = false; diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PUnpickler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PUnpickler.java index 85ab4c5bba..b70dc9e32c 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PUnpickler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PUnpickler.java @@ -111,12 +111,15 @@ import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.PICKLE_PROTOCOL_HIGHEST; import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.PREFETCH; import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.READ_WHOLE_LINE; +import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.T_CP_IMPORT_MAPPING; +import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.T_CP_NAME_MAPPING; import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.T_METHOD_PEEK; import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.T_METHOD_PERSISTENT_LOAD; import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.T_METHOD_READ; import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.T_METHOD_READINTO; import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.T_METHOD_READLINE; import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.getValidIntString; +import static com.oracle.graal.python.builtins.objects.PNone.NO_VALUE; import static com.oracle.graal.python.nodes.BuiltinNames.T_ADD; import static com.oracle.graal.python.nodes.BuiltinNames.T_APPEND; import static com.oracle.graal.python.nodes.BuiltinNames.T_EXTEND; @@ -126,11 +129,14 @@ import static com.oracle.graal.python.nodes.SpecialMethodNames.T___SETSTATE__; import static com.oracle.graal.python.nodes.StringLiterals.T_ASCII_UPPERCASE; import static com.oracle.graal.python.nodes.StringLiterals.T_STRICT; +import static com.oracle.graal.python.runtime.exception.PythonErrorType.AttributeError; +import static com.oracle.graal.python.runtime.exception.PythonErrorType.TypeError; import static com.oracle.graal.python.util.PythonUtils.TS_ENCODING; import org.graalvm.collections.Pair; import com.oracle.graal.python.PythonLanguage; +import com.oracle.graal.python.builtins.Python3Core; import com.oracle.graal.python.builtins.PythonBuiltinClassType; import com.oracle.graal.python.builtins.objects.PNone; import com.oracle.graal.python.builtins.objects.buffer.BufferFlags; @@ -166,6 +172,7 @@ import com.oracle.graal.python.lib.PyObjectCallMethodObjArgs; import com.oracle.graal.python.lib.PyObjectGetIter; import com.oracle.graal.python.lib.PyObjectLookupAttr; +import com.oracle.graal.python.lib.PyObjectReprAsTruffleStringNode; import com.oracle.graal.python.lib.PyObjectSetAttrO; import com.oracle.graal.python.lib.PyObjectSetItem; import com.oracle.graal.python.lib.PyTupleCheckNode; @@ -174,7 +181,7 @@ import com.oracle.graal.python.nodes.PRaiseNode; import com.oracle.graal.python.nodes.argument.keywords.ExpandKeywordStarargsNode; import com.oracle.graal.python.nodes.argument.positional.ExecutePositionalStarargsNode; -import com.oracle.graal.python.runtime.IndirectCallData.BoundaryCallData; +import com.oracle.graal.python.nodes.util.CastToTruffleStringNode; import com.oracle.graal.python.runtime.IndirectCallData.InteropCallData; import com.oracle.graal.python.runtime.PythonContext; import com.oracle.graal.python.runtime.exception.PException; @@ -183,6 +190,7 @@ import com.oracle.graal.python.util.NumericSupport; import com.oracle.graal.python.util.PythonUtils; import com.oracle.truffle.api.CompilerDirectives; +import com.oracle.truffle.api.CompilerDirectives.CompilationFinal; import com.oracle.truffle.api.dsl.Bind; import com.oracle.truffle.api.dsl.Cached; import com.oracle.truffle.api.dsl.Specialization; @@ -361,7 +369,7 @@ public Object memoGet(int idx) { } public void memoPut(int idx, Object value) { - if (idx >= memo.length) { + if (CompilerDirectives.injectBranchProbability(CompilerDirectives.SLOWPATH_PROBABILITY, idx >= memo.length)) { resizeMemoList(idx * 2); assert idx < memo.length; } @@ -396,6 +404,8 @@ public void clearMemo() { // inner nodes public abstract static class BasePickleReadNode extends PicklerNodes.BasePickleNode { + @Child private CastToTruffleStringNode castToTruffleStringNode; + @Child private PyObjectReprAsTruffleStringNode reprNode; @Child private PyMemoryViewFromObject memoryViewNode; @Child private MemoryViewBuiltins.ToReadonlyNode toReadonlyNode; @Child private PythonBufferAcquireLibrary bufferAcquireLibrary; @@ -405,9 +415,69 @@ public abstract static class BasePickleReadNode extends PicklerNodes.BasePickleN @Child private TruffleString.ParseIntNode tsParseIntNode; + @CompilationFinal private boolean seenFileRead; + // we only need the reference, doesn't matter that the object may not yet be fully // constructed - @SuppressWarnings("this-escape") private InteropCallData interopCallData = InteropCallData.createFor(this); + @SuppressWarnings("this-escape") private final InteropCallData interopCallData = InteropCallData.createFor(this); + + protected TruffleString castToString(Object value) { + if (castToTruffleStringNode == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + castToTruffleStringNode = insert(CastToTruffleStringNode.create()); + } + return castToTruffleStringNode.executeCached(value); + } + + private PyObjectReprAsTruffleStringNode getReprNode() { + if (reprNode == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + reprNode = insert(PyObjectReprAsTruffleStringNode.create()); + } + return reprNode; + } + + public Object getattribute(VirtualFrame frame, Object obj, TruffleString name, boolean allowQualname) { + if (allowQualname) { + TruffleString[] dottedPath = getDottedPath(obj, name); + Pair result = getDeepAttribute(frame, getLookupGlobalAttrNode(), obj, dottedPath); + if (result != null) { + return result.getLeft(); + } else { + throw raise(AttributeError, ErrorMessages.CANT_GET_ATTRIBUTE_S_ON_S, name, getReprNode().execute(frame, null, obj)); + } + } else { + Object attr = getLookupGlobalAttrNode().execute(frame, obj, name); + if (attr == NO_VALUE) { + throw raise(TypeError, ErrorMessages.OBJ_P_HAS_NO_ATTR_S, attr, name); + } + return attr; + } + } + + protected Pair get2To3Mapping(VirtualFrame frame, Python3Core core, TruffleString moduleName, TruffleString globalName) { + PickleState state = getGlobalState(core); + return getMapping(frame, state.nameMapping2To3, state.importMapping2To3, T_CP_NAME_MAPPING, T_CP_IMPORT_MAPPING, moduleName, globalName); + } + + public Object findClass(VirtualFrame frame, Python3Core core, PUnpickler self, Object moduleName, Object globalName) { + return findClass(frame, core, self, castToString(moduleName), castToString(globalName)); + } + + public Object findClass(VirtualFrame frame, Python3Core core, PUnpickler self, TruffleString moduleName, TruffleString globalName) { + // Try to map the old names used in Python 2.x to the new ones used in Python 3.x. We do + // this only with old pickle protocols and when the user has not disabled the feature. + TruffleString mName = moduleName; + TruffleString gName = globalName; + if (self.getProto() < 3 && self.isFixImports()) { + final Pair to3Mapping = get2To3Mapping(frame, core, mName, gName); + mName = to3Mapping.getLeft(); + gName = to3Mapping.getRight(); + } + + Object module = importModule(frame, PythonContext.get(this), mName); + return getattribute(frame, module, gName, self.getProto() >= 4); + } protected TruffleString.ParseLongNode ensureTsParseLongNode() { if (tsParseLongNode == null) { @@ -534,19 +604,24 @@ protected int readFromFile(VirtualFrame frame, PUnpickler self, int n) { } protected byte read(VirtualFrame frame, PUnpickler self) { - return read(frame, self, 1).get(0); + int offset = read(frame, self, 1); + return self.inputBuffer[offset]; } - protected ByteArrayView read(VirtualFrame frame, PUnpickler self, int n) { - if (n <= self.inputLen - self.nextReadIdx) { - ByteArrayView bytesView = new ByteArrayView(self.inputBuffer, self.nextReadIdx); + protected int read(VirtualFrame frame, PUnpickler self, int n) { + if (CompilerDirectives.injectBranchProbability(CompilerDirectives.FASTPATH_PROBABILITY, n <= self.inputLen - self.nextReadIdx)) { + int offset = self.nextReadIdx; self.nextReadIdx += n; - return bytesView; + return offset; + } + if (!seenFileRead) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + seenFileRead = true; } return readImpl(frame, self, n); } - private ByteArrayView readImpl(VirtualFrame frame, PUnpickler self, int n) { + private int readImpl(VirtualFrame frame, PUnpickler self, int n) { int numRead; // TODO: when GR-24978 is completed we should use PY_SSIZE_T_MAX if (self.nextReadIdx > Integer.MAX_VALUE - 1) { @@ -563,7 +638,7 @@ private ByteArrayView readImpl(VirtualFrame frame, PUnpickler self, int n) { throw badReadLine(); } self.nextReadIdx = n; - return new ByteArrayView(self.inputBuffer); + return 0; } protected int readInto(VirtualFrame frame, PUnpickler self, byte[] buffer) { @@ -676,13 +751,84 @@ public abstract static class FindClassNode extends BasePickleReadNode { public abstract Object execute(VirtualFrame frame, PUnpickler unpickler, TruffleString module, TruffleString name); @Specialization - Object find(VirtualFrame frame, PUnpickler unpickler, TruffleString module, TruffleString name, - @Cached("createFor($node)") BoundaryCallData boundaryCallData) { - return findClass(frame, boundaryCallData, PythonContext.get(this).getCore(), unpickler, module, name); + Object find(VirtualFrame frame, PUnpickler unpickler, TruffleString module, TruffleString name) { + return findClass(frame, PythonContext.get(this).getCore(), unpickler, module, name); } } public abstract static class LoadNode extends BasePickleReadNode { + private static final int SEEN_NONE = 0; + private static final int SEEN_BININT = 1; + private static final int SEEN_BININT1 = 2; + private static final int SEEN_BININT2 = 3; + private static final int SEEN_INT = 4; + private static final int SEEN_LONG = 5; + private static final int SEEN_LONG1 = 6; + private static final int SEEN_LONG4 = 7; + private static final int SEEN_FLOAT = 8; + private static final int SEEN_BINFLOAT = 9; + private static final int SEEN_SHORT_BINBYTES = 10; + private static final int SEEN_BINBYTES = 11; + private static final int SEEN_BINBYTES8 = 12; + private static final int SEEN_BYTEARRAY8 = 13; + private static final int SEEN_NEXT_BUFFER = 14; + private static final int SEEN_READONLY_BUFFER = 15; + private static final int SEEN_SHORT_BINSTRING = 16; + private static final int SEEN_BINSTRING = 17; + private static final int SEEN_STRING = 18; + private static final int SEEN_UNICODE = 19; + private static final int SEEN_SHORT_BINUNICODE = 20; + private static final int SEEN_BINUNICODE = 21; + private static final int SEEN_BINUNICODE8 = 22; + private static final int SEEN_EMPTY_TUPLE = 23; + private static final int SEEN_TUPLE1 = 24; + private static final int SEEN_TUPLE2 = 25; + private static final int SEEN_TUPLE3 = 26; + private static final int SEEN_TUPLE = 27; + private static final int SEEN_EMPTY_LIST = 28; + private static final int SEEN_LIST = 29; + private static final int SEEN_EMPTY_DICT = 30; + private static final int SEEN_DICT = 31; + private static final int SEEN_EMPTY_SET = 32; + private static final int SEEN_ADDITEMS = 33; + private static final int SEEN_FROZENSET = 34; + private static final int SEEN_OBJ = 35; + private static final int SEEN_INST = 36; + private static final int SEEN_NEWOBJ = 37; + private static final int SEEN_NEWOBJ_EX = 38; + private static final int SEEN_GLOBAL = 39; + private static final int SEEN_STACK_GLOBAL = 40; + private static final int SEEN_APPEND = 41; + private static final int SEEN_APPENDS = 42; + private static final int SEEN_BUILD = 43; + private static final int SEEN_DUP = 44; + private static final int SEEN_BINGET = 45; + private static final int SEEN_LONG_BINGET = 46; + private static final int SEEN_GET = 47; + private static final int SEEN_MARK = 48; + private static final int SEEN_BINPUT = 49; + private static final int SEEN_LONG_BINPUT = 50; + private static final int SEEN_PUT = 51; + private static final int SEEN_MEMOIZE = 52; + private static final int SEEN_POP = 53; + private static final int SEEN_POP_MARK = 54; + private static final int SEEN_SETITEM = 55; + private static final int SEEN_SETITEMS = 56; + private static final int SEEN_PERSID = 57; + private static final int SEEN_BINPERSID = 58; + private static final int SEEN_REDUCE = 59; + private static final int SEEN_PROTO = 60; + private static final int SEEN_FRAME = 61; + private static final int SEEN_EXT1 = 62; + private static final int SEEN_EXT2 = 63; + private static final int SEEN_EXT4 = 64; + private static final int SEEN_NEWTRUE = 65; + private static final int SEEN_NEWFALSE = 66; + private static final int SEEN_STOP = 67; + + @CompilationFinal private long seenOpcodes0; + @CompilationFinal private long seenOpcodes1; + @Child private PData.PDataPushNode pDataPushNode; @Child private PData.PDataPopNode pDataPopNode; @Child private PData.PDataPopTupleNode pDataPopTupleNode; @@ -698,6 +844,22 @@ public abstract static class LoadNode extends BasePickleReadNode { public abstract Object execute(VirtualFrame frame, PUnpickler self); + private void profileSeen(int seen) { + if (seen < Long.SIZE) { + long mask = 1L << seen; + if ((seenOpcodes0 & mask) == 0) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + seenOpcodes0 |= mask; + } + } else { + long mask = 1L << (seen - Long.SIZE); + if ((seenOpcodes1 & mask) == 0) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + seenOpcodes1 |= mask; + } + } + } + protected HashingStorageCopy ensureHashingStorageCopy() { if (hashCopy == null) { CompilerDirectives.transferToInterpreterAndInvalidate(); @@ -722,12 +884,12 @@ protected boolean isTuple(Object object) { return tupleCheck.execute(object); } - protected Object longFromBytes(byte[] data, boolean littleEndian) { + protected Object longFromBytes(byte[] data, int offset, int length, boolean littleEndian) { if (pyLongFromByteArray == null) { CompilerDirectives.transferToInterpreterAndInvalidate(); pyLongFromByteArray = insert(IntNodesFactory.PyLongFromByteArrayNodeGen.create()); } - return pyLongFromByteArray.executeCached(data, littleEndian, true); + return pyLongFromByteArray.executeCached(data, offset, length, littleEndian, true); } protected void setAttribute(VirtualFrame frame, Object object, Object key, Object value) { @@ -809,11 +971,11 @@ void loadNone(PUnpickler self) { pDataPush(self, PNone.NONE); } - private static long calcBinInt(ByteArrayView s, int nBytes) { + private static long calcBinInt(byte[] s, int offset, int nBytes) { long x = 0; for (int i = 0; i < nBytes; i++) { - x |= (long) s.getUnsigned(i) << (8 * i); + x |= (long) (s[offset + i] & 0xff) << (8 * i); } // Unlike BININT1 and BININT2, BININT (more accurately BININT4) is signed, so on a box @@ -826,7 +988,7 @@ private static long calcBinInt(ByteArrayView s, int nBytes) { return x; } - private int calcBinSize(ByteArrayView s, int nbytes) { + private int calcBinSize(byte[] s, int offset, int nbytes) { int i; int x = 0; int n = nbytes; @@ -836,7 +998,7 @@ private int calcBinSize(ByteArrayView s, int nbytes) { // Check for integer overflow. BINBYTES8 and BINUNICODE8 opcodes have 64-bit size // that can't be represented on 32-bit platform. for (i = Integer.BYTES; i < n; i++) { - if (s.get(i) != 0) { + if (s[offset + i] != 0) { throw raise(PythonBuiltinClassType.OverflowError); } } @@ -844,7 +1006,7 @@ private int calcBinSize(ByteArrayView s, int nbytes) { } for (i = 0; i < n; i++) { - x |= s.getUnsigned(i) << (8 * i); + x |= (s[offset + i] & 0xff) << (8 * i); } // TODO: GR-24978 check for PY_SSIZE_T_MAX (see: _cpickle.c:calc_binsize) @@ -855,24 +1017,24 @@ private int calcBinSize(ByteArrayView s, int nbytes) { return x; } - private void loadBinIntX(PUnpickler self, ByteArrayView s, int size) { - long x = calcBinInt(s, size); + private void loadBinIntX(PUnpickler self, int offset, int size) { + long x = calcBinInt(self.inputBuffer, offset, size); pDataPush(self, x); } private void loadBinInt(VirtualFrame frame, PUnpickler self) { - final ByteArrayView s = read(frame, self, 4); - loadBinIntX(self, s, 4); + int offset = read(frame, self, 4); + loadBinIntX(self, offset, 4); } private void loadBinInt1(VirtualFrame frame, PUnpickler self) { - final ByteArrayView s = read(frame, self, 1); - loadBinIntX(self, s, 1); + int offset = read(frame, self, 1); + loadBinIntX(self, offset, 1); } private void loadBinInt2(VirtualFrame frame, PUnpickler self) { - final ByteArrayView s = read(frame, self, 2); - loadBinIntX(self, s, 2); + int offset = read(frame, self, 2); + loadBinIntX(self, offset, 2); } private void loadInt(VirtualFrame frame, PUnpickler self) { @@ -922,8 +1084,8 @@ private void loadLong(VirtualFrame frame, PUnpickler self) { private void loadCountedLong(VirtualFrame frame, PUnpickler self, int n) { assert n == 1 || n == 4; int size = n; - final ByteArrayView nbytes = read(frame, self, size); - size = (int) calcBinInt(nbytes, size); + int sizeOffset = read(frame, self, size); + size = (int) calcBinInt(self.inputBuffer, sizeOffset, size); Object value; @@ -935,8 +1097,8 @@ private void loadCountedLong(VirtualFrame frame, PUnpickler self, int n) { value = 0L; } else { // Read the raw little-endian bytes and convert. - final ByteArrayView pdata = read(frame, self, size); - value = longFromBytes(pdata.getBytes(size), true); + int dataOffset = read(frame, self, size); + value = longFromBytes(self.inputBuffer, dataOffset, size, true); } pDataPush(self, value); } @@ -956,15 +1118,15 @@ private void loadFloat(VirtualFrame frame, PUnpickler self) { private void loadBinFloat(VirtualFrame frame, PUnpickler self) { Object value; - ByteArrayView s = read(frame, self, 8); + int offset = read(frame, self, 8); - value = NumericSupport.bigEndian().getDouble(s.getBytes(Double.BYTES), 0); + value = NumericSupport.bigEndian().getDouble(self.inputBuffer, offset); pDataPush(self, value); } private void loadCountedBinBytes(VirtualFrame frame, PUnpickler self, int nbytes) { - final ByteArrayView s = read(frame, self, nbytes); - int size = calcBinSize(s, nbytes); + int offset = read(frame, self, nbytes); + int size = calcBinSize(self.inputBuffer, offset, nbytes); if (size < 0) { throw raise(PythonBuiltinClassType.OverflowError, ErrorMessages.S_EXCEEDS_MAX_SIZE_N_BYTES, "BINBYTES", Integer.MAX_VALUE); } @@ -977,8 +1139,8 @@ private void loadCountedBinBytes(VirtualFrame frame, PUnpickler self, int nbytes } private void loadCountedByteArray(VirtualFrame frame, PUnpickler self) { - final ByteArrayView s = read(frame, self, 8); - int size = calcBinSize(s, 8); + int offset = read(frame, self, 8); + int size = calcBinSize(self.inputBuffer, offset, 8); if (size < 0) { throw raise(PythonBuiltinClassType.OverflowError, ErrorMessages.S_EXCEEDS_MAX_SIZE_N_BYTES, "BYTEARRAY8", Integer.MAX_VALUE); } @@ -1025,18 +1187,18 @@ private void loadReadOnlyBuffer(VirtualFrame frame, PUnpickler self) { private void loadCountedBinString(VirtualFrame frame, PUnpickler self, int nbytes) { Object obj; - ByteArrayView s = read(frame, self, nbytes); + int offset = read(frame, self, nbytes); - int size = calcBinSize(s, nbytes); + int size = calcBinSize(self.inputBuffer, offset, nbytes); if (size < 0) { throw raise(PythonBuiltinClassType.UnpicklingError, ErrorMessages.S_EXCEEDS_MAX_SIZE_N_BYTES, "BINSTRING", Integer.MAX_VALUE); } - s = read(frame, self, size); + offset = read(frame, self, size); // Convert Python 2.x strings to bytes if the *encoding* given to the Unpickler was // 'bytes'. Otherwise, convert them to unicode. - final PBytes bytes = PFactory.createBytes(PythonLanguage.get(this), s.getBytes(size), size); + final PBytes bytes = PFactory.createBytes(PythonLanguage.get(this), PythonUtils.arrayCopyOfRange(self.inputBuffer, offset, offset + size)); if (ensureTsEqualNode().execute(self.encoding, T_CODEC_BYTES, TS_ENCODING)) { obj = bytes; } else { @@ -1095,16 +1257,16 @@ private void loadUnicode(VirtualFrame frame, PUnpickler self) { private void loadBinCountedUnicode(VirtualFrame frame, PUnpickler self, int nbytes) { Object str; - ByteArrayView s = read(frame, self, nbytes); + int offset = read(frame, self, nbytes); - int size = calcBinSize(s, nbytes); + int size = calcBinSize(self.inputBuffer, offset, nbytes); if (size < 0) { throw raise(PythonBuiltinClassType.OverflowError, ErrorMessages.S_EXCEEDS_MAX_SIZE_N_BYTES, "BINUNICODE", Integer.MAX_VALUE); } - s = read(frame, self, size); + offset = read(frame, self, size); - str = decodeUTF8(frame, s, size, T_ERRORS_SURROGATEPASS); + str = decodeUTF8(frame, self.inputBuffer, offset, size, T_ERRORS_SURROGATEPASS); pDataPush(self, str); } @@ -1149,13 +1311,14 @@ private void loadDict(VirtualFrame frame, PUnpickler self) { throw raise(PythonBuiltinClassType.UnpicklingError, ErrorMessages.ODD_NR_ITEMS_FOR_S, "DICT"); } - HashingStorage storage = EmptyStorage.INSTANCE; + HashingStorage storage = PDict.createNewStorage((j - i) / 2); for (k = i + 1; k < j; k += 2) { key = self.stack.data[k - 1]; value = self.stack.data[k]; storage = setHashingStorageItem(frame, storage, key, value); } + LoopNode.reportLoopCount(this, (j - i) / 2); self.stack.clear(i); pDataPush(self, PFactory.createDict(PythonLanguage.get(this), storage)); } @@ -1235,7 +1398,7 @@ private void loadObj(VirtualFrame frame, Node inliningTarget, PUnpickler self, P pDataPush(self, obj); } - private void loadInst(VirtualFrame frame, Node inliningTarget, BoundaryCallData boundaryCallData, PythonContext ctx, PUnpickler self, PyObjectCallMethodObjArgs callMethod) { + private void loadInst(VirtualFrame frame, Node inliningTarget, PythonContext ctx, PUnpickler self, PyObjectCallMethodObjArgs callMethod) { Object cls = null; Object obj = null; int i = marker(self); @@ -1255,7 +1418,7 @@ private void loadInst(VirtualFrame frame, Node inliningTarget, BoundaryCallData throw badReadLine(); } Object className = decodeASCII(frame, s, s.length - 1, T_ERRORS_STRICT); - cls = findClass(frame, boundaryCallData, ctx.getCore(), self, moduleName, className); + cls = findClass(frame, ctx.getCore(), self, moduleName, className); } assert cls != null; @@ -1355,7 +1518,7 @@ private void doAppend(VirtualFrame frame, PUnpickler self, int x) { } } - private void loadGlobal(VirtualFrame frame, BoundaryCallData boundaryCallData, PythonContext ctx, PUnpickler self) { + private void loadGlobal(VirtualFrame frame, PythonContext ctx, PUnpickler self) { Object global = null; TruffleString globalName; byte[] s = readLine(frame, self); @@ -1371,14 +1534,14 @@ private void loadGlobal(VirtualFrame frame, BoundaryCallData boundaryCallData, P } globalName = PickleUtils.decodeUTF8Strict(s, s.length - 1, ensureTsFromByteArray(), ensureTsSwitchEncodingNode()); if (globalName != null) { - global = findClass(frame, boundaryCallData, ctx.getCore(), self, moduleName, globalName); + global = findClass(frame, ctx.getCore(), self, moduleName, globalName); } } pDataPush(self, global); } - private void loadStackGlobal(VirtualFrame frame, BoundaryCallData boundaryCallData, PythonContext ctx, PUnpickler self) { + private void loadStackGlobal(VirtualFrame frame, PythonContext ctx, PUnpickler self) { Object globalName = null; Object moduleName = null; try { @@ -1390,7 +1553,7 @@ private void loadStackGlobal(VirtualFrame frame, BoundaryCallData boundaryCallDa if (!PGuards.isString(moduleName) || !PGuards.isString(globalName)) { throw raise(PythonBuiltinClassType.UnpicklingError, ErrorMessages.S_REQ_STR, "STACK_GLOBAL"); } - Object global = findClass(frame, boundaryCallData, ctx.getCore(), self, moduleName, globalName); + Object global = findClass(frame, ctx.getCore(), self, moduleName, globalName); pDataPush(self, global); } @@ -1499,8 +1662,8 @@ private void loadBinGet(VirtualFrame frame, PUnpickler self) { } private void loadLongBinGet(VirtualFrame frame, PUnpickler self) { - ByteArrayView s = read(frame, self, 4); - int idx = calcBinSize(s, 4); + int offset = read(frame, self, 4); + int idx = calcBinSize(self.inputBuffer, offset, 4); Object value = self.memoGet(idx); if (value == null) { @@ -1559,13 +1722,13 @@ private void loadBinPut(VirtualFrame frame, PUnpickler self) { } private void loadLongBinPut(VirtualFrame frame, PUnpickler self) { - ByteArrayView s = read(frame, self, 4); + int offset = read(frame, self, 4); if (self.stack.size <= self.stack.fence) { throw pDataStackRaiseUnderflow(self); } Object value = self.stack.data[self.stack.size - 1]; - int idx = calcBinSize(s, 4); + int idx = calcBinSize(self.inputBuffer, offset, 4); if (idx < 0) { throw raise(PythonBuiltinClassType.ValueError, ErrorMessages.NEG_S_ARG, "LONG_BINPUT"); } @@ -1646,6 +1809,9 @@ private void doSetItems(VirtualFrame frame, PUnpickler self, int x) { Object dict = self.stack.data[x - 1]; final boolean isBuiltinDict = dict instanceof PDict pDict && PGuards.isBuiltinDict(pDict); + if (isBuiltinDict && ((PDict) dict).getDictStorage() == EmptyStorage.INSTANCE) { + ((PDict) dict).setDictStorage(PDict.createNewStorage((len - x) / 2)); + } for (i = x + 1; i < len; i += 2) { key = self.stack.data[i - 1]; value = self.stack.data[i]; @@ -1656,6 +1822,7 @@ private void doSetItems(VirtualFrame frame, PUnpickler self, int x) { } } + LoopNode.reportLoopCount(this, (len - x) / 2); self.stack.clear(x); } @@ -1717,9 +1884,9 @@ private void loadReduce(VirtualFrame frame, PUnpickler self) { } private void loadProto(VirtualFrame frame, PUnpickler self) { - final ByteArrayView s = read(frame, self, 1); + int offset = read(frame, self, 1); - int i = s.getUnsigned(0); + int i = self.inputBuffer[offset] & 0xff; if (i <= PICKLE_PROTOCOL_HIGHEST) { self.proto = i; return; @@ -1729,25 +1896,25 @@ private void loadProto(VirtualFrame frame, PUnpickler self) { } private void loadFrame(VirtualFrame frame, PUnpickler self) { - ByteArrayView s = read(frame, self, 8); + int offset = read(frame, self, 8); - int frameLen = calcBinSize(s, 8); + int frameLen = calcBinSize(self.inputBuffer, offset, 8); if (frameLen < 0) { throw raise(PythonBuiltinClassType.OverflowError, ErrorMessages.S_EXCEEDS_MAX_SIZE_N_BYTES, "FRAME", Integer.MAX_VALUE); } - s = read(frame, self, frameLen); + read(frame, self, frameLen); // Rewind to start of frame self.nextReadIdx -= frameLen; } - private void loadExtension(VirtualFrame frame, BoundaryCallData boundaryCallData, PythonContext ctx, PUnpickler self, int nbytes) { + private void loadExtension(VirtualFrame frame, PythonContext ctx, PUnpickler self, int nbytes) { assert (nbytes == 1 || nbytes == 2 || nbytes == 4); // the nbytes bytes after the opcode - ByteArrayView codebytes = read(frame, self, nbytes); + int codeOffset = read(frame, self, nbytes); // calc_binint returns long - long code = calcBinInt(codebytes, nbytes); + long code = calcBinInt(self.inputBuffer, codeOffset, nbytes); if (code <= 0) { // note that 0 is forbidden @@ -1788,7 +1955,7 @@ private void loadExtension(VirtualFrame frame, BoundaryCallData boundaryCallData } // Load the object. - obj = findClass(frame, boundaryCallData, ctx.getCore(), self, moduleName, className); + obj = findClass(frame, ctx.getCore(), self, moduleName, className); // Cache code -> obj. setDictItem(frame, st.extensionCache, code, obj); @@ -1802,7 +1969,6 @@ private void loadBool(PUnpickler self, boolean bool) { @Specialization public Object load(VirtualFrame frame, PUnpickler self, @Bind Node inliningTarget, - @Cached("createFor($node)") BoundaryCallData boundaryCallData, @Cached GetCachedTpSlotsNode getSlots, @Cached CallSlotTpNewNode callNew, @Cached ExecutePositionalStarargsNode expandArgs, @@ -1828,207 +1994,275 @@ public Object load(VirtualFrame frame, PUnpickler self, } switch (s) { case OPCODE_NONE: + profileSeen(SEEN_NONE); loadNone(self); continue; case OPCODE_BININT: + profileSeen(SEEN_BININT); loadBinInt(frame, self); continue; case OPCODE_BININT1: + profileSeen(SEEN_BININT1); loadBinInt1(frame, self); continue; case OPCODE_BININT2: + profileSeen(SEEN_BININT2); loadBinInt2(frame, self); continue; case OPCODE_INT: + profileSeen(SEEN_INT); loadInt(frame, self); continue; case OPCODE_LONG: + profileSeen(SEEN_LONG); loadLong(frame, self); continue; case OPCODE_LONG1: + profileSeen(SEEN_LONG1); loadCountedLong(frame, self, 1); continue; case OPCODE_LONG4: + profileSeen(SEEN_LONG4); loadCountedLong(frame, self, 4); continue; case OPCODE_FLOAT: + profileSeen(SEEN_FLOAT); loadFloat(frame, self); continue; case OPCODE_BINFLOAT: + profileSeen(SEEN_BINFLOAT); loadBinFloat(frame, self); continue; case OPCODE_SHORT_BINBYTES: + profileSeen(SEEN_SHORT_BINBYTES); loadCountedBinBytes(frame, self, 1); continue; case OPCODE_BINBYTES: + profileSeen(SEEN_BINBYTES); loadCountedBinBytes(frame, self, 4); continue; case OPCODE_BINBYTES8: + profileSeen(SEEN_BINBYTES8); loadCountedBinBytes(frame, self, 8); continue; case OPCODE_BYTEARRAY8: + profileSeen(SEEN_BYTEARRAY8); loadCountedByteArray(frame, self); continue; case OPCODE_NEXT_BUFFER: + profileSeen(SEEN_NEXT_BUFFER); loadNextBuffer(frame, self); continue; case OPCODE_READONLY_BUFFER: + profileSeen(SEEN_READONLY_BUFFER); loadReadOnlyBuffer(frame, self); continue; case OPCODE_SHORT_BINSTRING: + profileSeen(SEEN_SHORT_BINSTRING); loadCountedBinString(frame, self, 1); continue; case OPCODE_BINSTRING: + profileSeen(SEEN_BINSTRING); loadCountedBinString(frame, self, 4); continue; case OPCODE_STRING: + profileSeen(SEEN_STRING); loadString(frame, self); continue; case OPCODE_UNICODE: + profileSeen(SEEN_UNICODE); loadUnicode(frame, self); continue; case OPCODE_SHORT_BINUNICODE: + profileSeen(SEEN_SHORT_BINUNICODE); loadBinCountedUnicode(frame, self, 1); continue; case OPCODE_BINUNICODE: + profileSeen(SEEN_BINUNICODE); loadBinCountedUnicode(frame, self, 4); continue; case OPCODE_BINUNICODE8: + profileSeen(SEEN_BINUNICODE8); loadBinCountedUnicode(frame, self, 8); continue; case OPCODE_EMPTY_TUPLE: + profileSeen(SEEN_EMPTY_TUPLE); loadCountedTuple(self, 0); continue; case OPCODE_TUPLE1: + profileSeen(SEEN_TUPLE1); loadCountedTuple(self, 1); continue; case OPCODE_TUPLE2: + profileSeen(SEEN_TUPLE2); loadCountedTuple(self, 2); continue; case OPCODE_TUPLE3: + profileSeen(SEEN_TUPLE3); loadCountedTuple(self, 3); continue; case OPCODE_TUPLE: + profileSeen(SEEN_TUPLE); loadTuple(self); continue; case OPCODE_EMPTY_LIST: + profileSeen(SEEN_EMPTY_LIST); loadEmptyList(self); continue; case OPCODE_LIST: + profileSeen(SEEN_LIST); loadList(self); continue; case OPCODE_EMPTY_DICT: + profileSeen(SEEN_EMPTY_DICT); loadEmptyDict(self); continue; case OPCODE_DICT: + profileSeen(SEEN_DICT); loadDict(frame, self); continue; case OPCODE_EMPTY_SET: + profileSeen(SEEN_EMPTY_SET); loadEmptySet(self); continue; case OPCODE_ADDITEMS: + profileSeen(SEEN_ADDITEMS); loadAddItems(frame, self); continue; case OPCODE_FROZENSET: + profileSeen(SEEN_FROZENSET); loadFrozenSet(frame, self); continue; case OPCODE_OBJ: + profileSeen(SEEN_OBJ); loadObj(frame, inliningTarget, self, callMethod); continue; case OPCODE_INST: - loadInst(frame, inliningTarget, boundaryCallData, ctx, self, callMethod); + profileSeen(SEEN_INST); + loadInst(frame, inliningTarget, ctx, self, callMethod); continue; case OPCODE_NEWOBJ: + profileSeen(SEEN_NEWOBJ); loadNewObj(frame, inliningTarget, self, getSlots, callNew, expandArgs); continue; case OPCODE_NEWOBJ_EX: + profileSeen(SEEN_NEWOBJ_EX); loadNewObjEx(frame, inliningTarget, self, getSlots, callNew, expandArgs, expandKwargs); continue; case OPCODE_GLOBAL: - loadGlobal(frame, boundaryCallData, ctx, self); + profileSeen(SEEN_GLOBAL); + loadGlobal(frame, ctx, self); continue; case OPCODE_STACK_GLOBAL: - loadStackGlobal(frame, boundaryCallData, ctx, self); + profileSeen(SEEN_STACK_GLOBAL); + loadStackGlobal(frame, ctx, self); continue; case OPCODE_APPEND: + profileSeen(SEEN_APPEND); loadAppend(frame, self); continue; case OPCODE_APPENDS: + profileSeen(SEEN_APPENDS); loadAppends(frame, self); continue; case OPCODE_BUILD: + profileSeen(SEEN_BUILD); loadBuild(frame, self); continue; case OPCODE_DUP: + profileSeen(SEEN_DUP); loadDup(self); continue; case OPCODE_BINGET: + profileSeen(SEEN_BINGET); loadBinGet(frame, self); continue; case OPCODE_LONG_BINGET: + profileSeen(SEEN_LONG_BINGET); loadLongBinGet(frame, self); continue; case OPCODE_GET: + profileSeen(SEEN_GET); loadGet(frame, self); continue; case OPCODE_MARK: + profileSeen(SEEN_MARK); loadMark(self); continue; case OPCODE_BINPUT: + profileSeen(SEEN_BINPUT); loadBinPut(frame, self); continue; case OPCODE_LONG_BINPUT: + profileSeen(SEEN_LONG_BINPUT); loadLongBinPut(frame, self); continue; case OPCODE_PUT: + profileSeen(SEEN_PUT); loadPut(frame, self); continue; case OPCODE_MEMOIZE: + profileSeen(SEEN_MEMOIZE); loadMemoize(self); continue; case OPCODE_POP: + profileSeen(SEEN_POP); loadPop(self); continue; case OPCODE_POP_MARK: + profileSeen(SEEN_POP_MARK); loadPopMark(self); continue; case OPCODE_SETITEM: + profileSeen(SEEN_SETITEM); loadSetItem(frame, self); continue; case OPCODE_SETITEMS: + profileSeen(SEEN_SETITEMS); loadSetItems(frame, self); continue; case OPCODE_PERSID: + profileSeen(SEEN_PERSID); loadPersId(frame, self); continue; case OPCODE_BINPERSID: + profileSeen(SEEN_BINPERSID); loadBinPersId(frame, self); continue; case OPCODE_REDUCE: + profileSeen(SEEN_REDUCE); loadReduce(frame, self); continue; case OPCODE_PROTO: + profileSeen(SEEN_PROTO); loadProto(frame, self); continue; case OPCODE_FRAME: + profileSeen(SEEN_FRAME); loadFrame(frame, self); continue; case OPCODE_EXT1: - loadExtension(frame, boundaryCallData, ctx, self, 1); + profileSeen(SEEN_EXT1); + loadExtension(frame, ctx, self, 1); continue; case OPCODE_EXT2: - loadExtension(frame, boundaryCallData, ctx, self, 2); + profileSeen(SEEN_EXT2); + loadExtension(frame, ctx, self, 2); continue; case OPCODE_EXT4: - loadExtension(frame, boundaryCallData, ctx, self, 4); + profileSeen(SEEN_EXT4); + loadExtension(frame, ctx, self, 4); continue; case OPCODE_NEWTRUE: + profileSeen(SEEN_NEWTRUE); loadBool(self, true); continue; case OPCODE_NEWFALSE: + profileSeen(SEEN_NEWFALSE); loadBool(self, false); continue; case OPCODE_STOP: + profileSeen(SEEN_STOP); break; default: if (0x20 <= s && s <= 0x7e && s != '\'' && s != '\\') { diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PickleModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PickleModuleBuiltins.java index 9b8518028a..539038ee45 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PickleModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PickleModuleBuiltins.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -63,6 +63,8 @@ import com.oracle.graal.python.nodes.function.builtins.clinic.ArgumentClinicProvider; import com.oracle.graal.python.runtime.IndirectCallData.InteropCallData; import com.oracle.graal.python.runtime.object.PFactory; +import com.oracle.truffle.api.CompilerDirectives; +import com.oracle.truffle.api.CompilerDirectives.CompilationFinal; import com.oracle.truffle.api.dsl.Bind; import com.oracle.truffle.api.dsl.Cached; import com.oracle.truffle.api.dsl.GenerateNodeFactory; @@ -80,6 +82,60 @@ protected List> getNodeFa return PickleModuleBuiltinsFactory.getFactories(); } + /** Learns the capacities reached by successful calls at one builtin node. */ + private static final class PicklerSizeProfile { + private static final int SHIFT = 3; + private static final long MAX_ESTIMATE = (16L * 1024 * 1024) << SHIFT; + + @CompilationFinal private long outputEstimate = (long) PickleUtils.WRITE_BUF_SIZE << SHIFT; + @CompilationFinal private long memoEstimate = (long) MemoTable.INITIAL_CAPACITY << SHIFT; + + int outputCapacity() { + return (int) (outputEstimate >> SHIFT); + } + + int memoCapacity() { + int estimate = (int) (memoEstimate >> SHIFT); + return estimate <= MemoTable.INITIAL_CAPACITY ? MemoTable.INITIAL_CAPACITY : Integer.highestOneBit(estimate - 1) << 1; + } + + void report(PPickler pickler) { + if (CompilerDirectives.inInterpreter()) { + if (pickler.getMaxOutputLen() > outputCapacity()) { + outputEstimate = update(outputEstimate, pickler.getMaxOutputLen()); + } + int memoCapacity = pickler.getMemo().capacity(); + if (memoCapacity > (memoEstimate >> SHIFT)) { + memoEstimate = update(memoEstimate, memoCapacity); + } + } + } + + private static long update(long shiftedEstimate, int capacity) { + long estimate = shiftedEstimate >> SHIFT; + return Math.min(shiftedEstimate + capacity - estimate, MAX_ESTIMATE); + } + } + + abstract static class ProfiledPickleDumpNode extends PythonClinicBuiltinNode { + @CompilationFinal private PicklerSizeProfile sizes = new PicklerSizeProfile(); + + final PPickler createPickler(PythonLanguage language) { + return PFactory.createPickler(language, sizes.outputCapacity(), sizes.memoCapacity()); + } + + final void reportSizes(PPickler pickler) { + sizes.report(pickler); + } + + @Override + public Node copy() { + ProfiledPickleDumpNode copy = (ProfiledPickleDumpNode) super.copy(); + copy.sizes = new PicklerSizeProfile(); + return copy; + } + } + @Override public void postInitialize(Python3Core core) { super.postInitialize(core); @@ -95,26 +151,27 @@ public void postInitialize(Python3Core core) { @ArgumentClinic(name = "protocol", conversion = ArgumentClinic.ClinicConversion.Int, defaultValue = J_DEFAULT_PICKLE_PROTOCOL, useDefaultForNone = true) @ArgumentClinic(name = "fix_imports", conversion = ArgumentClinic.ClinicConversion.Boolean, defaultValue = "true") @GenerateNodeFactory - abstract static class PickleDumpNode extends PythonClinicBuiltinNode { + abstract static class PickleDumpNode extends ProfiledPickleDumpNode { @Override protected ArgumentClinicProvider getArgumentClinic() { return PickleModuleBuiltinsClinicProviders.PickleDumpNodeClinicProviderGen.INSTANCE; } @Specialization - static Object dump(VirtualFrame frame, @SuppressWarnings("unused") PythonModule self, Object obj, Object file, int protocol, boolean fixImports, Object bufferCallback, + Object dump(VirtualFrame frame, @SuppressWarnings("unused") PythonModule self, Object obj, Object file, int protocol, boolean fixImports, Object bufferCallback, @Bind Node inliningTarget, @Bind PythonLanguage language, @Cached PPickler.DumpNode dumpNode, @Cached PPickler.FlushToFileNode flushToFileNode, @Cached PyObjectLookupAttr lookup, @Cached PRaiseNode raiseNode) { - PPickler pickler = PFactory.createPickler(language); + PPickler pickler = createPickler(language); pickler.setProtocol(inliningTarget, raiseNode, protocol, fixImports); pickler.setOutputStream(frame, inliningTarget, raiseNode, lookup, file); pickler.setBufferCallback(inliningTarget, raiseNode, bufferCallback); dumpNode.execute(frame, pickler, obj); flushToFileNode.execute(frame, pickler); + reportSizes(pickler); return PNone.NONE; } } @@ -125,23 +182,25 @@ static Object dump(VirtualFrame frame, @SuppressWarnings("unused") PythonModule @ArgumentClinic(name = "protocol", conversion = ArgumentClinic.ClinicConversion.Int, defaultValue = J_DEFAULT_PICKLE_PROTOCOL, useDefaultForNone = true) @ArgumentClinic(name = "fix_imports", conversion = ArgumentClinic.ClinicConversion.Boolean, defaultValue = "true") @GenerateNodeFactory - abstract static class PickleDumpsNode extends PythonClinicBuiltinNode { + abstract static class PickleDumpsNode extends ProfiledPickleDumpNode { @Override protected ArgumentClinicProvider getArgumentClinic() { return PickleModuleBuiltinsClinicProviders.PickleDumpsNodeClinicProviderGen.INSTANCE; } @Specialization - static Object dump(VirtualFrame frame, @SuppressWarnings("unused") PythonModule self, Object obj, int protocol, boolean fixImports, Object bufferCallback, + Object dump(VirtualFrame frame, @SuppressWarnings("unused") PythonModule self, Object obj, int protocol, boolean fixImports, Object bufferCallback, @Bind Node inliningTarget, @Bind PythonLanguage language, @Cached PPickler.DumpNode dumpNode, @Cached PRaiseNode raiseNode) { - PPickler pickler = PFactory.createPickler(language); + PPickler pickler = createPickler(language); pickler.setProtocol(inliningTarget, raiseNode, protocol, fixImports); pickler.setBufferCallback(inliningTarget, raiseNode, bufferCallback); dumpNode.execute(frame, pickler, obj); - return pickler.getString(language); + Object result = pickler.getString(language, false); + reportSizes(pickler); + return result; } } @@ -198,8 +257,8 @@ static Object loads(VirtualFrame frame, @SuppressWarnings("unused") PythonModule @Cached PyObjectGetIter getIter) { try { PUnpickler unpickler = PFactory.createUnpickler(language); - byte[] data = bufferLib.getCopiedByteArray(buffer); - unpickler.setStringInput(data, data.length); + byte[] data = bufferLib.getInternalOrCopiedByteArray(buffer); + unpickler.setStringInput(data, bufferLib.getBufferLength(buffer)); unpickler.setInputEncoding(encoding, errors); unpickler.setBuffers(frame, inliningTarget, getIter, buffers); unpickler.setFixImports(fixImports); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PickleUtils.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PickleUtils.java index 276a5c3d0c..feb9b8f210 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PickleUtils.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PickleUtils.java @@ -40,7 +40,6 @@ */ package com.oracle.graal.python.builtins.modules.pickle; -import static com.oracle.graal.python.builtins.modules.pickle.PPickler.BasePickleWriteNode.NEW_LINE_BYTE; import static com.oracle.graal.python.nodes.StringLiterals.T_DOT; import static com.oracle.graal.python.util.PythonUtils.TS_ENCODING; import static com.oracle.graal.python.util.PythonUtils.toTruffleStringUncached; @@ -53,8 +52,6 @@ import com.oracle.graal.python.PythonLanguage; import com.oracle.graal.python.builtins.PythonBuiltinClassType; import com.oracle.graal.python.builtins.objects.PNone; -import com.oracle.graal.python.builtins.objects.bytes.ByteArrayBuffer; -import com.oracle.graal.python.builtins.objects.bytes.BytesUtils; import com.oracle.graal.python.builtins.objects.method.PBuiltinMethod; import com.oracle.graal.python.builtins.objects.method.PMethod; import com.oracle.graal.python.builtins.objects.str.StringUtils; @@ -62,10 +59,9 @@ import com.oracle.graal.python.nodes.PRaiseNode; import com.oracle.graal.python.nodes.StringLiterals; import com.oracle.graal.python.runtime.object.PFactory; -import com.oracle.graal.python.util.PythonUtils; +import com.oracle.graal.python.util.NumericSupport; import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; import com.oracle.truffle.api.frame.VirtualFrame; -import com.oracle.truffle.api.nodes.ExplodeLoop; import com.oracle.truffle.api.nodes.Node; import com.oracle.truffle.api.strings.TruffleString; @@ -218,37 +214,8 @@ public final class PickleUtils { public static final TruffleString T_PROTO_LE2_TRUE = tsLiteral("I01\n"); public static final TruffleString T_PROTO_LE2_FALSE = tsLiteral("I00\n"); - @ExplodeLoop public static void writeSize64(byte[] out, int start, int value) { - final int sizeofSizeT = Integer.BYTES; - for (int i = 0; i < sizeofSizeT; i++) { - out[start + i] = (byte) ((value >> (8 * i)) & 0xff); - } - for (int i = sizeofSizeT; i < 8; i++) { - out[start + i] = 0; - } - } - - public static byte[] resize(byte[] src, int newSize) { - byte[] newBytes = new byte[newSize]; - if (src != null) { - PythonUtils.arraycopy(src, 0, newBytes, 0, Math.min(newSize, src.length)); - } - return newBytes; - } - - private static int arrayCopyWithNewLine(byte[] dst, int offset, byte[] src) { - PythonUtils.arraycopy(src, 0, dst, offset, src.length); - int len = src.length + 2; - dst[len - 1] = NEW_LINE_BYTE; - return len; - } - - public static int toAsciiBytesWithNewLine(byte[] dst, int offset, long value, TruffleString.FromLongNode fromLongNode, TruffleString.CopyToByteArrayNode copyToByteArrayNode) { - TruffleString s = fromLongNode.execute(value, TruffleString.Encoding.US_ASCII, true); - byte[] buf = new byte[s.byteLength(TruffleString.Encoding.US_ASCII)]; - copyToByteArrayNode.execute(s, 0, buf, 0, buf.length, TruffleString.Encoding.US_ASCII); - return arrayCopyWithNewLine(dst, offset, buf); + NumericSupport.littleEndian().putLong(out, start, Integer.toUnsignedLong(value)); } public static int getStringSize(byte[] bytes) { @@ -295,23 +262,6 @@ public static double asciiBytesToDouble(Node node, byte[] bytes, PythonBuiltinCl } } - public static byte[] encodeUTF8Strict(TruffleString str, TruffleString.SwitchEncodingNode switchEncodingNode, TruffleString.CopyToByteArrayNode copyToByteArrayNode, - TruffleString.GetCodeRangeNode getCodeRangeNode) { - return encodeStrict(str, switchEncodingNode, TruffleString.Encoding.UTF_8, copyToByteArrayNode, getCodeRangeNode); - } - - public static byte[] encodeStrict(TruffleString ts, TruffleString.SwitchEncodingNode switchEncodingNode, TruffleString.Encoding encoding, TruffleString.CopyToByteArrayNode copyToByteArrayNode, - TruffleString.GetCodeRangeNode getCodeRangeNode) { - // TODO: [GR-39571] TruffleStrings: allow preservation of UTF-16 surrogate - if (getCodeRangeNode.execute(ts, TS_ENCODING) == TruffleString.CodeRange.BROKEN) { - return null; - } - TruffleString s = switchEncodingNode.execute(ts, encoding); - byte[] buf = new byte[s.byteLength(encoding)]; - copyToByteArrayNode.execute(s, 0, buf, 0, buf.length, encoding); - return buf; - } - public static TruffleString decodeUTF8Strict(byte[] data, int len, TruffleString.FromByteArrayNode fromByteArrayNode, TruffleString.SwitchEncodingNode switchEncodingNode) { return decodeStrict(data, len, fromByteArrayNode, TruffleString.Encoding.UTF_8, switchEncodingNode); } @@ -326,39 +276,6 @@ private static TruffleString decodeStrict(byte[] data, int len, TruffleString.Fr return switchEncodingNode.execute(ret, TS_ENCODING); } - public static byte[] rawUnicodeEscape(TruffleString unicode, TruffleString.CodePointLengthNode codePointLengthNode, TruffleString.CodePointAtIndexUTF32Node codePointAtIndexNode) { - int len = codePointLengthNode.execute(unicode, TS_ENCODING); - ByteArrayBuffer buffer = new ByteArrayBuffer(len); - for (int i = 0; i < len; i++) { - final int ch = codePointAtIndexNode.execute(unicode, i); - if (ch >= 0x10000) { - // Map 32-bit characters to \Uxxxxxxxx - buffer.append('\\'); - buffer.append('U'); - buffer.append((char) BytesUtils.HEXDIGITS[(ch >> 28) & 0xf]); - buffer.append((char) BytesUtils.HEXDIGITS[(ch >> 24) & 0xf]); - buffer.append((char) BytesUtils.HEXDIGITS[(ch >> 20) & 0xf]); - buffer.append((char) BytesUtils.HEXDIGITS[(ch >> 16) & 0xf]); - buffer.append((char) BytesUtils.HEXDIGITS[(ch >> 12) & 0xf]); - buffer.append((char) BytesUtils.HEXDIGITS[(ch >> 8) & 0xf]); - buffer.append((char) BytesUtils.HEXDIGITS[(ch >> 4) & 0xf]); - buffer.append((char) BytesUtils.HEXDIGITS[ch & 15]); - } else if (ch >= 256 || ch == '\\' || ch == 0 || ch == '\n' || ch == '\r' || ch == 0x1a) { - // Map 16-bit characters, \\ and \n to \Uxxxx - buffer.append('\\'); - buffer.append('u'); - buffer.append((char) BytesUtils.HEXDIGITS[(ch >> 12) & 0xf]); - buffer.append((char) BytesUtils.HEXDIGITS[(ch >> 8) & 0xf]); - buffer.append((char) BytesUtils.HEXDIGITS[(ch >> 4) & 0xf]); - buffer.append((char) BytesUtils.HEXDIGITS[ch & 15]); - } else { - // Copy everything else as-is - buffer.append((char) ch); - } - } - return buffer.getByteArray(); - } - public static Pair initMethodRef(VirtualFrame frame, Node inliningTarget, PyObjectLookupAttr lookup, Object receiver, TruffleString identifier) { final Object func = lookup.execute(frame, inliningTarget, receiver, identifier); if (func == PNone.NO_VALUE) { diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerMemoProxyBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerMemoProxyBuiltins.java index 9b5ad2622f..5d8ea60e57 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerMemoProxyBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerMemoProxyBuiltins.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -53,7 +53,6 @@ import com.oracle.graal.python.builtins.CoreFunctions; import com.oracle.graal.python.builtins.PythonBuiltinClassType; import com.oracle.graal.python.builtins.PythonBuiltins; -import com.oracle.graal.python.builtins.modules.pickle.MemoTable.MemoIterator; import com.oracle.graal.python.builtins.objects.PNone; import com.oracle.graal.python.builtins.objects.dict.PDict; import com.oracle.graal.python.builtins.objects.tuple.PTuple; @@ -106,11 +105,12 @@ Object clear(PPicklerMemoProxy self) { public static PDict picklerMemoCopyImpl(MemoTable memoTable) { PythonLanguage language = PythonLanguage.get(null); LinkedHashMap copy = new LinkedHashMap<>(); - MemoIterator iterator = memoTable.iterator(); - while (iterator.advance()) { - copy.put(System.identityHashCode(iterator.key()), - PFactory.createTuple(language, new Object[]{iterator.value(), iterator.key()})); - + for (int i = 0; i < memoTable.capacity(); i++) { + Object key = memoTable.getKeyAt(i); + if (key != null) { + copy.put(System.identityHashCode(key), + PFactory.createTuple(language, new Object[]{memoTable.getValueAt(i), key})); + } } return PFactory.createDictFromMapGeneric(language, copy); } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java index 8ac0b1339e..3815619a8d 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/pickle/PicklerNodes.java @@ -40,19 +40,11 @@ */ package com.oracle.graal.python.builtins.modules.pickle; -import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.T_CP_IMPORT_MAPPING; -import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.T_CP_NAME_MAPPING; -import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.T_CP_REVERSE_IMPORT_MAPPING; -import static com.oracle.graal.python.builtins.modules.pickle.PickleUtils.T_CP_REVERSE_NAME_MAPPING; import static com.oracle.graal.python.builtins.objects.PNone.NO_VALUE; -import static com.oracle.graal.python.nodes.BuiltinNames.T___MAIN__; -import static com.oracle.graal.python.nodes.ErrorMessages.MUST_BE_STR_NOT_P; import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___CLASS__; -import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___MODULE__; -import static com.oracle.graal.python.nodes.StringLiterals.T_DOT; import static com.oracle.graal.python.nodes.StringLiterals.T_UTF8; -import static com.oracle.graal.python.nodes.statement.AbstractImportNode.importModule; import static com.oracle.graal.python.runtime.exception.PythonErrorType.AttributeError; +import static com.oracle.graal.python.runtime.exception.PythonErrorType.KeyError; import static com.oracle.graal.python.runtime.exception.PythonErrorType.TypeError; import static com.oracle.graal.python.util.PythonUtils.TS_ENCODING; import static com.oracle.graal.python.util.PythonUtils.tsLiteral; @@ -78,7 +70,7 @@ import com.oracle.graal.python.builtins.objects.common.SequenceStorageNodes; import com.oracle.graal.python.builtins.objects.dict.PDict; import com.oracle.graal.python.builtins.objects.function.PKeyword; -import com.oracle.graal.python.builtins.objects.str.StringUtils; +import com.oracle.graal.python.builtins.objects.module.PythonModule; import com.oracle.graal.python.builtins.objects.tuple.PTuple; import com.oracle.graal.python.lib.PyIterCheckNode; import com.oracle.graal.python.lib.PyIterNextNode; @@ -86,7 +78,6 @@ import com.oracle.graal.python.lib.PyNumberAsSizeNode; import com.oracle.graal.python.lib.PyObjectGetItem; import com.oracle.graal.python.lib.PyObjectLookupAttr; -import com.oracle.graal.python.lib.PyObjectReprAsTruffleStringNode; import com.oracle.graal.python.lib.PyObjectSetItem; import com.oracle.graal.python.lib.PyObjectSizeNode; import com.oracle.graal.python.nodes.ErrorMessages; @@ -100,14 +91,21 @@ import com.oracle.graal.python.nodes.object.BuiltinClassProfiles.InlineIsBuiltinClassProfile; import com.oracle.graal.python.nodes.object.BuiltinClassProfiles.IsBuiltinObjectProfile; import com.oracle.graal.python.nodes.object.GetClassNode; +import com.oracle.graal.python.nodes.statement.AbstractImportNode.ImportName; import com.oracle.graal.python.nodes.util.CannotCastException; import com.oracle.graal.python.nodes.util.CastToTruffleStringNode; -import com.oracle.graal.python.runtime.IndirectCallData.BoundaryCallData; import com.oracle.graal.python.runtime.PythonContext; import com.oracle.graal.python.runtime.exception.PException; import com.oracle.graal.python.runtime.object.PFactory; import com.oracle.graal.python.runtime.sequence.storage.SequenceStorage; +import com.oracle.graal.python.util.PythonUtils; import com.oracle.truffle.api.CompilerDirectives; +import com.oracle.truffle.api.dsl.Bind; +import com.oracle.truffle.api.dsl.Cached; +import com.oracle.truffle.api.dsl.Cached.Exclusive; +import com.oracle.truffle.api.dsl.GenerateInline; +import com.oracle.truffle.api.dsl.ImportStatic; +import com.oracle.truffle.api.dsl.Specialization; import com.oracle.truffle.api.frame.Frame; import com.oracle.truffle.api.frame.VirtualFrame; import com.oracle.truffle.api.nodes.LoopNode; @@ -116,9 +114,55 @@ import com.oracle.truffle.api.strings.TruffleString; public final class PicklerNodes { + @GenerateInline(false) + abstract static class ImportModuleNode extends Node { + abstract PythonModule execute(VirtualFrame frame, PythonContext context, TruffleString name); + + @Specialization + static PythonModule doImport(VirtualFrame frame, PythonContext context, TruffleString name, + @Bind Node inliningTarget, + @Cached ImportName importNameNode, + @Cached CachedHashingStorageGetItem getItemNode, + @Cached PRaiseNode raiseNode) { + // ImportName checks initialization for cached modules and honors custom import hooks. + // With an empty fromlist, a dotted import returns the top-level package, so look up + // the requested module by its full name after importing for side effects. + importNameNode.execute(frame, context, context.getBuiltins(), name, PNone.NONE, PythonUtils.EMPTY_TRUFFLESTRING_ARRAY, 0); + Object module = getItemNode.execute(frame, context.getSysModules().getDictStorage(), name); + if (module == null) { + throw raiseNode.raise(inliningTarget, KeyError, new Object[]{name}); + } + if (module instanceof PythonModule pythonModule) { + return pythonModule; + } + throw raiseNode.raise(inliningTarget, PythonBuiltinClassType.NotImplementedError, ErrorMessages.PUTTING_NON_MODULE_OBJECTS_IN_SYS_MODULES_IS_NOT_SUPPORTED); + } + } + + @GenerateInline(false) + @ImportStatic(PythonUtils.class) + abstract static class LookupGlobalAttributeNode extends Node { + abstract Object execute(VirtualFrame frame, Object receiver, TruffleString name); + + // Unpickling decodes fresh strings. Give attribute lookup a stable name identity without + // caching the attribute value or interning arbitrary pickle contents. + @Specialization(guards = "equal.execute(name, cachedName, TS_ENCODING)", limit = "3") + static Object cached(VirtualFrame frame, Object receiver, TruffleString name, + @Cached("name") TruffleString cachedName, + @Cached TruffleString.EqualNode equal, + @Exclusive @Cached(inline = false) PyObjectLookupAttr lookup) { + return lookup.executeCached(frame, receiver, cachedName); + } + + @Specialization(replaces = "cached") + static Object generic(VirtualFrame frame, Object receiver, TruffleString name, + @Exclusive @Cached(inline = false) PyObjectLookupAttr lookup) { + return lookup.executeCached(frame, receiver, name); + } + } + abstract static class BasePickleNode extends Node { private static final TruffleString T_LOCALS = tsLiteral(""); - static final TruffleString T_SYS_MODULES = tsLiteral("sys.modules"); public static final TruffleString T_CODEC_RAW_UNICODE_ESCAPE = tsLiteral("raw_unicode_escape"); public static final TruffleString T_CODEC_BYTES = tsLiteral("bytes"); public static final TruffleString T_CODEC_ASCII = tsLiteral("ascii"); @@ -135,9 +179,8 @@ abstract static class BasePickleNode extends Node { @Child private HashingStorageSetItem setHashingStorageItemNode; @Child private PyObjectSetItem pyObjectSetItemNode; @Child private CachedHashingStorageGetItem getHashingStorageItemNode; - @Child private SequenceStorageNodes.GetItemNode getSeqStorageItemNode; + @Child private SequenceStorageNodes.GetItemScalarNode getSeqStorageItemNode; @Child private PyNumberAsSizeNode asSizeNode; - @Child private CastToTruffleStringNode castToTruffleStringNode; @Child private SequenceNodes.GetSequenceStorageNode getSequenceStorageNode; @Child private CallNode callNode; @Child private ExecutePositionalStarargsNode getArgsNode; @@ -150,24 +193,22 @@ abstract static class BasePickleNode extends Node { @Child private GetClassNode getClassNode; @Child private PyObjectSizeNode sizeNode; @Child private PyObjectLookupAttr lookupAttrNode; + @Child private LookupGlobalAttributeNode lookupGlobalAttrNode; + @Child private ImportModuleNode importModuleNode; @Child private BytesNodes.ToBytesNode toBytesNode; - @Child private PyObjectReprAsTruffleStringNode reprNode; @Child private TruffleString.FromByteArrayNode tsFromByteArrayNode; @Child private TruffleString.FromByteArrayWithCompactionUTF32Node tsFromByteArrayWithCompactionNode; @Child private TruffleString.CodePointLengthNode tsCodePointLengthNode; - @Child private TruffleString.CodePointAtIndexUTF32Node tsCodePointAtIndexUTF32Node; - @Child private TruffleString.FromLongNode tsFromLongNode; - @Child private TruffleString.IndexOfStringNode tsIndexOfStringNode; + @Child private TruffleString.IndexOfCodePointNode tsIndexOfCodePointNode; @Child private TruffleString.SubstringNode tsSubstringNode; @Child private TruffleString.EqualNode tsEqualNode; - @Child private TruffleString.CopyToByteArrayNode tsCopyToByteArrayNode; - @Child private TruffleString.GetCodeRangeNode tsGetCodeRangeNode; @Child private TruffleString.SwitchEncodingNode tsSwitchEncodingNode; + @Child private TruffleString.IsValidNode tsIsValidNode; @Child private HashingStorageGetIterator getHashingStorageIteratorNode; @Child private HashingStorageIteratorNext hashingStorageItNext; @Child private HashingStorageIteratorKey hashingStorageItKey; @Child private HashingStorageIteratorValue hashingStorageItValue; - private final BranchProfile errorProfile = BranchProfile.create(); + protected final BranchProfile errorProfile = BranchProfile.create(); protected PException raise(PythonBuiltinClassType type, TruffleString string) { errorProfile.enter(); @@ -208,28 +249,12 @@ protected TruffleString.CodePointLengthNode ensureTsCodePointLengthNode() { return tsCodePointLengthNode; } - protected TruffleString.CodePointAtIndexUTF32Node ensureTsCodePointAtIndexUTF32Node() { - if (tsCodePointAtIndexUTF32Node == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - tsCodePointAtIndexUTF32Node = insert(TruffleString.CodePointAtIndexUTF32Node.create()); - } - return tsCodePointAtIndexUTF32Node; - } - - protected TruffleString.FromLongNode ensureTsFromLongNode() { - if (tsFromLongNode == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - tsFromLongNode = insert(TruffleString.FromLongNode.create()); - } - return tsFromLongNode; - } - - protected TruffleString.IndexOfStringNode ensureTsIndexOfStringNode() { - if (tsIndexOfStringNode == null) { + protected TruffleString.IndexOfCodePointNode ensureTsIndexOfCodePointNode() { + if (tsIndexOfCodePointNode == null) { CompilerDirectives.transferToInterpreterAndInvalidate(); - tsIndexOfStringNode = insert(TruffleString.IndexOfStringNode.create()); + tsIndexOfCodePointNode = insert(TruffleString.IndexOfCodePointNode.create()); } - return tsIndexOfStringNode; + return tsIndexOfCodePointNode; } protected TruffleString.SubstringNode ensureTsSubstringNode() { @@ -248,22 +273,6 @@ protected TruffleString.EqualNode ensureTsEqualNode() { return tsEqualNode; } - protected TruffleString.CopyToByteArrayNode ensureTsCopyToByteArrayNode() { - if (tsCopyToByteArrayNode == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - tsCopyToByteArrayNode = insert(TruffleString.CopyToByteArrayNode.create()); - } - return tsCopyToByteArrayNode; - } - - protected TruffleString.GetCodeRangeNode ensureTsGetCodeRangeNode() { - if (tsGetCodeRangeNode == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - tsGetCodeRangeNode = insert(TruffleString.GetCodeRangeNode.create()); - } - return tsGetCodeRangeNode; - } - protected TruffleString.SwitchEncodingNode ensureTsSwitchEncodingNode() { if (tsSwitchEncodingNode == null) { CompilerDirectives.transferToInterpreterAndInvalidate(); @@ -272,20 +281,20 @@ protected TruffleString.SwitchEncodingNode ensureTsSwitchEncodingNode() { return tsSwitchEncodingNode; } - protected byte[] toBytes(VirtualFrame frame, Object obj) { - if (toBytesNode == null) { + protected TruffleString.IsValidNode ensureTsIsValidNode() { + if (tsIsValidNode == null) { CompilerDirectives.transferToInterpreterAndInvalidate(); - toBytesNode = insert(BytesNodes.ToBytesNode.create()); + tsIsValidNode = insert(TruffleString.IsValidNode.create()); } - return toBytesNode.execute(frame, obj); + return tsIsValidNode; } - private PyObjectReprAsTruffleStringNode getReprNode() { - if (reprNode == null) { + protected byte[] toBytes(VirtualFrame frame, Object obj) { + if (toBytesNode == null) { CompilerDirectives.transferToInterpreterAndInvalidate(); - reprNode = insert(PyObjectReprAsTruffleStringNode.create()); + toBytesNode = insert(BytesNodes.ToBytesNode.create()); } - return reprNode; + return toBytesNode.execute(frame, obj); } protected HashingStorageIterator getHashingStorageIterator(HashingStorage s) { @@ -296,6 +305,14 @@ protected HashingStorageIterator getHashingStorageIterator(HashingStorage s) { return getHashingStorageIteratorNode.executeCached(s); } + protected PythonModule importModule(VirtualFrame frame, PythonContext context, TruffleString name) { + if (importModuleNode == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + importModuleNode = insert(PicklerNodesFactory.ImportModuleNodeGen.create()); + } + return importModuleNode.execute(frame, context, name); + } + protected HashingStorageIteratorNext ensureHashingStorageIteratorNext() { if (hashingStorageItNext == null) { CompilerDirectives.transferToInterpreterAndInvalidate(); @@ -368,8 +385,12 @@ protected Object decodeASCII(VirtualFrame frame, byte[] bytes, int len, TruffleS return decode(frame, PFactory.createBytes(PythonLanguage.get(this), bytes, len), T_CODEC_ASCII, errors); } - protected Object decodeUTF8(VirtualFrame frame, ByteArrayView bytes, int len, TruffleString errors) { - return decode(frame, PFactory.createBytes(PythonLanguage.get(this), bytes.getBytes(len), len), T_UTF8, errors); + protected Object decodeUTF8(VirtualFrame frame, byte[] bytes, int offset, int len, TruffleString errors) { + TruffleString utf8 = ensureTsFromByteArray().execute(bytes, offset, len, TruffleString.Encoding.UTF_8, true); + if (ensureTsIsValidNode().execute(utf8, TruffleString.Encoding.UTF_8)) { + return ensureTsSwitchEncodingNode().execute(utf8, TS_ENCODING); + } + return decode(frame, PFactory.createBytes(PythonLanguage.get(this), PythonUtils.arrayCopyOfRange(bytes, offset, offset + len)), T_UTF8, errors); } protected Object decode(VirtualFrame frame, Object value, TruffleString encoding) { @@ -428,6 +449,14 @@ protected PyObjectLookupAttr getLookupAttrNode() { return lookupAttrNode; } + protected LookupGlobalAttributeNode getLookupGlobalAttrNode() { + if (lookupGlobalAttrNode == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + lookupGlobalAttrNode = insert(PicklerNodesFactory.LookupGlobalAttributeNodeGen.create()); + } + return lookupGlobalAttrNode; + } + protected Object lookupAttribute(Frame frame, Object receiver, TruffleString name) { return getLookupAttrNode().executeCached(frame, receiver, name); } @@ -477,14 +506,6 @@ protected SequenceStorage getSequenceStorage(Object iterator) { return getSequenceStorageNode.executeCached(iterator); } - protected TruffleString castToString(Object value) { - if (castToTruffleStringNode == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - castToTruffleStringNode = insert(CastToTruffleStringNode.create()); - } - return castToTruffleStringNode.executeCached(value); - } - public int asSizeExact(VirtualFrame frame, Object pyNumber) { if (asSizeNode == null) { CompilerDirectives.transferToInterpreterAndInvalidate(); @@ -556,12 +577,12 @@ public Object getNextItem(VirtualFrame frame, Object iterator) { return getNextNode.executeCached(frame, iterator); } - public Object getItem(VirtualFrame frame, SequenceStorage storage, int i) { + public Object getItem(SequenceStorage storage, int i) { if (getSeqStorageItemNode == null) { CompilerDirectives.transferToInterpreterAndInvalidate(); - getSeqStorageItemNode = insert(SequenceStorageNodes.GetItemNode.create()); + getSeqStorageItemNode = insert(SequenceStorageNodes.GetItemScalarNode.create()); } - return getSeqStorageItemNode.execute(frame, storage, i); + return getSeqStorageItemNode.executeCached(storage, i); } public Object getItem(VirtualFrame frame, Object obj, Object slice) { @@ -587,24 +608,12 @@ public TruffleString asString(Object value) { } } - public TruffleString asStringStrict(Object value) { - if (value instanceof TruffleString) { - return (TruffleString) value; - } else { - try { - return toStringNode.executeCached(value); - } catch (CannotCastException e) { - throw raise(PythonBuiltinClassType.TypeError, MUST_BE_STR_NOT_P, value); - } - } - } - - public static Pair getDeepAttribute(VirtualFrame frame, PyObjectLookupAttr lookup, Object obj, TruffleString[] names) { + public static Pair getDeepAttribute(VirtualFrame frame, LookupGlobalAttributeNode lookup, Object obj, TruffleString[] names) { Object parent = null; Object object = obj; for (int i = 0; i < names.length; i++) { parent = object; - object = lookup.executeCached(frame, parent, names[i]); + object = lookup.execute(frame, parent, names[i]); if (object == NO_VALUE) { LoopNode.reportLoopCount(lookup, i); return null; @@ -614,9 +623,41 @@ public static Pair getDeepAttribute(VirtualFrame frame, PyObject return Pair.create(object, parent); } + private TruffleString[] splitDottedPath(TruffleString name) { + int length = ensureTsCodePointLengthNode().execute(name, TS_ENCODING); + TruffleString.IndexOfCodePointNode indexOf = ensureTsIndexOfCodePointNode(); + int firstDot = length == 0 ? -1 : indexOf.execute(name, '.', 0, length, TS_ENCODING); + if (firstDot < 0) { + return new TruffleString[]{name}; + } + + // Count the components so that we can allocate the result without growing or copying it. + int parts = 2; + int start = firstDot + 1; + while (start < length) { + int dot = indexOf.execute(name, '.', start, length, TS_ENCODING); + if (dot < 0) { + break; + } + parts++; + start = dot + 1; + } + LoopNode.reportLoopCount(this, parts - 2 + (start < length ? 1 : 0)); + TruffleString[] dottedPath = new TruffleString[parts]; + TruffleString.SubstringNode substring = ensureTsSubstringNode(); + start = 0; + int part = 0; + for (int dot = firstDot; dot >= 0; dot = start < length ? indexOf.execute(name, '.', start, length, TS_ENCODING) : -1) { + dottedPath[part++] = substring.execute(name, start, dot - start, TS_ENCODING, false); + start = dot + 1; + } + LoopNode.reportLoopCount(this, part); + dottedPath[part] = substring.execute(name, start, length - start, TS_ENCODING, false); + return dottedPath; + } + public TruffleString[] getDottedPath(Object obj, TruffleString name) { - TruffleString[] dottedPath = StringUtils.split(name, T_DOT, ensureTsCodePointLengthNode(), ensureTsIndexOfStringNode(), ensureTsSubstringNode(), ensureTsEqualNode()); - assert dottedPath.length > 0; + TruffleString[] dottedPath = splitDottedPath(name); for (int i = 0; i < dottedPath.length; i++) { if (ensureTsEqualNode().execute(dottedPath[i], T_LOCALS, TS_ENCODING)) { if (obj == null) { @@ -630,40 +671,17 @@ public TruffleString[] getDottedPath(Object obj, TruffleString name) { return dottedPath; } - public Object getattribute(VirtualFrame frame, Object obj, TruffleString name, boolean allowQualname) { - if (allowQualname) { - TruffleString[] dottedPath = getDottedPath(obj, name); - Pair result = getDeepAttribute(frame, getLookupAttrNode(), obj, dottedPath); - if (result != null) { - return result.getLeft(); - } else { - throw raise(AttributeError, ErrorMessages.CANT_GET_ATTRIBUTE_S_ON_S, name, getReprNode().execute(frame, null, obj)); - } - } else { - return lookupAttributeStrict(frame, obj, name); - } - } - - protected Pair get3to2Mapping(VirtualFrame frame, Python3Core core, TruffleString moduleName, TruffleString globalName) { - PickleState state = getGlobalState(core); - return getMapping(frame, state.nameMapping3To2, state.importMapping3To2, T_CP_REVERSE_NAME_MAPPING, T_CP_REVERSE_IMPORT_MAPPING, moduleName, globalName); - } - - protected Pair get2To3Mapping(VirtualFrame frame, Python3Core core, TruffleString moduleName, TruffleString globalName) { - PickleState state = getGlobalState(core); - return getMapping(frame, state.nameMapping2To3, state.importMapping2To3, T_CP_NAME_MAPPING, T_CP_IMPORT_MAPPING, moduleName, globalName); - } - - private Pair getMapping(VirtualFrame frame, PDict nameMapping, PDict importMapping, + protected Pair getMapping(VirtualFrame frame, PDict nameMapping, PDict importMapping, TruffleString nameMappingLabel, TruffleString importMappingLabel, TruffleString moduleName, TruffleString globalName) { Object key = PFactory.createTuple(PythonLanguage.get(this), new Object[]{moduleName, globalName}); Object item = getDictItem(frame, nameMapping, key); if (item != null) { - if (!(item instanceof PTuple) || length(frame, item) != 2) { + if (!(item instanceof PTuple) || ((PTuple) item).getSequenceStorage().length() != 2) { throw raise(PythonBuiltinClassType.RuntimeError, ErrorMessages.S_SHOULD_BE_S_NOT_P, nameMappingLabel, "2-tuples", item); } - Object mappedModuleName = getItem(frame, item, 0); - Object mappedGlobalName = getItem(frame, item, 1); + SequenceStorage storage = ((PTuple) item).getSequenceStorage(); + Object mappedModuleName = getItem(storage, 0); + Object mappedGlobalName = getItem(storage, 1); if (!PGuards.isString(mappedModuleName) || !PGuards.isString(mappedGlobalName)) { throw raise(PythonBuiltinClassType.RuntimeError, ErrorMessages.S_SHOULD_BE_S_NOT_P_P, nameMappingLabel, "str", mappedModuleName, mappedGlobalName); } @@ -681,75 +699,5 @@ private Pair getMapping(VirtualFrame frame, PDict return Pair.create(moduleName, globalName); } - - public Object findClass(VirtualFrame frame, BoundaryCallData boundaryCallData, Python3Core core, PUnpickler self, Object moduleName, Object globalName) { - return findClass(frame, boundaryCallData, core, self, castToString(moduleName), castToString(globalName)); - } - - public Object findClass(VirtualFrame frame, BoundaryCallData boundaryCallData, Python3Core core, PUnpickler self, TruffleString moduleName, TruffleString globalName) { - // Try to map the old names used in Python 2.x to the new ones used in Python 3.x. We do - // this only with old pickle protocols and when the user has not disabled the feature. - TruffleString mName = moduleName; - TruffleString gName = globalName; - if (self.getProto() < 3 && self.isFixImports()) { - final Pair to3Mapping = get2To3Mapping(frame, core, mName, gName); - mName = to3Mapping.getLeft(); - gName = to3Mapping.getRight(); - } - - // we don't use PyImport_GetModule here, because it can return partially-initialised - // modules, which then cause the getattribute to fail. - Object module = importModule(frame, boundaryCallData, mName); - return getattribute(frame, module, gName, self.getProto() >= 4); - } - - private static boolean checkModule(VirtualFrame frame, PyObjectLookupAttr lookup, Object moduleName, Object module, Object global, TruffleString[] dottedPath) { - if (module == PNone.NONE) { - return false; - } - - assert !(moduleName instanceof String) : "moduleName shouldn't be j.l.String"; - - if (moduleName instanceof TruffleString && ((TruffleString) moduleName).equalsUncached(T___MAIN__, TS_ENCODING)) { - return false; - } - final Pair pair = getDeepAttribute(frame, lookup, module, dottedPath); - if (pair == null) { - return false; - } - return pair.getLeft() == global; - } - - public TruffleString whichModule(VirtualFrame frame, PythonContext context, Object global, TruffleString[] dottedPath) { - Object moduleName = lookupAttribute(frame, global, T___MODULE__); - if (moduleName != NO_VALUE) { - // In some rare cases (e.g., bound methods of extension types), __module__ can be - // None. If it is so, then search sys.modules for the module of global - if (moduleName != PNone.NONE) { - return asStringStrict(moduleName); - } - } - - // Fallback on walking sys.modules - final PDict sysModules = context.getSysModules(); - if (sysModules == null) { - throw raise(PythonBuiltinClassType.RuntimeError, ErrorMessages.UNABLE_TO_GET_S, T_SYS_MODULES); - } - - final HashingStorage storage = sysModules.getDictStorage(); - HashingStorageIterator it = getHashingStorageIterator(storage); - HashingStorageIteratorNext nextNode = ensureHashingStorageIteratorNext(); - HashingStorageIteratorKey getKeyNode = ensureHashingStorageIteratorKey(); - HashingStorageIteratorValue getValueNode = ensureHashingStorageIteratorValue(); - while (nextNode.executeCached(storage, it)) { - Object value = getValueNode.executeCached(storage, it); - Object key = getKeyNode.executeCached(storage, it); - if (checkModule(frame, getLookupAttrNode(), moduleName, value, global, dottedPath)) { - return asStringStrict(key); - } - } - // If no module is found, use __main__ - return T___MAIN__; - } } } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/ints/IntNodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/ints/IntNodes.java index 4cd53ffff2..5767768938 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/ints/IntNodes.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/ints/IntNodes.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -181,10 +181,14 @@ static byte[] doPInt(Node inliningTarget, PInt value, int size, boolean bigEndia @GenerateInline(inlineByDefault = true) @GenerateUncached public abstract static class PyLongFromByteArray extends Node { - public abstract Object execute(Node inliningTarget, byte[] data, boolean littleEndian, boolean signed); + public abstract Object execute(Node inliningTarget, byte[] data, int offset, int length, boolean littleEndian, boolean signed); - public final Object executeCached(byte[] data, boolean littleEndian, boolean signed) { - return execute(this, data, littleEndian, signed); + public final Object execute(Node inliningTarget, byte[] data, boolean littleEndian, boolean signed) { + return execute(inliningTarget, data, 0, data.length, littleEndian, signed); + } + + public final Object executeCached(byte[] data, int offset, int length, boolean littleEndian, boolean signed) { + return execute(this, data, offset, length, littleEndian, signed); } public static Object executeUncached(byte[] data, boolean littleEndian, boolean signed) { @@ -192,37 +196,65 @@ public static Object executeUncached(byte[] data, boolean littleEndian, boolean } @Specialization - static Object doOther(Node inliningTarget, byte[] data, boolean littleEndian, boolean signed, + static Object doOther(Node inliningTarget, byte[] data, int offset, int length, boolean littleEndian, boolean signed, @Cached InlinedBranchProfile fastPath1, @Cached InlinedBranchProfile fastPath2, @Cached InlinedBranchProfile fastPath4, @Cached InlinedBranchProfile fastPath8, + @Cached InlinedBranchProfile variablePath, + @Cached InlinedBranchProfile fastPathVariable, + @Cached InlinedBranchProfile overlong, @Cached InlinedBranchProfile generic, @Cached PRaiseNode raiseNode) { NumericSupport support = littleEndian ? NumericSupport.littleEndian() : NumericSupport.bigEndian(); if (signed) { - switch (data.length) { + switch (length) { case 1 -> { fastPath1.enter(inliningTarget); - return (int) support.getByte(data, 0); + return (int) support.getByte(data, offset); } case 2 -> { fastPath2.enter(inliningTarget); - return (int) support.getShort(data, 0); + return (int) support.getShort(data, offset); } case 4 -> { fastPath4.enter(inliningTarget); - return support.getInt(data, 0); + return support.getInt(data, offset); } case 8 -> { fastPath8.enter(inliningTarget); - return support.getLong(data, 0); + return support.getLong(data, offset); } } } + variablePath.enter(inliningTarget); + int remaining = length; + int index = littleEndian ? offset + length - 1 : offset; + int step = littleEndian ? -1 : 1; + boolean negative = signed && length > 0 && data[index] < 0; + if (remaining > Long.BYTES) { + overlong.enter(inliningTarget); + // Ignore redundant sign or zero extension beyond the width of a long. + byte padding = (byte) (negative ? -1 : 0); + while (remaining > Long.BYTES && data[index] == padding) { + index += step; + remaining--; + } + } + // For eight bytes, the remaining sign bit must agree with the original sign. + // This also excludes unsigned values greater than Long.MAX_VALUE. + if (remaining < Long.BYTES || (remaining == Long.BYTES && (data[index] < 0) == negative)) { + fastPathVariable.enter(inliningTarget); + long value = negative ? -1L : 0L; + for (int i = 0; i < remaining; i++) { + value = (value << Byte.SIZE) | (data[index] & 0xffL); + index += step; + } + return value; + } generic.enter(inliningTarget); try { - BigInteger integer = support.getBigInteger(data, signed); + BigInteger integer = support.getBigInteger(data, offset, length, signed); if (PInt.bigIntegerFitsInLong(integer)) { long longValue = PInt.longValue(integer); return PInt.isIntRange(longValue) ? (int) longValue : longValue; diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/PGuards.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/PGuards.java index 2f2a5daa95..f86db3fc36 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/PGuards.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/PGuards.java @@ -309,6 +309,10 @@ public static boolean isBuiltinPInt(PInt obj) { return obj.getPythonClass() == PythonBuiltinClassType.PInt; } + public static boolean isBuiltinPFloat(PFloat obj) { + return obj.getPythonClass() == PythonBuiltinClassType.PFloat; + } + public static boolean isPString(Object obj) { return obj instanceof PString; } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/object/PFactory.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/object/PFactory.java index 35e58a8b29..989b5a5c52 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/object/PFactory.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/object/PFactory.java @@ -1579,8 +1579,8 @@ public static PPickleBuffer createPickleBuffer(Object view, Object cls, Shape sh return new PPickleBuffer(cls, shape, view); } - public static PPickler createPickler(PythonLanguage language) { - return createPickler(PythonBuiltinClassType.Pickler, PythonBuiltinClassType.Pickler.getInstanceShape(language)); + public static PPickler createPickler(PythonLanguage language, int outputCapacity, int memoCapacity) { + return new PPickler(PythonBuiltinClassType.Pickler, PythonBuiltinClassType.Pickler.getInstanceShape(language), outputCapacity, memoCapacity); } public static PPickler createPickler(Object cls, Shape shape) {