From efa4fec205422042d3f48386ed0fd366850d3cf2 Mon Sep 17 00:00:00 2001 From: Harold Cindy <120691094+HaroldCindy@users.noreply.github.com> Date: Wed, 16 Sep 2026 22:35:34 -0700 Subject: [PATCH] Add extension masks to improve forward / backwards compatibility This is much less error-prone than juggling minor versions and whatnot. It also lets you only reject a newer payload if it actually _uses_ a feature that you don't know about (for example, coroutine finalizers). Also, switch to a separate string table and use varints to greatly reduce the size of ares payloads. --- ARES.bt | 194 ++++-- Executor/include/Luau/Script.h | 6 +- Executor/src/Script.cpp | 24 + VM/include/lua.h | 10 +- VM/src/ares.cpp | 558 +++++++++++------- tests/SLExecutor.test.cpp | 31 +- tests/SLGoldenFixtures.test.cpp | 8 + tests/conformance/ares.lua | 11 + tests/conformance/ares_errors.lua | 26 + ...exec2.0-ares6.0-between-handlers.lsl.state | Bin 587 -> 0 bytes ...exec2.0-ares6.0-between-handlers.lua.state | Bin 1459 -> 0 bytes .../exec2.0-ares6.0-errored-handler.lua.state | Bin 948 -> 0 bytes .../exec2.0-ares6.0-errored-main.lsl.state | Bin 86538 -> 0 bytes ...2.0-ares6.0-state-change-pending.lsl.state | Bin 668 -> 0 bytes .../exec2.0-ares6.0-yielded-handler.lsl.state | Bin 658 -> 0 bytes .../exec2.0-ares6.0-yielded-handler.lua.state | Bin 1149 -> 0 bytes .../exec2.0-ares6.0-yielded-main.lua.state | Bin 725 -> 0 bytes ...xec3.0-ares7.0-between-handlers.lsl.sluac} | Bin ...exec3.0-ares7.0-between-handlers.lsl.state | Bin 0 -> 459 bytes ...xec3.0-ares7.0-between-handlers.lua.sluac} | Bin ...exec3.0-ares7.0-between-handlers.lua.state | Bin 0 -> 1038 bytes ...exec3.0-ares7.0-errored-handler.lua.sluac} | Bin .../exec3.0-ares7.0-errored-handler.lua.state | Bin 0 -> 758 bytes ...=> exec3.0-ares7.0-errored-main.lsl.sluac} | Bin .../exec3.0-ares7.0-errored-main.lsl.state | Bin 0 -> 53004 bytes ....0-ares7.0-state-change-pending.lsl.sluac} | Bin ...3.0-ares7.0-state-change-pending.lsl.state | Bin 0 -> 525 bytes ...exec3.0-ares7.0-yielded-handler.lsl.sluac} | Bin .../exec3.0-ares7.0-yielded-handler.lsl.state | Bin 0 -> 477 bytes ...exec3.0-ares7.0-yielded-handler.lua.sluac} | Bin .../exec3.0-ares7.0-yielded-handler.lua.state | Bin 0 -> 829 bytes ...=> exec3.0-ares7.0-yielded-main.lua.sluac} | Bin .../exec3.0-ares7.0-yielded-main.lua.state | Bin 0 -> 550 bytes 33 files changed, 576 insertions(+), 292 deletions(-) delete mode 100644 tests/conformance/fixtures/exec/exec2.0-ares6.0-between-handlers.lsl.state delete mode 100644 tests/conformance/fixtures/exec/exec2.0-ares6.0-between-handlers.lua.state delete mode 100644 tests/conformance/fixtures/exec/exec2.0-ares6.0-errored-handler.lua.state delete mode 100644 tests/conformance/fixtures/exec/exec2.0-ares6.0-errored-main.lsl.state delete mode 100644 tests/conformance/fixtures/exec/exec2.0-ares6.0-state-change-pending.lsl.state delete mode 100644 tests/conformance/fixtures/exec/exec2.0-ares6.0-yielded-handler.lsl.state delete mode 100644 tests/conformance/fixtures/exec/exec2.0-ares6.0-yielded-handler.lua.state delete mode 100644 tests/conformance/fixtures/exec/exec2.0-ares6.0-yielded-main.lua.state rename tests/conformance/fixtures/exec/{exec2.0-ares6.0-between-handlers.lsl.sluac => exec3.0-ares7.0-between-handlers.lsl.sluac} (100%) create mode 100644 tests/conformance/fixtures/exec/exec3.0-ares7.0-between-handlers.lsl.state rename tests/conformance/fixtures/exec/{exec2.0-ares6.0-between-handlers.lua.sluac => exec3.0-ares7.0-between-handlers.lua.sluac} (100%) create mode 100644 tests/conformance/fixtures/exec/exec3.0-ares7.0-between-handlers.lua.state rename tests/conformance/fixtures/exec/{exec2.0-ares6.0-errored-handler.lua.sluac => exec3.0-ares7.0-errored-handler.lua.sluac} (100%) create mode 100644 tests/conformance/fixtures/exec/exec3.0-ares7.0-errored-handler.lua.state rename tests/conformance/fixtures/exec/{exec2.0-ares6.0-errored-main.lsl.sluac => exec3.0-ares7.0-errored-main.lsl.sluac} (100%) create mode 100644 tests/conformance/fixtures/exec/exec3.0-ares7.0-errored-main.lsl.state rename tests/conformance/fixtures/exec/{exec2.0-ares6.0-state-change-pending.lsl.sluac => exec3.0-ares7.0-state-change-pending.lsl.sluac} (100%) create mode 100644 tests/conformance/fixtures/exec/exec3.0-ares7.0-state-change-pending.lsl.state rename tests/conformance/fixtures/exec/{exec2.0-ares6.0-yielded-handler.lsl.sluac => exec3.0-ares7.0-yielded-handler.lsl.sluac} (100%) create mode 100644 tests/conformance/fixtures/exec/exec3.0-ares7.0-yielded-handler.lsl.state rename tests/conformance/fixtures/exec/{exec2.0-ares6.0-yielded-handler.lua.sluac => exec3.0-ares7.0-yielded-handler.lua.sluac} (100%) create mode 100644 tests/conformance/fixtures/exec/exec3.0-ares7.0-yielded-handler.lua.state rename tests/conformance/fixtures/exec/{exec2.0-ares6.0-yielded-main.lua.sluac => exec3.0-ares7.0-yielded-main.lua.sluac} (100%) create mode 100644 tests/conformance/fixtures/exec/exec3.0-ares7.0-yielded-main.lua.state diff --git a/ARES.bt b/ARES.bt index 929f2ab7..2a88730e 100644 --- a/ARES.bt +++ b/ARES.bt @@ -108,6 +108,48 @@ struct CallInfo; struct ExecString; +/* Counts, sizes and indices are the unsigned LEB128 varints Luau bytecode + * uses, see write_uvarint() in ares.cpp. `v` is the decoded value. */ +typedef struct { + local uint64 v = 0; + local int shift = 0; + local uchar cur; + do { + cur = ReadUByte(FTell()); + uchar raw; + v |= (uint64)(cur & 0x7F) << shift; + shift += 7; + } while (cur & 0x80); +} UVarint ; + +/* An `int` field: the same encoding, holding the 32-bit two's complement. */ +typedef struct { + local uint64 v = 0; + local int shift = 0; + local uchar cur; + do { + cur = ReadUByte(FTell()); + uchar raw; + v |= (uint64)(cur & 0x7F) << shift; + shift += 7; + } while (cur & 0x80); +} IntVarint ; + +/* A genuinely signed field: 64-bit zigzag varint, see write_ares_sint() in + * ares.cpp. Integer payloads and the two signed CallInfo fields use it. */ +typedef struct { + local uint64 v = 0; + local int shift = 0; + local uchar cur; + do { + cur = ReadUByte(FTell()); + uchar raw; + v |= (uint64)(cur & 0x7F) << shift; + shift += 7; + } while (cur & 0x80); + local int64 s = (int64)(v >> 1) ^ -(int64)(v & 1); +} SVarint ; + // From LuauBytecode.bt typedef enum { // NOP: noop @@ -502,6 +544,10 @@ typedef uint32 Instruction ; local uint64 refNum = 0; // tracks where we saw each reference local uint64 refPositions[0xFFFF] = {0}; +// string table entries parsed so far, and where each one's bytes start +local uint64 strNum = 0; +local uint64 strPositions[0xFFFF] = {0}; +local uint64 strLengths[0xFFFF] = {0}; /* Closes a length-prefixed record: anything left before record_end is test * padding or fields a newer minor appended, both of which the reader skips. @@ -520,20 +566,22 @@ typedef struct { /* Mirrors ARES_FORMAT_MAJOR in VM/include/lua.h. Any minor under it * parses: fields are only ever appended inside records, and each record's * length word steps over what this template doesn't know. */ - Assert(major == 6); + Assert(major == 7); uint32_t record_len; local int64 record_end = FTell() + record_len; uint8_t sizeof_number; /* sizeof(lua_Number) to check type compatibility */ lua_Number test; /* -1.234567890 to check representation compatibility */ - uint8_t sizeof_int; /* sizeof(int) in persisted data */ - uint8_t sizeof_size_t; /* sizeof(size_t) in persisted data */ uint8_t vector_components; /* how many components are stored in vectors */ - /* Note that the last two fields determine the size of the int and size_t - * fields in the following definitions. We write each value in the native - * "size" and check for truncation when reading, if necessary. */ - /* Reserved when the header is written and patched in after the root - * object, see p_header_refcount() in ares.cpp. */ + /* The root is serialized to a scratch buffer before the header is + * written, so this is final. Strings do not count. */ uint32_t final_refcount; + /* Every feature bit the writer knows, i.e. every appended field whose + * bytes are in this stream's records. Mirrors kAresFeaturesSupported in + * ares.cpp. */ + uint64_t features_present; + /* The subset a reader has to understand: what populated fields asked for + * through require_feature(). An unknown bit here refuses the stream. */ + uint64_t features_required; ParseRecordEnd(record_end); } Header ; @@ -544,6 +592,10 @@ void ParseGCHeader() { } string ReadObject(Object &o) { + if (o.type == ARES_T_STRING) { + return Str("%s $%Lu \"%s\"", EnumToString(o.type), o.val.v, + ReadString(strPositions[o.val.v], strLengths[o.val.v])); + } // ref should be non-zero if present if (o.ourRef) { // ARES_T_REFERENCE refNums are references _to_ something @@ -571,6 +623,8 @@ typedef struct { case ARES_T_VECTOR: // storing a reference to a reference? no. case ARES_T_REFERENCE: + // strings are indexes into the string table, not references + case ARES_T_STRING: break; // Permanents take a number too: persist_keyed() allocates it before // the permanents table gets a look, and u_permanent() reserves it @@ -604,13 +658,13 @@ typedef struct { case ARES_T_NIL: break; case ARES_T_BOOLEAN: - int32_t val; break; + IntVarint val; break; case ARES_T_LIGHTUSERDATA: LightUserdata val; break; case ARES_T_NUMBER: Number val; break; case ARES_T_INTEGER: - int64_t val; break; + SVarint val; break; case ARES_T_VECTOR: float val[3]; break; /* Reserved but never emitted; p_vector refuses to write f64 vectors @@ -619,7 +673,8 @@ typedef struct { ParseGCHeader(); double val[3]; break; case ARES_T_STRING: - String val; break; + /* 1-based index into the string table ahead of the root */ + UVarint val ; break; case ARES_T_TABLE: Table val; break; case ARES_T_FUNCTION: @@ -643,8 +698,8 @@ typedef struct { /* Note that the types LUA_TNIL, LUA_TBOOLEAN, LUA_TNUMBER and * LUA_TLIGHTUSERDATA will never be "referenced", but always be written * directly. */ - uint32 reference ; /* The index the object was referenced with */ - ourRef = reference; + IntVarint reference ; /* The index the object was referenced with */ + ourRef = reference.v; break; case ARES_T_CLASS: case ARES_T_OBJECT: @@ -658,28 +713,45 @@ typedef struct { ParseRecordEnd(record_end); } Object; +/* A string table entry. Every string the root reaches, in first-sight order. */ typedef struct { ParseGCHeader(); - size_t length; /* The length of the string */ - char str[length]; /* The actual string (not always null terminated) */ + UVarint length; /* The length of the string */ + /* Where the bytes start, whether or not there are any */ + local int64 str_start = FTell(); + char str[length.v]; /* The actual string (not always null terminated) */ + strNum++; + strPositions[strNum] = str_start; + strLengths[strNum] = length.v; } String ; +typedef struct { + uint32_t record_len ; + local int64 record_end = FTell() + record_len; + UVarint count; + local int i; + for (i = 0; i < count.v; ++i) { + String entry; + } + ParseRecordEnd(record_end); +} StrTab ; + typedef struct { ParseGCHeader(); - size_t length; /* The length of the buffer */ - char data[length]; /* The actual buffer data */ + UVarint length; /* The length of the buffer */ + char data[length.v]; /* The actual buffer data */ } Buffer ; struct Table { ParseGCHeader(); uint8_t read_only; uint8_t safe_env; - int array_size; - int node_size; + IntVarint array_size; + IntVarint node_size; local int i; // This serialization format preserves `nil` holes, so we can't use `nil` as a // terminator. - for (i=0; i<(array_size+node_size); ++i) { + for (i=0; i<(array_size.v+node_size.v); ++i) { /* key/value pairs */ struct Pair { /* Both of these may legally be `nil` due to hole preservation */ @@ -704,7 +776,7 @@ typedef struct { } case UTAG_DETECTED_EVENT: { - int32_t index; + IntVarint index; uint8_t valid; uint8_t can_change_damage; break; @@ -723,16 +795,16 @@ typedef struct { } case UTAG_STRBUF: { - size_t capacity; - size_t used; - uchar data[used]; + UVarint capacity; + UVarint used; + uchar data[used.v]; break; } case UTAG_OPAQUE_BUFFER: default: { - size_t length; /* Size of the data */ - uchar data[length]; /* The actual data */ + UVarint length; /* Size of the data */ + uchar data[length.v]; /* The actual data */ } } Object metatable; /* The metatable (nil for none, otherwise LUA_TTABLE) */ @@ -740,7 +812,7 @@ typedef struct { typedef struct { uint8_t lutag; /* Userdata tag */ - size_t ptr; + uint64_t ptr; /* Pointer-sized, so fixed width rather than a varint */ } LightUserdata ; struct Closure { @@ -774,7 +846,7 @@ struct Proto { Object source; /* Textual source for the function, string or nil */ - int bytecode_id; + IntVarint bytecode_id; uint8_t maxstacksize; /* Size of stack reserved for the function */ uint8_t flags; /* Flags related to the function definition */ @@ -782,16 +854,16 @@ struct Proto { uint8_t nups; /* Number of upvalues */ uint8_t is_vararg; /* 1 if function accepts varargs, 0 otherwise */ - int sizecode; /* Number of instructions in code */ - Instruction code[sizecode]; /* The proto's code */ + IntVarint sizecode; /* Number of instructions in code */ + Instruction code[sizecode.v]; /* The proto's code */ - int sizek; /* Number of constants referenced */ - Object k[sizek]; /* Constants referenced */ + IntVarint sizek; /* Number of constants referenced */ + Object k[sizek.v]; /* Constants referenced */ - int sizep; /* Number of inner Protos referenced */ - Object p[sizep]; /* Inner Protos referenced */ + IntVarint sizep; /* Number of inner Protos referenced */ + Object p[sizep.v]; /* Inner Protos referenced */ - int linedefined; /* Start of line range */ + IntVarint linedefined; /* Start of line range */ Object debugname; /* Name of the function for debugging, string or nil */ // Neither of these are supported yet. @@ -817,53 +889,56 @@ struct Proto { Object upvalnames[sizeupvalues]; /* Upvalue names */ } - int sizeyieldpoints; - int yieldpoints[sizeyieldpoints]; + IntVarint sizeyieldpoints; + local int yp; + for (yp = 0; yp < sizeyieldpoints.v; ++yp) { + IntVarint yieldpoint; + } }; struct Thread { ParseGCHeader(); Object env; - uint32_t stacksize; /* Allocated stack slots, restored as-is */ - size_t top; /* top = L->top - L->stack; */ - Object stack[top]; /* All stack values, bottom up */ + UVarint stacksize; /* Allocated stack slots, restored as-is */ + UVarint top; /* top = L->top - L->stack; */ + Object stack[top.v]; /* All stack values, bottom up */ AresStatus status; /* current thread status (ok, yield) */ uint8_t activememcat ; Object namecall; /* The pending namecall string, nil for none */ - uint32_t size_ci; /* Allocated callinfo slots, never below BASIC_CI_SIZE */ - uint32_t num_cis; /* number of callinfo frames */ + UVarint size_ci; /* Allocated callinfo slots, never below BASIC_CI_SIZE */ + UVarint num_cis; /* number of callinfo frames */ /* The CallInfo stack, starting with base_ci. Each frame is its own record. */ struct CallInfo { uint32_t record_len ; local int64 record_end = FTell() + record_len; - size_t func; /* func = ci->func - thread->stack */ - size_t top; /* top = ci->top - thread->stack */ - size_t base; /* base = ci->base - thread-stack */ - int32_t nresults; /* expected number of results from this function */ + UVarint func; /* func = ci->func - thread->stack */ + UVarint top; /* top = ci->top - thread->stack */ + UVarint base; /* base = ci->base - thread-stack */ + SVarint nresults; /* expected number of results from this function, -1 for MULTRET */ uint8_t flags; /* What to do after completing this call, see lstate.h */ eris_CIKind ci_kind;/* What kind of CallInfo this is */ if (ci_kind == ERIS_CI_KIND_LUA) { - int yield_point; - int savedpc; /* savedpc = ci->u.l.savedpc - ci_func(ci)->p->code */ + SVarint yield_point; /* -1 on an errored thread */ + IntVarint savedpc; /* savedpc = ci->u.l.savedpc - ci_func(ci)->p->code */ } else if (ci_kind == ERIS_CI_KIND_C) { - int32_t errfunc; /* pcall's error handler, 1-based from ci->base, 0 for none */ + IntVarint errfunc; /* pcall's error handler, 1-based from ci->base, 0 for none */ Object function; } else { Assert(ci_kind == ERIS_CI_KIND_NONE); } ParseRecordEnd(record_end); - } ci[num_cis] ; + } ci[num_cis.v] ; while (TRUE) { struct OpenUpval { - size_t idx; /* stack index of the value + 1; 0 if end of list */ - if (idx) + UVarint idx; /* stack index of the value + 1; 0 if end of list */ + if (idx.v) Object upval; /* The upvalue */ } openupval ; - if (!openupval.idx) + if (!openupval.idx.v) break; } }; @@ -878,7 +953,9 @@ struct PermKey { typedef struct { refNum = 0; + strNum = 0; Header header; + StrTab strtab; /* Read in full before the root, see u_strtab() */ Object rootobj; /* The root object that was persisted. */ /* The same check the reader makes in u_finish() */ if (refNum != header.final_refcount) @@ -904,12 +981,19 @@ typedef struct { Assert(tag == "EXEC"); uint32_t major; uint32_t minor; - Assert(major == 2); + Assert(major == 3); + /* Feature masks for the core section: the features whose fields the + * writer emits, then the subset a reader has to understand. Both are + * zero until a feature exists; a nonzero required mask is refused. */ + uint64_t features_present; + uint64_t features_required; /* The concrete Script subclass's fingerprint (EXEC again for the base - * class), covering the extra section at the end. */ + * class), covering the extra section at the end, with its own masks. */ char class_tag[4] ; uint32_t class_major; uint32_t class_minor; + uint64_t class_features_present; + uint64_t class_features_required; struct CoreSection { uint32_t record_len ; diff --git a/Executor/include/Luau/Script.h b/Executor/include/Luau/Script.h index 8bcc574c..fb8be550 100644 --- a/Executor/include/Luau/Script.h +++ b/Executor/include/Luau/Script.h @@ -54,8 +54,10 @@ struct RunResult // Payload format written by Script::serializeState(). Leads every payload as // the family magic and versions the core section only; the concrete class's // fingerprint follows it and versions the extra section, so the two evolve -// independently. -constexpr StateFingerprint kScriptStateFingerprint{{'E', 'X', 'E', 'C'}, 2, 0}; +// independently. Each fingerprint is followed by a present and a required +// feature mask for its section, both zero until a feature exists; see +// serializeState(). +constexpr StateFingerprint kScriptStateFingerprint{{'E', 'X', 'E', 'C'}, 3, 0}; // We need to carry around the error messages in our state, but // the messages can be arbitrarily large. Cap them. diff --git a/Executor/src/Script.cpp b/Executor/src/Script.cpp index 8c9bd988..28d946ae 100644 --- a/Executor/src/Script.cpp +++ b/Executor/src/Script.cpp @@ -674,9 +674,17 @@ bool Script::serializeState(std::string& out) writer.writeBytes(kScriptStateFingerprint.tag, sizeof(kScriptStateFingerprint.tag)); writer.writeU32(kScriptStateFingerprint.major); writer.writeU32(kScriptStateFingerprint.minor); + // Each fingerprint is followed by two feature masks for its section: the + // features whose fields this writer emits, then the subset a reader has + // to understand to load this payload. No feature exists yet, so both are + // zero, and a reader refuses any nonzero required mask. + writer.writeU64(0); + writer.writeU64(0); writer.writeBytes(fingerprint.tag, sizeof(fingerprint.tag)); writer.writeU32(fingerprint.major); writer.writeU32(fingerprint.minor); + writer.writeU64(0); + writer.writeU64(0); size_t core = writer.beginSection(); writer.writeF32(mSleep); @@ -733,6 +741,8 @@ bool Script::restoreState(const char* data, size_t len) uint64_t sticky_handler = 0; uint64_t current_events = 0; uint64_t event_handlers = 0; + uint64_t present_features = 0; + uint64_t required_features = 0; ByteReader core{nullptr, 0}; ByteReader extra{nullptr, 0}; @@ -752,6 +762,14 @@ bool Script::restoreState(const char* data, size_t len) setFault(FaultKind::Runtime, "invalid script state"); return false; } + // Present features are skipped through their section lengths. + // We don't have any yet, so just bail if it's non-zero. + if (!reader.readU64(present_features) || !reader.readU64(required_features) || required_features != 0) + { + logWarn(logSource(), "Script state requires features 0x%llx this build doesn't know", (unsigned long long)required_features); + setFault(FaultKind::Runtime, "invalid script state"); + return false; + } // A payload meant for another class is refused here, before anything is forked if (!reader.readBytes(tag, sizeof(tag)) || memcmp(tag, fingerprint.tag, sizeof(tag)) != 0) { @@ -765,6 +783,12 @@ bool Script::restoreState(const char* data, size_t len) setFault(FaultKind::Runtime, "invalid script state"); return false; } + if (!reader.readU64(present_features) || !reader.readU64(required_features) || required_features != 0) + { + logWarn(logSource(), "Script extra state requires features 0x%llx this build doesn't know", (unsigned long long)required_features); + setFault(FaultKind::Runtime, "invalid script state"); + return false; + } // Every field fails the same way, keep each read to one line #define READ_OR_BAIL(read_expr) \ do \ diff --git a/VM/include/lua.h b/VM/include/lua.h index 20185054..ec5ceea3 100644 --- a/VM/include/lua.h +++ b/VM/include/lua.h @@ -785,14 +785,10 @@ static void populateperms(lua_State *L, bool forUnpersist) #undef eris_persist_base_cont #undef eris_persist_cont -// Ares stream format version. A reader accepts any minor under a major it -// supports: fields are only ever appended to length-prefixed records, so a -// newer minor's extra bytes are skipped and an older minor's missing ones -// default. The major moves only for a change that can't be expressed that way, -// and the reader keeps every major back to ARES_MIN_SUPPORTED_MAJOR readable. -#define ARES_FORMAT_MAJOR 6 +// Ares stream format versions +#define ARES_FORMAT_MAJOR 7 #define ARES_FORMAT_MINOR 0 -#define ARES_MIN_SUPPORTED_MAJOR 6 +#define ARES_MIN_SUPPORTED_MAJOR 7 LUA_API lua_State *eris_make_forkserver(lua_State *Lsrc); diff --git a/VM/src/ares.cpp b/VM/src/ares.cpp index f9c6369d..dcab72bc 100644 --- a/VM/src/ares.cpp +++ b/VM/src/ares.cpp @@ -63,6 +63,7 @@ THE SOFTWARE. #include "lstrbuf.h" #include "lljson.h" #include "Luau/Bytecode.h" +#include "Luau/BytecodeWire.h" LUAU_FASTFLAG(LuauCIProto) LUAU_FASTFLAG(LuauManagedDebugNames) @@ -192,8 +193,7 @@ typedef uint64_t ares_size_t; #define ERIS_ERR_TRUNC_INT "int value would get truncated" #define ERIS_ERR_TRUNC_SIZE "size_t value would get truncated" #define ERIS_ERR_TYPE_FLOAT "unsupported lua_Number type" -#define ERIS_ERR_TYPE_INT "unsupported int type" -#define ERIS_ERR_TYPE_SIZE "unsupported size_t type" +#define ERIS_ERR_VARINT "malformed data: unterminated varint" #define ERIS_ERR_TYPEP "trying to persist unknown type %d" #define ERIS_ERR_TYPEU "trying to unpersist unknown type %d" #define ERIS_ERR_UCFUNC "bad C closure (C function expected, got %s)" @@ -206,9 +206,12 @@ typedef uint64_t ares_size_t; #define ERIS_ERR_INVAL_PC "Tried to serialize thread yielded at invalid point" #define ERIS_ERR_RECORD "malformed data: record exceeds enclosing record" #define ERIS_ERR_REFCOUNT "malformed data: reference count mismatch (expected %u, got %u)" -#define ERIS_ERR_RAW_APPENDS "persisted a reference inside a record's raw-append section (refcount %d -> %d)" +#define ERIS_ERR_TRAILER "persisted a reference in a record's trailer without require_feature() (refcount %d -> %d)" #define ERIS_ERR_TRAILING "malformed data: trailing bytes after root object" #define ERIS_ERR_MAJOR "unsupported file format version %u.%u (want %u.x through %u.x)" +#define ERIS_ERR_FEATURE "file format version %u.%u uses features 0x%llx this reader doesn't know" +#define ERIS_ERR_FEATURE_SET "malformed data: required features 0x%llx are not among the present features 0x%llx" +#define ERIS_ERR_STRIDX "malformed data: string table index %llu out of range (table has %d)" /* ** ============================================================================ @@ -231,7 +234,9 @@ enum AresType : uint8_t ARES_T_VECTOR = 5, /* 6-15 reserved. */ - /* Collectable types, keyed into the reftable and carrying a memcat byte. */ + /* Collectable types, keyed into the reftable and carrying a memcat byte. + * Strings are the exception: the body is an index into the string table + * that precedes the root, which holds the memcat. */ ARES_T_STRING = 16, ARES_T_TABLE = 17, ARES_T_FUNCTION = 18, @@ -320,12 +325,17 @@ typedef struct PersistInfo { bool persistingCFunc; /* Junk bytes appended inside every record, for forward-compat tests. */ uint32_t testPadding; - /* info->refcount where the record being written reached BEGIN_RAW_APPENDS(), + /* info->refcount where the record being written reached BEGIN_TRAILER(), * -1 until its body gets there. */ - int rawAppendsRefcount; - /* Where the header reserved the final reference count, patched in once the - * root object has been written. A plain offset because this sits in a union. */ - std::streamoff refcountPos; + int trailerRefcount; + /* Whether require_feature() has been called since BEGIN_TRAILER(). A + * reference persisted in the trailer without it is an error, since a reader + * that skips the trailer would lose track of the numbering. */ + bool trailerRequired; + /* Every feature a populated field asked for through require_feature(). */ + uint64_t requiredFeatures; + /* Strings assigned a string table index so far. */ + int strcount; } PersistInfo; typedef uint8_t lu_byte; @@ -337,13 +347,15 @@ typedef struct UnpersistInfo { size_t pos; /* End of the innermost open record. Every read is bounded by it. */ size_t record_end; - size_t sizeof_int; - size_t sizeof_size_t; size_t vector_components; uint32_t major; uint32_t minor; /* The writer's final reference count, from the header. */ uint32_t expectedRefcount; + /* The features whose fields the writer emitted, from the header. */ + uint64_t presentFeatures; + /* Entries in the string table, once u_strtab has read it. */ + int strtabCount; /* When set, stamped as the threaddata of every thread of the unpersisted * tree; threads the tree spawns later inherit it through the userthread * callback. */ @@ -391,6 +403,21 @@ static char const kHeader[] = { 'A', 'R', 'E', 'S' }; /* Floating point number used to check compatibility of loaded data. */ static const lua_Number kHeaderNumber = (lua_Number)-1.234567890; +/* Feature bits, one per group of appended fields. Wire values: never + * renumbered or reused within a major; a major bump clears the set. A set bit + * in the header means "this build writes the field's bytes"; the field must be + * understood only when the writer also called require_feature() for it. To + * add one: define the bit, OR it into kAresFeaturesSupported, bump + * ARES_FORMAT_MINOR, and add a golden fixture. */ +enum AresFeature : uint64_t { + /* ARES_FEATURE_EXAMPLE = 1ull << 0, */ +}; + +/* The features this build implements. As a writer it emits the fields of all + * of them, so this is what the header's present mask says; as a reader it can + * be asked to require no more than this. */ +static const uint64_t kAresFeaturesSupported = 0; + /* Records that carry a length prefix: the VM-shaped ones whose layout tracks * the VM. Strings and buffers are self-delimiting, scalars never grow. Class * and object have no body yet but will be VM-shaped when they do. */ @@ -402,9 +429,10 @@ static inline bool type_is_framed(AresType type) { // The wire-tag equivalent of iscollectable(). ARES_T_PROTO and ARES_T_UPVAL are -// excluded, their stack stand-in needs a deref to get at the GCObject. +// excluded, their stack stand-in needs a deref to get at the GCObject. Strings +// carry theirs in the string table entry instead. static inline bool type_has_memcat(AresType type) { - return type == ARES_T_STRING || type == ARES_T_BUFFER || type == ARES_T_TABLE || + return type == ARES_T_BUFFER || type == ARES_T_TABLE || type == ARES_T_FUNCTION || type == ARES_T_USERDATA || type == ARES_T_THREAD || type == ARES_T_CLASS || type == ARES_T_OBJECT || type == ARES_T_VECTORD; } @@ -418,8 +446,10 @@ static inline bool ares_type_is_tvalue(AresType type) { /* Stack indices of some internal values/tables, to avoid magic numbers. */ #define PERMIDX 1 #define REFTIDX 2 -#define BUFFIDX 3 -#define PATHIDX 4 +/* Persisting: string -> index and index -> string. Unpersisting: index -> string. */ +#define STRTIDX 3 +#define BUFFIDX 4 +#define PATHIDX 5 /* Table indices for upvalue tables, keeping track of upvals to open. */ #define UVTOCL 1 @@ -748,6 +778,25 @@ set_setting(lua_State *L, void *key) { /* ... value */ lua_settable(L, LUA_REGISTRYINDEX); /* ... */ } +/* Marks the stream as needing a reader that knows this feature. Call it from + * the site that writes an appended field, under the same "is this the + * default" test the reader applies when the feature is absent, so a stream + * where the field was never populated still loads on older readers. Mandatory + * when the field takes a reference number, see RecordWriter::close. A call + * naming a bit this build doesn't know, such as one left over from before a + * major bump, fails the build. */ +#define require_feature(info, feature) do { \ + static_assert(((feature) & kAresFeaturesSupported) == (feature), "require_feature() names a feature this build doesn't support"); \ + static_assert((feature) != 0 && ((feature) & ((feature) - 1)) == 0, "require_feature() takes a single feature bit"); \ + (info)->u.pi.requiredFeatures |= (feature); \ + (info)->u.pi.trailerRequired = true; \ + } while(0) + +/* Whether the stream being read carries the field(s) behind this feature. The + * reader gates every appended field on this, never on the writer's minor or + * on bytes remaining in the record. */ +#define has_feature(info, feature) (((info)->u.upi.presentFeatures & (feature)) != 0) + /* Used as a callback for luaL_opt to check boolean setting values. */ static bool checkboolean(lua_State *L, int narg) { /* ... bool? ... */ @@ -848,16 +897,6 @@ write_uint64_t(Info *info, uint64_t value) { write_uint8_t(info, (uint8_t)(value >> 56)); } -static void -write_int32_t(Info *info, int32_t value) { - write_uint32_t(info, (uint32_t)value); -} - -static void -write_int64_t(Info *info, int64_t value) { - write_uint64_t(info, (uint64_t)value); -} - static void write_float32(Info *info, float value) { uint32_t rep; @@ -872,27 +911,36 @@ write_float64(Info *info, double value) { write_uint64_t(info, rep); } -/* Note regarding the following: any decent compiler should be able - * to reduce these to just the write call, since sizeof is constant. */ +/* Counts, sizes and indices are the same unsigned varints Luau bytecode uses, + * mirroring writeVarInt() in BytecodeBuilder.cpp. Nearly all fit in one byte. */ +static void +write_uvarint(Info *info, uint64_t value) { + do { + write_uint8_t(info, (uint8_t)((value & 127) | ((value > 127) << 7))); + value >>= 7; + } while (value); +} +/* Counts and offsets. Written as 32-bit two's complement, so a negative value + * costs five bytes; fields that are genuinely signed use ares_sint. */ static void write_int(Info *info, int value) { - if (sizeof(int) <= sizeof(int32_t)) { - write_int32_t(info, value); - } - else { - eris_error(info, ERIS_ERR_TYPE_INT); - } + static_assert(sizeof(int) == sizeof(int32_t), "int is assumed to be 32-bit on the wire"); + write_uvarint(info, (uint32_t)value); +} + +/* Zigzag varint for genuinely signed values: integer payloads, and the few + * fields where small negatives are normal, such as MULTRET in nresults. One + * 64-bit flavor covers them all; small values encode identically either way. */ +static void +write_ares_sint(Info *info, int64_t value) { + const uint64_t bits = (uint64_t)value; + write_uvarint(info, (bits << 1) ^ (uint64_t)(value >> 63)); } static void write_ares_size_t(Info *info, ares_size_t value) { - if (sizeof(size_t) <= sizeof(uint64_t)) { - write_uint64_t(info, (uint64_t)value); - } - else { - eris_error(info, ERIS_ERR_TYPE_SIZE); - } + write_uvarint(info, value); } static void @@ -931,13 +979,6 @@ read_uint8_t(Info *info) { return value; } -static uint16_t -read_uint16_t(Info *info) { - auto value = (uint16_t)read_uint8_t(info); - value |= (uint16_t)read_uint8_t(info) << 8; - return value; -} - static uint32_t read_uint32_t(Info *info) { auto value = (uint32_t)read_uint8_t(info); @@ -960,21 +1001,6 @@ read_uint64_t(Info *info) { return value; } -static int16_t -read_int16_t(Info *info) { - return (int16_t)read_uint16_t(info); -} - -static int32_t -read_int32_t(Info *info) { - return (int32_t)read_uint32_t(info); -} - -static int64_t -read_int64_t(Info *info) { - return (int64_t)read_uint64_t(info); -} - static float read_float32(Info *info) { float value; @@ -991,58 +1017,59 @@ read_float64(Info *info) { return value; } -/* Note regarding the following: unlike with writing the sizeof check will be - * impossible to optimize away, since it depends on the input; however, the - * truncation check may be optimized away in the case where the read data size - * equals the native one, so reading data written on the same machine should be - * reasonably quick. Doing a (rather rudimentary) benchmark this did not have - * any measurable impact on performance. */ +/* A 64-bit value is at most ten varint bytes. */ +static const size_t kMaxVarintBytes = 10; + +static uint64_t +read_uvarint(Info *info) { + /* Luau::readVarInt64 trusts its input, so find the terminating byte inside + * the record before handing it the buffer. */ + const char *data = info->u.upi.data; + size_t limit = RECORD_REMAINING(); + if (limit > kMaxVarintBytes) { + limit = kMaxVarintBytes; + } + size_t n = 0; + while (n < limit && ((uint8_t)data[info->u.upi.pos + n] & 0x80)) { + ++n; + } + if (n == limit) { + eris_error(info, ERIS_ERR_VARINT); + } + return Luau::readVarInt64(data, info->u.upi.pos); +} static int read_int(Info *info) { - int value; - if (info->u.upi.sizeof_int == sizeof(int16_t)) { - int16_t pvalue = read_int16_t(info); - value = (int)pvalue; - if ((int32_t)value != pvalue) { - eris_error(info, ERIS_ERR_TRUNC_INT); - } - } - else if (info->u.upi.sizeof_int == sizeof(int32_t)) { - int32_t pvalue = read_int32_t(info); - value = (int)pvalue; - if ((int32_t)value != pvalue) { - eris_error(info, ERIS_ERR_TRUNC_INT); - } - } - else if (info->u.upi.sizeof_int == sizeof(int64_t)) { - int64_t pvalue = read_int64_t(info); - value = (int)pvalue; - if ((int64_t)value != pvalue) { - eris_error(info, ERIS_ERR_TRUNC_INT); - } + const uint64_t value = read_uvarint(info); + if (value > UINT32_MAX) { + eris_error(info, ERIS_ERR_TRUNC_INT); } - else { - eris_error(info, ERIS_ERR_TYPE_INT); - value = 0; /* not reached */ + return (int)(uint32_t)value; +} + +static int64_t +read_ares_sint(Info *info) { + const uint64_t bits = read_uvarint(info); + return (int64_t)((bits >> 1) ^ (uint64_t)-(int64_t)(bits & 1)); +} + +/* An ares_sint that has to fit an int on this side. */ +static int +read_ares_sint32(Info *info) { + const int64_t value = read_ares_sint(info); + if (value < INT32_MIN || value > INT32_MAX) { + eris_error(info, ERIS_ERR_TRUNC_INT); } - return value; + return (int)value; } static ares_size_t read_ares_size_t(Info *info) { - ares_size_t value; - if (info->u.upi.sizeof_size_t <= sizeof(uint64_t)) { - value = read_uint64_t(info); - // Refuse to read sizes that would be truncated on 32-bit - if (value > (ares_size_t)SIZE_MAX) { - eris_error(info, "malformed data: size value out of range"); - } - return value; - } - else { - eris_error(info, ERIS_ERR_TYPE_SIZE); - value = 0; /* not reached */ + const uint64_t value = read_uvarint(info); + // Refuse to read sizes that would be truncated on 32-bit + if (value > (uint64_t)SIZE_MAX) { + eris_error(info, "malformed data: size value out of range"); } return value; } @@ -1067,18 +1094,25 @@ read_Instruction(Info *info) { /* * Length-prefixed records. A record is `u32 len` followed by `len` body bytes. - * Also keeps note of any reference types written so we can be sure we don't - * mess anything up. + * Everything after BEGIN_TRAILER() is the trailer: fields a later feature + * added, which an older reader steps over via the length word. A reference + * persisted there is only safe if require_feature() was called, so the readers + * that would have skipped it refuse the stream instead. */ -#define BEGIN_RAW_APPENDS() (info->u.pi.rawAppendsRefcount = info->refcount) +#define BEGIN_TRAILER() do { \ + info->u.pi.trailerRefcount = info->refcount; \ + info->u.pi.trailerRequired = false; \ + } while(0) // Write a length-prefixed binary section to struct RecordWriter { explicit RecordWriter(Info *info_) : info(info_), exceptions(std::uncaught_exceptions()), - outer_raw_appends_refcount(info_->u.pi.rawAppendsRefcount) { + outer_trailer_refcount(info_->u.pi.trailerRefcount), + outer_trailer_required(info_->u.pi.trailerRequired) { write_uint32_t(info, 0); - info->u.pi.rawAppendsRefcount = -1; + info->u.pi.trailerRefcount = -1; + info->u.pi.trailerRequired = false; body_start = info->u.pi.writer->tellp(); } @@ -1086,15 +1120,14 @@ struct RecordWriter { eris_assert(!closed); closed = true; - // A body that never called BEGIN_RAW_APPENDS() - eris_assert(info->u.pi.rawAppendsRefcount != -1); - // someone persist()ed something inside the raw trailer section. This is an error, - // and would definitely break compatibility. - if (info->u.pi.rawAppendsRefcount != info->refcount) { - eris_error(info, ERIS_ERR_RAW_APPENDS, info->u.pi.rawAppendsRefcount, info->refcount); + // A body that never called BEGIN_TRAILER() has no trailer to check + eris_assert(info->u.pi.trailerRefcount != -1); + if (info->u.pi.trailerRefcount != info->refcount && !info->u.pi.trailerRequired) { + eris_error(info, ERIS_ERR_TRAILER, info->u.pi.trailerRefcount, info->refcount); } - info->u.pi.rawAppendsRefcount = outer_raw_appends_refcount; + info->u.pi.trailerRefcount = outer_trailer_refcount; + info->u.pi.trailerRequired = outer_trailer_required; // write in some padding if we're in a test that wants malformed data for (uint32_t i = 0; i < info->u.pi.testPadding; ++i) { write_uint8_t(info, (uint8_t)(0xA5 ^ i) | 1); @@ -1121,7 +1154,8 @@ struct RecordWriter { Info *info; int exceptions; - int outer_raw_appends_refcount; + int outer_trailer_refcount; + bool outer_trailer_required; std::streampos body_start; bool closed = false; }; @@ -1165,14 +1199,14 @@ static void unpersist(Info*); static void p_boolean(Info *info) { /* ... bool */ // Have to use this rather than toboolean so that we keep the full int value - WRITE_VALUE(bvalue(info->L->top - 1), int32_t); + WRITE_VALUE(bvalue(info->L->top - 1), int); } static void u_boolean(Info *info) { /* ... */ eris_checkstack(info->L, 1); // Have to use this rather than pushboolean so that we keep the full int value - setbvalue(info->L->top, READ_VALUE(int32_t)); /* ... bool */ + setbvalue(info->L->top, READ_VALUE(int)); /* ... bool */ eris_incr_top(info->L); eris_checktype(info, -1, LUA_TBOOLEAN); @@ -1183,7 +1217,8 @@ u_boolean(Info *info) { /* ... */ static void p_pointer(Info *info) { /* ... ludata */ WRITE_VALUE((uint8_t)lua_lightuserdatatag(info->L, -1), uint8_t); - WRITE_VALUE((ares_size_t)lua_touserdata(info->L, -1), ares_size_t); + /* A pointer-sized value, not an index, so it stays fixed width. */ + WRITE_VALUE((uint64_t)(uintptr_t)lua_touserdata(info->L, -1), uint64_t); } static void @@ -1193,24 +1228,27 @@ u_pointer(Info *info) { /* ... */ if (tag >= LUTAG_ARES_START) { eris_error(info, "malformed data: invalid lightuserdata tag"); } - void *ptr = (void*)READ_VALUE(ares_size_t); + const uint64_t raw = READ_VALUE(uint64_t); + if (raw > (uint64_t)UINTPTR_MAX) { + eris_error(info, ERIS_ERR_TRUNC_SIZE); + } + void *ptr = (void*)(uintptr_t)raw; lua_pushlightuserdatatagged(info->L, ptr, tag); /* ... ludata */ eris_checktype(info, -1, LUA_TLIGHTUSERDATA); } /** ======================================================================== */ -// TODO: This could probably benefit from varint encoding. static void p_integer(Info *info) { /* ... int */ - WRITE_VALUE(lua_tointeger64(info->L, -1, NULL), int64_t); + WRITE_VALUE(lua_tointeger64(info->L, -1, NULL), ares_sint); } static void u_integer(Info *info) { /* ... */ eris_checkstack(info->L, 1); - lua_pushinteger64(info->L, READ_VALUE(int64_t)); /* ... int */ + lua_pushinteger64(info->L, READ_VALUE(ares_sint)); /* ... int */ eris_checktype(info, -1, LUA_TINTEGER); } @@ -1299,31 +1337,90 @@ u_vector_f64(Info *info) { /* ... */ /** ======================================================================== */ +/* Strings are written as an index into the string table, assigned on first + * sight. The table itself goes out ahead of the root, see p_strtab. */ static void p_string(Info *info) { /* ... str */ - size_t length; - const char *value = lua_tolstring(info->L, -1, &length); - WRITE_VALUE(length, ares_size_t); - WRITE_RAW(value, length); + eris_checkstack(info->L, 2); + lua_pushvalue(info->L, -1); /* ... str str */ + lua_rawget(info->L, STRTIDX); /* ... str idx? */ + int index; + if (lua_isnil(info->L, -1)) { /* ... str nil */ + lua_pop(info->L, 1); /* ... str */ + index = ++info->u.pi.strcount; + lua_pushvalue(info->L, -1); /* ... str str */ + lua_pushinteger(info->L, index); /* ... str str idx */ + lua_rawset(info->L, STRTIDX); /* ... str */ + lua_pushvalue(info->L, -1); /* ... str str */ + lua_rawseti(info->L, STRTIDX, index); /* ... str */ + } + else { /* ... str idx */ + index = lua_tointeger(info->L, -1); + lua_pop(info->L, 1); /* ... str */ + } + WRITE_VALUE((ares_size_t)index, ares_size_t); } static void u_string(Info *info) { /* ... */ - eris_checkstack(info->L, 2); - { - /* TODO Can we avoid this copy somehow? (Without it getting too nasty) */ - const size_t length = (size_t)READ_VALUE(ares_size_t); - VALIDATE_SIZE(length); - char *value = (char*)lua_newuserdata(info->L, length * sizeof(char)); /* ... tmp */ - READ_RAW(value, length); - lua_pushlstring(info->L, value, length); /* ... tmp str */ - lua_replace(info->L, -2); /* ... str */ + eris_checkstack(info->L, 1); + const ares_size_t index = READ_VALUE(ares_size_t); + if (index < 1 || index > (ares_size_t)info->u.upi.strtabCount) { + eris_error(info, ERIS_ERR_STRIDX, (unsigned long long)index, info->u.upi.strtabCount); } - registerobject(info); + lua_rawgeti(info->L, STRTIDX, (int)index); /* ... str */ eris_checktype(info, -1, LUA_TSTRING); } +/* The string table: every string the root reached, in first-sight order, each + * with the memcat it was allocated under. Written after the root has been + * serialized to a scratch buffer, placed before it in the stream. */ +static void +p_strtab(Info *info) { /* perms reftbl strtab ... */ + eris_checkstack(info->L, 1); + RecordWriter rec(info); + const int count = info->u.pi.strcount; + WRITE_VALUE((ares_size_t)count, ares_size_t); + for (int i = 1; i <= count; ++i) { + lua_rawgeti(info->L, STRTIDX, i); /* perms reftbl strtab ... str */ + const TValue *tv = luaA_toobject(info->L, -1); + eris_assert(ttisstring(tv)); + WRITE_VALUE(gcvalue(tv)->gch.memcat, uint8_t); + size_t length; + const char *value = lua_tolstring(info->L, -1, &length); + WRITE_VALUE(length, ares_size_t); + WRITE_RAW(value, length); + lua_pop(info->L, 1); /* perms reftbl strtab ... */ + } + BEGIN_TRAILER(); + rec.close(); +} + +static void +u_strtab(Info *info) { /* perms reftbl strtab ... */ + eris_checkstack(info->L, 2); + RecordReader rec(info); + const ares_size_t count = READ_VALUE(ares_size_t); + /* Every entry is at least its memcat byte. */ + if (count > RECORD_REMAINING()) { + eris_error(info, "malformed data: string table count exceeds stream data"); + } + for (ares_size_t i = 1; i <= count; ++i) { + const uint8_t memcat = READ_VALUE(uint8_t); + MemcatGuard guard(info->L, memcat); + const size_t length = (size_t)READ_VALUE(ares_size_t); + VALIDATE_SIZE(length); + /* The stream is already in memory, so the string is interned straight + * out of it. */ + lua_pushlstring(info->L, info->u.upi.data + info->u.upi.pos, length); + /* perms reftbl strtab ... str */ + info->u.upi.pos += length; + lua_rawseti(info->L, STRTIDX, (int)i); /* perms reftbl strtab ... */ + } + info->u.upi.strtabCount = (int)count; +} + /** ======================================================================== */ @@ -1469,7 +1566,7 @@ static void p_table(Info *info) { /* ... tbl */ } p_metatable(info); - BEGIN_RAW_APPENDS(); + BEGIN_TRAILER(); } static void u_table(Info *info) { /* ... */ @@ -1690,7 +1787,7 @@ static void p_userdata(Info *info) { /* ... udata case UTAG_DETECTED_EVENT: { const auto *detected_event = (lua_DetectedEvent*)value; - WRITE_VALUE(detected_event->index, int32_t); + WRITE_VALUE((int)detected_event->index, int); WRITE_VALUE(detected_event->valid, uint8_t); WRITE_VALUE(detected_event->can_change_damage, uint8_t); break; @@ -1735,7 +1832,7 @@ static void p_userdata(Info *info) { /* ... udata } p_metatable(info); /* ... udata */ eris_assert(top == lua_gettop(info->L)); - BEGIN_RAW_APPENDS(); + BEGIN_TRAILER(); } static void u_userdata(Info *info) { /* ... */ @@ -1770,10 +1867,6 @@ static void u_userdata(Info *info) { /* ... */ } case UTAG_UUID: { - // Because we have an inner, wrapped string reference we need to reserve - // the idx for the outer UUID first, since we saw it first. - int reference = allocate_ref_idx(info); - // Deserialize the wrapped string unpersist(info); /* ... str */ eris_checktype(info, -1, LUA_TSTRING); @@ -1781,10 +1874,7 @@ static void u_userdata(Info *info) { /* ... */ auto *str_val = luaL_checklstring(info->L, -1, &len); luaSL_pushuuidlstring(info->L, str_val, len); /* ... str uuid */ lua_replace(info->L, -2); /* ... uuid */ - - // Manually put the UUID in the references table at the correct reference index - lua_pushvalue(info->L, -1); /* perms reftbl ... obj obj */ - lua_rawseti(info->L, REFTIDX, reference); /* perms reftbl ... obj */ + registerobject(info); break; } case UTAG_DETECTED_EVENT: @@ -1796,7 +1886,7 @@ static void u_userdata(Info *info) { /* ... */ ); /* ... udata */ memset(detected_event, 0, sizeof(lua_DetectedEvent)); - detected_event->index = READ_VALUE(int32_t); + detected_event->index = READ_VALUE(int); detected_event->valid = (bool)READ_VALUE(uint8_t); detected_event->can_change_damage = (bool)READ_VALUE(uint8_t); registerobject(info); @@ -2022,12 +2112,12 @@ p_proto(Info *info) { /* ... proto */ // poppath(info); // } // poppath(info); - WRITE_VALUE(p->sizeyieldpoints, int32_t); + WRITE_VALUE(p->sizeyieldpoints, int); for (i=0; isizeyieldpoints; ++i) { - WRITE_VALUE(p->yieldpoints[i], int32_t); + WRITE_VALUE(p->yieldpoints[i], int); } - BEGIN_RAW_APPENDS(); + BEGIN_TRAILER(); } static void @@ -2179,7 +2269,7 @@ u_proto(Info *info) { /* ... proto */ SAFE_ALLOC_VECTOR(info->L, p->yieldpoints, 0, int, p->sizeyieldpoints, int); for (i=0; isizeyieldpoints; ++i) { - p->yieldpoints[i] = READ_VALUE(int32_t); + p->yieldpoints[i] = READ_VALUE(int); if (p->yieldpoints[i] < 0 || p->yieldpoints[i] >= p->sizecode) { eris_error(info, "malformed data: invalid yield point"); } @@ -2198,7 +2288,7 @@ u_proto(Info *info) { /* ... proto */ static void p_upval(Info *info) { /* ... obj */ persist(info); /* ... obj */ - BEGIN_RAW_APPENDS(); + BEGIN_TRAILER(); } static void @@ -2363,7 +2453,7 @@ p_closure(Info *info) { /* perms reftbl ... func */ } poppath(info); } - BEGIN_RAW_APPENDS(); + BEGIN_TRAILER(); } static void @@ -2610,7 +2700,7 @@ p_thread(Info *info) { /* ... thread */ eris_assert(lua_type(info->L, -1) == LUA_TTHREAD); /* Persist the stack. Save the total size and used space first. */ - WRITE_VALUE((uint32_t)thread->stacksize, uint32_t); + WRITE_VALUE((ares_size_t)thread->stacksize, ares_size_t); WRITE_VALUE(total, ares_size_t); /* The Lua stack looks like this: @@ -2679,8 +2769,8 @@ p_thread(Info *info) { /* ... thread */ // written above. The capacity has to survive the round trip on its own: the // VM assumes it never drops below BASIC_CI_SIZE, and lua_resetthread shrinks // to that rather than growing back up to it. - WRITE_VALUE((uint32_t)thread->size_ci, uint32_t); - WRITE_VALUE((uint32_t)num_cis, uint32_t); + WRITE_VALUE((ares_size_t)thread->size_ci, ares_size_t); + WRITE_VALUE((ares_size_t)num_cis, ares_size_t); for (int i=0; i < num_cis; ++i) { pushpath(info, "[%d]", level++); ci = thread->base_ci + i; @@ -2691,7 +2781,7 @@ p_thread(Info *info) { /* ... thread */ /* CallInfo.nresults and CallInfo.flags are only set for actual functions. * base_ci[0] is never set up by luau_precall, so reading them there would * put uninitialised bytes on the wire. */ - WRITE_VALUE(ttisfunction(ci->func) ? ci->nresults : 0, int); + WRITE_VALUE(ttisfunction(ci->func) ? ci->nresults : 0, ares_sint); WRITE_VALUE(ttisfunction(ci->func) ? ci->flags : 0, uint8_t); if (eris_isLua(ci)) { @@ -2736,7 +2826,7 @@ p_thread(Info *info) { /* ... thread */ } } - WRITE_VALUE(yield_point, int); + WRITE_VALUE(yield_point, ares_sint); WRITE_VALUE((int)pc_offset, int); } else if (ttisfunction(ci->func)) { @@ -2747,7 +2837,7 @@ p_thread(Info *info) { /* ... thread */ // Protected call's error function: 1-based index relative to ci->base, 0 for none. // C frames only, Lua frames use the same union member as savedpc. - WRITE_VALUE(ci->errfunc, int32_t); + WRITE_VALUE(ci->errfunc, int); eris_ifassert(const int pre_closure_top = lua_gettop(info->L)); // Copy the original closure from ci->func to info->L's stack for serialization. @@ -2763,7 +2853,7 @@ p_thread(Info *info) { /* ... thread */ WRITE_VALUE(ERIS_CI_KIND_NONE, uint8_t); eris_assert(ttisnil(ci->func)); } - BEGIN_RAW_APPENDS(); + BEGIN_TRAILER(); ci_rec.close(); poppath(info); } @@ -2796,7 +2886,7 @@ p_thread(Info *info) { /* ... thread */ lua_pop(info->L, 1); /* ... thread */ poppath(info); eris_assert(lua_type(info->L, -1) == LUA_TTHREAD); - BEGIN_RAW_APPENDS(); + BEGIN_TRAILER(); } /* Used in u_thread to validate read stack positions. */ @@ -2861,7 +2951,7 @@ u_thread(Info *info) { /* ... */ /* Unpersist the stack. Read size first and adjust accordingly. * stack_size is the full stacksize (including EXTRA_STACK) as written by p_thread. * luaD_reallocstack expects the usable size (without EXTRA_STACK) and adds it back. */ - uint32_t stack_size = READ_VALUE(uint32_t); + ares_size_t stack_size = READ_VALUE(ares_size_t); ares_size_t total = READ_VALUE(ares_size_t); if (stack_size < LUA_MINSTACK + EXTRA_STACK || stack_size > kMaxStackSize) { eris_error(info, "malformed data: invalid stack size"); @@ -2930,8 +3020,8 @@ u_thread(Info *info) { /* ... */ pushpath(info, ".callinfo"); UNLOCK(thread); - uint32_t size_ci = READ_VALUE(uint32_t); - uint32_t num_cis = READ_VALUE(uint32_t); + ares_size_t size_ci = READ_VALUE(ares_size_t); + ares_size_t num_cis = READ_VALUE(ares_size_t); if (num_cis < 1 || num_cis > LUAI_MAXCALLS) { eris_error(info, "malformed data: invalid call info count"); } @@ -2941,7 +3031,7 @@ u_thread(Info *info) { /* ... */ // bytes of its own in the stream. There's no floor: hardstacktests builds // shrink the array to exactly what's in use on every GC pass. if (size_ci < num_cis || - size_ci > (uint32_t)(LUAI_MAXCALLS + (LUAI_MAXCALLS >> 3))) { + size_ci > (ares_size_t)(LUAI_MAXCALLS + (LUAI_MAXCALLS >> 3))) { eris_error(info, "malformed data: invalid call info capacity"); } // conservative: each CI needs at least a few bytes @@ -2949,7 +3039,7 @@ u_thread(Info *info) { /* ... */ luaD_reallocCI(thread, (int)size_ci); thread->ci = thread->base_ci; level = 0; - for (uint32_t ci_idx=0; ci_idxci->func, thread->top - 1); u_stackidx(info, thread, &thread->ci->top, thread->stack_last); u_stackidx(info, thread, &thread->ci->base, thread->top); - thread->ci->nresults = READ_VALUE(int32_t); + thread->ci->nresults = READ_VALUE(ares_sint32); thread->ci->flags = READ_VALUE(uint8_t); // We'll set these later if relevant for the CI type. thread->ci->p = nullptr; @@ -2986,7 +3076,7 @@ u_thread(Info *info) { /* ... */ if (ci_kind == ERIS_CI_KIND_LUA) { Closure *lcl = eris_ci_func(thread->ci); - int yield_point = READ_VALUE(int); + int yield_point = READ_VALUE(ares_sint32); int real_pc = READ_VALUE(int); int pc_offset; @@ -3020,7 +3110,7 @@ u_thread(Info *info) { /* ... */ } } else if (ci_kind == ERIS_CI_KIND_C) { // resume_handle() resolves this as ci->base + (errfunc - 1). - int errfunc = READ_VALUE(int32_t); + int errfunc = READ_VALUE(int); if (errfunc != 0) { // Bounds-check the index before the pointer arithmetic can overflow if (errfunc < 1 || errfunc > (int)(thread->top - thread->ci->base) + 1) { @@ -3317,12 +3407,14 @@ persist(Info *info) { /* perms reftbl ... obj */ WRITE_VALUE(type, uint8_t); } /* Write simple values directly, because writing a "reference" would take up - * just as much space and we can save ourselves work this way. */ + * just as much space and we can save ourselves work this way. Strings have + * their own table and never take a reference number either. */ else if (type == ARES_T_BOOLEAN || type == ARES_T_LIGHTUSERDATA || type == ARES_T_NUMBER || type == ARES_T_INTEGER || - type == ARES_T_VECTOR) + type == ARES_T_VECTOR || + type == ARES_T_STRING) { persist_typed(info, type); /* perms reftbl ... obj */ } @@ -3470,6 +3562,8 @@ unpersist(Info *info) { /* perms reftbl ... */ ** ============================================================================ */ +/* Written after the root has gone to its scratch buffer, so the counts it + * carries are final. */ static void p_header(Info *info) { WRITE_RAW(kHeader, HEADER_LENGTH); @@ -3478,29 +3572,13 @@ p_header(Info *info) { RecordWriter rec(info); WRITE_VALUE(sizeof(lua_Number), uint8_t); WRITE_VALUE(kHeaderNumber, lua_Number); - WRITE_VALUE(sizeof(int), uint8_t); - WRITE_VALUE(sizeof(size_t), uint8_t); WRITE_VALUE(LUA_VECTOR_SIZE, uint8_t); - /* Final reference count, unknown until the root has been written. Reserved - * here and filled in by p_header_refcount. */ - info->u.pi.refcountPos = std::streamoff(info->u.pi.writer->tellp()); - WRITE_VALUE(0, uint32_t); - BEGIN_RAW_APPENDS(); - rec.close(); -} - -/* Patches the reference count the header reserved, so a reader that lost or - * gained a reference somewhere refuses the stream. */ -static void -p_header_refcount(Info *info) { - std::ostream *writer = info->u.pi.writer; - std::streampos end = writer->tellp(); - writer->seekp(info->u.pi.refcountPos); + /* So a reader that lost or gained a reference somewhere refuses the stream. */ WRITE_VALUE((uint32_t)info->refcount, uint32_t); - writer->seekp(end); - if (writer->fail()) { - eris_error(info, ERIS_ERR_WRITE); - } + WRITE_VALUE(kAresFeaturesSupported, uint64_t); + WRITE_VALUE(info->u.pi.requiredFeatures, uint64_t); + BEGIN_TRAILER(); + rec.close(); } static void @@ -3529,10 +3607,20 @@ u_header(Info *info) { if (READ_VALUE(lua_Number) != kHeaderNumber) { eris_error(info, "incompatible floating point representation"); } - info->u.upi.sizeof_int = READ_VALUE(uint8_t); - info->u.upi.sizeof_size_t = READ_VALUE(uint8_t); info->u.upi.vector_components = READ_VALUE(uint8_t); info->u.upi.expectedRefcount = READ_VALUE(uint32_t); + info->u.upi.presentFeatures = READ_VALUE(uint64_t); + const uint64_t required_features = READ_VALUE(uint64_t); + /* Present-but-unknown features are skipped through their record lengths; + * only required ones refuse the stream. */ + if (required_features & ~kAresFeaturesSupported) { + eris_error(info, ERIS_ERR_FEATURE, info->u.upi.major, info->u.upi.minor, + (unsigned long long)(required_features & ~kAresFeaturesSupported)); + } + if (required_features & ~info->u.upi.presentFeatures) { + eris_error(info, ERIS_ERR_FEATURE_SET, (unsigned long long)required_features, + (unsigned long long)info->u.upi.presentFeatures); + } } /* Once the root has been read, the stream must have assigned exactly the @@ -3899,9 +3987,12 @@ unchecked_persist(lua_State *L, std::ostream *writer) { info.u.pi.writeDebugInfo = kWriteDebugInformation; info.u.pi.persistingCFunc = false; info.u.pi.testPadding = 0; - info.u.pi.rawAppendsRefcount = -1; + info.u.pi.trailerRefcount = -1; + info.u.pi.trailerRequired = false; + info.u.pi.requiredFeatures = 0; + info.u.pi.strcount = 0; - eris_checkstack(L, 6); + eris_checkstack(L, 7); // Record lengths are back-patched if (writer->tellp() == std::streampos(-1)) { @@ -3931,9 +4022,11 @@ unchecked_persist(lua_State *L, std::ostream *writer) { lua_newtable(L); /* perms buff rootobj reftbl */ lua_insert(L, REFTIDX); /* perms reftbl buff rootobj */ + lua_newtable(L); /* perms reftbl buff rootobj strtab */ + lua_insert(L, STRTIDX); /* perms reftbl strtab buff rootobj */ if (info.generatePath) { - lua_newtable(L); /* perms reftbl buff rootobj path */ - lua_insert(L, PATHIDX); /* perms reftbl buff path rootobj */ + lua_newtable(L); /* perms reftbl strtab buff rootobj path */ + lua_insert(L, PATHIDX); /* perms reftbl strtab buff path rootobj */ pushpath(&info, "root"); } @@ -3942,23 +4035,37 @@ unchecked_persist(lua_State *L, std::ostream *writer) { lua_replace(L, PERMIDX); /* Populate perms table with Lua internals. */ - lua_pushvalue(L, PERMIDX); /* perms reftbl buff path? rootobj perms */ + lua_pushvalue(L, PERMIDX); /* perms reftbl strtab buff path? rootobj perms */ eris_populate_perms(L, false); - lua_pop(L, 1); /* perms reftbl buff path? rootobj */ + lua_pop(L, 1); /* perms reftbl strtab buff path? rootobj */ int pre_pad_top = lua_gettop(L); lua_hardenstack(L, 1); - p_header(&info); - persist(&info); /* perms reftbl buff path? rootobj */ - // go back and backpatch the header to include the refcount - p_header_refcount(&info); + /* The string table and the header's counts are only known once the root has + * been walked, and both precede it in the stream, so the root goes to a + * scratch buffer first. */ + { + std::ostringstream scratch; + info.u.pi.writer = &scratch; + persist(&info); /* perms reftbl strtab buff path? rootobj */ + info.u.pi.writer = writer; + + p_header(&info); + p_strtab(&info); + const std::string body = scratch.str(); + writer->write(body.data(), body.size()); + if (writer->fail()) { + eris_error(&info, ERIS_ERR_WRITE); + } + } lua_settop(L, pre_pad_top); - if (info.generatePath) { /* perms reftbl buff path rootobj */ - lua_remove(L, PATHIDX); /* perms reftbl buff rootobj */ - } /* perms reftbl buff rootobj */ + if (info.generatePath) { /* perms reftbl strtab buff path rootobj */ + lua_remove(L, PATHIDX); /* perms reftbl strtab buff rootobj */ + } /* perms reftbl strtab buff rootobj */ + lua_remove(L, STRTIDX); /* perms reftbl buff rootobj */ lua_remove(L, REFTIDX); /* perms buff rootobj */ eris_assert(lua_gettop(L) == old_top); } @@ -3981,8 +4088,9 @@ unchecked_unpersist(lua_State *L, const char *data, size_t size, void *threaddat info.u.upi.pos = 0; info.u.upi.record_end = size; info.u.upi.threaddata = threaddata; + info.u.upi.strtabCount = 0; - eris_checkstack(L, 6); + eris_checkstack(L, 7); if (get_setting(L, (void*)&kSettingMaxComplexity)) { /* perms buff rootobj value */ @@ -3997,13 +4105,15 @@ unchecked_unpersist(lua_State *L, const char *data, size_t size, void *threaddat lua_newtable(L); /* perms str? reftbl */ lua_insert(L, REFTIDX); /* perms reftbl str? */ + lua_newtable(L); /* perms reftbl str? strtab */ + lua_insert(L, STRTIDX); /* perms reftbl strtab str? */ if (info.generatePath) { - /* Make sure the path is always at index 4, so that it's the same for + /* Make sure the path is always at PATHIDX, so that it's the same for * persist and unpersist. */ - lua_pushnil(L); /* perms reftbl str? nil */ - lua_insert(L, BUFFIDX); /* perms reftbl nil str? */ - lua_newtable(L); /* perms reftbl nil str? path */ - lua_insert(L, PATHIDX); /* perms reftbl nil path str? */ + lua_pushnil(L); /* perms reftbl strtab str? nil */ + lua_insert(L, BUFFIDX); /* perms reftbl strtab nil str? */ + lua_newtable(L); /* perms reftbl strtab nil str? path */ + lua_insert(L, PATHIDX); /* perms reftbl strtab nil path str? */ pushpath(&info, "root"); } @@ -4012,15 +4122,16 @@ unchecked_unpersist(lua_State *L, const char *data, size_t size, void *threaddat lua_replace(L, PERMIDX); /* Populate perms table with Lua internals. */ - lua_pushvalue(L, PERMIDX); /* perms reftbl nil? path? str? perms */ + lua_pushvalue(L, PERMIDX); /* perms reftbl strtab nil? path? str? perms */ eris_populate_perms(L, true); - lua_pop(L, 1); /* perms reftbl nil? path? str? */ + lua_pop(L, 1); /* perms reftbl strtab nil? path? str? */ int pre_pad_top = lua_gettop(L); lua_hardenstack(L, 1); u_header(&info); - unpersist(&info); /* perms reftbl nil? path? str? rootobj */ + u_strtab(&info); + unpersist(&info); /* perms reftbl strtab nil? path? str? rootobj */ u_finish(&info); /* Get rid of any padding we might have added, leave just the result */ @@ -4029,10 +4140,11 @@ unchecked_unpersist(lua_State *L, const char *data, size_t size, void *threaddat lua_settop(L, pre_pad_top + 1); } - if (info.generatePath) { /* perms reftbl nil path str? rootobj */ - lua_remove(L, PATHIDX); /* perms reftbl nil str? rootobj */ - lua_remove(L, BUFFIDX); /* perms reftbl str? rootobj */ - } /* perms reftbl str? rootobj */ + if (info.generatePath) { /* perms reftbl strtab nil path str? rootobj */ + lua_remove(L, PATHIDX); /* perms reftbl strtab nil str? rootobj */ + lua_remove(L, BUFFIDX); /* perms reftbl strtab str? rootobj */ + } /* perms reftbl strtab str? rootobj */ + lua_remove(L, STRTIDX); /* perms reftbl str? rootobj */ lua_remove(L, REFTIDX); /* perms str? rootobj */ eris_assert(lua_gettop(L) == old_top + 1); } diff --git a/tests/SLExecutor.test.cpp b/tests/SLExecutor.test.cpp index 5cc0ab5e..3afbec8c 100644 --- a/tests/SLExecutor.test.cpp +++ b/tests/SLExecutor.test.cpp @@ -1710,7 +1710,7 @@ TEST_CASE_FIXTURE(SLuaFixture, "SLExecutor unserializable global is refused with // Size of the payload a `counter = 1` script serializes to. Update it when the // wire format moves; a change nobody meant to make is the thing worth catching. -constexpr size_t kExpectedDonorPayloadSize = 634; +constexpr size_t kExpectedDonorPayloadSize = 492; TEST_CASE_FIXTURE(SLuaFixture, "SLExecutor invalid restore") { @@ -1770,20 +1770,30 @@ TEST_CASE_FIXTURE(SLuaFixture, "SLExecutor invalid restore") bad_magic[0] = 'B'; CHECK_FALSE(second.exec.restoreState(bad_magic.data(), bad_magic.size())); - // The core section's major follows the magic, the class's own follows - // the class tag + // Each fingerprint is tag, major, minor, then a present and a required + // feature mask, so the class fingerprint starts at 28 std::string bad_core_version = payload; bad_core_version[4] = (char)(kScriptStateFingerprint.major + 99); CHECK_FALSE(second.exec.restoreState(bad_core_version.data(), bad_core_version.size())); std::string bad_class_tag = payload; - bad_class_tag[12] = 'B'; + bad_class_tag[28] = 'B'; CHECK_FALSE(second.exec.restoreState(bad_class_tag.data(), bad_class_tag.size())); std::string bad_class_version = payload; - bad_class_version[16] = (char)(kScriptStateFingerprint.major + 99); + bad_class_version[32] = (char)(kScriptStateFingerprint.major + 99); CHECK_FALSE(second.exec.restoreState(bad_class_version.data(), bad_class_version.size())); + // A required feature this build doesn't know is refused, for either + // section + std::string core_requires = payload; + core_requires[20] = 1; + CHECK_FALSE(second.exec.restoreState(core_requires.data(), core_requires.size())); + + std::string class_requires = payload; + class_requires[48] = 1; + CHECK_FALSE(second.exec.restoreState(class_requires.data(), class_requires.size())); + CHECK_FALSE(second.exec.restoreState("", 0)); for (size_t len : {size_t(4), size_t(8), size_t(16), payload.size() / 2, payload.size() - 1}) CHECK_FALSE(second.exec.restoreState(payload.data(), len)); @@ -1794,6 +1804,17 @@ TEST_CASE_FIXTURE(SLuaFixture, "SLExecutor invalid restore") CHECK(second.exec.getFaultKind() == FaultKind::None); } + SUBCASE("unknown present features are skipped") + { + // A feature that is present but not required belongs to a droppable + // field, and an older build loads the payload without it + std::string core_present = payload; + core_present[12] = 1; + core_present[40] = 1; + restore(second.exec, core_present); + CHECK(second.exec.getFaultKind() == FaultKind::None); + } + SUBCASE("restore directly after instantiate succeeds") { // Restoring persisted state is the common production start path, so diff --git a/tests/SLGoldenFixtures.test.cpp b/tests/SLGoldenFixtures.test.cpp index ce3fdfab..5c21e983 100644 --- a/tests/SLGoldenFixtures.test.cpp +++ b/tests/SLGoldenFixtures.test.cpp @@ -8,6 +8,14 @@ // asset and state under the current format versions. The load case restores // every committed state this build claims to read and runs its scenario's // verify against it, so a state written by an older build has to keep working. +// +// TODO: the other direction, an older reader loading states this build wrote, +// can only be checked by running the previous release's test binary against +// this tree's fixtures. That needs a runtime override for the fixture dir, +// unknown scenarios skipped rather than failed, and a way to mark a fixture as +// expected to be refused (a populated require_feature() field). +// Perhaps the mere presence of an unknown require_feature() field would be enough, +// given that we can parse those out ourselves. #include "SLExecutorFixture.h" #include "Luau/FileUtils.h" diff --git a/tests/conformance/ares.lua b/tests/conformance/ares.lua index 78f52226..cac32c49 100644 --- a/tests/conformance/ares.lua +++ b/tests/conformance/ares.lua @@ -227,6 +227,17 @@ assert(ares.unpersist(ares.persist(userdata)) ~= nil) assert_round_trips(vector(1, 2, 3)) +-- Strings live in a table ahead of the root, so a repeat costs an index, not a copy +local long_str = string.rep("abcdefgh", 64) +local once = #ares.persist({long_str}) +local four_times = #ares.persist({long_str, long_str, long_str, long_str}) +assert(four_times - once < #long_str) + +-- The same string as key and value resolves to the same table entry +local str_key_val = round_trip({[long_str] = long_str, other = "other"}) +assert(str_key_val[long_str] == long_str) +assert(str_key_val.other == "other") + -- Integer values survive round-trips at full 64-bit width. -- The integer library is only registered when the integer fflags are enabled, -- and this file must still parse without them, so no integer literals here. diff --git a/tests/conformance/ares_errors.lua b/tests/conformance/ares_errors.lua index 2cf60275..e6ca3d8a 100644 --- a/tests/conformance/ares_errors.lua +++ b/tests/conformance/ares_errors.lua @@ -11,8 +11,34 @@ local function assert_decode_fails(data) end +local function assert_decode_fails_with(data, pattern) + local success, err = pcall(function() ares.unpersist(uperms, data) end) + print(err) + assert(not success and string.find(err, pattern) ~= nil) +end + assert_decode_fails(tab_ser:sub(#tab_ser - 2)) assert_decode_fails(tab_ser:gsub("ARES", "ARTS")) +-- The header is "ARES", major, minor and the record length, then number size, +-- test number, vector size and the reference count ahead of the two feature +-- masks. Bit 63 will never be a feature this build knows. Luau numbers are +-- doubles, so the u64 is packed as two halves. +local features_present_pos = 4 + 4 + 4 + 4 + 1 + 8 + 1 + 4 +local features_required_pos = features_present_pos + 8 +local unknown_bit = string.pack("baCK&4fB;t{wmnpgfdM896kue8iy|@51sORQ1VGX+fD(>DuEA_jwID$b>3hvi zrw)AH&%(hh0_2Mv1}fqb)B}+WV1rnIq5@p}P(AUf2KvP%i6yD=sd*(ul|mXoaW0^- zAV)c93NQ#m)fE)wm*nf)<`n0|7bh2G7L@2408Qb9nUbEIUz%5vTEqf(3qlTPLrG?C zY7q%$8v@M+dl6_p$c4H2Wtn;D@j!%+0GJK@}KN=0C7R$ z&XKo3ya90LA^Hkb;sIc0f9#+}qGgnL*E8RI^Ru%aKOgTLBU;9D`z@m*#9Be&w95I- z0d0>kv_C#hi2r2#^n?MUvUhIp!}gEwzrHQ^mL9_U@H&7f|FjHGO^+x>P8%P0bYXY* zS(1sW9(;j7bd{e06E$XB20N%f&oAMk^_ViFz^p`~JAVm#FxzC8v~7QjZ==CEXy{+n z9zL>p*BBXS7gAP*EZhfa69#o$C8cyXz``gdhLRPtB%cPg3@gb$f?{DYHhrswUQfS^_Z> zxeQ)cVOfgGhqGJ2xw#IUjG8`^sCl(Q)mQ-f=hx4BNZ$#gcFf{pxDR@&>c{esUda*a zyhYmc#d5e|n7VOM6^%@C;l52oI&moJfepc{27}9r+1p{!ua{x58avZA>)#-bgH}xU z3jA&1XJLK}3A1ZO^?-+Y&^cbIsg6Z?q|KR_Y}_U z9}q-?F0RCtuLJ6nheb}r{m5x#m8E^PBPaMbe#O`miz4vceYRCcVjb|cBS+qgt+`Pzsym1crFi0yTY5Hn;KEeTZy`+E(FQDp#az%ZXS?2+km@_)MsWPY&|3(4qUUc>iXaGo4%s ro$)J?N%8{XP!4D9_pnhX58@kT0dsxgx#I%!<*4w>xC(jCB&hHgf;fk# diff --git a/tests/conformance/fixtures/exec/exec2.0-ares6.0-errored-main.lsl.state b/tests/conformance/fixtures/exec/exec2.0-ares6.0-errored-main.lsl.state deleted file mode 100644 index 38afcfb1a2d71f4d3f60e3a05eda0747aca11ec5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 86538 zcmeI5*=`(V5QWQ5AeP02Eg1=kMG-;>A&6#k_GO&J*~z|MSqZUBBqv63kjPz2JXrdg zhu|5Icmbx{9ek<}S6neYUoJeJ0q`JPT`g>V_ynbOm_WrHQ_b(eQVm{`VCvRTu zyg9u2YUh`&pANs=e7W=b?dSg{E^hB`Z|wf~`o+(07U$;vSwp^gFScLb-kFQFyX*Hp zY+{!tuM404@!R>|%fH?oTsW{fdB1Xh612Gd?)Kzs)C-L3=%vLE>eq;kt?k}udF4<{ z0^)fi9@u;R^W4Ekh(?KVTUjt=$&?jSJ~8DpQx2Q*r72&T^0g`7nDVVD-17Gp}>v=~$3mc^J7 zx7#seQwmc`Q{s-TV@hF4X-eF+bxbKtDNTucwvH) z#6w%hl){wKlz3$8m{OQhni7v~9a9QZN>k#={yG!-B~p%7A?3b1&1V8gS${}s+<=tg zMgb|s4MwPAVDTq`EQnoNUcrd(2+hrPTUJN2zYmQHmRoGLgXc zBq3$;$Mz(rZkMw9b5dV9qOXLOb&}Ip(pPG=FLAo$M)7 zM>!laeP#Mu9R+lhkTO+6NLkylcU(ZqlpC{1$`g%H#&5l)BMD zQl?$XKIgr{Ny&FOskIPJsvB@JxdA7oZgh~7?RySzA3I1X84oG78bV5S15ze8Af?oe z4w5o;lmj>kCnewEq}D<>scyi@scyi@scyi@bNy&FOskIPJsvB@JxdA7oZgicKBkqO{CnewEq}D<> zscyi@eKTQtAQy zq#phBlj;WjWOBpES(*~}+MmJ4rWB@>ro??)$CSd9(v)~$>zGoQ!tIR%tH^PY5^||YixhtH6leGhIGSx!%C*`<+lPNc5j+1?^3xShT z57?hf_2AO}9A>ZIE=-Bb7Gp|Wu^3b0s>PTR*DS`AxNb3~#0`rvg?>^q0ZytLGuTfa z<9ye4&4?qW^i1iSGB9Om%E**eQ;wQ)%#`D%oG|62DW^<1ZOR!_&YE)0lyO@rOo=s% zF(uAhj45%!VoZsP7Gp}RTZ}23$D`Ge^LUaQwqseE!g)OE*fj4?_P8Gp{bZfY>`yXx z)v886scz6usvEP{PxiUb0Gx!AwF7W6)k5|s<+y;8DK}=dTNEsHTFZntB`rWB@>roast{+%?rO=B{#F&`+k^ zn7Mv(2q)pB)C2m-R1fGUB{w+VRdb`T*=%VF=ew$7(|o?`ko)n#NvVZ!Qjct|3sE=V zWOBo(2q%+2)^koXCkI?V2`B6HX70+|RqGtrg{T|MUDb^lye@?EUEw60tQ~-psTQ(7 zDaQqzOu1pb;@FhJlxglKd(2(oq|^iEuBjfdKPkDv+*Na-GGzH4LB)vqwAdP(NDrj$#*!ZwGd9K z8*nnY0VkzybdZzFtCPQPajyO&tFKFOsQx};W8=lE=bOKdmzUzBM%vomdG>UBcjMX9 z-Oc!ZBF;BIh?Ogo?-tjp_m#f4UiHJpSZLmVQN3;cecZh6$G!ha{raDNY4V$Y19xwk Ang9R* diff --git a/tests/conformance/fixtures/exec/exec2.0-ares6.0-state-change-pending.lsl.state b/tests/conformance/fixtures/exec/exec2.0-ares6.0-state-change-pending.lsl.state deleted file mode 100644 index 45062e4b2af60cc139d7a2aa153ad085b107d003..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 668 zcmZ>baCK&4fB;t{wmnpgfdM896kue8iNYBmQK%qHAv2V32^4S)at&sK@&$l2hxEN> zr&9+$?`Pp)mH_faM1YF81nYrja6t^>0E!B5@j>;(ryA%Nmn4>?#-~twAu=dJ;T}BGOgxDDJNG1@z)Ablp%2-F8WW=-D^$h4d&sgfHO( z=uDH8y~)m@nfYeEuU~%hhx~Q}T;wbHjmH2&7lA11QL20q*3gS$5R_-goZRL21K%Jp zKKy>2etrDBvv@PbI^{UTlQY!dPLLrg_^#(jHOh=Ftdgp+`ep9~agMTNYJ4hS-_I>| zYuoI)HnlX5x?0+7geu!BDz;iEy$1gR`N6p?=WV0NSr>wL`@@{Uo?9*zr(Vx3i))a#k1x=Lq)KlD=WZyAvUN%M)Niep%Q8 v3%A5R{lI`jj4A0Hd5D$+pNDx@9^(>_@Q2{w7sI^_((A*?6X}2Xo`lIC9FauU diff --git a/tests/conformance/fixtures/exec/exec2.0-ares6.0-yielded-handler.lua.state b/tests/conformance/fixtures/exec/exec2.0-ares6.0-yielded-handler.lua.state deleted file mode 100644 index 8687cfb5128235735a08bdda58cf477c2f40adec..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1149 zcmcIjO>fjN5Ovy878i*)TQ03y^-uvtf-E2e98fof+XTxR=XpLnGs#c$cPp$zz9b)d9bCh8*+xe#V z-|hr-wzFMxGT$ zu_tsJ?Q+;0VK7|#pbtJ+@(OgsP#W|Yn3u*C!$_^Y@VTf)TKO8y2K^nn%=Z#!M9V$4 zf(*M{oglVoyKc}jH?-LuqBD#)!s7sp)23Bw(%5zGYK$iAI;LChdwCwmogdN~G-24o y>;DkCQeAqf!dDc*5!wrj)FIT zgug-X;!S7vqb>B{9QM6^v-5UlH@peY7eGZg!@F7oFxiP%W(mV=XBQyrJbaCK&8zz$sT$l1fSFu_?2jEo32f`u-~$iW~0lJx*eI0m@}vqP220cj5Dd(BR# z4t(Cv%ncPqrX~P6oLv0zsRsJRC5a`e@u_(wMU@P~1x5KK`TDjw#X0fC$wiq3CHe+j zobl<&`K5U!sYO60P(w*(ZfX%WrG{JrAo<+<5z-$IbCI?LcCI$g!79If>6hT%70X7r? wb_PKnkS1V=FbZ%u2nujA=rS>gh=If$nRJ1|TnrqHY(Or9D3AjL=zO3M0Hqc|AOHXW literal 0 HcmV?d00001 diff --git a/tests/conformance/fixtures/exec/exec2.0-ares6.0-between-handlers.lua.sluac b/tests/conformance/fixtures/exec/exec3.0-ares7.0-between-handlers.lua.sluac similarity index 100% rename from tests/conformance/fixtures/exec/exec2.0-ares6.0-between-handlers.lua.sluac rename to tests/conformance/fixtures/exec/exec3.0-ares7.0-between-handlers.lua.sluac diff --git a/tests/conformance/fixtures/exec/exec3.0-ares7.0-between-handlers.lua.state b/tests/conformance/fixtures/exec/exec3.0-ares7.0-between-handlers.lua.state new file mode 100644 index 0000000000000000000000000000000000000000..1831137fb9386cc4493ef9adc5bfa00e60a1d963 GIT binary patch literal 1038 zcmaJ=%Wl&^6umRf!<_^=lSk=`szTXNsko#pV1a6b!iH{mEZAALGenjgJF>?Cc1SE? z&sXpTNWhv6f6!k5i67vOYdMV)F|vKH&)jqFW9-qZ(PIPFuhD<@&MFG6JRl4AinoHA z!1eY=%}aGFGcc>(YJNVQ ztrI!XL6_mI9(z-nG2#L783|Y;mYI@cW+)jmV=e>M43a!nSY*?6!+0#;LVFS=z87V# z?`5)Q;0ZGE3unvl1UV<&uQ+seAR5{f^Y4Dv?oIKjL@1|Xc@S(TfL|F#vFQFTh?kX3PzP{gn?-k`h6SC{S>A(btP#ck1}u zbg)0pJKG51AdA8U92JZr4hA;wICb6?_lppqw3-v|l`h!4&$mQ5HX!^4sm6)?ZQwhY q?7jZ|Y`ko~TWN^4mc@z*-@`HN!!d>4md3DSgt05K=xX&HBmV*J^_ge@ literal 0 HcmV?d00001 diff --git a/tests/conformance/fixtures/exec/exec2.0-ares6.0-errored-handler.lua.sluac b/tests/conformance/fixtures/exec/exec3.0-ares7.0-errored-handler.lua.sluac similarity index 100% rename from tests/conformance/fixtures/exec/exec2.0-ares6.0-errored-handler.lua.sluac rename to tests/conformance/fixtures/exec/exec3.0-ares7.0-errored-handler.lua.sluac diff --git a/tests/conformance/fixtures/exec/exec3.0-ares7.0-errored-handler.lua.state b/tests/conformance/fixtures/exec/exec3.0-ares7.0-errored-handler.lua.state new file mode 100644 index 0000000000000000000000000000000000000000..a251eb974ab0a0610df11d74278dff2ac203b361 GIT binary patch literal 758 zcmbVK%}(Pm5FV#34Q?wrEpR~~ACS-<*syR~Bt)|)x1N9l5<-@tE?9Dt$Z>%?53~1u z0rs*NPCNn6fW!-|od)_N#DPeTXTEQ~(KjQVXB+h_n1s%gpTU?on=x7-oi=l**(p?r0(vy_M#gZVseS{Y^YwPY0vq(#SNSuBXOl zhvyTBUGt=`M0_I>gVDQ>nZTdP0DK??;sHcQ@o~MR*X`}EKHvX({QLS)S)LQQ^q)N~ zXoMTI*E?}ua$GAsK_}CxNyGh#?{FJ8hv|(>u2^okHXoJzfX1lL4G(vIhC>4zj0r9C z?@FnnYtD*`2{#uLlbSWPi}Pci~!aS0Wzqf}L@UpnN4S2Z7s~UHlZ(#^b8U>wY7y zkW{}QtXUy+1G3OwbnOu8wn;szSi~obQH|iJLK;ymXcjC0B2CM+V9SH06^BLMmKzd$ OI;Cy?N#bPR#ga%f6JIFx{AY)GI59H^ljTf(7Ky|4mADUy>^gi7t* zNPPeviIKQNh)3WVD)j}Jc<=a+5Na=2z59H@e6ze~##q)b9(V3_E)3cGADw@BZh3G> z)vEP!XJz%t#Nx{8?@PZ`-ZjNTCcnI;PIp1p42Lp!8zWFjp4i7UtXEb z&3A5%#I^g%?da~;f1Q6({PS$6_0}0Sa-lp$THDlJ>NQ&ZP3uJ1YxFnS$*>oOt+3Y$ z2Q;((b7@=>_L|N9uz4zM;#tC`8CGWDLTOl;hLdISMTFzB_*~d)g;O`;)x&dH7`EaS zX2a_YXG!A~!mt^K@deI=mGKJkm4*Lb7Oxi$Z^lhHOBRkxTK(zzzgDrfSyHLb53ldpLyMoRzt0VCq_qb*s!WvQ z%3aDm$_eFu<)m^-c}RIgc~p5!*-)m+OxaYnlqZxYl{3m&Jh0=a5*Orf@3N1D>c;9SR2`&9U`#H4q--oskftK(s zSd81D_#x0e&@vKv@{1z3Q^8DLNgC~XlG=3D#&@pI=Bm@>xMri3&E>A`D0j-oyZhzn4Sgc2rT_o{ literal 0 HcmV?d00001 diff --git a/tests/conformance/fixtures/exec/exec2.0-ares6.0-state-change-pending.lsl.sluac b/tests/conformance/fixtures/exec/exec3.0-ares7.0-state-change-pending.lsl.sluac similarity index 100% rename from tests/conformance/fixtures/exec/exec2.0-ares6.0-state-change-pending.lsl.sluac rename to tests/conformance/fixtures/exec/exec3.0-ares7.0-state-change-pending.lsl.sluac diff --git a/tests/conformance/fixtures/exec/exec3.0-ares7.0-state-change-pending.lsl.state b/tests/conformance/fixtures/exec/exec3.0-ares7.0-state-change-pending.lsl.state new file mode 100644 index 0000000000000000000000000000000000000000..ea1ed42ef555c39c34b2557d80bd622a531af0e8 GIT binary patch literal 525 zcmZ>baCK&8zz$sT$l1fSFu_?2jEpcgoB`<5Md#OBjR;it}VnpOlzW9G{d}oGLN{=zcE26c7mv6%GMLAuceR!I8;9Q-Fy=JXKrVwQ LkOKthe4r2jO?6Tk literal 0 HcmV?d00001 diff --git a/tests/conformance/fixtures/exec/exec2.0-ares6.0-yielded-handler.lsl.sluac b/tests/conformance/fixtures/exec/exec3.0-ares7.0-yielded-handler.lsl.sluac similarity index 100% rename from tests/conformance/fixtures/exec/exec2.0-ares6.0-yielded-handler.lsl.sluac rename to tests/conformance/fixtures/exec/exec3.0-ares7.0-yielded-handler.lsl.sluac diff --git a/tests/conformance/fixtures/exec/exec3.0-ares7.0-yielded-handler.lsl.state b/tests/conformance/fixtures/exec/exec3.0-ares7.0-yielded-handler.lsl.state new file mode 100644 index 0000000000000000000000000000000000000000..1bc7fda011a24890e15e394105101660cbe3ca51 GIT binary patch literal 477 zcmaJ-O-sW-5S`s@zSe*nttY{|NJ;fBHkKT{=taCNsa--KP0D5wJ$n{B_3!A}AK{7C)en-e!0HS~$TR z9Dl!Fe|`RZID2dMZ+gQbEJ2=2a`Fd8-CXd=Zat?2Z3rf*kN) z3@II-3JSv6As7f3!`mT!h$GvC>mVW+$0CFwg~6HSQaW@Yby&nGdv`nz0rilFBfHc9 QgSeF7+7Y5gwwg7;3b2Yn2mk;8 literal 0 HcmV?d00001 diff --git a/tests/conformance/fixtures/exec/exec2.0-ares6.0-yielded-handler.lua.sluac b/tests/conformance/fixtures/exec/exec3.0-ares7.0-yielded-handler.lua.sluac similarity index 100% rename from tests/conformance/fixtures/exec/exec2.0-ares6.0-yielded-handler.lua.sluac rename to tests/conformance/fixtures/exec/exec3.0-ares7.0-yielded-handler.lua.sluac diff --git a/tests/conformance/fixtures/exec/exec3.0-ares7.0-yielded-handler.lua.state b/tests/conformance/fixtures/exec/exec3.0-ares7.0-yielded-handler.lua.state new file mode 100644 index 0000000000000000000000000000000000000000..28b0ec3cb03acfacb39b6157ba11de9513847e16 GIT binary patch literal 829 zcmbVKv2N5r5S{hK_MYz`yS|Wuh$ux0C=y>nMS*e)a+eAy=vGe7IxOwkmc4hRB+AF| z4>VBo5lH+25Qa6>Yl9y(c z>GrZIYMV@>@v>SMQCd42`sg{dww)9Q02wZZ%9)9%cNHV$i?Q;sF*vgd-1ju8C9LQ3!E&D-5vP_GFjngBZ5K!+mUykAakwBC0n~98{==C=@{i oe2X>(k4Q%YG9O!bpmyn0{V)K=JWh#V4;?Xq*N7^%QDF|?5A%9?g8%>k literal 0 HcmV?d00001 diff --git a/tests/conformance/fixtures/exec/exec2.0-ares6.0-yielded-main.lua.sluac b/tests/conformance/fixtures/exec/exec3.0-ares7.0-yielded-main.lua.sluac similarity index 100% rename from tests/conformance/fixtures/exec/exec2.0-ares6.0-yielded-main.lua.sluac rename to tests/conformance/fixtures/exec/exec3.0-ares7.0-yielded-main.lua.sluac diff --git a/tests/conformance/fixtures/exec/exec3.0-ares7.0-yielded-main.lua.state b/tests/conformance/fixtures/exec/exec3.0-ares7.0-yielded-main.lua.state new file mode 100644 index 0000000000000000000000000000000000000000..b58cc01ec605b5742c79e57499472bb9dbc821e2 GIT binary patch literal 550 zcmaKpyH3L}6o!wT=F+qhPEsT!M8(jBp@f1uFrX%&Y?-)7Ojs_pP*B=NNeb*d1|tsw zGaE0!Ga#`rVuz%l2w}-SzyHt2KJp39gChn09>G6sv`oQERTHS(SFwiSQ2_rqIMGYc z713CKyFYw+{&-SW>c_?YK}-``Zf`RTPopVMGiWU)ENAZSzB|pKM=4LEY&(wQyuh>w zl;7}KPH$6|Bs{HB_I6~75r%=JnN-!C#RH|SQyR@C{0`bz@qEnUjE-5x!Aa72K6ek| z1*6$z8YQ_q=)B7Mtc#*b1Py|mq10VVD=e{qV@bL3%keCN?WS}Q@@%5`if|f4^;M$z z3ekPlY8Mm$5ks&h^f7eKr03(l2DT}s_)F|<5fjR84!v)2{aFflAC+j$(@Ze1EDBa- UcMLI+h7m#_njskaVe$}s0cq<=2><{9 literal 0 HcmV?d00001