diff --git a/JSTests/wasm/stress/armv7-fused-branch-compare-unsigned-ge-zero.js b/JSTests/wasm/stress/armv7-fused-branch-compare-unsigned-ge-zero.js new file mode 100644 index 0000000000000..19db69dd731b8 --- /dev/null +++ b/JSTests/wasm/stress/armv7-fused-branch-compare-unsigned-ge-zero.js @@ -0,0 +1,74 @@ +//@ requireOptions("--useBBQJIT=1", "--useWasmLLInt=0", "--useOMGJIT=0") + +import { instantiate } from "../wabt-wrapper.js" +import * as assert from "../assert.js" + +// BBQ fuses a comparison into a following br_if/if and emits the inverted condition, so +// the jump it hands back is the "branch not taken" edge. i64.ge_u inverts to Below, and +// every unsigned value is >= 0, so these reach the never-taken case of +// MacroAssemblerARMv7::branch64(Below, hi, lo, TrustedImm64(0)). +let wat = ` +(module + (func (export "geUnsignedZeroBrIf") (param $x i64) (result i32) + (block $done + (br_if $done (i64.ge_u (local.get $x) (i64.const 0))) + (return (i32.const 0)) + ) + (i32.const 1) + ) + + ;; Constant on the left: i64.le_u inverts to Above, which emitBranchI64 commutes to Below. + (func (export "zeroLeUnsignedBrIf") (param $x i64) (result i32) + (block $done + (br_if $done (i64.le_u (i64.const 0) (local.get $x))) + (return (i32.const 0)) + ) + (i32.const 1) + ) + + ;; The if-fusion path, which stores the jump in ControlData::m_ifBranch. + (func (export "geUnsignedZeroIf") (param $x i64) (result i32) + (if (result i32) (i64.ge_u (local.get $x) (i64.const 0)) + (then (i32.const 1)) + (else (i32.const 0)) + ) + ) + + ;; i64.lt_u inverts to AboveOrEqual, the always-taken sibling of the case above. + (func (export "ltUnsignedZeroBrIf") (param $x i64) (result i32) + (block $done + (br_if $done (i64.lt_u (local.get $x) (i64.const 0))) + (return (i32.const 0)) + ) + (i32.const 1) + ) + + ;; A non-zero constant is genuinely conditional and misses the compare-with-zero paths. + (func (export "geUnsignedTenBrIf") (param $x i64) (result i32) + (block $done + (br_if $done (i64.ge_u (local.get $x) (i64.const 10))) + (return (i32.const 0)) + ) + (i32.const 1) + ) +) +` + +async function test() { + const instance = await instantiate(wat, {}, {}) + const { geUnsignedZeroBrIf, zeroLeUnsignedBrIf, geUnsignedZeroIf, ltUnsignedZeroBrIf, geUnsignedTenBrIf } = instance.exports + + // Cover values whose high word, low word, or neither is zero. + for (const x of [0n, 1n, 0xffffffffn, 0x100000000n, 0xffffffff00000000n, 0xffffffffffffffffn]) { + assert.eq(geUnsignedZeroBrIf(x), 1) + assert.eq(zeroLeUnsignedBrIf(x), 1) + assert.eq(geUnsignedZeroIf(x), 1) + assert.eq(ltUnsignedZeroBrIf(x), 0) + } + + assert.eq(geUnsignedTenBrIf(9n), 0) + assert.eq(geUnsignedTenBrIf(10n), 1) + assert.eq(geUnsignedTenBrIf(0xffffffffffffffffn), 1) +} + +await assert.asyncTest(test()) diff --git a/JSTests/wasm/stress/array-element-creation.js b/JSTests/wasm/stress/array-element-creation.js index 7f8b8da9a5b5d..7387e29be3575 100644 --- a/JSTests/wasm/stress/array-element-creation.js +++ b/JSTests/wasm/stress/array-element-creation.js @@ -1,3 +1,4 @@ +//@ skip if $memoryLimited //@ runDefault("--useConcurrentJIT=0") function main() { diff --git a/Source/JavaScriptCore/assembler/ARMv7Assembler.h b/Source/JavaScriptCore/assembler/ARMv7Assembler.h index b07a820c2188d..2b96e7b733219 100644 --- a/Source/JavaScriptCore/assembler/ARMv7Assembler.h +++ b/Source/JavaScriptCore/assembler/ARMv7Assembler.h @@ -417,6 +417,11 @@ class ARMv7Assembler { ConditionInvalid } Condition; + static Condition invert(Condition cond) + { + return static_cast(cond ^ 1); + } + #define JUMP_ENUM_WITH_SIZE(index, value) (((value) << 3) | (index)) #define JUMP_ENUM_SIZE(jump) ((jump) >> 3) enum JumpType { JumpFixed = JUMP_ENUM_WITH_SIZE(0, 0), @@ -617,6 +622,8 @@ class ARMv7Assembler { OP_VLDR = 0xED10, OP_VMOV_CtoS = 0xEE00, OP_VMOV_StoC = 0xEE10, + OP_VMLA_T2 = 0xEE00, + OP_VMLS_T2 = 0xEE10, OP_VMUL_T2 = 0xEE20, OP_VADD_T2 = 0xEE30, OP_VSUB_T2 = 0xEE30, @@ -711,6 +718,8 @@ class ARMv7Assembler { OP_VLDRb = 0x0A00, OP_VMOV_IMM_T2b = 0x0A00, OP_VMOV_T2b = 0x0A40, + OP_VMLA_T2b = 0x0A00, + OP_VMLS_T2b = 0x0A00, OP_VMUL_T2b = 0x0A00, OP_FSTSb = 0x0A00, OP_VSTRb = 0x0A00, @@ -2230,6 +2239,11 @@ class ARMv7Assembler { m_formatter.vfpOp(OP_VCMP, OP_VCMPb, true, VFPOperand(4), rd, rm); } + void vcmpz(FPSingleRegisterID rd) + { + m_formatter.vfpOp(OP_VCMP, OP_VCMPb, false, VFPOperand(5), rd, VFPOperand(0)); + } + void vcmpz(FPDoubleRegisterID rd) { m_formatter.vfpOp(OP_VCMP, OP_VCMPb, true, VFPOperand(5), rd, VFPOperand(0)); @@ -2344,6 +2358,11 @@ class ARMv7Assembler { m_formatter.vfpOp(OP_VMUL_T2, OP_VMUL_T2b, true, rn, rd, rm); } + void vmla(FPDoubleRegisterID rd, FPDoubleRegisterID rn, FPDoubleRegisterID rm) + { + m_formatter.vfpOp(OP_VMLA_T2, OP_VMLA_T2b, true, rn, rd, rm); + } + void vstr(FPDoubleRegisterID rd, RegisterID rn, int32_t imm) { m_formatter.vfpMemOp(OP_VSTR, OP_VSTRb, true, rn, rd, imm); diff --git a/Source/JavaScriptCore/assembler/MacroAssemblerARMv7.h b/Source/JavaScriptCore/assembler/MacroAssemblerARMv7.h index e2b80afe45cbd..9c6b6f38579c8 100644 --- a/Source/JavaScriptCore/assembler/MacroAssemblerARMv7.h +++ b/Source/JavaScriptCore/assembler/MacroAssemblerARMv7.h @@ -164,6 +164,57 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { LessThanOrEqual = ARMv7Assembler::ConditionLE }; + static constexpr ARMv7Assembler::Condition armV7ConditionForHigh32(RelationalCondition cond) + { + switch (cond) { + case GreaterThan: + case GreaterThanOrEqual: + return ARMv7Assembler::ConditionGT; + case LessThan: + case LessThanOrEqual: + return ARMv7Assembler::ConditionLT; + case Above: + case AboveOrEqual: + return ARMv7Assembler::ConditionHI; + case Below: + case BelowOrEqual: + return ARMv7Assembler::ConditionLO; + case NotEqual: + return ARMv7Assembler::ConditionNE; + case Equal: + // Equal can never be determined from high alone (needs both parts to match) + return ARMv7Assembler::ConditionInvalid; + default: + RELEASE_ASSERT_NOT_REACHED(); + return ARMv7Assembler::ConditionInvalid; + } + } + + static constexpr ARMv7Assembler::Condition armV7ConditionForLow32(RelationalCondition cond) + { + switch (cond) { + case GreaterThan: + case Above: + return ARMv7Assembler::ConditionHI; + case GreaterThanOrEqual: + case AboveOrEqual: + return ARMv7Assembler::ConditionHS; + case LessThan: + case Below: + return ARMv7Assembler::ConditionLO; + case LessThanOrEqual: + case BelowOrEqual: + return ARMv7Assembler::ConditionLS; + case NotEqual: + return ARMv7Assembler::ConditionNE; + case Equal: + return ARMv7Assembler::ConditionEQ; + default: + RELEASE_ASSERT_NOT_REACHED(); + return ARMv7Assembler::ConditionInvalid; + } + } + enum ResultCondition { Carry = ARMv7Assembler::ConditionCS, Overflow = ARMv7Assembler::ConditionVS, @@ -372,6 +423,125 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { m_assembler.clz(dest, dest); } + void countLeadingZeros64(RegisterID srcHi, RegisterID srcLo, RegisterID destHi, RegisterID destLo) + { + if (destLo != srcLo) { + m_assembler.clz(destLo, srcHi); + m_assembler.cmp(destLo, ARMThumbImmediate::makeEncodedImm(32)); + Jump done = makeBranch(ARMv7Assembler::ConditionNE); + m_assembler.clz(destLo, srcLo); + m_assembler.add(destLo, destLo, ARMThumbImmediate::makeEncodedImm(32)); + done.link(this); + xor32(destHi, destHi); + } else { + RegisterID scratch = getCachedDataTempRegisterIDAndInvalidate(); + move(srcLo, scratch); + m_assembler.clz(destLo, srcHi); + m_assembler.cmp(destLo, ARMThumbImmediate::makeEncodedImm(32)); + Jump done = makeBranch(ARMv7Assembler::ConditionNE); + m_assembler.clz(destLo, scratch); + m_assembler.add(destLo, destLo, ARMThumbImmediate::makeEncodedImm(32)); + done.link(this); + xor32(destHi, destHi); + } + } + + void countTrailingZeros64(RegisterID srcHi, RegisterID srcLo, RegisterID destHi, RegisterID destLo) + { + if (destLo != srcHi) { + countTrailingZeros32(srcLo, destLo); + m_assembler.cmp(destLo, ARMThumbImmediate::makeEncodedImm(32)); + Jump done = makeBranch(ARMv7Assembler::ConditionNE); + countTrailingZeros32(srcHi, destLo); + m_assembler.add(destLo, destLo, ARMThumbImmediate::makeEncodedImm(32)); + done.link(this); + xor32(destHi, destHi); + } else { + RegisterID scratch = getCachedDataTempRegisterIDAndInvalidate(); + move(srcHi, scratch); + countTrailingZeros32(srcLo, destLo); + m_assembler.cmp(destLo, ARMThumbImmediate::makeEncodedImm(32)); + Jump done = makeBranch(ARMv7Assembler::ConditionNE); + countTrailingZeros32(scratch, destLo); + m_assembler.add(destLo, destLo, ARMThumbImmediate::makeEncodedImm(32)); + done.link(this); + xor32(destHi, destHi); + } + } + + void compare64(RelationalCondition cond, RegisterID lhsHi, RegisterID lhsLo, RegisterID rhsHi, RegisterID rhsLo, RegisterID dest) + { + if (cond == RelationalCondition::Equal || cond == RelationalCondition::NotEqual) { + // For Equal/NotEqual, we can optimize to only set one value conditionally + // NotEqual: default to 1, change to 0 only if both parts equal + // Equal: default to 0, change to 1 only if both parts equal + if (dest != lhsHi && dest != rhsHi && dest != lhsLo && dest != rhsLo) { + m_assembler.mov(dest, ARMThumbImmediate::makeEncodedImm(cond == RelationalCondition::NotEqual ? 1 : 0)); + m_assembler.cmp(lhsHi, rhsHi); + Jump done = makeBranch(ARMv7Assembler::ConditionNE); + m_assembler.cmp(lhsLo, rhsLo); + // Only need to set the "opposite" value when both parts match + m_assembler.it(ARMv7Assembler::ConditionEQ); + m_assembler.mov(dest, ARMThumbImmediate::makeEncodedImm(cond == RelationalCondition::NotEqual ? 0 : 1)); + done.link(this); + } else { + RegisterID scratch = getCachedDataTempRegisterIDAndInvalidate(); + m_assembler.mov(scratch, ARMThumbImmediate::makeEncodedImm(cond == RelationalCondition::NotEqual ? 1 : 0)); + m_assembler.cmp(lhsHi, rhsHi); + Jump done = makeBranch(ARMv7Assembler::ConditionNE); + m_assembler.cmp(lhsLo, rhsLo); + m_assembler.it(ARMv7Assembler::ConditionEQ); + m_assembler.mov(scratch, ARMThumbImmediate::makeEncodedImm(cond == RelationalCondition::NotEqual ? 0 : 1)); + done.link(this); + move(scratch, dest); + } + return; + } + + ARMv7Assembler::Condition hiCondition = armV7ConditionForHigh32(cond); + ARMv7Assembler::Condition loCondition = armV7ConditionForLow32(cond); + + if (dest != lhsLo && dest != rhsLo && dest != lhsHi && dest != rhsHi) { + // No aliasing - use ITE blocks with 1 branch + m_assembler.cmp(lhsHi, rhsHi); + m_assembler.it(hiCondition, false); + m_assembler.mov(dest, ARMThumbImmediate::makeEncodedImm(1)); + m_assembler.mov(dest, ARMThumbImmediate::makeEncodedImm(0)); + + Jump done = makeBranch(ARMv7Assembler::ConditionNE); + + m_assembler.cmp(lhsLo, rhsLo); + m_assembler.it(loCondition, false); + m_assembler.mov(dest, ARMThumbImmediate::makeEncodedImm(1)); + m_assembler.mov(dest, ARMThumbImmediate::makeEncodedImm(0)); + + done.link(this); + } else { + // dest aliases with source - use scratch + RegisterID scratch = getCachedDataTempRegisterIDAndInvalidate(); + + m_assembler.cmp(lhsHi, rhsHi); + m_assembler.it(hiCondition, false); + m_assembler.mov(scratch, ARMThumbImmediate::makeEncodedImm(1)); + m_assembler.mov(scratch, ARMThumbImmediate::makeEncodedImm(0)); + + Jump done = makeBranch(ARMv7Assembler::ConditionNE); + + m_assembler.cmp(lhsLo, rhsLo); + m_assembler.it(loCondition, false); + m_assembler.mov(scratch, ARMThumbImmediate::makeEncodedImm(1)); + m_assembler.mov(scratch, ARMThumbImmediate::makeEncodedImm(0)); + + done.link(this); + move(scratch, dest); + } + } + + void lshiftUnchecked(RegisterID src, RegisterID shiftAmount, RegisterID dest) + { + m_assembler.lsl(dest, src, shiftAmount); + } + void lshift32(RegisterID src, RegisterID shiftAmount, RegisterID dest) { RegisterID scratch = getCachedDataTempRegisterIDAndInvalidate(); @@ -385,7 +555,10 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { void lshift32(RegisterID src, TrustedImm32 imm, RegisterID dest) { - m_assembler.lsl(dest, src, imm.m_value & 0x1f); + if (!(imm.m_value & 0x1f)) + move(src, dest); + else + m_assembler.lsl(dest, src, imm.m_value & 0x1f); } void lshift32(TrustedImm32 imm, RegisterID shiftAmount, RegisterID dest) @@ -574,6 +747,12 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { m_assembler.sub(scratch, ARMThumbImmediate::makeUInt12(32), scratch); m_assembler.ror(dest, src, scratch); } + + void rshiftUnchecked(RegisterID src, RegisterID shiftAmount, RegisterID dest) + { + m_assembler.asr(dest, src, shiftAmount); + } + void rshift32(RegisterID src, RegisterID shiftAmount, RegisterID dest) { RegisterID scratch = getCachedDataTempRegisterIDAndInvalidate(); @@ -597,12 +776,25 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { { rshift32(dest, shiftAmount, dest); } - + void rshift32(TrustedImm32 imm, RegisterID dest) { rshift32(dest, imm, dest); } + void rshift32(TrustedImm32 imm, RegisterID shiftAmount, RegisterID dest) + { + // Clamp the shift to the range 0..31 + m_assembler.ARM_and(dest, shiftAmount, ARMThumbImmediate::makeEncodedImm(0x1f)); + move(imm, getCachedDataTempRegisterIDAndInvalidate()); + m_assembler.asr(dest, dataTempRegister, dest); + } + + void urshiftUnchecked(RegisterID src, RegisterID shiftAmount, RegisterID dest) + { + m_assembler.lsr(dest, src, shiftAmount); + } + void urshift32(RegisterID src, RegisterID shiftAmount, RegisterID dest) { RegisterID scratch = getCachedDataTempRegisterIDAndInvalidate(); @@ -610,10 +802,10 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { ARMThumbImmediate armImm = ARMThumbImmediate::makeEncodedImm(0x1f); ASSERT(armImm.isValid()); m_assembler.ARM_and(scratch, shiftAmount, armImm); - + m_assembler.lsr(dest, src, scratch); } - + void urshift32(RegisterID src, TrustedImm32 imm, RegisterID dest) { if (!(imm.m_value & 0x1f)) @@ -626,7 +818,7 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { { urshift32(dest, shiftAmount, dest); } - + void urshift32(TrustedImm32 imm, RegisterID dest) { urshift32(dest, imm, dest); @@ -671,6 +863,17 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { } } + void sub32(TrustedImm32 imm, RegisterID src, RegisterID dest) + { + ARMThumbImmediate armImm = ARMThumbImmediate::makeUInt12OrEncodedImm(imm.m_value); + if (armImm.isValid()) + m_assembler.sub(dest, armImm, src); + else { + move(imm, dataTempRegister); + m_assembler.sub(dest, dataTempRegister, src); + } + } + void sub32(TrustedImm32 imm, Address address) { load32(address, dataTempRegister); @@ -1080,23 +1283,32 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { loadPair32(Address(scratch), dest1, dest2); } else { ASSERT(dest1 != dest2); // If it is the same, ldp becomes illegal instruction. - int32_t absOffset = address.u.offset; - if (absOffset < 0) - absOffset = -absOffset; - if (!(absOffset & ~0x3fc)) { - if ((dest1 == addressTempRegister) || (dest2 == addressTempRegister)) - invalidateCachedAddressTempRegister(); - if ((dest1 == dataTempRegister) || (dest2 == dataTempRegister)) - cachedDataTempRegister().invalidate(); - m_assembler.ldrd(dest1, dest2, address.base, address.u.offset, /* index: */ true, /* wback: */ false); - } else if (address.base == dest1) { + // Check if dest1 or dest2 aliases the base register to avoid UNPREDICTABLE ldrd behavior + if (address.base == dest1) { + // Load high word first to avoid clobbering base register ArmAddress highAddress(address.base, address.u.offset + 4); load32(highAddress, dest2); load32(address, dest1); - } else { + } else if (address.base == dest2) { + // Load low word first to avoid clobbering base register load32(address, dest1); ArmAddress highAddress(address.base, address.u.offset + 4); load32(highAddress, dest2); + } else { + int32_t absOffset = address.u.offset; + if (absOffset < 0) + absOffset = -absOffset; + if (!(absOffset & ~0x3fc)) { + if ((dest1 == addressTempRegister) || (dest2 == addressTempRegister)) + invalidateCachedAddressTempRegister(); + if ((dest1 == dataTempRegister) || (dest2 == dataTempRegister)) + cachedDataTempRegister().invalidate(); + m_assembler.ldrd(dest1, dest2, address.base, address.u.offset, /* index: */ true, /* wback: */ false); + } else { + load32(address, dest1); + ArmAddress highAddress(address.base, address.u.offset + 4); + load32(highAddress, dest2); + } } } } @@ -1106,6 +1318,18 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { loadPair32(setupArmAddress(address), dest1, dest2); } + void loadPair32Unaligned(Address address, RegisterID dest1, RegisterID dest2) + { + ASSERT(dest1 != dest2); + if (address.base == dest1) { + load32(address.withOffset(4), dest2); + load32(address, dest1); + } else { + load32(address, dest1); + load32(address.withOffset(4), dest2); + } + } + void loadPair32(BaseIndex address, RegisterID dest1, RegisterID dest2) { // Using r0-r7 can often be encoded with a shorter (16-bit vs 32-bit) instruction, so use @@ -1283,6 +1507,12 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { store16(dataTempRegister, address); } + void store16(TrustedImm32 imm, Address address) + { + move(imm, dataTempRegister); + store16(dataTempRegister, address); + } + void storeRel16(RegisterID src, Address address) { storeFence(); @@ -1303,26 +1533,11 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { void storePair32(TrustedImm32 imm1, TrustedImm32 imm2, Address address) { - // We cannot re-use the two register version of `storePair32` defined - // below here because we can only use the `addressTempRegister` as a - // scratch register if the `strd` case is taken. int32_t absOffset = address.offset; if (absOffset < 0) absOffset = -absOffset; - if (!(absOffset & ~0x3fc)) { - RegisterID src1 = getCachedAddressTempRegisterIDAndInvalidate(); - move(imm1, src1); - RegisterID src2 = src1; - if (imm1.m_value != imm2.m_value) { - src2 = getCachedDataTempRegisterIDAndInvalidate(); - move(imm2, src2); - } - ASSERT(src1 != address.base && src2 != address.base); - m_assembler.strd(src1, src2, address.base, address.offset, /* index: */ true, /* wback: */ false); - } else { - store32(imm1, address); - store32(imm2, address.withOffset(4)); - } + store32(imm1, address); + store32(imm2, address.withOffset(4)); } void storePair32(RegisterID src1, RegisterID src2, RegisterID dest) @@ -1340,12 +1555,9 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { int32_t absOffset = address.offset; if (absOffset < 0) absOffset = -absOffset; - if (!(absOffset & ~0x3fc)) - m_assembler.strd(src1, src2, address.base, address.offset, /* index: */ true, /* wback: */ false); - else { - store32(src1, address); - store32(src2, address.withOffset(4)); - } + // strd does not support unaligned accesses on some chips, so we avoid it. + store32(src1, address); + store32(src2, address.withOffset(4)); } void storePair32(RegisterID src1, RegisterID src2, BaseIndex address) @@ -1984,7 +2196,7 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { { m_assembler.vcvtds(dst, ARMRegisters::asSingle(src)); } - + void convertDoubleToFloat(FPRegisterID src, FPRegisterID dst) { m_assembler.vcvtsd(ARMRegisters::asSingle(dst), src); @@ -2063,10 +2275,8 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { Jump branchFloatWithZero(DoubleCondition cond, FPRegisterID left) { - UNUSED_PARAM(cond); - UNUSED_PARAM(left); - UNREACHABLE_FOR_PLATFORM(); - return { }; + m_assembler.vcmpz(asSingle(left)); + return makeFPBranch(cond); } Jump branchDouble(DoubleCondition cond, FPRegisterID left, FPRegisterID right) @@ -2077,10 +2287,8 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { Jump branchDoubleWithZero(DoubleCondition cond, FPRegisterID left) { - UNUSED_PARAM(cond); - UNUSED_PARAM(left); - UNREACHABLE_FOR_PLATFORM(); - return { }; + m_assembler.vcmpz(left); + return makeFPBranch(cond); } enum BranchTruncateType { BranchIfTruncateFailed, BranchIfTruncateSuccessful }; @@ -2133,7 +2341,7 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { m_assembler.vcvt_floatingPointToUnsigned(fpTempRegisterAsSingle(), asSingle(src)); m_assembler.vmov(dest, fpTempRegisterAsSingle()); } - + // Convert 'src' to an integer, and places the resulting 'dest'. // If the result is not representable as a 32 bit value, branch. // May also branch for some values that are representable in 32 bits @@ -2179,6 +2387,84 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { notEqual.link(this); return result; } +private: + void convertDoubleToUint64(FPRegisterID src, RegisterID destLo, RegisterID destHi, FPRegisterID scratch1, FPRegisterID scratch2) + { + // We override src on the vmla, so we require a temp fp register here + ASSERT(src == fpTempRegister); + + // Constant materialization + move(TrustedImm32(0x00000000), destLo); + move(TrustedImm32(0x3DF00000), destHi); + move64ToDouble(destHi, destLo, scratch1); + move(TrustedImm32(0x00000000), destLo); + move(TrustedImm32(0xC1F00000), destHi); + move64ToDouble(destHi, destLo, scratch2); + + m_assembler.vmul(scratch1, src, scratch1); + + m_assembler.vcvt_floatingPointToUnsigned(ARMRegisters::asSingle(scratch1), scratch1); + m_assembler.vmov(destHi, ARMRegisters::asSingle(scratch1)); + + m_assembler.vcvt_unsignedToFloatingPoint(scratch1, ARMRegisters::asSingle(scratch1)); + + m_assembler.vmla(src, scratch1, scratch2); + + m_assembler.vcvt_floatingPointToUnsigned(ARMRegisters::asSingle(src), src); + m_assembler.vmov(destLo, ARMRegisters::asSingle(src)); + } + +public: + void truncateDoubleToUint64(FPRegisterID src, RegisterID destLo, RegisterID destHi, FPRegisterID scratch1, FPRegisterID scratch2) + { + Jump notPositive = branchDoubleWithZero(DoubleLessThanOrEqualOrUnordered, src); + + moveDouble(src, fpTempRegister); + convertDoubleToUint64(fpTempRegister, destLo, destHi, scratch1, scratch2); + + Jump done = jump(); + + notPositive.link(this); + move(TrustedImm32(0), destLo); + move(TrustedImm32(0), destHi); + + done.link(this); + } + + void truncateDoubleToInt64(FPRegisterID src, RegisterID destLo, RegisterID destHi, FPRegisterID scratch1, FPRegisterID scratch2) + { + RegisterID signFlag = getCachedAddressTempRegisterIDAndInvalidate(); + moveDouble(src, fpTempRegister); + + Jump isNegative = branchDoubleWithZero(DoubleLessThanAndOrdered, fpTempRegister); + move(TrustedImm32(0), signFlag); + Jump join = jump(); + + isNegative.link(this); + m_assembler.vneg(fpTempRegister, fpTempRegister); + move(TrustedImm32(1), signFlag); + + join.link(this); + convertDoubleToUint64(fpTempRegister, destLo, destHi, scratch1, scratch2); + + Jump wasNonNegative = branch32(Equal, signFlag, TrustedImm32(0)); + m_assembler.sub_S(destLo, ARMThumbImmediate::makeUInt12OrEncodedImm(0), destLo); + m_assembler.mvn(destHi, destHi); + m_assembler.adc(destHi, destHi, ARMThumbImmediate::makeEncodedImm(0)); + wasNonNegative.link(this); + } + + void truncateFloatToUint64(FPRegisterID src, RegisterID destLo, RegisterID destHi, FPRegisterID scratch1, FPRegisterID scratch2) + { + convertFloatToDouble(src, fpTempRegister); + truncateDoubleToUint64(fpTempRegister, destLo, destHi, scratch1, scratch2); + } + + void truncateFloatToInt64(FPRegisterID src, RegisterID destLo, RegisterID destHi, FPRegisterID scratch1, FPRegisterID scratch2) + { + convertFloatToDouble(src, fpTempRegister); + truncateDoubleToInt64(fpTempRegister, destLo, destHi, scratch1, scratch2); + } // Stack manipulation operations: // @@ -2187,7 +2473,7 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { // operations add and remove a single register sized unit of data // to or from the stack. Peek and poke operations read or write // values on the stack, without moving the current stack position. - + void pop(RegisterID dest) { m_assembler.pop(dest); @@ -2472,6 +2758,11 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { m_assembler.cmp(left, scratch); } + void compare32AndSetFlags(RegisterID left, RegisterID right) + { + m_assembler.cmp(left, right); + } + void add32Impl(TrustedImm32 imm, Address address, bool updateFlags = false) { load32(address, dataTempRegister); @@ -2694,6 +2985,91 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { return branch32(cond, addressTempRegister, right16); } +private: + template + Jump branch64Impl(RelationalCondition cond, RegisterID leftHi, RegisterID leftLo, T rightHi, T rightLo) + { + if (cond == Equal) { + // Equal: bne done; cmp lo; beq target; done: + compare32AndSetFlags(leftHi, rightHi); + Jump done = makeBranch(ARMv7Assembler::ConditionNE); + compare32AndSetFlags(leftLo, rightLo); + Jump result = makeBranch(ARMv7Assembler::ConditionEQ); + done.link(this); + return result; + } + + if (cond == NotEqual) { + // NotEqual: branch taken if ANY part differs + compare32AndSetFlags(leftHi, rightHi); + Jump fromHi = makeBranch(ARMv7Assembler::ConditionNE); + compare32AndSetFlags(leftLo, rightLo); + Jump fromLo = makeBranch(ARMv7Assembler::ConditionNE); + Jump notTaken = jump(); + fromHi.link(this); + fromLo.link(this); + Jump result = jump(); + notTaken.link(this); + return result; + } + + ARMv7Assembler::Condition hiCond = armV7ConditionForHigh32(cond); + ARMv7Assembler::Condition loCond = armV7ConditionForLow32(cond); + ARMv7Assembler::Condition inverseLoCond = ARMv7Assembler::invert(loCond); + compare32AndSetFlags(leftHi, rightHi); + Jump fromHi = makeBranch(hiCond); + Jump notTakenHi = makeBranch(ARMv7Assembler::ConditionNE); + compare32AndSetFlags(leftLo, rightLo); + Jump notTakenLo = makeBranch(inverseLoCond); + fromHi.link(this); + Jump result = jump(); + notTakenHi.link(this); + notTakenLo.link(this); + return result; + } + +public: + Jump branch64(RelationalCondition cond, RegisterID leftHi, RegisterID leftLo, RegisterID rightHi, RegisterID rightLo) + { + return branch64Impl(cond, leftHi, leftLo, rightHi, rightLo); + } + + Jump branch64(RelationalCondition cond, RegisterID leftHi, RegisterID leftLo, TrustedImm64 right) + { + TrustedImm32 rightHi(static_cast(right.m_value >> 32)); + TrustedImm32 rightLo(static_cast(right.m_value)); + + // Optimize for comparing with zero (unsigned comparisons only) + if (!rightHi.m_value && !rightLo.m_value) { + if (cond == Below) + return branch32(Below, leftHi, TrustedImm32(0)); + if (cond == AboveOrEqual) + return jump(); // all unsigned values are >= 0 + + if (cond == Equal || cond == BelowOrEqual || cond == NotEqual || cond == Above) { + RegisterID scratch = getCachedDataTempRegisterIDAndInvalidate(); + m_assembler.orr_S(scratch, leftHi, leftLo); + return Jump(makeBranch((cond == Equal || cond == BelowOrEqual) ? ARMv7Assembler::ConditionEQ : ARMv7Assembler::ConditionNE)); + } + } + + return branch64Impl(cond, leftHi, leftLo, rightHi, rightLo); + } + + Jump branchTest64(ResultCondition cond, RegisterID regHi, RegisterID regLo) + { + if (cond == Signed || cond == PositiveOrZero) { + // For sign tests, only check the sign bit of the high 32 bits + m_assembler.tst(regHi, regHi); + return Jump(makeBranch((cond == Signed) ? ARMv7Assembler::ConditionMI : ARMv7Assembler::ConditionPL)); + } + + ASSERT(cond == Zero || cond == NonZero); + RegisterID scratch = getCachedDataTempRegisterIDAndInvalidate(); + m_assembler.orr_S(scratch, regHi, regLo); + return Jump(makeBranch(cond == Zero ? ARMv7Assembler::ConditionEQ : ARMv7Assembler::ConditionNE)); + } + Jump branchTest32(ResultCondition cond, RegisterID reg, RegisterID mask) { ASSERT(cond == Zero || cond == NonZero || cond == Signed || cond == PositiveOrZero); diff --git a/Source/JavaScriptCore/bytecode/DeferredCompilationCallback.cpp b/Source/JavaScriptCore/bytecode/DeferredCompilationCallback.cpp index 67ff2b2ffd927..f879fbc479335 100644 --- a/Source/JavaScriptCore/bytecode/DeferredCompilationCallback.cpp +++ b/Source/JavaScriptCore/bytecode/DeferredCompilationCallback.cpp @@ -26,6 +26,8 @@ #include "config.h" #include "DeferredCompilationCallback.h" +#include "StrongInlines.h" + namespace JSC { DeferredCompilationCallback::DeferredCompilationCallback() = default; diff --git a/Source/JavaScriptCore/dfg/DFGAdaptiveInferredPropertyValueWatchpoint.cpp b/Source/JavaScriptCore/dfg/DFGAdaptiveInferredPropertyValueWatchpoint.cpp index 492bac685f228..8f901f638b93f 100644 --- a/Source/JavaScriptCore/dfg/DFGAdaptiveInferredPropertyValueWatchpoint.cpp +++ b/Source/JavaScriptCore/dfg/DFGAdaptiveInferredPropertyValueWatchpoint.cpp @@ -29,6 +29,7 @@ #if ENABLE(DFG_JIT) #include "CodeBlock.h" +#include "CodeBlockInlines.h" #include "DFGCommon.h" #include diff --git a/Source/JavaScriptCore/heap/BlockDirectory.cpp b/Source/JavaScriptCore/heap/BlockDirectory.cpp index 2e5bf80bb5ce4..baeef2d509c6d 100644 --- a/Source/JavaScriptCore/heap/BlockDirectory.cpp +++ b/Source/JavaScriptCore/heap/BlockDirectory.cpp @@ -28,6 +28,7 @@ #include "BlockDirectoryInlines.h" #include "Heap.h" +#include "HeapInlines.h" #include "MarkedSpaceInlines.h" #include "SubspaceInlines.h" #include "SuperSampler.h" diff --git a/Source/JavaScriptCore/parser/SourceTaintedOrigin.cpp b/Source/JavaScriptCore/parser/SourceTaintedOrigin.cpp index 52923690182a1..47a83a9e69e71 100644 --- a/Source/JavaScriptCore/parser/SourceTaintedOrigin.cpp +++ b/Source/JavaScriptCore/parser/SourceTaintedOrigin.cpp @@ -27,6 +27,7 @@ #include "SourceTaintedOrigin.h" #include "CodeBlock.h" +#include "JSCellInlines.h" #include "StackVisitor.h" #include "VM.h" diff --git a/Source/JavaScriptCore/runtime/ExceptionScope.cpp b/Source/JavaScriptCore/runtime/ExceptionScope.cpp index 89af72bab77da..fe22f6895aecf 100644 --- a/Source/JavaScriptCore/runtime/ExceptionScope.cpp +++ b/Source/JavaScriptCore/runtime/ExceptionScope.cpp @@ -28,6 +28,7 @@ #include "ErrorInstance.h" #include "Exception.h" +#include "JSCJSValueInlines.h" #include #include #include diff --git a/Source/JavaScriptCore/wasm/WasmBBQJIT.cpp b/Source/JavaScriptCore/wasm/WasmBBQJIT.cpp index b9cb2b4578a1a..772777ec38dab 100644 --- a/Source/JavaScriptCore/wasm/WasmBBQJIT.cpp +++ b/Source/JavaScriptCore/wasm/WasmBBQJIT.cpp @@ -4631,6 +4631,396 @@ ALWAYS_INLINE void BBQJIT::didParseOpcode() { } +BBQJIT::BranchFoldResult BBQJIT::tryFoldFusedBranchCompare(OpType opType, ExpressionType operand) +{ + if (!operand.isConst()) + return BranchNotFolded; + switch (opType) { + case OpType::I32Eqz: + return operand.asI32() ? BranchNeverTaken : BranchAlwaysTaken; + case OpType::I64Eqz: + return operand.asI64() ? BranchNeverTaken : BranchAlwaysTaken; + default: + RELEASE_ASSERT_NOT_REACHED_WITH_MESSAGE("Op type '%s' is not a unary comparison and should not have been fused.\n", makeString(opType).characters()); + } + return BranchNotFolded; +} + +BBQJIT::Jump BBQJIT::emitFusedBranchCompareBranch(OpType opType, ExpressionType, Location operandLocation) +{ + // Emit the negation of the intended branch. + switch (opType) { + case OpType::I32Eqz: + return m_jit.branchTest32(ResultCondition::NonZero, operandLocation.asGPR()); + case OpType::I64Eqz: +#if USE(JSVALUE64) + return m_jit.branchTest64(ResultCondition::NonZero, operandLocation.asGPR()); +#else + return m_jit.branchTest64(ResultCondition::NonZero, operandLocation.asGPRhi(), operandLocation.asGPRlo()); +#endif + default: + RELEASE_ASSERT_NOT_REACHED_WITH_MESSAGE("Op type '%s' is not a unary comparison and should not have been fused.\n", makeString(opType).characters()); + } +} + +PartialResult BBQJIT::addFusedBranchCompare(OpType opType, ControlType& target, ExpressionType operand, Stack& results) +{ + ASSERT(!operand.isNone()); + + switch (tryFoldFusedBranchCompare(opType, operand)) { + case BranchNeverTaken: + return { }; + case BranchAlwaysTaken: + currentControlData().flushAndSingleExit(*this, target, results, false, false); + target.addBranch(m_jit.jump()); + return { }; + case BranchNotFolded: + break; + } + + { + // Like in normal addBranch(), we can directly use the operand location + // because it shouldn't interfere with flushAtBlockBoundary(). + Location operandLocation = loadIfNecessary(operand); + consume(operand); + + LOG_INSTRUCTION("BranchCompare", makeString(opType).characters(), operand, operandLocation); + + currentControlData().flushAtBlockBoundary(*this, 0, results, false); + Jump ifNotTaken = emitFusedBranchCompareBranch(opType, operand, operandLocation); + currentControlData().addExit(*this, target.targetLocations(), results); + target.addBranch(m_jit.jump()); + ifNotTaken.link(&m_jit); + currentControlData().finalizeBlock(*this, target.targetLocations().size(), results, true); + } + + return { }; +} + +PartialResult WARN_UNUSED_RETURN BBQJIT::addFusedIfCompare(OpType op, ExpressionType operand, BlockSignature signature, Stack& enclosingStack, ControlData& result, Stack& newStack) +{ + BranchFoldResult foldResult = tryFoldFusedBranchCompare(op, operand); + + ScratchScope<0, 1> scratches(*this); + Location operandLocation; + RegisterSet liveScratchGPRs, liveScratchFPRs; + if (foldResult == BranchNotFolded) { + if (!operand.isConst()) + operandLocation = loadIfNecessary(operand); + else if (operand.isFloat()) { + operandLocation = Location::fromFPR(scratches.fpr(0)); + emitMove(operand, operandLocation); + } + + if (operandLocation.isGPR()) + liveScratchGPRs.add(operandLocation.asGPR(), IgnoreVectors); + else if (operandLocation.isGPR2()) { + liveScratchGPRs.add(operandLocation.asGPRlo(), IgnoreVectors); + liveScratchGPRs.add(operandLocation.asGPRhi(), IgnoreVectors); + } else if (operandLocation.isFPR()) + liveScratchFPRs.add(operandLocation.asFPR(), operand.type() == TypeKind::V128 ? Width128 : Width64); + } + if (!liveScratchFPRs.contains(scratches.fpr(0), IgnoreVectors)) + scratches.unbindEarly(); + + consume(operand); + + result = ControlData(*this, BlockType::If, signature, currentControlData().enclosedHeight() + currentControlData().implicitSlots() + enclosingStack.size() - signature.m_signature->argumentCount(), liveScratchGPRs, liveScratchFPRs); + + // Despite being conditional, if doesn't need to worry about diverging expression stacks at block boundaries, so it doesn't need multiple exits. + currentControlData().flushAndSingleExit(*this, result, enclosingStack, true, false); + + LOG_INSTRUCTION("IfCompare", makeString(op).characters(), *signature.m_signature, operand, operandLocation); + LOG_INDENT(); + splitStack(signature, enclosingStack, newStack); + + result.startBlock(*this, newStack); + if (foldResult == BranchNeverTaken) + result.setIfBranch(m_jit.jump()); // Emit direct branch if we know the condition is false. + else if (foldResult == BranchNotFolded) // Otherwise, we only emit a branch at all if we don't know the condition statically. + result.setIfBranch(emitFusedBranchCompareBranch(op, operand, operandLocation)); + return { }; +} + +BBQJIT::BranchFoldResult BBQJIT::tryFoldFusedBranchCompare(OpType opType, ExpressionType left, ExpressionType right) +{ + if (!left.isConst() || !right.isConst()) + return BranchNotFolded; + switch (opType) { + case OpType::I32LtS: + return left.asI32() < right.asI32() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::I32LtU: + return static_cast(left.asI32()) < static_cast(right.asI32()) ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::I32GtS: + return left.asI32() > right.asI32() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::I32GtU: + return static_cast(left.asI32()) > static_cast(right.asI32()) ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::I32LeS: + return left.asI32() <= right.asI32() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::I32LeU: + return static_cast(left.asI32()) <= static_cast(right.asI32()) ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::I32GeS: + return left.asI32() >= right.asI32() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::I32GeU: + return static_cast(left.asI32()) >= static_cast(right.asI32()) ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::I32Eq: + return left.asI32() == right.asI32() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::I32Ne: + return left.asI32() != right.asI32() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::I64LtS: + return left.asI64() < right.asI64() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::I64LtU: + return static_cast(left.asI64()) < static_cast(right.asI64()) ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::I64GtS: + return left.asI64() > right.asI64() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::I64GtU: + return static_cast(left.asI64()) > static_cast(right.asI64()) ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::I64LeS: + return left.asI64() <= right.asI64() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::I64LeU: + return static_cast(left.asI64()) <= static_cast(right.asI64()) ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::I64GeS: + return left.asI64() >= right.asI64() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::I64GeU: + return static_cast(left.asI64()) >= static_cast(right.asI64()) ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::I64Eq: + return left.asI64() == right.asI64() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::I64Ne: + return left.asI64() != right.asI64() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::F32Lt: + return left.asF32() < right.asF32() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::F32Gt: + return left.asF32() > right.asF32() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::F32Le: + return left.asF32() <= right.asF32() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::F32Ge: + return left.asF32() >= right.asF32() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::F32Eq: + return left.asF32() == right.asF32() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::F32Ne: + return left.asF32() != right.asF32() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::F64Lt: + return left.asF64() < right.asF64() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::F64Gt: + return left.asF64() > right.asF64() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::F64Le: + return left.asF64() <= right.asF64() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::F64Ge: + return left.asF64() >= right.asF64() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::F64Eq: + return left.asF64() == right.asF64() ? BranchAlwaysTaken : BranchNeverTaken; + case OpType::F64Ne: + return left.asF64() != right.asF64() ? BranchAlwaysTaken : BranchNeverTaken; + default: + RELEASE_ASSERT_NOT_REACHED_WITH_MESSAGE("Op type '%s' is not a binary comparison and should not have been fused.\n", makeString(opType).characters()); + } +} + +static MacroAssembler::Jump emitBranchI32(CCallHelpers& jit, MacroAssembler::RelationalCondition condition, Value left, Location leftLocation, Value right, Location rightLocation) +{ + if (right.isConst()) + return jit.branch32(condition, leftLocation.asGPR(), MacroAssembler::TrustedImm32(right.asI32())); + if (left.isConst()) + return jit.branch32(condition, MacroAssembler::TrustedImm32(left.asI32()), rightLocation.asGPR()); + return jit.branch32(condition, leftLocation.asGPR(), rightLocation.asGPR()); +} + +static MacroAssembler::Jump emitBranchI64(CCallHelpers& jit, MacroAssembler::RelationalCondition condition, Value left, Location leftLocation, Value right, Location rightLocation) +{ +#if USE(JSVALUE64) + if (right.isConst()) + return jit.branch64(condition, leftLocation.asGPR(), MacroAssembler::Imm64(right.asI64())); + if (left.isConst()) + return jit.branch64(MacroAssembler::commute(condition), rightLocation.asGPR(), MacroAssembler::Imm64(left.asI64())); + return jit.branch64(condition, leftLocation.asGPR(), rightLocation.asGPR()); +#else + if (right.isConst()) + return jit.branch64(condition, leftLocation.asGPRhi(), leftLocation.asGPRlo(), MacroAssembler::TrustedImm64(right.asI64())); + if (left.isConst()) + return jit.branch64(MacroAssembler::commute(condition), rightLocation.asGPRhi(), rightLocation.asGPRlo(), MacroAssembler::TrustedImm64(left.asI64())); + return jit.branch64(condition, leftLocation.asGPRhi(), leftLocation.asGPRlo(), rightLocation.asGPRhi(), rightLocation.asGPRlo()); +#endif +} + +static MacroAssembler::Jump emitBranchF32(CCallHelpers& jit, MacroAssembler::DoubleCondition condition, Value, Location leftLocation, Value, Location rightLocation) +{ + return jit.branchFloat(condition, leftLocation.asFPR(), rightLocation.asFPR()); +} + +static MacroAssembler::Jump emitBranchF64(CCallHelpers& jit, MacroAssembler::DoubleCondition condition, Value, Location leftLocation, Value, Location rightLocation) +{ + return jit.branchDouble(condition, leftLocation.asFPR(), rightLocation.asFPR()); +} + +BBQJIT::Jump BBQJIT::emitFusedBranchCompareBranch(OpType opType, ExpressionType left, Location leftLocation, ExpressionType right, Location rightLocation) +{ + // Emit a branch with the inverse of the comparison. We're generating the "branch-if-false" case. + switch (opType) { + case OpType::I32LtS: + return emitBranchI32(m_jit, RelationalCondition::GreaterThanOrEqual, left, leftLocation, right, rightLocation); + case OpType::I32LtU: + return emitBranchI32(m_jit, RelationalCondition::AboveOrEqual, left, leftLocation, right, rightLocation); + case OpType::I32GtS: + return emitBranchI32(m_jit, RelationalCondition::LessThanOrEqual, left, leftLocation, right, rightLocation); + case OpType::I32GtU: + return emitBranchI32(m_jit, RelationalCondition::BelowOrEqual, left, leftLocation, right, rightLocation); + case OpType::I32LeS: + return emitBranchI32(m_jit, RelationalCondition::GreaterThan, left, leftLocation, right, rightLocation); + case OpType::I32LeU: + return emitBranchI32(m_jit, RelationalCondition::Above, left, leftLocation, right, rightLocation); + case OpType::I32GeS: + return emitBranchI32(m_jit, RelationalCondition::LessThan, left, leftLocation, right, rightLocation); + case OpType::I32GeU: + return emitBranchI32(m_jit, RelationalCondition::Below, left, leftLocation, right, rightLocation); + case OpType::I32Eq: + return emitBranchI32(m_jit, RelationalCondition::NotEqual, left, leftLocation, right, rightLocation); + case OpType::I32Ne: + return emitBranchI32(m_jit, RelationalCondition::Equal, left, leftLocation, right, rightLocation); + case OpType::I64LtS: + return emitBranchI64(m_jit, RelationalCondition::GreaterThanOrEqual, left, leftLocation, right, rightLocation); + case OpType::I64LtU: + return emitBranchI64(m_jit, RelationalCondition::AboveOrEqual, left, leftLocation, right, rightLocation); + case OpType::I64GtS: + return emitBranchI64(m_jit, RelationalCondition::LessThanOrEqual, left, leftLocation, right, rightLocation); + case OpType::I64GtU: + return emitBranchI64(m_jit, RelationalCondition::BelowOrEqual, left, leftLocation, right, rightLocation); + case OpType::I64LeS: + return emitBranchI64(m_jit, RelationalCondition::GreaterThan, left, leftLocation, right, rightLocation); + case OpType::I64LeU: + return emitBranchI64(m_jit, RelationalCondition::Above, left, leftLocation, right, rightLocation); + case OpType::I64GeS: + return emitBranchI64(m_jit, RelationalCondition::LessThan, left, leftLocation, right, rightLocation); + case OpType::I64GeU: + return emitBranchI64(m_jit, RelationalCondition::Below, left, leftLocation, right, rightLocation); + case OpType::I64Eq: + return emitBranchI64(m_jit, RelationalCondition::NotEqual, left, leftLocation, right, rightLocation); + case OpType::I64Ne: + return emitBranchI64(m_jit, RelationalCondition::Equal, left, leftLocation, right, rightLocation); + case OpType::F32Lt: + return emitBranchF32(m_jit, MacroAssembler::invert(DoubleCondition::DoubleLessThanAndOrdered), left, leftLocation, right, rightLocation); + case OpType::F32Gt: + return emitBranchF32(m_jit, MacroAssembler::invert(DoubleCondition::DoubleGreaterThanAndOrdered), left, leftLocation, right, rightLocation); + case OpType::F32Le: + return emitBranchF32(m_jit, MacroAssembler::invert(DoubleCondition::DoubleLessThanOrEqualAndOrdered), left, leftLocation, right, rightLocation); + case OpType::F32Ge: + return emitBranchF32(m_jit, MacroAssembler::invert(DoubleCondition::DoubleGreaterThanOrEqualAndOrdered), left, leftLocation, right, rightLocation); + case OpType::F32Eq: + return emitBranchF32(m_jit, MacroAssembler::invert(DoubleCondition::DoubleEqualAndOrdered), left, leftLocation, right, rightLocation); + case OpType::F32Ne: + return emitBranchF32(m_jit, MacroAssembler::invert(DoubleCondition::DoubleNotEqualOrUnordered), left, leftLocation, right, rightLocation); + case OpType::F64Lt: + return emitBranchF64(m_jit, MacroAssembler::invert(DoubleCondition::DoubleLessThanAndOrdered), left, leftLocation, right, rightLocation); + case OpType::F64Gt: + return emitBranchF64(m_jit, MacroAssembler::invert(DoubleCondition::DoubleGreaterThanAndOrdered), left, leftLocation, right, rightLocation); + case OpType::F64Le: + return emitBranchF64(m_jit, MacroAssembler::invert(DoubleCondition::DoubleLessThanOrEqualAndOrdered), left, leftLocation, right, rightLocation); + case OpType::F64Ge: + return emitBranchF64(m_jit, MacroAssembler::invert(DoubleCondition::DoubleGreaterThanOrEqualAndOrdered), left, leftLocation, right, rightLocation); + case OpType::F64Eq: + return emitBranchF64(m_jit, MacroAssembler::invert(DoubleCondition::DoubleEqualAndOrdered), left, leftLocation, right, rightLocation); + case OpType::F64Ne: + return emitBranchF64(m_jit, MacroAssembler::invert(DoubleCondition::DoubleNotEqualOrUnordered), left, leftLocation, right, rightLocation); + default: + RELEASE_ASSERT_NOT_REACHED_WITH_MESSAGE("Op type '%s' is not a binary comparison and should not have been fused.\n", makeString(opType).characters()); + } +} + +PartialResult BBQJIT::addFusedBranchCompare(OpType opType, ControlType& target, ExpressionType left, ExpressionType right, Stack& results) +{ + switch (tryFoldFusedBranchCompare(opType, left, right)) { + case BranchNeverTaken: + return { }; + case BranchAlwaysTaken: + currentControlData().flushAndSingleExit(*this, target, results, false, false); + target.addBranch(m_jit.jump()); + return { }; + case BranchNotFolded: + break; + } + + { + Location leftLocation, rightLocation; + + if (!left.isConst()) + leftLocation = loadIfNecessary(left); + else if (left.isFloat()) // Materialize floats here too, since they don't have a good immediate lowering. + emitMove(left, leftLocation = Location::fromFPR(wasmScratchFPR)); + if (!right.isConst()) + rightLocation = loadIfNecessary(right); + else if (right.isFloat()) + emitMove(right, rightLocation = Location::fromFPR(wasmScratchFPR)); + + consume(left); + consume(right); + + LOG_INSTRUCTION("BranchCompare", makeString(opType).characters(), left, leftLocation, right, rightLocation); + + currentControlData().flushAtBlockBoundary(*this, 0, results, false); + Jump ifNotTaken = emitFusedBranchCompareBranch(opType, left, leftLocation, right, rightLocation); + currentControlData().addExit(*this, target.targetLocations(), results); + target.addBranch(m_jit.jump()); + if (ifNotTaken.isSet()) + ifNotTaken.link(&m_jit); + currentControlData().finalizeBlock(*this, target.targetLocations().size(), results, true); + } + + return { }; +} + +PartialResult WARN_UNUSED_RETURN BBQJIT::addFusedIfCompare(OpType op, ExpressionType left, ExpressionType right, BlockSignature signature, Stack& enclosingStack, ControlData& result, Stack& newStack) +{ + BranchFoldResult foldResult = tryFoldFusedBranchCompare(op, left, right); + + Location leftLocation, rightLocation; + RegisterSet liveScratchGPRs, liveScratchFPRs; + if (foldResult == BranchNotFolded) { + ASSERT(!left.isConst() || !right.isConst()); // If they're both constants, we should have folded. + + if (!left.isConst()) + leftLocation = loadIfNecessary(left); + else if (left.isFloat()) + emitMove(left, leftLocation = Location::fromFPR(wasmScratchFPR)); + if (leftLocation.isGPR()) + liveScratchGPRs.add(leftLocation.asGPR(), IgnoreVectors); + else if (leftLocation.isGPR2()) { + liveScratchGPRs.add(leftLocation.asGPRlo(), IgnoreVectors); + liveScratchGPRs.add(leftLocation.asGPRhi(), IgnoreVectors); + } else if (leftLocation.isFPR()) + liveScratchFPRs.add(leftLocation.asFPR(), left.type() == TypeKind::V128 ? Width128 : Width64); + + if (!right.isConst()) + rightLocation = loadIfNecessary(right); + else if (right.isFloat()) + emitMove(right, rightLocation = Location::fromFPR(wasmScratchFPR)); + if (rightLocation.isGPR()) + liveScratchGPRs.add(rightLocation.asGPR(), IgnoreVectors); + else if (rightLocation.isGPR2()) { + liveScratchGPRs.add(rightLocation.asGPRlo(), IgnoreVectors); + liveScratchGPRs.add(rightLocation.asGPRhi(), IgnoreVectors); + } else if (rightLocation.isFPR()) + liveScratchFPRs.add(rightLocation.asFPR(), right.type() == TypeKind::V128 ? Width128 : Width64); + } + consume(left); + consume(right); + + + result = ControlData(*this, BlockType::If, signature, currentControlData().enclosedHeight() + currentControlData().implicitSlots() + enclosingStack.size() - signature.m_signature->argumentCount(), liveScratchGPRs, liveScratchFPRs); + + // Despite being conditional, if doesn't need to worry about diverging expression stacks at block boundaries, so it doesn't need multiple exits. + currentControlData().flushAndSingleExit(*this, result, enclosingStack, true, false); + + LOG_INSTRUCTION("IfCompare", makeString(op).characters(), *signature.m_signature, left, leftLocation, right, rightLocation); + LOG_INDENT(); + splitStack(signature, enclosingStack, newStack); + + result.startBlock(*this, newStack); + if (foldResult == BranchNeverTaken) + result.setIfBranch(m_jit.jump()); // Emit direct branch if we know the condition is false. + else if (foldResult == BranchNotFolded) // Otherwise, we only emit a branch at all if we don't know the condition statically. + result.setIfBranch(emitFusedBranchCompareBranch(op, left, leftLocation, right, rightLocation)); + return { }; +} + // SIMD bool BBQJIT::usesSIMD() diff --git a/Source/JavaScriptCore/wasm/WasmBBQJIT.h b/Source/JavaScriptCore/wasm/WasmBBQJIT.h index ea45678cd87cb..ae7980168119d 100644 --- a/Source/JavaScriptCore/wasm/WasmBBQJIT.h +++ b/Source/JavaScriptCore/wasm/WasmBBQJIT.h @@ -861,8 +861,8 @@ class BBQJIT { #define LOG_DEDENT() do { if (UNLIKELY(Options::verboseBBQJITInstructions())) { m_loggingIndent -= 2; } } while (false); public: - // FIXME: Support fused branch compare on 32-bit platforms. - static constexpr bool shouldFuseBranchCompare = is64Bit(); + // Enable fused branch compare for all platforms + static constexpr bool shouldFuseBranchCompare = true; static constexpr bool tierSupportsSIMD = true; static constexpr bool validateFunctionBodySize = true; @@ -1163,7 +1163,6 @@ class BBQJIT { FloatingPointRange lookupTruncationRange(TruncationKind truncationKind); void truncInBounds(TruncationKind truncationKind, Location operandLocation, Location resultLocation, FPRReg scratch1FPR, FPRReg scratch2FPR); - void truncInBounds(TruncationKind truncationKind, Location operandLocation, Value& result, Location resultLocation); PartialResult WARN_UNUSED_RETURN truncTrapping(OpType truncationOp, Value operand, Value& result, Type returnType, Type operandType); PartialResult WARN_UNUSED_RETURN truncSaturated(Ext1OpType truncationOp, Value operand, Value& result, Type returnType, Type operandType); @@ -2248,9 +2247,6 @@ class BBQJIT { constexpr static int tempSlotSize = 16; // Size of the stack slot for a stack temporary. Currently the size of the largest possible temporary (a v128). - enum class ShiftI64HelperOp { Lshift, Urshift, Rshift }; - void shiftI64Helper(ShiftI64HelperOp op, Location lhsLocation, Location rhsLocation, Location resultLocation); - enum class RotI64HelperOp { Left, Right }; void rotI64Helper(RotI64HelperOp op, Location lhsLocation, Location rhsLocation, Location resultLocation); diff --git a/Source/JavaScriptCore/wasm/WasmBBQJIT32_64.cpp b/Source/JavaScriptCore/wasm/WasmBBQJIT32_64.cpp index 2dda9962e276b..2f8ec8db3951a 100644 --- a/Source/JavaScriptCore/wasm/WasmBBQJIT32_64.cpp +++ b/Source/JavaScriptCore/wasm/WasmBBQJIT32_64.cpp @@ -477,14 +477,14 @@ PartialResult WARN_UNUSED_RETURN BBQJIT::load(LoadOpType loadOp, Value pointer, m_jit.rshift32(resultLocation.asGPRlo(), TrustedImm32(31), resultLocation.asGPRhi()); break; case LoadOpType::I64Load: - m_jit.loadPair32(location, resultLocation.asGPRlo(), resultLocation.asGPRhi()); + m_jit.loadPair32Unaligned(location, resultLocation.asGPRlo(), resultLocation.asGPRhi()); break; case LoadOpType::F32Load: m_jit.load32(location, wasmScratchGPR); m_jit.move32ToFloat(wasmScratchGPR, resultLocation.asFPR()); break; case LoadOpType::F64Load: - m_jit.loadPair32(location, wasmScratchGPR, wasmScratchGPR2); + m_jit.loadPair32Unaligned(location, wasmScratchGPR, wasmScratchGPR2); m_jit.move64ToDouble(wasmScratchGPR2, wasmScratchGPR, resultLocation.asFPR()); break; } @@ -513,42 +513,57 @@ PartialResult WARN_UNUSED_RETURN BBQJIT::store(StoreOpType storeOp, Value pointe ScratchScope<0, 1> scratches(*this); valueLocation = Location::fromFPR(scratches.fpr(0)); emitMoveConst(value, valueLocation); - } else if (value.isConst() && typeNeedsGPR2(value.type())) { - ScratchScope<2, 0> scratches(*this); - valueLocation = Location::fromGPR2(scratches.gpr(1), scratches.gpr(0)); - emitMoveConst(value, valueLocation); - } else if (value.isConst()) { - ScratchScope<1, 0> scratches(*this); - valueLocation = Location::fromGPR(scratches.gpr(0)); - emitMoveConst(value, valueLocation); - } else + } else if (!value.isConst()) { valueLocation = loadIfNecessary(value); - ASSERT(valueLocation.isRegister()); + ASSERT(valueLocation.isRegister()); + } consume(value); consume(pointer); switch (storeOp) { case StoreOpType::I64Store8: - m_jit.store8(valueLocation.asGPRlo(), location); + if (value.isConst()) + m_jit.store8(TrustedImm32(static_cast(value.asI64())), location); + else + m_jit.store8(valueLocation.asGPRlo(), location); return; case StoreOpType::I32Store8: - m_jit.store8(valueLocation.asGPR(), location); + if (value.isConst()) + m_jit.store8(TrustedImm32(value.asI32()), location); + else + m_jit.store8(valueLocation.asGPR(), location); return; case StoreOpType::I64Store16: - m_jit.store16(valueLocation.asGPRlo(), location); + if (value.isConst()) + m_jit.store16(TrustedImm32(static_cast(value.asI64())), location); + else + m_jit.store16(valueLocation.asGPRlo(), location); return; case StoreOpType::I32Store16: - m_jit.store16(valueLocation.asGPR(), location); + if (value.isConst()) + m_jit.store16(TrustedImm32(value.asI32()), location); + else + m_jit.store16(valueLocation.asGPR(), location); return; case StoreOpType::I64Store32: - m_jit.store32(valueLocation.asGPRlo(), location); + if (value.isConst()) + m_jit.store32(TrustedImm32(static_cast(value.asI64())), location); + else + m_jit.store32(valueLocation.asGPRlo(), location); return; case StoreOpType::I32Store: - m_jit.store32(valueLocation.asGPR(), location); + if (value.isConst()) + m_jit.store32(TrustedImm32(value.asI32()), location); + else + m_jit.store32(valueLocation.asGPR(), location); return; case StoreOpType::I64Store: - m_jit.storePair32(valueLocation.asGPRlo(), valueLocation.asGPRhi(), location); + if (value.isConst()) { + int64_t val = value.asI64(); + m_jit.storePair32(TrustedImm32(static_cast(val)), TrustedImm32(static_cast(val >> 32)), location); + } else + m_jit.storePair32(valueLocation.asGPRlo(), valueLocation.asGPRhi(), location); return; case StoreOpType::F32Store: { ScratchScope<1, 0> scratches(*this); @@ -1161,7 +1176,7 @@ Value WARN_UNUSED_RETURN BBQJIT::emitAtomicCompareExchange(ExtAtomicOpType op, T return result; } -void BBQJIT::truncInBounds(TruncationKind truncationKind, Location operandLocation, Value& result, Location resultLocation) +void BBQJIT::truncInBounds(TruncationKind truncationKind, Location operandLocation, Location resultLocation, FPRReg scratch1FPR, FPRReg scratch2FPR) { switch (truncationKind) { case TruncationKind::I32TruncF32S: @@ -1177,27 +1192,19 @@ void BBQJIT::truncInBounds(TruncationKind truncationKind, Location operandLocati m_jit.truncateDoubleToUint32(operandLocation.asFPR(), resultLocation.asGPR()); break; case TruncationKind::I64TruncF32S: { - auto operand = Value::pinned(TypeKind::F32, operandLocation); - consume(result); - emitCCall(Math::i64_trunc_s_f32, ArgumentList { operand }, result); + m_jit.truncateFloatToInt64(operandLocation.asFPR(), resultLocation.asGPRlo(), resultLocation.asGPRhi(), scratch1FPR, scratch2FPR); break; } case TruncationKind::I64TruncF64S: { - auto operand = Value::pinned(TypeKind::F64, operandLocation); - consume(result); - emitCCall(Math::i64_trunc_s_f64, ArgumentList { operand }, result); + m_jit.truncateDoubleToInt64(operandLocation.asFPR(), resultLocation.asGPRlo(), resultLocation.asGPRhi(), scratch1FPR, scratch2FPR); break; } case TruncationKind::I64TruncF32U: { - auto operand = Value::pinned(TypeKind::F32, operandLocation); - consume(result); - emitCCall(Math::i64_trunc_u_f32, ArgumentList { operand }, result); + m_jit.truncateFloatToUint64(operandLocation.asFPR(), resultLocation.asGPRlo(), resultLocation.asGPRhi(), scratch1FPR, scratch2FPR); break; } case TruncationKind::I64TruncF64U: { - auto operand = Value::pinned(TypeKind::F64, operandLocation); - consume(result); - emitCCall(Math::i64_trunc_u_f64, ArgumentList { operand }, result); + m_jit.truncateDoubleToUint64(operandLocation.asFPR(), resultLocation.asGPRlo(), resultLocation.asGPRhi(), scratch1FPR, scratch2FPR); break; } } @@ -1231,7 +1238,7 @@ PartialResult WARN_UNUSED_RETURN BBQJIT::truncTrapping(OpType truncationOp, Valu emitMoveConst(minFloatConst, minFloat); emitMoveConst(maxFloatConst, maxFloat); - LOG_INSTRUCTION("TruncSaturated", operand, operandLocation, RESULT(result)); + LOG_INSTRUCTION("TruncTrapping", operand, operandLocation, RESULT(result)); DoubleCondition minCondition = range.closedLowerEndpoint ? DoubleCondition::DoubleLessThanOrUnordered : DoubleCondition::DoubleLessThanOrEqualOrUnordered; Jump belowMin = operandType == Types::F32 @@ -1244,7 +1251,7 @@ PartialResult WARN_UNUSED_RETURN BBQJIT::truncTrapping(OpType truncationOp, Valu : m_jit.branchDouble(DoubleCondition::DoubleGreaterThanOrEqualOrUnordered, operandLocation.asFPR(), maxFloat.asFPR()); throwExceptionIf(ExceptionType::OutOfBoundsTrunc, aboveMax); - truncInBounds(kind, operandLocation, result, resultLocation); + truncInBounds(kind, operandLocation, resultLocation, scratches.fpr(0), scratches.fpr(1)); return { }; } @@ -1314,7 +1321,7 @@ PartialResult WARN_UNUSED_RETURN BBQJIT::truncSaturated(Ext1OpType truncationOp, : m_jit.branchDouble(DoubleCondition::DoubleGreaterThanOrEqualOrUnordered, operandLocation.asFPR(), maxFloat.asFPR()); // In-bounds case. Emit normal truncation instructions. - truncInBounds(kind, operandLocation, result, resultLocation); + truncInBounds(kind, operandLocation, resultLocation, scratches.fpr(0), scratches.fpr(1)); resultLocation = locationOf(result); Jump afterInBounds = m_jit.jump(); @@ -2251,108 +2258,132 @@ PartialResult WARN_UNUSED_RETURN BBQJIT::addI64Or(Value lhs, Value rhs, Value& r PartialResult WARN_UNUSED_RETURN BBQJIT::addI64Shl(Value lhs, Value rhs, Value& result) { - PREPARE_FOR_SHIFT; + auto emitI64Shl = [&](Location lhsLocation, Location rhsLocation, Location resultLocation) { + ScratchScope<2, 0> scratches(*this, lhsLocation, rhsLocation, resultLocation); + + auto shiftReg = rhsLocation.asGPRlo(); + auto resultLo = resultLocation.asGPRlo(); + auto resultHi = resultLocation.asGPRhi(); + auto lhsLo = lhsLocation.asGPRlo(); + auto lhsHi = lhsLocation.asGPRhi(); + + auto shift = scratches.gpr(0); + auto tmp = scratches.gpr(1); + + m_jit.and32(TrustedImm32(63), shiftReg, shift); + + m_jit.sub32(shift, TrustedImm32(32), tmp); + m_jit.lshiftUnchecked(lhsHi, shift, resultHi); + m_jit.lshiftUnchecked(lhsLo, tmp, tmp); + m_jit.or32(resultHi, tmp, resultHi); + + m_jit.sub32(TrustedImm32(32), shift, tmp); + m_jit.urshiftUnchecked(lhsLo, tmp, tmp); + m_jit.or32(resultHi, tmp, resultHi); + m_jit.lshiftUnchecked(lhsLo, shift, resultLo); + }; + EMIT_BINARY( "I64Shl", TypeKind::I64, BLOCK(Value::fromI64(lhs.asI64() << rhs.asI64())), BLOCK( - shiftI64Helper(ShiftI64HelperOp::Lshift, lhsLocation, rhsLocation, resultLocation); + emitI64Shl(lhsLocation, rhsLocation, resultLocation); ), BLOCK( ImmHelpers::immLocation(lhsLocation, rhsLocation) = Location::fromGPR2(wasmScratchGPR, wasmScratchGPR2); emitMoveConst(ImmHelpers::imm(lhs, rhs), Location::fromGPR2(wasmScratchGPR, wasmScratchGPR2)); - shiftI64Helper(ShiftI64HelperOp::Lshift, lhsLocation, rhsLocation, resultLocation); + emitI64Shl(lhsLocation, rhsLocation, resultLocation); ) ); } PartialResult WARN_UNUSED_RETURN BBQJIT::addI64ShrS(Value lhs, Value rhs, Value& result) { - PREPARE_FOR_SHIFT; + auto emitI64ShrS = [&](Location lhsLocation, Location rhsLocation, Location resultLocation) { + ScratchScope<2, 0> scratches(*this, lhsLocation, rhsLocation, resultLocation); + + auto shiftReg = rhsLocation.asGPRlo(); + auto resultLo = resultLocation.asGPRlo(); + auto resultHi = resultLocation.asGPRhi(); + auto lhsLo = lhsLocation.asGPRlo(); + auto lhsHi = lhsLocation.asGPRhi(); + + auto shift = scratches.gpr(0); + auto tmp = scratches.gpr(1); + + m_jit.and32(TrustedImm32(63), shiftReg, shift); + + m_jit.urshiftUnchecked(lhsLo, shift, resultLo); + + m_jit.sub32(TrustedImm32(32), shift, tmp); + m_jit.lshiftUnchecked(lhsHi, tmp, tmp); + m_jit.or32(tmp, resultLo); + + m_jit.sub32(shift, TrustedImm32(32), tmp); + m_jit.rshiftUnchecked(lhsHi, tmp, tmp); + m_jit.or32(resultLo, tmp, tmp); + m_jit.moveConditionally32(RelationalCondition::AboveOrEqual, shift, TrustedImm32(32), tmp, resultLo, resultLo); + + m_jit.rshiftUnchecked(lhsHi, shift, resultHi); + }; + EMIT_BINARY( "I64ShrS", TypeKind::I64, BLOCK(Value::fromI64(lhs.asI64() >> rhs.asI64())), BLOCK( - shiftI64Helper(ShiftI64HelperOp::Rshift, lhsLocation, rhsLocation, resultLocation); + emitI64ShrS(lhsLocation, rhsLocation, resultLocation); ), BLOCK( ImmHelpers::immLocation(lhsLocation, rhsLocation) = Location::fromGPR2(wasmScratchGPR, wasmScratchGPR2); emitMoveConst(ImmHelpers::imm(lhs, rhs), Location::fromGPR2(wasmScratchGPR, wasmScratchGPR2)); - shiftI64Helper(ShiftI64HelperOp::Rshift, lhsLocation, rhsLocation, resultLocation); + emitI64ShrS(lhsLocation, rhsLocation, resultLocation); ) ); } PartialResult WARN_UNUSED_RETURN BBQJIT::addI64ShrU(Value lhs, Value rhs, Value& result) { - PREPARE_FOR_SHIFT; + auto emitI64ShrU = [&](Location lhsLocation, Location rhsLocation, Location resultLocation) { + ScratchScope<2, 0> scratches(*this, lhsLocation, rhsLocation, resultLocation); + + auto shiftReg = rhsLocation.asGPRlo(); + auto resultLo = resultLocation.asGPRlo(); + auto resultHi = resultLocation.asGPRhi(); + auto lhsLo = lhsLocation.asGPRlo(); + auto lhsHi = lhsLocation.asGPRhi(); + + auto shift = scratches.gpr(0); + auto tmp = scratches.gpr(1); + + m_jit.and32(TrustedImm32(63), shiftReg, shift); + + m_jit.urshiftUnchecked(lhsLo, shift, resultLo); + + m_jit.sub32(TrustedImm32(32), shift, tmp); + m_jit.lshiftUnchecked(lhsHi, tmp, tmp); + m_jit.or32(tmp, resultLo); + + m_jit.sub32(shift, TrustedImm32(32), tmp); + m_jit.urshiftUnchecked(lhsHi, tmp, tmp); + m_jit.or32(tmp, resultLo); + + m_jit.urshiftUnchecked(lhsHi, shift, resultHi); + }; + EMIT_BINARY( "I64ShrU", TypeKind::I64, BLOCK(Value::fromI64(static_cast(lhs.asI64()) >> static_cast(rhs.asI64()))), BLOCK( - shiftI64Helper(ShiftI64HelperOp::Urshift, lhsLocation, rhsLocation, resultLocation); + emitI64ShrU(lhsLocation, rhsLocation, resultLocation); ), BLOCK( ImmHelpers::immLocation(lhsLocation, rhsLocation) = Location::fromGPR2(wasmScratchGPR, wasmScratchGPR2); emitMoveConst(ImmHelpers::imm(lhs, rhs), Location::fromGPR2(wasmScratchGPR, wasmScratchGPR2)); - shiftI64Helper(ShiftI64HelperOp::Urshift, lhsLocation, rhsLocation, resultLocation); + emitI64ShrU(lhsLocation, rhsLocation, resultLocation); ) ); } -void BBQJIT::shiftI64Helper(ShiftI64HelperOp op, Location lhsLocation, Location rhsLocation, Location resultLocation) -{ - auto shift = rhsLocation.asGPRlo(); - m_jit.and32(TrustedImm32(63), rhsLocation.asGPRlo(), shift); - auto zero = m_jit.branch32(RelationalCondition::Equal, shift, TrustedImm32(0)); - auto aboveOrEqual32 = m_jit.branch32(RelationalCondition::AboveOrEqual, shift, TrustedImm32(32)); - // shift < 32 - ScratchScope<1, 0> scratches(*this, lhsLocation, rhsLocation, resultLocation); - auto carry = scratches.gpr(0); - m_jit.move(TrustedImm32(32), carry); - m_jit.sub32(carry, shift, carry); - if (op == ShiftI64HelperOp::Lshift) { - ASSERT(resultLocation.asGPRhi() != shift); - ASSERT(resultLocation.asGPRhi() != lhsLocation.asGPRlo()); - m_jit.lshift32(lhsLocation.asGPRhi(), shift, resultLocation.asGPRhi()); - m_jit.urshift32(lhsLocation.asGPRlo(), carry, carry); - m_jit.or32(carry, resultLocation.asGPRhi()); - m_jit.lshift32(lhsLocation.asGPRlo(), shift, resultLocation.asGPRlo()); - } else if (op == ShiftI64HelperOp::Urshift) { - m_jit.lshift32(lhsLocation.asGPRhi(), carry, carry); - ASSERT(resultLocation.asGPRhi() != shift); - ASSERT(resultLocation.asGPRhi() != lhsLocation.asGPRlo()); - m_jit.urshift32(lhsLocation.asGPRhi(), shift, resultLocation.asGPRhi()); - m_jit.urshift32(lhsLocation.asGPRlo(), shift, resultLocation.asGPRlo()); - m_jit.or32(carry, resultLocation.asGPRlo()); - } else if (op ==ShiftI64HelperOp::Rshift) { - m_jit.lshift32(lhsLocation.asGPRhi(), carry, carry); - ASSERT(resultLocation.asGPRhi() != shift); - ASSERT(resultLocation.asGPRhi() != lhsLocation.asGPRlo()); - m_jit.rshift32(lhsLocation.asGPRhi(), shift, resultLocation.asGPRhi()); - m_jit.urshift32(lhsLocation.asGPRlo(), shift, resultLocation.asGPRlo()); - m_jit.or32(carry, resultLocation.asGPRlo()); - } - auto done = m_jit.jump(); - // shift >= 32 - aboveOrEqual32.link(&m_jit); - m_jit.sub32(shift, TrustedImm32(32), shift); - if (op == ShiftI64HelperOp::Lshift) { - m_jit.lshift32(lhsLocation.asGPRlo(), shift, resultLocation.asGPRhi()); - m_jit.xor32(resultLocation.asGPRlo(), resultLocation.asGPRlo()); - } else if (op == ShiftI64HelperOp::Urshift) { - m_jit.urshift32(lhsLocation.asGPRhi(), shift, resultLocation.asGPRlo()); - m_jit.xor32(resultLocation.asGPRhi(), resultLocation.asGPRhi()); - } else if (op ==ShiftI64HelperOp::Rshift) { - ASSERT(resultLocation.asGPRlo() != lhsLocation.asGPRhi()); - m_jit.rshift32(lhsLocation.asGPRhi(), shift, resultLocation.asGPRlo()); - m_jit.rshift32(lhsLocation.asGPRhi(), TrustedImm32(31), resultLocation.asGPRhi()); - } - // shift == 0 - zero.link(&m_jit); - done.link(&m_jit); -} - PartialResult WARN_UNUSED_RETURN BBQJIT::addI64Rotl(Value lhs, Value rhs, Value& result) { PREPARE_FOR_SHIFT; @@ -3112,46 +3143,6 @@ PartialResult WARN_UNUSED_RETURN BBQJIT::addRethrow(unsigned, ControlType& data) return { }; } -BBQJIT::BranchFoldResult BBQJIT::tryFoldFusedBranchCompare(OpType, ExpressionType) -{ - RELEASE_ASSERT_NOT_REACHED(); -} - -BBQJIT::Jump BBQJIT::emitFusedBranchCompareBranch(OpType, ExpressionType, Location) -{ - RELEASE_ASSERT_NOT_REACHED(); -} - -BBQJIT::BranchFoldResult BBQJIT::tryFoldFusedBranchCompare(OpType, ExpressionType, ExpressionType) -{ - RELEASE_ASSERT_NOT_REACHED(); -} - -BBQJIT::Jump BBQJIT::emitFusedBranchCompareBranch(OpType, ExpressionType, Location, ExpressionType, Location) -{ - RELEASE_ASSERT_NOT_REACHED(); -} - -PartialResult BBQJIT::addFusedBranchCompare(OpType, ControlType&, ExpressionType, Stack&) -{ - RELEASE_ASSERT_NOT_REACHED(); -} - -PartialResult BBQJIT::addFusedBranchCompare(OpType, ControlType&, ExpressionType, ExpressionType, Stack&) -{ - RELEASE_ASSERT_NOT_REACHED(); -} - -PartialResult BBQJIT::addFusedIfCompare(OpType, ExpressionType, BlockSignature, Stack&, ControlType&, Stack&) -{ - RELEASE_ASSERT_NOT_REACHED(); -} - -PartialResult BBQJIT::addFusedIfCompare(OpType, ExpressionType, ExpressionType, BlockSignature, Stack&, ControlType&, Stack&) -{ - RELEASE_ASSERT_NOT_REACHED(); -} - PartialResult WARN_UNUSED_RETURN BBQJIT::addBranchNull(ControlData& data, ExpressionType reference, Stack& returnValues, bool shouldNegate, ExpressionType& result) { if (reference.isConst() && (reference.asRef() == JSValue::encode(jsNull())) == shouldNegate) { diff --git a/Source/JavaScriptCore/wasm/WasmBBQJIT32_64.h b/Source/JavaScriptCore/wasm/WasmBBQJIT32_64.h index 5a1f15a7cad5f..2cbcc2fadec7c 100644 --- a/Source/JavaScriptCore/wasm/WasmBBQJIT32_64.h +++ b/Source/JavaScriptCore/wasm/WasmBBQJIT32_64.h @@ -70,21 +70,68 @@ auto BBQJIT::emitCheckAndPrepareAndMaterializePointerApply(Value pointer, uint32 } return functor(CCallHelpers::Address(wasmBaseMemoryPointer, static_cast(finalOffset))); } - pointerLocation = Location::fromGPR(scratches.gpr(0)); - emitMoveConst(pointer, pointerLocation); + // Constant pointer, but finalOffset doesn't fit in addressing mode + // Fold constantPointer + boundary into a single immediate for bounds checking + GPRReg boundsCheckReg = wasmScratchGPR; + if (boundary) { + uint64_t foldedValue = constantPointer + boundary; + // Check for 32-bit overflow at compile time + if (foldedValue > std::numeric_limits::max()) { + // Overflow occurred - unconditional throw + throwExceptionIf(ExceptionType::OutOfBoundsMemoryAccess, m_jit.jump()); + } else { + m_jit.move(TrustedImm32(foldedValue), boundsCheckReg); + switch (m_mode) { + case MemoryMode::BoundsChecking: { + throwExceptionIf(ExceptionType::OutOfBoundsMemoryAccess, m_jit.branchPtr(RelationalCondition::AboveOrEqual, boundsCheckReg, wasmBoundsCheckingSizeRegister)); + break; + } + case MemoryMode::Signaling: { + if (uoffset >= Memory::fastMappedRedzoneBytes()) { + uint64_t maximum = m_info.memory.maximum() ? m_info.memory.maximum().bytes() : std::numeric_limits::max(); + throwExceptionIf(ExceptionType::OutOfBoundsMemoryAccess, m_jit.branchPtr(RelationalCondition::AboveOrEqual, boundsCheckReg, TrustedImmPtr(static_cast(maximum)))); + } + break; + } + } + } + } else { + switch (m_mode) { + case MemoryMode::BoundsChecking: + m_jit.move(TrustedImm32(constantPointer), boundsCheckReg); + throwExceptionIf(ExceptionType::OutOfBoundsMemoryAccess, m_jit.branchPtr(RelationalCondition::AboveOrEqual, boundsCheckReg, wasmBoundsCheckingSizeRegister)); + break; + case MemoryMode::Signaling: + if (uoffset >= Memory::fastMappedRedzoneBytes()) { + uint64_t maximum = m_info.memory.maximum() ? m_info.memory.maximum().bytes() : std::numeric_limits::max(); + m_jit.move(TrustedImm32(constantPointer), boundsCheckReg); + throwExceptionIf(ExceptionType::OutOfBoundsMemoryAccess, m_jit.branchPtr(RelationalCondition::AboveOrEqual, boundsCheckReg, TrustedImmPtr(static_cast(maximum)))); + } + break; + } + } + // Use original constantPointer (not folded) for address calculation + m_jit.add32(TrustedImm32(constantPointer), wasmBaseMemoryPointer, wasmScratchGPR); + if (static_cast(uoffset) > static_cast(std::numeric_limits::max()) || !B3::Air::Arg::isValidAddrForm(B3::Air::Move, static_cast(uoffset), Width::Width128)) { + m_jit.addPtr(TrustedImmPtr(static_cast(uoffset)), wasmScratchGPR); + return functor(Address(wasmScratchGPR)); + } + return functor(Address(wasmScratchGPR, static_cast(uoffset))); } else pointerLocation = loadIfNecessary(pointer); ASSERT(pointerLocation.isGPR()); + GPRReg pointerReg = pointerLocation.asGPR(); switch (m_mode) { case MemoryMode::BoundsChecking: { // We're not using signal handling only when the memory is not shared. // Regardless of signaling, we must check that no memory access exceeds the current memory size. - m_jit.zeroExtend32ToWord(pointerLocation.asGPR(), wasmScratchGPR); - if (boundary) - // NB: On 32-bit we have to check the addition for overflow - throwExceptionIf(ExceptionType::OutOfBoundsMemoryAccess, m_jit.branchAdd32(ResultCondition::Carry, wasmScratchGPR, TrustedImm32(boundary), wasmScratchGPR)); - throwExceptionIf(ExceptionType::OutOfBoundsMemoryAccess, m_jit.branchPtr(RelationalCondition::AboveOrEqual, wasmScratchGPR, wasmBoundsCheckingSizeRegister)); + if (boundary) { + m_jit.add32(TrustedImm32(boundary), pointerReg, wasmScratchGPR); + throwExceptionIf(ExceptionType::OutOfBoundsMemoryAccess, m_jit.branch32(RelationalCondition::Below, wasmScratchGPR, pointerReg)); + throwExceptionIf(ExceptionType::OutOfBoundsMemoryAccess, m_jit.branchPtr(RelationalCondition::AboveOrEqual, wasmScratchGPR, wasmBoundsCheckingSizeRegister)); + } else + throwExceptionIf(ExceptionType::OutOfBoundsMemoryAccess, m_jit.branchPtr(RelationalCondition::AboveOrEqual, pointerReg, wasmBoundsCheckingSizeRegister)); break; } @@ -101,17 +148,16 @@ auto BBQJIT::emitCheckAndPrepareAndMaterializePointerApply(Value pointer, uint32 // any access equal to or greater than 4GiB will trap, no need to add the redzone. if (uoffset >= Memory::fastMappedRedzoneBytes()) { uint64_t maximum = m_info.memory.maximum() ? m_info.memory.maximum().bytes() : std::numeric_limits::max(); - m_jit.zeroExtend32ToWord(pointerLocation.asGPR(), wasmScratchGPR); - if (boundary) - m_jit.addPtr(TrustedImmPtr(boundary), wasmScratchGPR); - throwExceptionIf(ExceptionType::OutOfBoundsMemoryAccess, m_jit.branchPtr(RelationalCondition::AboveOrEqual, wasmScratchGPR, TrustedImmPtr(static_cast(maximum)))); + if (boundary) { + m_jit.add32(TrustedImm32(boundary), pointerReg, wasmScratchGPR); + throwExceptionIf(ExceptionType::OutOfBoundsMemoryAccess, m_jit.branchPtr(RelationalCondition::AboveOrEqual, wasmScratchGPR, TrustedImmPtr(static_cast(maximum)))); + } else + throwExceptionIf(ExceptionType::OutOfBoundsMemoryAccess, m_jit.branchPtr(RelationalCondition::AboveOrEqual, pointerReg, TrustedImmPtr(static_cast(maximum)))); } break; } } - - m_jit.zeroExtend32ToWord(pointerLocation.asGPR(), wasmScratchGPR); - m_jit.addPtr(wasmBaseMemoryPointer, wasmScratchGPR); + m_jit.add32(pointerReg, wasmBaseMemoryPointer, wasmScratchGPR); if (static_cast(uoffset) > static_cast(std::numeric_limits::max()) || !B3::Air::Arg::isValidAddrForm(B3::Air::Move, uoffset, Width::Width128)) { m_jit.addPtr(TrustedImmPtr(static_cast(uoffset)), wasmScratchGPR); diff --git a/Source/JavaScriptCore/wasm/WasmBBQJIT64.cpp b/Source/JavaScriptCore/wasm/WasmBBQJIT64.cpp index 264b54786897c..54d3860368977 100644 --- a/Source/JavaScriptCore/wasm/WasmBBQJIT64.cpp +++ b/Source/JavaScriptCore/wasm/WasmBBQJIT64.cpp @@ -3175,376 +3175,6 @@ PartialResult WARN_UNUSED_RETURN BBQJIT::addRethrow(unsigned, ControlType& data) return { }; } -BBQJIT::BranchFoldResult BBQJIT::tryFoldFusedBranchCompare(OpType opType, ExpressionType operand) -{ - if (!operand.isConst()) - return BranchNotFolded; - switch (opType) { - case OpType::I32Eqz: - return operand.asI32() ? BranchNeverTaken : BranchAlwaysTaken; - case OpType::I64Eqz: - return operand.asI64() ? BranchNeverTaken : BranchAlwaysTaken; - default: - RELEASE_ASSERT_NOT_REACHED_WITH_MESSAGE("Op type '%s' is not a unary comparison and should not have been fused.\n", makeString(opType).characters()); - } - return BranchNotFolded; -} - -BBQJIT::Jump BBQJIT::emitFusedBranchCompareBranch(OpType opType, ExpressionType, Location operandLocation) -{ - // Emit the negation of the intended branch. - switch (opType) { - case OpType::I32Eqz: - return m_jit.branchTest32(ResultCondition::NonZero, operandLocation.asGPR()); - case OpType::I64Eqz: - return m_jit.branchTest64(ResultCondition::NonZero, operandLocation.asGPR()); - default: - RELEASE_ASSERT_NOT_REACHED_WITH_MESSAGE("Op type '%s' is not a unary comparison and should not have been fused.\n", makeString(opType).characters()); - } -} - -PartialResult BBQJIT::addFusedBranchCompare(OpType opType, ControlType& target, ExpressionType operand, Stack& results) -{ - ASSERT(!operand.isNone()); - - switch (tryFoldFusedBranchCompare(opType, operand)) { - case BranchNeverTaken: - return { }; - case BranchAlwaysTaken: - currentControlData().flushAndSingleExit(*this, target, results, false, false); - target.addBranch(m_jit.jump()); - return { }; - case BranchNotFolded: - break; - } - - { - // Like in normal addBranch(), we can directly use the operand location - // because it shouldn't interfere with flushAtBlockBoundary(). - Location operandLocation = loadIfNecessary(operand); - consume(operand); - - LOG_INSTRUCTION("BranchCompare", makeString(opType).characters(), operand, operandLocation); - - currentControlData().flushAtBlockBoundary(*this, 0, results, false); - Jump ifNotTaken = emitFusedBranchCompareBranch(opType, operand, operandLocation); - currentControlData().addExit(*this, target.targetLocations(), results); - target.addBranch(m_jit.jump()); - ifNotTaken.link(&m_jit); - currentControlData().finalizeBlock(*this, target.targetLocations().size(), results, true); - } - - return { }; -} - -PartialResult WARN_UNUSED_RETURN BBQJIT::addFusedIfCompare(OpType op, ExpressionType operand, BlockSignature signature, Stack& enclosingStack, ControlData& result, Stack& newStack) -{ - BranchFoldResult foldResult = tryFoldFusedBranchCompare(op, operand); - - ScratchScope<0, 1> scratches(*this); - Location operandLocation; - RegisterSet liveScratchGPRs, liveScratchFPRs; - if (foldResult == BranchNotFolded) { - if (!operand.isConst()) - operandLocation = loadIfNecessary(operand); - else if (operand.isFloat()) { - operandLocation = Location::fromFPR(scratches.fpr(0)); - emitMove(operand, operandLocation); - } - - if (operandLocation.isGPR()) - liveScratchGPRs.add(operandLocation.asGPR(), IgnoreVectors); - else - liveScratchFPRs.add(operandLocation.asFPR(), operand.type() == TypeKind::V128 ? Width128 : Width64); - } - if (!liveScratchFPRs.contains(scratches.fpr(0), IgnoreVectors)) - scratches.unbindEarly(); - - consume(operand); - - result = ControlData(*this, BlockType::If, signature, currentControlData().enclosedHeight() + currentControlData().implicitSlots() + enclosingStack.size() - signature.m_signature->argumentCount(), liveScratchGPRs, liveScratchFPRs); - - // Despite being conditional, if doesn't need to worry about diverging expression stacks at block boundaries, so it doesn't need multiple exits. - currentControlData().flushAndSingleExit(*this, result, enclosingStack, true, false); - - LOG_INSTRUCTION("IfCompare", makeString(op).characters(), *signature.m_signature, operand, operandLocation); - LOG_INDENT(); - splitStack(signature, enclosingStack, newStack); - - result.startBlock(*this, newStack); - if (foldResult == BranchNeverTaken) - result.setIfBranch(m_jit.jump()); // Emit direct branch if we know the condition is false. - else if (foldResult == BranchNotFolded) // Otherwise, we only emit a branch at all if we don't know the condition statically. - result.setIfBranch(emitFusedBranchCompareBranch(op, operand, operandLocation)); - return { }; -} - -BBQJIT::BranchFoldResult BBQJIT::tryFoldFusedBranchCompare(OpType opType, ExpressionType left, ExpressionType right) -{ - if (!left.isConst() || !right.isConst()) - return BranchNotFolded; - switch (opType) { - case OpType::I32LtS: - return left.asI32() < right.asI32() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::I32LtU: - return static_cast(left.asI32()) < static_cast(right.asI32()) ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::I32GtS: - return left.asI32() > right.asI32() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::I32GtU: - return static_cast(left.asI32()) > static_cast(right.asI32()) ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::I32LeS: - return left.asI32() <= right.asI32() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::I32LeU: - return static_cast(left.asI32()) <= static_cast(right.asI32()) ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::I32GeS: - return left.asI32() >= right.asI32() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::I32GeU: - return static_cast(left.asI32()) >= static_cast(right.asI32()) ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::I32Eq: - return left.asI32() == right.asI32() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::I32Ne: - return left.asI32() == right.asI32() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::I64LtS: - return left.asI64() < right.asI64() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::I64LtU: - return static_cast(left.asI64()) < static_cast(right.asI64()) ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::I64GtS: - return left.asI64() > right.asI64() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::I64GtU: - return static_cast(left.asI64()) > static_cast(right.asI64()) ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::I64LeS: - return left.asI64() <= right.asI64() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::I64LeU: - return static_cast(left.asI64()) <= static_cast(right.asI64()) ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::I64GeS: - return left.asI64() >= right.asI64() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::I64GeU: - return static_cast(left.asI64()) >= static_cast(right.asI64()) ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::I64Eq: - return left.asI64() == right.asI64() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::I64Ne: - return left.asI64() == right.asI64() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::F32Lt: - return left.asF32() < right.asF32() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::F32Gt: - return left.asF32() > right.asF32() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::F32Le: - return left.asF32() <= right.asF32() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::F32Ge: - return left.asF32() >= right.asF32() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::F32Eq: - return left.asF32() == right.asF32() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::F32Ne: - return left.asF32() != right.asF32() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::F64Lt: - return left.asF64() < right.asF64() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::F64Gt: - return left.asF64() > right.asF64() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::F64Le: - return left.asF64() <= right.asF64() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::F64Ge: - return left.asF64() >= right.asF64() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::F64Eq: - return left.asF64() == right.asF64() ? BranchAlwaysTaken : BranchNeverTaken; - case OpType::F64Ne: - return left.asF64() != right.asF64() ? BranchAlwaysTaken : BranchNeverTaken; - default: - RELEASE_ASSERT_NOT_REACHED_WITH_MESSAGE("Op type '%s' is not a binary comparison and should not have been fused.\n", makeString(opType).characters()); - } -} - -static MacroAssembler::Jump emitBranchI32(CCallHelpers& jit, MacroAssembler::RelationalCondition condition, Value left, Location leftLocation, Value right, Location rightLocation) -{ - if (right.isConst()) - return jit.branch32(condition, leftLocation.asGPR(), MacroAssembler::TrustedImm32(right.asI32())); - if (left.isConst()) - return jit.branch32(condition, MacroAssembler::TrustedImm32(left.asI32()), rightLocation.asGPR()); - return jit.branch32(condition, leftLocation.asGPR(), rightLocation.asGPR()); -} - -static MacroAssembler::Jump emitBranchI64(CCallHelpers& jit, MacroAssembler::RelationalCondition condition, Value left, Location leftLocation, Value right, Location rightLocation) -{ - if (right.isConst()) - return jit.branch64(condition, leftLocation.asGPR(), MacroAssembler::Imm64(right.asI64())); - if (left.isConst()) - return jit.branch64(MacroAssembler::commute(condition), rightLocation.asGPR(), MacroAssembler::Imm64(left.asI64())); - return jit.branch64(condition, leftLocation.asGPR(), rightLocation.asGPR()); -} - -static MacroAssembler::Jump emitBranchF32(CCallHelpers& jit, MacroAssembler::DoubleCondition condition, Value, Location leftLocation, Value, Location rightLocation) -{ - return jit.branchFloat(condition, leftLocation.asFPR(), rightLocation.asFPR()); -} - -static MacroAssembler::Jump emitBranchF64(CCallHelpers& jit, MacroAssembler::DoubleCondition condition, Value, Location leftLocation, Value, Location rightLocation) -{ - return jit.branchDouble(condition, leftLocation.asFPR(), rightLocation.asFPR()); -} - -BBQJIT::Jump BBQJIT::emitFusedBranchCompareBranch(OpType opType, ExpressionType left, Location leftLocation, ExpressionType right, Location rightLocation) -{ - // Emit a branch with the inverse of the comparison. We're generating the "branch-if-false" case. - switch (opType) { - case OpType::I32LtS: - return emitBranchI32(m_jit, RelationalCondition::GreaterThanOrEqual, left, leftLocation, right, rightLocation); - case OpType::I32LtU: - return emitBranchI32(m_jit, RelationalCondition::AboveOrEqual, left, leftLocation, right, rightLocation); - case OpType::I32GtS: - return emitBranchI32(m_jit, RelationalCondition::LessThanOrEqual, left, leftLocation, right, rightLocation); - case OpType::I32GtU: - return emitBranchI32(m_jit, RelationalCondition::BelowOrEqual, left, leftLocation, right, rightLocation); - case OpType::I32LeS: - return emitBranchI32(m_jit, RelationalCondition::GreaterThan, left, leftLocation, right, rightLocation); - case OpType::I32LeU: - return emitBranchI32(m_jit, RelationalCondition::Above, left, leftLocation, right, rightLocation); - case OpType::I32GeS: - return emitBranchI32(m_jit, RelationalCondition::LessThan, left, leftLocation, right, rightLocation); - case OpType::I32GeU: - return emitBranchI32(m_jit, RelationalCondition::Below, left, leftLocation, right, rightLocation); - case OpType::I32Eq: - return emitBranchI32(m_jit, RelationalCondition::NotEqual, left, leftLocation, right, rightLocation); - case OpType::I32Ne: - return emitBranchI32(m_jit, RelationalCondition::Equal, left, leftLocation, right, rightLocation); - case OpType::I64LtS: - return emitBranchI64(m_jit, RelationalCondition::GreaterThanOrEqual, left, leftLocation, right, rightLocation); - case OpType::I64LtU: - return emitBranchI64(m_jit, RelationalCondition::AboveOrEqual, left, leftLocation, right, rightLocation); - case OpType::I64GtS: - return emitBranchI64(m_jit, RelationalCondition::LessThanOrEqual, left, leftLocation, right, rightLocation); - case OpType::I64GtU: - return emitBranchI64(m_jit, RelationalCondition::BelowOrEqual, left, leftLocation, right, rightLocation); - case OpType::I64LeS: - return emitBranchI64(m_jit, RelationalCondition::GreaterThan, left, leftLocation, right, rightLocation); - case OpType::I64LeU: - return emitBranchI64(m_jit, RelationalCondition::Above, left, leftLocation, right, rightLocation); - case OpType::I64GeS: - return emitBranchI64(m_jit, RelationalCondition::LessThan, left, leftLocation, right, rightLocation); - case OpType::I64GeU: - return emitBranchI64(m_jit, RelationalCondition::Below, left, leftLocation, right, rightLocation); - case OpType::I64Eq: - return emitBranchI64(m_jit, RelationalCondition::NotEqual, left, leftLocation, right, rightLocation); - case OpType::I64Ne: - return emitBranchI64(m_jit, RelationalCondition::Equal, left, leftLocation, right, rightLocation); - case OpType::F32Lt: - return emitBranchF32(m_jit, MacroAssembler::invert(DoubleCondition::DoubleLessThanAndOrdered), left, leftLocation, right, rightLocation); - case OpType::F32Gt: - return emitBranchF32(m_jit, MacroAssembler::invert(DoubleCondition::DoubleGreaterThanAndOrdered), left, leftLocation, right, rightLocation); - case OpType::F32Le: - return emitBranchF32(m_jit, MacroAssembler::invert(DoubleCondition::DoubleLessThanOrEqualAndOrdered), left, leftLocation, right, rightLocation); - case OpType::F32Ge: - return emitBranchF32(m_jit, MacroAssembler::invert(DoubleCondition::DoubleGreaterThanOrEqualAndOrdered), left, leftLocation, right, rightLocation); - case OpType::F32Eq: - return emitBranchF32(m_jit, MacroAssembler::invert(DoubleCondition::DoubleEqualAndOrdered), left, leftLocation, right, rightLocation); - case OpType::F32Ne: - return emitBranchF32(m_jit, MacroAssembler::invert(DoubleCondition::DoubleNotEqualOrUnordered), left, leftLocation, right, rightLocation); - case OpType::F64Lt: - return emitBranchF64(m_jit, MacroAssembler::invert(DoubleCondition::DoubleLessThanAndOrdered), left, leftLocation, right, rightLocation); - case OpType::F64Gt: - return emitBranchF64(m_jit, MacroAssembler::invert(DoubleCondition::DoubleGreaterThanAndOrdered), left, leftLocation, right, rightLocation); - case OpType::F64Le: - return emitBranchF64(m_jit, MacroAssembler::invert(DoubleCondition::DoubleLessThanOrEqualAndOrdered), left, leftLocation, right, rightLocation); - case OpType::F64Ge: - return emitBranchF64(m_jit, MacroAssembler::invert(DoubleCondition::DoubleGreaterThanOrEqualAndOrdered), left, leftLocation, right, rightLocation); - case OpType::F64Eq: - return emitBranchF64(m_jit, MacroAssembler::invert(DoubleCondition::DoubleEqualAndOrdered), left, leftLocation, right, rightLocation); - case OpType::F64Ne: - return emitBranchF64(m_jit, MacroAssembler::invert(DoubleCondition::DoubleNotEqualOrUnordered), left, leftLocation, right, rightLocation); - default: - RELEASE_ASSERT_NOT_REACHED_WITH_MESSAGE("Op type '%s' is not a binary comparison and should not have been fused.\n", makeString(opType).characters()); - } -} - -PartialResult BBQJIT::addFusedBranchCompare(OpType opType, ControlType& target, ExpressionType left, ExpressionType right, Stack& results) -{ - switch (tryFoldFusedBranchCompare(opType, left, right)) { - case BranchNeverTaken: - return { }; - case BranchAlwaysTaken: - currentControlData().flushAndSingleExit(*this, target, results, false, false); - target.addBranch(m_jit.jump()); - return { }; - case BranchNotFolded: - break; - } - - { - Location leftLocation, rightLocation; - - if (!left.isConst()) - leftLocation = loadIfNecessary(left); - else if (left.isFloat()) // Materialize floats here too, since they don't have a good immediate lowering. - emitMove(left, leftLocation = Location::fromFPR(wasmScratchFPR)); - if (!right.isConst()) - rightLocation = loadIfNecessary(right); - else if (right.isFloat()) - emitMove(right, rightLocation = Location::fromFPR(wasmScratchFPR)); - - consume(left); - consume(right); - - LOG_INSTRUCTION("BranchCompare", makeString(opType).characters(), left, leftLocation, right, rightLocation); - - currentControlData().flushAtBlockBoundary(*this, 0, results, false); - Jump ifNotTaken = emitFusedBranchCompareBranch(opType, left, leftLocation, right, rightLocation); - currentControlData().addExit(*this, target.targetLocations(), results); - target.addBranch(m_jit.jump()); - ifNotTaken.link(&m_jit); - currentControlData().finalizeBlock(*this, target.targetLocations().size(), results, true); - } - - return { }; -} - -PartialResult WARN_UNUSED_RETURN BBQJIT::addFusedIfCompare(OpType op, ExpressionType left, ExpressionType right, BlockSignature signature, Stack& enclosingStack, ControlData& result, Stack& newStack) -{ - BranchFoldResult foldResult = tryFoldFusedBranchCompare(op, left, right); - - ScratchScope<0, 1> scratches(*this, RegisterSetBuilder::argumentGPRs(), RegisterSetBuilder::argumentFPRs()); - scratches.unbindPreserved(); - Location leftLocation, rightLocation; - RegisterSet liveScratchGPRs, liveScratchFPRs; - if (foldResult == BranchNotFolded) { - ASSERT(!left.isConst() || !right.isConst()); // If they're both constants, we should have folded. - - if (!left.isConst()) - leftLocation = loadIfNecessary(left); - else if (left.isFloat()) - emitMove(left, leftLocation = Location::fromFPR(scratches.fpr(0))); - if (leftLocation.isGPR()) - liveScratchGPRs.add(leftLocation.asGPR(), IgnoreVectors); - else if (leftLocation.isFPR()) - liveScratchFPRs.add(leftLocation.asFPR(), left.type() == TypeKind::V128 ? Width128 : Width64); - - if (!right.isConst()) - rightLocation = loadIfNecessary(right); - else if (right.isFloat()) - emitMove(right, rightLocation = Location::fromFPR(scratches.fpr(0))); - if (rightLocation.isGPR()) - liveScratchGPRs.add(rightLocation.asGPR(), IgnoreVectors); - else if (rightLocation.isFPR()) - liveScratchFPRs.add(rightLocation.asFPR(), right.type() == TypeKind::V128 ? Width128 : Width64); - } - consume(left); - consume(right); - - - result = ControlData(*this, BlockType::If, signature, currentControlData().enclosedHeight() + currentControlData().implicitSlots() + enclosingStack.size() - signature.m_signature->argumentCount(), liveScratchGPRs, liveScratchFPRs); - - // Despite being conditional, if doesn't need to worry about diverging expression stacks at block boundaries, so it doesn't need multiple exits. - currentControlData().flushAndSingleExit(*this, result, enclosingStack, true, false); - - LOG_INSTRUCTION("IfCompare", makeString(op).characters(), *signature.m_signature, left, leftLocation, right, rightLocation); - LOG_INDENT(); - splitStack(signature, enclosingStack, newStack); - - result.startBlock(*this, newStack); - if (foldResult == BranchNeverTaken) - result.setIfBranch(m_jit.jump()); // Emit direct branch if we know the condition is false. - else if (foldResult == BranchNotFolded) // Otherwise, we only emit a branch at all if we don't know the condition statically. - result.setIfBranch(emitFusedBranchCompareBranch(op, left, leftLocation, right, rightLocation)); - return { }; -} - PartialResult WARN_UNUSED_RETURN BBQJIT::addBranchNull(ControlData& data, ExpressionType reference, Stack& returnValues, bool shouldNegate, ExpressionType& result) { if (reference.isConst() && (reference.asRef() == JSValue::encode(jsNull())) == shouldNegate) { diff --git a/Source/JavaScriptCore/wasm/WasmOperationsInlines.h b/Source/JavaScriptCore/wasm/WasmOperationsInlines.h index 3be526d74075a..12e5c5f3c508a 100644 --- a/Source/JavaScriptCore/wasm/WasmOperationsInlines.h +++ b/Source/JavaScriptCore/wasm/WasmOperationsInlines.h @@ -433,7 +433,12 @@ inline bool arrayInitElem(JSWebAssemblyInstance* instance, EncodedJSValue dst, u if (lastSrcElementIndexChecked > lengthOfElementSegment) return false; - instance->copyElementSegment(dstObject, *instance->elementAt(srcElementIndex), srcOffset, size, dstObject->reftypeData() + dstOffset); + auto* elementSegment = instance->elementAt(srcElementIndex); + if (elementSegment) + instance->copyElementSegment(dstObject, *elementSegment, srcOffset, size, dstObject->reftypeData() + dstOffset); + else + ASSERT(!lastSrcElementIndexChecked); + return true; } diff --git a/Source/cmake/OptionsCommon.cmake b/Source/cmake/OptionsCommon.cmake index bd2055ca29505..e24d6c96d8989 100644 --- a/Source/cmake/OptionsCommon.cmake +++ b/Source/cmake/OptionsCommon.cmake @@ -204,7 +204,7 @@ if (DEBUG_FISSION) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -gsplit-dwarf") if (LD_SUPPORTS_GDB_INDEX) set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gdb-index") - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gdb-index") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--gdb-index") endif () if (WTF_CPU_ARM) string(REPLACE "-fdebug-types-section" "" CMAKE_C_FLAGS ${CMAKE_C_FLAGS})