diff --git a/FEXCore/Source/Interface/Core/CPUBackend.h b/FEXCore/Source/Interface/Core/CPUBackend.h index 8fc2de268..f9930fa89 100644 --- a/FEXCore/Source/Interface/Core/CPUBackend.h +++ b/FEXCore/Source/Interface/Core/CPUBackend.h @@ -102,22 +102,6 @@ namespace CPU { uint32_t _Pad; }; - // Entries that live after the JITCodeTail. - // These entries correlate JIT code regions with guest RIP regions. - // Using these entries FEX is able to reconstruct the guest RIP accurately when an instruction cause a signal fault. - // Packed using 16-bit entries to ensure the size isn't too large. - // These smaller sizes means that each entry is relative to each other instead of absolute offset from the start of the JIT block. - // When reconstructing the RIP, each entry must be walked linearly and accumulated with the previous entries. - // This is a trade-off between compression inside the JIT code space and execution time when reconstruction the RIP. - // RIP reconstruction when faulting is less likely so we are requiring the accumulation. - struct JITRIPReconstructEntries { - // The Host PC offset from the previous entry. - uint16_t HostPCOffset; - - // How much to offset the RIP from the previous entry. - int16_t GuestRIPOffset; - }; - /** * @brief Tells this CPUBackend to compile code for the provided IR and DebugData * diff --git a/FEXCore/Source/Interface/Core/Core.cpp b/FEXCore/Source/Interface/Core/Core.cpp index fcea48851..693c4fb2d 100644 --- a/FEXCore/Source/Interface/Core/Core.cpp +++ b/FEXCore/Source/Interface/Core/Core.cpp @@ -28,6 +28,7 @@ $end_info$ #include "Utils/Allocator.h" #include "Utils/Allocator/HostAllocator.h" #include "Utils/SpinWaitLock.h" +#include "Utils/variable_length_integer.h" #include #include @@ -144,24 +145,27 @@ uint64_t ContextImpl::RestoreRIPFromHostPC(FEXCore::Core::InternalThreadState* T auto [InlineHeader, InlineTail] = GetFrameBlockInfo(Thread->CurrentFrame); if (InlineHeader) { - auto RIPEntries = reinterpret_cast( - Frame->State.InlineJITBlockHeader + InlineHeader->OffsetToBlockTail + InlineTail->OffsetToRIPEntries); - // Check if the host PC is currently within a code block. // If it is then RIP can be reconstructed from the beginning of the code block. // This is currently as close as FEX can get RIP reconstructions. if (HostPC >= reinterpret_cast(BlockBegin) && HostPC < reinterpret_cast(BlockBegin + InlineTail->Size)) { + auto RIPEntry = + reinterpret_cast(Frame->State.InlineJITBlockHeader + InlineHeader->OffsetToBlockTail + InlineTail->OffsetToRIPEntries); + // Reconstruct RIP from JIT entries for this block. uint64_t StartingHostPC = BlockBegin; uint64_t StartingGuestRIP = InlineTail->RIP; for (uint32_t i = 0; i < InlineTail->NumberOfRIPEntries; ++i) { - const auto& RIPEntry = RIPEntries[i]; - if (HostPC >= (StartingHostPC + RIPEntry.HostPCOffset)) { + auto HostPCOffset = FEXCore::Utils::vl64::Decode(RIPEntry); + RIPEntry += HostPCOffset.Size; + auto GuestRIPOffset = FEXCore::Utils::vl64::Decode(RIPEntry); + RIPEntry += GuestRIPOffset.Size; + if (HostPC >= (StartingHostPC + HostPCOffset.Integer)) { // We are beyond this entry, keep going forward. - StartingHostPC += RIPEntry.HostPCOffset; - StartingGuestRIP += RIPEntry.GuestRIPOffset; + StartingHostPC += HostPCOffset.Integer; + StartingGuestRIP += GuestRIPOffset.Integer; } else { // Passed where the Host PC is at. Break now. break; diff --git a/FEXCore/Source/Interface/Core/JIT/JIT.cpp b/FEXCore/Source/Interface/Core/JIT/JIT.cpp index 1929719c8..65f28a0a3 100644 --- a/FEXCore/Source/Interface/Core/JIT/JIT.cpp +++ b/FEXCore/Source/Interface/Core/JIT/JIT.cpp @@ -21,6 +21,7 @@ $end_info$ #include "Interface/IR/Passes/RegisterAllocationPass.h" #include "Utils/MemberFunctionToPointer.h" +#include "Utils/variable_length_integer.h" #include #include @@ -852,10 +853,23 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size auto JITBlockTail = GetCursorAddress(); CursorIncrement(sizeof(JITCodeTail)); - auto JITRIPEntriesLocation = GetCursorAddress(); - auto JITRIPEntries = GetCursorAddress(); + // Entries that live after the JITCodeTail. + // These entries correlate JIT code regions with guest RIP regions. + // Using these entries FEX is able to reconstruct the guest RIP accurately when an instruction cause a signal fault. + // Packed using two variable length integer entries to ensure the size isn't too large. + // These smaller sizes means that each entry is relative to each other instead of absolute offset from the start of the JIT block. + // When reconstructing the RIP, each entry must be walked linearly and accumulated with the previous entries. + // This is a trade-off between compression inside the JIT code space and execution time when reconstruction the RIP. + // RIP reconstruction when faulting is less likely so we are requiring the accumulation. + // + // struct { + // // The Host PC offset from the previous entry. + // FEXCore::Utils::vl64 HostPCOffset; + // // How much to offset the RIP from the previous entry. + // FEXCore::Utils::vl64 GuestRIPOffset; + // }; - CursorIncrement(sizeof(JITRIPReconstructEntries) * DebugData->GuestOpcodes.size()); + auto JITRIPEntriesBegin = GetCursorAddress(); // Put the block's RIP entry in the tail. // This will be used for RIP reconstruction in the future. @@ -866,27 +880,34 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size JITBlockTail->SingleInst = SingleInst; JITBlockTail->SpinLockFutex = 0; + auto JITRIPEntriesLocation = JITRIPEntriesBegin; + { // Store the RIP entries. JITBlockTail->NumberOfRIPEntries = DebugData->GuestOpcodes.size(); - JITBlockTail->OffsetToRIPEntries = JITRIPEntriesLocation - JITBlockTailLocation; + JITBlockTail->OffsetToRIPEntries = JITRIPEntriesBegin - JITBlockTailLocation; uintptr_t CurrentRIPOffset = 0; uint64_t CurrentPCOffset = 0; + for (size_t i = 0; i < DebugData->GuestOpcodes.size(); i++) { const auto& GuestOpcode = DebugData->GuestOpcodes[i]; - auto& RIPEntry = JITRIPEntries[i]; - [[maybe_unused]] uint64_t HostPCOffset = GuestOpcode.HostEntryOffset - CurrentPCOffset; - [[maybe_unused]] int64_t GuestRIPOffset = GuestOpcode.GuestEntryOffset - CurrentRIPOffset; - LOGMAN_THROW_A_FMT(HostPCOffset <= std::numeric_limits::max(), "PC offset too large"); - LOGMAN_THROW_A_FMT(GuestRIPOffset >= std::numeric_limits::min(), "RIP offset too small"); - LOGMAN_THROW_A_FMT(GuestRIPOffset <= std::numeric_limits::max(), "RIP offset too large"); - RIPEntry.HostPCOffset = GuestOpcode.HostEntryOffset - CurrentPCOffset; - RIPEntry.GuestRIPOffset = GuestOpcode.GuestEntryOffset - CurrentRIPOffset; + int64_t HostPCOffset = GuestOpcode.HostEntryOffset - CurrentPCOffset; + int64_t GuestRIPOffset = GuestOpcode.GuestEntryOffset - CurrentRIPOffset; + + size_t Size = FEXCore::Utils::vl64::Encode(JITRIPEntriesLocation, HostPCOffset); + JITRIPEntriesLocation += Size; + + Size = FEXCore::Utils::vl64::Encode(JITRIPEntriesLocation, GuestRIPOffset); + JITRIPEntriesLocation += Size; + CurrentPCOffset = GuestOpcode.HostEntryOffset; CurrentRIPOffset = GuestOpcode.GuestEntryOffset; } } + CursorIncrement(JITRIPEntriesLocation - JITRIPEntriesBegin); + Align(); + CodeHeader->OffsetToBlockTail = JITBlockTailLocation - CodeData.BlockBegin; CodeData.Size = GetCursorAddress() - CodeData.BlockBegin; diff --git a/FEXCore/Source/Utils/variable_length_integer.h b/FEXCore/Source/Utils/variable_length_integer.h new file mode 100644 index 000000000..875eedbbf --- /dev/null +++ b/FEXCore/Source/Utils/variable_length_integer.h @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: MIT +#pragma once + +#include + +#include +#include +#include +#include + +namespace FEXCore::Utils { +// Variable length signed integer +// The most common encoded size is 8-bit positive, but other values can occur +// +// 8-bit: +// bit[7] = 0 - 8-bit +// bit[6:0] = 7-bit encoding +// +// 16-bit: +// byte1[7:6] = 0b10 - 16-bit +// byte1[5:0] = top 6-bits +// byte2[7:0] = Bottom 8-bits bits +// +// 32-bit +// byte1[7:5] = 0b110 - 32-bit +// byte1[4:0] = +// word[31:0] = signed word +// +// 64-bit +// byte1[7:5] = 0b111 - 64-bit +// byte1[4:0] = +// dword[63:0] = signed dword +struct vl64 final { + static size_t EncodedSize(int64_t Data) { + if (Data >= vl8_min && Data <= vl8_max) { + return sizeof(vl8_enc); + } else if (Data >= vl16_min && Data <= vl16_max) { + return sizeof(vl16_enc); + } else if (Data >= vl32_min && Data <= vl32_max) { + return sizeof(vl32_enc); + } + return sizeof(vl64_enc); + } + + struct Decoded { + int64_t Integer; + size_t Size; + }; + + static Decoded Decode(const uint8_t* data) { + auto vl8_type = reinterpret_cast(data); + auto vl16_type = reinterpret_cast(data); + auto vl32_type = reinterpret_cast(data); + auto vl64_type = reinterpret_cast(data); + + if (vl8_type->Type == vl8_type_header) { + return {vl8_type->Integer, sizeof(vl8_enc)}; + } else if (vl16_type->Type == vl16_type_header) { + return {vl16_type->Integer, sizeof(vl16_enc)}; + } else if (vl32_type->Type == vl32_type_header) { + return {vl32_type->Integer, sizeof(vl32_enc)}; + } + return {vl64_type->Integer, sizeof(vl64_enc)}; + } + + static size_t Encode(uint8_t* dst, int64_t Data) { + auto vl8_type = reinterpret_cast(dst); + auto vl16_type = reinterpret_cast(dst); + auto vl32_type = reinterpret_cast(dst); + auto vl64_type = reinterpret_cast(dst); + + if (Data >= vl8_min && Data <= vl8_max) { + *vl8_type = { + .Integer = static_cast(Data), + .Type = vl8_type_header, + }; + return sizeof(vl8_enc); + } else if (Data >= vl16_min && Data <= vl16_max) { + *vl16_type = { + .Integer = static_cast(Data), + .Type = vl16_type_header, + }; + return sizeof(vl16_enc); + } else if (Data >= vl32_min && Data <= vl32_max) { + *vl32_type = { + .Type = vl32_type_header, + .Integer = static_cast(Data), + }; + return sizeof(vl32_enc); + } + + *vl64_type = { + .Type = vl64_type_header, + .Integer = Data, + }; + return sizeof(vl64_enc); + } + +private: + + struct vl8_enc { + int8_t Integer : 7; + uint8_t Type : 1; + }; + static_assert(sizeof(vl8_enc) == 1); + + struct vl16_enc { + int16_t Integer : 14; + uint16_t Type : 2; + }; + static_assert(sizeof(vl16_enc) == 2); + + struct FEX_PACKED vl32_enc { + uint8_t Type; + int32_t Integer; + }; + static_assert(sizeof(vl32_enc) == 5); + + struct FEX_PACKED vl64_enc { + uint8_t Type; + int64_t Integer; + }; + static_assert(sizeof(vl64_enc) == 9); + + // Maximum ranges for encodings. + + // vl8 can hold a signed 7-bit integer. + // Encoded in one 8-bit value. + constexpr static int64_t vl8_encoded_bits = 7; + constexpr static int64_t vl8_type_header = 0; + constexpr static int64_t vl8_min = std::numeric_limits::min() >> ((sizeof(int64_t) * 8) - vl8_encoded_bits); + constexpr static int64_t vl8_max = std::numeric_limits::max() >> ((sizeof(int64_t) * 8) - vl8_encoded_bits); + + // vl16 can hold a signed 14-bit integer. + // Encoded in one 16-bit value. + constexpr static int64_t vl16_encoded_bits = 14; + constexpr static int64_t vl16_type_header = 0b10; + constexpr static int64_t vl16_min = std::numeric_limits::min() >> ((sizeof(int64_t) * 8) - vl16_encoded_bits); + constexpr static int64_t vl16_max = std::numeric_limits::max() >> ((sizeof(int64_t) * 8) - vl16_encoded_bits); + + // vl32 can hold a signed 32-bit integer. + // Encoded in 8-bit and 32-bit value; + constexpr static int64_t vl32_encoded_bits = 32; + constexpr static int64_t vl32_type_header = 0b1100'0000; + constexpr static int64_t vl32_min = std::numeric_limits::min(); + constexpr static int64_t vl32_max = std::numeric_limits::max(); + + // vl64 can hold a signed 32-bit integer. + // Encoded in 8-bit and 64-bit value. + constexpr static int64_t vl64_encoded_bits = 64; + constexpr static int64_t vl64_type_header = 0b1110'0000; + constexpr static int64_t vl64_min = std::numeric_limits::min(); + constexpr static int64_t vl64_max = std::numeric_limits::max(); +}; + +} // namespace FEXCore::Utils diff --git a/FEXCore/unittests/APITests/vl_integer.cpp b/FEXCore/unittests/APITests/vl_integer.cpp new file mode 100644 index 000000000..c9e5bb2ad --- /dev/null +++ b/FEXCore/unittests/APITests/vl_integer.cpp @@ -0,0 +1,114 @@ +#include +#include +#include + +#include "Utils/variable_length_integer.h" + +#include + +TEST_CASE("vl-size") { + // Check 8-bit minimum and maximum. + CHECK(FEXCore::Utils::vl64::EncodedSize(-64) == 1); + CHECK(FEXCore::Utils::vl64::EncodedSize(63) == 1); + + // Check 16-bit minimum and maximum. + CHECK(FEXCore::Utils::vl64::EncodedSize(-8192) == 2); + CHECK(FEXCore::Utils::vl64::EncodedSize(8191) == 2); + + // Check 32-bit minimum and maximum. + CHECK(FEXCore::Utils::vl64::EncodedSize(std::numeric_limits::min()) == 5); + CHECK(FEXCore::Utils::vl64::EncodedSize(std::numeric_limits::max()) == 5); + + // Check 64-bit minimum and maximum. + CHECK(FEXCore::Utils::vl64::EncodedSize(std::numeric_limits::min()) == 9); + CHECK(FEXCore::Utils::vl64::EncodedSize(std::numeric_limits::max()) == 9); +} + +TEST_CASE("vl8 - in memory - encode") { + uint8_t data[1]; + REQUIRE(FEXCore::Utils::vl64::Encode(data, 0) == 1); + CHECK(data[0] == 0); + + REQUIRE(FEXCore::Utils::vl64::Encode(data, 63) == 1); + CHECK(data[0] == 0b0011'1111); + + REQUIRE(FEXCore::Utils::vl64::Encode(data, -1) == 1); + CHECK(data[0] == 0b0111'1111); + + REQUIRE(FEXCore::Utils::vl64::Encode(data, -64) == 1); + CHECK(data[0] == 0b0100'0000); +} + +TEST_CASE("vl16 - in memory - encode") { + uint8_t data[2]; + + REQUIRE(FEXCore::Utils::vl64::Encode(data, -65) == 2); + CHECK(data[0] == 0b1011'1111); + CHECK(data[1] == 0b1011'1111); + + REQUIRE(FEXCore::Utils::vl64::Encode(data, -66) == 2); + CHECK(data[0] == 0b1011'1110); + CHECK(data[1] == 0b1011'1111); + + REQUIRE(FEXCore::Utils::vl64::Encode(data, 64) == 2); + CHECK(data[0] == 0b0100'0000); + CHECK(data[1] == 0b1000'0000); + + REQUIRE(FEXCore::Utils::vl64::Encode(data, 8191) == 2); + CHECK(data[0] == 0b1111'1111); + CHECK(data[1] == 0b1001'1111); + + REQUIRE(FEXCore::Utils::vl64::Encode(data, -8192) == 2); + CHECK(data[0] == 0b0000'0000); + CHECK(data[1] == 0b1010'0000); +} + +TEST_CASE("vl32 - in memory - encode") { + uint8_t data[5]; + int32_t result {}; + + REQUIRE(FEXCore::Utils::vl64::Encode(data, 8192) == 5); + CHECK(data[0] == 0b1100'0000); + memcpy(&result, &data[1], sizeof(int32_t)); + CHECK(result == 8192); + + REQUIRE(FEXCore::Utils::vl64::Encode(data, -8193) == 5); + CHECK(data[0] == 0b1100'0000); + memcpy(&result, &data[1], sizeof(int32_t)); + CHECK(result == -8193); + + REQUIRE(FEXCore::Utils::vl64::Encode(data, std::numeric_limits::min()) == 5); + CHECK(data[0] == 0b1100'0000); + memcpy(&result, &data[1], sizeof(int32_t)); + CHECK(result == std::numeric_limits::min()); + + REQUIRE(FEXCore::Utils::vl64::Encode(data, std::numeric_limits::max()) == 5); + CHECK(data[0] == 0b1100'0000); + memcpy(&result, &data[1], sizeof(int32_t)); + CHECK(result == std::numeric_limits::max()); +} + +TEST_CASE("vl64 - in memory - encode") { + uint8_t data[9]; + int64_t result {}; + + REQUIRE(FEXCore::Utils::vl64::Encode(data, static_cast(std::numeric_limits::min()) - 1) == 9); + CHECK(data[0] == 0b1110'0000); + memcpy(&result, &data[1], sizeof(int64_t)); + CHECK(result == static_cast(std::numeric_limits::min()) - 1); + + REQUIRE(FEXCore::Utils::vl64::Encode(data, static_cast(std::numeric_limits::max()) + 1) == 9); + CHECK(data[0] == 0b1110'0000); + memcpy(&result, &data[1], sizeof(int64_t)); + CHECK(result == static_cast(std::numeric_limits::max()) + 1); + + REQUIRE(FEXCore::Utils::vl64::Encode(data, std::numeric_limits::min()) == 9); + CHECK(data[0] == 0b1110'0000); + memcpy(&result, &data[1], sizeof(int64_t)); + CHECK(result == std::numeric_limits::min()); + + REQUIRE(FEXCore::Utils::vl64::Encode(data, std::numeric_limits::max()) == 9); + CHECK(data[0] == 0b1110'0000); + memcpy(&result, &data[1], sizeof(int64_t)); + CHECK(result == std::numeric_limits::max()); +}