mirror of
https://github.com/FEX-Emu/FEX.git
synced 2026-10-06 18:00:17 +02:00
Ryan Houdek
02f8ab5f87
The various places that were using the CodeBuffer object were using internal implementation details that are changing as we move over to a bitmap allocator. Preempt this by hiding some of the implementation details early without changing behaviour. `GetBufferBase` is still technically leaking some of the internal details, but it needs changes around how relocations are being handled and how the disk cache validation works in order to handle that right now. Should be no functional change.
360 lines
15 KiB
C++
360 lines
15 KiB
C++
// SPDX-License-Identifier: MIT
|
|
#include "FEXCore/Config/Config.h"
|
|
#include "Interface/Context/Context.h"
|
|
#include "Interface/Core/CPUBackend.h"
|
|
#include "Interface/Core/LookupCache.h"
|
|
#include "Interface/Core/Dispatcher/Dispatcher.h"
|
|
|
|
#include <FEXCore/IR/IR.h>
|
|
#include <FEXCore/Utils/AllocatorHooks.h>
|
|
#include <FEXCore/Utils/PrctlUtils.h>
|
|
|
|
#include <cstdint>
|
|
|
|
namespace FEXCore {
|
|
namespace CPU {
|
|
constexpr static uint64_t NamedVectorConstants[FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_CONST_POOL_MAX][2] = {
|
|
{0x0003'0002'0001'0000ULL, 0x0007'0006'0005'0004ULL}, // NAMED_VECTOR_INCREMENTAL_U16_INDEX
|
|
{0x000B'000A'0009'0008ULL, 0x000F'000E'000D'000CULL}, // NAMED_VECTOR_INCREMENTAL_U16_INDEX_UPPER
|
|
{0x0000'0000'8000'0000ULL, 0x0000'0000'8000'0000ULL}, // NAMED_VECTOR_PADDSUBPS_INVERT
|
|
{0x0000'0000'8000'0000ULL, 0x0000'0000'8000'0000ULL}, // NAMED_VECTOR_PADDSUBPS_INVERT_UPPER
|
|
{0x8000'0000'0000'0000ULL, 0x0000'0000'0000'0000ULL}, // NAMED_VECTOR_PADDSUBPD_INVERT
|
|
{0x8000'0000'0000'0000ULL, 0x0000'0000'0000'0000ULL}, // NAMED_VECTOR_PADDSUBPD_INVERT_UPPER
|
|
{0x8000'0000'0000'0000ULL, 0x8000'0000'0000'0000ULL}, // NAMED_VECTOR_PSUBADDPS_INVERT
|
|
{0x8000'0000'0000'0000ULL, 0x8000'0000'0000'0000ULL}, // NAMED_VECTOR_PSUBADDPS_INVERT_UPPER
|
|
{0x0000'0000'0000'0000ULL, 0x8000'0000'0000'0000ULL}, // NAMED_VECTOR_PSUBADDPD_INVERT
|
|
{0x0000'0000'0000'0000ULL, 0x8000'0000'0000'0000ULL}, // NAMED_VECTOR_PSUBADDPD_INVERT_UPPER
|
|
{0x0000'0001'0000'0000ULL, 0x0000'0003'0000'0002ULL}, // NAMED_VECTOR_MOVMSKPS_SHIFT
|
|
{0x040B'0E01'0B0E'0104ULL, 0x0C03'0609'0306'090CULL}, // NAMED_VECTOR_AESKEYGENASSIST_SWIZZLE
|
|
{0x0706'0504'FFFF'FFFFULL, 0xFFFF'FFFF'0B0A'0908ULL}, // NAMED_VECTOR_BLENDPS_0110B
|
|
{0x0706'0504'0302'0100ULL, 0xFFFF'FFFF'0B0A'0908ULL}, // NAMED_VECTOR_BLENDPS_0111B
|
|
{0xFFFF'FFFF'0302'0100ULL, 0x0F0E'0D0C'FFFF'FFFFULL}, // NAMED_VECTOR_BLENDPS_1001B
|
|
{0x0706'0504'0302'0100ULL, 0x0F0E'0D0C'FFFF'FFFFULL}, // NAMED_VECTOR_BLENDPS_1011B
|
|
{0xFFFF'FFFF'0302'0100ULL, 0x0F0E'0D0C'0B0A'0908ULL}, // NAMED_VECTOR_BLENDPS_1101B
|
|
{0x0706'0504'FFFF'FFFFULL, 0x0F0E'0D0C'0B0A'0908ULL}, // NAMED_VECTOR_BLENDPS_1110B
|
|
{0x8040'2010'0804'0201ULL, 0x8040'2010'0804'0201ULL}, // NAMED_VECTOR_MOVMASKB
|
|
{0x8040'2010'0804'0201ULL, 0x8040'2010'0804'0201ULL}, // NAMED_VECTOR_MOVMASKB_UPPER
|
|
{0x0706'0504'0302'0100ULL, 0x1716'1514'1312'1110ULL}, // NAMED_VECTOR_256_MID_ELEMENT_SWAP
|
|
{0x0F0E'0D0C'0B0A'0908ULL, 0x1F1E'1D1C'1B1A'1918ULL}, // NAMED_VECTOR_256_MID_ELEMENT_SWAP_UPPER
|
|
{0x8000'0000'0000'0000ULL, 0x0000'0000'0000'3FFFULL}, // NAMED_VECTOR_X87_ONE
|
|
{0xD49A'784B'CD1B'8AFEULL, 0x0000'0000'0000'4000ULL}, // NAMED_VECTOR_X87_LOG2_10
|
|
{0xB8AA'3B29'5C17'F0BCULL, 0x0000'0000'0000'3FFFULL}, // NAMED_VECTOR_X87_LOG2_E
|
|
{0xC90F'DAA2'2168'C235ULL, 0x0000'0000'0000'4000ULL}, // NAMED_VECTOR_X87_PI
|
|
{0x9A20'9A84'FBCF'F799ULL, 0x0000'0000'0000'3FFDULL}, // NAMED_VECTOR_X87_LOG10_2
|
|
{0xB172'17F7'D1CF'79ACULL, 0x0000'0000'0000'3FFEULL}, // NAMED_VECTOR_X87_LOG_2
|
|
{0x4F00'0000'4F00'0000ULL, 0x4F00'0000'4F00'0000ULL}, // NAMED_VECTOR_CVTMAX_F32_I32
|
|
{0x4F00'0000'4F00'0000ULL, 0x4F00'0000'4F00'0000ULL}, // NAMED_VECTOR_CVTMAX_F32_I32_UPPER
|
|
{0x5F00'0000'5F00'0000ULL, 0x5F00'0000'5F00'0000ULL}, // NAMED_VECTOR_CVTMAX_F32_I64
|
|
{0x41E0'0000'0000'0000ULL, 0x41E0'0000'0000'0000ULL}, // NAMED_VECTOR_CVTMAX_F64_I32
|
|
{0x41E0'0000'0000'0000ULL, 0x41E0'0000'0000'0000ULL}, // NAMED_VECTOR_CVTMAX_F64_I32_UPPER
|
|
{0x43E0'0000'0000'0000ULL, 0x43E0'0000'0000'0000ULL}, // NAMED_VECTOR_CVTMAX_F64_I64
|
|
{0x8000'0000'8000'0000ULL, 0x8000'0000'8000'0000ULL}, // NAMED_VECTOR_CVTMAX_I32
|
|
{0x8000'0000'0000'0000ULL, 0x8000'0000'0000'0000ULL}, // NAMED_VECTOR_CVTMAX_I64
|
|
{0x0000'0000'0000'0000ULL, 0x0000'0000'0000'8000ULL}, // NAMED_VECTOR_F80_SIGN_MASK
|
|
{0x5A82'7999'5A82'7999ULL, 0x5A82'7999'5A82'7999ULL}, // NAMED_VECTOR_SHA1RNDS_K0
|
|
{0x6ED9'EBA1'6ED9'EBA1ULL, 0x6ED9'EBA1'6ED9'EBA1ULL}, // NAMED_VECTOR_SHA1RNDS_K1
|
|
{0x8F1B'BCDC'8F1B'BCDCULL, 0x8F1B'BCDC'8F1B'BCDCULL}, // NAMED_VECTOR_SHA1RNDS_K2
|
|
{0xCA62'C1D6'CA62'C1D6ULL, 0xCA62'C1D6'CA62'C1D6ULL}, // NAMED_VECTOR_SHA1RNDS_K3
|
|
};
|
|
|
|
constexpr static auto PSHUFLW_LUT {[]() consteval {
|
|
struct LUTType {
|
|
uint64_t Val[2];
|
|
};
|
|
// Expectation for this LUT is to simulate PSHUFLW with ARM's TBL (single register) instruction
|
|
// PSHUFLW behaviour:
|
|
// 16-bit words in [63:48], [47:32], [31:16], [15:0] are selected using the 8-bit Index.
|
|
// For 128-bit PSHUFLW, bits [127:64] are identity copied.
|
|
constexpr uint64_t IdentityCopyUpper = 0x0f'0e'0d'0c'0b'0a'09'08;
|
|
std::array<LUTType, 256> TotalLUT {};
|
|
uint64_t WordSelection[4] = {
|
|
0x01'00,
|
|
0x03'02,
|
|
0x05'04,
|
|
0x07'06,
|
|
};
|
|
for (size_t i = 0; i < 256; ++i) {
|
|
auto& LUT = TotalLUT[i];
|
|
const auto Word0 = (i >> 0) & 0b11;
|
|
const auto Word1 = (i >> 2) & 0b11;
|
|
const auto Word2 = (i >> 4) & 0b11;
|
|
const auto Word3 = (i >> 6) & 0b11;
|
|
|
|
LUT.Val[0] = (WordSelection[Word0] << 0) | (WordSelection[Word1] << 16) | (WordSelection[Word2] << 32) | (WordSelection[Word3] << 48);
|
|
|
|
LUT.Val[1] = IdentityCopyUpper;
|
|
}
|
|
return TotalLUT;
|
|
}()};
|
|
|
|
constexpr static auto PSHUFHW_LUT {[]() consteval {
|
|
struct LUTType {
|
|
uint64_t Val[2];
|
|
};
|
|
// Expectation for this LUT is to simulate PSHUFHW with ARM's TBL (single register) instruction
|
|
// PSHUFHW behaviour:
|
|
// 16-bit words in [127:112], [111:96], [95:80], [79:64] are selected using the 8-bit Index.
|
|
// Incoming words come from bits [127:64] of the source.
|
|
// Bits [63:0] are identity copied.
|
|
constexpr uint64_t IdentityCopyLower = 0x07'06'05'04'03'02'01'00;
|
|
std::array<LUTType, 256> TotalLUT {};
|
|
uint64_t WordSelection[4] = {
|
|
0x09'08,
|
|
0x0b'0a,
|
|
0x0d'0c,
|
|
0x0f'0e,
|
|
};
|
|
for (size_t i = 0; i < 256; ++i) {
|
|
auto& LUT = TotalLUT[i];
|
|
const auto Word0 = (i >> 0) & 0b11;
|
|
const auto Word1 = (i >> 2) & 0b11;
|
|
const auto Word2 = (i >> 4) & 0b11;
|
|
const auto Word3 = (i >> 6) & 0b11;
|
|
|
|
LUT.Val[0] = IdentityCopyLower;
|
|
|
|
LUT.Val[1] = (WordSelection[Word0] << 0) | (WordSelection[Word1] << 16) | (WordSelection[Word2] << 32) | (WordSelection[Word3] << 48);
|
|
}
|
|
return TotalLUT;
|
|
}()};
|
|
|
|
constexpr static auto PSHUFD_LUT {[]() consteval {
|
|
struct LUTType {
|
|
uint64_t Val[2];
|
|
};
|
|
// Expectation for this LUT is to simulate PSHUFD with ARM's TBL (single register) instruction
|
|
// PSHUFD behaviour:
|
|
// 32-bit words in [127:96], [95:64], [63:32], [31:0] are selected using the 8-bit Index.
|
|
std::array<LUTType, 256> TotalLUT {};
|
|
uint64_t WordSelection[4] = {
|
|
0x03'02'01'00,
|
|
0x07'06'05'04,
|
|
0x0b'0a'09'08,
|
|
0x0f'0e'0d'0c,
|
|
};
|
|
for (size_t i = 0; i < 256; ++i) {
|
|
auto& LUT = TotalLUT[i];
|
|
const auto Word0 = (i >> 0) & 0b11;
|
|
const auto Word1 = (i >> 2) & 0b11;
|
|
const auto Word2 = (i >> 4) & 0b11;
|
|
const auto Word3 = (i >> 6) & 0b11;
|
|
|
|
LUT.Val[0] = (WordSelection[Word0] << 0) | (WordSelection[Word1] << 32);
|
|
|
|
LUT.Val[1] = (WordSelection[Word2] << 0) | (WordSelection[Word3] << 32);
|
|
}
|
|
return TotalLUT;
|
|
}()};
|
|
|
|
constexpr static auto SHUFPS_LUT {[]() consteval {
|
|
struct LUTType {
|
|
uint64_t Val[2];
|
|
};
|
|
// 32-bit words in [127:96], [95:64], [63:32], [31:0] are selected using the 8-bit Index.
|
|
// Expectation for this LUT is to simulate SHUFPS with ARM's TBL (two register) instruction.
|
|
// SHUFPS behaviour:
|
|
// Two 32-bits words from each source are selected from each source in the lower and upper halves of the 128-bit destination.
|
|
// Dest[31:0] = Src1[<Word0>]
|
|
// Dest[63:32] = Src1[<Word1>]
|
|
// Dest[95:64] = Src2[<Word2>]
|
|
// Dest[127:96] = Src2[<Word3>]
|
|
|
|
std::array<LUTType, 256> TotalLUT {};
|
|
const uint64_t WordSelectionSrc1[4] = {
|
|
0x03'02'01'00,
|
|
0x07'06'05'04,
|
|
0x0b'0a'09'08,
|
|
0x0f'0e'0d'0c,
|
|
};
|
|
|
|
// Src2 needs to offset each byte index by 16-bytes to pull from the second source.
|
|
const uint64_t WordSelectionSrc2[4] = {
|
|
0x03'02'01'00 + (0x10101010),
|
|
0x07'06'05'04 + (0x10101010),
|
|
0x0b'0a'09'08 + (0x10101010),
|
|
0x0f'0e'0d'0c + (0x10101010),
|
|
};
|
|
|
|
for (size_t i = 0; i < 256; ++i) {
|
|
auto& LUT = TotalLUT[i];
|
|
const auto Word0 = (i >> 0) & 0b11;
|
|
const auto Word1 = (i >> 2) & 0b11;
|
|
const auto Word2 = (i >> 4) & 0b11;
|
|
const auto Word3 = (i >> 6) & 0b11;
|
|
|
|
LUT.Val[0] = (WordSelectionSrc1[Word0] << 0) | (WordSelectionSrc1[Word1] << 32);
|
|
|
|
LUT.Val[1] = (WordSelectionSrc2[Word2] << 0) | (WordSelectionSrc2[Word3] << 32);
|
|
}
|
|
return TotalLUT;
|
|
}()};
|
|
|
|
constexpr static auto DPPS_MASK {[]() consteval {
|
|
struct LUTType {
|
|
uint32_t Val[4];
|
|
};
|
|
|
|
std::array<LUTType, 16> TotalLUT {};
|
|
for (size_t i = 0; i < TotalLUT.size(); ++i) {
|
|
auto& LUT = TotalLUT[i];
|
|
constexpr auto GetLUT = [](size_t i, size_t Index) {
|
|
if (i & (1U << Index)) {
|
|
return -1U;
|
|
}
|
|
return 0U;
|
|
};
|
|
|
|
LUT.Val[0] = GetLUT(i, 0);
|
|
LUT.Val[1] = GetLUT(i, 1);
|
|
LUT.Val[2] = GetLUT(i, 2);
|
|
LUT.Val[3] = GetLUT(i, 3);
|
|
}
|
|
return TotalLUT;
|
|
}()};
|
|
|
|
constexpr static auto DPPD_MASK {[]() consteval {
|
|
struct LUTType {
|
|
uint64_t Val[2];
|
|
};
|
|
|
|
std::array<LUTType, 4> TotalLUT {};
|
|
for (size_t i = 0; i < TotalLUT.size(); ++i) {
|
|
auto& LUT = TotalLUT[i];
|
|
constexpr auto GetLUT = [](size_t i, size_t Index) {
|
|
if (i & (1U << Index)) {
|
|
return -1ULL;
|
|
}
|
|
return 0ULL;
|
|
};
|
|
|
|
LUT.Val[0] = GetLUT(i, 0);
|
|
LUT.Val[1] = GetLUT(i, 1);
|
|
}
|
|
return TotalLUT;
|
|
}()};
|
|
|
|
constexpr static auto PBLENDW_LUT {[]() consteval {
|
|
struct LUTType {
|
|
uint16_t Val[8];
|
|
};
|
|
// 16-bit words in [127:112], [111:96], [95:80], [79:64], [63:48], [47:32], [31:16], [15:0] are selected using 8-bit swizzle.
|
|
// Expectation for this LUT is to simulate PBLENDW with ARM's TBX (one register) instruction.
|
|
// PBLENDW behaviour:
|
|
// 16-bit words from the source is moved in to the destination based on the bit in the swizzle.
|
|
// Dest[15:0] = Swizzle[0] ? Src[15:0] : Dest[15:0]
|
|
// Dest[31:16] = Swizzle[1] ? Src[31:16] : Dest[31:16]
|
|
// Dest[47:32] = Swizzle[2] ? Src[47:32] : Dest[47:32]
|
|
// Dest[63:48] = Swizzle[3] ? Src[63:48] : Dest[63:48]
|
|
// Dest[79:64] = Swizzle[4] ? Src[79:64] : Dest[79:64]
|
|
// Dest[95:80] = Swizzle[5] ? Src[95:80] : Dest[95:80]
|
|
// Dest[111:96] = Swizzle[6] ? Src[111:96] : Dest[111:96]
|
|
// Dest[127:112] = Swizzle[7] ? Src[127:112] : Dest[127:112]
|
|
|
|
std::array<LUTType, 256> TotalLUT {};
|
|
const uint16_t WordSelectionSrc[8] = {
|
|
0x01'00, 0x03'02, 0x05'04, 0x07'06, 0x09'08, 0x0B'0A, 0x0D'0C, 0x0F'0E,
|
|
};
|
|
|
|
constexpr uint16_t OriginalDest = 0xFF'FF;
|
|
|
|
for (size_t i = 0; i < 256; ++i) {
|
|
auto& LUT = TotalLUT[i];
|
|
for (size_t j = 0; j < 8; ++j) {
|
|
LUT.Val[j] = ((i >> j) & 1) ? WordSelectionSrc[j] : OriginalDest;
|
|
}
|
|
}
|
|
return TotalLUT;
|
|
}()};
|
|
|
|
CPUBackend::CPUBackend(SharedCodeBufferManager& SharedCodeBuffers, FEXCore::Core::InternalThreadState* ThreadState)
|
|
: ThreadState(ThreadState)
|
|
, SharedCodeBuffers(SharedCodeBuffers) {
|
|
|
|
auto& Ptrs = ThreadState->CurrentFrame->Pointers;
|
|
|
|
// Initialize named vector constants.
|
|
for (size_t i = 0; i < FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_CONST_POOL_MAX; ++i) {
|
|
Ptrs.NamedVectorConstantPointers[i] = reinterpret_cast<uint64_t>(NamedVectorConstants[i]);
|
|
}
|
|
|
|
// Copy named vector constants.
|
|
memcpy(Ptrs.NamedVectorConstants, NamedVectorConstants, sizeof(NamedVectorConstants));
|
|
|
|
// Initialize Indexed named vector constants.
|
|
Ptrs.IndexedNamedVectorConstantPointers[FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_PSHUFLW] =
|
|
reinterpret_cast<uint64_t>(PSHUFLW_LUT.data());
|
|
Ptrs.IndexedNamedVectorConstantPointers[FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_PSHUFHW] =
|
|
reinterpret_cast<uint64_t>(PSHUFHW_LUT.data());
|
|
Ptrs.IndexedNamedVectorConstantPointers[FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_PSHUFD] =
|
|
reinterpret_cast<uint64_t>(PSHUFD_LUT.data());
|
|
Ptrs.IndexedNamedVectorConstantPointers[FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_SHUFPS] =
|
|
reinterpret_cast<uint64_t>(SHUFPS_LUT.data());
|
|
Ptrs.IndexedNamedVectorConstantPointers[FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_DPPS_MASK] =
|
|
reinterpret_cast<uint64_t>(DPPS_MASK.data());
|
|
Ptrs.IndexedNamedVectorConstantPointers[FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_DPPD_MASK] =
|
|
reinterpret_cast<uint64_t>(DPPD_MASK.data());
|
|
Ptrs.IndexedNamedVectorConstantPointers[FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_PBLENDW] =
|
|
reinterpret_cast<uint64_t>(PBLENDW_LUT.data());
|
|
|
|
#ifndef FEX_DISABLE_TELEMETRY
|
|
// Fill in telemetry values
|
|
for (size_t i = 0; i < FEXCore::Telemetry::TYPE_LAST; ++i) {
|
|
auto& Telem = FEXCore::Telemetry::GetTelemetryValue(static_cast<FEXCore::Telemetry::TelemetryType>(i));
|
|
Ptrs.TelemetryValueAddresses[i] = reinterpret_cast<uint64_t>(&Telem);
|
|
}
|
|
#endif
|
|
}
|
|
|
|
CPUBackend::~CPUBackend() = default;
|
|
|
|
auto CPUBackend::AcquireNewSharedCodeBuffer() -> CodeBuffer* {
|
|
auto PrevCodeBuffer = CurrentCodeBuffer;
|
|
|
|
// Resize the code buffer and reallocate our code size
|
|
CurrentCodeBuffer = SharedCodeBuffers.StartLargerCodeBuffer();
|
|
|
|
RegisterForSignalHandler(std::move(PrevCodeBuffer));
|
|
return CurrentCodeBuffer.get();
|
|
}
|
|
|
|
void CPUBackend::RegisterForSignalHandler(fextl::shared_ptr<CodeBuffer> CodeBuffer) {
|
|
if (ThreadState->CurrentFrame->SignalHandlerRefCounter != 0) {
|
|
// We have signal handlers that have generated code
|
|
// This means that we can not safely clear the code at this point in time
|
|
// Keep a reference to the old code buffer to delay deallocation
|
|
SignalHandlerCodeBuffers.push_back(std::move(CodeBuffer));
|
|
} else {
|
|
SignalHandlerCodeBuffers.clear();
|
|
}
|
|
}
|
|
|
|
fextl::shared_ptr<CodeBuffer> CPUBackend::CheckCodeBufferUpdate() {
|
|
auto NewCodeBuffer = SharedCodeBuffers.GetLatest();
|
|
if (CurrentCodeBuffer != NewCodeBuffer) {
|
|
RegisterForSignalHandler(CurrentCodeBuffer);
|
|
return std::exchange(CurrentCodeBuffer, NewCodeBuffer);
|
|
}
|
|
return nullptr;
|
|
}
|
|
|
|
bool CPUBackend::IsAddressInCodeBuffer(uintptr_t Address) const {
|
|
const auto CheckCodeBuffer = [](const CodeBuffer& Buffer, uintptr_t Address) {
|
|
const auto BufferPtr = reinterpret_cast<uintptr_t>(Buffer.GetBufferBase());
|
|
const uintptr_t LastPageAddr = BufferPtr + Buffer.UsableSize();
|
|
return (Address >= BufferPtr && Address < LastPageAddr);
|
|
};
|
|
|
|
if (CheckCodeBuffer(*CurrentCodeBuffer, Address)) {
|
|
return true;
|
|
}
|
|
for (const auto& Buffer : SignalHandlerCodeBuffers) {
|
|
if (CheckCodeBuffer(*Buffer, Address)) {
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
} // namespace CPU
|
|
} // namespace FEXCore
|