// SPDX-License-Identifier: MIT #include "FEXCore/IR/IR.h" #include "FEXCore/Utils/AllocatorHooks.h" #include "Interface/Context/Context.h" #include "Interface/Core/Dispatcher/Dispatcher.h" #include namespace FEXCore { namespace CPU { constexpr static uint64_t NamedVectorConstants[FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_CONST_POOL_MAX][2] = { {0x0003'0002'0001'0000ULL, 0x0007'0006'0005'0004ULL}, // NAMED_VECTOR_INCREMENTAL_U16_INDEX {0x000B'000A'0009'0008ULL, 0x000F'000E'000D'000CULL}, // NAMED_VECTOR_INCREMENTAL_U16_INDEX_UPPER {0x0000'0000'8000'0000ULL, 0x0000'0000'8000'0000ULL}, // NAMED_VECTOR_PADDSUBPS_INVERT {0x0000'0000'8000'0000ULL, 0x0000'0000'8000'0000ULL}, // NAMED_VECTOR_PADDSUBPS_INVERT_UPPER {0x8000'0000'0000'0000ULL, 0x0000'0000'0000'0000ULL}, // NAMED_VECTOR_PADDSUBPD_INVERT {0x8000'0000'0000'0000ULL, 0x0000'0000'0000'0000ULL}, // NAMED_VECTOR_PADDSUBPD_INVERT_UPPER {0x0000'0001'0000'0000ULL, 0x0000'0003'0000'0002ULL}, // NAMED_VECTOR_MOVMSKPS_SHIFT {0x040B'0E01'0B0E'0104ULL, 0x0C03'0609'0306'090CULL}, // NAMED_VECTOR_AESKEYGENASSIST_SWIZZLE }; constexpr static auto PSHUFLW_LUT { []() consteval { struct LUTType { uint64_t Val[2]; }; // Expectation for this LUT is to simulate PSHUFLW with ARM's TBL (single register) instruction // PSHUFLW behaviour: // 16-bit words in [63:48], [47:32], [31:16], [15:0] are selected using the 8-bit Index. // For 128-bit PSHUFLW, bits [127:64] are identity copied. constexpr uint64_t IdentityCopyUpper = 0x0f'0e'0d'0c'0b'0a'09'08; std::array TotalLUT{}; uint64_t WordSelection[4] = { 0x01'00, 0x03'02, 0x05'04, 0x07'06, }; for (size_t i = 0; i < 256; ++i) { auto &LUT = TotalLUT[i]; const auto Word0 = (i >> 0) & 0b11; const auto Word1 = (i >> 2) & 0b11; const auto Word2 = (i >> 4) & 0b11; const auto Word3 = (i >> 6) & 0b11; LUT.Val[0] = (WordSelection[Word0] << 0) | (WordSelection[Word1] << 16) | (WordSelection[Word2] << 32) | (WordSelection[Word3] << 48); LUT.Val[1] = IdentityCopyUpper; } return TotalLUT; }() }; constexpr static auto PSHUFHW_LUT { []() consteval { struct LUTType { uint64_t Val[2]; }; // Expectation for this LUT is to simulate PSHUFHW with ARM's TBL (single register) instruction // PSHUFHW behaviour: // 16-bit words in [127:112], [111:96], [95:80], [79:64] are selected using the 8-bit Index. // Incoming words come from bits [127:64] of the source. // Bits [63:0] are identity copied. constexpr uint64_t IdentityCopyLower = 0x07'06'05'04'03'02'01'00; std::array TotalLUT{}; uint64_t WordSelection[4] = { 0x09'08, 0x0b'0a, 0x0d'0c, 0x0f'0e, }; for (size_t i = 0; i < 256; ++i) { auto &LUT = TotalLUT[i]; const auto Word0 = (i >> 0) & 0b11; const auto Word1 = (i >> 2) & 0b11; const auto Word2 = (i >> 4) & 0b11; const auto Word3 = (i >> 6) & 0b11; LUT.Val[0] = IdentityCopyLower; LUT.Val[1] = (WordSelection[Word0] << 0) | (WordSelection[Word1] << 16) | (WordSelection[Word2] << 32) | (WordSelection[Word3] << 48); } return TotalLUT; }() }; constexpr static auto PSHUFD_LUT { []() consteval { struct LUTType { uint64_t Val[2]; }; // Expectation for this LUT is to simulate PSHUFD with ARM's TBL (single register) instruction // PSHUFD behaviour: // 32-bit words in [127:96], [95:64], [63:32], [31:0] are selected using the 8-bit Index. std::array TotalLUT{}; uint64_t WordSelection[4] = { 0x03'02'01'00, 0x07'06'05'04, 0x0b'0a'09'08, 0x0f'0e'0d'0c, }; for (size_t i = 0; i < 256; ++i) { auto &LUT = TotalLUT[i]; const auto Word0 = (i >> 0) & 0b11; const auto Word1 = (i >> 2) & 0b11; const auto Word2 = (i >> 4) & 0b11; const auto Word3 = (i >> 6) & 0b11; LUT.Val[0] = (WordSelection[Word0] << 0) | (WordSelection[Word1] << 32); LUT.Val[1] = (WordSelection[Word2] << 0) | (WordSelection[Word3] << 32); } return TotalLUT; }() }; constexpr static auto SHUFPS_LUT { []() consteval { struct LUTType { uint64_t Val[2]; }; // 32-bit words in [127:96], [95:64], [63:32], [31:0] are selected using the 8-bit Index. // Expectation for this LUT is to simulate SHUFPS with ARM's TBL (two register) instruction. // SHUFPS behaviour: // Two 32-bits words from each source are selected from each source in the lower and upper halves of the 128-bit destination. // Dest[31:0] = Src1[] // Dest[63:32] = Src1[] // Dest[95:64] = Src2[] // Dest[127:96] = Src2[] std::array TotalLUT{}; const uint64_t WordSelectionSrc1[4] = { 0x03'02'01'00, 0x07'06'05'04, 0x0b'0a'09'08, 0x0f'0e'0d'0c, }; // Src2 needs to offset each byte index by 16-bytes to pull from the second source. const uint64_t WordSelectionSrc2[4] = { 0x03'02'01'00 + (0x10101010), 0x07'06'05'04 + (0x10101010), 0x0b'0a'09'08 + (0x10101010), 0x0f'0e'0d'0c + (0x10101010), }; for (size_t i = 0; i < 256; ++i) { auto &LUT = TotalLUT[i]; const auto Word0 = (i >> 0) & 0b11; const auto Word1 = (i >> 2) & 0b11; const auto Word2 = (i >> 4) & 0b11; const auto Word3 = (i >> 6) & 0b11; LUT.Val[0] = (WordSelectionSrc1[Word0] << 0) | (WordSelectionSrc1[Word1] << 32); LUT.Val[1] = (WordSelectionSrc2[Word2] << 0) | (WordSelectionSrc2[Word3] << 32); } return TotalLUT; }() }; constexpr static auto DPPS_MASK { []() consteval { struct LUTType { uint32_t Val[4]; }; std::array TotalLUT{}; for (size_t i = 0; i < TotalLUT.size(); ++i) { auto &LUT = TotalLUT[i]; constexpr auto GetLUT = [](size_t i, size_t Index) { if (i & (1U << Index)) { return -1U; } return 0U; }; LUT.Val[0] = GetLUT(i, 0); LUT.Val[1] = GetLUT(i, 1); LUT.Val[2] = GetLUT(i, 2); LUT.Val[3] = GetLUT(i, 3); } return TotalLUT; }() }; constexpr static auto DPPD_MASK { []() consteval { struct LUTType { uint64_t Val[2]; }; std::array TotalLUT{}; for (size_t i = 0; i < TotalLUT.size(); ++i) { auto &LUT = TotalLUT[i]; constexpr auto GetLUT = [](size_t i, size_t Index) { if (i & (1U << Index)) { return -1ULL; } return 0ULL; }; LUT.Val[0] = GetLUT(i, 0); LUT.Val[1] = GetLUT(i, 1); } return TotalLUT; }() }; CPUBackend::CPUBackend(FEXCore::Core::InternalThreadState *ThreadState, size_t InitialCodeSize, size_t MaxCodeSize) : ThreadState(ThreadState), InitialCodeSize(InitialCodeSize), MaxCodeSize(MaxCodeSize) { auto &Common = ThreadState->CurrentFrame->Pointers.Common; // Initialize named vector constants. for (size_t i = 0; i < FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_CONST_POOL_MAX; ++i) { Common.NamedVectorConstantPointers[i] = reinterpret_cast(NamedVectorConstants[i]); } // Copy named vector constants. memcpy(Common.NamedVectorConstants, NamedVectorConstants, sizeof(NamedVectorConstants)); // Initialize Indexed named vector constants. Common.IndexedNamedVectorConstantPointers[FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_PSHUFLW] = reinterpret_cast(PSHUFLW_LUT.data()); Common.IndexedNamedVectorConstantPointers[FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_PSHUFHW] = reinterpret_cast(PSHUFHW_LUT.data()); Common.IndexedNamedVectorConstantPointers[FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_PSHUFD] = reinterpret_cast(PSHUFD_LUT.data()); Common.IndexedNamedVectorConstantPointers[FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_SHUFPS] = reinterpret_cast(SHUFPS_LUT.data()); Common.IndexedNamedVectorConstantPointers[FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_DPPS_MASK] = reinterpret_cast(DPPS_MASK.data()); Common.IndexedNamedVectorConstantPointers[FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_DPPD_MASK] = reinterpret_cast(DPPD_MASK.data()); #ifndef FEX_DISABLE_TELEMETRY // Fill in telemetry values for (size_t i = 0; i < FEXCore::Telemetry::TYPE_LAST; ++i) { auto &Telem = FEXCore::Telemetry::GetTelemetryValue(static_cast(i)); Common.TelemetryValueAddresses[i] = reinterpret_cast(Telem.GetAddr()); } #endif } CPUBackend::~CPUBackend() { for (auto CodeBuffer : CodeBuffers) { FreeCodeBuffer(CodeBuffer); } CodeBuffers.clear(); } auto CPUBackend::GetEmptyCodeBuffer() -> CodeBuffer * { if (ThreadState->CurrentFrame->SignalHandlerRefCounter == 0) { if (CodeBuffers.empty()) { auto NewCodeBuffer = AllocateNewCodeBuffer(InitialCodeSize); EmplaceNewCodeBuffer(NewCodeBuffer); } else { if (CodeBuffers.size() > 1) { // If we have more than one code buffer we are tracking then walk them and delete // This is a cleanup step for (size_t i = 1; i < CodeBuffers.size(); i++) { FreeCodeBuffer(CodeBuffers[i]); } CodeBuffers.resize(1); } // Set the current code buffer to the initial CurrentCodeBuffer = &CodeBuffers[0]; if (CurrentCodeBuffer->Size != MaxCodeSize) { FreeCodeBuffer(*CurrentCodeBuffer); // Resize the code buffer and reallocate our code size CurrentCodeBuffer->Size *= 1.5; CurrentCodeBuffer->Size = std::min(CurrentCodeBuffer->Size, MaxCodeSize); *CurrentCodeBuffer = AllocateNewCodeBuffer(CurrentCodeBuffer->Size); } } } else { // We have signal handlers that have generated code // This means that we can not safely clear the code at this point in time // Allocate some new code buffers that we can switch over to instead auto NewCodeBuffer = AllocateNewCodeBuffer(InitialCodeSize); EmplaceNewCodeBuffer(NewCodeBuffer); } return CurrentCodeBuffer; } auto CPUBackend::AllocateNewCodeBuffer(size_t Size) -> CodeBuffer { CodeBuffer Buffer; Buffer.Size = Size; Buffer.Ptr = static_cast( FEXCore::Allocator::VirtualAlloc(Buffer.Size, true)); LOGMAN_THROW_AA_FMT(!!Buffer.Ptr, "Couldn't allocate code buffer"); if (static_cast(ThreadState->CTX)->Config.GlobalJITNaming()) { static_cast(ThreadState->CTX)->Symbols.RegisterJITSpace(Buffer.Ptr, Buffer.Size); } return Buffer; } void CPUBackend::FreeCodeBuffer(CodeBuffer Buffer) { FEXCore::Allocator::VirtualFree(Buffer.Ptr, Buffer.Size); } bool CPUBackend::IsAddressInCodeBuffer(uintptr_t Address) const { for (auto &Buffer: CodeBuffers) { auto start = (uintptr_t)Buffer.Ptr; auto end = start + Buffer.Size; if (Address >= start && Address < end) { return true; } } return false; } } }