Compare commits

..
1 Commits
Author SHA1 Message Date
Ryan Houdek e9e88968d7 Emitter: Remove unused header 2023-01-11 10:52:57 -08:00
284 changed files with 7176 additions and 16599 deletions

No files matched your search

+6 -16
View File
@@ -72,11 +72,6 @@ jobs:
# Execute the build. You can specify a specific target with "--target <NAME>"
run: cmake --build . --config $BUILD_TYPE
- name: Install
working-directory: ${{runner.workspace}}/build
shell: bash
run: cmake --build . --config $BUILD_TYPE --target install
- name: ASM Tests
working-directory: ${{runner.workspace}}/build
shell: bash
@@ -171,17 +166,6 @@ jobs:
working-directory: ${{runner.workspace}}/build
run: mv ${{runner.workspace}}/build/Testing/Temporary/LastTest.log ${{runner.workspace}}/build/Testing/Temporary/LastTest_APITests.log || true
- name: ARMEmitter tests
working-directory: ${{runner.workspace}}/build
shell: bash
run: cmake --build . --config $BUILD_TYPE --target emitter_tests
- name: ARMEmitter Test Results move
if: ${{ always() }}
shell: bash
working-directory: ${{runner.workspace}}/build
run: mv ${{runner.workspace}}/build/Testing/Temporary/LastTest.log ${{runner.workspace}}/build/Testing/Temporary/LastTest_ARMEmitterTests.log || true
- name: FEXLinuxTests
working-directory: ${{runner.workspace}}/build
shell: bash
@@ -204,6 +188,12 @@ jobs:
working-directory: ${{runner.workspace}}/build
run: mv ${{runner.workspace}}/build/Testing/Temporary/LastTest.log ${{runner.workspace}}/build/Testing/Temporary/LastTest_ThunkgenTests.log || true
- name: Install
if: matrix.arch[1] == 'x64'
working-directory: ${{runner.workspace}}/build
shell: bash
run: cmake --build . --config $BUILD_TYPE --target install
- name: Test GL No-Thunks
if: matrix.arch[1] == 'x64'
working-directory: ${{runner.workspace}}/build
-6
View File
@@ -30,7 +30,6 @@ option(ENABLE_CCACHE "Enables ccache for compile caching" TRUE)
option(ENABLE_TERMUX_BUILD "Forces building for Termux on a non-Termux build machine" FALSE)
option(ENABLE_VIXL_SIMULATOR "Forces the FEX JIT to use the VIXL simulator" FALSE)
option(ENABLE_VIXL_DISASSEMBLER "Enables debug disassembler output with VIXL" FALSE)
option(COMPILE_VIXL_DISASSEMBLER "Compiles the vixl disassembler in to vixl" FALSE)
option(ENABLE_FEXCORE_PROFILER "Enables use of the FEXCore timeline profiling capabilities" FALSE)
set (FEXCORE_PROFILER_BACKEND "gpuvis" CACHE STRING "Set which backend you want to use for the FEXCore profiler")
@@ -198,11 +197,6 @@ set (CMAKE_LINKER_FLAGS_RELEASE "${CMAKE_LINKER_FLAGS_RELEASE} -fomit-frame-poin
include_directories(External/robin-map/include/)
if (BUILD_TESTS)
# Enable vixl disassembler if tests are enabled.
set(COMPILE_VIXL_DISASSEMBLER TRUE)
endif()
add_subdirectory(External/vixl/)
include_directories(External/vixl/src/)
+5
View File
@@ -0,0 +1,5 @@
{
"Config": {
"x86dec_SynchronizeRIPOnAllBlocks": "1"
}
}
+5
View File
@@ -0,0 +1,5 @@
{
"Config": {
"x86dec_SynchronizeRIPOnAllBlocks": "1"
}
}
-5
View File
@@ -1,5 +0,0 @@
{
"Config": {
"HideHypervisorBit": "1"
}
}
+5
View File
@@ -0,0 +1,5 @@
{
"Config": {
"x86dec_SynchronizeRIPOnAllBlocks": "1"
}
}
+5
View File
@@ -0,0 +1,5 @@
{
"Config": {
"x86dec_SynchronizeRIPOnAllBlocks": "1"
}
}
+11 -38
View File
@@ -281,7 +281,9 @@ def print_ir_structs(defines):
output_file.write("\tvoid* Data[0];\n")
output_file.write("\tIROps Op;\n\n")
output_file.write("\tuint8_t Size;\n")
output_file.write("\tuint8_t ElementSize;\n")
output_file.write("\tuint8_t NumArgs;\n")
output_file.write("\tuint8_t ElementSize : 7;\n")
output_file.write("\tbool HasDest : 1;\n")
output_file.write("\ttemplate<typename T>\n")
output_file.write("\tT const* C() const { return reinterpret_cast<T const*>(Data); }\n")
@@ -356,10 +358,8 @@ def print_ir_sizes():
output_file.write("[[nodiscard, gnu::const, gnu::visibility(\"default\")]] std::string_view const& GetName(IROps Op);\n")
output_file.write("[[nodiscard, gnu::const, gnu::visibility(\"default\")]] uint8_t GetArgs(IROps Op);\n")
output_file.write("[[nodiscard, gnu::const, gnu::visibility(\"default\")]] uint8_t GetRAArgs(IROps Op);\n")
output_file.write("[[nodiscard, gnu::const, gnu::visibility(\"default\")]] FEXCore::IR::RegisterClassType GetRegClass(IROps Op);\n\n")
output_file.write("[[nodiscard, gnu::const, gnu::visibility(\"default\")]] bool HasSideEffects(IROps Op);\n")
output_file.write("[[nodiscard, gnu::const, gnu::visibility(\"default\")]] bool GetHasDest(IROps Op);\n")
output_file.write("#undef IROP_SIZES\n")
output_file.write("#endif\n\n")
@@ -417,7 +417,7 @@ def print_ir_getname():
def print_ir_getraargs():
output_file.write("#ifdef IROP_GETRAARGS_IMPL\n")
output_file.write("constexpr std::array<uint8_t, OP_LAST + 1> IRRAArgs = {\n")
output_file.write("constexpr std::array<uint8_t, OP_LAST + 1> IRArgs = {\n")
for op in IROps:
SSAArgs = op.SSAArgNum
@@ -430,18 +430,6 @@ def print_ir_getraargs():
output_file.write("};\n\n")
output_file.write("constexpr std::array<uint8_t, OP_LAST + 1> IRArgs = {\n")
for op in IROps:
SSAArgs = op.SSAArgNum
output_file.write("\t{},\n".format(SSAArgs))
output_file.write("};\n\n")
output_file.write("uint8_t GetRAArgs(IROps Op) {\n")
output_file.write(" return IRRAArgs[Op];\n")
output_file.write("}\n")
output_file.write("uint8_t GetArgs(IROps Op) {\n")
output_file.write(" return IRArgs[Op];\n")
output_file.write("}\n")
@@ -465,25 +453,6 @@ def print_ir_hassideeffects():
output_file.write("#undef IROP_HASSIDEEFFECTS_IMPL\n")
output_file.write("#endif\n\n")
def print_ir_gethasdest():
output_file.write("#ifdef IROP_GETHASDEST_IMPL\n")
output_file.write("constexpr std::array<bool, OP_LAST + 1> IRDest = {\n")
for op in IROps:
if op.HasDest:
output_file.write("\ttrue,\n")
else:
output_file.write("\tfalse,\n")
output_file.write("};\n\n")
output_file.write("bool GetHasDest(IROps Op) {\n")
output_file.write(" return IRDest[Op];\n")
output_file.write("}\n")
output_file.write("#undef IROP_GETHASDEST_IMPL\n")
output_file.write("#endif\n\n")
# Print out IR argument printing
def print_ir_arg_printer():
output_file.write("#ifdef IROP_ARGPRINTER_HELPER\n")
@@ -578,13 +547,13 @@ def print_ir_allocator_helpers():
output_file.write("\tuint8_t GetOpElements(const OrderedNode *Op) const {\n")
output_file.write("\t\tauto HeaderOp = Op->Header.Value.GetNode(DualListData.DataBegin());\n")
output_file.write("\t\tLOGMAN_THROW_A_FMT(OpHasDest(Op), \"Op {} has no dest\\n\", GetName(HeaderOp->Op));\n")
output_file.write("\t\tLOGMAN_THROW_A_FMT(HeaderOp->HasDest, \"Op {} has no dest\\n\", GetName(HeaderOp->Op));\n")
output_file.write("\t\treturn HeaderOp->Size / HeaderOp->ElementSize;\n")
output_file.write("\t}\n\n")
output_file.write("\tbool OpHasDest(const OrderedNode *Op) const {\n")
output_file.write("\t\tauto HeaderOp = Op->Header.Value.GetNode(DualListData.DataBegin());\n")
output_file.write("\t\treturn GetHasDest(HeaderOp->Op);\n")
output_file.write("\t\treturn HeaderOp->HasDest;\n")
output_file.write("\t}\n\n")
output_file.write("\tIROps GetOpType(const OrderedNode *Op) const {\n")
@@ -662,6 +631,8 @@ def print_ir_allocator_helpers():
output_file.write("\t\tOp.first->Header.Size = InferSize;\n")
output_file.write("\t\tOp.first->Header.NumArgs = {};\n".format(op.SSAArgNum))
# Some ops without a destination still need an operating size
# Effectively reusing the destination size value for operation size
if op.DestSize != None:
@@ -672,6 +643,9 @@ def print_ir_allocator_helpers():
else:
output_file.write("\t\tOp.first->Header.ElementSize = Op.first->Header.Size / ({});\n".format(op.NumElements))
if (op.HasDest):
output_file.write("\t\tOp.first->Header.HasDest = true;\n")
# Insert validation here
if op.EmitValidation != None:
output_file.write("\t\t#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED\n")
@@ -759,7 +733,6 @@ print_ir_reg_classes()
print_ir_getname()
print_ir_getraargs()
print_ir_hassideeffects()
print_ir_gethasdest()
print_ir_arg_printer()
print_ir_allocator_helpers()
print_ir_parser_switch_helper()
+17 -39
View File
@@ -1,86 +1,64 @@
#include "Common/JitSymbols.h"
#include <fcntl.h>
#include <string>
#include <unistd.h>
#include <fmt/format.h>
namespace FEXCore {
JITSymbols::JITSymbols() {
JITSymbols::JITSymbols() : fp{nullptr, std::fclose} {
}
JITSymbols::~JITSymbols() {
if (fd != -1) {
close(fd);
}
}
JITSymbols::~JITSymbols() = default;
void JITSymbols::InitFile() {
// We can't use FILE here since we must be robust against forking processes closing our FD from under us.
const auto PerfMap = fmt::format("/tmp/perf-{}.map", getpid());
fd = open(PerfMap.c_str(), O_CREAT | O_TRUNC | O_WRONLY | O_APPEND, 0644);
fp.reset(fopen(PerfMap.c_str(), "wb"));
if (fp) {
// Disable buffering on this file
setvbuf(fp.get(), nullptr, _IONBF, 0);
}
}
void JITSymbols::Register(const void *HostAddr, uint64_t GuestAddr, uint32_t CodeSize) {
if (fd == -1) return;
if (!fp) return;
// Linux perf format is very straightforward
// `<HostPtr> <Size> <Name>\n`
const auto Buffer = fmt::format("{} {:x} JIT_0x{:x}_{}\n", HostAddr, CodeSize, GuestAddr, HostAddr);
auto Result = write(fd, Buffer.c_str(), Buffer.size());
if (Result == -1 && errno == EBADF) {
fd = -1;
}
fmt::print(fp.get(), "{} {:x} JIT_0x{:x}_{}\n", HostAddr, CodeSize, GuestAddr, HostAddr);
}
void JITSymbols::Register(const void *HostAddr, uint32_t CodeSize, std::string_view Name) {
if (fd == -1) return;
if (!fp) return;
// Linux perf format is very straightforward
// `<HostPtr> <Size> <Name>\n`
const auto Buffer = fmt::format("{} {:x} {}_{}\n", HostAddr, CodeSize, Name, HostAddr);
auto Result = write(fd, Buffer.c_str(), Buffer.size());
if (Result == -1 && errno == EBADF) {
fd = -1;
}
fmt::print(fp.get(), "{} {:x} {}_{}\n", HostAddr, CodeSize, Name, HostAddr);
}
void JITSymbols::Register(const void *HostAddr, uint32_t CodeSize, std::string_view Name, uintptr_t Offset) {
if (fd == -1) return;
if (!fp) return;
// Linux perf format is very straightforward
// `<HostPtr> <Size> <Name>\n`
const auto Buffer = fmt::format("{} {:x} {}+0x{:x} ({})\n", HostAddr, CodeSize, Name, Offset, HostAddr);
auto Result = write(fd, Buffer.c_str(), Buffer.size());
if (Result == -1 && errno == EBADF) {
fd = -1;
}
fmt::print(fp.get(), "{} {:x} {}+0x{:x} ({})\n", HostAddr, CodeSize, Name, Offset, HostAddr);
}
void JITSymbols::RegisterNamedRegion(const void *HostAddr, uint32_t CodeSize, std::string_view Name) {
if (fd == -1) return;
if (!fp) return;
// Linux perf format is very straightforward
// `<HostPtr> <Size> <Name>\n`
const auto Buffer = fmt::format("{} {:x} {}\n", HostAddr, CodeSize, Name);
auto Result = write(fd, Buffer.c_str(), Buffer.size());
if (Result == -1 && errno == EBADF) {
fd = -1;
}
fmt::print(fp.get(), "{} {:x} {}\n", HostAddr, CodeSize, Name);
}
void JITSymbols::RegisterJITSpace(const void *HostAddr, uint32_t CodeSize) {
if (fd == -1) return;
if (!fp) return;
// Linux perf format is very straightforward
// `<HostPtr> <Size> <Name>\n`
const auto Buffer = fmt::format("{} {:x} FEXJIT\n", HostAddr, CodeSize);
auto Result = write(fd, Buffer.c_str(), Buffer.size());
if (Result == -1 && errno == EBADF) {
fd = -1;
}
fmt::print(fp.get(), "{} {:x} FEXJIT\n", HostAddr, CodeSize);
}
} // namespace FEXCore
+3 -1
View File
@@ -19,6 +19,8 @@ public:
void RegisterJITSpace(const void *HostAddr, uint32_t CodeSize);
private:
int fd{-1};
using FILEPtr = std::unique_ptr<FILE, decltype(&std::fclose)>;
FILEPtr fp;
};
}
+1 -1
View File
@@ -30,7 +30,7 @@
#include <tiny-json.h>
namespace FEXCore::Context {
class Context;
struct Context;
}
namespace FEXCore::Config {
+5 -14
View File
@@ -240,17 +240,6 @@
"Also needs x86_64-linux-gnu-objdump in PATH.",
"Can be very slow."
]
},
"InjectLibSegFault": {
"Type": "bool",
"Default": "false",
"Desc": [
"Sets the environment variable LD_PRELOAD=libSegFault.so",
"This allows the user to very easily enable libSegFault without dealing with environment variables",
"Very useful for applications that have launch scripts that set the variable to nothing at launch",
"Set this in an application configuration for injecting in to only specific applications.",
"\tNote: If x86/x86_64 libSegFault.so isn't installed then this option won't work."
]
}
},
"Logging": {
@@ -343,12 +332,14 @@
"Useful for a process that keeps restarting and doesn't work"
]
},
"HideHypervisorBit": {
"x86dec_SynchronizeRIPOnAllBlocks": {
"Type": "bool",
"Default": "false",
"Desc": [
"Hides the hypervisor CPUID bit when set.",
"Should only be used for applications that have issues with this set."
"An application that uses try-catch or longjump extensively needs the ability to do context aware state flushing",
"In the case of FEX's block-linking, it won't always ensure that RIP is synchronized.",
"If an exception occurs and RIP isn't synchronized, then FEX's exception stack restore may not long jump as expected",
"Can be useful for Wine applications that rely on stack unwinding"
]
}
},
+152 -57
View File
@@ -28,108 +28,203 @@ namespace FEXCore::Context {
FEXCore::Paths::ShutdownPaths();
}
FEXCore::Context::Context *FEXCore::Context::Context::CreateNewContext() {
return new FEXCore::Context::ContextImpl{};
FEXCore::Context::Context *CreateNewContext() {
return new FEXCore::Context::Context{};
}
void FEXCore::Context::Context::DestroyContext(FEXCore::Context::Context *CTX) {
CTX->DestroyContext();
bool InitializeContext(FEXCore::Context::Context *CTX) {
return FEXCore::CPU::CreateCPUCore(CTX);
}
void DestroyContext(FEXCore::Context::Context *CTX) {
if (CTX->ParentThread) {
CTX->DestroyThread(CTX->ParentThread);
}
delete CTX;
}
bool FEXCore::Context::ContextImpl::InitializeContext() {
return FEXCore::CPU::CreateCPUCore(this);
FEXCore::Core::InternalThreadState* InitCore(FEXCore::Context::Context *CTX, uint64_t InitialRIP, uint64_t StackPointer) {
return CTX->InitCore(InitialRIP, StackPointer);
}
void FEXCore::Context::ContextImpl::DestroyContext() {
if (ParentThread) {
DestroyThread(ParentThread);
}
void SetExitHandler(FEXCore::Context::Context *CTX, ExitHandler handler) {
CTX->CustomExitHandler = std::move(handler);
}
void FEXCore::Context::ContextImpl::SetExitHandler(ExitHandler handler) {
CustomExitHandler = std::move(handler);
ExitHandler GetExitHandler(const FEXCore::Context::Context *CTX) {
return CTX->CustomExitHandler;
}
ExitHandler FEXCore::Context::ContextImpl::GetExitHandler() const {
return CustomExitHandler;
void Run(FEXCore::Context::Context *CTX) {
CTX->Run();
}
void FEXCore::Context::ContextImpl::Stop() {
Stop(false);
void Step(FEXCore::Context::Context *CTX) {
CTX->Step();
}
void FEXCore::Context::ContextImpl::CompileRIP(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP) {
CompileBlock(Thread->CurrentFrame, GuestRIP);
void CompileRIP(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP) {
Thread->CTX->CompileBlock(Thread->CurrentFrame, GuestRIP);
}
FEXCore::Context::ExitReason FEXCore::Context::ContextImpl::GetExitReason() {
return ParentThread->ExitReason;
FEXCore::Context::ExitReason RunUntilExit(FEXCore::Context::Context *CTX) {
return CTX->RunUntilExit();
}
bool FEXCore::Context::ContextImpl::IsDone() const {
return IsPaused();
int GetProgramStatus(const FEXCore::Context::Context *CTX) {
return CTX->GetProgramStatus();
}
void FEXCore::Context::ContextImpl::GetCPUState(FEXCore::Core::CPUState *State) const {
memcpy(State, ParentThread->CurrentFrame, sizeof(FEXCore::Core::CPUState));
FEXCore::Context::ExitReason GetExitReason(const FEXCore::Context::Context *CTX) {
return CTX->ParentThread->ExitReason;
}
void FEXCore::Context::ContextImpl::SetCPUState(const FEXCore::Core::CPUState *State) {
memcpy(ParentThread->CurrentFrame, State, sizeof(FEXCore::Core::CPUState));
bool IsDone(const FEXCore::Context::Context *CTX) {
return CTX->IsPaused();
}
void FEXCore::Context::ContextImpl::SetCustomCPUBackendFactory(CustomCPUFactoryType Factory) {
CustomCPUFactory = std::move(Factory);
void GetCPUState(const FEXCore::Context::Context *CTX, FEXCore::Core::CPUState *State) {
memcpy(State, CTX->ParentThread->CurrentFrame, sizeof(FEXCore::Core::CPUState));
}
bool FEXCore::Context::ContextImpl::AddVirtualMemoryMapping([[maybe_unused]] uint64_t VirtualAddress, [[maybe_unused]] uint64_t PhysicalAddress, [[maybe_unused]] uint64_t Size) {
void SetCPUState(FEXCore::Context::Context *CTX, const FEXCore::Core::CPUState *State) {
memcpy(CTX->ParentThread->CurrentFrame, State, sizeof(FEXCore::Core::CPUState));
}
void Pause(FEXCore::Context::Context *CTX) {
CTX->Pause();
}
void Stop(FEXCore::Context::Context *CTX) {
CTX->Stop(false);
}
void SetCustomCPUBackendFactory(FEXCore::Context::Context *CTX, CustomCPUFactoryType Factory) {
CTX->CustomCPUFactory = std::move(Factory);
}
bool AddVirtualMemoryMapping([[maybe_unused]] FEXCore::Context::Context *CTX, [[maybe_unused]] uint64_t VirtualAddress, [[maybe_unused]] uint64_t PhysicalAddress, [[maybe_unused]] uint64_t Size) {
return false;
}
HostFeatures FEXCore::Context::ContextImpl::GetHostFeatures() const {
return HostFeatures;
void RegisterExternalSyscallVisitor(FEXCore::Context::Context *CTX, [[maybe_unused]] uint64_t Syscall, [[maybe_unused]] FEXCore::HLE::SyscallVisitor *Visitor) {
}
void FEXCore::Context::ContextImpl::SetSignalDelegator(FEXCore::SignalDelegator *_SignalDelegation) {
SignalDelegation = _SignalDelegation;
HostFeatures GetHostFeatures(const FEXCore::Context::Context *CTX) {
return CTX->HostFeatures;
}
void FEXCore::Context::ContextImpl::SetSyscallHandler(FEXCore::HLE::SyscallHandler *Handler) {
SyscallHandler = Handler;
SourcecodeResolver = Handler->GetSourcecodeResolver();
void HandleCallback(FEXCore::Context::Context *CTX, FEXCore::Core::InternalThreadState *Thread, uint64_t RIP) {
CTX->HandleCallback(Thread, RIP);
}
FEXCore::CPUID::FunctionResults FEXCore::Context::ContextImpl::RunCPUIDFunction(uint32_t Function, uint32_t Leaf) {
return CPUID.RunFunction(Function, Leaf);
void RegisterHostSignalHandler(FEXCore::Context::Context *CTX, int Signal, HostSignalDelegatorFunction Func, bool Required) {
CTX->RegisterHostSignalHandler(Signal, std::move(Func), Required);
}
FEXCore::CPUID::FunctionResults FEXCore::Context::ContextImpl::RunCPUIDFunctionName(uint32_t Function, uint32_t Leaf, uint32_t CPU) {
return CPUID.RunFunctionName(Function, Leaf, CPU);
void RegisterFrontendHostSignalHandler(FEXCore::Context::Context *CTX, int Signal, HostSignalDelegatorFunction Func, bool Required) {
CTX->RegisterFrontendHostSignalHandler(Signal, std::move(Func), Required);
}
void SetVDSOSigReturn(FEXCore::Context::Context *CTX, const VDSOSigReturn &Pointers) {
CTX->SetVDSOSigReturn(Pointers);
FEXCore::Core::InternalThreadState* CreateThread(FEXCore::Context::Context *CTX, FEXCore::Core::CPUState *NewThreadState, uint64_t ParentTID) {
return CTX->CreateThread(NewThreadState, ParentTID);
}
void ExecutionThread(FEXCore::Context::Context *CTX, FEXCore::Core::InternalThreadState *Thread) {
return CTX->ExecutionThread(Thread);
}
void InitializeThread(FEXCore::Context::Context *CTX, FEXCore::Core::InternalThreadState *Thread) {
return CTX->InitializeThread(Thread);
}
void RunThread(FEXCore::Context::Context *CTX, FEXCore::Core::InternalThreadState *Thread) {
CTX->RunThread(Thread);
}
void StopThread(FEXCore::Context::Context *CTX, FEXCore::Core::InternalThreadState *Thread) {
CTX->StopThread(Thread);
}
void DestroyThread(FEXCore::Context::Context *CTX, FEXCore::Core::InternalThreadState *Thread) {
CTX->DestroyThread(Thread);
}
void CleanupAfterFork(FEXCore::Context::Context *CTX, FEXCore::Core::InternalThreadState *Thread) {
CTX->CleanupAfterFork(Thread);
}
void SetSignalDelegator(FEXCore::Context::Context *CTX, FEXCore::SignalDelegator *SignalDelegation) {
CTX->SignalDelegation = SignalDelegation;
}
void SetSyscallHandler(FEXCore::Context::Context *CTX, FEXCore::HLE::SyscallHandler *Handler) {
CTX->SyscallHandler = Handler;
CTX->SourcecodeResolver = Handler->GetSourcecodeResolver();
}
FEXCore::CPUID::FunctionResults RunCPUIDFunction(FEXCore::Context::Context *CTX, uint32_t Function, uint32_t Leaf) {
return CTX->CPUID.RunFunction(Function, Leaf);
}
FEX_DEFAULT_VISIBILITY FEXCore::CPUID::FunctionResults RunCPUIDFunctionName(FEXCore::Context::Context *CTX, uint32_t Function, uint32_t Leaf, uint32_t CPU) {
return CTX->CPUID.RunFunctionName(Function, Leaf, CPU);
}
void SetAOTIRLoader(FEXCore::Context::Context *CTX, std::function<int(const std::string&)> CacheReader) {
CTX->SetAOTIRLoader(CacheReader);
}
void SetAOTIRWriter(FEXCore::Context::Context *CTX, std::function<std::unique_ptr<std::ofstream>(const std::string&)> CacheWriter) {
CTX->SetAOTIRWriter(CacheWriter);
}
void SetAOTIRRenamer(FEXCore::Context::Context *CTX, std::function<void(const std::string&)> CacheRenamer) {
CTX->SetAOTIRRenamer(CacheRenamer);
}
void FinalizeAOTIRCache(FEXCore::Context::Context *CTX) {
CTX->FinalizeAOTIRCache();
}
void WriteFilesWithCode(FEXCore::Context::Context *CTX, std::function<void(const std::string& fileid, const std::string& filename)> Writer) {
CTX->WriteFilesWithCode(Writer);
}
IR::AOTIRCacheEntry *LoadAOTIRCacheEntry(FEXCore::Context::Context *CTX, const std::string &Name) {
return CTX->LoadAOTIRCacheEntry(Name);
}
void UnloadAOTIRCacheEntry(FEXCore::Context::Context *CTX, IR::AOTIRCacheEntry *Entry) {
return CTX->UnloadAOTIRCacheEntry(Entry);
}
CustomIRResult AddCustomIREntrypoint(FEXCore::Context::Context *CTX, uintptr_t Entrypoint, std::function<void(uintptr_t Entrypoint, FEXCore::IR::IREmitter *)> Handler, void *Creator, void *Data) {
return CTX->AddCustomIREntrypoint(Entrypoint, Handler, Creator, Data);
}
void AppendThunkDefinitions(FEXCore::Context::Context *CTX, std::vector<FEXCore::IR::ThunkDefinition> const& Definitions) {
CTX->AppendThunkDefinitions(Definitions);
}
namespace Debug {
//void CompileRIP(FEXCore::Context::Context *CTX, uint64_t RIP) {
// CTX->CompileRIP(CTX->ParentThread, RIP);
//}
//uint64_t GetThreadCount(FEXCore::Context::Context *CTX) {
// return CTX->GetThreadCount();
//}
void CompileRIP(FEXCore::Context::Context *CTX, uint64_t RIP) {
CTX->CompileRIP(CTX->ParentThread, RIP);
}
uint64_t GetThreadCount(FEXCore::Context::Context *CTX) {
return CTX->GetThreadCount();
}
//FEXCore::Core::RuntimeStats *GetRuntimeStatsForThread(FEXCore::Context::Context *CTX, uint64_t Thread) {
// return CTX->GetRuntimeStatsForThread(Thread);
//}
FEXCore::Core::RuntimeStats *GetRuntimeStatsForThread(FEXCore::Context::Context *CTX, uint64_t Thread) {
return CTX->GetRuntimeStatsForThread(Thread);
}
//bool GetDebugDataForRIP(FEXCore::Context::Context *CTX, uint64_t RIP, FEXCore::Core::DebugData *Data) {
// return CTX->GetDebugDataForRIP(RIP, Data);
//}
bool GetDebugDataForRIP(FEXCore::Context::Context *CTX, uint64_t RIP, FEXCore::Core::DebugData *Data) {
return CTX->GetDebugDataForRIP(RIP, Data);
}
//bool FindHostCodeForRIP(FEXCore::Context::Context *CTX, uint64_t RIP, uint8_t **Code) {
// return CTX->FindHostCodeForRIP(RIP, Code);
//}
bool FindHostCodeForRIP(FEXCore::Context::Context *CTX, uint64_t RIP, uint8_t **Code) {
return CTX->FindHostCodeForRIP(RIP, Code);
}
// XXX:
// bool FindIRForRIP(FEXCore::Context::Context *CTX, uint64_t RIP, FEXCore::IR::IntrusiveIRList **ir) {
+96 -139
View File
@@ -70,130 +70,7 @@ namespace FEXCore::Context {
MODE_SINGLESTEP = 1,
};
class ContextImpl final : public FEXCore::Context::Context {
public:
// Context base class implementation.
bool InitializeContext() override;
void DestroyContext() override;
FEXCore::Core::InternalThreadState* InitCore(uint64_t InitialRIP, uint64_t StackPointer) override;
void SetExitHandler(ExitHandler handler) override;
ExitHandler GetExitHandler() const override;
void Pause() override;
void Run() override;
void Stop() override;
void Step() override;
ExitReason RunUntilExit() override;
void CompileRIP(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP) override;
int GetProgramStatus() const override;
ExitReason GetExitReason() override;
bool IsDone() const override;
void GetCPUState(FEXCore::Core::CPUState *State) const override;
void SetCPUState(const FEXCore::Core::CPUState *State) override;
void SetCustomCPUBackendFactory(CustomCPUFactoryType Factory) override;
bool AddVirtualMemoryMapping(uint64_t VirtualAddress, uint64_t PhysicalAddress, uint64_t Size) override;
HostFeatures GetHostFeatures() const override;
void HandleCallback(FEXCore::Core::InternalThreadState *Thread, uint64_t RIP) override;
void RegisterHostSignalHandler(int Signal, HostSignalDelegatorFunction Func, bool Required) override;
[[noreturn]] void HandleSignalHandlerReturn(bool RT) override ;
void RegisterFrontendHostSignalHandler(int Signal, HostSignalDelegatorFunction Func, bool Required) override;
/**
* @brief Used to create FEX thread objects in preparation for creating a true OS thread. Does set a TID or PID.
*
* @param NewThreadState The initial thread state to setup for our state
* @param ParentTID The PID that was the parent thread that created this
*
* @return The InternalThreadState object that tracks all of the emulated thread's state
*
* Usecases:
* OS thread Creation:
* - Thread = CreateThread(NewState, PPID);
* - InitializeThread(Thread);
* OS fork (New thread created with a clone of thread state):
* - clone{2, 3}
* - Thread = CreateThread(CopyOfThreadState, PPID);
* - ExecutionThread(Thread); // Starts executing without creating another host thread
* Thunk callback executing guest code from native host thread
* - Thread = CreateThread(NewState, PPID);
* - InitializeThreadTLSData(Thread);
* - HandleCallback(Thread, RIP);
*/
FEXCore::Core::InternalThreadState* CreateThread(FEXCore::Core::CPUState *NewThreadState, uint64_t ParentTID) override;
// Public for threading
void ExecutionThread(FEXCore::Core::InternalThreadState *Thread) override;
/**
* @brief Initializes the OS thread object and prepares to start executing on that new OS thread
*
* @param Thread The internal FEX thread state object
*
* The OS thread will wait until RunThread is executed
*/
void InitializeThread(FEXCore::Core::InternalThreadState *Thread) override;
/**
* @brief Starts the OS thread object to start executing guest code
*
* @param Thread The internal FEX thread state object
*/
void RunThread(FEXCore::Core::InternalThreadState *Thread) override;
void StopThread(FEXCore::Core::InternalThreadState *Thread) override;
/**
* @brief Destroys this FEX thread object and stops tracking it internally
*
* @param Thread The internal FEX thread state object
*/
void DestroyThread(FEXCore::Core::InternalThreadState *Thread) override;
void CleanupAfterFork(FEXCore::Core::InternalThreadState *Thread) override;
void SetSignalDelegator(FEXCore::SignalDelegator *SignalDelegation) override;
void SetSyscallHandler(FEXCore::HLE::SyscallHandler *Handler) override;
FEXCore::CPUID::FunctionResults RunCPUIDFunction(uint32_t Function, uint32_t Leaf) override;
FEXCore::CPUID::FunctionResults RunCPUIDFunctionName(uint32_t Function, uint32_t Leaf, uint32_t CPU) override;
FEXCore::IR::AOTIRCacheEntry *LoadAOTIRCacheEntry(const std::string& Name) override;
void UnloadAOTIRCacheEntry(FEXCore::IR::AOTIRCacheEntry *Entry) override;
void SetAOTIRLoader(std::function<int(const std::string&)> CacheReader) override {
IRCaptureCache.SetAOTIRLoader(CacheReader);
}
void SetAOTIRWriter(std::function<std::unique_ptr<std::ofstream>(const std::string&)> CacheWriter) override {
IRCaptureCache.SetAOTIRWriter(CacheWriter);
}
void SetAOTIRRenamer(std::function<void(const std::string&)> CacheRenamer) override {
IRCaptureCache.SetAOTIRRenamer(CacheRenamer);
}
void FinalizeAOTIRCache() override {
IRCaptureCache.FinalizeAOTIRCache();
}
void WriteFilesWithCode(std::function<void(const std::string& fileid, const std::string& filename)> Writer) override {
IRCaptureCache.WriteFilesWithCode(Writer);
}
void InvalidateGuestCodeRange(uint64_t Start, uint64_t Length) override;
void InvalidateGuestCodeRange(uint64_t Start, uint64_t Length, std::function<void(uint64_t start, uint64_t Length)> callback) override;
void MarkMemoryShared() override;
void ConfigureAOTGen(FEXCore::Core::InternalThreadState *Thread, std::set<uint64_t> *ExternalBranches, uint64_t SectionMaxAddress) override;
// returns false if a handler was already registered
CustomIRResult AddCustomIREntrypoint(uintptr_t Entrypoint, std::function<void(uintptr_t Entrypoint, FEXCore::IR::IREmitter *)> Handler, void *Creator = nullptr, void *Data = nullptr) override;
void AppendThunkDefinitions(std::vector<FEXCore::IR::ThunkDefinition> const& Definitions) override;
public:
struct Context {
friend class FEXCore::HLE::SyscallHandler;
#ifdef JIT_ARM64
friend class FEXCore::CPU::Arm64JITCore;
@@ -239,13 +116,14 @@ namespace FEXCore::Context {
FEX_CONFIG_OPT(ParanoidTSO, PARANOIDTSO);
FEX_CONFIG_OPT(CacheObjectCodeCompilation, CACHEOBJECTCODECOMPILATION);
FEX_CONFIG_OPT(x87ReducedPrecision, X87REDUCEDPRECISION);
FEX_CONFIG_OPT(x86dec_SynchronizeRIPOnAllBlocks, X86DEC_SYNCHRONIZERIPONALLBLOCKS);
FEX_CONFIG_OPT(EnableAVX, ENABLEAVX);
} Config;
FEXCore::HostFeatures HostFeatures;
std::mutex ThreadCreationMutex;
FEXCore::Core::InternalThreadState* ParentThread{};
FEXCore::Core::InternalThreadState* ParentThread;
std::vector<FEXCore::Core::InternalThreadState*> Threads;
std::atomic_bool CoreShuttingDown{false};
bool NeedToCheckXID{true};
@@ -274,27 +152,36 @@ namespace FEXCore::Context {
SignalDelegator *SignalDelegation{};
X86GeneratedCode X86CodeGen;
VDSOSigReturn VDSOPointers{};
ContextImpl();
~ContextImpl();
Context();
~Context();
FEXCore::Core::InternalThreadState* InitCore(uint64_t InitialRIP, uint64_t StackPointer);
FEXCore::Context::ExitReason RunUntilExit();
int GetProgramStatus() const;
bool IsPaused() const { return !Running; }
void Pause();
void Run();
void WaitForThreadsToRun();
void Step();
void Stop(bool IgnoreCurrentThread);
void WaitForIdle();
void StopThread(FEXCore::Core::InternalThreadState *Thread);
void SignalThread(FEXCore::Core::InternalThreadState *Thread, FEXCore::Core::SignalEvent Event);
bool GetGdbServerStatus() const { return DebugServer != nullptr; }
void StartGdbServer();
void StopGdbServer();
void HandleCallback(FEXCore::Core::InternalThreadState *Thread, uint64_t RIP);
void RegisterHostSignalHandler(int Signal, HostSignalDelegatorFunction Func, bool Required);
void RegisterFrontendHostSignalHandler(int Signal, HostSignalDelegatorFunction Func, bool Required);
static void ThreadRemoveCodeEntry(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP);
static void ThreadAddBlockLink(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestDestination, uintptr_t HostLink, const std::function<void()> &delinker);
template<auto Fn>
static uint64_t ThreadExitFunctionLink(FEXCore::Core::CpuStateFrame *Frame, uint64_t *record) {
FHU::ScopedSignalMaskWithSharedLock lk(static_cast<ContextImpl*>(Frame->Thread->CTX)->CodeInvalidationMutex);
FHU::ScopedSignalMaskWithSharedLock lk(Frame->Thread->CTX->CodeInvalidationMutex);
return Fn(Frame, record);
}
@@ -303,17 +190,21 @@ namespace FEXCore::Context {
// Must be called from owning thread
static void ThreadRemoveCodeEntryFromJit(FEXCore::Core::CpuStateFrame *Frame, uint64_t GuestRIP) {
auto Thread = Frame->Thread;
LogMan::Throw::AFmt(Thread->ThreadManager.GetTID() == FHU::Syscalls::gettid(), "Must be called from owning thread {}, not {}", Thread->ThreadManager.GetTID(), FHU::Syscalls::gettid());
FHU::ScopedSignalMaskWithUniqueLock lk(static_cast<ContextImpl*>(Thread->CTX)->CodeInvalidationMutex);
FHU::ScopedSignalMaskWithUniqueLock lk(Thread->CTX->CodeInvalidationMutex);
ThreadRemoveCodeEntry(Thread, GuestRIP);
}
// returns false if a handler was already registered
CustomIRResult AddCustomIREntrypoint(uintptr_t Entrypoint, std::function<void(uintptr_t Entrypoint, FEXCore::IR::IREmitter *)> Handler, void *Creator, void *Data);
void RemoveCustomIREntrypoint(uintptr_t Entrypoint);
// Debugger interface
void CompileRIP(FEXCore::Core::InternalThreadState *Thread, uint64_t RIP);
uint64_t GetThreadCount() const;
FEXCore::Core::RuntimeStats *GetRuntimeStatsForThread(uint64_t Thread);
bool GetDebugDataForRIP(uint64_t RIP, FEXCore::Core::DebugData *Data);
@@ -345,6 +236,29 @@ namespace FEXCore::Context {
void CompileBlockJit(FEXCore::Core::CpuStateFrame *Frame, uint64_t GuestRIP);
// Used for thread creation from syscalls
/**
* @brief Used to create FEX thread objects in preparation for creating a true OS thread. Does set a TID or PID.
*
* @param NewThreadState The initial thread state to setup for our state
* @param ParentTID The PID that was the parent thread that created this
*
* @return The InternalThreadState object that tracks all of the emulated thread's state
*
* Usecases:
* OS thread Creation:
* - Thread = CreateThread(NewState, PPID);
* - InitializeThread(Thread);
* OS fork (New thread created with a clone of thread state):
* - clone{2, 3}
* - Thread = CreateThread(CopyOfThreadState, PPID);
* - ExecutionThread(Thread); // Starts executing without creating another host thread
* Thunk callback executing guest code from native host thread
* - Thread = CreateThread(NewState, PPID);
* - InitializeThreadTLSData(Thread);
* - HandleCallback(Thread, RIP);
*/
FEXCore::Core::InternalThreadState* CreateThread(FEXCore::Core::CPUState *NewThreadState, uint64_t ParentTID);
/**
* @brief Initializes TID, PID and TLS data for a thread
*
@@ -352,28 +266,71 @@ namespace FEXCore::Context {
*/
void InitializeThreadTLSData(FEXCore::Core::InternalThreadState *Thread);
/**
* @brief Initializes the OS thread object and prepares to start executing on that new OS thread
*
* @param Thread The internal FEX thread state object
*
* The OS thread will wait until RunThread is executed
*/
void InitializeThread(FEXCore::Core::InternalThreadState *Thread);
/**
* @brief Starts the OS thread object to start executing guest code
*
* @param Thread The internal FEX thread state object
*/
void RunThread(FEXCore::Core::InternalThreadState *Thread);
/**
* @brief Destroys this FEX thread object and stops tracking it internally
*
* @param Thread The internal FEX thread state object
*/
void DestroyThread(FEXCore::Core::InternalThreadState *Thread);
void CopyMemoryMapping(FEXCore::Core::InternalThreadState *ParentThread, FEXCore::Core::InternalThreadState *ChildThread);
void CleanupAfterFork(FEXCore::Core::InternalThreadState *ExceptForThread);
std::vector<FEXCore::Core::InternalThreadState*>* GetThreads() { return &Threads; }
uint8_t GetGPRSize() const { return Config.Is64BitMode ? 8 : 4; }
IR::AOTIRCacheEntry *LoadAOTIRCacheEntry(const std::string &filename);
void UnloadAOTIRCacheEntry(IR::AOTIRCacheEntry *Entry);
FEXCore::JITSymbols Symbols;
void SetVDSOSigReturn(const VDSOSigReturn &Pointers) override {
VDSOPointers = Pointers;
if (VDSOPointers.VDSO_kernel_sigreturn == nullptr) {
VDSOPointers.VDSO_kernel_sigreturn = reinterpret_cast<void*>(X86CodeGen.sigreturn_32);
}
// Public for threading
void ExecutionThread(FEXCore::Core::InternalThreadState *Thread);
if (VDSOPointers.VDSO_kernel_rt_sigreturn == nullptr) {
VDSOPointers.VDSO_kernel_rt_sigreturn = reinterpret_cast<void*>(X86CodeGen.rt_sigreturn_32);
}
void FinalizeAOTIRCache() {
IRCaptureCache.FinalizeAOTIRCache();
}
void WriteFilesWithCode(std::function<void(const std::string& fileid, const std::string& filename)> Writer) {
IRCaptureCache.WriteFilesWithCode(Writer);
}
void SetAOTIRLoader(std::function<int(const std::string&)> CacheReader) {
IRCaptureCache.SetAOTIRLoader(CacheReader);
}
void SetAOTIRWriter(std::function<std::unique_ptr<std::ofstream>(const std::string&)> CacheWriter) {
IRCaptureCache.SetAOTIRWriter(CacheWriter);
}
void SetAOTIRRenamer(std::function<void(const std::string&)> CacheRenamer) {
IRCaptureCache.SetAOTIRRenamer(CacheRenamer);
}
void AppendThunkDefinitions(std::vector<FEXCore::IR::ThunkDefinition> const& Definitions);
FEXCore::Utils::PooledAllocatorMMap OpDispatcherAllocator;
FEXCore::Utils::PooledAllocatorMMap FrontendAllocator;
void MarkMemoryShared();
bool IsTSOEnabled() { return (IsMemoryShared || !Config.TSOAutoMigration) && Config.TSOEnabled; }
protected:
@@ -5,7 +5,6 @@
#include "Interface/HLE/Thunks/Thunks.h"
#include <FEXCore/Core/CoreState.h>
#include <FEXCore/Utils/BitUtils.h>
#include <FEXCore/Utils/LogManager.h>
#include <FEXCore/Utils/MathUtils.h>
@@ -21,7 +20,7 @@
namespace FEXCore::CPU {
// We want vixl to not allocate a default buffer. Jit and dispatcher will manually create one.
Arm64Emitter::Arm64Emitter(FEXCore::Context::ContextImpl *ctx, size_t size)
Arm64Emitter::Arm64Emitter(FEXCore::Context::Context *ctx, size_t size)
: Emitter(size ? (uint8_t*)FEXCore::Allocator::mmap(nullptr, size, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0) : nullptr, size)
, EmitterCTX {ctx} {
CPU.SetUp();
@@ -217,13 +216,13 @@ void Arm64Emitter::SpillStaticRegs(bool FPRs, uint32_t GPRSpillMask, uint32_t FP
if (((1U << Reg.Idx()) & FPRSpillMask) != 0) {
mov(ARMEmitter::Size::i64Bit, TMP4.R(), offsetof(Core::CpuStateFrame, State.xmm.avx.data[i][0]));
st1b<ARMEmitter::SubRegSize::i8Bit>(Reg.Z(), PRED_TMP_32B, STATE.R(), TMP4.R());
st1b<ARMEmitter::SubRegSize::i8Bit>(Reg, PRED_TMP_32B, STATE.R(), TMP4.R());
}
}
} else {
if (GPRSpillMask && FPRSpillMask == ~0U) {
// Optimize the common case where we can spill four registers per instruction
auto TmpReg = SRA64[FindFirstSetBit(GPRSpillMask)];
auto TmpReg = SRA64[__builtin_ffs(GPRSpillMask)];
// Load the sse offset in to the temporary register
add(ARMEmitter::Size::i64Bit, TmpReg, STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.xmm.sse.data[0][0]));
@@ -274,14 +273,14 @@ void Arm64Emitter::FillStaticRegs(bool FPRs, uint32_t GPRFillMask, uint32_t FPRF
const auto Reg = SRAFPR[i];
if (((1U << Reg.Idx()) & FPRFillMask) != 0) {
mov(ARMEmitter::Size::i64Bit, TMP4.R(), offsetof(Core::CpuStateFrame, State.xmm.avx.data[i][0]));
ld1b<ARMEmitter::SubRegSize::i8Bit>(Reg.Z(), PRED_TMP_32B.Zeroing(), STATE.R(), TMP4.R());
ld1b<ARMEmitter::SubRegSize::i8Bit>(Reg, PRED_TMP_32B, STATE.R(), TMP4.R());
}
}
} else {
if (GPRFillMask && FPRFillMask == ~0U) {
// Optimize the common case where we can fill four registers per instruction.
// Use one of the filling static registers before we fill it.
auto TmpReg = SRA64[FindFirstSetBit(GPRFillMask)];
auto TmpReg = SRA64[__builtin_ffs(GPRFillMask)];
// Load the sse offset in to the temporary register
add(ARMEmitter::Size::i64Bit, TmpReg, STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.xmm.sse.data[0][0]));
@@ -348,7 +347,7 @@ void Arm64Emitter::PushDynamicRegsAndLR(FEXCore::ARMEmitter::Register TmpReg) {
const auto Reg2 = RAFPR[i + 1];
const auto Reg3 = RAFPR[i + 2];
const auto Reg4 = RAFPR[i + 3];
st4b(Reg1.Z(), Reg2.Z(), Reg3.Z(), Reg4.Z(), PRED_TMP_32B, TmpReg, 0);
st4b(Reg1, Reg2, Reg3, Reg4, PRED_TMP_32B, TmpReg, 0);
add(ARMEmitter::Size::i64Bit, TmpReg, TmpReg, 32 * 4);
}
} else {
@@ -374,7 +373,7 @@ void Arm64Emitter::PopDynamicRegsAndLR() {
const auto Reg2 = RAFPR[i + 1];
const auto Reg3 = RAFPR[i + 2];
const auto Reg4 = RAFPR[i + 3];
ld4b(Reg1.Z(), Reg2.Z(), Reg3.Z(), Reg4.Z(), PRED_TMP_32B.Zeroing(), ARMEmitter::Reg::rsp);
ld4b(Reg1, Reg2, Reg3, Reg4, PRED_TMP_32B, ARMEmitter::Reg::rsp);
add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, 32 * 4);
}
} else {
@@ -85,10 +85,10 @@ constexpr FEXCore::ARMEmitter::PRegister PRED_TMP_32B = FEXCore::ARMEmitter::PRe
// be used by both Arm64 JIT and ARM64 Dispatcher
class Arm64Emitter : public FEXCore::ARMEmitter::Emitter {
protected:
Arm64Emitter(FEXCore::Context::ContextImpl *ctx, size_t size);
Arm64Emitter(FEXCore::Context::Context *ctx, size_t size);
~Arm64Emitter();
FEXCore::Context::ContextImpl *EmitterCTX;
FEXCore::Context::Context *EmitterCTX;
vixl::aarch64::CPU CPU;
void LoadConstant(FEXCore::ARMEmitter::Size s, FEXCore::ARMEmitter::Register Reg, uint64_t Constant, bool NOPPad = false);
@@ -10,16 +10,6 @@
* FEX-Emu ALU operations usually have a 32-bit or 64-bit operating size encoded in the IR operation,
* This allows FEX to use a single helper function which decodes to both handlers.
*/
private:
static bool IsADRRange(int64_t Imm) {
return Imm >= -1048576 && Imm <= 1048575;
}
static bool IsADRPRange(int64_t Imm) {
return Imm >= -4294967296 && Imm <= 4294963200;
}
static bool IsADRPAligned(int64_t Imm) {
return (Imm & 0xFFF) == 0;
}
public:
// PC relative
void adr(FEXCore::ARMEmitter::Register rd, uint32_t Imm) {
@@ -29,7 +19,7 @@ public:
void adr(FEXCore::ARMEmitter::Register rd, BackwardLabel const* Label) {
int32_t Imm = static_cast<int32_t>(Label->Location - GetCursorAddress<uint8_t*>());
LOGMAN_THROW_A_FMT(IsADRRange(Imm), "Unscaled offset too large");
LOGMAN_THROW_A_FMT(Imm >= -1048576 && Imm <= 1048575, "Unscaled offset too large");
constexpr uint32_t Op = 0b0001'0000 << 24;
DataProcessing_PCRel_Imm(Op, rd, Imm);
@@ -56,7 +46,7 @@ public:
void adrp(FEXCore::ARMEmitter::Register rd, BackwardLabel const* Label) {
int64_t Imm = reinterpret_cast<int64_t>(Label->Location) - (GetCursorAddress<int64_t>() & ~0xFFFLL);
LOGMAN_THROW_A_FMT(IsADRPRange(Imm) && IsADRPAligned(Imm), "Unscaled offset too large");
LOGMAN_THROW_A_FMT(Imm >= -4294967296 && Imm <= 4294963200 && (Imm & 0xFFF) == 0, "Unscaled offset too large");
constexpr uint32_t Op = 0b1001'0000 << 24;
DataProcessing_PCRel_Imm(Op, rd, Imm);
@@ -76,49 +66,6 @@ public:
}
}
void LongAddressGen(FEXCore::ARMEmitter::Register rd, BackwardLabel const* Label) {
int64_t Imm = reinterpret_cast<int64_t>(Label->Location) - (GetCursorAddress<int64_t>());
if (IsADRRange(Imm)) {
// If the range is in ADR range then we can just use ADR.
adr(rd, Label);
}
else if (IsADRPRange(Imm)) {
int64_t ADRPImm = (reinterpret_cast<int64_t>(Label->Location) & ~0xFFFLL)
- (GetCursorAddress<int64_t>() & ~0xFFFLL);
// If the range is in the ADRP range then we can use ADRP.
bool NeedsOffset = !IsADRPAligned(reinterpret_cast<uint64_t>(Label->Location));
uint64_t AlignedOffset = reinterpret_cast<uint64_t>(Label->Location) & 0xFFFULL;
// First emit ADRP
adrp(rd, ADRPImm >> 12);
if (NeedsOffset) {
// Now even an add
add(ARMEmitter::Size::i64Bit, rd, rd, AlignedOffset);
}
}
else {
LOGMAN_MSG_A_FMT("Unscaled offset too large");
FEX_UNREACHABLE;
}
}
void LongAddressGen(FEXCore::ARMEmitter::Register rd, ForwardLabel* Label) {
Label->Insts.emplace_back(ForwardLabel::Instructions{ .Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::Instructions::InstType::LONG_ADDRESS_GEN });
// Emit a register index and a nop. These will be backpatched.
dc32(rd.Idx());
nop();
}
void LongAddressGen(FEXCore::ARMEmitter::Register rd, BiDirectionalLabel *Label) {
if (Label->Backward.Location) {
LongAddressGen(rd, &Label->Backward);
}
else {
LongAddressGen(rd, &Label->Forward);
}
}
// Add/subtract immediate
void add(FEXCore::ARMEmitter::Size s, FEXCore::ARMEmitter::Register rd, FEXCore::ARMEmitter::Register rn, uint32_t Imm, bool LSL12 = false) {
constexpr uint32_t Op = 0b0001'0001'0 << 23;
@@ -257,8 +204,8 @@ public:
void sxth(FEXCore::ARMEmitter::Size s, FEXCore::ARMEmitter::Register rd, FEXCore::ARMEmitter::Register rn) {
sbfm(s, rd, rn, 0, 15);
}
void sxtw(FEXCore::ARMEmitter::XRegister rd, FEXCore::ARMEmitter::WRegister rn) {
sbfm(ARMEmitter::Size::i64Bit, rd, rn.X(), 0, 31);
void sxtw(FEXCore::ARMEmitter::XRegister rd, FEXCore::ARMEmitter::XRegister rn) {
sbfm(ARMEmitter::Size::i64Bit, rd, rn, 0, 31);
}
void sbfx(FEXCore::ARMEmitter::Size s, FEXCore::ARMEmitter::Register rd, FEXCore::ARMEmitter::Register rn, uint32_t lsb, uint32_t width) {
LOGMAN_THROW_A_FMT(width > 0, "sbfx needs width > 0");
@@ -287,12 +234,12 @@ public:
void lsl(FEXCore::ARMEmitter::Size s, FEXCore::ARMEmitter::Register rd, FEXCore::ARMEmitter::Register rn, uint32_t shift) {
const auto RegSize = RegSizeInBits(s);
LOGMAN_THROW_A_FMT(shift < RegSize, "Tried to lsl a region larger than the register");
LOGMAN_THROW_A_FMT(shift < RegSize, "Tried to asr a region larger than the register");
ubfm(s, rd, rn, (RegSize - shift) % RegSize, RegSize - shift - 1);
}
void lsr(FEXCore::ARMEmitter::Size s, FEXCore::ARMEmitter::Register rd, FEXCore::ARMEmitter::Register rn, uint32_t shift) {
const auto RegSize = RegSizeInBits(s);
LOGMAN_THROW_A_FMT(shift < RegSize, "Tried to lsr a region larger than the register");
LOGMAN_THROW_A_FMT(shift < RegSize, "Tried to asr a region larger than the register");
ubfm(s, rd, rn, shift, RegSize - 1);
}
void ubfx(FEXCore::ARMEmitter::Size s, FEXCore::ARMEmitter::Register rd, FEXCore::ARMEmitter::Register rn, uint32_t lsb, uint32_t width) {
@@ -303,8 +250,8 @@ public:
void bfi(FEXCore::ARMEmitter::Size s, FEXCore::ARMEmitter::Register rd, FEXCore::ARMEmitter::Register rn, uint32_t lsb, uint32_t width) {
const auto RegSize = RegSizeInBits(s);
LOGMAN_THROW_A_FMT(width > 0, "bfi needs width > 0");
LOGMAN_THROW_A_FMT((lsb + width) <= RegSize, "Tried to bfi a region larger than the register");
LOGMAN_THROW_A_FMT(width > 0, "sbfx needs width > 0");
LOGMAN_THROW_A_FMT((lsb + width) <= RegSize, "Tried to sbfx a region larger than the register");
bfm(s, rd, rn, (RegSize - lsb) & (RegSize - 1), width - 1);
}
@@ -316,6 +263,7 @@ public:
}
void ror(FEXCore::ARMEmitter::Size s, FEXCore::ARMEmitter::Register rd, FEXCore::ARMEmitter::Register rn, uint32_t Imm) {
LOGMAN_THROW_A_FMT(Imm < RegSizeInBits(s), "Tried to extr a region larger than the register");
extr(s, rd, rn, rn, Imm);
}
@@ -636,30 +584,10 @@ public:
constexpr uint32_t Op = 0b0111'1010'000U << 21;
DataProcessing_Extended_Reg(Op, s, rd, rn, rm, FEXCore::ARMEmitter::ExtendedType::UXTB, 0);
}
// Rotate right into flags
void rmif(XRegister rn, uint32_t shift, uint32_t mask) {
LOGMAN_THROW_AA_FMT(shift <= 63, "Shift must be within 0-63. Shift: {}", shift);
LOGMAN_THROW_AA_FMT(mask <= 15, "Mask must be within 0-15. Mask: {}", mask);
uint32_t Op = 0b1011'1010'0000'0000'0000'0100'0000'0000;
Op |= rn.Idx() << 5;
Op |= shift << 15;
Op |= mask;
dc32(Op);
}
// TODO
// Evaluate into flags
void setf8(WRegister rn) {
constexpr uint32_t Op = 0b0011'1010'0000'0000'0000'1000'0000'1101;
EvaluateIntoFlags(Op, 0, rn);
}
void setf16(WRegister rn) {
constexpr uint32_t Op = 0b0011'1010'0000'0000'0000'1000'0000'1101;
EvaluateIntoFlags(Op, 1, rn);
}
// TODO
// Conditional compare - register
void ccmn(FEXCore::ARMEmitter::Size s, FEXCore::ARMEmitter::Register rn, FEXCore::ARMEmitter::Register rm, FEXCore::ARMEmitter::StatusFlags flags, FEXCore::ARMEmitter::Condition Cond) {
constexpr uint32_t Op = 0b0011'1010'010 << 21;
@@ -710,28 +638,28 @@ public:
DataProcessing_3Source(Op, 0, s, rd, rn, rm, ra);
}
void mul(FEXCore::ARMEmitter::Size s, FEXCore::ARMEmitter::Register rd, FEXCore::ARMEmitter::Register rn, FEXCore::ARMEmitter::Register rm) {
madd(s, rd, rn, rm, XReg::zr);
madd(s, rd, rn, rm, FEXCore::ARMEmitter::Reg::zr);
}
void msub(FEXCore::ARMEmitter::Size s, FEXCore::ARMEmitter::Register rd, FEXCore::ARMEmitter::Register rn, FEXCore::ARMEmitter::Register rm, FEXCore::ARMEmitter::Register ra) {
constexpr uint32_t Op = 0b001'1011'000U << 21;
DataProcessing_3Source(Op, 1, s, rd, rn, rm, ra);
}
void mneg(FEXCore::ARMEmitter::Size s, FEXCore::ARMEmitter::Register rd, FEXCore::ARMEmitter::Register rn, FEXCore::ARMEmitter::Register rm) {
msub(s, rd, rn, rm, XReg::zr);
msub(s, rd, rn, rm, FEXCore::ARMEmitter::Reg::zr);
}
void smaddl(FEXCore::ARMEmitter::XRegister rd, FEXCore::ARMEmitter::WRegister rn, FEXCore::ARMEmitter::WRegister rm, FEXCore::ARMEmitter::XRegister ra) {
constexpr uint32_t Op = 0b001'1011'001U << 21;
DataProcessing_3Source(Op, 0, FEXCore::ARMEmitter::Size::i64Bit, rd, rn, rm, ra);
}
void smull(FEXCore::ARMEmitter::XRegister rd, FEXCore::ARMEmitter::WRegister rn, FEXCore::ARMEmitter::WRegister rm) {
smaddl(rd, rn, rm, XReg::zr);
smaddl(rd, rn, rm, FEXCore::ARMEmitter::Reg::zr);
}
void smsubl(FEXCore::ARMEmitter::XRegister rd, FEXCore::ARMEmitter::WRegister rn, FEXCore::ARMEmitter::WRegister rm, FEXCore::ARMEmitter::XRegister ra) {
constexpr uint32_t Op = 0b001'1011'001U << 21;
DataProcessing_3Source(Op, 1, FEXCore::ARMEmitter::Size::i64Bit, rd, rn, rm, ra);
}
void smnegl(FEXCore::ARMEmitter::XRegister rd, FEXCore::ARMEmitter::WRegister rn, FEXCore::ARMEmitter::WRegister rm) {
smsubl(rd, rn, rm, XReg::zr);
smsubl(rd, rn, rm, FEXCore::ARMEmitter::Reg::zr);
}
void smulh(FEXCore::ARMEmitter::XRegister rd, FEXCore::ARMEmitter::XRegister rn, FEXCore::ARMEmitter::XRegister rm) {
constexpr uint32_t Op = 0b001'1011'010U << 21;
@@ -742,14 +670,14 @@ public:
DataProcessing_3Source(Op, 0, FEXCore::ARMEmitter::Size::i64Bit, rd, rn, rm, ra);
}
void umull(FEXCore::ARMEmitter::XRegister rd, FEXCore::ARMEmitter::WRegister rn, FEXCore::ARMEmitter::WRegister rm) {
umaddl(rd, rn, rm, XReg::zr);
umaddl(rd, rn, rm, FEXCore::ARMEmitter::Reg::zr);
}
void umsubl(FEXCore::ARMEmitter::XRegister rd, FEXCore::ARMEmitter::WRegister rn, FEXCore::ARMEmitter::WRegister rm, FEXCore::ARMEmitter::XRegister ra) {
constexpr uint32_t Op = 0b001'1011'101U << 21;
DataProcessing_3Source(Op, 1, FEXCore::ARMEmitter::Size::i64Bit, rd, rn, rm, ra);
}
void umnegl(FEXCore::ARMEmitter::XRegister rd, FEXCore::ARMEmitter::WRegister rn, FEXCore::ARMEmitter::WRegister rm) {
umsubl(rd, rn, rm, XReg::zr);
umsubl(rd, rn, rm, FEXCore::ARMEmitter::Reg::zr);
}
void umulh(FEXCore::ARMEmitter::XRegister rd, FEXCore::ARMEmitter::XRegister rn, FEXCore::ARMEmitter::XRegister rm) {
constexpr uint32_t Op = 0b001'1011'110U << 21;
@@ -981,11 +909,4 @@ private:
dc32(Instr);
}
void EvaluateIntoFlags(uint32_t op, uint32_t size, WRegister rn) {
uint32_t Instr = op;
Instr |= size << 14;
Instr |= rn.Idx() << 5;
dc32(Instr);
}
File diff suppressed because it is too large. Load diff
@@ -87,11 +87,6 @@ namespace FEXCore::ARMEmitter {
return Size;
}
template<typename T>
size_t GetCursorOffsetFromAddress(const T* Address) const {
return static_cast<size_t>(reinterpret_cast<const uint8_t*>(Address) - BufferBase);
}
protected:
void ResetBuffer() {
@@ -519,7 +519,6 @@ namespace FEXCore::ARMEmitter {
BC,
TEST_BRANCH,
RELATIVE_LOAD,
LONG_ADDRESS_GEN,
};
uint8_t *Location{};
InstType Type;
@@ -536,44 +535,6 @@ namespace FEXCore::ARMEmitter {
ForwardLabel Forward;
};
// Some FCMA ASIMD instructions support a rotation argument.
enum class Rotation : uint32_t {
ROTATE_0 = 0b00,
ROTATE_90 = 0b01,
ROTATE_180 = 0b10,
ROTATE_270 = 0b11,
};
// Concept for contraining some instructions to accept only an XRegister or WRegister.
// Particularly for operations that differ encodings depending on which one is used.
template <typename T>
concept IsXOrWRegister = std::is_same_v<T, XRegister> || std::is_same_v<T, WRegister>;
// Whether or not a given set of vector registers are sequential
// in increasing order as far as the register file is concerned (modulo its size)
//
// For example, a set of registers like:
//
// v1, v2, v3 and
// v31, v0, v1
//
// would both be considered sequential sequences, and some instructions in particular
// limit register lists to these kind of sequences.
//
template <typename T, typename... Args>
constexpr bool AreVectorsSequential(T first, const Args&... args) {
// Ensure we always have a pair of registers to compare against.
static_assert(sizeof...(args) >= 1, "Number of arguments must be greater than 1");
const auto fn = [](auto& lhs, const auto& rhs) {
const auto result = ((lhs.Idx() + 1) % 32) == rhs.Idx();
lhs = rhs;
return result;
};
return (fn(first, args) && ...);
}
// This is an emitter that is designed around the smallest code bloat as possible.
// Eschewing most developer convenience in order to keep code as small as possible.
@@ -610,7 +571,7 @@ namespace FEXCore::ARMEmitter {
case ForwardLabel::Instructions::InstType::ADR: {
uint32_t *Instruction = reinterpret_cast<uint32_t*>(Inst.Location);
int64_t Imm = reinterpret_cast<int64_t>(CurrentAddress) - reinterpret_cast<int64_t>(Instruction);
LOGMAN_THROW_A_FMT(IsADRRange(Imm), "Unscaled offset too large");
LOGMAN_THROW_A_FMT(Imm >= -1048576 && Imm <= 1048575, "Unscaled offset too large");
uint32_t InstMask = 0b11 << 29 | 0b1111'1111'1111'1111'111 << 5;
uint32_t Offset = static_cast<uint32_t>(Imm) & 0x3F'FFFF;
uint32_t Inst = *Instruction & ~InstMask;
@@ -622,7 +583,7 @@ namespace FEXCore::ARMEmitter {
case ForwardLabel::Instructions::InstType::ADRP: {
uint32_t *Instruction = reinterpret_cast<uint32_t*>(Inst.Location);
int64_t Imm = reinterpret_cast<int64_t>(CurrentAddress) - reinterpret_cast<int64_t>(Instruction);
LOGMAN_THROW_A_FMT(IsADRPRange(Imm) && IsADRPAligned(Imm), "Unscaled offset too large");
LOGMAN_THROW_A_FMT(Imm >= -4294967296 && Imm <= 4294963200 && (Imm & 0xFFF) == 0, "Unscaled offset too large");
Imm >>= 12;
uint32_t InstMask = 0b11 << 29 | 0b1111'1111'1111'1111'111 << 5;
uint32_t Offset = static_cast<uint32_t>(Imm) & 0x3F'FFFF;
@@ -673,47 +634,6 @@ namespace FEXCore::ARMEmitter {
*Instruction = Inst;
break;
}
case ForwardLabel::Instructions::InstType::LONG_ADDRESS_GEN: {
uint32_t *Instructions = reinterpret_cast<uint32_t*>(Inst.Location);
int64_t ImmInstOne = reinterpret_cast<int64_t>(CurrentAddress) - reinterpret_cast<int64_t>(&Instructions[0]);
int64_t ImmInstTwo = reinterpret_cast<int64_t>(CurrentAddress) - reinterpret_cast<int64_t>(&Instructions[1]);
auto OriginalOffset = GetCursorOffset();
auto InstOffset = GetCursorOffsetFromAddress(Instructions);
SetCursorOffset(InstOffset);
// We encoded the destination register in to the first instruction space.
// Read it back.
ARMEmitter::Register DestReg(Instructions[0]);
if (IsADRRange(ImmInstTwo)) {
// If within ADR range from the second instruction, then we can emit NOP+ADR
nop();
adr(DestReg, static_cast<uint32_t>(ImmInstTwo) & 0x7FFF);
}
else if (IsADRPRange(ImmInstOne)) {
// If within ADRP range from the first instruction, then we are /definitely/ in range for the second instruction.
// First check if we are in non-offset range for second instruction.
if (IsADRPAligned(reinterpret_cast<uint64_t>(CurrentAddress))) {
// We can emit nop + adrp
nop();
adrp(DestReg, static_cast<uint32_t>(ImmInstTwo >> 12) & 0x7FFF);
}
else {
// Not aligned, need adrp + add
adrp(DestReg, static_cast<uint32_t>(ImmInstOne >> 12) & 0x7FFF);
add(ARMEmitter::Size::i64Bit, DestReg, DestReg, ImmInstOne & 0xFFF);
}
}
else {
LOGMAN_MSG_A_FMT("Unscaled offset is too large");
FEX_UNREACHABLE;
}
SetCursorOffset(OriginalOffset);
break;
}
default: LOGMAN_MSG_A_FMT("Unexpected inst type in label fixup");
}
}
File diff suppressed because it is too large. Load diff
@@ -1,8 +1,5 @@
#pragma once
#include <FEXCore/Utils/EnumUtils.h>
#include <compare>
#include <cstdint>
namespace FEXCore::ARMEmitter {
@@ -18,12 +15,13 @@ namespace FEXCore::ARMEmitter {
constexpr explicit Register(uint32_t Idx)
: Index {Idx} {}
friend constexpr auto operator<=>(const Register&, const Register&) = default;
uint32_t Idx() const {
return Index;
}
operator WRegister() const;
operator XRegister() const;
WRegister W() const;
XRegister X() const;
@@ -43,7 +41,9 @@ namespace FEXCore::ARMEmitter {
constexpr explicit WRegister(uint32_t Idx)
: Index {Idx} {}
friend constexpr auto operator<=>(const WRegister&, const WRegister&) = default;
bool operator==(const WRegister &rhs) {
return Idx() == rhs.Idx();
}
uint32_t Idx() const {
return Index;
@@ -53,7 +53,10 @@ namespace FEXCore::ARMEmitter {
return Register(Index);
}
operator XRegister() const;
XRegister X() const;
Register R() const;
private:
@@ -72,7 +75,9 @@ namespace FEXCore::ARMEmitter {
constexpr explicit XRegister(uint32_t Idx)
: Index {Idx} {}
friend constexpr auto operator<=>(const XRegister&, const XRegister&) = default;
bool operator==(const XRegister &rhs) {
return Idx() == rhs.Idx();
}
uint32_t Idx() const {
return Index;
@@ -82,7 +87,10 @@ namespace FEXCore::ARMEmitter {
return Register(Index);
}
operator WRegister() const;
WRegister W() const;
Register R() const;
private:
@@ -93,29 +101,45 @@ namespace FEXCore::ARMEmitter {
static_assert(std::is_standard_layout_v<Register>, "Needs to be standard");
inline WRegister Register::W() const {
return WRegister{Index};
return *this;
}
inline XRegister Register::X() const {
return XRegister{Index};
return *this;
}
inline Register::operator WRegister () const {
return WRegister(Index);
}
inline Register::operator XRegister () const {
return XRegister(Index);
}
inline XRegister WRegister::X() const {
return XRegister{Index};
return *this;
}
inline Register WRegister::R() const {
return *this;
}
inline WRegister::operator XRegister () const {
return XRegister(Index);
}
inline WRegister XRegister::W() const {
return WRegister{Index};
return *this;
}
inline Register XRegister::R() const {
return *this;
}
inline XRegister::operator WRegister () const {
return WRegister(Index);
}
// Namespace containing all unsized GPR register objects.
namespace Reg {
constexpr static Register r0(0);
@@ -267,15 +291,20 @@ namespace FEXCore::ARMEmitter {
class VRegister {
public:
VRegister() = delete;
constexpr explicit VRegister(uint32_t Idx)
constexpr VRegister(uint32_t Idx)
: Index {Idx} {}
friend constexpr auto operator<=>(const VRegister&, const VRegister&) = default;
uint32_t Idx() const {
return Index;
}
operator BRegister() const;
operator HRegister() const;
operator SRegister() const;
operator DRegister() const;
operator QRegister() const;
operator ZRegister() const;
BRegister B() const;
HRegister H() const;
SRegister S() const;
@@ -299,15 +328,16 @@ namespace FEXCore::ARMEmitter {
constexpr explicit BRegister(uint32_t Idx)
: Index {Idx} {}
friend constexpr auto operator<=>(const BRegister&, const BRegister&) = default;
uint32_t Idx() const {
return Index;
}
operator VRegister () const {
return VRegister(Index);
}
operator VRegister() const;
operator HRegister() const;
operator SRegister() const;
operator DRegister() const;
operator QRegister() const;
operator ZRegister() const;
BRegister V() const;
HRegister H() const;
@@ -332,15 +362,16 @@ namespace FEXCore::ARMEmitter {
constexpr explicit HRegister(uint32_t Idx)
: Index {Idx} {}
friend constexpr auto operator<=>(const HRegister&, const HRegister&) = default;
uint32_t Idx() const {
return Index;
}
operator VRegister() const {
return VRegister(Index);
}
operator VRegister() const;
operator BRegister() const;
operator SRegister() const;
operator DRegister() const;
operator QRegister() const;
operator ZRegister() const;
HRegister V() const;
BRegister B() const;
@@ -365,15 +396,16 @@ namespace FEXCore::ARMEmitter {
constexpr explicit SRegister(uint32_t Idx)
: Index {Idx} {}
friend constexpr auto operator<=>(const SRegister&, const SRegister&) = default;
uint32_t Idx() const {
return Index;
}
operator VRegister() const {
return VRegister(Index);
}
operator VRegister() const;
operator BRegister() const;
operator HRegister() const;
operator DRegister() const;
operator QRegister() const;
operator ZRegister() const;
SRegister V() const;
BRegister B() const;
@@ -399,15 +431,16 @@ namespace FEXCore::ARMEmitter {
constexpr explicit DRegister(uint32_t Idx)
: Index {Idx} {}
friend constexpr auto operator<=>(const DRegister&, const DRegister&) = default;
uint32_t Idx() const {
return Index;
}
operator VRegister() const {
return VRegister(Index);
}
operator VRegister() const;
operator BRegister() const;
operator HRegister() const;
operator SRegister() const;
operator QRegister() const;
operator ZRegister() const;
DRegister V() const;
BRegister B() const;
@@ -433,15 +466,16 @@ namespace FEXCore::ARMEmitter {
constexpr explicit QRegister(uint32_t Idx)
: Index {Idx} {}
friend constexpr auto operator<=>(const QRegister&, const QRegister&) = default;
uint32_t Idx() const {
return Index;
}
operator VRegister () const {
return VRegister(Index);
}
operator VRegister() const;
operator BRegister() const;
operator HRegister() const;
operator SRegister() const;
operator DRegister() const;
operator ZRegister() const;
QRegister V() const;
BRegister B() const;
@@ -466,8 +500,6 @@ namespace FEXCore::ARMEmitter {
constexpr explicit ZRegister(uint32_t Idx)
: Index {Idx} {}
friend constexpr auto operator<=>(const ZRegister&, const ZRegister&) = default;
uint32_t Idx() const {
return Index;
}
@@ -488,22 +520,41 @@ namespace FEXCore::ARMEmitter {
// VRegister
inline BRegister VRegister::B() const {
return BRegister{Index};
return *this;
}
inline HRegister VRegister::H() const {
return HRegister{Index};
return *this;
}
inline SRegister VRegister::S() const {
return SRegister{Index};
return *this;
}
inline DRegister VRegister::D() const {
return DRegister{Index};
return *this;
}
inline QRegister VRegister::Q() const {
return QRegister{Index};
return *this;
}
inline ZRegister VRegister::Z() const {
return ZRegister{Index};
return *this;
}
inline VRegister::operator BRegister () const {
return BRegister(Index);
}
inline VRegister::operator HRegister () const {
return HRegister(Index);
}
inline VRegister::operator SRegister () const {
return SRegister(Index);
}
inline VRegister::operator DRegister () const {
return DRegister(Index);
}
inline VRegister::operator QRegister () const {
return QRegister(Index);
}
inline VRegister::operator ZRegister () const {
return ZRegister(Index);
}
// BRegister
@@ -511,19 +562,38 @@ namespace FEXCore::ARMEmitter {
return *this;
}
inline HRegister BRegister::H() const {
return HRegister{Index};
return *this;
}
inline SRegister BRegister::S() const {
return SRegister{Index};
return *this;
}
inline DRegister BRegister::D() const {
return DRegister{Index};
return *this;
}
inline QRegister BRegister::Q() const {
return QRegister{Index};
return *this;
}
inline ZRegister BRegister::Z() const {
return ZRegister{Index};
return *this;
}
inline BRegister::operator VRegister () const {
return VRegister(Index);
}
inline BRegister::operator HRegister () const {
return HRegister(Index);
}
inline BRegister::operator SRegister () const {
return SRegister(Index);
}
inline BRegister::operator DRegister () const {
return DRegister(Index);
}
inline BRegister::operator QRegister () const {
return QRegister(Index);
}
inline BRegister::operator ZRegister () const {
return ZRegister(Index);
}
// HRegister
@@ -531,19 +601,38 @@ namespace FEXCore::ARMEmitter {
return *this;
}
inline BRegister HRegister::B() const {
return BRegister{Index};
return *this;
}
inline SRegister HRegister::S() const {
return SRegister{Index};
return *this;
}
inline DRegister HRegister::D() const {
return DRegister{Index};
return *this;
}
inline QRegister HRegister::Q() const {
return QRegister{Index};
return *this;
}
inline ZRegister HRegister::Z() const {
return ZRegister{Index};
return *this;
}
inline HRegister::operator VRegister () const {
return VRegister(Index);
}
inline HRegister::operator BRegister () const {
return BRegister(Index);
}
inline HRegister::operator SRegister () const {
return SRegister(Index);
}
inline HRegister::operator DRegister () const {
return DRegister(Index);
}
inline HRegister::operator QRegister () const {
return QRegister(Index);
}
inline HRegister::operator ZRegister () const {
return ZRegister(Index);
}
// SRegister
@@ -551,39 +640,77 @@ namespace FEXCore::ARMEmitter {
return *this;
}
inline BRegister SRegister::B() const {
return BRegister{Index};
return *this;
}
inline HRegister SRegister::H() const {
return HRegister{Index};
return *this;
}
inline DRegister SRegister::D() const {
return DRegister{Index};
return *this;
}
inline QRegister SRegister::Q() const {
return QRegister{Index};
return *this;
}
inline ZRegister SRegister::Z() const {
return ZRegister{Index};
return *this;
}
inline SRegister::operator VRegister () const {
return VRegister(Index);
}
inline SRegister::operator BRegister () const {
return BRegister(Index);
}
inline SRegister::operator HRegister () const {
return HRegister(Index);
}
inline SRegister::operator DRegister () const {
return DRegister(Index);
}
inline SRegister::operator QRegister () const {
return QRegister(Index);
}
inline SRegister::operator ZRegister () const {
return ZRegister(Index);
}
// DRegister
inline DRegister DRegister::V() const {
return DRegister{Index};
return *this;
}
inline BRegister DRegister::B() const {
return BRegister{Index};
return *this;
}
inline HRegister DRegister::H() const {
return HRegister{Index};
return *this;
}
inline SRegister DRegister::S() const {
return SRegister{Index};
return *this;
}
inline QRegister DRegister::Q() const {
return QRegister{Index};
return *this;
}
inline ZRegister DRegister::Z() const {
return ZRegister{Index};
return *this;
}
inline DRegister::operator VRegister () const {
return VRegister(Index);
}
inline DRegister::operator BRegister () const {
return BRegister(Index);
}
inline DRegister::operator HRegister () const {
return HRegister(Index);
}
inline DRegister::operator SRegister () const {
return SRegister(Index);
}
inline DRegister::operator QRegister () const {
return QRegister(Index);
}
inline DRegister::operator ZRegister () const {
return ZRegister(Index);
}
// QRegister
@@ -591,19 +718,38 @@ namespace FEXCore::ARMEmitter {
return *this;
}
inline BRegister QRegister::B() const {
return BRegister{Index};
return *this;
}
inline HRegister QRegister::H() const {
return HRegister{Index};
return *this;
}
inline SRegister QRegister::S() const {
return SRegister{Index};
return *this;
}
inline DRegister QRegister::D() const {
return DRegister{Index};
return *this;
}
inline ZRegister QRegister::Z() const {
return ZRegister{Index};
return *this;
}
inline QRegister::operator VRegister () const {
return VRegister(Index);
}
inline QRegister::operator BRegister () const {
return BRegister(Index);
}
inline QRegister::operator HRegister () const {
return HRegister(Index);
}
inline QRegister::operator SRegister () const {
return SRegister(Index);
}
inline QRegister::operator DRegister () const {
return DRegister(Index);
}
inline QRegister::operator ZRegister () const {
return ZRegister(Index);
}
// ZRegister
@@ -923,12 +1069,17 @@ namespace FEXCore::ARMEmitter {
constexpr PRegister(uint32_t Idx)
: Index {Idx} {}
friend constexpr auto operator<=>(const PRegister&, const PRegister&) = default;
operator uint32_t() const {
return Index;
}
uint32_t Idx() const {
return Index;
}
operator PRegisterZero() const;
operator PRegisterMerge() const;
PRegisterZero Zeroing() const;
PRegisterMerge Merging() const;
@@ -946,13 +1097,16 @@ namespace FEXCore::ARMEmitter {
constexpr PRegisterZero(uint32_t Idx)
: Index {Idx} {}
friend constexpr auto operator<=>(const PRegisterZero&, const PRegisterZero&) = default;
operator uint32_t() const {
return Index;
}
uint32_t Idx() const {
return Index;
}
operator PRegister() const;
operator PRegisterMerge() const;
PRegister P() const;
PRegisterMerge Merging() const;
@@ -971,13 +1125,16 @@ namespace FEXCore::ARMEmitter {
constexpr PRegisterMerge(uint32_t Idx)
: Index {Idx} {}
friend constexpr auto operator<=>(const PRegisterMerge&, const PRegisterMerge&) = default;
operator uint32_t() const {
return Index;
}
uint32_t Idx() const {
return Index;
}
operator PRegister() const;
operator PRegisterZero() const;
PRegister P() const;
PRegisterZero Zeroing() const;
@@ -991,6 +1148,14 @@ namespace FEXCore::ARMEmitter {
// PRegister
inline PRegister::operator PRegisterZero() const {
return PRegisterZero(Index);
}
inline PRegister::operator PRegisterMerge() const {
return PRegisterMerge(Index);
}
inline PRegisterZero PRegister::Zeroing() const {
return PRegisterZero(Idx());
}
@@ -1004,6 +1169,10 @@ namespace FEXCore::ARMEmitter {
return PRegister(Index);
}
inline PRegisterZero::operator PRegisterMerge() const {
return PRegisterMerge(Index);
}
inline PRegister PRegisterZero::P() const {
return PRegister(Idx());
}
@@ -1017,6 +1186,10 @@ namespace FEXCore::ARMEmitter {
return PRegisterZero(Index);
}
inline PRegisterMerge::operator PRegisterZero() const {
return PRegisterZero(Index);
}
inline PRegister PRegisterMerge::P() const {
return PRegister(Idx());
}
File diff suppressed because it is too large. Load diff
@@ -17,17 +17,25 @@
*/
public:
// Advanced SIMD scalar copy
void dup(ScalarRegSize size, VRegister rd, VRegister rn, uint32_t Index) {
void dup(FEXCore::ARMEmitter::ScalarRegSize size, FEXCore::ARMEmitter::VRegister rd, FEXCore::ARMEmitter::VRegister rn, uint32_t Index) {
constexpr uint32_t Op = 0b0101'1110'0000'0000'0000'01 << 10;
const uint32_t SizeImm = FEXCore::ToUnderlying(size);
const uint32_t IndexShift = SizeImm + 1;
const uint32_t ElementSize = 1U << SizeImm;
const uint32_t MaxIndex = 128U / (ElementSize * 8);
LOGMAN_THROW_AA_FMT(Index < MaxIndex, "Index too large. Index={}, Max Index: {}", Index, MaxIndex);
const uint32_t imm5 = (Index << IndexShift) | ElementSize;
uint32_t imm5 = 0b00000;
if (size == ScalarRegSize::i8Bit) {
LOGMAN_THROW_AA_FMT(Index < 16, "Index too large");
imm5 = (Index << 1) | 1;
}
else if (size == ScalarRegSize::i16Bit) {
LOGMAN_THROW_AA_FMT(Index < 8, "Index too large");
imm5 = (Index << 2) | 0b10;
}
else if (size == ScalarRegSize::i32Bit) {
LOGMAN_THROW_AA_FMT(Index < 4, "Index too large");
imm5 = (Index << 3) | 0b100;
}
else if (size == ScalarRegSize::i64Bit) {
LOGMAN_THROW_AA_FMT(Index < 2, "Index too large");
imm5 = (Index << 4) | 0b1000;
}
ASIMDScalarCopy(Op, 1, imm5, 0b0000, rd, rn);
}
@@ -15,21 +15,11 @@ namespace FEXCore::ArchHelpers::Context {
enum ContextFlags : uint32_t {
CONTEXT_FLAG_INJIT = (1U << 0),
CONTEXT_FLAG_32BIT = (1U << 1),
};
#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
constexpr uint64_t STACK_COOKIE_MAGIC = 0x4142434445464748ULL;
#endif
struct X86ContextBackup {
// Host State
#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
// During debug builds, insert a cookie on the stack.
// This is useful for validation that the stack is trying to be restored from the correct location.
// During stack restore, we ensure this is set to the value we expect.
// If given an incorrect stack location, or corrupted stack then this cookie will be wrong.
uint64_t StackCookie;
#endif
// RIP and RSP is stored in GPRs here
uint64_t GPRs[23];
FEXCore::x86_64::_libc_fpstate FPRState;
@@ -49,9 +39,6 @@ struct X86ContextBackup {
struct ArmContextBackup {
// Host State
#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
uint64_t StackCookie;
#endif
uint64_t GPRs[31];
uint64_t PrevSP;
uint64_t PrevPC;
@@ -224,10 +211,6 @@ static inline void BackupContext(void* ucontext, T *Backup) {
// Save the signal mask so we can restore it
memcpy(&Backup->sa_mask, &_ucontext->uc_sigmask, sizeof(uint64_t));
#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
Backup->StackCookie = STACK_COOKIE_MAGIC;
#endif
} else {
// This must be a runtime error
ERROR_AND_DIE_FMT("Wrong context type");
@@ -237,10 +220,8 @@ static inline void BackupContext(void* ucontext, T *Backup) {
template <typename T>
static inline void RestoreContext(void* ucontext, T *Backup) {
if constexpr (std::is_same<T, ArmContextBackup>::value) {
LOGMAN_THROW_A_FMT(Backup->StackCookie == STACK_COOKIE_MAGIC, "Stack cookie didn't match! 0x{:x}", Backup->StackCookie);
auto _ucontext = GetUContext(ucontext);
auto _mcontext = GetMContext(ucontext);
auto _mcontext = GetMContext(ucontext);
HostFPRState *HostState = reinterpret_cast<HostFPRState*>(&_mcontext->__reserved[0]);
LOGMAN_THROW_AA_FMT(HostState->Head.Magic == FPR_MAGIC, "Wrong FPR Magic: 0x{:08x}", HostState->Head.Magic);
@@ -321,10 +302,6 @@ static inline void BackupContext(void* ucontext, T *Backup) {
// Save the signal mask so we can restore it
memcpy(&Backup->sa_mask, &_ucontext->uc_sigmask, sizeof(uint64_t));
#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
Backup->StackCookie = STACK_COOKIE_MAGIC;
#endif
} else {
// This must be a runtime error
ERROR_AND_DIE_FMT("Wrong context type");
@@ -334,8 +311,6 @@ static inline void BackupContext(void* ucontext, T *Backup) {
template <typename T>
static inline void RestoreContext(void* ucontext, T *Backup) {
if constexpr (std::is_same<T, X86ContextBackup>::value) {
LOGMAN_THROW_A_FMT(Backup->StackCookie == STACK_COOKIE_MAGIC, "Stack cookie didn't match! 0x{:x}", Backup->StackCookie);
auto _ucontext = GetUContext(ucontext);
auto _mcontext = GetMContext(ucontext);
+2 -2
View File
@@ -60,8 +60,8 @@ auto CPUBackend::AllocateNewCodeBuffer(size_t Size) -> CodeBuffer {
FEXCore::Allocator::mmap(nullptr, Buffer.Size, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0));
LOGMAN_THROW_AA_FMT(!!Buffer.Ptr, "Couldn't allocate code buffer");
if (static_cast<Context::ContextImpl*>(ThreadState->CTX)->Config.GlobalJITNaming()) {
static_cast<Context::ContextImpl*>(ThreadState->CTX)->Symbols.RegisterJITSpace(Buffer.Ptr, Buffer.Size);
if (ThreadState->CTX->Config.GlobalJITNaming()) {
ThreadState->CTX->Symbols.RegisterJITSpace(Buffer.Ptr, Buffer.Size);
}
return Buffer;
}
+100 -23
View File
@@ -88,6 +88,7 @@ static uint32_t CalculateNumberOfCPUs() {
// when AVX implementations are further along.
constexpr uint32_t SUPPORTS_AVX = 0;
// #define CPUID_AMD
#ifdef CPUID_AMD
constexpr uint32_t FAMILY_IDENTIFIER =
0 | // Stepping
@@ -121,24 +122,25 @@ void CPUIDEmu::SetupHostHybridFlag() {
uint64_t MIDR{};
for (size_t i = 0; i < CPUs; ++i) {
std::error_code ec{};
std::string MIDRPath = fmt::format("/sys/devices/system/cpu/cpu{}/regs/identification/midr_el1", i);
std::string MIDRPath = "/sys/devices/system/cpu/cpu" + std::to_string(i) + "/regs/identification/midr_el1";
if (std::filesystem::exists(MIDRPath, ec)) {
std::vector<char> Data{};
// Needs to be a fixed size since depending on kernel it will try to read a full page of data and fail
// Only read 18 bytes for a 64bit value prefixed with 0x
if (FEXCore::FileLoading::LoadFile(Data, MIDRPath, 18)) {
uint64_t NewMIDR{};
std::string_view MIDRView(&Data.at(0), 18);
if (FEXCore::StrConv::Conv(MIDRView, &NewMIDR)) {
if (MIDR != 0 && MIDR != NewMIDR) {
// CPU mismatch, claim hybrid
Hybrid = true;
}
std::array<char, 18> Data;
// Needs to be a fixed size since depending on kernel it will try to read a full page of data and fail
// Only read 18 bytes for a 64bit value prefixed with 0x
if (FEXCore::FileLoading::LoadFileToBuffer(MIDRPath, Data) == sizeof(Data)) {
uint64_t NewMIDR{};
std::string_view MIDRView(Data.data(), sizeof(Data));
if (FEXCore::StrConv::Conv(MIDRView, &NewMIDR)) {
if (MIDR != 0 && MIDR != NewMIDR) {
// CPU mismatch, claim hybrid
Hybrid = true;
// Truncate to 32-bits, top 32-bits are all reserved in MIDR
PerCPUData[i].ProductName = ProductNames::ARM_UNKNOWN;
PerCPUData[i].MIDR = NewMIDR;
MIDR = NewMIDR;
}
// Truncate to 32-bits, top 32-bits are all reserved in MIDR
PerCPUData[i].ProductName = ProductNames::ARM_UNKNOWN;
PerCPUData[i].MIDR = NewMIDR;
MIDR = NewMIDR;
}
}
}
@@ -410,9 +412,6 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_01h(uint32_t Leaf) {
// XXX: Enable once the rest of the SSE4.2 instructions are emulated
uint32_t SupportsSSE42 = CTX->HostFeatures.SupportsCRC && false ? 1 : 0;
// Hypervisor bit is normally set but some applications have issues with it.
uint32_t Hypervisor = HideHypervisorBit() ? 0 : 1;
Res.eax = FAMILY_IDENTIFIER;
Res.ebx = 0 | // Brand index
@@ -452,7 +451,7 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_01h(uint32_t Leaf) {
(SUPPORTS_AVX << 28) | // AVX
(0 << 29) | // F16C
(CTX->HostFeatures.SupportsRAND << 30) | // RDRAND
(Hypervisor << 31);
(1 << 31); // Hypervisor always returns one
Res.edx =
(1 << 0) | // FPU
@@ -658,8 +657,8 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_07h(uint32_t Leaf) {
(0 << 20) | // SMAP Supervisor mode access prevention and CLAC/STAC instructions
(0 << 21) | // Reserved
(0 << 22) | // Reserved
(1 << 23) | // CLFLUSHOPT instruction
(CTX->HostFeatures.SupportsCLWB << 24) | // CLWB instruction
(0 << 23) | // CLFLUSHOPT instruction
(0 << 24) | // CLWB instruction
(0 << 25) | // Intel processor trace
(0 << 26) | // Reserved
(0 << 27) | // Reserved
@@ -1213,9 +1212,87 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_Reserved(uint32_t Leaf) {
return Res;
}
void CPUIDEmu::Init(FEXCore::Context::ContextImpl *ctx) {
void CPUIDEmu::Init(FEXCore::Context::Context *ctx) {
CTX = ctx;
RegisterFunction(0, &CPUIDEmu::Function_0h);
RegisterFunction(1, &CPUIDEmu::Function_01h);
RegisterFunction(2, &CPUIDEmu::Function_02h);
// 3: Serial Number(previously), now reserved
#ifndef CPUID_AMD
// Deterministic cache parameters for each level
RegisterFunction(0x4, &CPUIDEmu::Function_04h);
#endif
// 5: Monitor/mwait
// Thermal and power management
RegisterFunction(6, &CPUIDEmu::Function_06h);
// Extended feature flags
RegisterFunction(7, &CPUIDEmu::Function_07h);
// 9: Direct Cache Access information
// 0x0A: Architectural performance monitoring
// 0x0B: Extended topology enumeration
// 0x0D: Processor extended state enumeration
RegisterFunction(0x0D, &CPUIDEmu::Function_0Dh);
// 0x0F: Intel RDT monitoring
// 0x10: Intel RDT allocation enumeration
// 0x12: Intel SGX capability enumeration
// 0x13: Reserved
// 0x14: Intel Processor trace
#ifndef CPUID_AMD
// Timestamp counter information
// Doesn't exist on AMD hardware
RegisterFunction(0x15, &CPUIDEmu::Function_15h);
#endif
// 0x16: Processor frequency information
// 0x17: SoC vendor attribute enumeration
// 0x1A: Hybrid Information Sub-leaf
#ifndef CPUID_AMD
RegisterFunction(0x1A, &CPUIDEmu::Function_1Ah);
#endif
// Hypervisor CPUID information leaf
RegisterFunction(0x4000'0000, &CPUIDEmu::Function_4000_0000h);
RegisterFunction(0x4000'0001, &CPUIDEmu::Function_4000_0001h);
// Largest extended function number
RegisterFunction(0x8000'0000, &CPUIDEmu::Function_8000_0000h);
// Processor vendor
RegisterFunction(0x8000'0001, &CPUIDEmu::Function_8000_0001h);
// Processor brand string
RegisterFunction(0x8000'0002, &CPUIDEmu::Function_8000_0002h);
// Processor brand string continued
RegisterFunction(0x8000'0003, &CPUIDEmu::Function_8000_0003h);
// Processor brand string continued
RegisterFunction(0x8000'0004, &CPUIDEmu::Function_8000_0004h);
// 0x8000'0005: L1 Cache and TLB identifiers
#ifdef CPUID_AMD
RegisterFunction(0x8000'0005, &CPUIDEmu::Function_8000_0005h);
#else
// This is full reserved on Intel platforms
RegisterFunction(0x8000'0005, &CPUIDEmu::Function_Reserved);
#endif
// 0x8000'0006: L2 Cache identifiers
RegisterFunction(0x8000'0006, &CPUIDEmu::Function_8000_0006h);
// Advanced power management information
RegisterFunction(0x8000'0007, &CPUIDEmu::Function_8000_0007h);
// Virtual and physical address sizes
RegisterFunction(0x8000'0008, &CPUIDEmu::Function_8000_0008h);
// 0x8000'000A: SVM Revision
// TLB 1GB page identifiers
RegisterFunction(0x8000'0019, &CPUIDEmu::Function_8000_0019h);
// 0x8000'001A: Performance optimization identifiers
// 0x8000'001B: Instruction based sampling identifiers
// 0x8000'001C: Lightweight profiling capabilities
// 0x8000'001D: Cache properties
#ifdef CPUID_AMD
// Deterministic cache parameters for each level
RegisterFunction(0x8000'001D, &CPUIDEmu::Function_8000_001Dh);
#endif
// 0x8000'001E: Extended APIC ID
// 0x8000'001F: AMD Secure Encryption
// Setup some state tracking
SetupHostHybridFlag();
}
+14 -173
View File
@@ -10,12 +10,9 @@
namespace FEXCore {
namespace Context {
class ContextImpl;
struct Context;
}
// Debugging define to switch what family of CPU we execute as.
// Might be useful if an application makes an assumption about a CPU.
// #define CPUID_AMD
class CPUIDEmu final {
private:
constexpr static uint32_t CPUID_VENDOR_INTEL1 = 0x756E6547; // "Genu"
@@ -31,27 +28,16 @@ public:
// if we report anything differently then applications are likely to break
constexpr static uint64_t CACHELINE_SIZE = 64;
void Init(FEXCore::Context::ContextImpl *ctx);
void Init(FEXCore::Context::Context *ctx);
FEXCore::CPUID::FunctionResults RunFunction(uint32_t Function, uint32_t Leaf) {
if (Function < Primary.size()) {
const auto Handler = Primary[Function];
return (this->*Handler)(Leaf);
const auto Handler = FunctionHandlers.find(Function);
if (Handler == FunctionHandlers.end()) {
return Function_Reserved(Leaf);
}
constexpr uint32_t HypervisorBase = 0x4000'0000;
if (Function >= HypervisorBase && Function < (HypervisorBase + Hypervisor.size())) {
const auto Handler = Hypervisor[Function - HypervisorBase];
return (this->*Handler)(Leaf);
}
constexpr uint32_t ExtendedBase = 0x8000'0000;
if (Function >= ExtendedBase && Function < (ExtendedBase + Extended.size())) {
const auto Handler = Extended[Function - ExtendedBase];
return (this->*Handler)(Leaf);
}
return Function_Reserved(Leaf);
return (this->*Handler->second)(Leaf);
}
FEXCore::CPUID::FunctionResults RunFunctionName(uint32_t Function, uint32_t Leaf, uint32_t CPU) {
@@ -64,12 +50,16 @@ public:
}
private:
FEXCore::Context::ContextImpl *CTX;
FEXCore::Context::Context *CTX;
bool Hybrid{};
FEX_CONFIG_OPT(Cores, THREADS);
FEX_CONFIG_OPT(HideHypervisorBit, HIDEHYPERVISORBIT);
using FunctionHandler = FEXCore::CPUID::FunctionResults (CPUIDEmu::*)(uint32_t Leaf);
void RegisterFunction(uint32_t Function, FunctionHandler Handler) {
FunctionHandlers.insert_or_assign(Function, Handler);
}
std::unordered_map<uint32_t, FunctionHandler> FunctionHandlers;
struct CPUData {
const char *ProductName{};
#ifdef _M_ARM_64
@@ -105,161 +95,12 @@ private:
FEXCore::CPUID::FunctionResults Function_8000_0006h(uint32_t Leaf);
FEXCore::CPUID::FunctionResults Function_8000_0007h(uint32_t Leaf);
FEXCore::CPUID::FunctionResults Function_8000_0008h(uint32_t Leaf);
FEXCore::CPUID::FunctionResults Function_8000_0009h(uint32_t Leaf);
FEXCore::CPUID::FunctionResults Function_8000_0019h(uint32_t Leaf);
FEXCore::CPUID::FunctionResults Function_8000_001Dh(uint32_t Leaf);
FEXCore::CPUID::FunctionResults Function_Reserved(uint32_t Leaf);
void SetupHostHybridFlag();
static constexpr std::array<FunctionHandler, 27> Primary = {
// 0: Highest function parameter and ID
&CPUIDEmu::Function_0h,
// 1: Processor info
&CPUIDEmu::Function_01h,
// 2: Cache and TLB info
&CPUIDEmu::Function_02h,
// 3: Serial Number(previously), now reserved
&CPUIDEmu::Function_Reserved,
#ifndef CPUID_AMD
// 4: Deterministic cache parameters for each level
&CPUIDEmu::Function_04h,
#else
&CPUIDEmu::Function_Reserved,
#endif
// 5: Monitor/mwait
&CPUIDEmu::Function_Reserved,
// 6: Thermal and power management
&CPUIDEmu::Function_06h,
// 7: Extended feature flags
&CPUIDEmu::Function_07h,
// 0x08: Reserved?
&CPUIDEmu::Function_Reserved,
// 9: Direct Cache Access information
&CPUIDEmu::Function_Reserved,
// 0x0A: Architectural performance monitoring
&CPUIDEmu::Function_Reserved,
// 0x0B: Extended topology enumeration
&CPUIDEmu::Function_Reserved,
// 0x0C: Reserved?
&CPUIDEmu::Function_Reserved,
// 0x0D: Processor extended state enumeration
&CPUIDEmu::Function_0Dh,
// 0x0E: Reserved?
&CPUIDEmu::Function_Reserved,
// 0x0F: Intel RDT monitoring
&CPUIDEmu::Function_Reserved,
// 0x10: Intel RDT allocation enumeration
&CPUIDEmu::Function_Reserved,
// 0x12: Reserved?
&CPUIDEmu::Function_Reserved,
// 0x12: Intel SGX capability enumeration
&CPUIDEmu::Function_Reserved,
// 0x13: Reserved
&CPUIDEmu::Function_Reserved,
// 0x14: Intel Processor trace
&CPUIDEmu::Function_Reserved,
#ifndef CPUID_AMD
// Timestamp counter information
// Doesn't exist on AMD hardware
&CPUIDEmu::Function_15h,
#else
&CPUIDEmu::Function_Reserved,
#endif
// 0x16: Processor frequency information
&CPUIDEmu::Function_Reserved,
// 0x17: SoC vendor attribute enumeration
&CPUIDEmu::Function_Reserved,
// 0x18: Reserved?
&CPUIDEmu::Function_Reserved,
// 0x19: Reserved?
&CPUIDEmu::Function_Reserved,
#ifndef CPUID_AMD
// 0x1A: Hybrid Information Sub-leaf
&CPUIDEmu::Function_1Ah,
#else
&CPUIDEmu::Function_Reserved,
#endif
};
static constexpr std::array<FunctionHandler, 2> Hypervisor = {
// Hypervisor CPUID information leaf
&CPUIDEmu::Function_4000_0000h,
// FEX-Emu specific leaf
&CPUIDEmu::Function_4000_0001h,
};
static constexpr std::array<FunctionHandler, 32> Extended = {
// Largest extended function number
&CPUIDEmu::Function_8000_0000h,
// Processor vendor
&CPUIDEmu::Function_8000_0001h,
// Processor brand string
&CPUIDEmu::Function_8000_0002h,
// Processor brand string continued
&CPUIDEmu::Function_8000_0003h,
// Processor brand string continued
&CPUIDEmu::Function_8000_0004h,
#ifdef CPUID_AMD
// 0x8000'0005: L1 Cache and TLB identifiers
&CPUIDEmu::Function_8000_0005h,
#else
&CPUIDEmu::Function_Reserved,
#endif
// 0x8000'0006: L2 Cache identifiers
&CPUIDEmu::Function_8000_0006h,
// 0x8000'0007: Advanced power management information
&CPUIDEmu::Function_8000_0007h,
// 0x8000'0008: Virtual and physical address sizes
&CPUIDEmu::Function_8000_0008h,
// 0x8000'0009: Reserved?
&CPUIDEmu::Function_Reserved,
// 0x8000'000A: SVM Revision
&CPUIDEmu::Function_Reserved,
// 0x8000'000B: Reserved?
&CPUIDEmu::Function_Reserved,
// 0x8000'000C: Reserved?
&CPUIDEmu::Function_Reserved,
// 0x8000'000D: Reserved?
&CPUIDEmu::Function_Reserved,
// 0x8000'000E: Reserved?
&CPUIDEmu::Function_Reserved,
// 0x8000'000F: Reserved?
&CPUIDEmu::Function_Reserved,
// 0x8000'0010: Reserved?
&CPUIDEmu::Function_Reserved,
// 0x8000'0011: Reserved?
&CPUIDEmu::Function_Reserved,
// 0x8000'0012: Reserved?
&CPUIDEmu::Function_Reserved,
// 0x8000'0013: Reserved?
&CPUIDEmu::Function_Reserved,
// 0x8000'0014: Reserved?
&CPUIDEmu::Function_Reserved,
// 0x8000'0015: Reserved?
&CPUIDEmu::Function_Reserved,
// 0x8000'0016: Reserved?
&CPUIDEmu::Function_Reserved,
// 0x8000'0017: Reserved?
&CPUIDEmu::Function_Reserved,
// 0x8000'0018: Reserved?
&CPUIDEmu::Function_Reserved,
// 0x8000'0019: TLB 1GB page identifiers
&CPUIDEmu::Function_8000_0019h,
// 0x8000'001A: Performance optimization identifiers
&CPUIDEmu::Function_Reserved,
// 0x8000'001B: Instruction based sampling identifiers
&CPUIDEmu::Function_Reserved,
// 0x8000'001C: Lightweight profiling capabilities
&CPUIDEmu::Function_Reserved,
#ifdef CPUID_AMD
// 0x8000'001D: Cache properties
&CPUIDEmu::Function_8000_001Dh,
#else
&CPUIDEmu::Function_Reserved,
#endif
// 0x8000'001E: Extended APIC ID
&CPUIDEmu::Function_Reserved,
// 0x8000'001F: AMD Secure Encryption
&CPUIDEmu::Function_Reserved,
};
};
}
+89 -102
View File
@@ -79,7 +79,7 @@ $end_info$
namespace FEXCore::CPU {
bool CreateCPUCore(Context::ContextImpl *CTX) {
bool CreateCPUCore(FEXCore::Context::Context *CTX) {
// This should be used for generating things that are shared between threads
CTX->CPUID.Init(CTX);
return true;
@@ -147,7 +147,7 @@ std::string_view const& GetGRegName(unsigned Reg) {
} // namespace FEXCore::Core
namespace FEXCore::Context {
ContextImpl::ContextImpl()
Context::Context()
: IRCaptureCache {this} {
#ifdef BLOCKSTATS
BlockData = std::make_unique<FEXCore::BlockSamplingData>();
@@ -159,11 +159,6 @@ namespace FEXCore::Context {
HostFeatures.SupportsAVX = false;
}
if (!Config.Is64BitMode()) {
// When operating in 32-bit mode, the virtual memory we care about is only the lower 32-bits.
Config.VirtualMemSize = 1ULL << 32;
}
if (Config.BlockJITNaming() ||
Config.GlobalJITNaming() ||
Config.LibraryJITNaming()) {
@@ -172,7 +167,7 @@ namespace FEXCore::Context {
}
}
ContextImpl::~ContextImpl() {
Context::~Context() {
{
if (CodeObjectCacheService) {
CodeObjectCacheService->Shutdown();
@@ -214,7 +209,7 @@ namespace FEXCore::Context {
return NewThreadState;
}
FEXCore::Core::InternalThreadState* ContextImpl::InitCore(uint64_t InitialRIP, uint64_t StackPointer) {
FEXCore::Core::InternalThreadState* Context::InitCore(uint64_t InitialRIP, uint64_t StackPointer) {
// Initialize the CPU core signal handlers & DispatcherConfig
switch (Config.Core) {
#ifdef INTERPRETER_ENABLED
@@ -255,13 +250,13 @@ namespace FEXCore::Context {
// Initialize common signal handlers
auto PauseHandler = [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext) -> bool {
return static_cast<ContextImpl*>(Thread->CTX)->Dispatcher->HandleSignalPause(Thread, Signal, info, ucontext);
return Thread->CTX->Dispatcher->HandleSignalPause(Thread, Signal, info, ucontext);
};
SignalDelegation->RegisterHostSignalHandler(SignalDelegator::SIGNAL_FOR_PAUSE, PauseHandler, true);
auto GuestSignalHandler = [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext, GuestSigAction *GuestAction, stack_t *GuestStack) -> bool {
return static_cast<ContextImpl*>(Thread->CTX)->Dispatcher->HandleGuestSignal(Thread, Signal, info, ucontext, GuestAction, GuestStack);
return Thread->CTX->Dispatcher->HandleGuestSignal(Thread, Signal, info, ucontext, GuestAction, GuestStack);
};
for (uint32_t Signal = 0; Signal <= SignalDelegator::MAX_SIGNALS; ++Signal) {
@@ -295,45 +290,30 @@ namespace FEXCore::Context {
return Thread;
}
void ContextImpl::StartGdbServer() {
void Context::StartGdbServer() {
if (!DebugServer) {
DebugServer = std::make_unique<GdbServer>(this);
StartPaused = true;
}
}
void ContextImpl::StopGdbServer() {
void Context::StopGdbServer() {
DebugServer.reset();
}
void ContextImpl::HandleCallback(FEXCore::Core::InternalThreadState *Thread, uint64_t RIP) {
static_cast<ContextImpl*>(Thread->CTX)->Dispatcher->ExecuteJITCallback(Thread->CurrentFrame, RIP);
void Context::HandleCallback(FEXCore::Core::InternalThreadState *Thread, uint64_t RIP) {
Thread->CTX->Dispatcher->ExecuteJITCallback(Thread->CurrentFrame, RIP);
}
void ContextImpl::HandleSignalHandlerReturn(bool RT) {
using SignalHandlerReturnFunc = void(*)();
SignalHandlerReturnFunc SignalHandlerReturn{};
if (RT) {
SignalHandlerReturn = reinterpret_cast<SignalHandlerReturnFunc>(Dispatcher->SignalHandlerReturnAddressRT);
}
else {
SignalHandlerReturn = reinterpret_cast<SignalHandlerReturnFunc>(Dispatcher->SignalHandlerReturnAddress);
}
SignalHandlerReturn();
FEX_UNREACHABLE;
}
void ContextImpl::RegisterHostSignalHandler(int Signal, HostSignalDelegatorFunction Func, bool Required) {
void Context::RegisterHostSignalHandler(int Signal, HostSignalDelegatorFunction Func, bool Required) {
SignalDelegation->RegisterHostSignalHandler(Signal, Func, Required);
}
void ContextImpl::RegisterFrontendHostSignalHandler(int Signal, HostSignalDelegatorFunction Func, bool Required) {
void Context::RegisterFrontendHostSignalHandler(int Signal, HostSignalDelegatorFunction Func, bool Required) {
SignalDelegation->RegisterFrontendHostSignalHandler(Signal, Func, Required);
}
void ContextImpl::WaitForIdle() {
void Context::WaitForIdle() {
std::unique_lock<std::mutex> lk(IdleWaitMutex);
IdleWaitCV.wait(lk, [this] {
return IdleWaitRefCount.load() == 0;
@@ -342,7 +322,7 @@ namespace FEXCore::Context {
Running = false;
}
void ContextImpl::WaitForIdleWithTimeout() {
void Context::WaitForIdleWithTimeout() {
std::unique_lock<std::mutex> lk(IdleWaitMutex);
bool WaitResult = IdleWaitCV.wait_for(lk, std::chrono::milliseconds(1500),
[this] {
@@ -360,7 +340,7 @@ namespace FEXCore::Context {
WaitForIdle();
}
void ContextImpl::NotifyPause() {
void Context::NotifyPause() {
// Tell all the threads that they should pause
std::lock_guard<std::mutex> lk(ThreadCreationMutex);
@@ -373,7 +353,7 @@ namespace FEXCore::Context {
}
}
void ContextImpl::Pause() {
void Context::Pause() {
// If we aren't running, WaitForIdle will never compete.
if (Running) {
NotifyPause();
@@ -382,7 +362,7 @@ namespace FEXCore::Context {
}
}
void ContextImpl::Run() {
void Context::Run() {
// Spin up all the threads
std::lock_guard<std::mutex> lk(ThreadCreationMutex);
for (auto &Thread : Threads) {
@@ -394,7 +374,7 @@ namespace FEXCore::Context {
}
}
void ContextImpl::WaitForThreadsToRun() {
void Context::WaitForThreadsToRun() {
size_t NumThreads{};
{
std::lock_guard<std::mutex> lk(ThreadCreationMutex);
@@ -410,7 +390,7 @@ namespace FEXCore::Context {
Running = true;
}
void ContextImpl::Step() {
void Context::Step() {
{
std::lock_guard<std::mutex> lk(ThreadCreationMutex);
// Walk the threads and tell them to clear their caches
@@ -430,7 +410,7 @@ namespace FEXCore::Context {
this->Config.MaxInstPerBlock = PreviousMaxIntPerBlock;
}
void ContextImpl::Stop(bool IgnoreCurrentThread) {
void Context::Stop(bool IgnoreCurrentThread) {
pid_t tid = FHU::Syscalls::gettid();
FEXCore::Core::InternalThreadState* CurrentThread{};
@@ -468,21 +448,21 @@ namespace FEXCore::Context {
}
}
void ContextImpl::StopThread(FEXCore::Core::InternalThreadState *Thread) {
void Context::StopThread(FEXCore::Core::InternalThreadState *Thread) {
if (Thread->RunningEvents.Running.exchange(false)) {
Thread->SignalReason.store(FEXCore::Core::SignalEvent::Stop);
FHU::Syscalls::tgkill(Thread->ThreadManager.PID, Thread->ThreadManager.TID, SignalDelegator::SIGNAL_FOR_PAUSE);
}
}
void ContextImpl::SignalThread(FEXCore::Core::InternalThreadState *Thread, FEXCore::Core::SignalEvent Event) {
void Context::SignalThread(FEXCore::Core::InternalThreadState *Thread, FEXCore::Core::SignalEvent Event) {
if (Thread->RunningEvents.Running.load()) {
Thread->SignalReason.store(Event);
FHU::Syscalls::tgkill(Thread->ThreadManager.PID, Thread->ThreadManager.TID, SignalDelegator::SIGNAL_FOR_PAUSE);
}
}
FEXCore::Context::ExitReason ContextImpl::RunUntilExit() {
FEXCore::Context::ExitReason Context::RunUntilExit() {
if(!StartPaused) {
// We will only have one thread at this point, but just in case run notify everything
std::lock_guard lk(ThreadCreationMutex);
@@ -503,16 +483,16 @@ namespace FEXCore::Context {
}
}
int ContextImpl::GetProgramStatus() const {
int Context::GetProgramStatus() const {
return ParentThread->StatusCode;
}
void ContextImpl::InitializeThreadData(FEXCore::Core::InternalThreadState *Thread) {
void Context::InitializeThreadData(FEXCore::Core::InternalThreadState *Thread) {
Thread->CPUBackend->Initialize();
}
struct ExecutionThreadHandler {
ContextImpl *This;
FEXCore::Context::Context *This;
FEXCore::Core::InternalThreadState *Thread;
};
@@ -523,7 +503,7 @@ namespace FEXCore::Context {
return nullptr;
}
void ContextImpl::InitializeThread(FEXCore::Core::InternalThreadState *Thread) {
void Context::InitializeThread(FEXCore::Core::InternalThreadState *Thread) {
// This will create the execution thread but it won't actually start executing
ExecutionThreadHandler *Arg = reinterpret_cast<ExecutionThreadHandler*>(FEXCore::Allocator::malloc(sizeof(ExecutionThreadHandler)));
Arg->This = this;
@@ -545,7 +525,7 @@ namespace FEXCore::Context {
}
}
void ContextImpl::InitializeThreadTLSData(FEXCore::Core::InternalThreadState *Thread) {
void Context::InitializeThreadTLSData(FEXCore::Core::InternalThreadState *Thread) {
// Let's do some initial bookkeeping here
Thread->ThreadManager.TID = FHU::Syscalls::gettid();
Thread->ThreadManager.PID = ::getpid();
@@ -553,12 +533,12 @@ namespace FEXCore::Context {
ThunkHandler->RegisterTLSState(Thread);
}
void ContextImpl::RunThread(FEXCore::Core::InternalThreadState *Thread) {
void Context::RunThread(FEXCore::Core::InternalThreadState *Thread) {
// Tell the thread to start executing
Thread->StartRunning.NotifyAll();
}
void ContextImpl::InitializeCompiler(FEXCore::Core::InternalThreadState* Thread) {
void Context::InitializeCompiler(FEXCore::Core::InternalThreadState* Thread) {
Thread->OpDispatcher = std::make_unique<FEXCore::IR::OpDispatchBuilder>(this);
Thread->OpDispatcher->SetMultiblock(Config.Multiblock);
Thread->LookupCache = std::make_unique<FEXCore::LookupCache>(this);
@@ -610,7 +590,7 @@ namespace FEXCore::Context {
}
}
FEXCore::Core::InternalThreadState* ContextImpl::CreateThread(FEXCore::Core::CPUState *NewThreadState, uint64_t ParentTID) {
FEXCore::Core::InternalThreadState* Context::CreateThread(FEXCore::Core::CPUState *NewThreadState, uint64_t ParentTID) {
FEXCore::Core::InternalThreadState *Thread = new FEXCore::Core::InternalThreadState{};
// Copy over the new thread state to the new object
@@ -632,7 +612,7 @@ namespace FEXCore::Context {
return Thread;
}
void ContextImpl::DestroyThread(FEXCore::Core::InternalThreadState *Thread) {
void Context::DestroyThread(FEXCore::Core::InternalThreadState *Thread) {
// remove new thread object
{
std::lock_guard lk(ThreadCreationMutex);
@@ -651,7 +631,7 @@ namespace FEXCore::Context {
delete Thread;
}
void ContextImpl::CleanupAfterFork(FEXCore::Core::InternalThreadState *LiveThread) {
void Context::CleanupAfterFork(FEXCore::Core::InternalThreadState *LiveThread) {
// This function is called after fork
// We need to cleanup some of the thread data that is dead
for (auto &DeadThread : Threads) {
@@ -690,11 +670,11 @@ namespace FEXCore::Context {
FEXCore::Threads::Thread::CleanupAfterFork();
}
void ContextImpl::AddBlockMapping(FEXCore::Core::InternalThreadState *Thread, uint64_t Address, void *Ptr) {
void Context::AddBlockMapping(FEXCore::Core::InternalThreadState *Thread, uint64_t Address, void *Ptr) {
Thread->LookupCache->AddBlockMapping(Address, Ptr);
}
void ContextImpl::ClearCodeCache(FEXCore::Core::InternalThreadState *Thread) {
void Context::ClearCodeCache(FEXCore::Core::InternalThreadState *Thread) {
FEXCORE_PROFILE_INSTANT("ClearCodeCache");
{
@@ -712,7 +692,7 @@ namespace FEXCore::Context {
static void IRDumper(FEXCore::Core::InternalThreadState *Thread, IR::IREmitter *IREmitter, uint64_t GuestRIP, IR::RegisterAllocationData* RA) {
FILE* f = nullptr;
bool CloseAfter = false;
const auto DumpIRStr = static_cast<ContextImpl*>(Thread->CTX)->Config.DumpIR();
const auto DumpIRStr = Thread->CTX->Config.DumpIR();
// DumpIRStr might be no if not dumping but ShouldDump is set in OpDisp
if (DumpIRStr =="stderr" || DumpIRStr =="no") {
@@ -739,7 +719,7 @@ namespace FEXCore::Context {
}
};
static void ValidateIR(ContextImpl *ctx, IR::IREmitter *IREmitter) {
static void ValidateIR(FEXCore::Context::Context *ctx, IR::IREmitter *IREmitter) {
// Convert to text, Parse, Convert to text again and make sure the texts match
std::stringstream out;
static auto compaction = IR::CreateIRCompaction(ctx->OpDispatcherAllocator);
@@ -763,7 +743,7 @@ namespace FEXCore::Context {
}
}
ContextImpl::GenerateIRResult ContextImpl::GenerateIR(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP, bool ExtendedDebugInfo) {
Context::GenerateIRResult Context::GenerateIR(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP, bool ExtendedDebugInfo) {
FEXCORE_PROFILE_SCOPED("GenerateIR");
Thread->OpDispatcher->ReownOrClaimBuffer();
@@ -790,7 +770,7 @@ namespace FEXCore::Context {
Thread->FrontendDecoder->DecodeInstructionsAtEntry(GuestCode, GuestRIP, [Thread](uint64_t BlockEntry, uint64_t Start, uint64_t Length) {
if (Thread->LookupCache->AddBlockExecutableRange(BlockEntry, Start, Length)) {
static_cast<ContextImpl*>(Thread->CTX)->SyscallHandler->MarkGuestExecutableRange(Start, Length);
Thread->CTX->SyscallHandler->MarkGuestExecutableRange(Start, Length);
}
});
@@ -810,6 +790,13 @@ namespace FEXCore::Context {
// Reset any block-specific state
Thread->OpDispatcher->StartNewBlock();
if (Config.x86dec_SynchronizeRIPOnAllBlocks) {
// Ensure the RIP is synchronized to the context on block entry.
// In the case of block linking, the RIP may not have synchronized.
auto NewRIP = Thread->OpDispatcher->_EntrypointOffset(Block.Entry - GuestRIP, GPRSize);
Thread->OpDispatcher->_StoreContext(GPRSize, IR::GPRClass, NewRIP, offsetof(FEXCore::Core::CPUState, rip));
}
uint64_t InstsInBlock = Block.NumInstructions;
for (size_t i = 0; i < InstsInBlock; ++i) {
@@ -901,14 +888,14 @@ namespace FEXCore::Context {
IR::IREmitter *IREmitter = Thread->OpDispatcher.get();
auto ShouldDump = static_cast<ContextImpl*>(Thread->CTX)->Config.DumpIR() != "no" || Thread->OpDispatcher->ShouldDump;
auto ShouldDump = Thread->CTX->Config.DumpIR() != "no" || Thread->OpDispatcher->ShouldDump;
// Debug
{
if (ShouldDump) {
IRDumper(Thread, IREmitter, GuestRIP, nullptr);
}
if (static_cast<ContextImpl*>(Thread->CTX)->Config.ValidateIRarser) {
if (Thread->CTX->Config.ValidateIRarser) {
ValidateIR(this, IREmitter);
}
}
@@ -938,7 +925,7 @@ namespace FEXCore::Context {
};
}
ContextImpl::CompileCodeResult ContextImpl::CompileCode(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP) {
Context::CompileCodeResult Context::CompileCode(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP) {
FEXCore::IR::IRListView *IRList {};
FEXCore::Core::DebugData *DebugData {};
FEXCore::IR::RegisterAllocationData::UniquePtr RAData {};
@@ -1010,10 +997,7 @@ namespace FEXCore::Context {
}
// Attempt to get the CPU backend to compile this code
return {
// FEX currently throws away the CPUBackend::CompiledCode object other than the entrypoint
// In the future with code caching getting wired up, we will pass the rest of the data forward.
// TODO: Pass the data forward when code caching is wired up to this.
.CompiledCode = Thread->CPUBackend->CompileCode(GuestRIP, IRList, DebugData, RAData.get(), GetGdbServerStatus()).BlockEntry,
.CompiledCode = Thread->CPUBackend->CompileCode(GuestRIP, IRList, DebugData, RAData.get(), GetGdbServerStatus()),
.IRData = IRList,
.DebugData = DebugData,
.RAData = std::move(RAData),
@@ -1023,7 +1007,7 @@ namespace FEXCore::Context {
};
}
void ContextImpl::CompileBlockJit(FEXCore::Core::CpuStateFrame *Frame, uint64_t GuestRIP) {
void Context::CompileBlockJit(FEXCore::Core::CpuStateFrame *Frame, uint64_t GuestRIP) {
auto NewBlock = CompileBlock(Frame, GuestRIP);
if (NewBlock == 0) {
@@ -1034,7 +1018,7 @@ namespace FEXCore::Context {
}
}
uintptr_t ContextImpl::CompileBlock(FEXCore::Core::CpuStateFrame *Frame, uint64_t GuestRIP) {
uintptr_t Context::CompileBlock(FEXCore::Core::CpuStateFrame *Frame, uint64_t GuestRIP) {
FEXCORE_PROFILE_SCOPED("CompileBlock");
auto Thread = Frame->Thread;
@@ -1134,7 +1118,7 @@ namespace FEXCore::Context {
return (uintptr_t)CodePtr;
}
void ContextImpl::ExecutionThread(FEXCore::Core::InternalThreadState *Thread) {
void Context::ExecutionThread(FEXCore::Core::InternalThreadState *Thread) {
Core::ThreadData.Thread = Thread;
Thread->ExitReason = FEXCore::Context::ExitReason::EXIT_WAITING;
@@ -1145,7 +1129,7 @@ namespace FEXCore::Context {
// Now notify the thread that we are initialized
Thread->ThreadWaiting.NotifyAll();
if (Thread != static_cast<ContextImpl*>(Thread->CTX)->ParentThread || StartPaused || Thread->StartPaused) {
if (Thread != Thread->CTX->ParentThread || StartPaused || Thread->StartPaused) {
// Parent thread doesn't need to wait to run
Thread->StartRunning.Wait();
}
@@ -1157,7 +1141,7 @@ namespace FEXCore::Context {
Thread->RunningEvents.Running = true;
static_cast<ContextImpl*>(Thread->CTX)->Dispatcher->ExecuteDispatch(Thread->CurrentFrame);
Thread->CTX->Dispatcher->ExecuteDispatch(Thread->CurrentFrame);
Thread->RunningEvents.Running = false;
}
@@ -1186,7 +1170,7 @@ namespace FEXCore::Context {
SignalDelegation->UninstallTLSState(Thread);
// If the parent thread is waiting to join, then we can't destroy our thread object
if (!Thread->DestroyedByParent && Thread != static_cast<ContextImpl*>(Thread->CTX)->ParentThread) {
if (!Thread->DestroyedByParent && Thread != Thread->CTX->ParentThread) {
Thread->CTX->DestroyThread(Thread);
}
}
@@ -1199,34 +1183,34 @@ namespace FEXCore::Context {
for (auto it = lower; it != upper; it++) {
for (auto Address: it->second) {
ContextImpl::ThreadRemoveCodeEntry(Thread, Address);
Context::ThreadRemoveCodeEntry(Thread, Address);
}
it->second.clear();
}
}
static void InvalidateGuestCodeRangeInternal(ContextImpl *CTX, uint64_t Start, uint64_t Length) {
std::lock_guard lk(static_cast<ContextImpl*>(CTX)->ThreadCreationMutex);
static void InvalidateGuestCodeRangeInternal(FEXCore::Context::Context *CTX, uint64_t Start, uint64_t Length) {
std::lock_guard lk(CTX->ThreadCreationMutex);
for (auto &Thread : static_cast<ContextImpl*>(CTX)->Threads) {
for (auto &Thread : CTX->Threads) {
InvalidateGuestThreadCodeRange(Thread, Start, Length);
}
}
void ContextImpl::InvalidateGuestCodeRange(uint64_t Start, uint64_t Length) {
FHU::ScopedSignalMaskWithUniqueLock CodeInvalidationLock(CodeInvalidationMutex);
void InvalidateGuestCodeRange(FEXCore::Context::Context *CTX, uint64_t Start, uint64_t Length) {
FHU::ScopedSignalMaskWithUniqueLock CodeInvalidationLock(CTX->CodeInvalidationMutex);
InvalidateGuestCodeRangeInternal(this, Start, Length);
InvalidateGuestCodeRangeInternal(CTX, Start, Length);
}
void ContextImpl::InvalidateGuestCodeRange(uint64_t Start, uint64_t Length, std::function<void(uint64_t start, uint64_t Length)> CallAfter) {
FHU::ScopedSignalMaskWithUniqueLock CodeInvalidationLock(CodeInvalidationMutex);
void InvalidateGuestCodeRange(FEXCore::Context::Context *CTX, uint64_t Start, uint64_t Length, std::function<void(uint64_t start, uint64_t Length)> CallAfter) {
FHU::ScopedSignalMaskWithUniqueLock CodeInvalidationLock(CTX->CodeInvalidationMutex);
InvalidateGuestCodeRangeInternal(this, Start, Length);
InvalidateGuestCodeRangeInternal(CTX, Start, Length);
CallAfter(Start, Length);
}
void ContextImpl::MarkMemoryShared() {
void Context::MarkMemoryShared() {
if (!IsMemoryShared) {
IsMemoryShared = true;
@@ -1246,14 +1230,18 @@ namespace FEXCore::Context {
}
}
void ContextImpl::ThreadAddBlockLink(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestDestination, uintptr_t HostLink, const std::function<void()> &delinker) {
std::shared_lock lk(static_cast<ContextImpl*>(Thread->CTX)->CodeInvalidationMutex);
void MarkMemoryShared(FEXCore::Context::Context *CTX) {
CTX->MarkMemoryShared();
}
void Context::ThreadAddBlockLink(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestDestination, uintptr_t HostLink, const std::function<void()> &delinker) {
std::shared_lock lk(Thread->CTX->CodeInvalidationMutex);
Thread->LookupCache->AddBlockLink(GuestDestination, HostLink, delinker);
}
void ContextImpl::ThreadRemoveCodeEntry(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP) {
LogMan::Throw::AFmt(static_cast<ContextImpl*>(Thread->CTX)->CodeInvalidationMutex.try_lock() == false, "CodeInvalidationMutex needs to be unique_locked here");
void Context::ThreadRemoveCodeEntry(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP) {
LogMan::Throw::AFmt(Thread->CTX->CodeInvalidationMutex.try_lock() == false, "CodeInvalidationMutex needs to be unique_locked here");
std::lock_guard<std::recursive_mutex> lk(Thread->LookupCache->WriteLock);
@@ -1261,7 +1249,7 @@ namespace FEXCore::Context {
Thread->LookupCache->Erase(GuestRIP);
}
CustomIRResult ContextImpl::AddCustomIREntrypoint(uintptr_t Entrypoint, std::function<void(uintptr_t Entrypoint, FEXCore::IR::IREmitter *)> Handler, void *Creator, void *Data) {
CustomIRResult Context::AddCustomIREntrypoint(uintptr_t Entrypoint, std::function<void(uintptr_t Entrypoint, FEXCore::IR::IREmitter *)> Handler, void *Creator, void *Data) {
LOGMAN_THROW_A_FMT(Config.Is64BitMode || !(Entrypoint >> 32), "64-bit Entrypoint in 32-bit mode {:x}", Entrypoint);
std::unique_lock lk(CustomIRMutex);
@@ -1277,12 +1265,12 @@ namespace FEXCore::Context {
}
}
void ContextImpl::RemoveCustomIREntrypoint(uintptr_t Entrypoint) {
void Context::RemoveCustomIREntrypoint(uintptr_t Entrypoint) {
LOGMAN_THROW_A_FMT(Config.Is64BitMode || !(Entrypoint >> 32), "64-bit Entrypoint in 32-bit mode {:x}", Entrypoint);
std::scoped_lock lk(CustomIRMutex);
InvalidateGuestCodeRange(Entrypoint, 1, [this](uint64_t Entrypoint, uint64_t) {
InvalidateGuestCodeRange(this, Entrypoint, 1, [this](uint64_t Entrypoint, uint64_t) {
CustomIRHandlers.erase(Entrypoint);
});
}
@@ -1292,26 +1280,24 @@ namespace FEXCore::Context {
uint64_t RIPBackup = Thread->CurrentFrame->State.rip;
Thread->CurrentFrame->State.rip = RIP;
auto CTX = static_cast<ContextImpl*>(Thread->CTX);
// Erase the RIP from all the storage backings if it exists
CTX->ThreadRemoveCodeEntry(Thread, RIP);
ThreadRemoveCodeEntry(Thread, RIP);
// We don't care if compilation passes or not
CTX->CompileBlock(Thread->CurrentFrame, RIP);
CompileBlock(Thread->CurrentFrame, RIP);
Thread->CurrentFrame->State.rip = RIPBackup;
}
uint64_t ContextImpl::GetThreadCount() const {
uint64_t Context::GetThreadCount() const {
return Threads.size();
}
FEXCore::Core::RuntimeStats *ContextImpl::GetRuntimeStatsForThread(uint64_t Thread) {
FEXCore::Core::RuntimeStats *Context::GetRuntimeStatsForThread(uint64_t Thread) {
return &Threads[Thread]->Stats;
}
bool ContextImpl::GetDebugDataForRIP(uint64_t RIP, FEXCore::Core::DebugData *Data) {
bool Context::GetDebugDataForRIP(uint64_t RIP, FEXCore::Core::DebugData *Data) {
std::lock_guard<std::recursive_mutex> lk(ParentThread->LookupCache->WriteLock);
auto it = ParentThread->DebugStore.find(RIP);
if (it == ParentThread->DebugStore.end()) {
@@ -1322,7 +1308,7 @@ namespace FEXCore::Context {
return true;
}
bool ContextImpl::FindHostCodeForRIP(uint64_t RIP, uint8_t **Code) {
bool Context::FindHostCodeForRIP(uint64_t RIP, uint8_t **Code) {
uintptr_t HostCode = ParentThread->LookupCache->FindBlock(RIP);
if (!HostCode) {
return false;
@@ -1338,7 +1324,7 @@ namespace FEXCore::Context {
return Result;
}
IR::AOTIRCacheEntry *ContextImpl::LoadAOTIRCacheEntry(const std::string &filename) {
IR::AOTIRCacheEntry *Context::LoadAOTIRCacheEntry(const std::string &filename) {
auto rv = IRCaptureCache.LoadAOTIRCacheEntry(filename);
if (DebugServer) {
DebugServer->AlertLibrariesChanged();
@@ -1346,18 +1332,19 @@ namespace FEXCore::Context {
return rv;
}
void ContextImpl::UnloadAOTIRCacheEntry(IR::AOTIRCacheEntry *Entry) {
void Context::UnloadAOTIRCacheEntry(IR::AOTIRCacheEntry *Entry) {
IRCaptureCache.UnloadAOTIRCacheEntry(Entry);
if (DebugServer) {
DebugServer->AlertLibrariesChanged();
}
}
void ContextImpl::AppendThunkDefinitions(std::vector<FEXCore::IR::ThunkDefinition> const& Definitions) {
void Context::AppendThunkDefinitions(std::vector<FEXCore::IR::ThunkDefinition> const& Definitions) {
ThunkHandler->AppendThunkDefinitions(Definitions);
}
void ContextImpl::ConfigureAOTGen(FEXCore::Core::InternalThreadState *Thread, std::set<uint64_t> *ExternalBranches, uint64_t SectionMaxAddress) {
void ConfigureAOTGen(FEXCore::Core::InternalThreadState *Thread, std::set<uint64_t> *ExternalBranches, uint64_t SectionMaxAddress) {
Thread->FrontendDecoder->SetExternalBranches(ExternalBranches);
Thread->FrontendDecoder->SetSectionMaxAddress(SectionMaxAddress);
}
+3 -3
View File
@@ -5,7 +5,7 @@ namespace FEXCore {
}
namespace FEXCore::Context {
class ContextImpl;
struct Context;
}
namespace FEXCore::CPU {
@@ -17,7 +17,7 @@ namespace FEXCore::CPU {
*
* @return true if core was able to be create
*/
bool CreateCPUCore(FEXCore::Context::ContextImpl *CTX);
bool CreateCPUCore(FEXCore::Context::Context *CTX);
bool LoadCode(FEXCore::Context::ContextImpl *CTX, FEXCore::CodeLoader *Loader);
bool LoadCode(FEXCore::Context::Context *CTX, FEXCore::CodeLoader *Loader);
}
@@ -35,7 +35,7 @@ namespace FEXCore::CPU {
constexpr size_t MAX_DISPATCHER_CODE_SIZE = 4096;
Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::ContextImpl *ctx, const DispatcherConfig &config)
Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const DispatcherConfig &config)
: FEXCore::CPU::Dispatcher(ctx, config), Arm64Emitter(ctx, MAX_DISPATCHER_CODE_SIZE)
#ifdef VIXL_SIMULATOR
, Simulator {&Decoder}
@@ -155,16 +155,14 @@ void Arm64Dispatcher::EmitDispatcher() {
// Shift the offset by the size of the block cache entry
add(ARMEmitter::XReg::x0, ARMEmitter::XReg::x0, ARMEmitter::XReg::x1, ARMEmitter::ShiftType::LSL, (int)log2(sizeof(FEXCore::LookupCache::LookupCacheEntry)));
// The the full LookupCacheEntry with a single LDP.
// Check the guest address first to ensure it maps to the address we are currently at.
// Load the guest address first to ensure it maps to the address we are currently at
// This fixes aliasing problems
ldp<ARMEmitter::IndexType::OFFSET>(ARMEmitter::XReg::x3, ARMEmitter::XReg::x1, ARMEmitter::Reg::r0, 0);
// If the guest address doesn't match, Compile the block.
ldr(ARMEmitter::XReg::x1, ARMEmitter::Reg::r0, offsetof(FEXCore::LookupCache::LookupCacheEntry, GuestCode));
cmp(ARMEmitter::XReg::x1, RipReg);
b(ARMEmitter::Condition::CC_NE, &NoBlock);
// Check the host address to see if it matches, else compile the block.
// Now load the actual host block to execute if we can
ldr(ARMEmitter::XReg::x3, ARMEmitter::Reg::r0, offsetof(FEXCore::LookupCache::LookupCacheEntry, HostCode));
cbz(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r3, &NoBlock);
// If we've made it here then we have a real compiled block
@@ -320,14 +318,6 @@ void Arm64Dispatcher::EmitDispatcher() {
hlt(0);
}
{
SignalHandlerReturnAddressRT = GetCursorAddress<uint64_t>();
// Now to get back to our old location we need to do a fault dance
// We can't use SIGTRAP here since gdb catches it and never gives it to the application!
hlt(0);
}
{
// Guest SIGILL handler
// Needs to be distinct from the SignalHandlerReturnAddress
@@ -578,10 +568,10 @@ size_t Arm64Dispatcher::GenerateGDBPauseCheck(uint8_t *CodeBuffer, uint64_t Gues
// If we have a gdb server running then run in a less efficient mode that checks if we need to exit
// This happens when single stepping
static_assert(sizeof(FEXCore::Context::ContextImpl::Config.RunningMode) == 4, "This is expected to be size of 4");
static_assert(sizeof(FEXCore::Context::Context::Config.RunningMode) == 4, "This is expected to be size of 4");
emit.ldr(ARMEmitter::XReg::x0, STATE_PTR(CpuStateFrame, Thread));
emit.ldr(ARMEmitter::XReg::x0, ARMEmitter::Reg::r0, offsetof(FEXCore::Core::InternalThreadState, CTX)); // Get Context
emit.ldr(ARMEmitter::WReg::w0, ARMEmitter::Reg::r0, offsetof(FEXCore::Context::ContextImpl, Config.RunningMode));
emit.ldr(ARMEmitter::WReg::w0, ARMEmitter::Reg::r0, offsetof(FEXCore::Context::Context, Config.RunningMode));
// If the value == 0 then we don't need to stop
emit.cbz(ARMEmitter::Size::i32Bit, ARMEmitter::Reg::r0, &RunBlock);
@@ -662,7 +652,6 @@ void Arm64Dispatcher::InitThreadPointers(FEXCore::Core::InternalThreadState *Thr
Common.GuestSignal_SIGTRAP = GuestSignal_SIGTRAP;
Common.GuestSignal_SIGSEGV = GuestSignal_SIGSEGV;
Common.SignalReturnHandler = SignalHandlerReturnAddress;
Common.SignalReturnHandlerRT = SignalHandlerReturnAddressRT;
auto &AArch64 = Thread->CurrentFrame->Pointers.AArch64;
AArch64.LUDIVHandler = LUDIVHandlerAddress;
@@ -672,7 +661,7 @@ void Arm64Dispatcher::InitThreadPointers(FEXCore::Core::InternalThreadState *Thr
}
}
std::unique_ptr<Dispatcher> Dispatcher::CreateArm64(FEXCore::Context::ContextImpl *CTX, const DispatcherConfig &Config) {
std::unique_ptr<Dispatcher> Dispatcher::CreateArm64(FEXCore::Context::Context *CTX, const DispatcherConfig &Config) {
return std::make_unique<Arm64Dispatcher>(CTX, Config);
}
@@ -7,6 +7,10 @@
#include <aarch64/simulator-aarch64.h>
#endif
namespace FEXCore::Context {
struct Context;
}
namespace FEXCore::Core {
struct InternalThreadState;
}
@@ -18,7 +22,7 @@ namespace FEXCore::CPU {
class Arm64Dispatcher final : public Dispatcher, public Arm64Emitter {
public:
Arm64Dispatcher(FEXCore::Context::ContextImpl *ctx, const DispatcherConfig &config);
Arm64Dispatcher(FEXCore::Context::Context *ctx, const DispatcherConfig &config);
void InitThreadPointers(FEXCore::Core::InternalThreadState *Thread) override;
size_t GenerateGDBPauseCheck(uint8_t *CodeBuffer, uint64_t GuestRIP) override;
size_t GenerateInterpreterTrampoline(uint8_t *CodeBuffer) override;
File diff suppressed because it is too large. Load diff
@@ -20,7 +20,7 @@ struct InternalThreadState;
}
namespace FEXCore::Context {
class ContextImpl;
struct Context;
}
namespace FEXCore::CPU {
@@ -44,7 +44,6 @@ public:
uint64_t ThreadPauseHandlerAddressSpillSRA{};
uint64_t ExitFunctionLinkerAddress{};
uint64_t SignalHandlerReturnAddress{};
uint64_t SignalHandlerReturnAddressRT{};
uint64_t GuestSignal_SIGILL{};
uint64_t GuestSignal_SIGTRAP{};
uint64_t GuestSignal_SIGSEGV{};
@@ -74,8 +73,8 @@ public:
virtual size_t GenerateGDBPauseCheck(uint8_t *CodeBuffer, uint64_t GuestRIP) = 0;
virtual size_t GenerateInterpreterTrampoline(uint8_t *CodeBuffer) = 0;
static std::unique_ptr<Dispatcher> CreateX86(FEXCore::Context::ContextImpl *CTX, const DispatcherConfig &Config);
static std::unique_ptr<Dispatcher> CreateArm64(FEXCore::Context::ContextImpl *CTX, const DispatcherConfig &Config);
static std::unique_ptr<Dispatcher> CreateX86(FEXCore::Context::Context *CTX, const DispatcherConfig &Config);
static std::unique_ptr<Dispatcher> CreateArm64(FEXCore::Context::Context *CTX, const DispatcherConfig &Config);
virtual void ExecuteDispatch(FEXCore::Core::CpuStateFrame *Frame) {
DispatchPtr(Frame);
@@ -86,83 +85,21 @@ public:
}
protected:
Dispatcher(FEXCore::Context::ContextImpl *ctx, const DispatcherConfig &Config)
Dispatcher(FEXCore::Context::Context *ctx, const DispatcherConfig &Config)
: CTX {ctx}
, config {Config}
{}
uint64_t ReconstructRIPFromContext(FEXCore::Core::CpuStateFrame *Frame, void *ucontext) const;
void RestoreFrame_x64(ArchHelpers::Context::ContextBackup* Context, FEXCore::Core::CpuStateFrame *Frame, void *ucontext);
void RestoreFrame_ia32(ArchHelpers::Context::ContextBackup* Context, FEXCore::Core::CpuStateFrame *Frame, void *ucontext);
void RestoreRTFrame_ia32(ArchHelpers::Context::ContextBackup* Context, FEXCore::Core::CpuStateFrame *Frame, void *ucontext);
///< Setup the signal frame for x64.
uint64_t SetupFrame_x64(FEXCore::Core::InternalThreadState *Thread, ArchHelpers::Context::ContextBackup* ContextBackup, FEXCore::Core::CpuStateFrame *Frame,
int Signal, siginfo_t *HostSigInfo, void *ucontext,
GuestSigAction *GuestAction, stack_t *GuestStack,
uint64_t NewGuestSP, const uint32_t eflags);
///< Setup the signal frame for a 32-bit signal without SA_SIGINFO.
uint64_t SetupFrame_ia32(ArchHelpers::Context::ContextBackup* ContextBackup, FEXCore::Core::CpuStateFrame *Frame,
int Signal, siginfo_t *HostSigInfo, void *ucontext,
GuestSigAction *GuestAction, stack_t *GuestStack,
uint64_t NewGuestSP, const uint32_t eflags);
///< Setup the signal frame for a 32-bit signal with SA_SIGINFO.
uint64_t SetupRTFrame_ia32(ArchHelpers::Context::ContextBackup* ContextBackup, FEXCore::Core::CpuStateFrame *Frame,
int Signal, siginfo_t *HostSigInfo, void *ucontext,
GuestSigAction *GuestAction, stack_t *GuestStack,
uint64_t NewGuestSP, const uint32_t eflags);
ArchHelpers::Context::ContextBackup* StoreThreadState(FEXCore::Core::InternalThreadState *Thread, int Signal, void *ucontext);
enum class RestoreType {
TYPE_REALTIME, ///< Signal restore type is from a `realtime` signal.
TYPE_NONREALTIME, ///< Signal restore type is from a `non-realtime` signal.
TYPE_PAUSE, ///< Signal restore type is from a GDB pause event.
};
/*
* Signal frames on 32-bit architecture needs to match exactly how the kernel generates the frame.
* This is because large parts of the signal frame definition is part of the UAPI.
* This means that when FEX sets up the signal frame, it needs to match the UAPI stack setup.
*
* The two signal stack frame types below describe the two different 32-bit frame types.
*/
// The 32-bit non-realtime signal frame.
// This frame type is used when the guest signal is used without the `SA_SIGINFO` flag.
struct SigFrame_i32 {
uint32_t pretcode; ///< sigreturn return branch point.
int32_t Signal; ///< The signal hit.
FEXCore::x86::sigcontext sc; ///< The signal context.
x86::_libc_fpstate fpstate_unused; ///< Unused fpstate. Retained for backwards compatibility.
uint32_t extramask[1]; ///< Upper 32-bits of the signal mask. Lower 32-bits is in the sigcontext.
char retcode[8]; ///< Unused but needs to be filled. GDB seemingly uses as a debug marker.
///< FP state now follows after this.
};
// The 32-bit realtime signal frame.
// This frame type is used when the guest signal is used with the `SA_SIGINFO` flag.
struct RTSigFrame_i32 {
uint32_t pretcode; ///< sigreturn return branch point.
int32_t Signal; ///< The signal hit.
uint32_t pinfo; ///< Pointer to siginfo_t
uint32_t puc; ///< Pointer to ucontext_t
FEXCore::x86::siginfo_t info;
FEXCore::x86::ucontext_t uc;
char retcode[8]; ///< Unused but needs to be filled. GDB seemingly uses as a debug marker.
///< FP state now follows after this.
};
void RestoreThreadState(FEXCore::Core::InternalThreadState *Thread, void *ucontext, RestoreType Type);
void RestoreThreadState(FEXCore::Core::InternalThreadState *Thread, void *ucontext);
std::stack<uint64_t, std::vector<uint64_t>> SignalFrames;
virtual void SpillSRA(FEXCore::Core::InternalThreadState *Thread, void *ucontext, uint32_t IgnoreMask) {}
FEXCore::Context::ContextImpl *CTX;
FEXCore::Context::Context *CTX;
DispatcherConfig config;
static void SleepThread(FEXCore::Context::ContextImpl *ctx, FEXCore::Core::CpuStateFrame *Frame);
static void SleepThread(FEXCore::Context::Context *ctx, FEXCore::Core::CpuStateFrame *Frame);
static uint64_t GetCompileBlockPtr();
@@ -27,7 +27,7 @@ namespace FEXCore::CPU {
static constexpr size_t MAX_DISPATCHER_CODE_SIZE = 4096;
#define STATE r14
X86Dispatcher::X86Dispatcher(FEXCore::Context::ContextImpl *ctx, const DispatcherConfig &config)
X86Dispatcher::X86Dispatcher(FEXCore::Context::Context *ctx, const DispatcherConfig &config)
: Dispatcher(ctx, config)
, Xbyak::CodeGenerator(MAX_DISPATCHER_CODE_SIZE,
FEXCore::Allocator::mmap(nullptr, MAX_DISPATCHER_CODE_SIZE, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0),
@@ -344,12 +344,6 @@ X86Dispatcher::X86Dispatcher(FEXCore::Context::ContextImpl *ctx, const Dispatche
ud2();
}
{
// RT Signal return handler
SignalHandlerReturnAddressRT = getCurr<uint64_t>();
ud2();
}
{
// Guest SIGILL handler
// Needs to be distinct from the SignalHandlerReturnAddress
@@ -433,7 +427,7 @@ size_t X86Dispatcher::GenerateGDBPauseCheck(uint8_t *CodeBuffer, uint64_t GuestR
emit.mov(rax, reinterpret_cast<uint64_t>(CTX));
// If the value == 0 then we don't need to stop
emit.cmp(dword [rax + (offsetof(FEXCore::Context::ContextImpl, Config.RunningMode))], 0);
emit.cmp(dword [rax + (offsetof(FEXCore::Context::Context, Config.RunningMode))], 0);
emit.je(RunBlock);
{
// Make sure RIP is syncronized to the context
@@ -492,14 +486,13 @@ void X86Dispatcher::InitThreadPointers(FEXCore::Core::InternalThreadState *Threa
Common.GuestSignal_SIGTRAP = GuestSignal_SIGTRAP;
Common.GuestSignal_SIGSEGV = GuestSignal_SIGSEGV;
Common.SignalReturnHandler = SignalHandlerReturnAddress;
Common.SignalReturnHandlerRT = SignalHandlerReturnAddressRT;
auto &Interpreter = Thread->CurrentFrame->Pointers.Interpreter;
(uintptr_t&)Interpreter.CallbackReturn = IntCallbackReturnAddress;
}
}
std::unique_ptr<Dispatcher> Dispatcher::CreateX86(FEXCore::Context::ContextImpl *CTX, const DispatcherConfig &Config) {
std::unique_ptr<Dispatcher> Dispatcher::CreateX86(FEXCore::Context::Context *CTX, const DispatcherConfig &Config) {
return std::make_unique<X86Dispatcher>(CTX, Config);
}
@@ -5,6 +5,10 @@
#define XBYAK64
#include <xbyak/xbyak.h>
namespace FEXCore::Context {
struct Context;
}
namespace FEXCore::Core {
struct InternalThreadState;
}
@@ -13,7 +17,7 @@ namespace FEXCore::CPU {
class X86Dispatcher final : public Dispatcher, public Xbyak::CodeGenerator {
public:
X86Dispatcher(FEXCore::Context::ContextImpl *ctx, const DispatcherConfig &config);
X86Dispatcher(FEXCore::Context::Context *ctx, const DispatcherConfig &config);
void InitThreadPointers(FEXCore::Core::InternalThreadState *Thread) override;
size_t GenerateGDBPauseCheck(uint8_t *CodeBuffer, uint64_t GuestRIP) override;
size_t GenerateInterpreterTrampoline(uint8_t *CodeBuffer) override;
+241 -135
View File
@@ -32,6 +32,26 @@ using namespace FEXCore::X86Tables;
static uint32_t MapModRMToReg(uint8_t REX, uint8_t bits, bool HighBits, bool HasREX, bool HasXMM, bool HasMM, uint8_t InvalidOffset = 16) {
using GPRArray = std::array<uint32_t, 16>;
static constexpr GPRArray GPRIndexes = {
// Classical ordering?
FEXCore::X86State::REG_RAX,
FEXCore::X86State::REG_RCX,
FEXCore::X86State::REG_RDX,
FEXCore::X86State::REG_RBX,
FEXCore::X86State::REG_RSP,
FEXCore::X86State::REG_RBP,
FEXCore::X86State::REG_RSI,
FEXCore::X86State::REG_RDI,
FEXCore::X86State::REG_R8,
FEXCore::X86State::REG_R9,
FEXCore::X86State::REG_R10,
FEXCore::X86State::REG_R11,
FEXCore::X86State::REG_R12,
FEXCore::X86State::REG_R13,
FEXCore::X86State::REG_R14,
FEXCore::X86State::REG_R15,
};
static constexpr GPRArray GPR8BitHighIndexes = {
// Classical ordering?
FEXCore::X86State::REG_RAX,
@@ -52,34 +72,112 @@ static uint32_t MapModRMToReg(uint8_t REX, uint8_t bits, bool HighBits, bool Has
FEXCore::X86State::REG_R15,
};
static constexpr GPRArray XMMIndexes = {
FEXCore::X86State::REG_XMM_0,
FEXCore::X86State::REG_XMM_1,
FEXCore::X86State::REG_XMM_2,
FEXCore::X86State::REG_XMM_3,
FEXCore::X86State::REG_XMM_4,
FEXCore::X86State::REG_XMM_5,
FEXCore::X86State::REG_XMM_6,
FEXCore::X86State::REG_XMM_7,
FEXCore::X86State::REG_XMM_8,
FEXCore::X86State::REG_XMM_9,
FEXCore::X86State::REG_XMM_10,
FEXCore::X86State::REG_XMM_11,
FEXCore::X86State::REG_XMM_12,
FEXCore::X86State::REG_XMM_13,
FEXCore::X86State::REG_XMM_14,
FEXCore::X86State::REG_XMM_15,
};
static constexpr GPRArray MMIndexes = {
FEXCore::X86State::REG_MM_0,
FEXCore::X86State::REG_MM_1,
FEXCore::X86State::REG_MM_2,
FEXCore::X86State::REG_MM_3,
FEXCore::X86State::REG_MM_4,
FEXCore::X86State::REG_MM_5,
FEXCore::X86State::REG_MM_6,
FEXCore::X86State::REG_MM_7,
FEXCore::X86State::REG_INVALID,
FEXCore::X86State::REG_INVALID,
FEXCore::X86State::REG_INVALID,
FEXCore::X86State::REG_INVALID,
FEXCore::X86State::REG_INVALID,
FEXCore::X86State::REG_INVALID,
FEXCore::X86State::REG_INVALID,
FEXCore::X86State::REG_INVALID
};
const GPRArray *GPRs = &GPRIndexes;
if (HasXMM) {
GPRs = &XMMIndexes;
}
else if (HasMM) {
GPRs = &MMIndexes;
}
else if (HighBits && !HasREX) {
GPRs = &GPR8BitHighIndexes;
}
uint8_t Offset = (REX << 3) | bits;
if (Offset == InvalidOffset) {
return FEXCore::X86State::REG_INVALID;
}
if (HasXMM) {
return FEXCore::X86State::REG_XMM_0 + Offset;
}
else if (HasMM) {
return FEXCore::X86State::REG_MM_0 + Offset;
}
else if (!(HighBits && !HasREX)) {
return FEXCore::X86State::REG_RAX + Offset;
}
return GPR8BitHighIndexes[Offset];
return (*GPRs)[(REX << 3) | bits];
}
static uint32_t MapVEXToReg(uint8_t vvvv, bool HasXMM) {
using GPRArray = std::array<uint32_t, 16>;
static constexpr GPRArray GPRIndexes = {
FEXCore::X86State::REG_RAX,
FEXCore::X86State::REG_RCX,
FEXCore::X86State::REG_RDX,
FEXCore::X86State::REG_RBX,
FEXCore::X86State::REG_RSP,
FEXCore::X86State::REG_RBP,
FEXCore::X86State::REG_RSI,
FEXCore::X86State::REG_RDI,
FEXCore::X86State::REG_R8,
FEXCore::X86State::REG_R9,
FEXCore::X86State::REG_R10,
FEXCore::X86State::REG_R11,
FEXCore::X86State::REG_R12,
FEXCore::X86State::REG_R13,
FEXCore::X86State::REG_R14,
FEXCore::X86State::REG_R15,
};
static constexpr GPRArray XMMIndexes = {
FEXCore::X86State::REG_XMM_0,
FEXCore::X86State::REG_XMM_1,
FEXCore::X86State::REG_XMM_2,
FEXCore::X86State::REG_XMM_3,
FEXCore::X86State::REG_XMM_4,
FEXCore::X86State::REG_XMM_5,
FEXCore::X86State::REG_XMM_6,
FEXCore::X86State::REG_XMM_7,
FEXCore::X86State::REG_XMM_8,
FEXCore::X86State::REG_XMM_9,
FEXCore::X86State::REG_XMM_10,
FEXCore::X86State::REG_XMM_11,
FEXCore::X86State::REG_XMM_12,
FEXCore::X86State::REG_XMM_13,
FEXCore::X86State::REG_XMM_14,
FEXCore::X86State::REG_XMM_15,
};
if (HasXMM) {
return FEXCore::X86State::REG_XMM_0 + vvvv;
return XMMIndexes[vvvv];
} else {
return FEXCore::X86State::REG_RAX + vvvv;
return GPRIndexes[vvvv];
}
}
Decoder::Decoder(FEXCore::Context::ContextImpl *ctx)
Decoder::Decoder(FEXCore::Context::Context *ctx)
: CTX {ctx}
, OSABI { ctx->SyscallHandler ? ctx->SyscallHandler->GetOSABI() : FEXCore::HLE::SyscallOSABI::OS_UNKNOWN }
, PoolObject {ctx->FrontendAllocator, sizeof(FEXCore::X86Tables::DecodedInst) * DefaultDecodedBufferSize} {
@@ -108,7 +206,7 @@ uint64_t Decoder::ReadData(uint8_t Size) {
uint64_t Res = 0;
std::memcpy(&Res, &InstStream[InstructionSize], Size);
#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
#ifndef NDEBUG
for(size_t i = 0; i < Size; ++i) {
ReadByte();
}
@@ -286,6 +384,12 @@ bool Decoder::NormalOp(FEXCore::X86Tables::X86InstInfo const *Info, uint16_t Op,
DecodeInst->OP = Op;
DecodeInst->TableInfo = Info;
// XXX: Once we support 32bit x86 then this will be necessary to support
if (Info->Type == FEXCore::X86Tables::TYPE_LEGACY_PREFIX) {
LogMan::Msg::DFmt("Legacy Prefix");
return false;
}
if (Info->Type == FEXCore::X86Tables::TYPE_UNKNOWN) {
LogMan::Msg::DFmt("Unknown instruction: {} 0x{:04x} 0x{:x}", Info->Name ?: "UND", Op, DecodeInst->PC);
return false;
@@ -321,17 +425,10 @@ bool Decoder::NormalOp(FEXCore::X86Tables::X86InstInfo const *Info, uint16_t Op,
const bool HasMODRM = !!(Info->Flags & FEXCore::X86Tables::InstFlags::FLAGS_MODRM);
const bool HasREX = !!(DecodeInst->Flags & DecodeFlags::FLAG_REX_PREFIX);
const bool HasHighXMM = HAS_XMM_SUBFLAG(Info->Flags, FEXCore::X86Tables::InstFlags::FLAGS_SF_HIGH_XMM_REG);
const bool Has16BitAddressing = !CTX->Config.Is64BitMode &&
DecodeInst->Flags & DecodeFlags::FLAG_ADDRESS_SIZE;
// This is used for ModRM register modification
// For both modrm.reg and modrm.rm(when mod == 0b11) when value is >= 0b100
// then it changes from expected registers to the high 8bits of the lower registers
// Bit annoying to support
// In the case of no modrm (REX in byte situation) then it is unaffected
bool Is8BitSrc{};
bool Is8BitDest{};
// If we require ModRM and haven't decoded it yet, do it now
// Some instructions have to read modrm upfront, others do it later
if (HasMODRM && !DecodeInst->DecodedModRM) {
@@ -348,7 +445,6 @@ bool Decoder::NormalOp(FEXCore::X86Tables::X86InstInfo const *Info, uint16_t Op,
if (DstSizeFlag == FEXCore::X86Tables::InstFlags::SIZE_8BIT) {
DecodeInst->Flags |= DecodeFlags::GenSizeDstSize(DecodeFlags::SIZE_8BIT);
DestSize = 1;
Is8BitDest = true;
}
else if (DstSizeFlag == FEXCore::X86Tables::InstFlags::SIZE_16BIT) {
DecodeInst->Flags |= DecodeFlags::GenSizeDstSize(DecodeFlags::SIZE_16BIT);
@@ -391,7 +487,6 @@ bool Decoder::NormalOp(FEXCore::X86Tables::X86InstInfo const *Info, uint16_t Op,
// Decode sources
if (SrcSizeFlag == FEXCore::X86Tables::InstFlags::SIZE_8BIT) {
DecodeInst->Flags |= DecodeFlags::GenSizeSrcSize(DecodeFlags::SIZE_8BIT);
Is8BitSrc = true;
}
else if (SrcSizeFlag == FEXCore::X86Tables::InstFlags::SIZE_16BIT) {
DecodeInst->Flags |= DecodeFlags::GenSizeSrcSize(DecodeFlags::SIZE_16BIT);
@@ -425,6 +520,14 @@ bool Decoder::NormalOp(FEXCore::X86Tables::X86InstInfo const *Info, uint16_t Op,
}
}
// This is used for ModRM register modification
// For both modrm.reg and modrm.rm(when mod == 0b11) when value is >= 0b100
// then it changes from expected registers to the high 8bits of the lower registers
// Bit annoying to support
// In the case of no modrm (REX in byte situation) then it is unaffected
const bool Is8BitSrc = (DecodeFlags::GetSizeSrcFlags(DecodeInst->Flags) == DecodeFlags::SIZE_8BIT);
const bool Is8BitDest = (DecodeFlags::GetSizeDstFlags(DecodeInst->Flags) == DecodeFlags::SIZE_8BIT);
auto *CurrentDest = &DecodeInst->Dest;
if (HAS_NON_XMM_SUBFLAG(Info->Flags, FEXCore::X86Tables::InstFlags::FLAGS_SF_DST_RAX) ||
@@ -435,7 +538,8 @@ bool Decoder::NormalOp(FEXCore::X86Tables::X86InstInfo const *Info, uint16_t Op,
CurrentDest->Data.GPR.GPR = HAS_NON_XMM_SUBFLAG(Info->Flags, FEXCore::X86Tables::InstFlags::FLAGS_SF_DST_RAX) ? FEXCore::X86State::REG_RAX : FEXCore::X86State::REG_RDX;
CurrentDest = &DecodeInst->Src[0];
}
else if (HAS_NON_XMM_SUBFLAG(Info->Flags, FEXCore::X86Tables::InstFlags::FLAGS_SF_REX_IN_BYTE)) {
if (HAS_NON_XMM_SUBFLAG(Info->Flags, FEXCore::X86Tables::InstFlags::FLAGS_SF_REX_IN_BYTE)) {
LOGMAN_THROW_AA_FMT(!HasMODRM, "This instruction shouldn't have ModRM!");
// If the REX is in the byte that means the lower nibble of the OP contains the destination GPR
@@ -443,7 +547,7 @@ bool Decoder::NormalOp(FEXCore::X86Tables::X86InstInfo const *Info, uint16_t Op,
// ADDITIONALLY:
// If there is a REX prefix then that allows extended GPR usage
CurrentDest->Type = DecodedOperand::OpType::GPR;
DecodeInst->Dest.Data.GPR.HighBits = (Is8BitDest && !HasREX && (Op & 0b111) >= 0b100);
DecodeInst->Dest.Data.GPR.HighBits = (Is8BitDest && !HasREX && (Op & 0b111) >= 0b100) || HasHighXMM;
CurrentDest->Data.GPR.GPR = MapModRMToReg(DecodeInst->Flags & DecodeFlags::FLAG_REX_XGPR_B ? 1 : 0, Op & 0b111, Is8BitDest, HasREX, false, false);
if (CurrentDest->Data.GPR.GPR == FEXCore::X86State::REG_INVALID)
@@ -471,7 +575,7 @@ bool Decoder::NormalOp(FEXCore::X86Tables::X86InstInfo const *Info, uint16_t Op,
// Decode the GPR source first
GPR.Type = DecodedOperand::OpType::GPR;
GPR.Data.GPR.HighBits = (GPR8Bit && ModRM.reg >= 0b100 && !HasREX);
GPR.Data.GPR.HighBits = (GPR8Bit && ModRM.reg >= 0b100 && !HasREX) || HasHighXMM;
GPR.Data.GPR.GPR = MapModRMToReg(DecodeInst->Flags & DecodeFlags::FLAG_REX_XGPR_R ? 1 : 0, ModRM.reg, GPR8Bit, HasREX, HasXMMGPR, HasMMGPR);
if (GPR.Data.GPR.GPR == FEXCore::X86State::REG_INVALID)
@@ -481,7 +585,7 @@ bool Decoder::NormalOp(FEXCore::X86Tables::X86InstInfo const *Info, uint16_t Op,
// ModRM.Mod != 0b11 == Register-direct addressing
if (ModRM.mod == 0b11) {
NonGPR.Type = DecodedOperand::OpType::GPR;
NonGPR.Data.GPR.HighBits = (NonGPR8Bit && ModRM.rm >= 0b100 && !HasREX);
NonGPR.Data.GPR.HighBits = (NonGPR8Bit && ModRM.rm >= 0b100 && !HasREX) || HasHighXMM;
NonGPR.Data.GPR.GPR = MapModRMToReg(DecodeInst->Flags & DecodeFlags::FLAG_REX_XGPR_B ? 1 : 0, ModRM.rm, NonGPR8Bit, HasREX, HasXMMNonGPR, HasMMNonGPR);
if (NonGPR.Data.GPR.GPR == FEXCore::X86State::REG_INVALID)
return false;
@@ -580,6 +684,12 @@ bool Decoder::NormalOpHeader(FEXCore::X86Tables::X86InstInfo const *Info, uint16
DecodeInst->OP = Op;
DecodeInst->TableInfo = Info;
// XXX: Once we support 32bit x86 then this will be necessary to support
if (Info->Type == FEXCore::X86Tables::TYPE_LEGACY_PREFIX) {
LogMan::Msg::DFmt("Legacy Prefix");
return false;
}
if (Info->Type == FEXCore::X86Tables::TYPE_UNKNOWN) {
LogMan::Msg::DFmt("Unknown instruction: {} 0x{:04x} 0x{:x}", Info->Name ?: "UND", Op, DecodeInst->PC);
return false;
@@ -593,11 +703,7 @@ bool Decoder::NormalOpHeader(FEXCore::X86Tables::X86InstInfo const *Info, uint16
LOGMAN_THROW_AA_FMT(Info->Type != FEXCore::X86Tables::TYPE_REX_PREFIX,
"REX PREFIX should have been decoded before this!");
// A normal instruction is the most likely.
if (Info->Type == FEXCore::X86Tables::TYPE_INST) [[likely]] {
return NormalOp(Info, Op);
}
else if (Info->Type >= FEXCore::X86Tables::TYPE_GROUP_1 &&
if (Info->Type >= FEXCore::X86Tables::TYPE_GROUP_1 &&
Info->Type <= FEXCore::X86Tables::TYPE_GROUP_11) {
uint8_t ModRMByte = ReadByte();
DecodeInst->ModRM = ModRMByte;
@@ -745,8 +851,7 @@ bool Decoder::NormalOpHeader(FEXCore::X86Tables::X86InstInfo const *Info, uint16
return NormalOp(&EVEXTableOps[EVEXOp], EVEXOp);
}
LOGMAN_MSG_A_FMT("Invalid instruction decoding type");
FEX_UNREACHABLE;
return NormalOp(Info, Op);
}
bool Decoder::DecodeInstruction(uint64_t PC) {
@@ -765,106 +870,105 @@ bool Decoder::DecodeInstruction(uint64_t PC) {
case 0x0F: {// Escape Op
uint8_t EscapeOp = ReadByte();
switch (EscapeOp) {
case 0x0F: [[unlikely]] { // 3DNow!
// 3DNow! Instruction Encoding: 0F 0F [ModRM] [SIB] [Displacement] [Opcode]
// Decode ModRM
uint8_t ModRMByte = ReadByte();
DecodeInst->ModRM = ModRMByte;
DecodeInst->DecodedModRM = true;
case 0x0F: [[unlikely]] { // 3DNow!
// 3DNow! Instruction Encoding: 0F 0F [ModRM] [SIB] [Displacement] [Opcode]
// Decode ModRM
uint8_t ModRMByte = ReadByte();
DecodeInst->ModRM = ModRMByte;
DecodeInst->DecodedModRM = true;
FEXCore::X86Tables::ModRMDecoded ModRM;
ModRM.Hex = DecodeInst->ModRM;
FEXCore::X86Tables::ModRMDecoded ModRM;
ModRM.Hex = DecodeInst->ModRM;
const bool Has16BitAddressing = !CTX->Config.Is64BitMode &&
DecodeInst->Flags & DecodeFlags::FLAG_ADDRESS_SIZE;
const bool Has16BitAddressing = !CTX->Config.Is64BitMode &&
DecodeInst->Flags & DecodeFlags::FLAG_ADDRESS_SIZE;
// All 3DNow! instructions have the second argument as the rm handler
// We need to decode it upfront to get the displacement out of the way
if (ModRM.mod != 0b11) {
auto Disp = DecodeModRMs_Disp[Has16BitAddressing];
(this->*Disp)(&DecodeInst->Src[0], ModRM);
}
// Take a peek at the op just past the displacement
uint8_t LocalOp = ReadByte();
return NormalOpHeader(&FEXCore::X86Tables::DDDNowOps[LocalOp], LocalOp);
break;
// All 3DNow! instructions have the second argument as the rm handler
// We need to decode it upfront to get the displacement out of the way
if (ModRM.mod != 0b11) {
auto Disp = DecodeModRMs_Disp[Has16BitAddressing];
(this->*Disp)(&DecodeInst->Src[0], ModRM);
}
case 0x38: { // F38 Table!
constexpr uint16_t PF_38_NONE = 0;
constexpr uint16_t PF_38_66 = (1U << 0);
constexpr uint16_t PF_38_F2 = (1U << 1);
constexpr uint16_t PF_38_F3 = (1U << 2);
uint16_t Prefix = PF_38_NONE;
if (DecodeInst->Flags & DecodeFlags::FLAG_OPERAND_SIZE) {
Prefix |= PF_38_66;
}
if (DecodeInst->Flags & DecodeFlags::FLAG_REPNE_PREFIX) {
Prefix |= PF_38_F2;
}
if (DecodeInst->Flags & DecodeFlags::FLAG_REP_PREFIX) {
Prefix |= PF_38_F3;
}
// Take a peek at the op just past the displacement
uint8_t LocalOp = ReadByte();
return NormalOpHeader(&FEXCore::X86Tables::DDDNowOps[LocalOp], LocalOp);
break;
}
case 0x38: { // F38 Table!
constexpr uint16_t PF_38_NONE = 0;
constexpr uint16_t PF_38_66 = (1U << 0);
constexpr uint16_t PF_38_F2 = (1U << 1);
constexpr uint16_t PF_38_F3 = (1U << 2);
uint16_t LocalOp = (Prefix << 8) | ReadByte();
return NormalOpHeader(&FEXCore::X86Tables::H0F38TableOps[LocalOp], LocalOp);
break;
uint16_t Prefix = PF_38_NONE;
if (DecodeInst->Flags & DecodeFlags::FLAG_OPERAND_SIZE) {
Prefix |= PF_38_66;
}
case 0x3A: { // F3A Table!
constexpr uint16_t PF_3A_NONE = 0;
constexpr uint16_t PF_3A_66 = (1 << 0);
constexpr uint16_t PF_3A_REX = (1 << 1);
uint16_t Prefix = PF_3A_NONE;
if (DecodeInst->LastEscapePrefix == 0x66) // Operand Size
Prefix = PF_3A_66;
if (DecodeInst->Flags & DecodeFlags::FLAG_REX_WIDENING)
Prefix |= PF_3A_REX;
uint16_t LocalOp = (Prefix << 8) | ReadByte();
return NormalOpHeader(&FEXCore::X86Tables::H0F3ATableOps[LocalOp], LocalOp);
break;
if (DecodeInst->Flags & DecodeFlags::FLAG_REPNE_PREFIX) {
Prefix |= PF_38_F2;
}
default: [[likely]] { // Two byte table!
// x86-64 abuses three legacy prefixes to extend the table encodings
// 0x66 - Operand Size prefix
// 0xF2 - REPNE prefix
// 0xF3 - REP prefix
// If any of these three prefixes are used then it falls down the subtable
// Additionally: If you hit repeat of differnt prefixes then only the LAST one before this one works for subtable selection
bool NoOverlay = (FEXCore::X86Tables::SecondBaseOps[EscapeOp].Flags & InstFlags::FLAGS_NO_OVERLAY) != 0;
bool NoOverlay66 = (FEXCore::X86Tables::SecondBaseOps[EscapeOp].Flags & InstFlags::FLAGS_NO_OVERLAY66) != 0;
if (NoOverlay) { // This section of the table ignores prefix extention
return NormalOpHeader(&FEXCore::X86Tables::SecondBaseOps[EscapeOp], EscapeOp);
}
else if (DecodeInst->LastEscapePrefix == 0xF3) { // REP
// Remove prefix so it doesn't effect calculations.
// This is only an escape prefix rather tan modifier now
DecodeInst->Flags &= ~DecodeFlags::FLAG_REP_PREFIX;
return NormalOpHeader(&FEXCore::X86Tables::RepModOps[EscapeOp], EscapeOp);
}
else if (DecodeInst->LastEscapePrefix == 0xF2) { // REPNE
// Remove prefix so it doesn't effect calculations.
// This is only an escape prefix rather tan modifier now
DecodeInst->Flags &= ~DecodeFlags::FLAG_REPNE_PREFIX;
return NormalOpHeader(&FEXCore::X86Tables::RepNEModOps[EscapeOp], EscapeOp);
}
else if (DecodeInst->LastEscapePrefix == 0x66 && !NoOverlay66) { // Operand Size
// Remove prefix so it doesn't effect calculations.
// This is only an escape prefix rather tan modifier now
DecodeInst->Flags &= ~DecodeFlags::FLAG_OPERAND_SIZE;
DecodeFlags::PopOpAddrIf(&DecodeInst->Flags, DecodeFlags::FLAG_OPERAND_SIZE_LAST);
return NormalOpHeader(&FEXCore::X86Tables::OpSizeModOps[EscapeOp], EscapeOp);
}
else {
return NormalOpHeader(&FEXCore::X86Tables::SecondBaseOps[EscapeOp], EscapeOp);
}
break;
if (DecodeInst->Flags & DecodeFlags::FLAG_REP_PREFIX) {
Prefix |= PF_38_F3;
}
uint16_t LocalOp = (Prefix << 8) | ReadByte();
return NormalOpHeader(&FEXCore::X86Tables::H0F38TableOps[LocalOp], LocalOp);
break;
}
case 0x3A: { // F3A Table!
constexpr uint16_t PF_3A_NONE = 0;
constexpr uint16_t PF_3A_66 = (1 << 0);
constexpr uint16_t PF_3A_REX = (1 << 1);
uint16_t Prefix = PF_3A_NONE;
if (DecodeInst->LastEscapePrefix == 0x66) // Operand Size
Prefix = PF_3A_66;
if (DecodeInst->Flags & DecodeFlags::FLAG_REX_WIDENING)
Prefix |= PF_3A_REX;
uint16_t LocalOp = (Prefix << 8) | ReadByte();
return NormalOpHeader(&FEXCore::X86Tables::H0F3ATableOps[LocalOp], LocalOp);
break;
}
default: // Two byte table!
// x86-64 abuses three legacy prefixes to extend the table encodings
// 0x66 - Operand Size prefix
// 0xF2 - REPNE prefix
// 0xF3 - REP prefix
// If any of these three prefixes are used then it falls down the subtable
// Additionally: If you hit repeat of differnt prefixes then only the LAST one before this one works for subtable selection
bool NoOverlay = (FEXCore::X86Tables::SecondBaseOps[EscapeOp].Flags & InstFlags::FLAGS_NO_OVERLAY) != 0;
bool NoOverlay66 = (FEXCore::X86Tables::SecondBaseOps[EscapeOp].Flags & InstFlags::FLAGS_NO_OVERLAY66) != 0;
if (NoOverlay) { // This section of the table ignores prefix extention
return NormalOpHeader(&FEXCore::X86Tables::SecondBaseOps[EscapeOp], EscapeOp);
}
else if (DecodeInst->LastEscapePrefix == 0xF3) { // REP
// Remove prefix so it doesn't effect calculations.
// This is only an escape prefix rather tan modifier now
DecodeInst->Flags &= ~DecodeFlags::FLAG_REP_PREFIX;
return NormalOpHeader(&FEXCore::X86Tables::RepModOps[EscapeOp], EscapeOp);
}
else if (DecodeInst->LastEscapePrefix == 0xF2) { // REPNE
// Remove prefix so it doesn't effect calculations.
// This is only an escape prefix rather tan modifier now
DecodeInst->Flags &= ~DecodeFlags::FLAG_REPNE_PREFIX;
return NormalOpHeader(&FEXCore::X86Tables::RepNEModOps[EscapeOp], EscapeOp);
}
else if (DecodeInst->LastEscapePrefix == 0x66 && !NoOverlay66) { // Operand Size
// Remove prefix so it doesn't effect calculations.
// This is only an escape prefix rather tan modifier now
DecodeInst->Flags &= ~DecodeFlags::FLAG_OPERAND_SIZE;
DecodeFlags::PopOpAddrIf(&DecodeInst->Flags, DecodeFlags::FLAG_OPERAND_SIZE_LAST);
return NormalOpHeader(&FEXCore::X86Tables::OpSizeModOps[EscapeOp], EscapeOp);
}
else {
return NormalOpHeader(&FEXCore::X86Tables::SecondBaseOps[EscapeOp], EscapeOp);
}
break;
}
break;
}
@@ -917,7 +1021,7 @@ bool Decoder::DecodeInstruction(uint64_t PC) {
case 0x65: // GS prefix
DecodeInst->Flags |= DecodeFlags::FLAG_GS_PREFIX;
break;
default: [[likely]] { // Default base table
default: { // Default base table
auto Info = &FEXCore::X86Tables::BaseOps[Op];
if (Info->Type == FEXCore::X86Tables::TYPE_REX_PREFIX) {
@@ -1136,19 +1240,24 @@ void Decoder::DecodeInstructionsAtEntry(uint8_t const* _InstStream, uint64_t PC,
auto OpMinPage = OpMinAddress & FHU::FEX_PAGE_MASK;
auto OpMaxPage = OpMaxAddress & FHU::FEX_PAGE_MASK;
if (OpMinPage != CurrentCodePage) {
CurrentCodePage = OpMinPage;
CodePages.insert(CurrentCodePage);
if (CodePages.insert(CurrentCodePage).second) {
AddContainedCodePage(PC, CurrentCodePage, FHU::FEX_PAGE_SIZE);
}
}
if (OpMaxPage != CurrentCodePage) {
CurrentCodePage = OpMaxPage;
CodePages.insert(CurrentCodePage);
if (CodePages.insert(CurrentCodePage).second) {
AddContainedCodePage(PC, CurrentCodePage, FHU::FEX_PAGE_SIZE);
}
}
bool ErrorDuringDecoding = !DecodeInstruction(RIPToDecode + PCOffset);
if (ErrorDuringDecoding) [[unlikely]] {
if (ErrorDuringDecoding) {
LogMan::Msg::DFmt("Couldn't Decode something at 0x{:x}, Started at 0x{:x}", RIPToDecode + PCOffset, PC);
// Put an invalid instruction in the stream so the core can raise SIGILL if hit
CurrentBlockDecoding.HasInvalidInstruction = true;
@@ -1205,9 +1314,6 @@ void Decoder::DecodeInstructionsAtEntry(uint8_t const* _InstStream, uint64_t PC,
CurrentBlockDecoding.DecodedInstructions = &DecodedBuffer[BlockStartOffset];
}
for (auto CodePage : CodePages) {
AddContainedCodePage(PC, CodePage, FHU::FEX_PAGE_SIZE);
}
// sort for better branching
std::sort(Blocks.begin(), Blocks.end(), [](const FEXCore::Frontend::Decoder::DecodedBlocks& a, const FEXCore::Frontend::Decoder::DecodedBlocks& b) {
+3 -3
View File
@@ -11,7 +11,7 @@
#include <vector>
namespace FEXCore::Context {
class ContextImpl;
struct Context;
}
namespace FEXCore::Frontend {
@@ -25,7 +25,7 @@ public:
bool HasInvalidInstruction{};
};
Decoder(FEXCore::Context::ContextImpl *ctx);
Decoder(FEXCore::Context::Context *ctx);
~Decoder();
void DecodeInstructionsAtEntry(uint8_t const* InstStream, uint64_t PC, std::function<void(uint64_t BlockEntry, uint64_t Start, uint64_t Length)> AddContainedCodePage);
@@ -52,7 +52,7 @@ private:
bool L; // VEX.L bit (if set then 256 bit operation, if unset then scalar or 128-bit operation)
};
FEXCore::Context::ContextImpl *CTX;
FEXCore::Context::Context *CTX;
const FEXCore::HLE::SyscallOSABI OSABI{};
bool DecodeInstruction(uint64_t PC);
+2 -2
View File
@@ -68,11 +68,11 @@ void GdbServer::WaitForThreadWakeup() {
ThreadBreakEvent.Wait();
}
GdbServer::GdbServer(FEXCore::Context::ContextImpl *ctx) : CTX(ctx) {
GdbServer::GdbServer(FEXCore::Context::Context *ctx) : CTX(ctx) {
// Pass all signals by default
std::fill(PassSignals.begin(), PassSignals.end(), true);
ctx->SetExitHandler([this](uint64_t ThreadId, FEXCore::Context::ExitReason ExitReason) {
Context::SetExitHandler(ctx, [this](uint64_t ThreadId, FEXCore::Context::ExitReason ExitReason) {
if (ExitReason == FEXCore::Context::ExitReason::EXIT_DEBUG) {
this->Break(SIGTRAP);
}
+3 -3
View File
@@ -19,12 +19,12 @@ $end_info$
namespace FEXCore {
namespace Context {
class ContextImpl;
struct Context;
}
class GdbServer {
public:
GdbServer(FEXCore::Context::ContextImpl *ctx);
GdbServer(FEXCore::Context::Context *ctx);
// Public for threading
void GdbServerLoop();
@@ -75,7 +75,7 @@ private:
std::string readRegs();
HandledPacketType readReg(const std::string& packet);
FEXCore::Context::ContextImpl *CTX;
FEXCore::Context::Context *CTX;
std::unique_ptr<FEXCore::Threads::Thread> gdbServerThread;
std::unique_ptr<std::iostream> CommsStream;
std::mutex sendMutex;
@@ -79,7 +79,6 @@ HostFeatures::HostFeatures() {
SupportsSHA = true;
SupportsBMI1 = true;
SupportsBMI2 = true;
SupportsCLWB = true;
if (!SupportsAtomics) {
WARN_ONCE_FMT("Host CPU doesn't support atomics. Expect bad performance");
@@ -129,7 +128,6 @@ HostFeatures::HostFeatures() {
SupportsSHA = Features.has(Xbyak::util::Cpu::tSHA);
SupportsBMI1 = Features.has(Xbyak::util::Cpu::tBMI1);
SupportsBMI2 = Features.has(Xbyak::util::Cpu::tBMI2);
SupportsBMI2 = Features.has(Xbyak::util::Cpu::tCLWB);
SupportsPMULL_128Bit = Features.has(Xbyak::util::Cpu::tPCLMULQDQ);
// xbyak doesn't know how to check for CLZero
@@ -17,8 +17,20 @@ $end_info$
#include <unistd.h>
namespace FEXCore::CPU {
[[noreturn]]
static void SignalReturn(FEXCore::Core::InternalThreadState *Thread) {
Thread->CTX->SignalThread(Thread, FEXCore::Core::SignalEvent::Return);
LOGMAN_MSG_A_FMT("unreachable");
FEX_UNREACHABLE;
}
#define DEF_OP(x) void InterpreterOps::Op_##x(IR::IROp_Header *IROp, IROpData *Data, IR::NodeID Node)
DEF_OP(SignalReturn) {
SignalReturn(Data->State);
}
DEF_OP(CallbackReturn) {
Data->State->CurrentFrame->Pointers.Interpreter.CallbackReturn(Data->State, Data->StackEntry);
}
@@ -79,7 +91,7 @@ DEF_OP(Syscall) {
Args.Argument[j] = *GetSrc<uint64_t*>(Data->SSAData, Op->Header.Args[j]);
}
uint64_t Res = FEXCore::Context::HandleSyscall(static_cast<Context::ContextImpl*>(Data->State->CTX)->SyscallHandler, Data->State->CurrentFrame, &Args);
uint64_t Res = FEXCore::Context::HandleSyscall(Data->State->CTX->SyscallHandler, Data->State->CurrentFrame, &Args);
GD = Res;
}
@@ -114,7 +126,7 @@ DEF_OP(InlineSyscall) {
DEF_OP(Thunk) {
auto Op = IROp->C<IR::IROp_Thunk>();
auto thunkFn = static_cast<Context::ContextImpl*>(Data->State->CTX)->ThunkHandler->LookupThunk(Op->ThunkNameHash);
auto thunkFn = Data->State->CTX->ThunkHandler->LookupThunk(Op->ThunkNameHash);
thunkFn(*GetSrc<void**>(Data->SSAData, Op->ArgPtr));
}
@@ -130,7 +142,7 @@ DEF_OP(ValidateCode) {
}
DEF_OP(ThreadRemoveCodeEntry) {
static_cast<Context::ContextImpl*>(Data->State->CTX)->ThreadRemoveCodeEntryFromJit(Data->State->CurrentFrame, Data->CurrentEntry);
Data->State->CTX->ThreadRemoveCodeEntryFromJit(Data->State->CurrentFrame, Data->CurrentEntry);
}
DEF_OP(CPUID) {
@@ -139,7 +151,7 @@ DEF_OP(CPUID) {
const uint64_t Arg = *GetSrc<uint64_t*>(Data->SSAData, Op->Function);
const uint64_t Leaf = *GetSrc<uint64_t*>(Data->SSAData, Op->Leaf);
auto Results = Data->State->CTX->RunCPUIDFunction(Arg, Leaf);
auto Results = Data->State->CTX->CPUID.RunFunction(Arg, Leaf);
memcpy(DstPtr, &Results, sizeof(uint32_t) * 4);
}
@@ -62,23 +62,6 @@ DEF_OP(VCastFromGPR) {
memcpy(GDP, GetSrc<void*>(Data->SSAData, Op->Src), Op->Header.ElementSize);
}
DEF_OP(VDupFromGPR) {
const auto Op = IROp->C<IR::IROp_VDupFromGPR>();
const auto OpSize = IROp->Size;
const auto ElementSize = IROp->ElementSize;
const auto NumElements = OpSize / IROp->ElementSize;
uint8_t Tmp[Core::CPUState::XMM_AVX_REG_SIZE]{};
const auto *Src = GetSrc<void*>(Data->SSAData, Op->Src);
for (size_t i = 0; i < NumElements; i++) {
memcpy(Tmp + (i * ElementSize), Src, ElementSize);
}
memcpy(GDP, Tmp, sizeof(Tmp));
}
DEF_OP(Float_FromGPR_S) {
auto Op = IROp->C<IR::IROp_Float_FromGPR_S>();
@@ -26,7 +26,7 @@ public:
[[nodiscard]] std::string GetName() override { return "Interpreter"; }
[[nodiscard]] CPUBackend::CompiledCode CompileCode(uint64_t Entry,
[[nodiscard]] void *CompileCode(uint64_t Entry,
FEXCore::IR::IRListView const *IR,
FEXCore::Core::DebugData *DebugData,
FEXCore::IR::RegisterAllocationData *RAData, bool GDBEnabled) override;
@@ -35,7 +35,7 @@ public:
[[nodiscard]] bool NeedsOpDispatch() override { return true; }
static void InitializeSignalHandlers(FEXCore::Context::ContextImpl *CTX);
static void InitializeSignalHandlers(FEXCore::Context::Context *CTX);
void ClearCache() override;
@@ -49,10 +49,7 @@ InterpreterCore::InterpreterCore(Dispatcher *Dispatcher, FEXCore::Core::Internal
ClearCache();
}
void InterpreterCore::InitializeSignalHandlers(FEXCore::Context::ContextImpl *CTX) {
CTX->SignalDelegation->RegisterHostSignalHandler(SIGILL, [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext) -> bool {
return reinterpret_cast<Context::ContextImpl*>(Thread->CTX)->Dispatcher->HandleSIGILL(Thread, Signal, info, ucontext);
}, true);
void InterpreterCore::InitializeSignalHandlers(FEXCore::Context::Context *CTX) {
#ifdef _M_ARM_64
CTX->SignalDelegation->RegisterHostSignalHandler(SIGBUS, [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext) -> bool {
@@ -61,21 +58,18 @@ void InterpreterCore::InitializeSignalHandlers(FEXCore::Context::ContextImpl *CT
#endif
}
CPUBackend::CompiledCode InterpreterCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IRListView const *IR, [[maybe_unused]] FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData, bool GDBEnabled) {
void *InterpreterCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IRListView const *IR, [[maybe_unused]] FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData, bool GDBEnabled) {
const auto IRSize = AlignUp(IR->GetInlineSize(), 16);
const auto MaxSize = IRSize + Dispatcher::MaxInterpreterTrampolineSize + GDBEnabled * Dispatcher::MaxGDBPauseCheckSize;
if ((BufferUsed + MaxSize) > CurrentCodeBuffer->Size) {
static_cast<Context::ContextImpl*>(ThreadState->CTX)->ClearCodeCache(ThreadState);
ThreadState->CTX->ClearCodeCache(ThreadState);
}
CPUBackend::CompiledCode CodeData{};
const auto BufferStart = CurrentCodeBuffer->Ptr + BufferUsed;
const auto BufferStartOffset = BufferUsed;
CodeData.BlockBegin = CodeData.BlockEntry = CurrentCodeBuffer->Ptr + BufferStartOffset;
auto DestBuffer = CodeData.BlockBegin;
auto DestBuffer = BufferStart;
if (GDBEnabled) {
const auto GDBSize = Dispatch->GenerateGDBPauseCheck(DestBuffer, Entry);
@@ -92,9 +86,7 @@ CPUBackend::CompiledCode InterpreterCore::CompileCode(uint64_t Entry, [[maybe_un
DestBuffer += IRSize;
BufferUsed += IRSize;
CodeData.Size = BufferUsed - BufferStartOffset;
return CodeData;
return BufferStart;
}
void InterpreterCore::ClearCache() {
@@ -103,11 +95,11 @@ void InterpreterCore::ClearCache() {
BufferUsed = 0;
}
std::unique_ptr<CPUBackend> CreateInterpreterCore(FEXCore::Context::ContextImpl *ctx, FEXCore::Core::InternalThreadState *Thread) {
std::unique_ptr<CPUBackend> CreateInterpreterCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread) {
return std::make_unique<InterpreterCore>(ctx->Dispatcher.get(), Thread);
}
void InitializeInterpreterSignalHandlers(FEXCore::Context::ContextImpl *CTX) {
void InitializeInterpreterSignalHandlers(FEXCore::Context::Context *CTX) {
InterpreterCore::InitializeSignalHandlers(CTX);
}
@@ -3,7 +3,7 @@
#include <memory>
namespace FEXCore::Context {
class ContextImpl;
struct Context;
}
namespace FEXCore::Core {
@@ -14,9 +14,9 @@ namespace FEXCore::CPU {
class CPUBackend;
struct DispatcherConfig;
[[nodiscard]] std::unique_ptr<CPUBackend> CreateInterpreterCore(FEXCore::Context::ContextImpl *ctx,
[[nodiscard]] std::unique_ptr<CPUBackend> CreateInterpreterCore(FEXCore::Context::Context *ctx,
FEXCore::Core::InternalThreadState *Thread);
void InitializeInterpreterSignalHandlers(FEXCore::Context::ContextImpl *CTX);
void InitializeInterpreterSignalHandlers(FEXCore::Context::Context *CTX);
CPUBackendFeatures GetInterpreterBackendFeatures();
} // namespace FEXCore::CPU
@@ -113,6 +113,7 @@ constexpr OpHandlerArray InterpreterOpHandlers = [] {
REGISTER_OP(ATOMICFETCHNEG, AtomicFetchNeg);
// Branch ops
REGISTER_OP(SIGNALRETURN, SignalReturn);
REGISTER_OP(CALLBACKRETURN, CallbackReturn);
REGISTER_OP(EXITFUNCTION, ExitFunction);
REGISTER_OP(JUMP, Jump);
@@ -127,7 +128,6 @@ constexpr OpHandlerArray InterpreterOpHandlers = [] {
// Conversion ops
REGISTER_OP(VINSGPR, VInsGPR);
REGISTER_OP(VCASTFROMGPR, VCastFromGPR);
REGISTER_OP(VDUPFROMGPR, VDupFromGPR);
REGISTER_OP(FLOAT_FROMGPR_S, Float_FromGPR_S);
REGISTER_OP(FLOAT_FTOF, Float_FToF);
REGISTER_OP(VECTOR_STOF, Vector_SToF);
@@ -154,9 +154,7 @@ constexpr OpHandlerArray InterpreterOpHandlers = [] {
REGISTER_OP(STOREMEM, StoreMem);
REGISTER_OP(LOADMEMTSO, LoadMem);
REGISTER_OP(STOREMEMTSO, StoreMem);
REGISTER_OP(MEMSET, MemSet);
REGISTER_OP(CACHELINECLEAR, CacheLineClear);
REGISTER_OP(CACHELINECLEAN, CacheLineClean);
REGISTER_OP(CACHELINEZERO, CacheLineZero);
// Misc ops
@@ -223,8 +221,6 @@ constexpr OpHandlerArray InterpreterOpHandlers = [] {
REGISTER_OP(VZIP2, VZip);
REGISTER_OP(VUNZIP, VUnZip);
REGISTER_OP(VUNZIP2, VUnZip);
REGISTER_OP(VTRN, VTrn);
REGISTER_OP(VTRN2, VTrn);
REGISTER_OP(VBSL, VBSL);
REGISTER_OP(VCMPEQ, VCMPEQ);
REGISTER_OP(VCMPEQZ, VCMPEQZ);
@@ -333,6 +329,7 @@ void InterpreterOps::InterpretIR(FEXCore::Core::CpuStateFrame *Frame, FEXCore::I
const uintptr_t ListSize = CurrentIR->GetSSACount();
static_assert(sizeof(FEXCore::IR::IROp_Header) == 4);
static_assert(sizeof(FEXCore::IR::OrderedNode) == 16);
auto BlockEnd = CurrentIR->GetBlocks().end();
@@ -142,6 +142,7 @@ namespace FEXCore::CPU {
DEF_OP(AtomicFetchNeg);
///< Branch ops
DEF_OP(SignalReturn);
DEF_OP(CallbackReturn);
DEF_OP(ExitFunction);
DEF_OP(Jump);
@@ -156,7 +157,6 @@ namespace FEXCore::CPU {
///< Conversion ops
DEF_OP(VInsGPR);
DEF_OP(VCastFromGPR);
DEF_OP(VDupFromGPR);
DEF_OP(Float_FromGPR_S);
DEF_OP(Float_FToF);
DEF_OP(Vector_SToF);
@@ -181,9 +181,7 @@ namespace FEXCore::CPU {
DEF_OP(StoreFlag);
DEF_OP(LoadMem);
DEF_OP(StoreMem);
DEF_OP(MemSet);
DEF_OP(CacheLineClear);
DEF_OP(CacheLineClean);
DEF_OP(CacheLineZero);
///< Misc ops
@@ -243,7 +241,6 @@ namespace FEXCore::CPU {
DEF_OP(VSMax);
DEF_OP(VZip);
DEF_OP(VUnZip);
DEF_OP(VTrn);
DEF_OP(VBSL);
DEF_OP(VCMPEQ);
DEF_OP(VCMPEQZ);
@@ -23,22 +23,6 @@ static inline void CacheLineFlush(char *Addr) {
#endif
}
static inline void CacheLineClean(char *Addr) {
#ifdef _M_X86_64
__asm volatile (
"clwb (%[Addr]);"
:: [Addr] "r" (Addr)
: "memory");
#elif _M_ARM_64
__asm volatile (
"dc cvac, %[Addr]"
:: [Addr] "r" (Addr)
: "memory");
#else
LOGMAN_THROW_A_FMT("Unsupported architecture with cacheline clean");
#endif
}
#define DEF_OP(x) void InterpreterOps::Op_##x(IR::IROp_Header *IROp, IROpData *Data, IR::NodeID Node)
DEF_OP(LoadContext) {
const auto Op = IROp->C<IR::IROp_LoadContext>();
@@ -288,111 +272,6 @@ DEF_OP(StoreMem) {
}
}
DEF_OP(MemSet) {
const auto Op = IROp->C<IR::IROp_MemSet>();
const int32_t Size = Op->Size;
char *MemData = *GetSrc<char **>(Data->SSAData, Op->Addr);
const auto Value = *GetSrc<uint64_t*>(Data->SSAData, Op->Value);
const auto Length = *GetSrc<uint64_t*>(Data->SSAData, Op->Length);
const auto Direction = *GetSrc<uint8_t*>(Data->SSAData, Op->Direction);
auto MemSetElements = [](auto* Memory, uint64_t Value, size_t Length) {
for (size_t i = 0; i < Length; ++i) {
Memory[i] = Value;
}
};
auto MemSetElementsInverse = [](auto* Memory, uint64_t Value, size_t Length) {
for (size_t i = 0; i < Length; ++i) {
Memory[-i] = Value;
}
};
if (Direction == 0) { // Forward
if (Op->IsAtomic) {
switch (Size) {
case 1:
MemSetElements(reinterpret_cast<std::atomic<uint8_t>*>(MemData), Value, Length);
break;
case 2:
MemSetElements(reinterpret_cast<std::atomic<uint16_t>*>(MemData), Value, Length);
break;
case 4:
MemSetElements(reinterpret_cast<std::atomic<uint32_t>*>(MemData), Value, Length);
break;
case 8:
MemSetElements(reinterpret_cast<std::atomic<uint64_t>*>(MemData), Value, Length);
break;
default:
LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, Size);
break;
}
}
else {
switch (Size) {
case 1:
MemSetElements(reinterpret_cast<uint8_t*>(MemData), Value, Length);
break;
case 2:
MemSetElements(reinterpret_cast<uint16_t*>(MemData), Value, Length);
break;
case 4:
MemSetElements(reinterpret_cast<uint32_t*>(MemData), Value, Length);
break;
case 8:
MemSetElements(reinterpret_cast<uint64_t*>(MemData), Value, Length);
break;
default:
LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, Size);
break;
}
}
GD = reinterpret_cast<uint64_t>(MemData + (Length * Size));
}
else { // Backward
if (Op->IsAtomic) {
switch (Size) {
case 1:
MemSetElementsInverse(reinterpret_cast<std::atomic<uint8_t>*>(MemData), Value, Length);
break;
case 2:
MemSetElementsInverse(reinterpret_cast<std::atomic<uint16_t>*>(MemData), Value, Length);
break;
case 4:
MemSetElementsInverse(reinterpret_cast<std::atomic<uint32_t>*>(MemData), Value, Length);
break;
case 8:
MemSetElementsInverse(reinterpret_cast<std::atomic<uint64_t>*>(MemData), Value, Length);
break;
default:
LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, Size);
break;
}
}
else {
switch (Size) {
case 1:
MemSetElementsInverse(reinterpret_cast<uint8_t*>(MemData), Value, Length);
break;
case 2:
MemSetElementsInverse(reinterpret_cast<uint16_t*>(MemData), Value, Length);
break;
case 4:
MemSetElementsInverse(reinterpret_cast<uint32_t*>(MemData), Value, Length);
break;
case 8:
MemSetElementsInverse(reinterpret_cast<uint64_t*>(MemData), Value, Length);
break;
default:
LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, Size);
break;
}
}
GD = reinterpret_cast<uint64_t>(MemData - (Length * Size));
}
}
DEF_OP(CacheLineClear) {
auto Op = IROp->C<IR::IROp_CacheLineClear>();
@@ -402,15 +281,6 @@ DEF_OP(CacheLineClear) {
CacheLineFlush(MemData);
}
DEF_OP(CacheLineClean) {
auto Op = IROp->C<IR::IROp_CacheLineClean>();
char *MemData = *GetSrc<char **>(Data->SSAData, Op->Addr);
// 64-byte cache line clear
CacheLineClean(MemData);
}
DEF_OP(CacheLineZero) {
auto Op = IROp->C<IR::IROp_CacheLineZero>();
@@ -902,67 +902,6 @@ DEF_OP(VZip) {
memcpy(GDP, Tmp, OpSize);
}
DEF_OP(VTrn) {
const auto Op = IROp->C<IR::IROp_VTrn>();
const uint8_t OpSize = IROp->Size;
void *Src1 = GetSrc<void*>(Data->SSAData, Op->VectorLower);
void *Src2 = GetSrc<void*>(Data->SSAData, Op->VectorUpper);
uint8_t Tmp[Core::CPUState::XMM_AVX_REG_SIZE]{};
const uint8_t ElementSize = Op->Header.ElementSize;
uint8_t Elements = OpSize / ElementSize;
const uint8_t BaseOffset = IROp->Op == IR::OP_VTRN2 ? 1 : 0;
Elements >>= 1;
switch (ElementSize) {
case 1: {
auto *Dst_d = reinterpret_cast<uint8_t*>(Tmp);
auto *Src1_d = reinterpret_cast<uint8_t*>(Src1);
auto *Src2_d = reinterpret_cast<uint8_t*>(Src2);
for (unsigned i = 0; i < Elements; ++i) {
Dst_d[i*2] = Src1_d[i*2 + BaseOffset];
Dst_d[i*2+1] = Src2_d[i*2 + BaseOffset];
}
break;
}
case 2: {
auto *Dst_d = reinterpret_cast<uint16_t*>(Tmp);
auto *Src1_d = reinterpret_cast<uint16_t*>(Src1);
auto *Src2_d = reinterpret_cast<uint16_t*>(Src2);
for (unsigned i = 0; i < Elements; ++i) {
Dst_d[i*2] = Src1_d[i*2 + BaseOffset];
Dst_d[i*2+1] = Src2_d[i*2 + BaseOffset];
}
break;
}
case 4: {
auto *Dst_d = reinterpret_cast<uint32_t*>(Tmp);
auto *Src1_d = reinterpret_cast<uint32_t*>(Src1);
auto *Src2_d = reinterpret_cast<uint32_t*>(Src2);
for (unsigned i = 0; i < Elements; ++i) {
Dst_d[i*2] = Src1_d[i*2 + BaseOffset];
Dst_d[i*2+1] = Src2_d[i*2 + BaseOffset];
}
break;
}
case 8: {
auto *Dst_d = reinterpret_cast<uint64_t*>(Tmp);
auto *Src1_d = reinterpret_cast<uint64_t*>(Src1);
auto *Src2_d = reinterpret_cast<uint64_t*>(Src2);
for (unsigned i = 0; i < Elements; ++i) {
Dst_d[i*2] = Src1_d[i*2 + BaseOffset];
Dst_d[i*2+1] = Src2_d[i*2 + BaseOffset];
}
break;
}
default:
LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize);
break;
}
memcpy(GDP, Tmp, OpSize);
}
DEF_OP(VUnZip) {
const auto Op = IROp->C<IR::IROp_VUnZip>();
const uint8_t OpSize = IROp->Size;
@@ -1025,9 +964,7 @@ DEF_OP(VUnZip) {
}
DEF_OP(VBSL) {
const auto Op = IROp->C<IR::IROp_VBSL>();
const auto OpSize = IROp->Size;
auto Op = IROp->C<IR::IROp_VBSL>();
const auto Src1 = *GetSrc<InterpVector256*>(Data->SSAData, Op->VectorMask);
const auto Src2 = *GetSrc<InterpVector256*>(Data->SSAData, Op->VectorTrue);
const auto Src3 = *GetSrc<InterpVector256*>(Data->SSAData, Op->VectorFalse);
@@ -1037,8 +974,7 @@ DEF_OP(VBSL) {
.Upper = (Src2.Upper & Src1.Upper) | (Src3.Upper & ~Src1.Upper),
};
memset(GDP, 0, sizeof(InterpVector256));
memcpy(GDP, &Tmp, OpSize);
memcpy(GDP, &Tmp, sizeof(Tmp));
}
DEF_OP(VCMPEQ) {
+61 -8
View File
@@ -262,13 +262,13 @@ DEF_OP(MulH) {
const auto Src2 = GetReg(Op->Src2.ID());
if (OpSize == 4) {
sxtw(TMP1, Src1.W());
sxtw(TMP2, Src2.W());
sxtw(TMP1, Src1);
sxtw(TMP2, Src2);
mul(ARMEmitter::Size::i32Bit, Dst, TMP1, TMP2);
ubfx(ARMEmitter::Size::i32Bit, Dst, Dst, 32, 32);
}
else {
smulh(Dst.X(), Src1.X(), Src2.X());
smulh(Dst, Src1, Src2);
}
}
@@ -289,7 +289,7 @@ DEF_OP(UMulH) {
ubfx(ARMEmitter::Size::i64Bit, Dst, Dst, 32, 32);
}
else {
umulh(Dst.X(), Src1.X(), Src2.X());
umulh(Dst, Src1, Src2);
}
}
@@ -610,7 +610,7 @@ DEF_OP(LDiv) {
case 4: {
mov(EmitSize, TMP1, Lower);
bfi(EmitSize, TMP1, Upper, 32, 32);
sxtw(TMP2, Divisor.W());
sxtw(TMP2, Divisor);
sdiv(EmitSize, Dst, TMP1, TMP2);
break;
}
@@ -744,7 +744,7 @@ DEF_OP(LRem) {
case 4: {
mov(EmitSize, TMP1, Lower);
bfi(EmitSize, TMP1, Upper, 32, 32);
sxtw(TMP3, Divisor.W());
sxtw(TMP3, Divisor);
sdiv(EmitSize, TMP2, TMP1, TMP3);
msub(EmitSize, Dst, TMP2, TMP3, TMP1);
break;
@@ -1173,8 +1173,8 @@ DEF_OP(VExtractToGPR) {
// Inverting our dedicated predicate for 128-bit operations selects
// all of the top lanes. We can then compact those into a temporary.
const auto CompactPred = ARMEmitter::PReg::p0;
not_(CompactPred, PRED_TMP_32B.Zeroing(), PRED_TMP_16B);
compact(ARMEmitter::SubRegSize::i64Bit, VTMP1.Z(), CompactPred, Vector.Z());
not_(CompactPred, PRED_TMP_32B, PRED_TMP_16B);
compact(ARMEmitter::SubRegSize::i64Bit, VTMP1, CompactPred, Vector);
// Sanitize the zero-based index to work on the now-moved
// upper half of the vector.
@@ -1274,4 +1274,57 @@ DEF_OP(FCmp) {
#undef DEF_OP
void Arm64JITCore::RegisterALUHandlers() {
#define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &Arm64JITCore::Op_##x
REGISTER_OP(TRUNCELEMENTPAIR, TruncElementPair);
REGISTER_OP(CONSTANT, Constant);
REGISTER_OP(ENTRYPOINTOFFSET, EntrypointOffset);
REGISTER_OP(INLINECONSTANT, InlineConstant);
REGISTER_OP(INLINEENTRYPOINTOFFSET, InlineEntrypointOffset);
REGISTER_OP(CYCLECOUNTER, CycleCounter);
REGISTER_OP(ADD, Add);
REGISTER_OP(SUB, Sub);
REGISTER_OP(NEG, Neg);
REGISTER_OP(MUL, Mul);
REGISTER_OP(UMUL, UMul);
REGISTER_OP(DIV, Div);
REGISTER_OP(UDIV, UDiv);
REGISTER_OP(REM, Rem);
REGISTER_OP(UREM, URem);
REGISTER_OP(MULH, MulH);
REGISTER_OP(UMULH, UMulH);
REGISTER_OP(OR, Or);
REGISTER_OP(AND, And);
REGISTER_OP(ANDN, Andn);
REGISTER_OP(XOR, Xor);
REGISTER_OP(LSHL, Lshl);
REGISTER_OP(LSHR, Lshr);
REGISTER_OP(ASHR, Ashr);
REGISTER_OP(ROR, Ror);
REGISTER_OP(EXTR, Extr);
REGISTER_OP(PDEP, PDep);
REGISTER_OP(PEXT, PExt);
REGISTER_OP(LDIV, LDiv);
REGISTER_OP(LUDIV, LUDiv);
REGISTER_OP(LREM, LRem);
REGISTER_OP(LUREM, LURem);
REGISTER_OP(NOT, Not);
REGISTER_OP(POPCOUNT, Popcount);
REGISTER_OP(FINDLSB, FindLSB);
REGISTER_OP(FINDMSB, FindMSB);
REGISTER_OP(FINDTRAILINGZEROS, FindTrailingZeros);
REGISTER_OP(COUNTLEADINGZEROES, CountLeadingZeroes);
REGISTER_OP(REV, Rev);
REGISTER_OP(BFI, Bfi);
REGISTER_OP(BFE, Bfe);
REGISTER_OP(SBFE, Sbfe);
REGISTER_OP(SELECT, Select);
REGISTER_OP(VEXTRACTTOGPR, VExtractToGPR);
REGISTER_OP(FLOAT_TOGPR_ZS, Float_ToGPR_ZS);
REGISTER_OP(FLOAT_TOGPR_S, Float_ToGPR_S);
REGISTER_OP(FCMP, FCmp);
#undef REGISTER_OP
}
}
@@ -27,7 +27,7 @@ void Arm64JITCore::InsertNamedThunkRelocation(ARMEmitter::Register Reg, const IR
MoveABI.NamedThunkMove.Header.Type = FEXCore::CPU::RelocationTypes::RELOC_NAMED_THUNK_MOVE;
// Offset is the offset from the entrypoint of the block
auto CurrentCursor = GetCursorAddress<uint8_t *>();
MoveABI.NamedThunkMove.Offset = CurrentCursor - CodeData.BlockBegin;
MoveABI.NamedThunkMove.Offset = CurrentCursor - GuestEntry;
MoveABI.NamedThunkMove.Symbol = Sum;
MoveABI.NamedThunkMove.RegisterIndex = Reg.Idx();
@@ -58,7 +58,7 @@ Arm64JITCore::NamedSymbolLiteralPair Arm64JITCore::InsertNamedSymbolLiteral(FEXC
void Arm64JITCore::PlaceNamedSymbolLiteral(NamedSymbolLiteralPair &Lit) {
// Offset is the offset from the entrypoint of the block
auto CurrentCursor = GetCursorAddress<uint8_t *>();
Lit.MoveABI.NamedSymbolLiteral.Offset = CurrentCursor - CodeData.BlockBegin;
Lit.MoveABI.NamedSymbolLiteral.Offset = CurrentCursor - GuestEntry;
Bind(&Lit.Loc);
dc64(Lit.Lit);
@@ -70,7 +70,7 @@ void Arm64JITCore::InsertGuestRIPMove(ARMEmitter::Register Reg, uint64_t Constan
MoveABI.GuestRIPMove.Header.Type = FEXCore::CPU::RelocationTypes::RELOC_GUEST_RIP_MOVE;
// Offset is the offset from the entrypoint of the block
auto CurrentCursor = GetCursorAddress<uint8_t *>();
MoveABI.GuestRIPMove.Offset = CurrentCursor - CodeData.BlockBegin;
MoveABI.GuestRIPMove.Offset = CurrentCursor - GuestEntry;
MoveABI.GuestRIPMove.GuestRIP = Constant;
MoveABI.GuestRIPMove.RegisterIndex = Reg.Idx();
@@ -438,5 +438,23 @@ DEF_OP(AtomicFetchNeg) {
}
#undef DEF_OP
void Arm64JITCore::RegisterAtomicHandlers() {
#define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &Arm64JITCore::Op_##x
REGISTER_OP(CASPAIR, CASPair);
REGISTER_OP(CAS, CAS);
REGISTER_OP(ATOMICADD, AtomicAdd);
REGISTER_OP(ATOMICSUB, AtomicSub);
REGISTER_OP(ATOMICAND, AtomicAnd);
REGISTER_OP(ATOMICOR, AtomicOr);
REGISTER_OP(ATOMICXOR, AtomicXor);
REGISTER_OP(ATOMICSWAP, AtomicSwap);
REGISTER_OP(ATOMICFETCHADD, AtomicFetchAdd);
REGISTER_OP(ATOMICFETCHSUB, AtomicFetchSub);
REGISTER_OP(ATOMICFETCHAND, AtomicFetchAnd);
REGISTER_OP(ATOMICFETCHOR, AtomicFetchOr);
REGISTER_OP(ATOMICFETCHXOR, AtomicFetchXor);
REGISTER_OP(ATOMICFETCHNEG, AtomicFetchNeg);
#undef REGISTER_OP
}
}
@@ -20,6 +20,16 @@ $end_info$
namespace FEXCore::CPU {
#define DEF_OP(x) void Arm64JITCore::Op_##x(IR::IROp_Header const *IROp, IR::NodeID Node)
DEF_OP(SignalReturn) {
// First we must reset the stack
ResetStack();
// Now branch to our signal return helper
// This can't be a direct branch since the code needs to live at a constant location
ldr(ARMEmitter::XReg::x0, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.SignalReturnHandler));
br(ARMEmitter::Reg::r0);
}
DEF_OP(CallbackReturn) {
// spill back to CTX
SpillStaticRegs();
@@ -167,22 +177,13 @@ DEF_OP(Syscall) {
FEXCore::IR::SyscallFlags Flags = Op->Flags;
PushDynamicRegsAndLR(TMP1);
uint32_t GPRSpillMask = ~0U;
uint32_t FPRSpillMask = ~0U;
if ((Flags & FEXCore::IR::SyscallFlags::NOSYNCSTATEONENTRY) == FEXCore::IR::SyscallFlags::NOSYNCSTATEONENTRY) {
if ((Flags & FEXCore::IR::SyscallFlags::NOSYNCSTATEONENTRY) != FEXCore::IR::SyscallFlags::NOSYNCSTATEONENTRY) {
SpillStaticRegs();
}
else {
// Need to spill all caller saved registers still
GPRSpillMask = CALLER_GPR_MASK;
FPRSpillMask = CALLER_FPR_MASK;
SpillStaticRegs(true, CALLER_GPR_MASK, CALLER_FPR_MASK);
}
SpillStaticRegs(true, GPRSpillMask, FPRSpillMask);
// Now that we are spilled, store in the state that we are in a syscall
// Still without overwriting registers that matter
// 16bit LoadConstant to be a single instruction
// This gives the signal handler a value to check to see if we are in a syscall at all
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, GPRSpillMask & 0xFFFF);
str(ARMEmitter::XReg::x0, STATE, offsetof(FEXCore::Core::CpuStateFrame, InSyscallInfo));
uint64_t SPOffset = AlignUp(FEXCore::HLE::SyscallArguments::MAX_ARGS * 8, 16);
sub(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, SPOffset);
@@ -205,17 +206,19 @@ DEF_OP(Syscall) {
add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, SPOffset);
if ((Flags & FEXCore::IR::SyscallFlags::NORETURN) != FEXCore::IR::SyscallFlags::NORETURN) {
if ((Flags & FEXCore::IR::SyscallFlags::NOSYNCSTATEONENTRY) != FEXCore::IR::SyscallFlags::NOSYNCSTATEONENTRY &&
(Flags & FEXCore::IR::SyscallFlags::NORETURN) != FEXCore::IR::SyscallFlags::NORETURN) {
FillStaticRegs();
}
else {
// Result is now in x0
// Fix the stack and any values that were stepped on
FillStaticRegs(true, GPRSpillMask, FPRSpillMask);
FillStaticRegs(true, CALLER_GPR_MASK, CALLER_FPR_MASK);
}
// Now the registers we've spilled are back in their original host registers
// We can safely claim we are no longer in a syscall
str(ARMEmitter::XReg::zr, STATE, offsetof(FEXCore::Core::CpuStateFrame, InSyscallInfo));
PopDynamicRegsAndLR();
PopDynamicRegsAndLR();
if ((Flags & FEXCore::IR::SyscallFlags::NORETURN) != FEXCore::IR::SyscallFlags::NORETURN) {
// Move result to its destination register
mov(ARMEmitter::Size::i64Bit, GetReg(Node), ARMEmitter::Reg::r0);
}
@@ -245,9 +248,9 @@ DEF_OP(InlineSyscall) {
if (Op->Header.Args[i].IsInvalid()) break;
auto Reg = GetReg(Op->Header.Args[i].ID());
if (Reg == ARMEmitter::Reg::r8 ||
Reg == ARMEmitter::Reg::r4 ||
Reg == ARMEmitter::Reg::r5) {
if (Reg.Idx() == ARMEmitter::Reg::r8.Idx() ||
Reg.Idx() == ARMEmitter::Reg::r4.Idx() ||
Reg.Idx() == ARMEmitter::Reg::r5.Idx()) {
SpillMask |= (1U << Reg.Idx());
Intersects = true;
@@ -278,13 +281,13 @@ DEF_OP(InlineSyscall) {
// In the case of intersection with x4, x5, or x8 then these are currently SRA
// for registers RAX, RBX, and RSI. Which have just been spilled
// Just load back from the context. Could be slightly smarter but this is fairly uncommon
if (Reg == ARMEmitter::Reg::r8) {
if (Reg.Idx() == FEXCore::ARMEmitter::Reg::r8.Idx()) {
ldr(EmitSubSize, RegArgs[i].R(), STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RSI]));
}
else if (Reg == ARMEmitter::Reg::r4) {
else if (Reg.Idx() == FEXCore::ARMEmitter::Reg::r4.Idx()) {
ldr(EmitSubSize, RegArgs[i].R(), STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RAX]));
}
else if (Reg == ARMEmitter::Reg::r5) {
else if (Reg.Idx() == FEXCore::ARMEmitter::Reg::r5.Idx()) {
ldr(EmitSubSize, RegArgs[i].R(), STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RBX]));
}
else {
@@ -331,7 +334,7 @@ DEF_OP(Thunk) {
mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, GetReg(Op->ArgPtr.ID()));
auto thunkFn = static_cast<Context::ContextImpl*>(ThreadState->CTX)->ThunkHandler->LookupThunk(Op->ThunkNameHash);
auto thunkFn = ThreadState->CTX->ThunkHandler->LookupThunk(Op->ThunkNameHash);
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r2, (uintptr_t)thunkFn);
#ifdef VIXL_SIMULATOR
GenerateIndirectRuntimeCall<void, void*, void*>(ARMEmitter::Reg::r2);
@@ -448,5 +451,20 @@ DEF_OP(CPUID) {
}
#undef DEF_OP
void Arm64JITCore::RegisterBranchHandlers() {
#define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &Arm64JITCore::Op_##x
REGISTER_OP(SIGNALRETURN, SignalReturn);
REGISTER_OP(CALLBACKRETURN, CallbackReturn);
REGISTER_OP(EXITFUNCTION, ExitFunction);
REGISTER_OP(JUMP, Jump);
REGISTER_OP(CONDJUMP, CondJump);
REGISTER_OP(SYSCALL, Syscall);
REGISTER_OP(INLINESYSCALL, InlineSyscall);
REGISTER_OP(THUNK, Thunk);
REGISTER_OP(VALIDATECODE, ValidateCode);
REGISTER_OP(THREADREMOVECODEENTRY, ThreadRemoveCodeEntry);
REGISTER_OP(CPUID, CPUID);
#undef REGISTER_OP
}
}
@@ -55,7 +55,7 @@ DEF_OP(VInsGPR) {
// Move the upper lane down for the insertion.
const auto CompactPred = ARMEmitter::PReg::p0;
not_(CompactPred, PRED_TMP_32B.Zeroing(), PRED_TMP_16B);
compact(ARMEmitter::SubRegSize::i64Bit, VTMP1.Z(), CompactPred, DestVector.Z());
compact(ARMEmitter::SubRegSize::i64Bit, VTMP1.Z(), CompactPred, DestVector);
}
// Put data in place for destructive SPLICE below.
@@ -108,32 +108,6 @@ DEF_OP(VCastFromGPR) {
}
}
DEF_OP(VDupFromGPR) {
const auto Op = IROp->C<IR::IROp_VDupFromGPR>();
const auto OpSize = IROp->Size;
const auto Dst = GetVReg(Node);
const auto Src = GetReg(Op->Src.ID());
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto ElementSize = IROp->ElementSize;
LOGMAN_THROW_AA_FMT(ElementSize == 8 || ElementSize == 4 || ElementSize == 2 || ElementSize == 1,
"Unexpected {} element size: {}", __func__, ElementSize);
const auto SubEmitSize =
ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit :
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit :
ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : ARMEmitter::SubRegSize::i8Bit;
if (HostSupportsSVE && Is256Bit) {
dup(SubEmitSize, Dst.Z(), Src);
} else {
dup(SubEmitSize, Dst.Q(), Src);
}
}
DEF_OP(Float_FromGPR_S) {
const auto Op = IROp->C<IR::IROp_Float_FromGPR_S>();
@@ -225,7 +199,7 @@ DEF_OP(Vector_FToZS) {
const auto Vector = GetVReg(Op->Vector.ID());
if (HostSupportsSVE && Is256Bit) {
const auto Mask = PRED_TMP_32B;
fcvtzs(Dst.Z(), SubEmitSize, Mask.Merging(), Vector.Z(), SubEmitSize);
fcvtzs(Dst, SubEmitSize, Mask.Merging(), Vector, SubEmitSize);
} else {
fcvtzs(SubEmitSize, Dst.Q(), Vector.Q());
}
@@ -248,8 +222,8 @@ DEF_OP(Vector_FToS) {
if (HostSupportsSVE && Is256Bit) {
const auto Mask = PRED_TMP_32B;
frinti(SubEmitSize, Dst.Z(), Mask.Merging(), Vector.Z());
fcvtzs(Dst.Z(), SubEmitSize, Mask.Merging(), Dst.Z(), SubEmitSize);
frinti(SubEmitSize, Dst, Mask.Merging(), Vector);
fcvtzs(Dst, SubEmitSize, Mask.Merging(), Dst, SubEmitSize);
} else {
const auto Dst = GetVReg(Node);
const auto Vector = GetVReg(Op->Vector.ID());
@@ -302,12 +276,12 @@ DEF_OP(Vector_FToF) {
break;
}
case 0x0204: { // Half <- Float
fcvtnt(FEXCore::ARMEmitter::SubRegSize::i16Bit, Dst.Z(), Mask, Vector.Z());
fcvtnt(FEXCore::ARMEmitter::SubRegSize::i16Bit, Dst, Mask, Vector);
uzp2(FEXCore::ARMEmitter::SubRegSize::i16Bit, Dst.Z(), Dst.Z(), Dst.Z());
break;
}
case 0x0408: { // Float <- Double
fcvtnt(FEXCore::ARMEmitter::SubRegSize::i32Bit, Dst.Z(), Mask, Vector.Z());
fcvtnt(FEXCore::ARMEmitter::SubRegSize::i32Bit, Dst, Mask, Vector);
uzp2(FEXCore::ARMEmitter::SubRegSize::i32Bit, Dst.Z(), Dst.Z(), Dst.Z());
break;
}
@@ -391,5 +365,18 @@ DEF_OP(Vector_FToI) {
}
#undef DEF_OP
void Arm64JITCore::RegisterConversionHandlers() {
#define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &Arm64JITCore::Op_##x
REGISTER_OP(VINSGPR, VInsGPR);
REGISTER_OP(VCASTFROMGPR, VCastFromGPR);
REGISTER_OP(FLOAT_FROMGPR_S, Float_FromGPR_S);
REGISTER_OP(FLOAT_FTOF, Float_FToF);
REGISTER_OP(VECTOR_STOF, Vector_SToF);
REGISTER_OP(VECTOR_FTOZS, Vector_FToZS);
REGISTER_OP(VECTOR_FTOS, Vector_FToS);
REGISTER_OP(VECTOR_FTOF, Vector_FToF);
REGISTER_OP(VECTOR_FTOI, Vector_FToI);
#undef REGISTER_OP
}
}
@@ -17,73 +17,37 @@ DEF_OP(AESImc) {
}
DEF_OP(AESEnc) {
const auto Op = IROp->C<IR::IROp_VAESEnc>();
const auto OpSize = IROp->Size;
const auto Dst = GetVReg(Node);
const auto Key = GetVReg(Op->Key.ID());
const auto State = GetVReg(Op->State.ID());
LOGMAN_THROW_AA_FMT(OpSize == Core::CPUState::XMM_SSE_REG_SIZE,
"Currently only supports 128-bit operations.");
auto Op = IROp->C<IR::IROp_VAESEnc>();
eor(VTMP2.Q(), VTMP2.Q(), VTMP2.Q());
mov(VTMP1.Q(), State.Q());
mov(VTMP1.Q(), GetVReg(Op->State.ID()).Q());
aese(VTMP1, VTMP2);
aesmc(VTMP1, VTMP1);
eor(Dst.Q(), VTMP1.Q(), Key.Q());
eor(GetVReg(Node).Q(), VTMP1.Q(), GetVReg(Op->Key.ID()).Q());
}
DEF_OP(AESEncLast) {
const auto Op = IROp->C<IR::IROp_VAESEncLast>();
const auto OpSize = IROp->Size;
const auto Dst = GetVReg(Node);
const auto Key = GetVReg(Op->Key.ID());
const auto State = GetVReg(Op->State.ID());
LOGMAN_THROW_AA_FMT(OpSize == Core::CPUState::XMM_SSE_REG_SIZE,
"Currently only supports 128-bit operations.");
auto Op = IROp->C<IR::IROp_VAESEncLast>();
eor(VTMP2.Q(), VTMP2.Q(), VTMP2.Q());
mov(VTMP1.Q(), State.Q());
mov(VTMP1.Q(), GetVReg(Op->State.ID()).Q());
aese(VTMP1, VTMP2);
eor(Dst.Q(), VTMP1.Q(), Key.Q());
eor(GetVReg(Node).Q(), VTMP1.Q(), GetVReg(Op->Key.ID()).Q());
}
DEF_OP(AESDec) {
const auto Op = IROp->C<IR::IROp_VAESDec>();
const auto OpSize = IROp->Size;
const auto Dst = GetVReg(Node);
const auto Key = GetVReg(Op->Key.ID());
const auto State = GetVReg(Op->State.ID());
LOGMAN_THROW_AA_FMT(OpSize == Core::CPUState::XMM_SSE_REG_SIZE,
"Currently only supports 128-bit operations.");
auto Op = IROp->C<IR::IROp_VAESDec>();
eor(VTMP2.Q(), VTMP2.Q(), VTMP2.Q());
mov(VTMP1.Q(), State.Q());
mov(VTMP1.Q(), GetVReg(Op->State.ID()).Q());
aesd(VTMP1, VTMP2);
aesimc(VTMP1, VTMP1);
eor(Dst.Q(), VTMP1.Q(), Key.Q());
eor(GetVReg(Node).Q(), VTMP1.Q(), GetVReg(Op->Key.ID()).Q());
}
DEF_OP(AESDecLast) {
const auto Op = IROp->C<IR::IROp_VAESDecLast>();
const auto OpSize = IROp->Size;
const auto Dst = GetVReg(Node);
const auto Key = GetVReg(Op->Key.ID());
const auto State = GetVReg(Op->State.ID());
LOGMAN_THROW_AA_FMT(OpSize == Core::CPUState::XMM_SSE_REG_SIZE,
"Currently only supports 128-bit operations.");
auto Op = IROp->C<IR::IROp_VAESDecLast>();
eor(VTMP2.Q(), VTMP2.Q(), VTMP2.Q());
mov(VTMP1.Q(), State.Q());
mov(VTMP1.Q(), GetVReg(Op->State.ID()).Q());
aesd(VTMP1, VTMP2);
eor(Dst.Q(), VTMP1.Q(), Key.Q());
eor(GetVReg(Node).Q(), VTMP1.Q(), GetVReg(Op->Key.ID()).Q());
}
DEF_OP(AESKeyGenAssist) {
@@ -137,22 +101,18 @@ DEF_OP(CRC32) {
crc32cw(Dst.W(), Src1.W(), Src2.W());
break;
case 8:
crc32cx(Dst.X(), Src1.X(), Src2.X());
crc32cx(Dst, Src1, Src2);
break;
default: LOGMAN_MSG_A_FMT("Unknown CRC32 size: {}", Op->SrcSize);
}
}
DEF_OP(PCLMUL) {
const auto Op = IROp->C<IR::IROp_PCLMUL>();
const auto OpSize = IROp->Size;
auto Op = IROp->C<IR::IROp_PCLMUL>();
const auto Dst = GetVReg(Node);
const auto Src1 = GetVReg(Op->Src1.ID());
const auto Src2 = GetVReg(Op->Src2.ID());
LOGMAN_THROW_AA_FMT(OpSize == Core::CPUState::XMM_SSE_REG_SIZE,
"Currently only supports 128-bit operations.");
auto Dst = GetVReg(Node);
auto Src1 = GetVReg(Op->Src1.ID());
auto Src2 = GetVReg(Op->Src2.ID());
switch (Op->Selector) {
case 0b00000000:
@@ -176,4 +136,16 @@ DEF_OP(PCLMUL) {
}
#undef DEF_OP
void Arm64JITCore::RegisterEncryptionHandlers() {
#define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &Arm64JITCore::Op_##x
REGISTER_OP(VAESIMC, AESImc);
REGISTER_OP(VAESENC, AESEnc);
REGISTER_OP(VAESENCLAST, AESEncLast);
REGISTER_OP(VAESDEC, AESDec);
REGISTER_OP(VAESDECLAST, AESDecLast);
REGISTER_OP(VAESKEYGENASSIST, AESKeyGenAssist);
REGISTER_OP(CRC32, CRC32);
REGISTER_OP(PCLMUL, PCLMUL);
#undef REGISTER_OP
}
}
@@ -14,5 +14,10 @@ DEF_OP(GetHostFlag) {
}
#undef DEF_OP
void Arm64JITCore::RegisterFlagHandlers() {
#define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &Arm64JITCore::Op_##x
REGISTER_OP(GETHOSTFLAG, GetHostFlag);
#undef REGISTER_OP
}
}
+46 -310
View File
@@ -163,7 +163,7 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header const *IROp, IR::NodeID Node) {
const auto Src1 = GetReg(IROp->Args[0].ID());
if (Info.ABI == FABI_F80_I16) {
sxth(ARMEmitter::Size::i32Bit, ARMEmitter::Reg::r0, Src1);
uxth(ARMEmitter::Size::i32Bit, ARMEmitter::Reg::r0, Src1);
}
else {
mov(ARMEmitter::Size::i32Bit, ARMEmitter::Reg::r0, Src1);
@@ -310,7 +310,7 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header const *IROp, IR::NodeID Node) {
FillStaticRegs();
const auto Dst = GetReg(Node);
sxth(ARMEmitter::Size::i64Bit, Dst, ARMEmitter::Reg::r0);
uxth(ARMEmitter::Size::i64Bit, Dst, ARMEmitter::Reg::r0);
}
break;
case FABI_I32_F80:{
@@ -484,7 +484,7 @@ static uint64_t Arm64JITCore_ExitFunctionLink(FEXCore::Core::CpuStateFrame *Fram
FEXCore::ARMEmitter::Emitter::ClearICache((void*)branch, 24);
// Add de-linking handler
Context::ContextImpl::ThreadAddBlockLink(Thread, GuestRip, (uintptr_t)record, [branch, LinkerAddress]{
Context::Context::ThreadAddBlockLink(Thread, GuestRip, (uintptr_t)record, [branch, LinkerAddress]{
FEXCore::ARMEmitter::Emitter emit((uint8_t*)(branch), 24);
FEXCore::ARMEmitter::ForwardLabel l_BranchHost;
emit.ldr(FEXCore::ARMEmitter::XReg::x0, &l_BranchHost);
@@ -498,7 +498,7 @@ static uint64_t Arm64JITCore_ExitFunctionLink(FEXCore::Core::CpuStateFrame *Fram
record[0] = HostCode;
// Add de-linking handler
Context::ContextImpl::ThreadAddBlockLink(Thread, GuestRip, (uintptr_t)record, [record, LinkerAddress]{
Context::Context::ThreadAddBlockLink(Thread, GuestRip, (uintptr_t)record, [record, LinkerAddress]{
record[0] = LinkerAddress;
});
}
@@ -509,7 +509,7 @@ static uint64_t Arm64JITCore_ExitFunctionLink(FEXCore::Core::CpuStateFrame *Fram
void Arm64JITCore::Op_NoOp(IR::IROp_Header const *IROp, IR::NodeID Node) {
}
Arm64JITCore::Arm64JITCore(FEXCore::Context::ContextImpl *ctx, FEXCore::Core::InternalThreadState *Thread)
Arm64JITCore::Arm64JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread)
: CPUBackend(Thread, INITIAL_CODE_SIZE, MAX_CODE_SIZE)
, Arm64Emitter(ctx, 0)
, HostSupportsSVE{ctx->HostFeatures.SupportsAVX}
@@ -535,6 +535,21 @@ Arm64JITCore::Arm64JITCore(FEXCore::Context::ContextImpl *ctx, FEXCore::Core::In
RAPass->AddRegisterConflict(FEXCore::IR::GPRClass, i * 2 + 1, FEXCore::IR::GPRPairClass, i);
}
for (uint32_t i = 0; i < FEXCore::IR::IROps::OP_LAST + 1; ++i) {
OpHandlers[i] = &Arm64JITCore::Op_Unhandled;
}
RegisterALUHandlers();
RegisterAtomicHandlers();
RegisterBranchHandlers();
RegisterConversionHandlers();
RegisterFlagHandlers();
RegisterMemoryHandlers();
RegisterMiscHandlers();
RegisterMoveHandlers();
RegisterVectorHandlers();
RegisterEncryptionHandlers();
{
// Set up pointers that the JIT needs to load
@@ -543,7 +558,7 @@ Arm64JITCore::Arm64JITCore(FEXCore::Context::ContextImpl *ctx, FEXCore::Core::In
Common.PrintValue = reinterpret_cast<uint64_t>(PrintValue);
Common.PrintVectorValue = reinterpret_cast<uint64_t>(PrintVectorValue);
Common.ThreadRemoveCodeEntryFromJIT = reinterpret_cast<uintptr_t>(&Context::ContextImpl::ThreadRemoveCodeEntryFromJit);
Common.ThreadRemoveCodeEntryFromJIT = reinterpret_cast<uintptr_t>(&Context::Context::ThreadRemoveCodeEntryFromJit);
Common.CPUIDObj = reinterpret_cast<uint64_t>(&CTX->CPUID);
{
@@ -553,7 +568,7 @@ Arm64JITCore::Arm64JITCore(FEXCore::Context::ContextImpl *ctx, FEXCore::Core::In
Common.SyscallHandlerObj = reinterpret_cast<uint64_t>(CTX->SyscallHandler);
Common.SyscallHandlerFunc = reinterpret_cast<uint64_t>(FEXCore::Context::HandleSyscall);
Common.ExitFunctionLink = reinterpret_cast<uintptr_t>(&Context::ContextImpl::ThreadExitFunctionLink<Arm64JITCore_ExitFunctionLink>);
Common.ExitFunctionLink = reinterpret_cast<uintptr_t>(&Context::Context::ThreadExitFunctionLink<Arm64JITCore_ExitFunctionLink>);
// Fill in the fallback handlers
@@ -572,9 +587,9 @@ Arm64JITCore::Arm64JITCore(FEXCore::Context::ContextImpl *ctx, FEXCore::Core::In
ClearCache();
}
void Arm64JITCore::InitializeSignalHandlers(FEXCore::Context::ContextImpl *CTX) {
void Arm64JITCore::InitializeSignalHandlers(FEXCore::Context::Context *CTX) {
CTX->SignalDelegation->RegisterHostSignalHandler(SIGILL, [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext) -> bool {
return reinterpret_cast<Context::ContextImpl*>(Thread->CTX)->Dispatcher->HandleSIGILL(Thread, Signal, info, ucontext);
return Thread->CTX->Dispatcher->HandleSIGILL(Thread, Signal, info, ucontext);
}, true);
#ifdef _M_ARM_64
@@ -584,7 +599,7 @@ void Arm64JITCore::InitializeSignalHandlers(FEXCore::Context::ContextImpl *CTX)
return false;
}
return FEXCore::ArchHelpers::Arm64::HandleSIGBUS(static_cast<Context::ContextImpl*>(Thread->CTX)->Config.ParanoidTSO(), Signal, info, ucontext);
return FEXCore::ArchHelpers::Arm64::HandleSIGBUS(Thread->CTX->Config.ParanoidTSO(), Signal, info, ucontext);
}, true);
#endif
}
@@ -656,7 +671,7 @@ bool Arm64JITCore::IsGPR(IR::NodeID Node) const {
return Class == IR::GPRClass || Class == IR::GPRFixedClass;
}
CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry,
void *Arm64JITCore::CompileCode(uint64_t Entry,
FEXCore::IR::IRListView const *IR,
FEXCore::Core::DebugData *DebugData,
FEXCore::IR::RegisterAllocationData *RAData,
@@ -669,21 +684,6 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry,
this->Entry = Entry;
this->RAData = RAData;
this->DebugData = DebugData;
this->IR = IR;
// Fairly excessive buffer range to make sure we don't overflow
uint32_t BufferRange = SSACount * 16 + GDBEnabled * Dispatcher::MaxGDBPauseCheckSize;
if ((GetCursorOffset() + BufferRange) > CurrentCodeBuffer->Size) {
CTX->ClearCodeCache(ThreadState);
}
CodeData.BlockBegin = GetCursorAddress<uint8_t*>();
// Put the code header at the start of the data block.
ARMEmitter::BackwardLabel JITCodeHeaderLabel{};
Bind(&JITCodeHeaderLabel);
JITCodeHeader *CodeHeader = GetCursorAddress<JITCodeHeader *>();
CursorIncrement(sizeof(JITCodeHeader));
#ifdef VIXL_DISASSEMBLER
const auto DisasmBegin = GetCursorAddress<const vixl::aarch64::Instruction*>();
@@ -693,6 +693,14 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry,
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, Entry);
#endif
this->IR = IR;
// Fairly excessive buffer range to make sure we don't overflow
uint32_t BufferRange = SSACount * 16 + GDBEnabled * Dispatcher::MaxGDBPauseCheckSize;
if ((GetCursorOffset() + BufferRange) > CurrentCodeBuffer->Size) {
CTX->ClearCodeCache(ThreadState);
}
// AAPCS64
// r30 = LR
// r29 = FP
@@ -713,15 +721,10 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry,
// X1-X3 = Temp
// X4-r18 = RA
CodeData.BlockEntry = GetCursorAddress<uint8_t*>();
// Get the address of the JITCodeHeader and store in to the core state.
// Two instruction cost, each 1 cycle.
adr(TMP1, &JITCodeHeaderLabel);
str(TMP1, STATE, offsetof(FEXCore::Core::CPUState, InlineJITBlockHeader));
GuestEntry = GetCursorAddress<uint8_t *>();
if (GDBEnabled) {
auto GDBSize = CTX->Dispatcher->GenerateGDBPauseCheck(CodeData.BlockEntry, Entry);
auto GDBSize = CTX->Dispatcher->GenerateGDBPauseCheck(GuestEntry, Entry);
CursorIncrement(GDBSize);
}
@@ -766,266 +769,15 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry,
for (auto [CodeNode, IROp] : IR->GetCode(BlockNode)) {
const auto ID = IR->GetID(CodeNode);
switch (IROp->Op) {
#define REGISTER_OP(op, x) case FEXCore::IR::IROps::OP_##op: Op_##x(IROp, ID); break
// ALU ops
REGISTER_OP(TRUNCELEMENTPAIR, TruncElementPair);
REGISTER_OP(CONSTANT, Constant);
REGISTER_OP(ENTRYPOINTOFFSET, EntrypointOffset);
REGISTER_OP(INLINECONSTANT, InlineConstant);
REGISTER_OP(INLINEENTRYPOINTOFFSET, InlineEntrypointOffset);
REGISTER_OP(CYCLECOUNTER, CycleCounter);
REGISTER_OP(ADD, Add);
REGISTER_OP(SUB, Sub);
REGISTER_OP(NEG, Neg);
REGISTER_OP(MUL, Mul);
REGISTER_OP(UMUL, UMul);
REGISTER_OP(DIV, Div);
REGISTER_OP(UDIV, UDiv);
REGISTER_OP(REM, Rem);
REGISTER_OP(UREM, URem);
REGISTER_OP(MULH, MulH);
REGISTER_OP(UMULH, UMulH);
REGISTER_OP(OR, Or);
REGISTER_OP(AND, And);
REGISTER_OP(ANDN, Andn);
REGISTER_OP(XOR, Xor);
REGISTER_OP(LSHL, Lshl);
REGISTER_OP(LSHR, Lshr);
REGISTER_OP(ASHR, Ashr);
REGISTER_OP(ROR, Ror);
REGISTER_OP(EXTR, Extr);
REGISTER_OP(PDEP, PDep);
REGISTER_OP(PEXT, PExt);
REGISTER_OP(LDIV, LDiv);
REGISTER_OP(LUDIV, LUDiv);
REGISTER_OP(LREM, LRem);
REGISTER_OP(LUREM, LURem);
REGISTER_OP(NOT, Not);
REGISTER_OP(POPCOUNT, Popcount);
REGISTER_OP(FINDLSB, FindLSB);
REGISTER_OP(FINDMSB, FindMSB);
REGISTER_OP(FINDTRAILINGZEROS, FindTrailingZeros);
REGISTER_OP(COUNTLEADINGZEROES, CountLeadingZeroes);
REGISTER_OP(REV, Rev);
REGISTER_OP(BFI, Bfi);
REGISTER_OP(BFE, Bfe);
REGISTER_OP(SBFE, Sbfe);
REGISTER_OP(SELECT, Select);
REGISTER_OP(VEXTRACTTOGPR, VExtractToGPR);
REGISTER_OP(FLOAT_TOGPR_ZS, Float_ToGPR_ZS);
REGISTER_OP(FLOAT_TOGPR_S, Float_ToGPR_S);
REGISTER_OP(FCMP, FCmp);
// Atomic ops
REGISTER_OP(CASPAIR, CASPair);
REGISTER_OP(CAS, CAS);
REGISTER_OP(ATOMICADD, AtomicAdd);
REGISTER_OP(ATOMICSUB, AtomicSub);
REGISTER_OP(ATOMICAND, AtomicAnd);
REGISTER_OP(ATOMICOR, AtomicOr);
REGISTER_OP(ATOMICXOR, AtomicXor);
REGISTER_OP(ATOMICSWAP, AtomicSwap);
REGISTER_OP(ATOMICFETCHADD, AtomicFetchAdd);
REGISTER_OP(ATOMICFETCHSUB, AtomicFetchSub);
REGISTER_OP(ATOMICFETCHAND, AtomicFetchAnd);
REGISTER_OP(ATOMICFETCHOR, AtomicFetchOr);
REGISTER_OP(ATOMICFETCHXOR, AtomicFetchXor);
REGISTER_OP(ATOMICFETCHNEG, AtomicFetchNeg);
// Branch ops
REGISTER_OP(CALLBACKRETURN, CallbackReturn);
REGISTER_OP(EXITFUNCTION, ExitFunction);
REGISTER_OP(JUMP, Jump);
REGISTER_OP(CONDJUMP, CondJump);
REGISTER_OP(SYSCALL, Syscall);
REGISTER_OP(INLINESYSCALL, InlineSyscall);
REGISTER_OP(THUNK, Thunk);
REGISTER_OP(VALIDATECODE, ValidateCode);
REGISTER_OP(THREADREMOVECODEENTRY, ThreadRemoveCodeEntry);
REGISTER_OP(CPUID, CPUID);
// Conversion ops
REGISTER_OP(VINSGPR, VInsGPR);
REGISTER_OP(VCASTFROMGPR, VCastFromGPR);
REGISTER_OP(VDUPFROMGPR, VDupFromGPR);
REGISTER_OP(FLOAT_FROMGPR_S, Float_FromGPR_S);
REGISTER_OP(FLOAT_FTOF, Float_FToF);
REGISTER_OP(VECTOR_STOF, Vector_SToF);
REGISTER_OP(VECTOR_FTOZS, Vector_FToZS);
REGISTER_OP(VECTOR_FTOS, Vector_FToS);
REGISTER_OP(VECTOR_FTOF, Vector_FToF);
REGISTER_OP(VECTOR_FTOI, Vector_FToI);
// Encryption ops
REGISTER_OP(VAESIMC, AESImc);
REGISTER_OP(VAESENC, AESEnc);
REGISTER_OP(VAESENCLAST, AESEncLast);
REGISTER_OP(VAESDEC, AESDec);
REGISTER_OP(VAESDECLAST, AESDecLast);
REGISTER_OP(VAESKEYGENASSIST, AESKeyGenAssist);
REGISTER_OP(CRC32, CRC32);
REGISTER_OP(PCLMUL, PCLMUL);
// Flag ops
REGISTER_OP(GETHOSTFLAG, GetHostFlag);
// Memory ops
REGISTER_OP(LOADCONTEXT, LoadContext);
REGISTER_OP(STORECONTEXT, StoreContext);
REGISTER_OP(LOADREGISTER, LoadRegister);
REGISTER_OP(STOREREGISTER, StoreRegister);
REGISTER_OP(LOADCONTEXTINDEXED, LoadContextIndexed);
REGISTER_OP(STORECONTEXTINDEXED, StoreContextIndexed);
REGISTER_OP(SPILLREGISTER, SpillRegister);
REGISTER_OP(FILLREGISTER, FillRegister);
REGISTER_OP(LOADFLAG, LoadFlag);
REGISTER_OP(STOREFLAG, StoreFlag);
REGISTER_OP(LOADMEM, LoadMem);
REGISTER_OP(STOREMEM, StoreMem);
case FEXCore::IR::IROps::OP_LOADMEMTSO:
if (ParanoidTSO()) {
Op_ParanoidLoadMemTSO(IROp, ID);
}
else {
Op_LoadMemTSO(IROp, ID);
}
break;
case FEXCore::IR::IROps::OP_STOREMEMTSO:
if (ParanoidTSO()) {
Op_ParanoidStoreMemTSO(IROp, ID);
}
else {
Op_StoreMemTSO(IROp, ID);
}
break;
REGISTER_OP(MEMSET, MemSet);
REGISTER_OP(CACHELINECLEAR, CacheLineClear);
REGISTER_OP(CACHELINECLEAN, CacheLineClean);
REGISTER_OP(CACHELINEZERO, CacheLineZero);
// Misc ops
REGISTER_OP(DUMMY, NoOp);
REGISTER_OP(IRHEADER, NoOp);
REGISTER_OP(CODEBLOCK, NoOp);
REGISTER_OP(BEGINBLOCK, NoOp);
REGISTER_OP(ENDBLOCK, NoOp);
REGISTER_OP(GUESTOPCODE, GuestOpcode);
REGISTER_OP(FENCE, Fence);
REGISTER_OP(BREAK, Break);
REGISTER_OP(PHI, NoOp);
REGISTER_OP(PHIVALUE, NoOp);
REGISTER_OP(PRINT, Print);
REGISTER_OP(GETROUNDINGMODE, GetRoundingMode);
REGISTER_OP(SETROUNDINGMODE, SetRoundingMode);
REGISTER_OP(INVALIDATEFLAGS, NoOp);
REGISTER_OP(PROCESSORID, ProcessorID);
REGISTER_OP(RDRAND, RDRAND);
REGISTER_OP(YIELD, Yield);
// Move ops
REGISTER_OP(EXTRACTELEMENTPAIR, ExtractElementPair);
REGISTER_OP(CREATEELEMENTPAIR, CreateElementPair);
// Vector ops
REGISTER_OP(VECTORZERO, VectorZero);
REGISTER_OP(VECTORIMM, VectorImm);
REGISTER_OP(VMOV, VMov);
REGISTER_OP(VAND, VAnd);
REGISTER_OP(VBIC, VBic);
REGISTER_OP(VOR, VOr);
REGISTER_OP(VXOR, VXor);
REGISTER_OP(VADD, VAdd);
REGISTER_OP(VSUB, VSub);
REGISTER_OP(VUQADD, VUQAdd);
REGISTER_OP(VUQSUB, VUQSub);
REGISTER_OP(VSQADD, VSQAdd);
REGISTER_OP(VSQSUB, VSQSub);
REGISTER_OP(VADDP, VAddP);
REGISTER_OP(VADDV, VAddV);
REGISTER_OP(VUMINV, VUMinV);
REGISTER_OP(VURAVG, VURAvg);
REGISTER_OP(VABS, VAbs);
REGISTER_OP(VPOPCOUNT, VPopcount);
REGISTER_OP(VFADD, VFAdd);
REGISTER_OP(VFADDP, VFAddP);
REGISTER_OP(VFSUB, VFSub);
REGISTER_OP(VFMUL, VFMul);
REGISTER_OP(VFDIV, VFDiv);
REGISTER_OP(VFMIN, VFMin);
REGISTER_OP(VFMAX, VFMax);
REGISTER_OP(VFRECP, VFRecp);
REGISTER_OP(VFSQRT, VFSqrt);
REGISTER_OP(VFRSQRT, VFRSqrt);
REGISTER_OP(VNEG, VNeg);
REGISTER_OP(VFNEG, VFNeg);
REGISTER_OP(VNOT, VNot);
REGISTER_OP(VUMIN, VUMin);
REGISTER_OP(VSMIN, VSMin);
REGISTER_OP(VUMAX, VUMax);
REGISTER_OP(VSMAX, VSMax);
REGISTER_OP(VZIP, VZip);
REGISTER_OP(VZIP2, VZip2);
REGISTER_OP(VUNZIP, VUnZip);
REGISTER_OP(VUNZIP2, VUnZip2);
REGISTER_OP(VTRN, VTrn);
REGISTER_OP(VTRN2, VTrn2);
REGISTER_OP(VBSL, VBSL);
REGISTER_OP(VCMPEQ, VCMPEQ);
REGISTER_OP(VCMPEQZ, VCMPEQZ);
REGISTER_OP(VCMPGT, VCMPGT);
REGISTER_OP(VCMPGTZ, VCMPGTZ);
REGISTER_OP(VCMPLTZ, VCMPLTZ);
REGISTER_OP(VFCMPEQ, VFCMPEQ);
REGISTER_OP(VFCMPNEQ, VFCMPNEQ);
REGISTER_OP(VFCMPLT, VFCMPLT);
REGISTER_OP(VFCMPGT, VFCMPGT);
REGISTER_OP(VFCMPLE, VFCMPLE);
REGISTER_OP(VFCMPORD, VFCMPORD);
REGISTER_OP(VFCMPUNO, VFCMPUNO);
REGISTER_OP(VUSHL, VUShl);
REGISTER_OP(VUSHR, VUShr);
REGISTER_OP(VSSHR, VSShr);
REGISTER_OP(VUSHLS, VUShlS);
REGISTER_OP(VUSHRS, VUShrS);
REGISTER_OP(VSSHRS, VSShrS);
REGISTER_OP(VINSELEMENT, VInsElement);
REGISTER_OP(VDUPELEMENT, VDupElement);
REGISTER_OP(VEXTR, VExtr);
REGISTER_OP(VUSHRI, VUShrI);
REGISTER_OP(VSSHRI, VSShrI);
REGISTER_OP(VSHLI, VShlI);
REGISTER_OP(VUSHRNI, VUShrNI);
REGISTER_OP(VUSHRNI2, VUShrNI2);
REGISTER_OP(VSXTL, VSXTL);
REGISTER_OP(VSXTL2, VSXTL2);
REGISTER_OP(VUXTL, VUXTL);
REGISTER_OP(VUXTL2, VUXTL2);
REGISTER_OP(VSQXTN, VSQXTN);
REGISTER_OP(VSQXTN2, VSQXTN2);
REGISTER_OP(VSQXTUN, VSQXTUN);
REGISTER_OP(VSQXTUN2, VSQXTUN2);
REGISTER_OP(VUMUL, VMul);
REGISTER_OP(VSMUL, VMul);
REGISTER_OP(VUMULL, VUMull);
REGISTER_OP(VSMULL, VSMull);
REGISTER_OP(VUMULL2, VUMull2);
REGISTER_OP(VSMULL2, VSMull2);
REGISTER_OP(VUABDL, VUABDL);
REGISTER_OP(VTBL1, VTBL1);
REGISTER_OP(VREV64, VRev64);
#undef REGISTER_OP
default:
Op_Unhandled(IROp, ID);
break;
}
// Execute handler
OpHandler Handler = OpHandlers[IROp->Op];
(this->*Handler)(IROp, ID);
}
if (DebugData) {
DebugData->Subblocks.push_back({
static_cast<uint32_t>(BlockStartHostCode - CodeData.BlockEntry),
static_cast<uint32_t>(BlockStartHostCode - GuestEntry),
static_cast<uint32_t>(GetCursorAddress<uint8_t *>() - BlockStartHostCode)
});
}
@@ -1038,24 +790,8 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry,
}
PendingTargetLabel = nullptr;
// Add the JitCodeTail
auto JITBlockTailLocation = GetCursorAddress<uint8_t *>();
auto JITBlockTail = GetCursorAddress<JITCodeTail*>();
CursorIncrement(sizeof(JITCodeTail));
// Put the block's RIP entry in the tail.
// This will be used for RIP reconstruction in the future.
// TODO: This needs to be a data RIP relocation once code caching works.
// Current relocation code doesn't support this feature yet.
JITBlockTail->RIP = Entry;
CodeHeader->OffsetToBlockTail = JITBlockTailLocation - CodeData.BlockBegin;
CodeData.Size = GetCursorAddress<uint8_t *>() - CodeData.BlockBegin;
JITBlockTail->Size = CodeData.Size;
ClearICache(CodeData.BlockBegin, CodeData.Size);
auto CodeEnd = GetCursorAddress<uint8_t *>();
ClearICache(GuestEntry, CodeEnd - GuestEntry);
#ifdef VIXL_DISASSEMBLER
const auto DisasmEnd = GetCursorAddress<const vixl::aarch64::Instruction*>();
@@ -1063,13 +799,13 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry,
#endif
if (DebugData) {
DebugData->HostCodeSize = CodeData.Size;
DebugData->HostCodeSize = CodeEnd - GuestEntry;
DebugData->Relocations = &Relocations;
}
this->IR = nullptr;
return CodeData;
return GuestEntry;
}
void Arm64JITCore::ResetStack() {
@@ -1088,11 +824,11 @@ void Arm64JITCore::ResetStack() {
}
}
std::unique_ptr<CPUBackend> CreateArm64JITCore(FEXCore::Context::ContextImpl *ctx, FEXCore::Core::InternalThreadState *Thread) {
std::unique_ptr<CPUBackend> CreateArm64JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread) {
return std::make_unique<Arm64JITCore>(ctx, Thread);
}
void InitializeArm64JITSignalHandlers(FEXCore::Context::ContextImpl *CTX) {
void InitializeArm64JITSignalHandlers(FEXCore::Context::Context *CTX) {
Arm64JITCore::InitializeSignalHandlers(CTX);
}
+22 -10
View File
@@ -31,13 +31,13 @@ namespace FEXCore::Core {
namespace FEXCore::CPU {
class Arm64JITCore final : public CPUBackend, public Arm64Emitter {
public:
explicit Arm64JITCore(FEXCore::Context::ContextImpl *ctx,
explicit Arm64JITCore(FEXCore::Context::Context *ctx,
FEXCore::Core::InternalThreadState *Thread);
~Arm64JITCore() override;
[[nodiscard]] std::string GetName() override { return "JIT"; }
[[nodiscard]] CPUBackend::CompiledCode CompileCode(uint64_t Entry,
[[nodiscard]] void *CompileCode(uint64_t Entry,
FEXCore::IR::IRListView const *IR,
FEXCore::Core::DebugData *DebugData,
FEXCore::IR::RegisterAllocationData *RAData, bool GDBEnabled) override;
@@ -48,7 +48,7 @@ public:
void ClearCache() override;
static void InitializeSignalHandlers(FEXCore::Context::ContextImpl *CTX);
static void InitializeSignalHandlers(FEXCore::Context::Context *CTX);
void ClearRelocations() override { Relocations.clear(); }
@@ -57,10 +57,9 @@ private:
const bool HostSupportsSVE{};
ARMEmitter::BiDirectionalLabel *PendingTargetLabel;
FEXCore::Context::ContextImpl *CTX;
FEXCore::Context::Context *CTX;
FEXCore::IR::IRListView const *IR;
uint64_t Entry;
CPUBackend::CompiledCode CodeData{};
std::map<IR::NodeID, ARMEmitter::BiDirectionalLabel> JumpTargets;
@@ -231,6 +230,23 @@ private:
/** @} */
uint32_t SpillSlots{};
/**
* @brief Current guest RIP entrypoint
*/
uint8_t *GuestEntry{};
using OpHandler = void (Arm64JITCore::*)(IR::IROp_Header const *IROp, IR::NodeID Node);
std::array<OpHandler, IR::IROps::OP_LAST + 1> OpHandlers {};
void RegisterALUHandlers();
void RegisterAtomicHandlers();
void RegisterBranchHandlers();
void RegisterConversionHandlers();
void RegisterFlagHandlers();
void RegisterMemoryHandlers();
void RegisterMiscHandlers();
void RegisterMoveHandlers();
void RegisterVectorHandlers();
void RegisterEncryptionHandlers();
#define DEF_OP(x) void Op_##x(IR::IROp_Header const *IROp, IR::NodeID Node)
///< Unhandled handler
@@ -308,6 +324,7 @@ private:
DEF_OP(AtomicFetchNeg);
///< Branch ops
DEF_OP(SignalReturn);
DEF_OP(CallbackReturn);
DEF_OP(ExitFunction);
DEF_OP(Jump);
@@ -322,7 +339,6 @@ private:
///< Conversion ops
DEF_OP(VInsGPR);
DEF_OP(VCastFromGPR);
DEF_OP(VDupFromGPR);
DEF_OP(Float_FromGPR_S);
DEF_OP(Float_FToF);
DEF_OP(Vector_SToF);
@@ -349,11 +365,9 @@ private:
DEF_OP(StoreMem);
DEF_OP(LoadMemTSO);
DEF_OP(StoreMemTSO);
DEF_OP(MemSet);
DEF_OP(ParanoidLoadMemTSO);
DEF_OP(ParanoidStoreMemTSO);
DEF_OP(CacheLineClear);
DEF_OP(CacheLineClean);
DEF_OP(CacheLineZero);
///< Misc ops
@@ -414,8 +428,6 @@ private:
DEF_OP(VZip2);
DEF_OP(VUnZip);
DEF_OP(VUnZip2);
DEF_OP(VTrn);
DEF_OP(VTrn2);
DEF_OP(VBSL);
DEF_OP(VCMPEQ);
DEF_OP(VCMPEQZ);
+50 -288
View File
@@ -703,43 +703,19 @@ DEF_OP(SpillRegister) {
const auto Src = GetReg(Op->Value.ID());
switch (OpSize) {
case 1: {
if (SlotOffset > LSByteMaxUnsignedOffset) {
LoadConstant(ARMEmitter::Size::i64Bit, TMP1, SlotOffset);
strb(Src, ARMEmitter::Reg::rsp, TMP1.R(), ARMEmitter::ExtendedType::LSL_64, 0);
}
else {
strb(Src, ARMEmitter::Reg::rsp, SlotOffset);
}
strb(Src, ARMEmitter::Reg::rsp, SlotOffset);
break;
}
case 2: {
if (SlotOffset > LSHalfMaxUnsignedOffset) {
LoadConstant(ARMEmitter::Size::i64Bit, TMP1, SlotOffset);
strh(Src, ARMEmitter::Reg::rsp, TMP1.R(), ARMEmitter::ExtendedType::LSL_64, 0);
}
else {
strh(Src, ARMEmitter::Reg::rsp, SlotOffset);
}
strh(Src, ARMEmitter::Reg::rsp, SlotOffset);
break;
}
case 4: {
if (SlotOffset > LSWordMaxUnsignedOffset) {
LoadConstant(ARMEmitter::Size::i64Bit, TMP1, SlotOffset);
str(Src.W(), ARMEmitter::Reg::rsp, TMP1.R(), ARMEmitter::ExtendedType::LSL_64, 0);
}
else {
str(Src.W(), ARMEmitter::Reg::rsp, SlotOffset);
}
str(Src.W(), ARMEmitter::Reg::rsp, SlotOffset);
break;
}
case 8: {
if (SlotOffset > LSDWordMaxUnsignedOffset) {
LoadConstant(ARMEmitter::Size::i64Bit, TMP1, SlotOffset);
str(Src.X(), ARMEmitter::Reg::rsp, TMP1.R(), ARMEmitter::ExtendedType::LSL_64, 0);
}
else {
str(Src.X(), ARMEmitter::Reg::rsp, SlotOffset);
}
str(Src.X(), ARMEmitter::Reg::rsp, SlotOffset);
break;
}
default:
@@ -751,33 +727,15 @@ DEF_OP(SpillRegister) {
switch (OpSize) {
case 4: {
if (SlotOffset > LSWordMaxUnsignedOffset) {
LoadConstant(ARMEmitter::Size::i64Bit, TMP1, SlotOffset);
str(Src.S(), ARMEmitter::Reg::rsp, TMP1.R(), ARMEmitter::ExtendedType::LSL_64, 0);
}
else {
str(Src.S(), ARMEmitter::Reg::rsp, SlotOffset);
}
str(Src.S(), ARMEmitter::Reg::rsp, SlotOffset);
break;
}
case 8: {
if (SlotOffset > LSDWordMaxUnsignedOffset) {
LoadConstant(ARMEmitter::Size::i64Bit, TMP1, SlotOffset);
str(Src.D(), ARMEmitter::Reg::rsp, TMP1.R(), ARMEmitter::ExtendedType::LSL_64, 0);
}
else {
str(Src.D(), ARMEmitter::Reg::rsp, SlotOffset);
}
str(Src.D(), ARMEmitter::Reg::rsp, SlotOffset);
break;
}
case 16: {
if (SlotOffset > LSQWordMaxUnsignedOffset) {
LoadConstant(ARMEmitter::Size::i64Bit, TMP1, SlotOffset);
str(Src.Q(), ARMEmitter::Reg::rsp, TMP1.R(), ARMEmitter::ExtendedType::LSL_64, 0);
}
else {
str(Src.Q(), ARMEmitter::Reg::rsp, SlotOffset);
}
str(Src.Q(), ARMEmitter::Reg::rsp, SlotOffset);
break;
}
case 32: {
@@ -803,43 +761,19 @@ DEF_OP(FillRegister) {
const auto Dst = GetReg(Node);
switch (OpSize) {
case 1: {
if (SlotOffset > LSByteMaxUnsignedOffset) {
LoadConstant(ARMEmitter::Size::i64Bit, TMP1, SlotOffset);
ldrb(Dst, ARMEmitter::Reg::rsp, TMP1.R(), ARMEmitter::ExtendedType::LSL_64, 0);
}
else {
ldrb(Dst, ARMEmitter::Reg::rsp, SlotOffset);
}
ldrb(Dst, ARMEmitter::Reg::rsp, SlotOffset);
break;
}
case 2: {
if (SlotOffset > LSHalfMaxUnsignedOffset) {
LoadConstant(ARMEmitter::Size::i64Bit, TMP1, SlotOffset);
ldrh(Dst, ARMEmitter::Reg::rsp, TMP1.R(), ARMEmitter::ExtendedType::LSL_64, 0);
}
else {
ldrh(Dst, ARMEmitter::Reg::rsp, SlotOffset);
}
ldrh(Dst, ARMEmitter::Reg::rsp, SlotOffset);
break;
}
case 4: {
if (SlotOffset > LSWordMaxUnsignedOffset) {
LoadConstant(ARMEmitter::Size::i64Bit, TMP1, SlotOffset);
ldr(Dst.W(), ARMEmitter::Reg::rsp, TMP1.R(), ARMEmitter::ExtendedType::LSL_64, 0);
}
else {
ldr(Dst.W(), ARMEmitter::Reg::rsp, SlotOffset);
}
ldr(Dst.W(), ARMEmitter::Reg::rsp, SlotOffset);
break;
}
case 8: {
if (SlotOffset > LSDWordMaxUnsignedOffset) {
LoadConstant(ARMEmitter::Size::i64Bit, TMP1, SlotOffset);
ldr(Dst.X(), ARMEmitter::Reg::rsp, TMP1.R(), ARMEmitter::ExtendedType::LSL_64, 0);
}
else {
ldr(Dst.X(), ARMEmitter::Reg::rsp, SlotOffset);
}
ldr(Dst.X(), ARMEmitter::Reg::rsp, SlotOffset);
break;
}
default:
@@ -851,33 +785,15 @@ DEF_OP(FillRegister) {
switch (OpSize) {
case 4: {
if (SlotOffset > LSWordMaxUnsignedOffset) {
LoadConstant(ARMEmitter::Size::i64Bit, TMP1, SlotOffset);
ldr(Dst.S(), ARMEmitter::Reg::rsp, TMP1.R(), ARMEmitter::ExtendedType::LSL_64, 0);
}
else {
ldr(Dst.S(), ARMEmitter::Reg::rsp, SlotOffset);
}
ldr(Dst.S(), ARMEmitter::Reg::rsp, SlotOffset);
break;
}
case 8: {
if (SlotOffset > LSDWordMaxUnsignedOffset) {
LoadConstant(ARMEmitter::Size::i64Bit, TMP1, SlotOffset);
ldr(Dst.D(), ARMEmitter::Reg::rsp, TMP1.R(), ARMEmitter::ExtendedType::LSL_64, 0);
}
else {
ldr(Dst.D(), ARMEmitter::Reg::rsp, SlotOffset);
}
ldr(Dst.D(), ARMEmitter::Reg::rsp, SlotOffset);
break;
}
case 16: {
if (SlotOffset > LSQWordMaxUnsignedOffset) {
LoadConstant(ARMEmitter::Size::i64Bit, TMP1, SlotOffset);
ldr(Dst.Q(), ARMEmitter::Reg::rsp, TMP1.R(), ARMEmitter::ExtendedType::LSL_64, 0);
}
else {
ldr(Dst.Q(), ARMEmitter::Reg::rsp, SlotOffset);
}
ldr(Dst.Q(), ARMEmitter::Reg::rsp, SlotOffset);
break;
}
case 32: {
@@ -911,20 +827,20 @@ FEXCore::ARMEmitter::ExtendedMemOperand Arm64JITCore::GenerateMemOperand(uint8_t
IR::MemOffsetType OffsetType,
uint8_t OffsetScale) {
if (Offset.IsInvalid()) {
return ARMEmitter::ExtendedMemOperand(Base.X(), ARMEmitter::IndexType::OFFSET, 0);
return FEXCore::ARMEmitter::ExtendedMemOperand(Base, ARMEmitter::IndexType::OFFSET, 0);
} else {
if (OffsetScale != 1 && OffsetScale != AccessSize) {
LOGMAN_MSG_A_FMT("Unhandled GenerateMemOperand OffsetScale: {}", OffsetScale);
}
uint64_t Const;
if (IsInlineConstant(Offset, &Const)) {
return ARMEmitter::ExtendedMemOperand(Base.X(), ARMEmitter::IndexType::OFFSET, Const);
return FEXCore::ARMEmitter::ExtendedMemOperand(Base, ARMEmitter::IndexType::OFFSET, Const);
} else {
auto RegOffset = GetReg(Offset.ID());
switch(OffsetType.Val) {
case IR::MEM_OFFSET_SXTX.Val: return ARMEmitter::ExtendedMemOperand(Base.X(), RegOffset.X(), ARMEmitter::ExtendedType::SXTX, (int)std::log2(OffsetScale) );
case IR::MEM_OFFSET_UXTW.Val: return ARMEmitter::ExtendedMemOperand(Base.X(), RegOffset.X(), ARMEmitter::ExtendedType::UXTW, (int)std::log2(OffsetScale) );
case IR::MEM_OFFSET_SXTW.Val: return ARMEmitter::ExtendedMemOperand(Base.X(), RegOffset.X(), ARMEmitter::ExtendedType::SXTW, (int)std::log2(OffsetScale) );
case IR::MEM_OFFSET_SXTX.Val: return FEXCore::ARMEmitter::ExtendedMemOperand(Base, RegOffset, FEXCore::ARMEmitter::ExtendedType::SXTX, (int)std::log2(OffsetScale) );
case IR::MEM_OFFSET_UXTW.Val: return FEXCore::ARMEmitter::ExtendedMemOperand(Base, RegOffset, FEXCore::ARMEmitter::ExtendedType::UXTW, (int)std::log2(OffsetScale) );
case IR::MEM_OFFSET_SXTW.Val: return FEXCore::ARMEmitter::ExtendedMemOperand(Base, RegOffset, FEXCore::ARMEmitter::ExtendedType::SXTW, (int)std::log2(OffsetScale) );
default: LOGMAN_MSG_A_FMT("Unhandled GenerateMemOperand OffsetType: {}", OffsetType.Val); break;
}
}
@@ -1101,14 +1017,14 @@ DEF_OP(LoadMemTSO) {
const auto Dst = GetReg(Node);
if (OpSize == 1) {
// 8bit load is always aligned to natural alignment
ldaprb(Dst.W(), MemReg);
ldaprb(Dst, MemReg);
}
else {
// Aligned
nop();
switch (OpSize) {
case 2:
ldaprh(Dst.W(), MemReg);
ldaprh(Dst, MemReg);
break;
case 4:
ldapr(Dst.W(), MemReg);
@@ -1340,169 +1256,6 @@ DEF_OP(StoreMemTSO) {
}
}
DEF_OP(MemSet) {
// TODO: A future looking task would be to support this with ARM's MOPS instructions.
// The 8-bit non-atomic forward path directly matches ARM's SETP/SETM/SETE instruction,
// while the backward version needs some fixup to convert it to a forward direction.
//
// Assuming non-atomicity and non-faulting behaviour, this can accelerate this implementation.
// Additionally: This is commonly used as a memset to zero. If we know up-front with an inline constant
// that the value is zero, we can optimize any operation larger than 8-bit down to 8-bit to use the MOPS implementation.
const auto Op = IROp->C<IR::IROp_MemSet>();
const int32_t Size = Op->Size;
const auto MemReg = GetReg(Op->Addr.ID());
const auto Value = GetReg(Op->Value.ID());
const auto Length = GetReg(Op->Length.ID());
const auto Direction = GetReg(Op->Direction.ID());
const auto Dst = GetReg(Node);
// If Direction == 0 then:
// MemReg is incremented (by size)
// else:
// MemReg is decremented (by size)
//
// Counter is decremented regardless.
ARMEmitter::ForwardLabel BackwardImpl{};
ARMEmitter::ForwardLabel Done{};
mov(TMP1, Length.X());
if (Op->Prefix.IsInvalid()) {
mov(TMP2, MemReg.X());
}
else {
const auto Prefix = GetReg(Op->Prefix.ID());
add(TMP2, Prefix.X(), MemReg.X());
}
// Backward or forwards implementation depends on flag
cbnz(ARMEmitter::Size::i64Bit, Direction, &BackwardImpl);
auto MemStore = [this](auto Value, uint32_t OpSize, int32_t Size) {
switch (OpSize) {
case 1:
strb<ARMEmitter::IndexType::POST>(Value.W(), TMP2, Size);
break;
case 2:
strh<ARMEmitter::IndexType::POST>(Value.W(), TMP2, Size);
break;
case 4:
str<ARMEmitter::IndexType::POST>(Value.W(), TMP2, Size);
break;
case 8:
str<ARMEmitter::IndexType::POST>(Value.X(), TMP2, Size);
break;
default:
LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, Size);
break;
}
};
auto MemStoreTSO = [this](auto Value, uint32_t OpSize, int32_t Size) {
if (OpSize == 1) {
// 8bit load is always aligned to natural alignment
stlrb(Value.W(), TMP2);
}
else {
nop();
switch (OpSize) {
case 2:
stlrh(Value.W(), TMP2);
break;
case 4:
stlr(Value.W(), TMP2);
break;
case 8:
stlr(Value.X(), TMP2);
break;
default:
LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, Size);
break;
}
nop();
}
if (Size >= 0) {
add(ARMEmitter::Size::i64Bit, TMP2, TMP2, OpSize);
}
else {
sub(ARMEmitter::Size::i64Bit, TMP2, TMP2, OpSize);
}
};
// Emit forward direction memset then backward direction memset.
for (int32_t Direction : { 1, -1 }) {
const int32_t OpSize = Size;
const int32_t SizeDirection = Size * Direction;
ARMEmitter::BackwardLabel AgainInternal{};
ARMEmitter::ForwardLabel DoneInternal{};
// Early exit if zero count.
cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
Bind(&AgainInternal);
if (Op->IsAtomic) {
MemStoreTSO(Value, OpSize, SizeDirection);
}
else {
MemStore(Value, OpSize, SizeDirection);
}
sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 1);
cbnz(ARMEmitter::Size::i64Bit, TMP1, &AgainInternal);
Bind(&DoneInternal);
if (SizeDirection >= 0) {
switch (OpSize) {
case 1:
add(Dst.X(), MemReg.X(), Length.X());
break;
case 2:
add(Dst.X(), MemReg.X(), Length.X(), ARMEmitter::ShiftType::LSL, 1);
break;
case 4:
add(Dst.X(), MemReg.X(), Length.X(), ARMEmitter::ShiftType::LSL, 2);
break;
case 8:
add(Dst.X(), MemReg.X(), Length.X(), ARMEmitter::ShiftType::LSL, 3);
break;
default:
LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, OpSize);
break;
}
}
else {
switch (OpSize) {
case 1:
sub(Dst.X(), MemReg.X(), Length.X());
break;
case 2:
sub(Dst.X(), MemReg.X(), Length.X(), ARMEmitter::ShiftType::LSL, 1);
break;
case 4:
sub(Dst.X(), MemReg.X(), Length.X(), ARMEmitter::ShiftType::LSL, 2);
break;
case 8:
sub(Dst.X(), MemReg.X(), Length.X(), ARMEmitter::ShiftType::LSL, 3);
break;
default:
LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, OpSize);
break;
}
}
if (Direction == 1) {
b(&Done);
Bind(&BackwardImpl);
}
}
Bind(&Done);
// Destination already set to the final pointer.
}
DEF_OP(ParanoidLoadMemTSO) {
const auto Op = IROp->C<IR::IROp_LoadMemTSO>();
const auto OpSize = IROp->Size;
@@ -1636,7 +1389,7 @@ DEF_OP(ParanoidStoreMemTSO) {
}
case 32: {
dmb(FEXCore::ARMEmitter::BarrierScope::ISH);
st1b<ARMEmitter::SubRegSize::i8Bit>(Src.Z(), PRED_TMP_32B, Addr, 0);
st1b<ARMEmitter::SubRegSize::i8Bit>(Src, PRED_TMP_32B, Addr, 0);
dmb(FEXCore::ARMEmitter::BarrierScope::ISH);
break;
}
@@ -1656,27 +1409,10 @@ DEF_OP(CacheLineClear) {
// icache doesn't matter here since the guest application shouldn't be calling clflush on JIT code.
mov(TMP1, MemReg.X());
for (size_t i = 0; i < std::max(1U, CTX->HostFeatures.DCacheLineSize / 64U); ++i) {
dc(ARMEmitter::DataCacheOperation::CIVAC, TMP1);
add(ARMEmitter::Size::i64Bit, TMP1, TMP1, CTX->HostFeatures.DCacheLineSize);
}
if (Op->Serialize) {
// If requested, serialized all of the data cache operations.
dsb(FEXCore::ARMEmitter::BarrierScope::ISH);
}
}
DEF_OP(CacheLineClean) {
auto Op = IROp->C<IR::IROp_CacheLineClean>();
auto MemReg = GetReg(Op->Addr.ID());
// Clean dcache only
mov(TMP1, MemReg.X());
for (size_t i = 0; i < std::max(1U, CTX->HostFeatures.DCacheLineSize / 64U); ++i) {
dc(ARMEmitter::DataCacheOperation::CVAC, TMP1);
dc(ARMEmitter::DataCacheOperation::CVAU, TMP1);
add(ARMEmitter::Size::i64Bit, TMP1, TMP1, CTX->HostFeatures.DCacheLineSize);
}
dsb(FEXCore::ARMEmitter::BarrierScope::ISH);
}
DEF_OP(CacheLineZero) {
@@ -1702,5 +1438,31 @@ DEF_OP(CacheLineZero) {
}
#undef DEF_OP
void Arm64JITCore::RegisterMemoryHandlers() {
#define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &Arm64JITCore::Op_##x
REGISTER_OP(LOADCONTEXT, LoadContext);
REGISTER_OP(STORECONTEXT, StoreContext);
REGISTER_OP(LOADREGISTER, LoadRegister);
REGISTER_OP(STOREREGISTER, StoreRegister);
REGISTER_OP(LOADCONTEXTINDEXED, LoadContextIndexed);
REGISTER_OP(STORECONTEXTINDEXED, StoreContextIndexed);
REGISTER_OP(SPILLREGISTER, SpillRegister);
REGISTER_OP(FILLREGISTER, FillRegister);
REGISTER_OP(LOADFLAG, LoadFlag);
REGISTER_OP(STOREFLAG, StoreFlag);
REGISTER_OP(LOADMEM, LoadMem);
REGISTER_OP(STOREMEM, StoreMem);
if (ParanoidTSO()) {
REGISTER_OP(LOADMEMTSO, ParanoidLoadMemTSO);
REGISTER_OP(STOREMEMTSO, ParanoidStoreMemTSO);
}
else {
REGISTER_OP(LOADMEMTSO, LoadMemTSO);
REGISTER_OP(STOREMEMTSO, StoreMemTSO);
}
REGISTER_OP(CACHELINECLEAR, CacheLineClear);
REGISTER_OP(CACHELINEZERO, CacheLineZero);
#undef REGISTER_OP
}
}
@@ -15,7 +15,7 @@ namespace FEXCore::CPU {
DEF_OP(GuestOpcode) {
auto Op = IROp->C<IR::IROp_GuestOpcode>();
// metadata
DebugData->GuestOpcodes.push_back({Op->GuestEntryOffset, GetCursorAddress<uint8_t*>() - CodeData.BlockBegin});
DebugData->GuestOpcodes.push_back({Op->GuestEntryOffset, GetCursorAddress<uint8_t*>() - GuestEntry});
}
DEF_OP(Fence) {
@@ -231,5 +231,27 @@ DEF_OP(Yield) {
}
#undef DEF_OP
void Arm64JITCore::RegisterMiscHandlers() {
#define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &Arm64JITCore::Op_##x
REGISTER_OP(DUMMY, NoOp);
REGISTER_OP(IRHEADER, NoOp);
REGISTER_OP(CODEBLOCK, NoOp);
REGISTER_OP(BEGINBLOCK, NoOp);
REGISTER_OP(ENDBLOCK, NoOp);
REGISTER_OP(GUESTOPCODE, GuestOpcode);
REGISTER_OP(FENCE, Fence);
REGISTER_OP(BREAK, Break);
REGISTER_OP(PHI, NoOp);
REGISTER_OP(PHIVALUE, NoOp);
REGISTER_OP(PRINT, Print);
REGISTER_OP(GETROUNDINGMODE, GetRoundingMode);
REGISTER_OP(SETROUNDINGMODE, SetRoundingMode);
REGISTER_OP(INVALIDATEFLAGS, NoOp);
REGISTER_OP(PROCESSORID, ProcessorID);
REGISTER_OP(RDRAND, RDRAND);
REGISTER_OP(YIELD, Yield);
#undef REGISTER_OP
}
}
@@ -42,5 +42,11 @@ DEF_OP(CreateElementPair) {
}
#undef DEF_OP
void Arm64JITCore::RegisterMoveHandlers() {
#define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &Arm64JITCore::Op_##x
REGISTER_OP(EXTRACTELEMENTPAIR, ExtractElementPair);
REGISTER_OP(CREATEELEMENTPAIR, CreateElementPair);
#undef REGISTER_OP
}
}
+226 -242
View File
@@ -11,14 +11,12 @@ $end_info$
namespace FEXCore::CPU {
#define DEF_OP(x) void Arm64JITCore::Op_##x(IR::IROp_Header const *IROp, IR::NodeID Node)
DEF_OP(VectorZero) {
const auto OpSize = IROp->Size;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto Dst = GetVReg(Node);
if (HostSupportsSVE && Is256Bit) {
if (HostSupportsSVE) {
eor(Dst.Z(), Dst.Z(), Dst.Z());
} else {
const uint8_t OpSize = IROp->Size;
switch (OpSize) {
case 8: {
eor(Dst.D(), Dst.D(), Dst.D());
@@ -36,11 +34,8 @@ DEF_OP(VectorZero) {
}
DEF_OP(VectorImm) {
const auto Op = IROp->C<IR::IROp_VectorImm>();
const auto OpSize = IROp->Size;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto ElementSize = Op->Header.ElementSize;
auto Op = IROp->C<IR::IROp_VectorImm>();
const uint8_t ElementSize = Op->Header.ElementSize;
LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size");
const auto SubRegSize =
@@ -51,7 +46,7 @@ DEF_OP(VectorImm) {
const auto Dst = GetVReg(Node);
if (HostSupportsSVE && Is256Bit) {
if (HostSupportsSVE) {
if (ElementSize > 1 && (Op->Immediate & 0x80)) {
// SVE dup uses sign extension where VectorImm wants zext
LoadConstant(ARMEmitter::Size::i64Bit, TMP1, Op->Immediate);
@@ -125,15 +120,13 @@ DEF_OP(VMov) {
}
DEF_OP(VAnd) {
const auto Op = IROp->C<IR::IROp_VAnd>();
const auto OpSize = IROp->Size;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
auto Op = IROp->C<IR::IROp_VAnd>();
const auto Dst = GetVReg(Node);
const auto Vector1 = GetVReg(Op->Vector1.ID());
const auto Vector2 = GetVReg(Op->Vector2.ID());
if (HostSupportsSVE && Is256Bit) {
if (HostSupportsSVE) {
and_(Dst.Z(), Vector1.Z(), Vector2.Z());
} else {
and_(Dst.Q(), Vector1.Q(), Vector2.Q());
@@ -141,15 +134,13 @@ DEF_OP(VAnd) {
}
DEF_OP(VBic) {
const auto Op = IROp->C<IR::IROp_VBic>();
const auto OpSize = IROp->Size;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
auto Op = IROp->C<IR::IROp_VBic>();
const auto Dst = GetVReg(Node);
const auto Vector1 = GetVReg(Op->Vector1.ID());
const auto Vector2 = GetVReg(Op->Vector2.ID());
if (HostSupportsSVE && Is256Bit) {
if (HostSupportsSVE) {
bic(Dst.Z(), Vector1.Z(), Vector2.Z());
} else {
bic(Dst.Q(), Vector1.Q(), Vector2.Q());
@@ -157,15 +148,13 @@ DEF_OP(VBic) {
}
DEF_OP(VOr) {
const auto Op = IROp->C<IR::IROp_VOr>();
const auto OpSize = IROp->Size;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
auto Op = IROp->C<IR::IROp_VOr>();
const auto Dst = GetVReg(Node);
const auto Vector1 = GetVReg(Op->Vector1.ID());
const auto Vector2 = GetVReg(Op->Vector2.ID());
if (HostSupportsSVE && Is256Bit) {
if (HostSupportsSVE) {
orr(Dst.Z(), Vector1.Z(), Vector2.Z());
} else {
orr(Dst.Q(), Vector1.Q(), Vector2.Q());
@@ -173,15 +162,13 @@ DEF_OP(VOr) {
}
DEF_OP(VXor) {
const auto Op = IROp->C<IR::IROp_VXor>();
const auto OpSize = IROp->Size;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
auto Op = IROp->C<IR::IROp_VXor>();
const auto Dst = GetVReg(Node);
const auto Vector1 = GetVReg(Op->Vector1.ID());
const auto Vector2 = GetVReg(Op->Vector2.ID());
if (HostSupportsSVE && Is256Bit) {
if (HostSupportsSVE) {
eor(Dst.Z(), Vector1.Z(), Vector2.Z());
} else {
eor(Dst.Q(), Vector1.Q(), Vector2.Q());
@@ -189,10 +176,8 @@ DEF_OP(VXor) {
}
DEF_OP(VAdd) {
const auto Op = IROp->C<IR::IROp_VAdd>();
const auto OpSize = IROp->Size;
auto Op = IROp->C<IR::IROp_VAdd>();
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto ElementSize = Op->Header.ElementSize;
const auto Dst = GetVReg(Node);
@@ -205,19 +190,17 @@ DEF_OP(VAdd) {
ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit :
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit;
if (HostSupportsSVE && Is256Bit) {
if (HostSupportsSVE) {
add(SubRegSize, Dst.Z(), Vector1.Z(), Vector2.Z());
} else {
}
else {
add(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q());
}
}
DEF_OP(VSub) {
const auto Op = IROp->C<IR::IROp_VSub>();
const auto OpSize = IROp->Size;
auto Op = IROp->C<IR::IROp_VSub>();
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto ElementSize = Op->Header.ElementSize;
const auto Dst = GetVReg(Node);
@@ -230,19 +213,17 @@ DEF_OP(VSub) {
ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit :
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit;
if (HostSupportsSVE && Is256Bit) {
if (HostSupportsSVE) {
sub(SubRegSize, Dst.Z(), Vector1.Z(), Vector2.Z());
} else {
}
else {
sub(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q());
}
}
DEF_OP(VUQAdd) {
const auto Op = IROp->C<IR::IROp_VUQAdd>();
const auto OpSize = IROp->Size;
auto Op = IROp->C<IR::IROp_VUQAdd>();
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto ElementSize = Op->Header.ElementSize;
const auto Dst = GetVReg(Node);
@@ -255,19 +236,17 @@ DEF_OP(VUQAdd) {
ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit :
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit;
if (HostSupportsSVE && Is256Bit) {
if (HostSupportsSVE) {
uqadd(SubRegSize, Dst.Z(), Vector1.Z(), Vector2.Z());
} else {
}
else {
uqadd(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q());
}
}
DEF_OP(VUQSub) {
const auto Op = IROp->C<IR::IROp_VUQSub>();
const auto OpSize = IROp->Size;
auto Op = IROp->C<IR::IROp_VUQSub>();
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto ElementSize = Op->Header.ElementSize;
const auto Dst = GetVReg(Node);
@@ -280,19 +259,17 @@ DEF_OP(VUQSub) {
ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit :
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit;
if (HostSupportsSVE && Is256Bit) {
if (HostSupportsSVE) {
uqsub(SubRegSize, Dst.Z(), Vector1.Z(), Vector2.Z());
} else {
}
else {
uqsub(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q());
}
}
DEF_OP(VSQAdd) {
const auto Op = IROp->C<IR::IROp_VSQAdd>();
const auto OpSize = IROp->Size;
auto Op = IROp->C<IR::IROp_VSQAdd>();
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto ElementSize = Op->Header.ElementSize;
const auto Dst = GetVReg(Node);
@@ -305,19 +282,17 @@ DEF_OP(VSQAdd) {
ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit :
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit;
if (HostSupportsSVE && Is256Bit) {
if (HostSupportsSVE) {
sqadd(SubRegSize, Dst.Z(), Vector1.Z(), Vector2.Z());
} else {
}
else {
sqadd(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q());
}
}
DEF_OP(VSQSub) {
const auto Op = IROp->C<IR::IROp_VSQSub>();
const auto OpSize = IROp->Size;
auto Op = IROp->C<IR::IROp_VSQSub>();
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto ElementSize = Op->Header.ElementSize;
const auto Dst = GetVReg(Node);
@@ -330,10 +305,10 @@ DEF_OP(VSQSub) {
ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit :
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit;
if (HostSupportsSVE && Is256Bit) {
if (HostSupportsSVE) {
sqsub(SubRegSize, Dst.Z(), Vector1.Z(), Vector2.Z());
} else {
}
else {
sqsub(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q());
}
}
@@ -357,7 +332,7 @@ DEF_OP(VAddP) {
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit;
if (HostSupportsSVE && Is256Bit) {
if (HostSupportsSVE && Is256Bit && !IsScalar) {
const auto Pred = PRED_TMP_32B.Merging();
// SVE ADDP is a destructive operation, so we need a temporary
@@ -426,10 +401,9 @@ DEF_OP(VAddV) {
}
DEF_OP(VUMinV) {
const auto Op = IROp->C<IR::IROp_VUMinV>();
const auto OpSize = IROp->Size;
auto Op = IROp->C<IR::IROp_VUMinV>();
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto OpSize = IROp->Size;
const auto ElementSize = Op->Header.ElementSize;
const auto Dst = GetVReg(Node);
@@ -442,9 +416,14 @@ DEF_OP(VUMinV) {
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit;
if (HostSupportsSVE && Is256Bit) {
const auto Pred = PRED_TMP_32B;
uminv(SubRegSize, Dst, Pred, Vector.Z());
if (HostSupportsSVE) {
LOGMAN_THROW_AA_FMT(OpSize == 16 || OpSize == 32,
"Unsupported vector length: {}", OpSize);
const auto Pred = OpSize == 16 ? PRED_TMP_16B
: PRED_TMP_32B;
uminv(SubRegSize, Dst.Z(), Pred, Vector.Z());
} else {
// Vector
uminv(SubRegSize, Dst.Q(), Vector.Q());
@@ -486,8 +465,7 @@ DEF_OP(VAbs) {
const auto Op = IROp->C<IR::IROp_VAbs>();
const auto OpSize = IROp->Size;
const auto ElementSize = Op->Header.ElementSize;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const uint8_t ElementSize = Op->Header.ElementSize;
const auto Dst = GetVReg(Node);
const auto Src = GetVReg(Op->Vector.ID());
@@ -499,7 +477,7 @@ DEF_OP(VAbs) {
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit;
if (HostSupportsSVE && Is256Bit) {
if (HostSupportsSVE && OpSize == 32) {
abs(SubRegSize, Dst.Z(), PRED_TMP_32B.Merging(), Src.Z());
} else {
if (ElementSize == OpSize) {
@@ -515,9 +493,7 @@ DEF_OP(VAbs) {
DEF_OP(VPopcount) {
const auto Op = IROp->C<IR::IROp_VPopcount>();
const auto OpSize = IROp->Size;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto IsScalar = OpSize == 8;
const bool IsScalar = OpSize == 8;
const auto ElementSize = Op->Header.ElementSize;
@@ -531,13 +507,17 @@ DEF_OP(VPopcount) {
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit;
if (HostSupportsSVE && Is256Bit) {
const auto Pred = PRED_TMP_32B.Merging();
if (HostSupportsSVE && !IsScalar) {
const auto Pred = OpSize == 16 ? PRED_TMP_16B.Merging()
: PRED_TMP_32B.Merging();
cnt(SubRegSize, Dst.Z(), Pred, Src.Z());
} else {
if (IsScalar) {
// Scalar
cnt(SubRegSize, Dst.D(), Src.D());
} else {
// Scalar
cnt(SubRegSize, Dst.Q(), Src.Q());
}
}
@@ -548,7 +528,6 @@ DEF_OP(VFAdd) {
const auto OpSize = IROp->Size;
const auto ElementSize = Op->Header.ElementSize;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto IsScalar = ElementSize == OpSize;
const auto Dst = GetVReg(Node);
@@ -561,7 +540,7 @@ DEF_OP(VFAdd) {
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit;
if (HostSupportsSVE && Is256Bit) {
if (HostSupportsSVE && !IsScalar) {
fadd(SubRegSize, Dst.Z(), Vector1.Z(), Vector2.Z());
} else {
if (IsScalar) {
@@ -592,19 +571,21 @@ DEF_OP(VFAddP) {
const auto Op = IROp->C<IR::IROp_VFAddP>();
const auto OpSize = IROp->Size;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto ElementSize = Op->Header.ElementSize;
const auto Dst = GetVReg(Node);
const auto VectorLower = GetVReg(Op->VectorLower.ID());
const auto VectorUpper = GetVReg(Op->VectorUpper.ID());
const bool Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
LOGMAN_THROW_AA_FMT(ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size");
const auto SubRegSize =
ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit :
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit;
if (HostSupportsSVE && Is256Bit) {
const auto Pred = PRED_TMP_32B.Merging();
@@ -632,7 +613,6 @@ DEF_OP(VFSub) {
const auto OpSize = IROp->Size;
const auto ElementSize = Op->Header.ElementSize;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto IsScalar = ElementSize == OpSize;
const auto Dst = GetVReg(Node);
@@ -645,7 +625,7 @@ DEF_OP(VFSub) {
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit;
if (HostSupportsSVE && Is256Bit) {
if (HostSupportsSVE && !IsScalar) {
fsub(SubRegSize, Dst.Z(), Vector1.Z(), Vector2.Z());
} else {
if (IsScalar) {
@@ -677,7 +657,6 @@ DEF_OP(VFMul) {
const auto OpSize = IROp->Size;
const auto ElementSize = Op->Header.ElementSize;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto IsScalar = ElementSize == OpSize;
const auto Dst = GetVReg(Node);
@@ -690,7 +669,7 @@ DEF_OP(VFMul) {
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit;
if (HostSupportsSVE && Is256Bit) {
if (HostSupportsSVE && !IsScalar) {
fmul(SubRegSize, Dst.Z(), Vector1.Z(), Vector2.Z());
} else {
if (IsScalar) {
@@ -735,7 +714,7 @@ DEF_OP(VFDiv) {
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit;
if (HostSupportsSVE && Is256Bit) {
if (HostSupportsSVE && Is256Bit && !IsScalar) {
const auto Mask = PRED_TMP_32B.Merging();
// SVE VDIV is a destructive operation, so we need a temporary.
@@ -793,8 +772,9 @@ DEF_OP(VFMin) {
//
// * - Not exactly (differs slightly with SNaNs), but close enough for the explanation
if (HostSupportsSVE && Is256Bit) {
const auto Mask = PRED_TMP_32B;
if (HostSupportsSVE && !IsScalar) {
const auto Mask = Is256Bit ? PRED_TMP_32B
: PRED_TMP_16B;
const auto ComparePred = ARMEmitter::PReg::p0;
// General idea:
@@ -863,10 +843,10 @@ DEF_OP(VFMax) {
// NOTE: See VFMin implementation for reasons why we
// don't just use FMAX/FMIN for these implementations.
if (HostSupportsSVE && Is256Bit) {
const auto Mask = PRED_TMP_32B;
if (HostSupportsSVE && !IsScalar) {
const auto Mask = Is256Bit ? PRED_TMP_32B
: PRED_TMP_16B;
const auto ComparePred = ARMEmitter::PReg::p0;
fcmgt(SubRegSize, ComparePred, Mask.Zeroing(),
Vector2.Z(), Vector1.Z());
mov(VTMP1.Z(), Vector1.Z());
@@ -920,8 +900,9 @@ DEF_OP(VFRecp) {
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i128Bit);
if (HostSupportsSVE && Is256Bit) {
const auto Pred = PRED_TMP_32B.Merging();
if (HostSupportsSVE && !IsScalar) {
const auto Pred = Is256Bit ? PRED_TMP_32B.Merging()
: PRED_TMP_16B.Merging();
fmov(SubRegSize.Vector, VTMP1.Z(), 1.0);
fdiv(SubRegSize.Vector, VTMP1.Z(), Pred, VTMP1.Z(), Vector.Z());
@@ -970,8 +951,9 @@ DEF_OP(VFSqrt) {
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit;
if (HostSupportsSVE && Is256Bit) {
const auto Pred = PRED_TMP_32B.Merging();
if (HostSupportsSVE && !IsScalar) {
const auto Pred = Is256Bit ? PRED_TMP_32B.Merging()
: PRED_TMP_16B.Merging();
fsqrt(SubRegSize, Dst.Z(), Pred, Vector.Z());
} else {
@@ -1016,7 +998,7 @@ DEF_OP(VFRSqrt) {
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i128Bit);
if (HostSupportsSVE && Is256Bit) {
if (HostSupportsSVE && Is256Bit && !IsScalar) {
const auto Pred = PRED_TMP_32B.Merging();
fmov(SubRegSize.Vector, VTMP1.Z(), 1.0);
fsqrt(SubRegSize.Vector, VTMP2.Z(), Pred, Vector.Z());
@@ -1070,8 +1052,10 @@ DEF_OP(VNeg) {
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit;
if (HostSupportsSVE && Is256Bit) {
const auto Pred = PRED_TMP_32B.Merging();
if (HostSupportsSVE) {
const auto Pred = Is256Bit ? PRED_TMP_32B.Merging()
: PRED_TMP_16B.Merging();
neg(SubRegSize, Dst.Z(), Pred, Vector.Z());
} else {
neg(SubRegSize, Dst.Q(), Vector.Q());
@@ -1094,8 +1078,9 @@ DEF_OP(VFNeg) {
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit;
if (HostSupportsSVE && Is256Bit) {
const auto Pred = PRED_TMP_32B.Merging();
if (HostSupportsSVE) {
const auto Pred = Is256Bit ? PRED_TMP_32B.Merging()
: PRED_TMP_16B.Merging();
fneg(SubRegSize, Dst.Z(), Pred, Vector.Z());
} else {
@@ -1112,7 +1097,7 @@ DEF_OP(VNot) {
const auto Vector = GetVReg(Op->Vector.ID());
if (HostSupportsSVE && Is256Bit) {
not_(ARMEmitter::SubRegSize::i8Bit, Dst.Z(), PRED_TMP_32B.Merging(), Vector.Z());
not_(ARMEmitter::SubRegSize::i8Bit, Dst.Z(), PRED_TMP_32B, Vector.Z());
} else {
mvn(ARMEmitter::SubRegSize::i8Bit, Dst.Q(), Vector.Q());
}
@@ -1123,6 +1108,7 @@ DEF_OP(VUMin) {
const auto OpSize = IROp->Size;
const auto ElementSize = Op->Header.ElementSize;
const auto IsScalar = ElementSize == OpSize;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto Dst = GetVReg(Node);
@@ -1136,7 +1122,7 @@ DEF_OP(VUMin) {
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i128Bit;
if (HostSupportsSVE && Is256Bit) {
if (HostSupportsSVE && Is256Bit && !IsScalar) {
const auto Pred = PRED_TMP_32B.Merging();
// SVE UMIN is a destructive operation so we need a temporary.
@@ -1170,6 +1156,7 @@ DEF_OP(VSMin) {
const auto OpSize = IROp->Size;
const auto ElementSize = Op->Header.ElementSize;
const auto IsScalar = ElementSize == OpSize;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto Dst = GetVReg(Node);
@@ -1183,7 +1170,7 @@ DEF_OP(VSMin) {
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i128Bit;
if (HostSupportsSVE && Is256Bit) {
if (HostSupportsSVE && Is256Bit && !IsScalar) {
const auto Pred = PRED_TMP_32B.Merging();
// SVE SMIN is a destructive operation, so we need a temporary.
@@ -1217,6 +1204,7 @@ DEF_OP(VUMax) {
const auto OpSize = IROp->Size;
const auto ElementSize = Op->Header.ElementSize;
const auto IsScalar = ElementSize == OpSize;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto Dst = GetVReg(Node);
@@ -1230,7 +1218,7 @@ DEF_OP(VUMax) {
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i128Bit;
if (HostSupportsSVE && Is256Bit) {
if (HostSupportsSVE && Is256Bit && !IsScalar) {
const auto Pred = PRED_TMP_32B.Merging();
// SVE UMAX is a destructive operation, so we need a temporary.
@@ -1264,6 +1252,7 @@ DEF_OP(VSMax) {
const auto OpSize = IROp->Size;
const auto ElementSize = Op->Header.ElementSize;
const auto IsScalar = ElementSize == OpSize;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto Dst = GetVReg(Node);
@@ -1277,7 +1266,7 @@ DEF_OP(VSMax) {
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i128Bit;
if (HostSupportsSVE && Is256Bit) {
if (HostSupportsSVE && Is256Bit && !IsScalar) {
const auto Pred = PRED_TMP_32B.Merging();
// SVE SMAX is a destructive operation, so we need a temporary.
@@ -1422,79 +1411,20 @@ DEF_OP(VUnZip2) {
}
}
DEF_OP(VTrn) {
const auto Op = IROp->C<IR::IROp_VTrn>();
const auto OpSize = IROp->Size;
const auto ElementSize = Op->Header.ElementSize;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto Dst = GetVReg(Node);
const auto VectorLower = GetVReg(Op->VectorLower.ID());
const auto VectorUpper = GetVReg(Op->VectorUpper.ID());
LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size");
const auto SubRegSize =
ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit :
ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit :
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit;
if (HostSupportsSVE && Is256Bit) {
trn1(SubRegSize, Dst.Z(), VectorLower.Z(), VectorUpper.Z());
} else {
if (OpSize == 8) {
trn1(SubRegSize, Dst.D(), VectorLower.D(), VectorUpper.D());
} else {
trn1(SubRegSize, Dst.Q(), VectorLower.Q(), VectorUpper.Q());
}
}
}
DEF_OP(VTrn2) {
const auto Op = IROp->C<IR::IROp_VTrn2>();
const auto OpSize = IROp->Size;
const auto ElementSize = Op->Header.ElementSize;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto Dst = GetVReg(Node);
const auto VectorLower = GetVReg(Op->VectorLower.ID());
const auto VectorUpper = GetVReg(Op->VectorUpper.ID());
LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size");
const auto SubRegSize =
ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit :
ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit :
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit;
if (HostSupportsSVE && Is256Bit) {
trn2(SubRegSize, Dst.Z(), VectorLower.Z(), VectorUpper.Z());
} else {
if (OpSize == 8) {
trn2(SubRegSize, Dst.D(), VectorLower.D(), VectorUpper.D());
} else {
trn2(SubRegSize, Dst.Q(), VectorLower.Q(), VectorUpper.Q());
}
}
}
DEF_OP(VBSL) {
const auto Op = IROp->C<IR::IROp_VBSL>();
const auto OpSize = IROp->Size;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto Dst = GetVReg(Node);
const auto VectorFalse = GetVReg(Op->VectorFalse.ID());
const auto VectorTrue = GetVReg(Op->VectorTrue.ID());
const auto VectorMask = GetVReg(Op->VectorMask.ID());
if (HostSupportsSVE && Is256Bit) {
if (HostSupportsSVE) {
// NOTE: Slight parameter difference from ASIMD
// ASIMD -> BSL Mask, True, False
// SVE -> BSL True, True, False, Mask
movprfx(VTMP1.Z(), VectorTrue.Z());
mov(VTMP1.Z(), VectorTrue.Z());
bsl(VTMP1.Z(), VTMP1.Z(), VectorFalse.Z(), VectorMask.Z());
mov(Dst.Z(), VTMP1.Z());
} else {
@@ -1529,7 +1459,7 @@ DEF_OP(VCMPEQ) {
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i128Bit);
if (HostSupportsSVE && Is256Bit) {
if (HostSupportsSVE && Is256Bit && !IsScalar) {
const auto Mask = PRED_TMP_32B.Zeroing();
const auto ComparePred = ARMEmitter::PReg::p0;
@@ -1571,7 +1501,7 @@ DEF_OP(VCMPEQZ) {
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i128Bit);
if (HostSupportsSVE && Is256Bit) {
if (HostSupportsSVE && Is256Bit && !IsScalar) {
const auto Mask = PRED_TMP_32B.Zeroing();
const auto ComparePred = ARMEmitter::PReg::p0;
@@ -1613,7 +1543,7 @@ DEF_OP(VCMPGT) {
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i128Bit);
if (HostSupportsSVE && Is256Bit) {
if (HostSupportsSVE && Is256Bit && !IsScalar) {
const auto Mask = PRED_TMP_32B.Zeroing();
const auto ComparePred = ARMEmitter::PReg::p0;
@@ -1655,7 +1585,7 @@ DEF_OP(VCMPGTZ) {
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i128Bit);
if (HostSupportsSVE && Is256Bit) {
if (HostSupportsSVE && Is256Bit && !IsScalar) {
const auto Mask = PRED_TMP_32B.Zeroing();
const auto ComparePred = ARMEmitter::PReg::p0;
@@ -1693,7 +1623,7 @@ DEF_OP(VCMPLTZ) {
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i128Bit);
if (HostSupportsSVE && Is256Bit) {
if (HostSupportsSVE && Is256Bit && !IsScalar) {
const auto Mask = PRED_TMP_32B.Zeroing();
const auto ComparePred = ARMEmitter::PReg::p0;
@@ -1731,7 +1661,7 @@ DEF_OP(VFCMPEQ) {
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit);
if (HostSupportsSVE && Is256Bit) {
if (HostSupportsSVE && Is256Bit && !IsScalar) {
const auto Mask = PRED_TMP_32B.Zeroing();
const auto ComparePred = ARMEmitter::PReg::p0;
@@ -1780,7 +1710,7 @@ DEF_OP(VFCMPNEQ) {
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit);
if (HostSupportsSVE && Is256Bit) {
if (HostSupportsSVE && Is256Bit && !IsScalar) {
const auto Mask = PRED_TMP_32B.Zeroing();
const auto ComparePred = ARMEmitter::PReg::p0;
@@ -1831,7 +1761,7 @@ DEF_OP(VFCMPLT) {
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit);
if (HostSupportsSVE && Is256Bit) {
if (HostSupportsSVE && Is256Bit && !IsScalar) {
const auto Mask = PRED_TMP_32B.Zeroing();
const auto ComparePred = ARMEmitter::PReg::p0;
@@ -1880,7 +1810,7 @@ DEF_OP(VFCMPGT) {
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit);
if (HostSupportsSVE && Is256Bit) {
if (HostSupportsSVE && Is256Bit && !IsScalar) {
const auto Mask = PRED_TMP_32B.Zeroing();
const auto ComparePred = ARMEmitter::PReg::p0;
@@ -1929,7 +1859,7 @@ DEF_OP(VFCMPLE) {
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit);
if (HostSupportsSVE && Is256Bit) {
if (HostSupportsSVE && Is256Bit && !IsScalar) {
const auto Mask = PRED_TMP_32B.Zeroing();
const auto ComparePred = ARMEmitter::PReg::p0;
@@ -1979,7 +1909,7 @@ DEF_OP(VFCMPORD) {
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit);
if (HostSupportsSVE && Is256Bit) {
if (HostSupportsSVE && Is256Bit && !IsScalar) {
const auto Mask = PRED_TMP_32B.Zeroing();
const auto ComparePred = ARMEmitter::PReg::p0;
@@ -2040,7 +1970,7 @@ DEF_OP(VFCMPUNO) {
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit);
if (HostSupportsSVE && Is256Bit) {
if (HostSupportsSVE && Is256Bit && !IsScalar) {
const auto Mask = PRED_TMP_32B.Zeroing();
const auto ComparePred = ARMEmitter::PReg::p0;
@@ -2090,44 +2020,7 @@ DEF_OP(VUShr) {
}
DEF_OP(VSShr) {
const auto Op = IROp->C<IR::IROp_VSShr>();
const auto OpSize = IROp->Size;
const auto ElementSize = IROp->ElementSize;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto MaxShift = (ElementSize * 8) - 1;
const auto Dst = GetVReg(Node);
const auto ShiftVector = GetVReg(Op->ShiftVector.ID());
const auto Vector = GetVReg(Op->Vector.ID());
LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size");
const auto SubRegSize =
ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit :
ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit :
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit;
if (HostSupportsSVE && Is256Bit) {
const auto Mask = PRED_TMP_32B.Merging();
dup_imm(SubRegSize, VTMP2.Z(), MaxShift);
umin(SubRegSize, VTMP2.Z(), Mask, VTMP2.Z(), ShiftVector.Z());
movprfx(VTMP1.Z(), Vector.Z());
asr(SubRegSize, VTMP1.Z(), Mask, VTMP1.Z(), VTMP2.Z());
mov(Dst.Z(), VTMP1.Z());
} else {
LOGMAN_THROW_AA_FMT(ElementSize != 8, "Adv. SIMD UMIN doesn't handle 64-bit values");
movi(SubRegSize, VTMP1.Q(), MaxShift);
umin(SubRegSize, VTMP1.Q(), VTMP1.Q(), ShiftVector.Q());
// Need to invert shift values to perform a right shift with SSHL
// (SSHR only has an immediate variant).
neg(SubRegSize, VTMP1.Q(), VTMP1.Q());
sshl(SubRegSize, Dst.Q(), Vector.Q(), VTMP1.Q());
}
LOGMAN_MSG_A_FMT("Unimplemented");
}
DEF_OP(VUShlS) {
@@ -2288,10 +2181,10 @@ DEF_OP(VInsElement) {
dup(SubRegSize, VTMP2.Z(), SrcVector.Z(), SrcIdx);
mov(Dst.Z(), Reg.Z());
if (ElementSize == 16) {
mov(ARMEmitter::SubRegSize::i64Bit, Dst.Z(), Predicate.Merging(), VTMP2.Z());
mov(ARMEmitter::SubRegSize::i64Bit, Dst.Z(), Predicate, VTMP2.Z());
}
else {
mov(SubRegSize, Dst.Z(), Predicate.Merging(), VTMP2.Z());
mov(SubRegSize, Dst.Z(), Predicate, VTMP2.Z());
}
// Set up a label to jump over the data we inserted, so we don't try and execute it.
@@ -2526,13 +2419,14 @@ DEF_OP(VUShrNI2) {
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : ARMEmitter::SubRegSize::i8Bit;
if (HostSupportsSVE && Is256Bit) {
mov(VTMP1.Z(), VectorLower.Z());
const auto Mask = PRED_TMP_16B;
shrnb(SubRegSize, VTMP2.Z(), VectorUpper.Z(), BitShift);
uzp1(SubRegSize, VTMP2.Z(), VTMP2.Z(), VTMP2.Z());
movprfx(Dst.Z(), VectorLower.Z());
splice<ARMEmitter::OpType::Destructive>(SubRegSize, Dst.Z(), Mask, Dst.Z(), VTMP2.Z());
splice<ARMEmitter::OpType::Destructive>(SubRegSize, VTMP1.Z(), Mask, VTMP1.Z(), VTMP2.Z());
mov(Dst.Z(), VTMP1.Z());
} else {
mov(VTMP1.Q(), VectorLower.Q());
shrn2(SubRegSize, VTMP1.Q(), VectorUpper.Q(), BitShift);
@@ -2629,7 +2523,7 @@ DEF_OP(VUXTL2) {
if (HostSupportsSVE && Is256Bit) {
uunpkhi(SubRegSize, Dst.Z(), Vector.Z());
} else {
uxtl2(SubRegSize, Dst.Q(), Vector.Q());
uxtl2(SubRegSize, Dst.D(), Vector.D());
}
}
@@ -2708,6 +2602,12 @@ DEF_OP(VSQXTN2) {
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : ARMEmitter::SubRegSize::i8Bit;
if (HostSupportsSVE && Is256Bit) {
// Need to use the destructive variant of SPLICE, since
// the constructive variant requires a register list, and
// we can't guarantee VectorLower and VectorUpper will always
// have consecutive indexes with one another.
mov(VTMP1.Z(), VectorLower.Z());
// We use the 16 byte mask due to how SPLICE works. We only
// want to get at the first 16 bytes in the lower vector, so
// that SPLICE will then begin copying the first 16 bytes
@@ -2717,23 +2617,20 @@ DEF_OP(VSQXTN2) {
sqxtnb(SubRegSize, VTMP2.Z(), VectorUpper.Z());
uzp1(SubRegSize, VTMP2.Z(), VTMP2.Z(), VTMP2.Z());
splice<ARMEmitter::OpType::Destructive>(SubRegSize, VTMP1.Z(), Mask, VTMP1.Z(), VTMP2.Z());
// Need to use the destructive variant of SPLICE, since
// the constructive variant requires a register list, and
// we can't guarantee VectorLower and VectorUpper will always
// have consecutive indexes with one another.
movprfx(Dst.Z(), VectorLower.Z());
splice<ARMEmitter::OpType::Destructive>(SubRegSize, Dst.Z(), Mask, Dst.Z(), VTMP2.Z());
mov(Dst.Z(), VTMP1.Z());
} else {
mov(VTMP1.Q(), VectorLower.Q());
if (OpSize == 8) {
sqxtn(SubRegSize, VTMP2, VectorUpper);
mov(Dst.Q(), VectorLower.Q());
ins(ARMEmitter::SubRegSize::i32Bit, Dst, 1, VTMP2, 0);
ins(ARMEmitter::SubRegSize::i32Bit, VTMP1, 1, VTMP2, 0);
} else {
mov(VTMP1.Q(), VectorLower.Q());
sqxtn2(SubRegSize, VTMP1, VectorUpper);
mov(Dst.Q(), VTMP1.Q());
}
mov(Dst.Q(), VTMP1.Q());
}
}
@@ -2782,31 +2679,30 @@ DEF_OP(VSQXTUN2) {
// NOTE: See VSQXTN2 implementation for an in-depth explanation
// of everything going on here.
mov(VTMP1.Z(), VectorLower.Z());
const auto Mask = PRED_TMP_16B;
sqxtunb(SubRegSize, VTMP2.Z(), VectorUpper.Z());
uzp1(SubRegSize, VTMP2.Z(), VTMP2.Z(), VTMP2.Z());
splice<ARMEmitter::OpType::Destructive>(SubRegSize, VTMP1.Z(), Mask, VTMP1.Z(), VTMP2.Z());
movprfx(Dst.Z(), VectorLower.Z());
splice<ARMEmitter::OpType::Destructive>(SubRegSize, Dst.Z(), Mask, Dst.Z(), VTMP2.Z());
mov(Dst.Z(), VTMP1.Z());
} else {
mov(VTMP1.Q(), VectorLower.Q());
if (OpSize == 8) {
sqxtun(SubRegSize, VTMP2, VectorUpper);
mov(Dst.Q(), VectorLower.Q());
ins(ARMEmitter::SubRegSize::i32Bit, Dst, 1, VTMP2, 0);
ins(ARMEmitter::SubRegSize::i32Bit, VTMP1, 1, VTMP2, 0);
} else {
mov(VTMP1.Q(), VectorLower.Q());
sqxtun2(SubRegSize, VTMP1, VectorUpper);
mov(Dst.Q(), VTMP1.Q());
}
mov(Dst.Q(), VTMP1.Q());
}
}
DEF_OP(VMul) {
const auto Op = IROp->C<IR::IROp_VUMul>();
const auto OpSize = IROp->Size;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto ElementSize = Op->Header.ElementSize;
const auto Dst = GetVReg(Node);
@@ -2820,7 +2716,7 @@ DEF_OP(VMul) {
ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit :
ElementSize == 8 ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i8Bit;
if (HostSupportsSVE && Is256Bit) {
if (HostSupportsSVE) {
mul(SubRegSize, Dst.Z(), Vector1.Z(), Vector2.Z());
} else {
mul(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q());
@@ -2972,7 +2868,7 @@ DEF_OP(VTBL1) {
switch (OpSize) {
case 8: {
tbl(Dst.D(), VectorTable.Q(), VectorIndices.D());
tbl(Dst.D(), VectorTable.D(), VectorIndices.D());
break;
}
case 16: {
@@ -3039,5 +2935,93 @@ DEF_OP(VRev64) {
}
#undef DEF_OP
void Arm64JITCore::RegisterVectorHandlers() {
#define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &Arm64JITCore::Op_##x
REGISTER_OP(VECTORZERO, VectorZero);
REGISTER_OP(VECTORIMM, VectorImm);
REGISTER_OP(VMOV, VMov);
REGISTER_OP(VAND, VAnd);
REGISTER_OP(VBIC, VBic);
REGISTER_OP(VOR, VOr);
REGISTER_OP(VXOR, VXor);
REGISTER_OP(VADD, VAdd);
REGISTER_OP(VSUB, VSub);
REGISTER_OP(VUQADD, VUQAdd);
REGISTER_OP(VUQSUB, VUQSub);
REGISTER_OP(VSQADD, VSQAdd);
REGISTER_OP(VSQSUB, VSQSub);
REGISTER_OP(VADDP, VAddP);
REGISTER_OP(VADDV, VAddV);
REGISTER_OP(VUMINV, VUMinV);
REGISTER_OP(VURAVG, VURAvg);
REGISTER_OP(VABS, VAbs);
REGISTER_OP(VPOPCOUNT, VPopcount);
REGISTER_OP(VFADD, VFAdd);
REGISTER_OP(VFADDP, VFAddP);
REGISTER_OP(VFSUB, VFSub);
REGISTER_OP(VFMUL, VFMul);
REGISTER_OP(VFDIV, VFDiv);
REGISTER_OP(VFMIN, VFMin);
REGISTER_OP(VFMAX, VFMax);
REGISTER_OP(VFRECP, VFRecp);
REGISTER_OP(VFSQRT, VFSqrt);
REGISTER_OP(VFRSQRT, VFRSqrt);
REGISTER_OP(VNEG, VNeg);
REGISTER_OP(VFNEG, VFNeg);
REGISTER_OP(VNOT, VNot);
REGISTER_OP(VUMIN, VUMin);
REGISTER_OP(VSMIN, VSMin);
REGISTER_OP(VUMAX, VUMax);
REGISTER_OP(VSMAX, VSMax);
REGISTER_OP(VZIP, VZip);
REGISTER_OP(VZIP2, VZip2);
REGISTER_OP(VUNZIP, VUnZip);
REGISTER_OP(VUNZIP2, VUnZip2);
REGISTER_OP(VBSL, VBSL);
REGISTER_OP(VCMPEQ, VCMPEQ);
REGISTER_OP(VCMPEQZ, VCMPEQZ);
REGISTER_OP(VCMPGT, VCMPGT);
REGISTER_OP(VCMPGTZ, VCMPGTZ);
REGISTER_OP(VCMPLTZ, VCMPLTZ);
REGISTER_OP(VFCMPEQ, VFCMPEQ);
REGISTER_OP(VFCMPNEQ, VFCMPNEQ);
REGISTER_OP(VFCMPLT, VFCMPLT);
REGISTER_OP(VFCMPGT, VFCMPGT);
REGISTER_OP(VFCMPLE, VFCMPLE);
REGISTER_OP(VFCMPORD, VFCMPORD);
REGISTER_OP(VFCMPUNO, VFCMPUNO);
REGISTER_OP(VUSHL, VUShl);
REGISTER_OP(VUSHR, VUShr);
REGISTER_OP(VSSHR, VSShr);
REGISTER_OP(VUSHLS, VUShlS);
REGISTER_OP(VUSHRS, VUShrS);
REGISTER_OP(VSSHRS, VSShrS);
REGISTER_OP(VINSELEMENT, VInsElement);
REGISTER_OP(VDUPELEMENT, VDupElement);
REGISTER_OP(VEXTR, VExtr);
REGISTER_OP(VUSHRI, VUShrI);
REGISTER_OP(VSSHRI, VSShrI);
REGISTER_OP(VSHLI, VShlI);
REGISTER_OP(VUSHRNI, VUShrNI);
REGISTER_OP(VUSHRNI2, VUShrNI2);
REGISTER_OP(VSXTL, VSXTL);
REGISTER_OP(VSXTL2, VSXTL2);
REGISTER_OP(VUXTL, VUXTL);
REGISTER_OP(VUXTL2, VUXTL2);
REGISTER_OP(VSQXTN, VSQXTN);
REGISTER_OP(VSQXTN2, VSQXTN2);
REGISTER_OP(VSQXTUN, VSQXTUN);
REGISTER_OP(VSQXTUN2, VSQXTUN2);
REGISTER_OP(VUMUL, VMul);
REGISTER_OP(VSMUL, VMul);
REGISTER_OP(VUMULL, VUMull);
REGISTER_OP(VSMULL, VSMull);
REGISTER_OP(VUMULL2, VUMull2);
REGISTER_OP(VSMULL2, VSMull2);
REGISTER_OP(VUABDL, VUABDL);
REGISTER_OP(VTBL1, VTBL1);
REGISTER_OP(VREV64, VRev64);
#undef REGISTER_OP
}
}
+5 -5
View File
@@ -3,7 +3,7 @@
#include <memory>
namespace FEXCore::Context {
class ContextImpl;
struct Context;
}
namespace FEXCore::Core {
@@ -13,14 +13,14 @@ struct InternalThreadState;
namespace FEXCore::CPU {
class CPUBackend;
[[nodiscard]] std::unique_ptr<CPUBackend> CreateX86JITCore(FEXCore::Context::ContextImpl *ctx,
[[nodiscard]] std::unique_ptr<CPUBackend> CreateX86JITCore(FEXCore::Context::Context *ctx,
FEXCore::Core::InternalThreadState *Thread);
void InitializeX86JITSignalHandlers(FEXCore::Context::ContextImpl *CTX);
void InitializeX86JITSignalHandlers(FEXCore::Context::Context *CTX);
CPUBackendFeatures GetX86JITBackendFeatures();
[[nodiscard]] std::unique_ptr<CPUBackend> CreateArm64JITCore(FEXCore::Context::ContextImpl *ctx,
[[nodiscard]] std::unique_ptr<CPUBackend> CreateArm64JITCore(FEXCore::Context::Context *ctx,
FEXCore::Core::InternalThreadState *Thread);
void InitializeArm64JITSignalHandlers(FEXCore::Context::ContextImpl *CTX);
void InitializeArm64JITSignalHandlers(FEXCore::Context::Context *CTX);
CPUBackendFeatures GetArm64JITBackendFeatures();
} // namespace FEXCore::CPU
@@ -30,6 +30,15 @@ $end_info$
namespace FEXCore::CPU {
#define DEF_OP(x) void X86JITCore::Op_##x(IR::IROp_Header *IROp, IR::NodeID Node)
DEF_OP(SignalReturn) {
// Adjust the stack first for a regular return
if (SpillSlots) {
add(rsp, SpillSlots * MaxSpillSlotSize); // + 8 to consume return address
}
jmp(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.SignalReturnHandler)]);
}
DEF_OP(CallbackReturn) {
// Adjust the stack first for a regular return
if (SpillSlots) {
@@ -202,7 +211,7 @@ DEF_OP(Thunk) {
mov(rdi, GetSrc<RA_64>(Op->ArgPtr.ID()));
auto thunkFn = static_cast<Context::ContextImpl*>(ThreadState->CTX)->ThunkHandler->LookupThunk(Op->ThunkNameHash);
auto thunkFn = ThreadState->CTX->ThunkHandler->LookupThunk(Op->ThunkNameHash);
mov(rax, reinterpret_cast<uintptr_t>(thunkFn));
call(rax);
@@ -305,6 +314,7 @@ DEF_OP(CPUID) {
#undef DEF_OP
void X86JITCore::RegisterBranchHandlers() {
#define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &X86JITCore::Op_##x
REGISTER_OP(SIGNALRETURN, SignalReturn);
REGISTER_OP(CALLBACKRETURN, CallbackReturn);
REGISTER_OP(EXITFUNCTION, ExitFunction);
REGISTER_OP(JUMP, Jump);
@@ -111,53 +111,6 @@ DEF_OP(VCastFromGPR) {
}
}
DEF_OP(VDupFromGPR) {
const auto Op = IROp->C<IR::IROp_VDupFromGPR>();
const auto OpSize = IROp->Size;
const auto ElementSize = IROp->ElementSize;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto Dst = GetDst(Node);
const auto Src = GetSrc<RA_64>(Op->Src.ID()).cvt64();
vmovq(Dst, Src);
switch (ElementSize) {
case 1:
if (Is256Bit) {
vpbroadcastb(ToYMM(Dst), Dst);
} else {
vpbroadcastb(Dst, Dst);
}
break;
case 2:
if (Is256Bit) {
vpbroadcastw(ToYMM(Dst), Dst);
} else {
vpbroadcastw(Dst, Dst);
}
break;
case 4:
if (Is256Bit) {
vpbroadcastd(ToYMM(Dst), Dst);
} else {
vpbroadcastd(Dst, Dst);
}
break;
case 8:
if (Is256Bit) {
vpbroadcastq(ToYMM(Dst), Dst);
} else {
vpbroadcastq(Dst, Dst);
}
break;
default:
LOGMAN_MSG_A_FMT("Unhandled element size: {}", ElementSize);
return;
}
}
DEF_OP(Float_FromGPR_S) {
const auto Op = IROp->C<IR::IROp_Float_FromGPR_S>();
@@ -404,7 +357,6 @@ void X86JITCore::RegisterConversionHandlers() {
#define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &X86JITCore::Op_##x
REGISTER_OP(VINSGPR, VInsGPR);
REGISTER_OP(VCASTFROMGPR, VCastFromGPR);
REGISTER_OP(VDUPFROMGPR, VDupFromGPR);
REGISTER_OP(FLOAT_FROMGPR_S, Float_FromGPR_S);
REGISTER_OP(FLOAT_FTOF, Float_FToF);
REGISTER_OP(VECTOR_STOF, Vector_SToF);
@@ -21,67 +21,23 @@ DEF_OP(AESImc) {
}
DEF_OP(AESEnc) {
const auto Op = IROp->C<IR::IROp_VAESEnc>();
const auto OpSize = IROp->Size;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto Dst = GetDst(Node);
const auto Key = GetSrc(Op->Key.ID());
const auto State = GetSrc(Op->State.ID());
if (Is256Bit) {
vaesenc(ToYMM(Dst), ToYMM(State), ToYMM(Key));
} else {
vaesenc(Dst, State, Key);
}
auto Op = IROp->C<IR::IROp_VAESEnc>();
vaesenc(GetDst(Node), GetSrc(Op->State.ID()), GetSrc(Op->Key.ID()));
}
DEF_OP(AESEncLast) {
const auto Op = IROp->C<IR::IROp_VAESEncLast>();
const auto OpSize = IROp->Size;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto Dst = GetDst(Node);
const auto Key = GetSrc(Op->Key.ID());
const auto State = GetSrc(Op->State.ID());
if (Is256Bit) {
vaesenclast(ToYMM(Dst), ToYMM(State), ToYMM(Key));
} else {
vaesenclast(Dst, State, Key);
}
auto Op = IROp->C<IR::IROp_VAESEncLast>();
vaesenclast(GetDst(Node), GetSrc(Op->State.ID()), GetSrc(Op->Key.ID()));
}
DEF_OP(AESDec) {
const auto Op = IROp->C<IR::IROp_VAESDec>();
const auto OpSize = IROp->Size;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto Dst = GetDst(Node);
const auto Key = GetSrc(Op->Key.ID());
const auto State = GetSrc(Op->State.ID());
if (Is256Bit) {
vaesdec(ToYMM(Dst), ToYMM(State), ToYMM(Key));
} else {
vaesdec(Dst, State, Key);
}
auto Op = IROp->C<IR::IROp_VAESDec>();
vaesdec(GetDst(Node), GetSrc(Op->State.ID()), GetSrc(Op->Key.ID()));
}
DEF_OP(AESDecLast) {
const auto Op = IROp->C<IR::IROp_VAESDecLast>();
const auto OpSize = IROp->Size;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto Dst = GetDst(Node);
const auto Key = GetSrc(Op->Key.ID());
const auto State = GetSrc(Op->State.ID());
if (Is256Bit) {
vaesdeclast(ToYMM(Dst), ToYMM(State), ToYMM(Key));
} else {
vaesdeclast(Dst, State, Key);
}
auto Op = IROp->C<IR::IROp_VAESDecLast>();
vaesdeclast(GetDst(Node), GetSrc(Op->State.ID()), GetSrc(Op->Key.ID()));
}
DEF_OP(AESKeyGenAssist) {
@@ -120,24 +76,18 @@ DEF_OP(CRC32) {
}
DEF_OP(PCLMUL) {
const auto Op = IROp->C<IR::IROp_PCLMUL>();
const auto OpSize = IROp->Size;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
auto Op = IROp->C<IR::IROp_PCLMUL>();
const auto Dst = GetDst(Node);
const auto Src1 = GetSrc(Op->Src1.ID());
const auto Src2 = GetSrc(Op->Src2.ID());
auto Dst = GetDst(Node);
auto Src1 = GetSrc(Op->Src1.ID());
auto Src2 = GetSrc(Op->Src2.ID());
switch (Op->Selector) {
case 0b00000000:
case 0b00000001:
case 0b00010000:
case 0b00010001:
if (Is256Bit) {
vpclmulqdq(ToYMM(Dst), ToYMM(Src1), ToYMM(Src2), Op->Selector);
} else {
vpclmulqdq(Dst, Src1, Src2, Op->Selector);
}
vpclmulqdq(Dst, Src1, Src2, Op->Selector);
break;
default:
LOGMAN_MSG_A_FMT("Unknown PCLMUL selector: {}", Op->Selector);
+17 -53
View File
@@ -147,12 +147,7 @@ void X86JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
case FABI_F80_I32: {
PushRegs();
if (Info.ABI == FABI_F80_I16) {
movsx(rdi, GetSrc<RA_32>(IROp->Args[0].ID()).cvt16());
}
else {
mov(edi, GetSrc<RA_32>(IROp->Args[0].ID()));
}
mov(edi, GetSrc<RA_32>(IROp->Args[0].ID()));
call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]);
PopRegs();
@@ -228,7 +223,7 @@ void X86JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
PopRegs();
movsx(GetDst<RA_64>(Node), ax);
movzx(GetDst<RA_64>(Node), ax);
}
break;
case FABI_I32_F80:{
@@ -330,7 +325,7 @@ static uint64_t X86JITCore_ExitFunctionLink(FEXCore::Core::CpuStateFrame *Frame,
}
auto LinkerAddress = Frame->Pointers.Common.ExitFunctionLinker;
Context::ContextImpl::ThreadAddBlockLink(Thread, GuestRip, (uintptr_t)record, [record, LinkerAddress]{
Context::Context::ThreadAddBlockLink(Thread, GuestRip, (uintptr_t)record, [record, LinkerAddress]{
// undo the link
record[0] = LinkerAddress;
});
@@ -342,7 +337,7 @@ static uint64_t X86JITCore_ExitFunctionLink(FEXCore::Core::CpuStateFrame *Frame,
void X86JITCore::Op_NoOp(IR::IROp_Header *IROp, IR::NodeID Node) {
}
X86JITCore::X86JITCore(FEXCore::Context::ContextImpl *ctx, FEXCore::Core::InternalThreadState *Thread)
X86JITCore::X86JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread)
: CPUBackend(Thread, INITIAL_CODE_SIZE, MAX_CODE_SIZE)
, CodeGenerator(0, this, nullptr) // this is not used here
, CTX {ctx} {
@@ -379,7 +374,7 @@ X86JITCore::X86JITCore(FEXCore::Context::ContextImpl *ctx, FEXCore::Core::Intern
Common.PrintValue = reinterpret_cast<uint64_t>(PrintValue);
Common.PrintVectorValue = reinterpret_cast<uint64_t>(PrintVectorValue);
Common.ThreadRemoveCodeEntryFromJIT = reinterpret_cast<uintptr_t>(&Context::ContextImpl::ThreadRemoveCodeEntryFromJit);
Common.ThreadRemoveCodeEntryFromJIT = reinterpret_cast<uintptr_t>(&Context::Context::ThreadRemoveCodeEntryFromJit);
Common.CPUIDObj = reinterpret_cast<uint64_t>(&CTX->CPUID);
{
@@ -389,7 +384,7 @@ X86JITCore::X86JITCore(FEXCore::Context::ContextImpl *ctx, FEXCore::Core::Intern
Common.SyscallHandlerObj = reinterpret_cast<uint64_t>(CTX->SyscallHandler);
Common.SyscallHandlerFunc = reinterpret_cast<uint64_t>(FEXCore::Context::HandleSyscall);
Common.ExitFunctionLink = reinterpret_cast<uintptr_t>(&Context::ContextImpl::ThreadExitFunctionLink<X86JITCore_ExitFunctionLink>);
Common.ExitFunctionLink = reinterpret_cast<uintptr_t>(&Context::Context::ThreadExitFunctionLink<X86JITCore_ExitFunctionLink>);
// Fill in the fallback handlers
InterpreterOps::FillFallbackIndexPointers(Common.FallbackHandlerPointers);
@@ -399,9 +394,9 @@ X86JITCore::X86JITCore(FEXCore::Context::ContextImpl *ctx, FEXCore::Core::Intern
ClearCache();
}
void X86JITCore::InitializeSignalHandlers(FEXCore::Context::ContextImpl *CTX) {
void X86JITCore::InitializeSignalHandlers(FEXCore::Context::Context *CTX) {
CTX->SignalDelegation->RegisterHostSignalHandler(SIGILL, [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext) -> bool {
return static_cast<Context::ContextImpl*>(Thread->CTX)->Dispatcher->HandleSIGILL(Thread, Signal, info, ucontext);
return Thread->CTX->Dispatcher->HandleSIGILL(Thread, Signal, info, ucontext);
}, true);
}
@@ -587,7 +582,7 @@ std::tuple<X86JITCore::SetCC, X86JITCore::CMovCC, X86JITCore::JCC> X86JITCore::G
return { &CodeGenerator::sete , &CodeGenerator::cmove , &CodeGenerator::je };
}
CPUBackend::CompiledCode X86JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IRListView const *IR, [[maybe_unused]] FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData, bool GDBEnabled) {
void *X86JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IRListView const *IR, [[maybe_unused]] FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData, bool GDBEnabled) {
FEXCORE_PROFILE_SCOPED("x86::CompileCode");
JumpTargets.clear();
@@ -603,27 +598,12 @@ CPUBackend::CompiledCode X86JITCore::CompileCode(uint64_t Entry, [[maybe_unused]
CTX->ClearCodeCache(ThreadState);
}
CodeData.BlockBegin = getCurr<uint8_t*>();
// Put the code header at the start of the data block.
Label JITCodeHeaderLabel{};
L(JITCodeHeaderLabel);
JITCodeHeader *CodeHeader = getCurr<JITCodeHeader *>();
setSize(getSize() + sizeof(JITCodeHeader));
CodeData.BlockEntry = getCurr<uint8_t*>();
// Get the address of the JITCodeHeader and store in to the core state.
// Only two instructions, so very low overhead.
lea(TMP1, ptr [rip + JITCodeHeaderLabel]);
mov(qword [STATE + offsetof(FEXCore::Core::CPUState, InlineJITBlockHeader)], TMP1);
GuestEntry = getCurr<uint8_t*>();
CursorEntry = getSize();
this->IR = IR;
if (GDBEnabled) {
auto GDBSize = CTX->Dispatcher->GenerateGDBPauseCheck(CodeData.BlockBegin, Entry);
auto GDBSize = CTX->Dispatcher->GenerateGDBPauseCheck(GuestEntry, Entry);
setSize(getSize() + GDBSize);
}
@@ -746,7 +726,7 @@ CPUBackend::CompiledCode X86JITCore::CompileCode(uint64_t Entry, [[maybe_unused]
if (DebugData) {
DebugData->Subblocks.push_back({
static_cast<uint32_t>(BlockStartHostCode - CodeData.BlockBegin),
static_cast<uint32_t>(BlockStartHostCode - GuestEntry),
static_cast<uint32_t>(getCurr<uint8_t *>() - BlockStartHostCode)
});
}
@@ -759,36 +739,20 @@ CPUBackend::CompiledCode X86JITCore::CompileCode(uint64_t Entry, [[maybe_unused]
}
PendingTargetLabel = nullptr;
// Add the JitCodeTail
auto JITBlockTailLocation = getCurr<uint8_t *>();
auto JITBlockTail = getCurr<JITCodeTail*>();
setSize(getSize() + sizeof(JITCodeTail));
// Put the block's RIP entry in the tail.
// This will be used for RIP reconstruction in the future.
// TODO: This needs to be a data RIP relocation once code caching works.
// Current relocation code doesn't support this feature yet.
JITBlockTail->RIP = Entry;
CodeHeader->OffsetToBlockTail = JITBlockTailLocation - CodeData.BlockBegin;
CodeData.Size = getCurr<uint8_t*>() - CodeData.BlockBegin;
JITBlockTail->Size = CodeData.Size;
void *GuestExit = getCurr<void*>();
this->IR = nullptr;
ready();
if (DebugData) {
DebugData->HostCodeSize = CodeData.Size;
DebugData->HostCodeSize = reinterpret_cast<uintptr_t>(GuestExit) - reinterpret_cast<uintptr_t>(GuestEntry);
DebugData->Relocations = &Relocations;
}
return CodeData;
return GuestEntry;
}
std::unique_ptr<CPUBackend> CreateX86JITCore(FEXCore::Context::ContextImpl *ctx, FEXCore::Core::InternalThreadState *Thread) {
std::unique_ptr<CPUBackend> CreateX86JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread) {
return std::make_unique<X86JITCore>(ctx, Thread);
}
@@ -796,7 +760,7 @@ CPUBackendFeatures GetX86JITBackendFeatures() {
return CPUBackendFeatures { };
}
void InitializeX86JITSignalHandlers(FEXCore::Context::ContextImpl *CTX) {
void InitializeX86JITSignalHandlers(FEXCore::Context::Context *CTX) {
X86JITCore::InitializeSignalHandlers(CTX);
}
@@ -51,13 +51,13 @@ const std::array<Xbyak::Xmm, 11> RAXMM_x = { xmm1, xmm2, xmm3, xmm4, xmm5, xmm6
class X86JITCore final : public CPUBackend, public Xbyak::CodeGenerator {
public:
explicit X86JITCore(FEXCore::Context::ContextImpl *ctx,
explicit X86JITCore(FEXCore::Context::Context *ctx,
FEXCore::Core::InternalThreadState *Thread);
~X86JITCore() override;
[[nodiscard]] std::string GetName() override { return "JIT"; }
[[nodiscard]] CPUBackend::CompiledCode CompileCode(uint64_t Entry,
[[nodiscard]] void *CompileCode(uint64_t Entry,
FEXCore::IR::IRListView const *IR,
FEXCore::Core::DebugData *DebugData,
FEXCore::IR::RegisterAllocationData *RAData, bool GDBEnabled) override;
@@ -68,7 +68,7 @@ public:
void ClearCache() override;
static void InitializeSignalHandlers(FEXCore::Context::ContextImpl *CTX);
static void InitializeSignalHandlers(FEXCore::Context::Context *CTX);
void ClearRelocations() override { Relocations.clear(); }
@@ -135,10 +135,9 @@ private:
/** @} */
Label* PendingTargetLabel{};
FEXCore::Context::ContextImpl *CTX;
FEXCore::Context::Context *CTX;
FEXCore::IR::IRListView const *IR;
uint64_t Entry;
CPUBackend::CompiledCode CodeData{};
std::unordered_map<IR::NodeID, Label> JumpTargets;
Xbyak::util::Cpu Features{};
@@ -206,6 +205,10 @@ private:
void EmitDetectionString();
uint32_t SpillSlots{};
/**
* @brief Current guest RIP entrypoint
*/
uint8_t *GuestEntry{};
using SetCC = void (X86JITCore::*)(const Operand& op);
using CMovCC = void (X86JITCore::*)(const Reg& reg, const Operand& op);
@@ -305,6 +308,7 @@ private:
DEF_OP(AtomicFetchNeg);
///< Branch ops
DEF_OP(SignalReturn);
DEF_OP(CallbackReturn);
DEF_OP(ExitFunction);
DEF_OP(Jump);
@@ -318,7 +322,6 @@ private:
///< Conversion ops
DEF_OP(VInsGPR);
DEF_OP(VCastFromGPR);
DEF_OP(VDupFromGPR);
DEF_OP(Float_FromGPR_S);
DEF_OP(Float_FToF);
DEF_OP(Vector_UToF);
@@ -344,9 +347,7 @@ private:
DEF_OP(StoreFlag);
DEF_OP(LoadMem);
DEF_OP(StoreMem);
DEF_OP(MemSet);
DEF_OP(CacheLineClear);
DEF_OP(CacheLineClean);
DEF_OP(CacheLineZero);
///< Misc ops
@@ -407,8 +408,6 @@ private:
DEF_OP(VZip2);
DEF_OP(VUnZip);
DEF_OP(VUnZip2);
DEF_OP(VTrn);
DEF_OP(VTrn2);
DEF_OP(VBSL);
DEF_OP(VCMPEQ);
DEF_OP(VCMPEQZ);
@@ -766,112 +766,12 @@ DEF_OP(StoreMem) {
}
}
DEF_OP(MemSet) {
const auto Op = IROp->C<IR::IROp_MemSet>();
const int32_t Size = Op->Size;
const auto MemReg = GetSrc<RA_64>(Op->Addr.ID());
const auto Value = GetSrc<RA_64>(Op->Value.ID());
const auto Length = GetSrc<RA_64>(Op->Length.ID());
const auto Direction = GetSrc<RA_64>(Op->Direction.ID());
const auto Dst = GetSrc<RA_64>(Node);
// If Direction == 0 then:
// MemReg is incremented (by size)
// else:
// MemReg is decremented (by size)
//
// Counter is decremented regardless.
// TMP1 = rax
// TMP2 = rcx
// TMP4 = rdi
// That leaves us with TMP3 and TMP5
mov(rax, Value);
mov(rcx, Length);
mov(rdi, MemReg);
{
mov(TMP3, Length);
auto CalculateDest = [&]() {
mov(Dst, MemReg);
switch (Size) {
case 1:
break;
case 2:
shl(TMP3, 1);
break;
case 4:
shl(TMP3, 2);
break;
case 8:
shl(TMP3, 3);
break;
default:
LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, Size);
break;
}
};
Label AfterDir;
Label BackwardDir;
cmp(Direction, 0);
jne(BackwardDir);
// Incrementing DF flag.
cld();
CalculateDest();
add(Dst, TMP3);
jmp(AfterDir);
L(BackwardDir);
// Decrementing DF flag.
std();
CalculateDest();
sub(Dst, TMP3);
L(AfterDir);
}
switch (Size) {
case 1:
rep(); stosb();
break;
case 2:
rep(); stosw();
break;
case 4:
rep(); stosd();
break;
case 8:
rep(); stosq();
break;
default:
LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, Size);
break;
}
// Ensure we set DF back to zero. Required by the ABI.
cld();
}
DEF_OP(CacheLineClear) {
auto Op = IROp->C<IR::IROp_CacheLineClear>();
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Addr.ID());
if (Op->Serialize) {
clflush(ptr [MemReg]);
}
else {
clflushopt(ptr [MemReg]);
}
}
DEF_OP(CacheLineClean) {
auto Op = IROp->C<IR::IROp_CacheLineClean>();
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Addr.ID());
clwb(ptr [MemReg]);
clflush(ptr [MemReg]);
}
DEF_OP(CacheLineZero) {
@@ -908,9 +808,7 @@ void X86JITCore::RegisterMemoryHandlers() {
REGISTER_OP(STOREMEM, StoreMem);
REGISTER_OP(LOADMEMTSO, LoadMem);
REGISTER_OP(STOREMEMTSO, StoreMem);
REGISTER_OP(MEMSET, MemSet);
REGISTER_OP(CACHELINECLEAR, CacheLineClear);
REGISTER_OP(CACHELINECLEAN, CacheLineClean);
REGISTER_OP(CACHELINEZERO, CacheLineZero);
#undef REGISTER_OP
}
@@ -24,7 +24,7 @@ namespace FEXCore::CPU {
DEF_OP(GuestOpcode) {
auto Op = IROp->C<IR::IROp_GuestOpcode>();
// metadata
DebugData->GuestOpcodes.push_back({Op->GuestEntryOffset, getCurr<uint8_t*>() - CodeData.BlockBegin});
DebugData->GuestOpcodes.push_back({Op->GuestEntryOffset, getCurr<uint8_t*>() - GuestEntry});
}
DEF_OP(Fence) {
@@ -1944,7 +1944,7 @@ DEF_OP(VUnZip2) {
}
case 8: {
if (Is256Bit) {
vshufpd(ToYMM(Dst), ToYMM(VectorLower), ToYMM(VectorUpper), 0b11'11);
vshufpd(ToYMM(Dst), ToYMM(VectorLower), ToYMM(VectorUpper), 0b1'1);
vpermq(ToYMM(Dst), ToYMM(Dst), 0b11'01'10'00);
} else {
vshufpd(Dst, VectorLower, VectorUpper, 0b1'1);
@@ -1958,191 +1958,6 @@ DEF_OP(VUnZip2) {
}
}
DEF_OP(VTrn) {
const auto Op = IROp->C<IR::IROp_VTrn>();
const auto OpSize = IROp->Size;
const auto ElementSize = Op->Header.ElementSize;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto Dst = GetDst(Node);
const auto VectorLower = GetSrc(Op->VectorLower.ID());
const auto VectorUpper = GetSrc(Op->VectorUpper.ID());
const auto LoadPshufbReg = [&](Xbyak::Xmm reg, uint64_t lower) {
mov(rax, lower);
mov(rcx, 0x80'80'80'80'80'80'80'80);
vmovq(reg, rax);
pinsrq(reg, rcx, 1);
};
switch (ElementSize) {
case 1: {
LoadPshufbReg(xmm15, 0x0E'0C'0A'08'06'04'02'00);
if (Is256Bit) {
vinserti128(ymm15, ymm15, xmm15, 1);
vpshufb(ymm14, ToYMM(VectorLower), ymm15);
vpshufb(ymm13, ToYMM(VectorUpper), ymm15);
vpunpcklbw(ToYMM(Dst), ymm14, ymm13);
} else {
vpshufb(xmm14, VectorLower, xmm15);
vpshufb(xmm13, VectorUpper, xmm15);
vpunpcklbw(Dst, xmm14, xmm13);
}
break;
}
case 2: {
LoadPshufbReg(xmm15, 0x0D'0C'09'08'05'04'01'00);
if (Is256Bit) {
vinserti128(ymm15, ymm15, xmm15, 1);
vpshufb(ymm14, ToYMM(VectorLower), ymm15);
vpshufb(ymm13, ToYMM(VectorUpper), ymm15);
vpunpcklwd(ToYMM(Dst), ymm14, ymm13);
} else {
vpshufb(xmm14, VectorLower, xmm15);
vpshufb(xmm13, VectorUpper, xmm15);
vpunpcklwd(Dst, xmm14, xmm13);
}
break;
}
case 4: {
LoadPshufbReg(xmm15, 0x0B'0A'09'08'03'02'01'00);
if (Is256Bit) {
vinserti128(ymm15, ymm15, xmm15, 1);
vpshufb(ymm14, ToYMM(VectorLower), ymm15);
vpshufb(ymm13, ToYMM(VectorUpper), ymm15);
vpunpckldq(ToYMM(Dst), ymm14, ymm13);
} else {
vpshufb(xmm14, VectorLower, xmm15);
vpshufb(xmm13, VectorUpper, xmm15);
vpunpckldq(Dst, xmm14, xmm13);
}
break;
}
case 8: {
LoadPshufbReg(xmm15, 0x07'06'05'04'03'02'01'00);
if (Is256Bit) {
vinserti128(ymm15, ymm15, xmm15, 1);
vpshufb(ymm14, ToYMM(VectorLower), ymm15);
vpshufb(ymm13, ToYMM(VectorUpper), ymm15);
vpunpcklqdq(ToYMM(Dst), ymm14, ymm13);
} else {
vpshufb(xmm14, VectorLower, xmm15);
vpshufb(xmm13, VectorUpper, xmm15);
vpunpcklqdq(Dst, xmm14, xmm13);
}
break;
}
default:
LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize);
return;
}
}
DEF_OP(VTrn2) {
const auto Op = IROp->C<IR::IROp_VTrn2>();
const auto OpSize = IROp->Size;
const auto ElementSize = Op->Header.ElementSize;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto Dst = GetDst(Node);
const auto VectorLower = GetSrc(Op->VectorLower.ID());
const auto VectorUpper = GetSrc(Op->VectorUpper.ID());
const auto LoadPshufbReg = [&](Xbyak::Xmm reg, uint64_t lower) {
mov(rax, lower);
mov(rcx, 0x80'80'80'80'80'80'80'80);
vmovq(reg, rax);
pinsrq(reg, rcx, 1);
};
switch (ElementSize) {
case 1: {
LoadPshufbReg(xmm15, 0x0F'0D'0B'09'07'05'03'01);
if (Is256Bit) {
vinserti128(ymm15, ymm15, xmm15, 1);
vpshufb(ymm14, ToYMM(VectorLower), ymm15);
vpshufb(ymm13, ToYMM(VectorUpper), ymm15);
vpunpcklbw(ToYMM(Dst), ymm14, ymm13);
} else {
vpshufb(xmm14, VectorLower, xmm15);
vpshufb(xmm13, VectorUpper, xmm15);
vpunpcklbw(Dst, xmm14, xmm13);
}
break;
}
case 2: {
LoadPshufbReg(xmm15, 0x0F'0E'0B'0A'07'06'03'02);
if (Is256Bit) {
vinserti128(ymm15, ymm15, xmm15, 1);
vpshufb(ymm14, ToYMM(VectorLower), ymm15);
vpshufb(ymm13, ToYMM(VectorUpper), ymm15);
vpunpcklwd(ToYMM(Dst), ymm14, ymm13);
} else {
vpshufb(xmm14, VectorLower, xmm15);
vpshufb(xmm13, VectorUpper, xmm15);
vpunpcklwd(Dst, xmm14, xmm13);
}
break;
}
case 4: {
LoadPshufbReg(xmm15, 0x0F'0E'0D'0C'07'06'05'04);
if (Is256Bit) {
vinserti128(ymm15, ymm15, xmm15, 1);
vpshufb(ymm14, ToYMM(VectorLower), ymm15);
vpshufb(ymm13, ToYMM(VectorUpper), ymm15);
vpunpckldq(ToYMM(Dst), ymm14, ymm13);
} else {
vpshufb(xmm14, VectorLower, xmm15);
vpshufb(xmm13, VectorUpper, xmm15);
vpunpckldq(Dst, xmm14, xmm13);
}
break;
}
case 8: {
LoadPshufbReg(xmm15, 0x0F'0E'0D'0C'0B'0A'09'08);
if (Is256Bit) {
vinserti128(ymm15, ymm15, xmm15, 1);
vpshufb(ymm14, ToYMM(VectorLower), ymm15);
vpshufb(ymm13, ToYMM(VectorUpper), ymm15);
vpunpcklqdq(ToYMM(Dst), ymm14, ymm13);
} else {
vpshufb(xmm14, VectorLower, xmm15);
vpshufb(xmm13, VectorUpper, xmm15);
vpunpcklqdq(Dst, xmm14, xmm13);
}
break;
}
default:
LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize);
return;
}
}
DEF_OP(VBSL) {
const auto Op = IROp->C<IR::IROp_VBSL>();
@@ -2734,22 +2549,7 @@ DEF_OP(VUShr) {
}
DEF_OP(VSShr) {
const auto Op = IROp->C<IR::IROp_VSShr>();
const auto OpSize = IROp->Size;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto ElementSize = IROp->ElementSize;
LOGMAN_THROW_AA_FMT(ElementSize == 4, "VSShr only supports 32-bit elements");
const auto Dst = GetDst(Node);
const auto ShiftVector = GetSrc(Op->ShiftVector.ID());
const auto Vector = GetSrc(Op->Vector.ID());
if (Is256Bit) {
vpsravd(ToYMM(Dst), ToYMM(Vector), ToYMM(ShiftVector));
} else {
vpsravd(Dst, Vector, ShiftVector);
}
LOGMAN_MSG_A_FMT("Unimplemented");
}
DEF_OP(VUShlS) {
@@ -3332,23 +3132,6 @@ DEF_OP(VShlI) {
const auto Vector = GetSrc(Op->Vector.ID());
switch (ElementSize) {
case 1: {
const auto Mask = 0xFFU >> BitShift;
mov(rax, Mask);
vmovq(xmm15, rax);
if (Is256Bit) {
vpsllw(ToYMM(Dst), ToYMM(Vector), BitShift);
vpbroadcastb(ymm15, xmm15);
vpand(ToYMM(Dst), ToYMM(Dst), ymm15);
} else {
vpsllw(Dst, Vector, BitShift);
vpbroadcastb(xmm15, xmm15);
vpand(Dst, Dst, ymm15);
}
break;
}
case 2: {
if (Is256Bit) {
vpsllw(ToYMM(Dst), ToYMM(Vector), BitShift);
@@ -4506,8 +4289,6 @@ void X86JITCore::RegisterVectorHandlers() {
REGISTER_OP(VZIP2, VZip2);
REGISTER_OP(VUNZIP, VUnZip);
REGISTER_OP(VUNZIP2, VUnZip2);
REGISTER_OP(VTRN, VTrn);
REGISTER_OP(VTRN2, VTrn2);
REGISTER_OP(VBSL, VBSL);
REGISTER_OP(VCMPEQ, VCMPEQ);
REGISTER_OP(VCMPEQZ, VCMPEQZ);
+1 -1
View File
@@ -14,7 +14,7 @@ $end_info$
#include <sys/mman.h>
namespace FEXCore {
LookupCache::LookupCache(FEXCore::Context::ContextImpl *CTX)
LookupCache::LookupCache(FEXCore::Context::Context *CTX)
: ctx {CTX} {
TotalCacheSize = ctx->Config.VirtualMemSize / 4096 * 8 + CODE_SIZE + L1_SIZE;
+5 -3
View File
@@ -1,5 +1,4 @@
#pragma once
#include "Interface/Context/Context.h"
#include <FEXCore/Utils/LogManager.h>
#include <cstdint>
@@ -13,6 +12,9 @@
#include <tsl/robin_map.h>
namespace FEXCore {
namespace Context {
struct Context;
}
class LookupCache {
public:
@@ -22,7 +24,7 @@ public:
uintptr_t GuestCode;
};
LookupCache(FEXCore::Context::ContextImpl *CTX);
LookupCache(FEXCore::Context::Context *CTX);
~LookupCache();
uintptr_t FindBlock(uint64_t Address) {
@@ -258,7 +260,7 @@ private:
size_t AllocateOffset {};
FEXCore::Context::ContextImpl *ctx;
FEXCore::Context::Context *ctx;
uint64_t VirtualMemSize{};
};
}
@@ -4,7 +4,7 @@
#include <FEXCore/Config/Config.h>
namespace FEXCore::CodeSerialize {
NamedRegionObjectHandler::NamedRegionObjectHandler(FEXCore::Context::ContextImpl *ctx) {
NamedRegionObjectHandler::NamedRegionObjectHandler(FEXCore::Context::Context *ctx) {
DefaultSerializationConfig.Cookie = CODE_COOKIE;
// Initialize the Arch from CPUID
@@ -13,7 +13,7 @@ namespace {
}
namespace FEXCore::CodeSerialize {
CodeObjectSerializeService::CodeObjectSerializeService(FEXCore::Context::ContextImpl *ctx)
CodeObjectSerializeService::CodeObjectSerializeService(FEXCore::Context::Context *ctx)
: CTX {ctx}
, AsyncHandler { &NamedRegionHandler , this }
, NamedRegionHandler { ctx } {
@@ -253,7 +253,7 @@ namespace FEXCore::CodeSerialize {
class NamedRegionObjectHandler final {
public:
NamedRegionObjectHandler(FEXCore::Context::ContextImpl *ctx);
NamedRegionObjectHandler(FEXCore::Context::Context *ctx);
void HandleNamedRegionObjectJobs();
@@ -338,7 +338,7 @@ namespace FEXCore::CodeSerialize {
*/
class CodeObjectSerializeService final {
public:
CodeObjectSerializeService(FEXCore::Context::ContextImpl *ctx);
CodeObjectSerializeService(FEXCore::Context::Context *ctx);
/**
* @brief Initialize the internal interface
@@ -440,7 +440,7 @@ namespace FEXCore::CodeSerialize {
void NotifyWork() { WorkAvailable.NotifyOne(); }
private:
FEXCore::Context::ContextImpl *CTX;
FEXCore::Context::Context *CTX;
Event WorkAvailable{};
std::unique_ptr<FEXCore::Threads::Thread> WorkerThread;
+266 -173
View File
@@ -277,6 +277,16 @@ void OpDispatchBuilder::IRETOp(OpcodeArgs) {
BlockSetRIP = true;
}
void OpDispatchBuilder::SIGRETOp(OpcodeArgs) {
const uint8_t GPRSize = CTX->GetGPRSize();
// Store the new RIP
_SignalReturn();
auto NewRIP = _LoadContext(GPRSize, GPRClass, offsetof(FEXCore::Core::CPUState, rip));
// This ExitFunction won't actually get hit but needs to exist
_ExitFunction(NewRIP);
BlockSetRIP = true;
}
void OpDispatchBuilder::CallbackReturnOp(OpcodeArgs) {
const uint8_t GPRSize = CTX->GetGPRSize();
// Store the new RIP
@@ -864,7 +874,7 @@ OrderedNode *OpDispatchBuilder::SelectCC(uint8_t OP, OrderedNode *TrueValue, Ord
case 0x7: { // JA - Jump if CF == 0 && ZF == 0
auto Flag1 = GetRFLAG(FEXCore::X86State::RFLAG_ZF_LOC);
auto Flag2 = GetRFLAG(FEXCore::X86State::RFLAG_CF_LOC);
auto Check = _Or(Flag1, Flag2);
auto Check = _Or(Flag1, _Lshl(Flag2, _Constant(1)));
SrcCond = _Select(FEXCore::IR::COND_EQ,
Check, ZeroConst, TrueValue, FalseValue);
break;
@@ -1603,17 +1613,17 @@ void OpDispatchBuilder::MOVSegOp(OpcodeArgs) {
OrderedNode *Src = LoadSource_WithOpSize(GPRClass, Op, Op->Src[0], 2, Op->Flags, -1);
switch (Op->Dest.Data.GPR.GPR) {
case FEXCore::X86State::REG_RAX: // ES
case 0: // ES
case FEXCore::X86State::REG_R8: // ES
_StoreContext(2, GPRClass, Src, offsetof(FEXCore::Core::CPUState, es_idx));
UpdatePrefixFromSegment(Src, FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX);
break;
case FEXCore::X86State::REG_RBX: // DS
case 1: // DS
case FEXCore::X86State::REG_R11: // DS
_StoreContext(2, GPRClass, Src, offsetof(FEXCore::Core::CPUState, ds_idx));
UpdatePrefixFromSegment(Src, FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX);
break;
case FEXCore::X86State::REG_RCX: // CS
case 2: // CS
case FEXCore::X86State::REG_R9: // CS
// CPL3 can't write to this
_Break(FEXCore::IR::BreakDefinition {
@@ -1623,12 +1633,12 @@ void OpDispatchBuilder::MOVSegOp(OpcodeArgs) {
.si_code = 0,
});
break;
case FEXCore::X86State::REG_RDX: // SS
case 3: // SS
case FEXCore::X86State::REG_R10: // SS
_StoreContext(2, GPRClass, Src, offsetof(FEXCore::Core::CPUState, ss_idx));
UpdatePrefixFromSegment(Src, FEXCore::X86Tables::DecodeFlags::FLAG_SS_PREFIX);
break;
case FEXCore::X86State::REG_RBP: // GS
case 6: // GS
case FEXCore::X86State::REG_R13: // GS
if (!CTX->Config.Is64BitMode) {
_StoreContext(2, GPRClass, Src, offsetof(FEXCore::Core::CPUState, gs_idx));
@@ -1638,7 +1648,7 @@ void OpDispatchBuilder::MOVSegOp(OpcodeArgs) {
DecodeFailure = true;
}
break;
case FEXCore::X86State::REG_RSP: // FS
case 7: // FS
case FEXCore::X86State::REG_R12: // FS
if (!CTX->Config.Is64BitMode) {
_StoreContext(2, GPRClass, Src, offsetof(FEXCore::Core::CPUState, fs_idx));
@@ -1658,23 +1668,23 @@ void OpDispatchBuilder::MOVSegOp(OpcodeArgs) {
OrderedNode *Segment{};
switch (Op->Src[0].Data.GPR.GPR) {
case FEXCore::X86State::REG_RAX: // ES
case 0: // ES
case FEXCore::X86State::REG_R8: // ES
Segment = _LoadContext(2, GPRClass, offsetof(FEXCore::Core::CPUState, es_idx));
break;
case FEXCore::X86State::REG_RBX: // DS
case 1: // DS
case FEXCore::X86State::REG_R11: // DS
Segment = _LoadContext(2, GPRClass, offsetof(FEXCore::Core::CPUState, ds_idx));
break;
case FEXCore::X86State::REG_RCX: // CS
case 2: // CS
case FEXCore::X86State::REG_R9: // CS
Segment = _LoadContext(2, GPRClass, offsetof(FEXCore::Core::CPUState, cs_idx));
break;
case FEXCore::X86State::REG_RDX: // SS
case 3: // SS
case FEXCore::X86State::REG_R10: // SS
Segment = _LoadContext(2, GPRClass, offsetof(FEXCore::Core::CPUState, ss_idx));
break;
case FEXCore::X86State::REG_RBP: // GS
case 6: // GS
case FEXCore::X86State::REG_R13: // GS
if (CTX->Config.Is64BitMode) {
Segment = _Constant(0);
@@ -1683,7 +1693,7 @@ void OpDispatchBuilder::MOVSegOp(OpcodeArgs) {
Segment = _LoadContext(2, GPRClass, offsetof(FEXCore::Core::CPUState, gs_idx));
}
break;
case FEXCore::X86State::REG_RSP: // FS
case 7: // FS
case FEXCore::X86State::REG_R12: // FS
if (CTX->Config.Is64BitMode) {
Segment = _Constant(0);
@@ -3793,21 +3803,73 @@ void OpDispatchBuilder::STOSOp(OpcodeArgs) {
StoreGPRRegister(X86State::REG_RDI, TailDest);
}
else {
// FEX doesn't support partial faulting REP instructions.
// Converting this to a `MemSet` IR op optimizes this quite significantly in our codegen.
// If FEX is to gain support for faulting REP instructions, then this implementation needs to change significantly.
OrderedNode *Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, -1);
OrderedNode *Dest = LoadGPRRegister(X86State::REG_RDI);
// Calculate deffered flags.
// This block is ending and it needs flag status
CalculateDeferredFlags();
// Only ES prefix
auto Segment = GetSegment(0, FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX, true);
// Create all our blocks
auto LoopHead = CreateNewCodeBlockAfter(GetCurrentBlock());
auto LoopTail = CreateNewCodeBlockAfter(LoopHead);
auto LoopEnd = CreateNewCodeBlockAfter(LoopTail);
OrderedNode *Counter = LoadGPRRegister(X86State::REG_RCX);
// At the time this was written, our RA can't handle accessing nodes across blocks.
// So we need to re-load and re-calculate essential values each iteration of the loop.
// First thing we need to do is finish this block and jump to the start of the loop.
// RA can now better allocate things, move these ops before the header, to avoid accessing
// DF on every iteration
auto SizeConst = _Constant(Size);
auto NegSizeConst = _Constant(-Size);
// Calculate direction.
auto DF = GetRFLAG(FEXCore::X86State::RFLAG_DF_LOC);
auto PtrDir = _Select(FEXCore::IR::COND_EQ,
DF, _Constant(0),
SizeConst, NegSizeConst);
auto Result = _MemSet(CTX->IsTSOEnabled(), Size, Segment ?: InvalidNode, Dest, Src, Counter, DF);
StoreGPRRegister(X86State::REG_RCX, _Constant(0));
StoreGPRRegister(X86State::REG_RDI, Result);
_Jump(LoopHead);
SetCurrentCodeBlock(LoopHead);
{
OrderedNode *Counter = LoadGPRRegister(X86State::REG_RCX);
// Can we end the block?
_CondJump(Counter, LoopEnd, LoopTail, {COND_EQ});
}
SetCurrentCodeBlock(LoopTail);
{
OrderedNode *Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, -1);
OrderedNode *Dest = LoadGPRRegister(X86State::REG_RDI);
// Only ES prefix
Dest = AppendSegmentOffset(Dest, 0, FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX, true);
// Store to memory where RDI points
_StoreMemAutoTSO(GPRClass, Size, Dest, Src, Size);
OrderedNode *TailCounter = LoadGPRRegister(X86State::REG_RCX);
OrderedNode *TailDest = LoadGPRRegister(X86State::REG_RDI);
// Decrement counter
TailCounter = _Sub(TailCounter, _Constant(1));
// Store the counter so we don't have to deal with PHI here
StoreGPRRegister(X86State::REG_RCX, TailCounter);
// Offset the pointer
TailDest = _Add(TailDest, PtrDir);
StoreGPRRegister(X86State::REG_RDI, TailDest);
// Jump back to the start, we have more work to do
_Jump(LoopHead);
}
// Make sure to start a new block after ending this one
SetCurrentCodeBlock(LoopEnd);
}
}
@@ -4780,19 +4842,20 @@ uint32_t OpDispatchBuilder::GetDstBitSize(X86Tables::DecodedOp Op) const {
return GetDstSize(Op) * 8;
}
OrderedNode *OpDispatchBuilder::GetSegment(uint32_t Flags, uint32_t DefaultPrefix, bool Override) {
OrderedNode *OpDispatchBuilder::AppendSegmentOffset(OrderedNode *Value, uint32_t Flags, uint32_t DefaultPrefix, bool Override) {
const uint8_t GPRSize = CTX->GetGPRSize();
if (CTX->Config.Is64BitMode) {
if (Flags & FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX) {
return _LoadContext(GPRSize, GPRClass, offsetof(FEXCore::Core::CPUState, fs_cached));
Value = _Add(Value, _LoadContext(GPRSize, GPRClass, offsetof(FEXCore::Core::CPUState, fs_cached)));
}
else if (Flags & FEXCore::X86Tables::DecodeFlags::FLAG_GS_PREFIX) {
return _LoadContext(GPRSize, GPRClass, offsetof(FEXCore::Core::CPUState, gs_cached));
Value = _Add(Value, _LoadContext(GPRSize, GPRClass, offsetof(FEXCore::Core::CPUState, gs_cached)));
}
// If there was any other segment in 64bit then it is ignored
}
else {
OrderedNode *Segment{};
uint32_t Prefix = Flags & FEXCore::X86Tables::DecodeFlags::FLAG_SEGMENTS;
if (!Prefix || Override) {
// If there was no prefix then use the default one if available
@@ -4802,28 +4865,29 @@ OrderedNode *OpDispatchBuilder::GetSegment(uint32_t Flags, uint32_t DefaultPrefi
// With the segment register optimization we store the GDT bases directly in the segment register to remove indexed loads
switch (Prefix) {
case FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX:
return _LoadContext(GPRSize, GPRClass, offsetof(FEXCore::Core::CPUState, es_cached));
Segment = _LoadContext(GPRSize, GPRClass, offsetof(FEXCore::Core::CPUState, es_cached));
break;
case FEXCore::X86Tables::DecodeFlags::FLAG_CS_PREFIX:
return _LoadContext(GPRSize, GPRClass, offsetof(FEXCore::Core::CPUState, cs_cached));
Segment = _LoadContext(GPRSize, GPRClass, offsetof(FEXCore::Core::CPUState, cs_cached));
break;
case FEXCore::X86Tables::DecodeFlags::FLAG_SS_PREFIX:
return _LoadContext(GPRSize, GPRClass, offsetof(FEXCore::Core::CPUState, ss_cached));
Segment = _LoadContext(GPRSize, GPRClass, offsetof(FEXCore::Core::CPUState, ss_cached));
break;
case FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX:
return _LoadContext(GPRSize, GPRClass, offsetof(FEXCore::Core::CPUState, ds_cached));
Segment = _LoadContext(GPRSize, GPRClass, offsetof(FEXCore::Core::CPUState, ds_cached));
break;
case FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX:
return _LoadContext(GPRSize, GPRClass, offsetof(FEXCore::Core::CPUState, fs_cached));
Segment = _LoadContext(GPRSize, GPRClass, offsetof(FEXCore::Core::CPUState, fs_cached));
break;
case FEXCore::X86Tables::DecodeFlags::FLAG_GS_PREFIX:
return _LoadContext(GPRSize, GPRClass, offsetof(FEXCore::Core::CPUState, gs_cached));
default:
break; // Do nothing
Segment = _LoadContext(GPRSize, GPRClass, offsetof(FEXCore::Core::CPUState, gs_cached));
break;
default: break; // Do nothing
}
}
return nullptr;
}
OrderedNode *OpDispatchBuilder::AppendSegmentOffset(OrderedNode *Value, uint32_t Flags, uint32_t DefaultPrefix, bool Override) {
auto Segment = GetSegment(Flags, DefaultPrefix, Override);
if (Segment) {
Value = _Add(Value, Segment);
if (Segment) {
Value = _Add(Value, Segment);
}
}
return Value;
@@ -4891,8 +4955,24 @@ OrderedNode *OpDispatchBuilder::LoadSource_WithOpSize(FEXCore::IR::RegisterClass
else if (gpr >= FEXCore::X86State::REG_XMM_0) {
const auto gprIndex = gpr - X86State::REG_XMM_0;
const auto regSize = CTX->HostFeatures.SupportsAVX ?
Core::CPUState::XMM_AVX_REG_SIZE :
Core::CPUState::XMM_SSE_REG_SIZE;
const auto VectorOffset = CTX->HostFeatures.SupportsAVX ?
offsetof(Core::CPUState, xmm.avx.data[gprIndex][0]) :
offsetof(Core::CPUState, xmm.sse.data[gprIndex][0]);
// Load the full register size if it is a XMM register source.
Src = LoadXMMRegister(gprIndex);
Src = _LoadRegister(false, VectorOffset, FPRClass, FPRFixedClass, regSize);
// If we are wanting a high-index then we need to extract an element from the upper half of the reg.
// We can only extract an element size here.
// TODO: Have the instruction doing this load do the extract instead of here.
// We don't have enough information here to know if we can avoid this dup.
if (highIndex && OpSize < Core::CPUState::XMM_SSE_REG_SIZE) {
Src = _VDupElement(regSize, OpSize, Src, 1);
}
// Now extract the subregister if it was a partial load /smaller/ than SSE size
// TODO: Instead of doing the VMov implicitly on load, hunt down all use cases that require partial loads and do it after load.
@@ -4902,11 +4982,11 @@ OrderedNode *OpDispatchBuilder::LoadSource_WithOpSize(FEXCore::IR::RegisterClass
}
}
else {
Src = LoadGPRRegister(gpr, OpSize, highIndex ? 8 : 0);
Src = _LoadRegister(false, offsetof(FEXCore::Core::CPUState, gregs[gpr]) + (highIndex ? 1 : 0), GPRClass, GPRFixedClass, OpSize);
}
}
else if (Operand.IsGPRDirect()) {
Src = LoadGPRRegister(Operand.Data.GPR.GPR, GPRSize);
Src = _LoadRegister(false, offsetof(FEXCore::Core::CPUState, gregs[Operand.Data.GPR.GPR]), GPRClass, GPRFixedClass, GPRSize);
LoadableType = true;
if (Operand.Data.GPR.GPR == FEXCore::X86State::REG_RSP && AccessType == MemoryAccessType::ACCESS_DEFAULT) {
@@ -4914,7 +4994,7 @@ OrderedNode *OpDispatchBuilder::LoadSource_WithOpSize(FEXCore::IR::RegisterClass
}
}
else if (Operand.IsGPRIndirect()) {
auto GPR = LoadGPRRegister(Operand.Data.GPRIndirect.GPR, GPRSize);
auto GPR = _LoadRegister(false, offsetof(FEXCore::Core::CPUState, gregs[Operand.Data.GPRIndirect.GPR]), GPRClass, GPRFixedClass, GPRSize);
auto Constant = _Constant(GPRSize * 8, Operand.Data.GPRIndirect.Displacement);
@@ -4939,7 +5019,7 @@ OrderedNode *OpDispatchBuilder::LoadSource_WithOpSize(FEXCore::IR::RegisterClass
else if (Operand.IsSIB()) {
OrderedNode *Tmp {};
if (Operand.Data.SIB.Index != FEXCore::X86State::REG_INVALID) {
Tmp = LoadGPRRegister(Operand.Data.SIB.Index, GPRSize);
Tmp = _LoadRegister(false, offsetof(FEXCore::Core::CPUState, gregs[Operand.Data.SIB.Index]), GPRClass, GPRFixedClass, GPRSize);
if (Operand.Data.SIB.Scale != 1) {
auto Constant = _Constant(GPRSize * 8, Operand.Data.SIB.Scale);
@@ -4951,7 +5031,7 @@ OrderedNode *OpDispatchBuilder::LoadSource_WithOpSize(FEXCore::IR::RegisterClass
}
if (Operand.Data.SIB.Base != FEXCore::X86State::REG_INVALID) {
auto GPR = LoadGPRRegister(Operand.Data.SIB.Base, GPRSize);
auto GPR = _LoadRegister(false, offsetof(FEXCore::Core::CPUState, gregs[Operand.Data.SIB.Base]), GPRClass, GPRFixedClass, GPRSize);
if (Tmp != nullptr) {
Tmp = _Add(Tmp, GPR);
@@ -5016,12 +5096,9 @@ OrderedNode *OpDispatchBuilder::GetRelocatedPC(FEXCore::X86Tables::DecodedOp con
OrderedNode *OpDispatchBuilder::LoadGPRRegister(uint32_t GPR, int8_t Size, uint8_t Offset) {
const uint8_t GPRSize = CTX->GetGPRSize();
if (Size == -1) {
Size = GPRSize;
}
OrderedNode *Reg = _LoadRegister(false, offsetof(FEXCore::Core::CPUState, gregs[GPR]), GPRClass, GPRFixedClass, GPRSize);
if (Size != GPRSize || Offset != 0) {
if (Size != -1 || Offset != 0) {
// Extract the subregister if requested.
Reg = _Bfe(Size, Size * 8, Offset, Reg);
}
@@ -5040,18 +5117,15 @@ OrderedNode *OpDispatchBuilder::LoadXMMRegister(uint32_t XMM) {
void OpDispatchBuilder::StoreGPRRegister(uint32_t GPR, OrderedNode *const Src, int8_t Size, uint8_t Offset) {
const uint8_t GPRSize = CTX->GetGPRSize();
if (Size == -1) {
Size = GPRSize;
}
OrderedNode *Reg = Src;
if (Size != GPRSize || Offset != 0) {
if (Size != -1 || Offset != 0) {
// Need to do an insert if not automatic size or zero offset.
Reg = LoadGPRRegister(GPR);
OrderedNode *Reg = LoadGPRRegister(GPR);
Reg = _Bfi(GPRSize, Size * 8, Offset, Reg, Src);
_StoreRegister(Reg, false, offsetof(FEXCore::Core::CPUState, gregs[GPR]), GPRClass, GPRFixedClass, GPRSize);
}
else {
_StoreRegister(Src, false, offsetof(FEXCore::Core::CPUState, gregs[GPR]), GPRClass, GPRFixedClass, GPRSize);
}
_StoreRegister(Reg, false, offsetof(FEXCore::Core::CPUState, gregs[GPR]), GPRClass, GPRFixedClass, GPRSize);
}
void OpDispatchBuilder::StoreXMMRegister(uint32_t XMM, OrderedNode *const Src) {
@@ -5095,34 +5169,45 @@ void OpDispatchBuilder::StoreResult_WithOpSize(FEXCore::IR::RegisterClassType Cl
}
else if (gpr >= FEXCore::X86State::REG_XMM_0) {
const auto gprIndex = gpr - X86State::REG_XMM_0;
const auto highIndex = Operand.Data.GPR.HighBits ? 1 : 0;
const auto VectorSize = CTX->HostFeatures.SupportsAVX ? 32 : 16;
const auto VectorOffset = CTX->HostFeatures.SupportsAVX ?
offsetof(Core::CPUState, xmm.avx.data[gprIndex][highIndex]) :
offsetof(Core::CPUState, xmm.sse.data[gprIndex][highIndex]);
auto Result = Src;
if (OpSize != VectorSize) {
// Partial writes can come from FPRs.
if (highIndex || OpSize != VectorSize) {
auto InsertResult = Src;
// Partial writes can come from GPR or FPR.
// TODO: Fix the instructions doing partial writes rather than dealing with it here.
auto SrcVector = LoadXMMRegister(gprIndex);
auto SrcVector = _LoadRegister(false, VectorOffset, FPRClass, FPRFixedClass, OpSize);
LOGMAN_THROW_AA_FMT(Class != IR::GPRClass, "Partial writes from GPR not allowed. Instruction: {}",
Op->TableInfo->Name);
// OpSize of 16 is special in that it is expected to zero the upper bits of the 256-bit operation.
// TODO: Longer term we should enforce the difference between zero and insert.
if (VectorSize == Core::CPUState::XMM_AVX_REG_SIZE && OpSize == Core::CPUState::XMM_SSE_REG_SIZE) {
Result = _VMov(OpSize, Src);
} else {
Result = _VInsElement(VectorSize, OpSize, 0, 0, SrcVector, Src);
if (Class == IR::GPRClass) {
InsertResult = _VInsGPR(VectorSize, OpSize, highIndex, SrcVector, Src);
}
else {
// OpSize of 16 is special in that it is expected to zero the upper bits of the 256-bit operation.
// TODO: Longer term we should enforce the difference between zero and insert.
if (VectorSize == Core::CPUState::XMM_AVX_REG_SIZE && OpSize == Core::CPUState::XMM_SSE_REG_SIZE) {
InsertResult = _VMov(OpSize, Src);
}
else {
InsertResult = _VInsElement(VectorSize, OpSize, highIndex, 0, SrcVector, Src);
}
}
}
StoreXMMRegister(gprIndex, Result);
_StoreRegister(InsertResult, false, VectorOffset, FPRClass, FPRFixedClass, VectorSize);
}
else {
_StoreRegister(Src, false, VectorOffset, FPRClass, FPRFixedClass, VectorSize);
}
}
else {
if (GPRSize == 8 && OpSize == 4) {
// If the Source IR op is 64 bits, we need to zext the upper bits
// For all other sizes, the upper bits are guaranteed to already be zero
OrderedNode *Value = GetOpSize(Src) == 8 ? _Bfe(4, 32, 0, Src) : Src;
StoreGPRRegister(gpr, Value, GPRSize);
_StoreRegister(Value, false, offsetof(FEXCore::Core::CPUState, gregs[gpr]), GPRClass, GPRFixedClass, GPRSize);
LOGMAN_THROW_AA_FMT(!Operand.Data.GPR.HighBits, "Can't handle 32bit store to high 8bit register");
}
@@ -5135,24 +5220,25 @@ void OpDispatchBuilder::StoreResult_WithOpSize(FEXCore::IR::RegisterClassType Cl
// mov al, 2 ; Move in to lower 8-bits.
// mov ah, 2 ; Move in to upper 8-bits of 16-bit reg.
// mov ax, 2 ; Move in to lower 16-bits of reg.
StoreGPRRegister(gpr, Src, OpSize, Operand.Data.GPR.HighBits * 8);
auto RegDest = _LoadRegister(false, offsetof(FEXCore::Core::CPUState, gregs[gpr]), GPRClass, GPRFixedClass, GPRSize);
auto Result = _Bfi(GPRSize, OpSize * 8, Operand.Data.GPR.HighBits * 8, RegDest, Src);
_StoreRegister(Result, false, offsetof(FEXCore::Core::CPUState, gregs[gpr]), GPRClass, GPRFixedClass, GPRSize);
}
else {
StoreGPRRegister(gpr, Src, std::min(GPRSize, OpSize));
_StoreRegister(Src, false, offsetof(FEXCore::Core::CPUState, gregs[gpr]), GPRClass, GPRFixedClass, std::min(GPRSize, OpSize));
}
}
}
}
else if (Operand.IsGPRDirect()) {
MemStoreDst = LoadGPRRegister(Operand.Data.GPR.GPR, GPRSize);
MemStoreDst = _LoadRegister(false, offsetof(FEXCore::Core::CPUState, gregs[Operand.Data.GPR.GPR]), GPRClass, GPRFixedClass, GPRSize);
MemStore = true;
if (Operand.Data.GPR.GPR == FEXCore::X86State::REG_RSP && AccessType == MemoryAccessType::ACCESS_DEFAULT) {
AccessType = MemoryAccessType::ACCESS_NONTSO;
}
}
else if (Operand.IsGPRIndirect()) {
auto GPR = LoadGPRRegister(Operand.Data.GPRIndirect.GPR, GPRSize);
auto GPR = _LoadRegister(false, offsetof(FEXCore::Core::CPUState, gregs[Operand.Data.GPRIndirect.GPR]), GPRClass, GPRFixedClass, GPRSize);
auto Constant = _Constant(GPRSize * 8, Operand.Data.GPRIndirect.Displacement);
MemStoreDst = _Add(GPR, Constant);
@@ -5174,7 +5260,7 @@ void OpDispatchBuilder::StoreResult_WithOpSize(FEXCore::IR::RegisterClassType Cl
else if (Operand.IsSIB()) {
OrderedNode *Tmp {};
if (Operand.Data.SIB.Index != FEXCore::X86State::REG_INVALID) {
Tmp = LoadGPRRegister(Operand.Data.SIB.Index, GPRSize);
Tmp = _LoadRegister(false, offsetof(FEXCore::Core::CPUState, gregs[Operand.Data.SIB.Index]), GPRClass, GPRFixedClass, GPRSize);
if (Operand.Data.SIB.Scale != 1) {
auto Constant = _Constant(GPRSize * 8, Operand.Data.SIB.Scale);
@@ -5183,7 +5269,7 @@ void OpDispatchBuilder::StoreResult_WithOpSize(FEXCore::IR::RegisterClassType Cl
}
if (Operand.Data.SIB.Base != FEXCore::X86State::REG_INVALID) {
auto GPR = LoadGPRRegister(Operand.Data.SIB.Base, GPRSize);
auto GPR = _LoadRegister(false, offsetof(FEXCore::Core::CPUState, gregs[Operand.Data.SIB.Base]), GPRClass, GPRFixedClass, GPRSize);
if (Tmp != nullptr) {
Tmp = _Add(Tmp, GPR);
@@ -5247,7 +5333,7 @@ void OpDispatchBuilder::StoreResult(FEXCore::IR::RegisterClassType Class, FEXCor
StoreResult(Class, Op, Op->Dest, Src, Align, AccessType);
}
OpDispatchBuilder::OpDispatchBuilder(FEXCore::Context::ContextImpl *ctx)
OpDispatchBuilder::OpDispatchBuilder(FEXCore::Context::Context *ctx)
: IREmitter {ctx->OpDispatcherAllocator}
, CTX {ctx} {
ResetWorkingList();
@@ -5282,7 +5368,58 @@ void OpDispatchBuilder::MOVGPRNTOp(OpcodeArgs) {
StoreResult(GPRClass, Op, Src, 1, MemoryAccessType::ACCESS_STREAM);
}
void OpDispatchBuilder::ALUOpImpl(OpcodeArgs, FEXCore::IR::IROps ALUIROp, FEXCore::IR::IROps AtomicFetchOp, bool RequiresMask) {
void OpDispatchBuilder::ALUOp(OpcodeArgs) {
bool RequiresMask = false;
FEXCore::IR::IROps IROp;
switch (Op->OP) {
case 0x0:
case 0x1:
case 0x2:
case 0x3:
case 0x4:
case 0x5:
IROp = FEXCore::IR::IROps::OP_ADD;
RequiresMask = true;
break;
case 0x8:
case 0x9:
case 0xA:
case 0xB:
case 0xC:
case 0xD:
IROp = FEXCore::IR::IROps::OP_OR;
break;
case 0x20:
case 0x21:
case 0x22:
case 0x23:
case 0x24:
case 0x25:
IROp = FEXCore::IR::IROps::OP_AND;
break;
case 0x28:
case 0x29:
case 0x2A:
case 0x2B:
case 0x2C:
case 0x2D:
IROp = FEXCore::IR::IROps::OP_SUB;
RequiresMask = true;
break;
case 0x30:
case 0x31:
case 0x32:
case 0x33:
case 0x34:
case 0x35:
IROp = FEXCore::IR::IROps::OP_XOR;
break;
default:
IROp = FEXCore::IR::IROps::OP_LAST;
LOGMAN_MSG_A_FMT("Unknown ALU Op: 0x{:x}", Op->OP);
break;
}
auto Size = GetDstSize(Op);
// X86 basic ALU ops just do the operation between the destination and a single source
@@ -5295,24 +5432,43 @@ void OpDispatchBuilder::ALUOpImpl(OpcodeArgs, FEXCore::IR::IROps ALUIROp, FEXCor
HandledLock = true;
OrderedNode *DestMem = LoadSource(GPRClass, Op, Op->Dest, Op->Flags, -1, false);
DestMem = AppendSegmentOffset(DestMem, Op->Flags);
auto FetchOp = _AtomicFetchAdd(Size, Src, DestMem);
// Overwrite our atomic op type
FetchOp.first->Header.Op = AtomicFetchOp;
Dest = FetchOp;
auto ALUOp = _Add(Dest, Src);
// Overwrite our IR's op type
ALUOp.first->Header.Op = ALUIROp;
Result = ALUOp;
switch (IROp) {
case FEXCore::IR::IROps::OP_ADD: {
Dest = _AtomicFetchAdd(Size, Src, DestMem);
Result = _Add(Dest, Src);
break;
}
case FEXCore::IR::IROps::OP_SUB: {
Dest = _AtomicFetchSub(Size, Src, DestMem);
Result = _Sub(Dest, Src);
break;
}
case FEXCore::IR::IROps::OP_OR: {
Dest = _AtomicFetchOr(Size, Src, DestMem);
Result = _Or(Dest, Src);
break;
}
case FEXCore::IR::IROps::OP_AND: {
Dest = _AtomicFetchAnd(Size, Src, DestMem);
Result = _And(Dest, Src);
break;
}
case FEXCore::IR::IROps::OP_XOR: {
Dest = _AtomicFetchXor(Size, Src, DestMem);
Result = _Xor(Dest, Src);
break;
}
default:
LOGMAN_MSG_A_FMT("Unknown Atomic IR Op: {}", ToUnderlying(IROp));
break;
}
}
else {
Dest = LoadSource(GPRClass, Op, Op->Dest, Op->Flags, -1);
auto ALUOp = _Add(Dest, Src);
// Overwrite our IR's op type
ALUOp.first->Header.Op = ALUIROp;
ALUOp.first->Header.Op = IROp;
Result = ALUOp;
StoreResult(GPRClass, Op, Result, -1);
@@ -5324,7 +5480,7 @@ void OpDispatchBuilder::ALUOpImpl(OpcodeArgs, FEXCore::IR::IROps ALUIROp, FEXCor
// Flags set
{
switch (ALUIROp) {
switch (IROp) {
case FEXCore::IR::IROps::OP_ADD:
GenerateFlags_ADD(Op, Result, Dest, Src);
break;
@@ -5342,11 +5498,6 @@ void OpDispatchBuilder::ALUOpImpl(OpcodeArgs, FEXCore::IR::IROps ALUIROp, FEXCor
}
}
template<FEXCore::IR::IROps ALUIROp, FEXCore::IR::IROps AtomicFetchOp, bool RequiresMask>
void OpDispatchBuilder::ALUOp(OpcodeArgs) {
ALUOpImpl(Op, ALUIROp, AtomicFetchOp, RequiresMask);
}
void OpDispatchBuilder::INTOp(OpcodeArgs) {
IR::BreakDefinition Reason;
bool SetRIPToNext = false;
@@ -5472,29 +5623,6 @@ void OpDispatchBuilder::FenceOp(OpcodeArgs) {
_Fence({FenceType});
}
void OpDispatchBuilder::CLWB(OpcodeArgs) {
OrderedNode *DestMem = LoadSource(GPRClass, Op, Op->Dest, Op->Flags, -1, false);
DestMem = AppendSegmentOffset(DestMem, Op->Flags);
_CacheLineClean(DestMem);
}
void OpDispatchBuilder::CLFLUSHOPT(OpcodeArgs) {
OrderedNode *DestMem = LoadSource(GPRClass, Op, Op->Dest, Op->Flags, -1, false);
DestMem = AppendSegmentOffset(DestMem, Op->Flags);
_CacheLineClear(DestMem, false);
}
void OpDispatchBuilder::MemFenceOrXSAVEOPT(OpcodeArgs) {
if (Op->ModRM == 0xF0) {
// 0xF0 is MFENCE
_Fence(FEXCore::IR::Fence_LoadStore);
}
else {
LogMan::Msg::EFmt("Application tried using XSAVEOPT");
UnimplementedOp(Op);
}
}
void OpDispatchBuilder::StoreFenceOrCLFlush(OpcodeArgs) {
if (Op->ModRM == 0xF8) {
// 0xF8 is SFENCE
@@ -5504,7 +5632,7 @@ void OpDispatchBuilder::StoreFenceOrCLFlush(OpcodeArgs) {
// This is a CLFlush
OrderedNode *DestMem = LoadSource(GPRClass, Op, Op->Dest, Op->Flags, -1, false);
DestMem = AppendSegmentOffset(DestMem, Op->Flags);
_CacheLineClear(DestMem, true);
_CacheLineClear(DestMem);
}
}
@@ -5678,12 +5806,8 @@ void OpDispatchBuilder::InstallHostSpecificOpcodeHandlers() {
static constexpr std::tuple<uint16_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr> AVXTable[] = {
{OPD(1, 0b00, 0x10), 1, &OpDispatchBuilder::VMOVUPS_VMOVUPD_Op},
{OPD(1, 0b01, 0x10), 1, &OpDispatchBuilder::VMOVUPS_VMOVUPD_Op},
{OPD(1, 0b10, 0x10), 1, &OpDispatchBuilder::VMOVSSOp},
{OPD(1, 0b11, 0x10), 1, &OpDispatchBuilder::VMOVSDOp},
{OPD(1, 0b00, 0x11), 1, &OpDispatchBuilder::VMOVUPS_VMOVUPD_Op},
{OPD(1, 0b01, 0x11), 1, &OpDispatchBuilder::VMOVUPS_VMOVUPD_Op},
{OPD(1, 0b10, 0x11), 1, &OpDispatchBuilder::VMOVSSOp},
{OPD(1, 0b11, 0x11), 1, &OpDispatchBuilder::VMOVSDOp},
{OPD(1, 0b00, 0x12), 1, &OpDispatchBuilder::VMOVLPOp},
{OPD(1, 0b01, 0x12), 1, &OpDispatchBuilder::VMOVLPOp},
@@ -5802,10 +5926,6 @@ void OpDispatchBuilder::InstallHostSpecificOpcodeHandlers() {
{OPD(1, 0b01, 0x6F), 1, &OpDispatchBuilder::VMOVAPS_VMOVAPD_Op},
{OPD(1, 0b10, 0x6F), 1, &OpDispatchBuilder::VMOVUPS_VMOVUPD_Op},
{OPD(1, 0b01, 0x70), 1, &OpDispatchBuilder::VPSHUFWOp<4, true>},
{OPD(1, 0b10, 0x70), 1, &OpDispatchBuilder::VPSHUFWOp<2, false>},
{OPD(1, 0b11, 0x70), 1, &OpDispatchBuilder::VPSHUFWOp<2, true>},
{OPD(1, 0b01, 0x74), 1, &OpDispatchBuilder::AVXVectorALUOp<IR::OP_VCMPEQ, 1>},
{OPD(1, 0b01, 0x75), 1, &OpDispatchBuilder::AVXVectorALUOp<IR::OP_VCMPEQ, 2>},
{OPD(1, 0b01, 0x76), 1, &OpDispatchBuilder::AVXVectorALUOp<IR::OP_VCMPEQ, 4>},
@@ -5814,8 +5934,6 @@ void OpDispatchBuilder::InstallHostSpecificOpcodeHandlers() {
{OPD(1, 0b01, 0x7C), 1, &OpDispatchBuilder::VHADDPOp<IR::OP_VFADDP, 8>},
{OPD(1, 0b11, 0x7C), 1, &OpDispatchBuilder::VHADDPOp<IR::OP_VFADDP, 4>},
{OPD(1, 0b01, 0x7D), 1, &OpDispatchBuilder::VHSUBPOp<8>},
{OPD(1, 0b11, 0x7D), 1, &OpDispatchBuilder::VHSUBPOp<4>},
{OPD(1, 0b01, 0x7E), 1, &OpDispatchBuilder::MOVBetweenGPR_FPR},
{OPD(1, 0b10, 0x7E), 1, &OpDispatchBuilder::MOVQOp},
@@ -5830,9 +5948,6 @@ void OpDispatchBuilder::InstallHostSpecificOpcodeHandlers() {
{OPD(1, 0b01, 0xC5), 1, &OpDispatchBuilder::PExtrOp<2>},
{OPD(1, 0b00, 0xC6), 1, &OpDispatchBuilder::VSHUFOp<4>},
{OPD(1, 0b01, 0xC6), 1, &OpDispatchBuilder::VSHUFOp<8>},
{OPD(1, 0b01, 0xD0), 1, &OpDispatchBuilder::VADDSUBPOp<8>},
{OPD(1, 0b11, 0xD0), 1, &OpDispatchBuilder::VADDSUBPOp<4>},
@@ -5880,7 +5995,6 @@ void OpDispatchBuilder::InstallHostSpecificOpcodeHandlers() {
{OPD(1, 0b01, 0xF2), 1, &OpDispatchBuilder::VPSLLOp<4>},
{OPD(1, 0b01, 0xF3), 1, &OpDispatchBuilder::VPSLLOp<8>},
{OPD(1, 0b01, 0xF4), 1, &OpDispatchBuilder::VPMULLOp<4, false>},
{OPD(1, 0b01, 0xF5), 1, &OpDispatchBuilder::VPMADDWDOp},
{OPD(1, 0b01, 0xF7), 1, &OpDispatchBuilder::MASKMOVOp},
{OPD(1, 0b01, 0xF8), 1, &OpDispatchBuilder::AVXVectorALUOp<IR::OP_VSUB, 1>},
@@ -5891,24 +6005,17 @@ void OpDispatchBuilder::InstallHostSpecificOpcodeHandlers() {
{OPD(1, 0b01, 0xFD), 1, &OpDispatchBuilder::AVXVectorALUOp<IR::OP_VADD, 2>},
{OPD(1, 0b01, 0xFE), 1, &OpDispatchBuilder::AVXVectorALUOp<IR::OP_VADD, 4>},
{OPD(2, 0b01, 0x00), 1, &OpDispatchBuilder::VPSHUFBOp},
{OPD(2, 0b01, 0x01), 1, &OpDispatchBuilder::VHADDPOp<IR::OP_VADDP, 2>},
{OPD(2, 0b01, 0x02), 1, &OpDispatchBuilder::VHADDPOp<IR::OP_VADDP, 4>},
{OPD(2, 0b01, 0x03), 1, &OpDispatchBuilder::VPHADDSWOp},
{OPD(2, 0b01, 0x05), 1, &OpDispatchBuilder::VPHSUBOp<2>},
{OPD(2, 0b01, 0x06), 1, &OpDispatchBuilder::VPHSUBOp<4>},
{OPD(2, 0b01, 0x07), 1, &OpDispatchBuilder::VPHSUBSWOp},
{OPD(2, 0b01, 0x08), 1, &OpDispatchBuilder::VPSIGN<1>},
{OPD(2, 0b01, 0x09), 1, &OpDispatchBuilder::VPSIGN<2>},
{OPD(2, 0b01, 0x0A), 1, &OpDispatchBuilder::VPSIGN<4>},
{OPD(2, 0b01, 0x0B), 1, &OpDispatchBuilder::VPMULHRSWOp},
{OPD(2, 0b01, 0x0C), 1, &OpDispatchBuilder::VPERMILRegOp<4>},
{OPD(2, 0b01, 0x0D), 1, &OpDispatchBuilder::VPERMILRegOp<8>},
{OPD(2, 0b01, 0x16), 1, &OpDispatchBuilder::VPERMDOp},
{OPD(2, 0b01, 0x17), 1, &OpDispatchBuilder::PTestOp},
{OPD(2, 0b01, 0x18), 1, &OpDispatchBuilder::VBROADCASTOp<4>},
{OPD(2, 0b01, 0x19), 1, &OpDispatchBuilder::VBROADCASTOp<8>},
{OPD(2, 0b01, 0x1A), 1, &OpDispatchBuilder::VBROADCASTOp<16>},
@@ -5934,7 +6041,6 @@ void OpDispatchBuilder::InstallHostSpecificOpcodeHandlers() {
{OPD(2, 0b01, 0x33), 1, &OpDispatchBuilder::AVXExtendVectorElements<2, 4, false>},
{OPD(2, 0b01, 0x34), 1, &OpDispatchBuilder::AVXExtendVectorElements<2, 8, false>},
{OPD(2, 0b01, 0x35), 1, &OpDispatchBuilder::AVXExtendVectorElements<4, 8, false>},
{OPD(2, 0b01, 0x36), 1, &OpDispatchBuilder::VPERMDOp},
{OPD(2, 0b01, 0x37), 1, &OpDispatchBuilder::AVXVectorALUOp<IR::OP_VCMPGT, 8>},
{OPD(2, 0b01, 0x38), 1, &OpDispatchBuilder::AVXVectorALUOp<IR::OP_VSMIN, 1>},
@@ -5948,7 +6054,6 @@ void OpDispatchBuilder::InstallHostSpecificOpcodeHandlers() {
{OPD(2, 0b01, 0x40), 1, &OpDispatchBuilder::AVXVectorALUOp<IR::OP_VSMUL, 4>},
{OPD(2, 0b01, 0x41), 1, &OpDispatchBuilder::VPHMINPOSUWOp},
{OPD(2, 0b01, 0x46), 1, &OpDispatchBuilder::VPSRAVDOp},
{OPD(2, 0b01, 0x58), 1, &OpDispatchBuilder::VBROADCASTOp<4>},
{OPD(2, 0b01, 0x59), 1, &OpDispatchBuilder::VBROADCASTOp<8>},
@@ -5965,7 +6070,6 @@ void OpDispatchBuilder::InstallHostSpecificOpcodeHandlers() {
{OPD(3, 0b01, 0x00), 1, &OpDispatchBuilder::VPERMQOp},
{OPD(3, 0b01, 0x01), 1, &OpDispatchBuilder::VPERMQOp},
{OPD(3, 0b01, 0x02), 1, &OpDispatchBuilder::VPBLENDDOp},
{OPD(3, 0b01, 0x04), 1, &OpDispatchBuilder::VPERMILImmOp<4>},
{OPD(3, 0b01, 0x05), 1, &OpDispatchBuilder::VPERMILImmOp<8>},
{OPD(3, 0b01, 0x06), 1, &OpDispatchBuilder::VPERM2Op},
@@ -5973,10 +6077,6 @@ void OpDispatchBuilder::InstallHostSpecificOpcodeHandlers() {
{OPD(3, 0b01, 0x09), 1, &OpDispatchBuilder::AVXVectorRound<8, false>},
{OPD(3, 0b01, 0x0A), 1, &OpDispatchBuilder::AVXVectorRound<4, true>},
{OPD(3, 0b01, 0x0B), 1, &OpDispatchBuilder::AVXVectorRound<8, true>},
{OPD(3, 0b01, 0x0C), 1, &OpDispatchBuilder::VPBLENDDOp},
{OPD(3, 0b01, 0x0D), 1, &OpDispatchBuilder::VBLENDPDOp},
{OPD(3, 0b01, 0x0E), 1, &OpDispatchBuilder::VPBLENDWOp},
{OPD(3, 0b01, 0x0F), 1, &OpDispatchBuilder::VPALIGNROp},
{OPD(3, 0b01, 0x14), 1, &OpDispatchBuilder::PExtrOp<1>},
{OPD(3, 0b01, 0x15), 1, &OpDispatchBuilder::PExtrOp<2>},
@@ -5984,22 +6084,16 @@ void OpDispatchBuilder::InstallHostSpecificOpcodeHandlers() {
{OPD(3, 0b01, 0x17), 1, &OpDispatchBuilder::PExtrOp<4>},
{OPD(3, 0b01, 0x18), 1, &OpDispatchBuilder::VINSERTOp},
{OPD(3, 0b01, 0x19), 1, &OpDispatchBuilder::VEXTRACT128Op},
{OPD(3, 0b01, 0x21), 1, &OpDispatchBuilder::VINSERTPSOp},
{OPD(3, 0b01, 0x38), 1, &OpDispatchBuilder::VINSERTOp},
{OPD(3, 0b01, 0x39), 1, &OpDispatchBuilder::VEXTRACT128Op},
{OPD(3, 0b01, 0x40), 1, &OpDispatchBuilder::VDPPOp<4>},
{OPD(3, 0b01, 0x41), 1, &OpDispatchBuilder::VDPPOp<8>},
{OPD(3, 0b01, 0x46), 1, &OpDispatchBuilder::VPERM2Op},
{OPD(3, 0b01, 0x4A), 1, &OpDispatchBuilder::AVXVectorVariableBlend<4>},
{OPD(3, 0b01, 0x4B), 1, &OpDispatchBuilder::AVXVectorVariableBlend<8>},
{OPD(3, 0b01, 0x4C), 1, &OpDispatchBuilder::AVXVectorVariableBlend<1>},
{OPD(3, 0b01, 0xDF), 1, &OpDispatchBuilder::VAESKeyGenAssistOp},
};
#undef OPD
@@ -6069,19 +6163,19 @@ void OpDispatchBuilder::InstallHostSpecificOpcodeHandlers() {
void InstallOpcodeHandlers(Context::OperatingMode Mode) {
constexpr std::tuple<uint8_t, uint8_t, X86Tables::OpDispatchPtr> BaseOpTable[] = {
// Instructions
{0x00, 6, &OpDispatchBuilder::ALUOp<FEXCore::IR::IROps::OP_ADD, FEXCore::IR::IROps::OP_ATOMICFETCHADD, true>},
{0x00, 6, &OpDispatchBuilder::ALUOp},
{0x08, 6, &OpDispatchBuilder::ALUOp<FEXCore::IR::IROps::OP_OR, FEXCore::IR::IROps::OP_ATOMICFETCHOR, false>},
{0x08, 6, &OpDispatchBuilder::ALUOp},
{0x10, 6, &OpDispatchBuilder::ADCOp<0>},
{0x18, 6, &OpDispatchBuilder::SBBOp<0>},
{0x20, 6, &OpDispatchBuilder::ALUOp<FEXCore::IR::IROps::OP_AND, FEXCore::IR::IROps::OP_ATOMICFETCHAND, false>},
{0x20, 6, &OpDispatchBuilder::ALUOp},
{0x28, 6, &OpDispatchBuilder::ALUOp<FEXCore::IR::IROps::OP_SUB, FEXCore::IR::IROps::OP_ATOMICFETCHSUB, true>},
{0x28, 6, &OpDispatchBuilder::ALUOp},
{0x30, 6, &OpDispatchBuilder::ALUOp<FEXCore::IR::IROps::OP_XOR, FEXCore::IR::IROps::OP_ATOMICFETCHXOR, false>},
{0x30, 6, &OpDispatchBuilder::ALUOp},
{0x38, 6, &OpDispatchBuilder::CMPOp<0>},
{0x50, 8, &OpDispatchBuilder::PUSHREGOp},
@@ -6210,8 +6304,9 @@ void InstallOpcodeHandlers(Context::OperatingMode Mode) {
{0x12, 2, &OpDispatchBuilder::MOVLPOp},
{0x14, 1, &OpDispatchBuilder::PUNPCKLOp<4>},
{0x15, 1, &OpDispatchBuilder::PUNPCKHOp<4>},
{0x16, 2, &OpDispatchBuilder::MOVHPDOp},
{0x28, 2, &OpDispatchBuilder::MOVAPSOp},
{0x16, 1, &OpDispatchBuilder::MOVLHPSOp},
{0x17, 1, &OpDispatchBuilder::MOVUPSOp},
{0x28, 2, &OpDispatchBuilder::MOVUPSOp},
{0x2A, 1, &OpDispatchBuilder::MMX_To_XMM_Vector_CVT_Int_To_Float<4, false>},
{0x2B, 1, &OpDispatchBuilder::MOVVectorNTOp},
{0x2C, 1, &OpDispatchBuilder::Vector_CVT_Float_To_Int<4, false, false>},
@@ -6301,6 +6396,7 @@ void InstallOpcodeHandlers(Context::OperatingMode Mode) {
{0xFE, 1, &OpDispatchBuilder::VectorALUOp<IR::OP_VADD, 4>},
// FEX reserved instructions
{0x36, 1, &OpDispatchBuilder::SIGRETOp},
{0x37, 1, &OpDispatchBuilder::CallbackReturnOp},
};
@@ -6674,15 +6770,12 @@ constexpr uint16_t PF_F2 = 3;
{OPD(FEXCore::X86Tables::TYPE_GROUP_15, PF_NONE, 2), 1, &OpDispatchBuilder::LDMXCSR},
{OPD(FEXCore::X86Tables::TYPE_GROUP_15, PF_NONE, 3), 1, &OpDispatchBuilder::STMXCSR},
{OPD(FEXCore::X86Tables::TYPE_GROUP_15, PF_NONE, 5), 1, &OpDispatchBuilder::FenceOp<FEXCore::IR::Fence_Load.Val>}, //LFENCE
{OPD(FEXCore::X86Tables::TYPE_GROUP_15, PF_NONE, 6), 1, &OpDispatchBuilder::MemFenceOrXSAVEOPT}, //MFENCE
{OPD(FEXCore::X86Tables::TYPE_GROUP_15, PF_NONE, 6), 1, &OpDispatchBuilder::FenceOp<FEXCore::IR::Fence_LoadStore.Val>}, //MFENCE
{OPD(FEXCore::X86Tables::TYPE_GROUP_15, PF_NONE, 7), 1, &OpDispatchBuilder::StoreFenceOrCLFlush}, //SFENCE
{OPD(FEXCore::X86Tables::TYPE_GROUP_15, PF_F3, 5), 1, &OpDispatchBuilder::UnimplementedOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_15, PF_F3, 6), 1, &OpDispatchBuilder::UnimplementedOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_15, PF_66, 6), 1, &OpDispatchBuilder::CLWB},
{OPD(FEXCore::X86Tables::TYPE_GROUP_15, PF_66, 7), 1, &OpDispatchBuilder::CLFLUSHOPT},
// GROUP 16
{OPD(FEXCore::X86Tables::TYPE_GROUP_16, PF_NONE, 0), 8, &OpDispatchBuilder::NOPOp},
{OPD(FEXCore::X86Tables::TYPE_GROUP_16, PF_F3, 0), 8, &OpDispatchBuilder::NOPOp},
+4 -71
View File
@@ -75,7 +75,7 @@ public:
OrderedNode* flagsOpDestSigned{};
OrderedNode* flagsOpSrcSigned{};
FEXCore::Context::ContextImpl *CTX{};
FEXCore::Context::Context *CTX{};
// Used during new op bringup
bool ShouldDump {false};
@@ -149,7 +149,7 @@ public:
return false;
}
OpDispatchBuilder(FEXCore::Context::ContextImpl *ctx);
OpDispatchBuilder(FEXCore::Context::Context *ctx);
OpDispatchBuilder(FEXCore::Utils::IntrusivePooledAllocator &Allocator);
void ResetWorkingList();
@@ -168,7 +168,6 @@ public:
void MOVGPRNTOp(OpcodeArgs);
void MOVVectorOp(OpcodeArgs);
void MOVVectorNTOp(OpcodeArgs);
template<FEXCore::IR::IROps ALUIROp, FEXCore::IR::IROps AtomicFetchOp, bool RequiresMask>
void ALUOp(OpcodeArgs);
void INTOp(OpcodeArgs);
void SyscallOp(OpcodeArgs);
@@ -177,6 +176,7 @@ public:
void NOPOp(OpcodeArgs);
void RETOp(OpcodeArgs);
void IRETOp(OpcodeArgs);
void SIGRETOp(OpcodeArgs);
void CallbackReturnOp(OpcodeArgs);
void SecondaryALUOp(OpcodeArgs);
template<uint32_t SrcIndex>
@@ -304,6 +304,7 @@ public:
// SSE
void MOVAPSOp(OpcodeArgs);
void MOVUPSOp(OpcodeArgs);
void MOVLHPSOp(OpcodeArgs);
void MOVLPOp(OpcodeArgs);
void MOVSHDUPOp(OpcodeArgs);
void MOVSLDUPOp(OpcodeArgs);
@@ -436,22 +437,14 @@ public:
void VANDNOp(OpcodeArgs);
void VBLENDPDOp(OpcodeArgs);
void VPBLENDDOp(OpcodeArgs);
void VPBLENDWOp(OpcodeArgs);
template <size_t ElementSize>
void VBROADCASTOp(OpcodeArgs);
template <size_t ElementSize>
void VDPPOp(OpcodeArgs);
void VEXTRACT128Op(OpcodeArgs);
template <IROps IROp, size_t ElementSize>
void VHADDPOp(OpcodeArgs);
template <size_t ElementSize>
void VHSUBPOp(OpcodeArgs);
void VINSERTOp(OpcodeArgs);
void VINSERTPSOp(OpcodeArgs);
@@ -466,9 +459,6 @@ public:
void VMOVSHDUPOp(OpcodeArgs);
void VMOVSLDUPOp(OpcodeArgs);
void VMOVSDOp(OpcodeArgs);
void VMOVSSOp(OpcodeArgs);
void VMOVVectorNTOp(OpcodeArgs);
template <size_t ElementSize>
@@ -477,26 +467,16 @@ public:
template <size_t ElementSize>
void VPACKUSOp(OpcodeArgs);
void VPALIGNROp(OpcodeArgs);
void VPERM2Op(OpcodeArgs);
void VPERMDOp(OpcodeArgs);
void VPERMQOp(OpcodeArgs);
template <size_t ElementSize>
void VPERMILImmOp(OpcodeArgs);
template <size_t ElementSize>
void VPERMILRegOp(OpcodeArgs);
void VPHADDSWOp(OpcodeArgs);
void VPHMINPOSUWOp(OpcodeArgs);
template <size_t ElementSize>
void VPHSUBOp(OpcodeArgs);
void VPHSUBSWOp(OpcodeArgs);
void VPMADDWDOp(OpcodeArgs);
void VPMULHRSWOp(OpcodeArgs);
@@ -506,11 +486,6 @@ public:
template <size_t ElementSize, bool Signed>
void VPMULLOp(OpcodeArgs);
void VPSHUFBOp(OpcodeArgs);
template <size_t ElementSize, bool Low>
void VPSHUFWOp(OpcodeArgs);
template <size_t ElementSize>
void VPSLLOp(OpcodeArgs);
void VPSLLDQOp(OpcodeArgs);
@@ -523,8 +498,6 @@ public:
template <size_t ElementSize>
void VPSRAIOp(OpcodeArgs);
void VPSRAVDOp(OpcodeArgs);
template <size_t ElementSize>
void VPSRLDOp(OpcodeArgs);
void VPSRLDQOp(OpcodeArgs);
@@ -538,9 +511,6 @@ public:
template <size_t ElementSize>
void VPSRLIOp(OpcodeArgs);
template <size_t ElementSize>
void VSHUFOp(OpcodeArgs);
void VZEROOp(OpcodeArgs);
// X87 Ops
@@ -719,9 +689,6 @@ public:
template<uint8_t FenceType>
void FenceOp(OpcodeArgs);
void CLWB(OpcodeArgs);
void CLFLUSHOPT(OpcodeArgs);
void MemFenceOrXSAVEOPT(OpcodeArgs);
void StoreFenceOrCLFlush(OpcodeArgs);
void CLZeroOp(OpcodeArgs);
void RDTSCPOp(OpcodeArgs);
@@ -781,8 +748,6 @@ private:
FEXCore::IR::IROp_IRHeader *Current_Header{};
OrderedNode *Current_HeaderNode{};
void ALUOpImpl(OpcodeArgs, FEXCore::IR::IROps ALUIROp, FEXCore::IR::IROps AtomicFetchOp, bool RequiresMask);
// Opcode helpers for generalizing behavior across VEX and non-VEX variants.
OrderedNode* ADDSUBPOpImpl(OpcodeArgs, size_t ElementSize,
@@ -792,9 +757,6 @@ private:
void AVXVectorScalarALUOpImpl(OpcodeArgs, IROps IROp, size_t ElementSize);
void AVXVectorUnaryOpImpl(OpcodeArgs, IROps IROp, size_t ElementSize, bool Scalar);
template <size_t ElementSize>
void AVXVectorVariableBlend(OpcodeArgs);
OrderedNode* AESKeyGenAssistImpl(OpcodeArgs);
OrderedNode* AESIMCImpl(OpcodeArgs);
@@ -805,10 +767,6 @@ private:
OrderedNode* ExtendVectorElementsImpl(OpcodeArgs, size_t ElementSize,
size_t DstElementSize, bool Signed);
OrderedNode* HSUBPOpImpl(OpcodeArgs, size_t ElementSize,
const X86Tables::DecodedOperand& Src1Op,
const X86Tables::DecodedOperand& Src2Op);
OrderedNode* InsertPSOpImpl(OpcodeArgs, const X86Tables::DecodedOperand& Src1,
const X86Tables::DecodedOperand& Src2,
const X86Tables::DecodedOperand& Imm);
@@ -819,24 +777,11 @@ private:
OrderedNode* PACKUSOpImpl(OpcodeArgs, size_t ElementSize,
OrderedNode *Src1, OrderedNode *Src2);
OrderedNode* PALIGNROpImpl(OpcodeArgs, const X86Tables::DecodedOperand& Src1,
const X86Tables::DecodedOperand& Src2,
const X86Tables::DecodedOperand& Imm);
OrderedNode* PHADDSOpImpl(OpcodeArgs, const X86Tables::DecodedOperand& Src1,
const X86Tables::DecodedOperand& Src2);
OrderedNode* PHMINPOSUWOpImpl(OpcodeArgs);
OrderedNode* PHSUBOpImpl(OpcodeArgs, const X86Tables::DecodedOperand& Src1,
const X86Tables::DecodedOperand& Src2, size_t ElementSize);
OrderedNode* PHSUBSOpImpl(OpcodeArgs, const X86Tables::DecodedOperand& Src1Op,
const X86Tables::DecodedOperand& Src2Op);
OrderedNode* PMADDWDOpImpl(OpcodeArgs, const X86Tables::DecodedOperand& Src1,
const X86Tables::DecodedOperand& Src2);
OrderedNode* PMULHRSWOpImpl(OpcodeArgs, OrderedNode *Src1, OrderedNode *Src2);
OrderedNode* PMULHWOpImpl(OpcodeArgs, bool Signed,
@@ -845,9 +790,6 @@ private:
OrderedNode* PMULLOpImpl(OpcodeArgs, size_t ElementSize, bool Signed,
OrderedNode *Src1, OrderedNode *Src2);
OrderedNode* PSHUFBOpImpl(OpcodeArgs, const X86Tables::DecodedOperand& Src1,
const X86Tables::DecodedOperand& Src2);
OrderedNode* PSIGNImpl(OpcodeArgs, size_t ElementSize,
OrderedNode *Src1, OrderedNode *Src2);
@@ -863,13 +805,6 @@ private:
OrderedNode* PSRLDOpImpl(OpcodeArgs, size_t ElementSize,
OrderedNode *Src, OrderedNode *ShiftVec);
OrderedNode* SHUFOpImpl(OpcodeArgs, size_t ElementSize,
const X86Tables::DecodedOperand& Src1,
const X86Tables::DecodedOperand& Src2,
const X86Tables::DecodedOperand& Imm);
void VMOVScalarOpImpl(OpcodeArgs, size_t ElementSize);
OrderedNode* VFCMPOpImpl(OpcodeArgs, size_t ElementSize, bool Scalar,
OrderedNode *Src1, OrderedNode *Src2, uint8_t CompType);
@@ -889,8 +824,6 @@ private:
#undef OpcodeArgs
OrderedNode *AppendSegmentOffset(OrderedNode *Value, uint32_t Flags, uint32_t DefaultPrefix = 0, bool Override = false);
OrderedNode *GetSegment(uint32_t Flags, uint32_t DefaultPrefix = 0, bool Override = false);
void UpdatePrefixFromSegment(OrderedNode *Segment, uint32_t SegmentReg);
enum class MemoryAccessType {
@@ -280,7 +280,7 @@ void OpDispatchBuilder::VAESIMCOp(OpcodeArgs) {
void OpDispatchBuilder::AESEncOp(OpcodeArgs) {
OrderedNode *Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags, -1);
OrderedNode *Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1);
OrderedNode *Result = _VAESEnc(16, Dest, Src);
OrderedNode *Result = _VAESEnc(Dest, Src);
StoreResult(FPRClass, Op, Result, -1);
}
@@ -293,7 +293,7 @@ void OpDispatchBuilder::VAESEncOp(OpcodeArgs) {
OrderedNode *State = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1);
OrderedNode *Key = LoadSource(FPRClass, Op, Op->Src[1], Op->Flags, -1);
OrderedNode *Result = _VAESEnc(DstSize, State, Key);
OrderedNode *Result = _VAESEnc(State, Key);
if (Is128Bit) {
Result = _VMov(16, Result);
@@ -304,7 +304,7 @@ void OpDispatchBuilder::VAESEncOp(OpcodeArgs) {
void OpDispatchBuilder::AESEncLastOp(OpcodeArgs) {
OrderedNode *Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags, -1);
OrderedNode *Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1);
OrderedNode *Result = _VAESEncLast(16, Dest, Src);
OrderedNode *Result = _VAESEncLast(Dest, Src);
StoreResult(FPRClass, Op, Result, -1);
}
@@ -317,7 +317,7 @@ void OpDispatchBuilder::VAESEncLastOp(OpcodeArgs) {
OrderedNode *State = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1);
OrderedNode *Key = LoadSource(FPRClass, Op, Op->Src[1], Op->Flags, -1);
OrderedNode *Result = _VAESEncLast(DstSize, State, Key);
OrderedNode *Result = _VAESEncLast(State, Key);
if (Is128Bit) {
Result = _VMov(16, Result);
@@ -328,7 +328,7 @@ void OpDispatchBuilder::VAESEncLastOp(OpcodeArgs) {
void OpDispatchBuilder::AESDecOp(OpcodeArgs) {
OrderedNode *Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags, -1);
OrderedNode *Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1);
OrderedNode *Result = _VAESDec(16, Dest, Src);
OrderedNode *Result = _VAESDec(Dest, Src);
StoreResult(FPRClass, Op, Result, -1);
}
@@ -341,7 +341,7 @@ void OpDispatchBuilder::VAESDecOp(OpcodeArgs) {
OrderedNode *State = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1);
OrderedNode *Key = LoadSource(FPRClass, Op, Op->Src[1], Op->Flags, -1);
OrderedNode *Result = _VAESDec(DstSize, State, Key);
OrderedNode *Result = _VAESDec(State, Key);
if (Is128Bit) {
Result = _VMov(16, Result);
@@ -352,7 +352,7 @@ void OpDispatchBuilder::VAESDecOp(OpcodeArgs) {
void OpDispatchBuilder::AESDecLastOp(OpcodeArgs) {
OrderedNode *Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags, -1);
OrderedNode *Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1);
OrderedNode *Result = _VAESDecLast(16, Dest, Src);
OrderedNode *Result = _VAESDecLast(Dest, Src);
StoreResult(FPRClass, Op, Result, -1);
}
@@ -365,7 +365,7 @@ void OpDispatchBuilder::VAESDecLastOp(OpcodeArgs) {
OrderedNode *State = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1);
OrderedNode *Key = LoadSource(FPRClass, Op, Op->Src[1], Op->Flags, -1);
OrderedNode *Result = _VAESDecLast(DstSize, State, Key);
OrderedNode *Result = _VAESDecLast(State, Key);
if (Is128Bit) {
Result = _VMov(16, Result);
@@ -399,7 +399,7 @@ void OpDispatchBuilder::PCLMULQDQOp(OpcodeArgs) {
OrderedNode *Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1);
const auto Selector = static_cast<uint8_t>(Op->Src[1].Data.Literal.Value);
auto Res = _PCLMUL(16, Dest, Src, Selector);
auto Res = _PCLMUL(Dest, Src, Selector);
StoreResult(FPRClass, Op, Res, -1);
}
@@ -413,7 +413,7 @@ void OpDispatchBuilder::VPCLMULQDQOp(OpcodeArgs) {
OrderedNode *Src2 = LoadSource(FPRClass, Op, Op->Src[1], Op->Flags, -1);
const auto Selector = static_cast<uint8_t>(Op->Src[2].Data.Literal.Value);
OrderedNode *Res = _PCLMUL(DstSize, Src1, Src2, Selector);
OrderedNode *Res = _PCLMUL(Src1, Src2, Selector);
if (Is128Bit) {
Res = _VMov(16, Res);
}
@@ -271,8 +271,10 @@ void OpDispatchBuilder::CalculcateFlags_ADC(uint8_t SrcSize, OrderedNode *Res, O
// SF
{
auto SignOp = _Bfe(1, SrcSize * 8 - 1, Res);
SetRFLAG<FEXCore::X86State::RFLAG_SF_LOC>(SignOp);
auto SignBitConst = _Constant(Size - 1);
auto LshrOp = _Lshr(Res, SignBitConst);
SetRFLAG<FEXCore::X86State::RFLAG_SF_LOC>(LshrOp);
}
// PF
@@ -340,8 +342,10 @@ void OpDispatchBuilder::CalculcateFlags_SBB(uint8_t SrcSize, OrderedNode *Res, O
// SF
{
auto SignOp = _Bfe(1, SrcSize * 8 - 1, Res);
SetRFLAG<FEXCore::X86State::RFLAG_SF_LOC>(SignOp);
auto SignBitConst = _Constant(SrcSize * 8 - 1);
auto LshrOp = _Lshr(Res, SignBitConst);
SetRFLAG<FEXCore::X86State::RFLAG_SF_LOC>(LshrOp);
}
// PF
@@ -408,8 +412,10 @@ void OpDispatchBuilder::CalculcateFlags_SUB(uint8_t SrcSize, OrderedNode *Res, O
// SF
{
auto SignOp = _Bfe(1, SrcSize * 8 - 1, Res);
SetRFLAG<FEXCore::X86State::RFLAG_SF_LOC>(SignOp);
auto SignBitConst = _Constant(SrcSize * 8 - 1);
auto LshrOp = _Lshr(Res, SignBitConst);
SetRFLAG<FEXCore::X86State::RFLAG_SF_LOC>(LshrOp);
}
// PF
@@ -463,8 +469,10 @@ void OpDispatchBuilder::CalculcateFlags_ADD(uint8_t SrcSize, OrderedNode *Res, O
// SF
{
auto SignOp = _Bfe(1, SrcSize * 8 - 1, Res);
SetRFLAG<FEXCore::X86State::RFLAG_SF_LOC>(SignOp);
auto SignBitConst = _Constant(SrcSize * 8 - 1);
auto LshrOp = _Lshr(Res, SignBitConst);
SetRFLAG<FEXCore::X86State::RFLAG_SF_LOC>(LshrOp);
}
// PF
@@ -575,8 +583,10 @@ void OpDispatchBuilder::CalculcateFlags_Logical(uint8_t SrcSize, OrderedNode *Re
// SF
{
auto SignOp = _Bfe(1, SrcSize * 8 - 1, Res);
SetRFLAG<FEXCore::X86State::RFLAG_SF_LOC>(SignOp);
auto SignBitConst = _Constant(SrcSize * 8 - 1);
auto LshrOp = _Lshr(Res, SignBitConst);
SetRFLAG<FEXCore::X86State::RFLAG_SF_LOC>(LshrOp);
}
// PF
@@ -740,8 +750,10 @@ void OpDispatchBuilder::CalculcateFlags_SignShiftRight(uint8_t SrcSize, OrderedN
// SF
{
auto SignBitOp = _Bfe(1, SrcSize * 8 - 1, Res);
COND_FLAG_SET(Src2, RFLAG_SF_LOC, SignBitOp);
auto SignBitConst = _Constant(SrcSize * 8 - 1);
auto LshrOp = _Lshr(Res, SignBitConst);
COND_FLAG_SET(Src2, RFLAG_SF_LOC, LshrOp);
}
// OF
@@ -790,14 +802,15 @@ void OpDispatchBuilder::CalculcateFlags_ShiftLeftImmediate(uint8_t SrcSize, Orde
// SF
{
auto SignOp = _Bfe(1, SrcSize * 8 - 1, Res);
SetRFLAG<FEXCore::X86State::RFLAG_SF_LOC>(SignOp);
auto LshrOp = _Bfe(1, SrcSize * 8 - 1, Res);
SetRFLAG<FEXCore::X86State::RFLAG_SF_LOC>(LshrOp);
// OF
// In the case of left shift. OF is only set from the result of <Top Source Bit> XOR <Top Result Bit>
if (Shift == 1) {
auto SourceBit = _Bfe(1, SrcSize * 8 - 1, Src1);
SetRFLAG<FEXCore::X86State::RFLAG_OF_LOC>(_Xor(SourceBit, SignOp));
SetRFLAG<FEXCore::X86State::RFLAG_OF_LOC>(_Xor(SourceBit, LshrOp));
}
}
}
@@ -838,8 +851,10 @@ void OpDispatchBuilder::CalculcateFlags_SignShiftRightImmediate(uint8_t SrcSize,
// SF
{
auto SignBitOp = _Bfe(1, SrcSize * 8 - 1, Res);
SetRFLAG<FEXCore::X86State::RFLAG_SF_LOC>(SignBitOp);
auto SignBitConst = _Constant(SrcSize * 8 - 1);
auto LshrOp = _Lshr(Res, SignBitConst);
SetRFLAG<FEXCore::X86State::RFLAG_SF_LOC>(LshrOp);
// OF
// Only defined when Shift is 1 else undefined
@@ -887,8 +902,10 @@ void OpDispatchBuilder::CalculcateFlags_ShiftRightImmediate(uint8_t SrcSize, Ord
// SF
{
auto SignBitOp = _Bfe(1, SrcSize * 8 - 1, Res);
SetRFLAG<FEXCore::X86State::RFLAG_SF_LOC>(SignBitOp);
auto SignBitConst = _Constant(SrcSize * 8 - 1);
auto LshrOp = _Lshr(Res, SignBitConst);
SetRFLAG<FEXCore::X86State::RFLAG_SF_LOC>(LshrOp);
}
// OF
@@ -1098,8 +1115,10 @@ void OpDispatchBuilder::CalculcateFlags_BLSI(uint8_t SrcSize, OrderedNode *Src)
// SF
{
auto SignOp = _Bfe(1, SrcSize * 8 - 1, Src);
SetRFLAG<X86State::RFLAG_SF_LOC>(SignOp);
auto SignBit = _Constant(SrcSize * 8 - 1);
auto SFOp = _Lshr(Src, SignBit);
SetRFLAG<X86State::RFLAG_SF_LOC>(SFOp);
}
}
@@ -1155,8 +1174,10 @@ void OpDispatchBuilder::CalculcateFlags_BLSR(uint8_t SrcSize, OrderedNode *Resul
// SF
{
auto SignOp = _Bfe(1, SrcSize * 8 - 1, Result);
SetRFLAG<X86State::RFLAG_SF_LOC>(SignOp);
auto SignBit = _Constant(SrcSize * 8 - 1);
auto SFOp = _Lshr(Result, SignBit);
SetRFLAG<X86State::RFLAG_SF_LOC>(SFOp);
}
}
@@ -1209,8 +1230,9 @@ void OpDispatchBuilder::CalculcateFlags_BZHI(uint8_t SrcSize, OrderedNode *Resul
// SF
{
auto SignOp = _Bfe(1, SrcSize * 8 - 1, Result);
SetRFLAG<X86State::RFLAG_SF_LOC>(SignOp);
auto SFOp = _Lshr(Result, Bounds);
SetRFLAG<X86State::RFLAG_SF_LOC>(SFOp);
}
}
File diff suppressed because it is too large. Load diff
@@ -1388,7 +1388,7 @@ void OpDispatchBuilder::X87FCMOV(OpcodeArgs) {
auto a = _LoadContextIndexed(top, 16, MMBaseOffset(), 16, FPRClass);
auto b = _LoadContextIndexed(arg, 16, MMBaseOffset(), 16, FPRClass);
auto Result = _VBSL(16, VecCond, b, a);
auto Result = _VBSL(VecCond, b, a);
// Write to ST[TOP]
_StoreContextIndexed(Result, top, 16, MMBaseOffset(), 16, FPRClass);
@@ -39,7 +39,7 @@ class OrderedNode;
//FST(register to register)
// State loading duplicated from X87.cpp, setting host rounding mode
// See issue
// See issue
void OpDispatchBuilder::FNINITF64(OpcodeArgs) {
// Init FCW to 0x037F
auto NewFCW = _Constant(16, 0x037F);
@@ -76,7 +76,7 @@ void OpDispatchBuilder::X87LDENVF64(OpcodeArgs) {
roundingMode = _And(roundingMode, roundMask);
_SetRoundingMode(roundingMode);
_F80LoadFCW(NewFCW);
_StoreContext(2, GPRClass, NewFCW, offsetof(FEXCore::Core::CPUState, FCW));
OrderedNode *MemLocation = _Add(Mem, _Constant(Size * 1));
@@ -184,7 +184,7 @@ void OpDispatchBuilder::FBLDF64(OpcodeArgs) {
void OpDispatchBuilder::FBSTPF64(OpcodeArgs) {
auto orig_top = GetX87Top();
auto data = _LoadContextIndexed(orig_top, 8, MMBaseOffset(), 16, FPRClass);
OrderedNode *converted = _F80CVTTo(data, 8);
converted = _F80BCDStore(converted);
@@ -256,7 +256,7 @@ void OpDispatchBuilder::FSTF64(OpcodeArgs) {
//Convert to 80-bit float
auto result = _F80CVTTo(data, 8);
StoreResult_WithOpSize(FPRClass, Op, Op->Dest, result, 10, 1);
}
}
if ((Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) != 0) {
// if we are popping then we must first mark this location as empty
@@ -315,10 +315,7 @@ void OpDispatchBuilder::FADDF64(OpcodeArgs) {
// Memory arg
if constexpr (Integer) {
arg = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, -1);
if(width == 16) {
arg = _Sext(16, arg);
}
b = _Float_FromGPR_S(8, width == 64 ? 8 : 4, arg);
b = _Float_FromGPR_S(8, 8, arg);
} else if constexpr (width == 32) {
arg = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1);
b = _Float_FToF(8, 4, arg);
@@ -376,10 +373,7 @@ void OpDispatchBuilder::FMULF64(OpcodeArgs) {
// Memory arg
if constexpr (Integer) {
arg = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, -1);
if(width == 16) {
arg = _Sext(16, arg);
}
b = _Float_FromGPR_S(8, width == 64 ? 8 : 4, arg);
b = _Float_FromGPR_S(8, 8, arg);
} else if constexpr (width == 32) {
arg = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1);
b = _Float_FToF(8, 4, arg);
@@ -440,10 +434,7 @@ void OpDispatchBuilder::FDIVF64(OpcodeArgs) {
// Memory arg
if constexpr (Integer) {
arg = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, -1);
if(width == 16) {
arg = _Sext(16, arg);
}
b = _Float_FromGPR_S(8, width == 64 ? 8 : 4, arg);
b = _Float_FromGPR_S(8, 8, arg);
} else if constexpr (width == 32) {
arg = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1);
b = _Float_FToF(8, 4, arg);
@@ -526,10 +517,7 @@ void OpDispatchBuilder::FSUBF64(OpcodeArgs) {
// Memory arg
if constexpr (Integer) {
arg = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, -1);
if(width == 16) {
arg = _Sext(16, arg);
}
b = _Float_FromGPR_S(8, width == 64 ? 8 : 4, arg);
b = _Float_FromGPR_S(8, 8, arg);
} else if constexpr (width == 32) {
arg = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1);
b = _Float_FToF(8, 4, arg);
@@ -688,10 +676,7 @@ void OpDispatchBuilder::FCOMIF64(OpcodeArgs) {
// Memory arg
if constexpr (Integer) {
arg = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, -1);
if(width == 16) {
arg = _Sext(16, arg);
}
b = _Float_FromGPR_S(8, width == 64 ? 8 : 4, arg);
b = _Float_FromGPR_S(8, 8, arg);
} else if constexpr (width == 32) {
arg = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1);
b = _Float_FToF(8, 4, arg);
@@ -715,7 +700,7 @@ void OpDispatchBuilder::FCOMIF64(OpcodeArgs) {
OrderedNode *HostFlag_CF = _GetHostFlag(Res, FCMP_FLAG_LT);
OrderedNode *HostFlag_ZF = _GetHostFlag(Res, FCMP_FLAG_EQ);
OrderedNode *HostFlag_Unordered = _GetHostFlag(Res, FCMP_FLAG_UNORDERED);
HostFlag_CF = _Or(HostFlag_CF, HostFlag_Unordered);
HostFlag_ZF = _Or(HostFlag_ZF, HostFlag_Unordered);
@@ -825,8 +810,8 @@ void OpDispatchBuilder::X87BinaryOpF64(OpcodeArgs) {
// Overwrite the op
result.first->Header.Op = IROp;
if constexpr (IROp == IR::OP_F64FPREM ||
IROp == IR::OP_F64FPREM1) {
if constexpr (IROp == IR::OP_F80FPREM ||
IROp == IR::OP_F80FPREM1) {
//TODO: Set C0 to Q2, C3 to Q1, C1 to Q0
SetRFLAG<FEXCore::X86State::X87FLAG_C2_LOC>(_Constant(0));
}
+6 -29
View File
@@ -23,38 +23,15 @@ X86GeneratedCode::X86GeneratedCode() {
// Allocate a page for our emulated guest
CodePtr = AllocateGuestCodeSpace(CODE_SIZE);
constexpr std::array<uint8_t, 2> SignalReturnCode = {
SignalReturn = reinterpret_cast<uint64_t>(CodePtr);
CallbackReturn = reinterpret_cast<uint64_t>(CodePtr) + 2;
const std::vector<uint8_t> SignalReturnCode = {
0x0F, 0x36, // SIGRET FEX instruction
0x0F, 0x37, // CALLBACKRET FEX Instruction
};
// Signal return handlers need to be bit-exact to what the Linux kernel provides in VDSO.
// GDB and unwinding libraries key off of these instructions to understand if the stack frame is a signal frame or not.
// This two code sections match exactly what libSegFault expects.
//
// Typically this handlers are provided by the 32-bit VDSO thunk library, but that isn't available in all cases.
// Falling back to this generated code segment still allows a backtrace to work, just might not show
// the symbol as VDSO since there is no ELF to parse.
constexpr std::array<uint8_t, 9> sigreturn_32_code = {
0x58, // pop eax
0xb8, 0x77, 0x00, 0x00, 0x00, // mov eax, 0x77
0xcd, 0x80, // int 0x80
0x90, // nop
};
constexpr std::array<uint8_t, 7> rt_sigreturn_32_code = {
0xb8, 0xad, 0x00, 0x00, 0x00, // mov eax, 0xad
0xcd, 0x80, // int 0x80
};
CallbackReturn = reinterpret_cast<uint64_t>(CodePtr);
sigreturn_32 = CallbackReturn + SignalReturnCode.size();
rt_sigreturn_32 = sigreturn_32 + sigreturn_32_code.size();
memcpy(reinterpret_cast<void*>(CallbackReturn), &SignalReturnCode.at(0), SignalReturnCode.size());
memcpy(reinterpret_cast<void*>(sigreturn_32), &sigreturn_32_code.at(0), sigreturn_32_code.size());
memcpy(reinterpret_cast<void*>(rt_sigreturn_32), &rt_sigreturn_32_code.at(0), rt_sigreturn_32_code.size());
mprotect(CodePtr, CODE_SIZE, PROT_READ);
memcpy(CodePtr, &SignalReturnCode.at(0), SignalReturnCode.size());
}
X86GeneratedCode::~X86GeneratedCode() {
+1 -2
View File
@@ -15,9 +15,8 @@ public:
X86GeneratedCode();
~X86GeneratedCode();
uint64_t SignalReturn{};
uint64_t CallbackReturn{};
uint64_t sigreturn_32{};
uint64_t rt_sigreturn_32{};
private:
void *CodePtr{};
@@ -338,7 +338,7 @@ void InitializeSecondaryGroupTables() {
{OPD(TYPE_GROUP_15, PF_NONE, 3), 1, X86InstInfo{"STMXCSR", TYPE_INST, GenFlagsSameSize(SIZE_32BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_MEM_ONLY, 0, nullptr}},
{OPD(TYPE_GROUP_15, PF_NONE, 4), 1, X86InstInfo{"XSAVE", TYPE_PRIV, FLAGS_NONE, 0, nullptr}},
{OPD(TYPE_GROUP_15, PF_NONE, 5), 1, X86InstInfo{"LFENCE/XRSTOR", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0, nullptr}},
{OPD(TYPE_GROUP_15, PF_NONE, 6), 1, X86InstInfo{"MFENCE/XSAVEOPT", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0, nullptr}},
{OPD(TYPE_GROUP_15, PF_NONE, 6), 1, X86InstInfo{"MFENCE/XSAVEOPT", TYPE_INST, FLAGS_MODRM, 0, nullptr}},
{OPD(TYPE_GROUP_15, PF_NONE, 7), 1, X86InstInfo{"SFENCE/CLFLUSH", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0, nullptr}},
{OPD(TYPE_GROUP_15, PF_F3, 0), 1, X86InstInfo{"RDFSBASE", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_REG_ONLY, 0, nullptr}},
@@ -356,8 +356,8 @@ void InitializeSecondaryGroupTables() {
{OPD(TYPE_GROUP_15, PF_66, 3), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE, 0, nullptr}},
{OPD(TYPE_GROUP_15, PF_66, 4), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE, 0, nullptr}},
{OPD(TYPE_GROUP_15, PF_66, 5), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE, 0, nullptr}},
{OPD(TYPE_GROUP_15, PF_66, 6), 1, X86InstInfo{"CLWB", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0, nullptr}},
{OPD(TYPE_GROUP_15, PF_66, 7), 1, X86InstInfo{"CLFLUSHOPT", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0, nullptr}},
{OPD(TYPE_GROUP_15, PF_66, 6), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE, 0, nullptr}},
{OPD(TYPE_GROUP_15, PF_66, 7), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE, 0, nullptr}},
{OPD(TYPE_GROUP_15, PF_F2, 0), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE, 0, nullptr}},
{OPD(TYPE_GROUP_15, PF_F2, 1), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE, 0, nullptr}},
@@ -42,7 +42,7 @@ void InitializeSecondaryTables(Context::OperatingMode Mode) {
{0x14, 1, X86InstInfo{"UNPCKLPS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{0x15, 1, X86InstInfo{"UNPCKHPS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{0x16, 1, X86InstInfo{"MOVLHPS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{0x17, 1, X86InstInfo{"MOVHPS", TYPE_INST, GenFlagsSizes(SIZE_64BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_MEM_ONLY | FLAGS_XMM_FLAGS, 0, nullptr}},
{0x17, 1, X86InstInfo{"MOVHPS", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_SF_HIGH_XMM_REG | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_MEM_ONLY | FLAGS_XMM_FLAGS, 0, nullptr}},
{0x18, 1, X86InstInfo{"", TYPE_GROUP_16, FLAGS_NO_OVERLAY, 0, nullptr}},
{0x19, 7, X86InstInfo{"NOP", TYPE_INST, FLAGS_DEBUG | FLAGS_MODRM | FLAGS_NO_OVERLAY, 0, nullptr}},
@@ -64,7 +64,6 @@ void InitializeSecondaryTables(Context::OperatingMode Mode) {
{0x33, 1, X86InstInfo{"RDPMC", TYPE_PRIV, FLAGS_NO_OVERLAY, 0, nullptr}},
{0x34, 1, X86InstInfo{"SYSENTER", TYPE_PRIV, FLAGS_NO_OVERLAY, 0, nullptr}},
{0x35, 1, X86InstInfo{"SYSEXIT", TYPE_PRIV, FLAGS_NO_OVERLAY, 0, nullptr}},
{0x36, 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NO_OVERLAY, 0, nullptr}},
{0x38, 1, X86InstInfo{"", TYPE_0F38_TABLE, FLAGS_NO_OVERLAY, 0, nullptr}},
{0x39, 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NO_OVERLAY, 0, nullptr}},
{0x3A, 1, X86InstInfo{"", TYPE_0F3A_TABLE, FLAGS_NO_OVERLAY, 0, nullptr}},
@@ -258,6 +257,8 @@ void InitializeSecondaryTables(Context::OperatingMode Mode) {
// FEX reserved instructions
// Unused x86 encoding instruction.
// Used by FEX to know when to do a signal return
{0x36, 1, X86InstInfo{"SIGRET", TYPE_INST, FLAGS_BLOCK_END | FLAGS_NO_OVERLAY | FLAGS_SETS_RIP, 0, nullptr}},
{0x37, 1, X86InstInfo{"CALLBACKRET", TYPE_INST, FLAGS_BLOCK_END | FLAGS_NO_OVERLAY | FLAGS_SETS_RIP, 0, nullptr}},
@@ -19,13 +19,13 @@ void InitializeVEXTables() {
// VEX Map 1
{OPD(1, 0b00, 0x10), 1, X86InstInfo{"VMOVUPS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(1, 0b01, 0x10), 1, X86InstInfo{"VMOVUPD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(1, 0b10, 0x10), 1, X86InstInfo{"VMOVSS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(1, 0b11, 0x10), 1, X86InstInfo{"VMOVSD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(1, 0b10, 0x10), 1, X86InstInfo{"VMOVSS", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(1, 0b11, 0x10), 1, X86InstInfo{"VMOVSD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(1, 0b00, 0x11), 1, X86InstInfo{"VMOVUPS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(1, 0b01, 0x11), 1, X86InstInfo{"VMOVUPD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(1, 0b10, 0x11), 1, X86InstInfo{"VMOVSS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(1, 0b11, 0x11), 1, X86InstInfo{"VMOVSD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(1, 0b10, 0x11), 1, X86InstInfo{"VMOVSS", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(1, 0b11, 0x11), 1, X86InstInfo{"VMOVSD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(1, 0b00, 0x12), 1, X86InstInfo{"VMOVLPS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_XMM_FLAGS | FLAGS_VEX_1ST_SRC, 0, nullptr}},
{OPD(1, 0b01, 0x12), 1, X86InstInfo{"VMOVLPD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_XMM_FLAGS | FLAGS_VEX_1ST_SRC, 0, nullptr}},
@@ -83,9 +83,9 @@ void InitializeVEXTables() {
{OPD(1, 0b01, 0x66), 1, X86InstInfo{"VPCMPGTD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(1, 0b01, 0x67), 1, X86InstInfo{"VPACKUSWB", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(1, 0b01, 0x70), 1, X86InstInfo{"VPSHUFD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(1, 0b10, 0x70), 1, X86InstInfo{"VPSHUFHW", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(1, 0b11, 0x70), 1, X86InstInfo{"VPSHUFLW", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(1, 0b01, 0x70), 1, X86InstInfo{"VPSHUFD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(1, 0b10, 0x70), 1, X86InstInfo{"VPSHUFHW", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(1, 0b11, 0x70), 1, X86InstInfo{"VPSHUFLW", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(1, 0b01, 0x71), 1, X86InstInfo{"", TYPE_VEX_GROUP_12, FLAGS_NONE, 0, nullptr}}, // VEX Group 12
{OPD(1, 0b01, 0x72), 1, X86InstInfo{"", TYPE_VEX_GROUP_13, FLAGS_NONE, 0, nullptr}}, // VEX Group 13
@@ -105,8 +105,8 @@ void InitializeVEXTables() {
{OPD(1, 0b01, 0xC4), 1, X86InstInfo{"VPINSRW", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(1, 0b01, 0xC5), 1, X86InstInfo{"VPEXTRW", TYPE_INST, GenFlagsSizes(SIZE_32BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_SF_DST_GPR | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(1, 0b00, 0xC6), 1, X86InstInfo{"VSHUFPS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(1, 0b01, 0xC6), 1, X86InstInfo{"VSHUFPD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(1, 0b00, 0xC6), 1, X86InstInfo{"VSHUFPS", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(1, 0b01, 0xC6), 1, X86InstInfo{"VSHUFPD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
// The above ops are defined from `Table A-17. VEX Opcode Map 1, Low Nibble = [0h:7h]` of AMD Architecture programmer's manual Volume 3
// This table doesn't state which VEX.pp is for which instruction
@@ -184,8 +184,8 @@ void InitializeVEXTables() {
{OPD(1, 0b01, 0x7C), 1, X86InstInfo{"VHADDPD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(1, 0b11, 0x7C), 1, X86InstInfo{"VHADDPS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(1, 0b01, 0x7D), 1, X86InstInfo{"VHSUBPD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(1, 0b11, 0x7D), 1, X86InstInfo{"VHSUBPS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(1, 0b01, 0x7D), 1, X86InstInfo{"VHSUBPD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(1, 0b11, 0x7D), 1, X86InstInfo{"VHSUBPS", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(1, 0b01, 0x7E), 1, X86InstInfo{"VMOV*", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_DST_GPR | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(1, 0b10, 0x7E), 1, X86InstInfo{"VMOVQ", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
@@ -246,7 +246,7 @@ void InitializeVEXTables() {
{OPD(1, 0b01, 0xF2), 1, X86InstInfo{"VPSLLD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(1, 0b01, 0xF3), 1, X86InstInfo{"VPSLLQ", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(1, 0b01, 0xF4), 1, X86InstInfo{"VPMULUDQ", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(1, 0b01, 0xF5), 1, X86InstInfo{"VPMADDWD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(1, 0b01, 0xF5), 1, X86InstInfo{"VPMADDWD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(1, 0b01, 0xF6), 1, X86InstInfo{"VPSADBW", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(1, 0b01, 0xF7), 1, X86InstInfo{"VMASKMOVDQU", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_REG_ONLY | FLAGS_XMM_FLAGS, 0, nullptr}},
@@ -259,27 +259,27 @@ void InitializeVEXTables() {
{OPD(1, 0b01, 0xFE), 1, X86InstInfo{"VPADDD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
// VEX Map 2
{OPD(2, 0b01, 0x00), 1, X86InstInfo{"VPSHUFB", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(2, 0b01, 0x00), 1, X86InstInfo{"VPSHUFB", TYPE_UNDEC, FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(2, 0b01, 0x01), 1, X86InstInfo{"VPHADDW", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(2, 0b01, 0x02), 1, X86InstInfo{"VPHADDD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(2, 0b01, 0x03), 1, X86InstInfo{"VPHADDSW", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(2, 0b01, 0x03), 1, X86InstInfo{"VPHADDSW", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(2, 0b01, 0x04), 1, X86InstInfo{"VPMADDUBSW", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(2, 0b01, 0x05), 1, X86InstInfo{"VPHSUBW", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(2, 0b01, 0x06), 1, X86InstInfo{"VPHSUBD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(2, 0b01, 0x07), 1, X86InstInfo{"VPHSUBSW", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(2, 0b01, 0x07), 1, X86InstInfo{"VPHSUBSW", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(2, 0b01, 0x08), 1, X86InstInfo{"VPSIGNB", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(2, 0b01, 0x09), 1, X86InstInfo{"VPSIGNW", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(2, 0b01, 0x0A), 1, X86InstInfo{"VPSIGND", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(2, 0b01, 0x0B), 1, X86InstInfo{"VPMULHRSW", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(2, 0b01, 0x0C), 1, X86InstInfo{"VPERMILPS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(2, 0b01, 0x0D), 1, X86InstInfo{"VPERMILPD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(2, 0b01, 0x0C), 1, X86InstInfo{"VPERMILPS", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(2, 0b01, 0x0D), 1, X86InstInfo{"VPERMILPD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(2, 0b01, 0x0E), 1, X86InstInfo{"VTESTPS", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(2, 0b01, 0x0F), 1, X86InstInfo{"VTESTPD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(2, 0b01, 0x13), 1, X86InstInfo{"VCVTPH2PS", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(2, 0b01, 0x16), 1, X86InstInfo{"VPERMPS", TYPE_INST, GenFlagsSameSize(SIZE_256BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(2, 0b01, 0x17), 1, X86InstInfo{"VPTEST", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(2, 0b01, 0x16), 1, X86InstInfo{"VPERMPS", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(2, 0b01, 0x17), 1, X86InstInfo{"VPTEST", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(2, 0b01, 0x18), 1, X86InstInfo{"VBROADCASTSS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(2, 0b01, 0x19), 1, X86InstInfo{"VBROADCASTSD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
@@ -310,7 +310,7 @@ void InitializeVEXTables() {
{OPD(2, 0b01, 0x33), 1, X86InstInfo{"VPMOVZXWD", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(2, 0b01, 0x34), 1, X86InstInfo{"VPMOVZXWQ", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(2, 0b01, 0x35), 1, X86InstInfo{"VPMOVZXDQ", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(2, 0b01, 0x36), 1, X86InstInfo{"VPERMD", TYPE_INST, GenFlagsSameSize(SIZE_256BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(2, 0b01, 0x36), 1, X86InstInfo{"VPERMD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(2, 0b01, 0x37), 1, X86InstInfo{"VPCMPGTQ", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(2, 0b01, 0x38), 1, X86InstInfo{"VPMINSB", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
@@ -325,7 +325,7 @@ void InitializeVEXTables() {
{OPD(2, 0b01, 0x40), 1, X86InstInfo{"VPMULLD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(2, 0b01, 0x41), 1, X86InstInfo{"VPHMINPOSUW", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(2, 0b01, 0x45), 1, X86InstInfo{"VPSRLV", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(2, 0b01, 0x46), 1, X86InstInfo{"VPSRAVD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(2, 0b01, 0x46), 1, X86InstInfo{"VPSRAVD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(2, 0b01, 0x47), 1, X86InstInfo{"VPSLLV", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(2, 0b01, 0x58), 1, X86InstInfo{"VPBROADCASTD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
@@ -407,7 +407,7 @@ void InitializeVEXTables() {
// VEX Map 3
{OPD(3, 0b01, 0x00), 1, X86InstInfo{"VPERMQ", TYPE_INST, GenFlagsSameSize(SIZE_256BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(3, 0b01, 0x01), 1, X86InstInfo{"VPERMPD", TYPE_INST, GenFlagsSameSize(SIZE_256BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(3, 0b01, 0x02), 1, X86InstInfo{"VPBLENDD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(3, 0b01, 0x02), 1, X86InstInfo{"VPBLENDD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(3, 0b01, 0x04), 1, X86InstInfo{"VPERMILPS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(3, 0b01, 0x05), 1, X86InstInfo{"VPERMILPD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(3, 0b01, 0x06), 1, X86InstInfo{"VPERM2F128", TYPE_INST, GenFlagsSameSize(SIZE_256BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
@@ -416,10 +416,10 @@ void InitializeVEXTables() {
{OPD(3, 0b01, 0x09), 1, X86InstInfo{"VROUNDPD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(3, 0b01, 0x0A), 1, X86InstInfo{"VROUNDSS", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(3, 0b01, 0x0B), 1, X86InstInfo{"VROUNDSD", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(3, 0b01, 0x0C), 1, X86InstInfo{"VBLENDPS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(3, 0b01, 0x0D), 1, X86InstInfo{"VBLENDPD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(3, 0b01, 0x0E), 1, X86InstInfo{"VPBLENDW", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(3, 0b01, 0x0F), 1, X86InstInfo{"VPALIGNR", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(3, 0b01, 0x0C), 1, X86InstInfo{"VBLENDPS", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(3, 0b01, 0x0D), 1, X86InstInfo{"VBLENDPD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(3, 0b01, 0x0E), 1, X86InstInfo{"VBLENDW", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(3, 0b01, 0x0F), 1, X86InstInfo{"VPALIGNR", TYPE_INST, FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(3, 0b01, 0x14), 1, X86InstInfo{"VPEXTRB", TYPE_INST, GenFlagsSizes(SIZE_32BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_DST_GPR | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(3, 0b01, 0x15), 1, X86InstInfo{"VPEXTRW", TYPE_INST, GenFlagsSizes(SIZE_16BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_DST_GPR | FLAGS_XMM_FLAGS, 1, nullptr}},
@@ -427,7 +427,7 @@ void InitializeVEXTables() {
{OPD(3, 0b01, 0x17), 1, X86InstInfo{"VEXTRACTPS", TYPE_INST, GenFlagsSizes(SIZE_32BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_DST_GPR | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(3, 0b01, 0x18), 1, X86InstInfo{"VINSERTF128", TYPE_INST, GenFlagsSameSize(SIZE_256BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(3, 0b01, 0x19), 1, X86InstInfo{"VEXTRACTF128", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_256BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(3, 0b01, 0x19), 1, X86InstInfo{"VEXTRACTF128", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(3, 0b01, 0x1D), 1, X86InstInfo{"VCVTPS2PH", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(3, 0b01, 0x20), 1, X86InstInfo{"VPINSRB", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
@@ -435,7 +435,7 @@ void InitializeVEXTables() {
{OPD(3, 0b01, 0x22), 1, X86InstInfo{"VPINSRD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(3, 0b01, 0x38), 1, X86InstInfo{"VINSERTI128", TYPE_INST, GenFlagsSameSize(SIZE_256BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(3, 0b01, 0x39), 1, X86InstInfo{"VEXTRACTI128", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_256BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(3, 0b01, 0x39), 1, X86InstInfo{"VEXTRACTI128", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(3, 0b01, 0x40), 1, X86InstInfo{"VDPPS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(3, 0b01, 0x41), 1, X86InstInfo{"VDPPD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
@@ -443,9 +443,11 @@ void InitializeVEXTables() {
{OPD(3, 0b01, 0x44), 1, X86InstInfo{"VPCLMULQDQ", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(3, 0b01, 0x46), 1, X86InstInfo{"VPERM2I128", TYPE_INST, GenFlagsSameSize(SIZE_256BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(3, 0b01, 0x4A), 1, X86InstInfo{"VBLENDVPS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(3, 0b01, 0x4B), 1, X86InstInfo{"VBLENDVPD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(3, 0b01, 0x4C), 1, X86InstInfo{"VPBLENDVB", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(3, 0b01, 0x48), 1, X86InstInfo{"VPERMILzz2PS", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(3, 0b01, 0x49), 1, X86InstInfo{"VPERMILzz2PD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(3, 0b01, 0x4A), 1, X86InstInfo{"VBLENDVPS", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(3, 0b01, 0x4B), 1, X86InstInfo{"VBLENDVPD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(3, 0b01, 0x4C), 1, X86InstInfo{"VBLENDVB", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(3, 0b01, 0x5C), 1, X86InstInfo{"VFMADDSUBPS", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(3, 0b01, 0x5D), 1, X86InstInfo{"VFMADDSUBPD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
+4 -4
View File
@@ -179,7 +179,7 @@ namespace FEXCore {
};
auto args = reinterpret_cast<args_t*>(argsv);
auto CTX = static_cast<Context::ContextImpl*>(Thread->CTX);
auto CTX = Thread->CTX;
LOGMAN_THROW_AA_FMT(args->original_callee, "Tried to link null pointer address to guest function");
LOGMAN_THROW_AA_FMT(args->target_addr, "Tried to link address to null pointer guest function");
@@ -264,7 +264,7 @@ namespace FEXCore {
}
static void LoadLib(void *ArgsV) {
auto CTX = static_cast<Context::ContextImpl*>(Thread->CTX);
auto CTX = Thread->CTX;
auto Args = reinterpret_cast<LoadlibArgs*>(ArgsV);
@@ -321,7 +321,7 @@ namespace FEXCore {
auto &[Name, rv] = *reinterpret_cast<ArgsRV_t*>(ArgsRV);
auto CTX = static_cast<Context::ContextImpl*>(Thread->CTX);
auto CTX = Thread->CTX;
auto That = reinterpret_cast<ThunkHandler_impl*>(CTX->ThunkHandler.get());
{
@@ -385,7 +385,7 @@ namespace FEXCore {
HostToGuestTrampolinePtr* MakeHostTrampolineForGuestFunction(void* HostPacker, uintptr_t GuestTarget, uintptr_t GuestUnpacker) {
LOGMAN_THROW_AA_FMT(GuestTarget, "Tried to create host-trampoline to null pointer guest function");
const auto CTX = static_cast<Context::ContextImpl*>(Thread->CTX);
const auto CTX = Thread->CTX;
const auto ThunkHandler = reinterpret_cast<ThunkHandler_impl *>(CTX->ThunkHandler.get());
const GuestcallInfo gci = { GuestUnpacker, GuestTarget };
+1 -1
View File
@@ -11,7 +11,7 @@ $end_info$
#include <vector>
namespace FEXCore::Context {
class ContextImpl;
struct Context;
}
namespace FEXCore::Core {
+2 -5
View File
@@ -17,9 +17,6 @@
namespace FEXCore::Core {
struct DebugData;
}
namespace FEXCore::Context {
class ContextImpl;
}
namespace FEXCore::IR {
class RegisterAllocationData;
@@ -90,7 +87,7 @@ namespace FEXCore::IR {
class AOTIRCaptureCache final {
public:
AOTIRCaptureCache(FEXCore::Context::ContextImpl *ctx) : CTX {ctx} {}
AOTIRCaptureCache(FEXCore::Context::Context *ctx) : CTX {ctx} {}
void FinalizeAOTIRCache();
void AOTIRCaptureCacheWriteoutQueue_Flush();
@@ -134,7 +131,7 @@ namespace FEXCore::IR {
}
private:
FEXCore::Context::ContextImpl *CTX;
FEXCore::Context::Context *CTX;
std::shared_mutex AOTIRCacheLock;
std::shared_mutex AOTIRCaptureCacheWriteoutLock;
+18 -45
View File
@@ -22,7 +22,7 @@
"",
"Eg:",
"IR op with no result and no arguments",
" CallbackReturn",
" SignalReturn",
"",
"IR op with result and no arguments",
" GPR = ProcessorID",
@@ -264,6 +264,9 @@
"Break BreakDefinition:$Reason": {
"HasSideEffects": true
},
"SignalReturn": {
"HasSideEffects": true
},
"CallbackReturn": {
"HasSideEffects": true
},
@@ -476,25 +479,9 @@
]
},
"GPR = MemSet i1:$IsAtomic, u8:$Size, GPR:$Prefix, GPR:$Addr, GPR:$Value, GPR:$Length, GPR:$Direction": {
"Desc": ["Duplicates behaviour of x86 STOS repeat",
"Returns the final address that gets generated without the prefix appended."
],
"HasSideEffects": true,
"DestSize": "8"
},
"CacheLineClear GPR:$Addr, i1:$Serialize": {
"CacheLineClear GPR:$Addr": {
"Desc": ["Does a 64 byte cacheline clear at the address specified",
"Only clears the data cachelines. Doesn't do any zeroing",
"Can skip serialization if requested."
],
"HasSideEffects": true
},
"CacheLineClean GPR:$Addr": {
"Desc": ["Does a 64 byte cacheline cleanat the address specified",
"Only cleans the data cachelines. Doesn't do any zeroing",
"Skips the invalidation step of the CacheLineClear operation"
"Only clears the data cachelines. Doesn't do any zeroing"
],
"HasSideEffects": true
},
@@ -1196,14 +1183,6 @@
"DestSize": "RegisterSize",
"NumElements": "RegisterSize / ElementSize"
},
"FPR = VTrn u8:#RegisterSize, u8:#ElementSize, FPR:$VectorLower, FPR:$VectorUpper": {
"DestSize": "RegisterSize",
"NumElements": "RegisterSize / ElementSize"
},
"FPR = VTrn2 u8:#RegisterSize, u8:#ElementSize, FPR:$VectorLower, FPR:$VectorUpper": {
"DestSize": "RegisterSize",
"NumElements": "RegisterSize / ElementSize"
},
"FPR = VFAdd u8:#RegisterSize, u8:#ElementSize, FPR:$Vector1, FPR:$Vector2": {
"DestSize": "RegisterSize",
@@ -1366,12 +1345,12 @@
"DestSize": "RegisterSize"
},
"FPR = VBSL u8:#RegisterSize, FPR:$VectorMask, FPR:$VectorTrue, FPR:$VectorFalse": {
"FPR = VBSL FPR:$VectorMask, FPR:$VectorTrue, FPR:$VectorFalse": {
"Desc": ["Does a vector bitwise select.",
"If the bit in the field is 1 then the corresponding bit is pulled from VectorTrue",
"If the bit in the field is 0 then the corresponding bit is pulled from VectorFalse"
],
"DestSize": "RegisterSize"
"DestSize": "16"
}
},
"Conv": {
@@ -1383,12 +1362,6 @@
"NumElements": "RegisterSize / ElementSize"
},
"FPR = VDupFromGPR u8:#RegisterSize, u8:#ElementSize, GPR:$Src": {
"Desc": ["Broadcasts a value in a GPR into each ElementSize-sized element in a vector"],
"DestSize": "RegisterSize",
"NumElements": "RegisterSize / ElementSize"
},
"FPR = Float_FromGPR_S u8:#DstElementSize, u8:$SrcElementSize, GPR:$Src": {
"Desc": ["Scalar op: Converts signed GPR to Scalar float",
"Zeroes the upper bits of the vector register"
@@ -1437,21 +1410,21 @@
"Desc": "Does a stage of the inverse mix column transformation",
"DestSize": "16"
},
"FPR = VAESEnc u8:#RegisterSize, FPR:$State, FPR:$Key": {
"FPR = VAESEnc FPR:$State, FPR:$Key": {
"Desc": "Does a step of AES encryption",
"DestSize": "RegisterSize"
"DestSize": "16"
},
"FPR = VAESEncLast u8:#RegisterSize, FPR:$State, FPR:$Key": {
"FPR = VAESEncLast FPR:$State, FPR:$Key": {
"Desc": "Does the last step of AES encryption",
"DestSize": "RegisterSize"
"DestSize": "16"
},
"FPR = VAESDec u8:#RegisterSize, FPR:$State, FPR:$Key": {
"FPR = VAESDec FPR:$State, FPR:$Key": {
"Desc": "Does a step of AES decryption",
"DestSize": "RegisterSize"
"DestSize": "16"
},
"FPR = VAESDecLast u8:#RegisterSize, FPR:$State, FPR:$Key": {
"FPR = VAESDecLast FPR:$State, FPR:$Key": {
"Desc": "Does the last step of AES decryption",
"DestSize": "RegisterSize"
"DestSize": "16"
},
"FPR = VAESKeyGenAssist FPR:$Src, u8:$RCON": {
"Desc": "Assists in key generation",
@@ -1462,7 +1435,7 @@
],
"DestSize": "std::max<uint8_t>(4, GetOpSize(_Src1))"
},
"FPR = PCLMUL u8:#RegisterSize, FPR:$Src1, FPR:$Src2, u8:$Selector": {
"FPR = PCLMUL FPR:$Src1, FPR:$Src2, u8:$Selector": {
"Desc": [
"Performs carryless multiplication of 64-bit elements depending on the selector.",
"Selector = 0b00000000: Uses low 64-bit elements from both input vectors",
@@ -1470,7 +1443,7 @@
"Selector = 0b00010000: Uses low 64-bit element from Src1 and high 64-bit element from Src2",
"Selector = 0b00010001: Uses high 64-bit elements from both input vectors"
],
"DestSize": "RegisterSize"
"DestSize": "16"
}
},
"F64": {
+2 -3
View File
@@ -23,7 +23,6 @@ namespace FEXCore::IR {
#define IROP_REG_CLASSES_IMPL
#define IROP_HASSIDEEFFECTS_IMPL
#define IROP_SIZES_IMPL
#define IROP_GETHASDEST_IMPL
#include <FEXCore/IR/IRDefines.inc>
@@ -126,7 +125,7 @@ static void PrintArg(std::stringstream *out, IRListView const* IR, OrderedNodeWr
}
}
if (GetHasDest(IROp->Op)) {
if (IROp->HasDest) {
uint32_t ElementSize = IROp->ElementSize;
uint32_t NumElements = IROp->Size;
if (!IROp->ElementSize) {
@@ -232,7 +231,7 @@ void Dump(std::stringstream *out, IRListView const* IR, IR::RegisterAllocationDa
if (!Skip) {
AddIndent();
if (GetHasDest(IROp->Op)) {
if (IROp->HasDest) {
uint32_t ElementSize = IROp->ElementSize;
uint32_t NumElements = IROp->Size;
+3 -2
View File
@@ -112,7 +112,7 @@ void IREmitter::ReplaceAllUsesWithRange(OrderedNode *Node, OrderedNode *NewNode,
while (Begin != End) {
auto [RealNode, IROp] = Begin();
const uint8_t NumArgs = IR::GetArgs(IROp->Op);
uint8_t NumArgs = IR::GetArgs(IROp->Op);
for (uint8_t i = 0; i < NumArgs; ++i) {
if (IROp->Args[i].ID() == NodeId) {
Node->RemoveUse();
@@ -148,7 +148,7 @@ void IREmitter::RemoveArgUses(OrderedNode *Node) {
FEXCore::IR::IROp_Header *IROp = Node->Op(DataBegin);
const uint8_t NumArgs = IR::GetArgs(IROp->Op);
uint8_t NumArgs = IR::GetArgs(IROp->Op);
for (uint8_t i = 0; i < NumArgs; ++i) {
auto ArgNode = IROp->Args[i].GetNode(ListBegin);
ArgNode->RemoveUse();
@@ -201,6 +201,7 @@ void IREmitter::ReplaceWithConstant(OrderedNode *Node, uint64_t Value) {
// Overwrite data with the new constant op
Header->Op = OP_CONSTANT;
Header->NumArgs = 0;
auto Const = Header->CW<IROp_Constant>();
Const->Constant = Value;
} else {
+1 -1
View File
@@ -17,7 +17,7 @@ $end_info$
namespace FEXCore::IR {
class IREmitter;
void PassManager::AddDefaultPasses(FEXCore::Context::ContextImpl *ctx, bool InlineConstants, bool StaticRegisterAllocation) {
void PassManager::AddDefaultPasses(FEXCore::Context::Context *ctx, bool InlineConstants, bool StaticRegisterAllocation) {
FEX_CONFIG_OPT(DisablePasses, O0);
if (!DisablePasses()) {
+1 -5
View File
@@ -14,10 +14,6 @@ $end_info$
#include <utility>
#include <vector>
namespace FEXCore::Context {
class ContextImpl;
}
namespace FEXCore::HLE {
class SyscallHandler;
}
@@ -44,7 +40,7 @@ protected:
class PassManager final {
friend class SyscallOptimization;
public:
void AddDefaultPasses(FEXCore::Context::ContextImpl *ctx, bool InlineConstants, bool StaticRegisterAllocation);
void AddDefaultPasses(FEXCore::Context::Context *ctx, bool InlineConstants, bool StaticRegisterAllocation);
void AddDefaultValidationPasses();
Pass* InsertPass(std::unique_ptr<Pass> Pass, std::string Name = "") {
Pass->RegisterPassManager(this);
+18 -31
View File
@@ -29,7 +29,6 @@ $end_info$
#include <string.h>
#include <tuple>
#include <unordered_map>
#include <tsl/robin_map.h>
#include <utility>
namespace FEXCore::IR {
@@ -199,17 +198,6 @@ private:
std::unordered_map<uint64_t, OrderedNode*> ConstPool;
std::map<OrderedNode*, uint64_t> AddressgenConsts;
// Pool inline constant generation. These are typically very small and pool efficiently.
tsl::robin_map<uint64_t, OrderedNode*> InlineConstantGen;
OrderedNode *CreateInlineConstant(IREmitter *IREmit, uint64_t Constant) {
const auto it = InlineConstantGen.find(Constant);
if (it != InlineConstantGen.end()) {
return it->second;
}
auto Result = InlineConstantGen.insert_or_assign(Constant, IREmit->_InlineConstant(Constant));
return Result.first->second;
}
bool SupportsTSOImm9{};
};
@@ -245,7 +233,7 @@ void ConstProp::CodeMotionAroundSelects(IREmitter *IREmit, const IRListView& Cur
for (auto [BlockNode, BlockIROp] : CurrentIR.GetBlocks()) {
auto BlockOp = BlockIROp->CW<FEXCore::IR::IROp_CodeBlock>();
for (auto [UnaryOpNode, UnaryOpHdr] : CurrentIR.GetCode(BlockNode)) {
if (IR::GetArgs(UnaryOpHdr->Op) == 1 && !HasSideEffects(UnaryOpHdr->Op)) {
if (UnaryOpHdr->NumArgs == 1 && !HasSideEffects(UnaryOpHdr->Op)) {
// could be moved
auto SelectOpNode = IREmit->UnwrapNode(UnaryOpHdr->Args[0]);
auto SelectOpHdr = IREmit->GetOpHeader(UnaryOpHdr->Args[0]);
@@ -267,7 +255,7 @@ void ConstProp::CodeMotionAroundSelects(IREmitter *IREmit, const IRListView& Cur
// Copy over the op
memcpy(NewUnaryOp1.first, UnaryOpHdr, OpSize);
for (int i = 0; i < IR::GetArgs(NewUnaryOp1.first->Op); i++) {
for (int i = 0; i < NewUnaryOp1.first->NumArgs; i++) {
NewUnaryOp1.first->Args[i] = IREmit->WrapNode(IREmit->Invalid());
}
// Set New Op to operate on the constant
@@ -281,7 +269,7 @@ void ConstProp::CodeMotionAroundSelects(IREmitter *IREmit, const IRListView& Cur
// Copy over the op
memcpy(NewUnaryOp2.first, UnaryOpHdr, OpSize);
for (int i = 0; i < IR::GetArgs(NewUnaryOp2.first->Op); i++) {
for (int i = 0; i < NewUnaryOp2.first->NumArgs; i++) {
NewUnaryOp2.first->Args[i] = IREmit->WrapNode(IREmit->Invalid());
}
// Set New Op to operate on the constant
@@ -378,7 +366,7 @@ bool ConstProp::ZextAndMaskingElimination(IREmitter *IREmit, const IRListView& C
case OP_ASHR:
case OP_LSHL:
case OP_ROR: {
for (int i = 0; i < IR::GetArgs(IROp->Op); i++) {
for (int i = 0; i < IROp->NumArgs; i++) {
auto newArg = RemoveUselessMasking(IREmit, IROp->Args[i], getMask(IROp));
if (newArg.ID() != IROp->Args[i].ID()) {
IREmit->ReplaceNodeArgument(CodeNode, i, IREmit->UnwrapNode(newArg));
@@ -390,7 +378,7 @@ bool ConstProp::ZextAndMaskingElimination(IREmitter *IREmit, const IRListView& C
case OP_AND: {
// if AND's arguments are imms, they are masking
for (int i = 0; i < IR::GetArgs(IROp->Op); i++) {
for (int i = 0; i < IROp->NumArgs; i++) {
auto mask = getMask(IROp);
uint64_t imm = 0;
if (IREmit->IsValueConstant(IROp->Args[i^1], &imm))
@@ -469,7 +457,7 @@ bool ConstProp::ZextAndMaskingElimination(IREmitter *IREmit, const IRListView& C
case OP_VFDIV:
case OP_FCMP: {
auto flopSize = IROp->Size;
for (int i = 0; i < IR::GetArgs(IROp->Op); i++) {
for (int i = 0; i < IROp->NumArgs; i++) {
auto argHeader = IREmit->GetOpHeader(IROp->Args[i]);
if (argHeader->Op == OP_VMOV) {
@@ -832,7 +820,6 @@ bool ConstProp::ConstantPropagation(IREmitter *IREmit, const IRListView& Current
}
bool ConstProp::ConstantInlining(IREmitter *IREmit, const IRListView& CurrentIR) {
InlineConstantGen.clear();
bool Changed = false;
for (auto [CodeNode, IROp] : CurrentIR.GetAllCode()) {
@@ -854,7 +841,7 @@ bool ConstProp::ConstantInlining(IREmitter *IREmit, const IRListView& CurrentIR)
else
Constant2 &= 63;
IREmit->ReplaceNodeArgument(CodeNode, 1, CreateInlineConstant(IREmit, Constant2));
IREmit->ReplaceNodeArgument(CodeNode, 1, IREmit->_InlineConstant(Constant2));
Changed = true;
}
@@ -870,7 +857,7 @@ bool ConstProp::ConstantInlining(IREmitter *IREmit, const IRListView& CurrentIR)
if (IsImmAddSub(Constant2)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Header.Args[1]));
IREmit->ReplaceNodeArgument(CodeNode, 1, CreateInlineConstant(IREmit, Constant2));
IREmit->ReplaceNodeArgument(CodeNode, 1, IREmit->_InlineConstant(Constant2));
Changed = true;
}
@@ -886,7 +873,7 @@ bool ConstProp::ConstantInlining(IREmitter *IREmit, const IRListView& CurrentIR)
if (IsImmAddSub(Constant1)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Header.Args[1]));
IREmit->ReplaceNodeArgument(CodeNode, 1, CreateInlineConstant(IREmit, Constant1));
IREmit->ReplaceNodeArgument(CodeNode, 1, IREmit->_InlineConstant(Constant1));
Changed = true;
}
@@ -901,8 +888,8 @@ bool ConstProp::ConstantInlining(IREmitter *IREmit, const IRListView& CurrentIR)
{
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Header.Args[2]));
IREmit->ReplaceNodeArgument(CodeNode, 2, CreateInlineConstant(IREmit, Constant2));
IREmit->ReplaceNodeArgument(CodeNode, 3, CreateInlineConstant(IREmit, Constant3));
IREmit->ReplaceNodeArgument(CodeNode, 2, IREmit->_InlineConstant(Constant2));
IREmit->ReplaceNodeArgument(CodeNode, 3, IREmit->_InlineConstant(Constant3));
}
break;
@@ -916,7 +903,7 @@ bool ConstProp::ConstantInlining(IREmitter *IREmit, const IRListView& CurrentIR)
if (IsImmAddSub(Constant2)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Header.Args[1]));
IREmit->ReplaceNodeArgument(CodeNode, 1, CreateInlineConstant(IREmit, Constant2));
IREmit->ReplaceNodeArgument(CodeNode, 1, IREmit->_InlineConstant(Constant2));
Changed = true;
}
@@ -932,7 +919,7 @@ bool ConstProp::ConstantInlining(IREmitter *IREmit, const IRListView& CurrentIR)
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->NewRIP));
IREmit->ReplaceNodeArgument(CodeNode, 0, CreateInlineConstant(IREmit, Constant));
IREmit->ReplaceNodeArgument(CodeNode, 0, IREmit->_InlineConstant(Constant));
Changed = true;
} else {
@@ -958,7 +945,7 @@ bool ConstProp::ConstantInlining(IREmitter *IREmit, const IRListView& CurrentIR)
if (IsImmLogical(Constant2, IROp->Size * 8)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Header.Args[1]));
IREmit->ReplaceNodeArgument(CodeNode, 1, CreateInlineConstant(IREmit, Constant2));
IREmit->ReplaceNodeArgument(CodeNode, 1, IREmit->_InlineConstant(Constant2));
Changed = true;
}
@@ -974,7 +961,7 @@ bool ConstProp::ConstantInlining(IREmitter *IREmit, const IRListView& CurrentIR)
if (IsImmMemory(Constant2, IROp->Size)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Offset));
IREmit->ReplaceNodeArgument(CodeNode, Op->Offset_Index, CreateInlineConstant(IREmit, Constant2));
IREmit->ReplaceNodeArgument(CodeNode, Op->Offset_Index, IREmit->_InlineConstant(Constant2));
Changed = true;
}
@@ -990,7 +977,7 @@ bool ConstProp::ConstantInlining(IREmitter *IREmit, const IRListView& CurrentIR)
if (IsImmMemory(Constant2, IROp->Size)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Offset));
IREmit->ReplaceNodeArgument(CodeNode, Op->Offset_Index, CreateInlineConstant(IREmit, Constant2));
IREmit->ReplaceNodeArgument(CodeNode, Op->Offset_Index, IREmit->_InlineConstant(Constant2));
Changed = true;
}
@@ -1007,7 +994,7 @@ bool ConstProp::ConstantInlining(IREmitter *IREmit, const IRListView& CurrentIR)
if (IsTSOImm9(Constant2)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Offset));
IREmit->ReplaceNodeArgument(CodeNode, Op->Offset_Index, CreateInlineConstant(IREmit, Constant2));
IREmit->ReplaceNodeArgument(CodeNode, Op->Offset_Index, IREmit->_InlineConstant(Constant2));
Changed = true;
}
@@ -1025,7 +1012,7 @@ bool ConstProp::ConstantInlining(IREmitter *IREmit, const IRListView& CurrentIR)
if (IsTSOImm9(Constant2)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Offset));
IREmit->ReplaceNodeArgument(CodeNode, Op->Offset_Index, CreateInlineConstant(IREmit, Constant2));
IREmit->ReplaceNodeArgument(CodeNode, Op->Offset_Index, IREmit->_InlineConstant(Constant2));
Changed = true;
}
@@ -219,8 +219,8 @@ namespace {
ContextClassification->emplace_back(ContextMemberInfo{
ContextMemberClassification {
offsetof(FEXCore::Core::CPUState, InlineJITBlockHeader),
sizeof(FEXCore::Core::CPUState::InlineJITBlockHeader),
offsetof(FEXCore::Core::CPUState, _pad2),
sizeof(FEXCore::Core::CPUState::_pad2),
},
ACCESS_INVALID,
FEXCore::IR::InvalidClass,
@@ -174,9 +174,9 @@ bool IRCompaction::Run(IREmitter *IREmit) {
for (auto [LocalNode, LocalIROp] : LocalIR.GetCode(Block.NewNode)) {
// Now that we have the op copied over, we need to modify SSA values to point to the new correct locations
// This doesn't use IR::GetRAArgs(Op) because we need to remap all SSA nodes
// This doesn't use IR::GetArgs(Op) because we need to remap all SSA nodes
// Including ones that we don't RA
const uint8_t NumArgs = IR::GetArgs(LocalIROp->Op);
const uint8_t NumArgs = LocalIROp->NumArgs;
for (uint8_t i = 0; i < NumArgs; ++i) {
const auto OldArg = LocalIROp->Args[i].ID();
const auto NewArg = OldToNewRemap[OldArg.Value].NodeID;
Loaded 100 of 284 files, more files were not shown because too many files have changed in this diff. Show more