steam-frame/FEX-Emu--FEX
1
0 Fork 0
mirror of https://github.com/FEX-Emu/FEX.git synced 2026-10-06 09:00:17 +02:00
Code Issues Packages Projects Releases 66 Wiki Activity

Merge pull request #3134 from Sonicadvance1/remove_x86_jit

Browse Source 
FEXCore: Removes x86 JIT.
This commit is contained in:
Ryan Houdek authored and GitHub committed 2023-09-22 15:36:47 -07:00
2 parents 65b6df9dbb 0fbf403787
commit 879b41c184
29 files changed
+191 -12166
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.github/workflows/ccpp.yml
+1 -1
View File
@@ -21,7 +21,7 @@ jobs:
runs-on: ${{ matrix.arch }}
strategy:
matrix:
arch: [[self-hosted, x64], [self-hosted, ARMv8.0], [self-hosted, ARMv8.2], [self-hosted, ARMv8.4]]
arch: [[self-hosted, ARMv8.0], [self-hosted, ARMv8.2], [self-hosted, ARMv8.4]]
fail-fast: false
steps:
.github/workflows/glibc_fault.yml
+1 -2
View File
@@ -28,8 +28,7 @@ jobs:
runs-on: ${{ matrix.arch }}
strategy:
matrix:
# Run on an x86 device and any ARM runner.
arch: [[self-hosted, x64], [self-hosted, ARM64]]
arch: [[self-hosted, ARM64]]
fail-fast: false
steps:
.github/workflows/hostrunner.yml
+107
View File
@@ -0,0 +1,107 @@
name: Hostrunner tests
on:
push:
branches:
- main
pull_request:
branches:
- main
env:
# Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)
BUILD_TYPE: Release
CC: clang
CXX: clang++
FEX_ENABLEAVX: 1
jobs:
build:
runs-on: ${{ matrix.arch }}
strategy:
matrix:
arch: [[self-hosted, x64]]
fail-fast: false
steps:
- uses: actions/checkout@v3
- name: Set runner label
run: echo "runner_label=${{ matrix.arch[1] }}" >> $GITHUB_ENV
- name: Set rootfs paths
run: |
echo "FEX_ROOTFS_MOUNT=/mnt/AutoNFS/rootfs/" >> $GITHUB_ENV
echo "FEX_ROOTFS_PATH=$HOME/Rootfs/" >> $GITHUB_ENV
echo "FEX_ROOTFS=$HOME/Rootfs/" >> $GITHUB_ENV
echo "ROOTFS=$HOME/Rootfs/" >> $GITHUB_ENV
- name: Update RootFS cache
# Use a bash shell so we can use the same syntax for environment variable
# access regardless of the host operating system
shell: bash
run: $GITHUB_WORKSPACE/Scripts/CI_FetchRootFS.py
- name : submodule checkout
# Need to update submodules
run: |
git submodule sync --recursive
git submodule update --init --depth 1
- name: Clean Build Environment
run: rm -Rf ${{runner.workspace}}/build
- name: Create Build Environment
# Some projects don't allow in-source building, so create a separate build directory
# We'll use this as our working directory for all subsequent commands
run: cmake -E make_directory ${{runner.workspace}}/build
- name: Configure CMake
# Use a bash shell so we can use the same syntax for environment variable
# access regardless of the host operating system
shell: bash
working-directory: ${{runner.workspace}}/build
# Note the current convention is to use the -S and -B options here to specify source
# and build directories, but this is only available with CMake 3.13 and higher.
# The CMake binaries on the Github Actions machines are (as of this writing) 3.12
run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -G Ninja -DENABLE_LTO=False -DENABLE_ASSERTIONS=True -DENABLE_X86_HOST_DEBUG=True
- name: Build
working-directory: ${{runner.workspace}}/build
shell: bash
# Execute the build. You can specify a specific target with "--target <NAME>"
run: cmake --build . --config $BUILD_TYPE
- name: ASM Tests
working-directory: ${{runner.workspace}}/build
shell: bash
# Execute the unit tests
run: cmake --build . --config $BUILD_TYPE --target asm_tests
- name: ASM Test Results move
if: ${{ always() }}
shell: bash
working-directory: ${{runner.workspace}}/build
run: mv ${{runner.workspace}}/build/Testing/Temporary/LastTest.log ${{runner.workspace}}/build/Testing/Temporary/LastTest_ASM.log || true
- name: Truncate test results
if: ${{ always() }}
shell: bash
working-directory: ${{runner.workspace}}/build
# Cap out the log files at 20M in case something crash spins and dumps fault text
# ASM tests get quite close to 10MB
run: truncate --size=<20M ${{runner.workspace}}/build/Testing/Temporary/LastTest_*.log || true
- name: Set runner name
if: ${{ always() }}
run: echo "runner_name=$(hostname)" >> $GITHUB_ENV
- name: Upload results
if: ${{ always() }}
uses: 'actions/upload-artifact@v3'
timeout-minutes: 1
with:
name: Results-${{ env.runner_name }}
path: ${{runner.workspace}}/build/Testing/Temporary/LastTest_*.log
retention-days: 3
.github/workflows/mingw_build.yml
+1 -1
View File
@@ -17,7 +17,7 @@ jobs:
runs-on: ${{ matrix.arch }}
strategy:
matrix:
arch: [[self-hosted, x64, mingw], [self-hosted, ARM64, mingw]]
arch: [[self-hosted, ARM64, mingw]]
fail-fast: false
steps:
CMakeLists.txt
-8
View File
@@ -112,14 +112,6 @@ else()
endif()
if (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")
option(ENABLE_X86_HOST_DEBUG "Enables compiling on x86_64 host" FALSE)
if (NOT ENABLE_X86_HOST_DEBUG)
message(FATAL_ERROR
" Be warned: FEX isn't optimized for x86_64 hosts!\n"
" Support for x86_64 hosts is only for debugging and convenience!\n"
" Don't expect amazing performance or optimal code generation!\n"
" Pass -DENABLE_X86_HOST_DEBUG=True to bypass this message!")
endif()
set(_M_X86_64 1)
add_definitions(-D_M_X86_64=1)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcx16")
FEXCore/CMakeLists.txt
-9
View File
@@ -13,15 +13,6 @@ if (CMAKE_SYSTEM_PROCESSOR MATCHES "^aarch64|^arm64|^armv8\.*")
set(_M_ARM_64 1)
endif()
if (ENABLE_VIXL_SIMULATOR)
# If the vixl simulator is enabled then we are using the ARM64 JIT
option(ENABLE_JIT_X86_64 "Enable the x86_64 JIT" FALSE)
option(ENABLE_JIT_ARM64 "Enable the ARM64 JIT" TRUE)
else()
option(ENABLE_JIT_X86_64 "Enable the x86_64 JIT" ${_M_X86_64})
option(ENABLE_JIT_ARM64 "Enable the ARM64 JIT" ${_M_ARM_64})
endif()
option(ENABLE_CLANG_FORMAT "Run clang format over the source" FALSE)
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
FEXCore/Source/CMakeLists.txt
+13 -38
View File
@@ -107,9 +107,20 @@ set (SRCS
Interface/Core/X86HelperGen.cpp
Interface/Core/ArchHelpers/Arm64Emitter.cpp
Interface/Core/Dispatcher/Dispatcher.cpp
Interface/Core/Dispatcher/X86Dispatcher.cpp
Interface/Core/Dispatcher/Arm64Dispatcher.cpp
Interface/Core/Interpreter/Fallbacks/InterpreterFallbacks.cpp
Interface/Core/JIT/Arm64/JIT.cpp
Interface/Core/JIT/Arm64/ALUOps.cpp
Interface/Core/JIT/Arm64/AtomicOps.cpp
Interface/Core/JIT/Arm64/BranchOps.cpp
Interface/Core/JIT/Arm64/ConversionOps.cpp
Interface/Core/JIT/Arm64/EncryptionOps.cpp
Interface/Core/JIT/Arm64/FlagOps.cpp
Interface/Core/JIT/Arm64/MemoryOps.cpp
Interface/Core/JIT/Arm64/MiscOps.cpp
Interface/Core/JIT/Arm64/MoveOps.cpp
Interface/Core/JIT/Arm64/VectorOps.cpp
Interface/Core/JIT/Arm64/Arm64Relocations.cpp
Interface/Core/X86Tables/BaseTables.cpp
Interface/Core/X86Tables/DDDTables.cpp
Interface/Core/X86Tables/EVEXTables.cpp
@@ -159,7 +170,7 @@ if (ENABLE_GLIBC_ALLOCATOR_HOOK_FAULT)
Utils/AllocatorOverride.cpp)
endif()
set(DEFINES -DTHREAD_LOCAL=_Thread_local)
set(DEFINES -DTHREAD_LOCAL=_Thread_local -DJIT_ARM64)
if (_M_X86_64)
list(APPEND DEFINES -D_M_X86_64=1)
@@ -178,42 +189,6 @@ if (ENABLE_VIXL_DISASSEMBLER)
list(APPEND DEFINES -DVIXL_DISASSEMBLER=1)
endif()
if (ENABLE_JIT_X86_64)
list(APPEND SRCS
Interface/Core/JIT/x86_64/JIT.cpp
Interface/Core/JIT/x86_64/ALUOps.cpp
Interface/Core/JIT/x86_64/AtomicOps.cpp
Interface/Core/JIT/x86_64/BranchOps.cpp
Interface/Core/JIT/x86_64/ConversionOps.cpp
Interface/Core/JIT/x86_64/EncryptionOps.cpp
Interface/Core/JIT/x86_64/FlagOps.cpp
Interface/Core/JIT/x86_64/MemoryOps.cpp
Interface/Core/JIT/x86_64/MiscOps.cpp
Interface/Core/JIT/x86_64/MoveOps.cpp
Interface/Core/JIT/x86_64/VectorOps.cpp
Interface/Core/JIT/x86_64/x64Relocations.cpp
)
list(APPEND DEFINES -DJIT_X86_64)
endif()
if (ENABLE_JIT_ARM64)
list(APPEND DEFINES -DJIT_ARM64)
list(APPEND SRCS
Interface/Core/JIT/Arm64/JIT.cpp
Interface/Core/JIT/Arm64/ALUOps.cpp
Interface/Core/JIT/Arm64/AtomicOps.cpp
Interface/Core/JIT/Arm64/BranchOps.cpp
Interface/Core/JIT/Arm64/ConversionOps.cpp
Interface/Core/JIT/Arm64/EncryptionOps.cpp
Interface/Core/JIT/Arm64/FlagOps.cpp
Interface/Core/JIT/Arm64/MemoryOps.cpp
Interface/Core/JIT/Arm64/MiscOps.cpp
Interface/Core/JIT/Arm64/MoveOps.cpp
Interface/Core/JIT/Arm64/VectorOps.cpp
Interface/Core/JIT/Arm64/Arm64Relocations.cpp
)
endif()
if (_M_ARM_64 AND HAS_CLANG_PRESERVE_ALL)
list(APPEND DEFINES "-DFEXCORE_PRESERVE_ALL_ATTR=__attribute__((preserve_all));-DFEXCORE_HAS_PRESERVE_ALL_ATTR=1")
else()
FEXCore/Source/Interface/Core/CPUID.cpp
-27
View File
@@ -21,9 +21,6 @@ $end_info$
#include "git_version.h"
#include <cstring>
#ifdef _M_X86_64
#include "Interface/Core/Dispatcher/X86Dispatcher.h"
#endif
namespace FEXCore {
namespace ProductNames {
@@ -67,7 +64,6 @@ namespace ProductNames {
static const char ARM_Firestorm[] = "Apple Firestorm";
static const char ARM_Icestorm[] = "Apple Icestorm";
#else
static const char UNKNOWN[] = "Unknown CPU";
#endif
}
@@ -343,33 +339,10 @@ void CPUIDEmu::SetupHostHybridFlag() {
#else
static uint32_t GetCycleCounterFrequency() {
uint32_t data[4];
Xbyak::util::Cpu::getCpuid(0, data);
if (data[0] >= 0x15) {
Xbyak::util::Cpu::getCpuid(0x15, data);
if (data[0] && data[1] && data[2]) {
return data[2] * data[1] / data[0];
}
}
return 0;
}
void CPUIDEmu::SetupHostHybridFlag() {
uint32_t data[4];
Xbyak::util::Cpu::getCpuid(0, data);
if (data[0] >= 0x7) {
Xbyak::util::Cpu::getCpuid(0x7, data);
// Bit 15 of edx claims hybrid CPU
Hybrid = (data[3] & (1U << 15)) != 0;
}
size_t CPUs = FEXCore::CPUInfo::CalculateNumberOfCPUs();
PerCPUData.resize(CPUs);
for (size_t i = 0; i < CPUs; ++i) {
PerCPUData[i].IsBig = true;
PerCPUData[i].ProductName = ProductNames::UNKNOWN;
}
}
#endif
FEXCore/Source/Interface/Core/Core.cpp
-21
View File
@@ -308,13 +308,7 @@ namespace FEXCore::Context {
// Initialize the CPU core signal handlers & DispatcherConfig
switch (Config.Core) {
case FEXCore::Config::CONFIG_IRJIT:
#if (_M_X86_64 && JIT_X86_64)
BackendFeatures = FEXCore::CPU::GetX86JITBackendFeatures();
#elif (_M_ARM_64 && JIT_ARM64) || defined(VIXL_SIMULATOR)
BackendFeatures = FEXCore::CPU::GetArm64JITBackendFeatures();
#else
ERROR_AND_DIE_FMT("FEXCore has been compiled without a viable JIT core");
#endif
break;
case FEXCore::Config::CONFIG_CUSTOM:
// Do nothing
@@ -325,14 +319,7 @@ namespace FEXCore::Context {
}
DispatcherConfig.StaticRegisterAllocation = Config.StaticRegisterAllocation && BackendFeatures.SupportsStaticRegisterAllocation;
#if JIT_ARM64
Dispatcher = FEXCore::CPU::Dispatcher::CreateArm64(this, DispatcherConfig);
#elif JIT_X86_64
Dispatcher = FEXCore::CPU::Dispatcher::CreateX86(this, DispatcherConfig);
#else
ERROR_AND_DIE_FMT("FEXCore has been compiled with an unknown target");
#endif
// Set up the SignalDelegator config since core is initialized.
FEXCore::SignalDelegator::SignalDelegatorConfig SignalConfig {
@@ -667,15 +654,7 @@ namespace FEXCore::Context {
switch (Config.Core) {
case FEXCore::Config::CONFIG_IRJIT:
Thread->PassManager->InsertRegisterAllocationPass(DoSRA, HostFeatures.SupportsAVX);
#if (_M_X86_64 && JIT_X86_64)
Thread->CPUBackend = FEXCore::CPU::CreateX86JITCore(this, Thread);
#elif (_M_ARM_64 && JIT_ARM64) || defined(VIXL_SIMULATOR)
Thread->CPUBackend = FEXCore::CPU::CreateArm64JITCore(this, Thread);
#else
ERROR_AND_DIE_FMT("FEXCore has been compiled without a viable JIT core");
#endif
break;
case FEXCore::Config::CONFIG_CUSTOM:
Thread->CPUBackend = CustomCPUFactory(this, Thread);
FEXCore/Source/Interface/Core/Dispatcher/X86Dispatcher.cpp
-409
View File
@@ -1,409 +0,0 @@
// SPDX-License-Identifier: MIT
#include "FEXCore/Utils/AllocatorHooks.h"
#include "Interface/Core/LookupCache.h"
#include "Interface/Core/Dispatcher/X86Dispatcher.h"
#include "Interface/Core/X86HelperGen.h"
#include "Interface/Context/Context.h"
#include <FEXCore/Core/X86Enums.h>
#include <FEXCore/Core/CoreState.h>
#include <FEXCore/Core/CPUBackend.h>
#include <FEXCore/Debug/InternalThreadState.h>
#include <FEXCore/Utils/Allocator.h>
#include <FEXCore/fextl/memory.h>
#include <FEXCore/fextl/string.h>
#include <FEXHeaderUtils/Syscalls.h>
#include <cmath>
#include <memory>
#include <stddef.h>
#include <stdint.h>
namespace FEXCore::CPU {
static constexpr size_t MAX_DISPATCHER_CODE_SIZE = 4096;
#define STATE r14
X86Dispatcher::X86Dispatcher(FEXCore::Context::ContextImpl *ctx, const DispatcherConfig &config)
: Dispatcher(ctx, config)
, Xbyak::CodeGenerator(MAX_DISPATCHER_CODE_SIZE,
FEXCore::Allocator::VirtualAlloc(MAX_DISPATCHER_CODE_SIZE, true),
nullptr) {
LOGMAN_THROW_AA_FMT(!config.StaticRegisterAllocation, "X86 dispatcher does not support SRA");
using namespace Xbyak;
using namespace Xbyak::util;
DispatchPtr = getCurr<AsmDispatch>();
// Temp registers
// rax, rcx, rdx, rsi, r8, r9,
// r10, r11
//
// Callee Saved
// rbx, rbp, r12, r13, r14, r15
//
// 1St Argument: rdi <ThreadState>
// XMM:
// All temp
// while (true) {
// Ptr = FindBlock(RIP)
// if (!Ptr)
// Ptr = CTX->CompileBlock(RIP);
//
// if (Ptr)
// Ptr();
// else
// {
// Ptr = FallbackCore->CompileBlock()
// if (Ptr)
// Ptr()
// else {
// ShouldStop = true;
// }
// }
// }
// Bunch of exit state stuff
// x86-64 ABI has the stack aligned when /call/ happens
// Which means the destination has a misaligned stack at that point
push(rbx);
push(rbp);
push(r12);
push(r13);
push(r14);
push(r15);
sub(rsp, 8);
mov(STATE, rdi);
// Save this stack pointer so we can cleanly shutdown the emulation with a long jump
// regardless of where we were in the stack
mov(qword STATE_PTR(CpuStateFrame, ReturningStackLocation), rsp);
Label LoopTop;
Label FullLookup;
Label NoBlock;
Label ExitBlock;
Label ThreadPauseHandler;
L(LoopTop);
AbsoluteLoopTopAddressFillSRA = AbsoluteLoopTopAddress = getCurr<uint64_t>();
{
// Load our RIP
mov(rdx, qword STATE_PTR(CPUState, rip));
// L1 Cache
mov(r13, qword STATE_PTR(CpuStateFrame, Pointers.Common.L1Pointer));
mov(rax, rdx);
and_(rax, LookupCache::L1_ENTRIES_MASK);
shl(rax, 4);
cmp(qword[r13 + rax + offsetof(FEXCore::LookupCache::LookupCacheEntry, GuestCode)], rdx);
jne(FullLookup);
jmp(qword[r13 + rax + offsetof(FEXCore::LookupCache::LookupCacheEntry, HostCode)]);
L(FullLookup);
mov(r13, qword STATE_PTR(CpuStateFrame, Pointers.Common.L2Pointer));
// Full lookup
uint64_t VirtualMemorySize = CTX->Config.VirtualMemSize;
mov(rax, rdx);
mov(rbx, VirtualMemorySize - 1);
and_(rax, rbx);
shr(rax, 12);
// Load page pointer
mov(rdi, qword [r13 + rax * 8]);
cmp(rdi, 0);
je(NoBlock);
mov (rax, rdx);
and_(rax, 0x0FFF);
shl(rax, (int)log2(sizeof(FEXCore::LookupCache::LookupCacheEntry)));
// check for aliasing
mov(rcx, qword [rdi + rax + 8]);
cmp(rcx, rdx);
jne(NoBlock);
// Load the block pointer
mov(rax, qword [rdi + rax]);
cmp(rax, 0);
je(NoBlock);
// Update L1
mov(r13, qword STATE_PTR(CpuStateFrame, Pointers.Common.L1Pointer));
mov(rcx, rdx);
and_(rcx, LookupCache::L1_ENTRIES_MASK);
shl(rcx, 1);
mov(qword[r13 + rcx*8 + 8], rdx);
mov(qword[r13 + rcx*8 + 0], rax);
// Real block if we made it here
jmp(rax);
}
{
L(ExitBlock);
ThreadStopHandlerAddress = getCurr<uint64_t>();
add(rsp, 8);
pop(r15);
pop(r14);
pop(r13);
pop(r12);
pop(rbp);
pop(rbx);
ret();
}
// Block creation
{
L(NoBlock);
inc(qword [STATE + offsetof(FEXCore::Core::CPUState, DeferredSignalRefCount)]);
// {rdi, rsi, rdx}
mov(rdi, reinterpret_cast<uint64_t>(CTX));
mov(rsi, STATE);
mov(rax, GetCompileBlockPtr());
call(rax);
dec(qword [STATE + offsetof(FEXCore::Core::CPUState, DeferredSignalRefCount)]);
Label AfterStore;
// Skip the deferred fault address if the refcount isn't zero
jne(AfterStore);
mov(rax, qword [STATE + offsetof(FEXCore::Core::CPUState, DeferredSignalFaultAddress)]);
mov(rax, qword [rax]);
L(AfterStore);
// rdx already contains RIP here
jmp(LoopTop);
}
{
ExitFunctionLinkerAddress = getCurr<uint64_t>();
inc(qword [STATE + offsetof(FEXCore::Core::CPUState, DeferredSignalRefCount)]);
// {rdi, rsi}
mov(rdi, STATE);
mov(rsi, rax); // rax is set at the block end
call(qword STATE_PTR(CpuStateFrame, Pointers.Common.ExitFunctionLink));
dec(qword [STATE + offsetof(FEXCore::Core::CPUState, DeferredSignalRefCount)]);
Label AfterStore;
// Skip the deferred fault address if the refcount isn't zero
jne(AfterStore);
mov(rbx, qword [STATE + offsetof(FEXCore::Core::CPUState, DeferredSignalFaultAddress)]);
mov(qword [rbx], rbx);
L(AfterStore);
jmp(rax);
}
{
// Pause handler
ThreadPauseHandlerAddress = getCurr<uint64_t>();
L(ThreadPauseHandler);
mov(rdi, reinterpret_cast<uintptr_t>(CTX));
mov(rsi, STATE);
mov(rax, reinterpret_cast<uint64_t>(SleepThread));
call(rax);
// XXX: Unsupported atm
PauseReturnInstruction = getCurr<uint64_t>();
ud2();
}
{
CallbackPtr = getCurr<JITCallback>();
push(rbx);
push(rbp);
push(r12);
push(r13);
push(r14);
push(r15);
sub(rsp, 8);
// First thing we need to move the thread state pointer back in to our register
mov(STATE, rdi);
// XXX: XMM?
// Make sure to adjust the refcounter so we don't clear the cache now
add(qword STATE_PTR(CpuStateFrame, SignalHandlerRefCounter), 1);
// Now push the callback return trampoline to the guest stack
// Guest will be misaligned because calling a thunk won't correct the guest's stack once we call the callback from the host
mov(rax, CTX->X86CodeGen.CallbackReturn);
// Store the trampoline to the guest stack
// Guest stack is now correctly misaligned after a regular call instruction
sub(qword STATE_PTR(CpuStateFrame, State.gregs[X86State::REG_RSP]), 16);
mov(rbx, qword STATE_PTR(CpuStateFrame, State.gregs[X86State::REG_RSP]));
mov(qword [rbx], rax);
// Store RIP to the context state
mov(qword STATE_PTR(CpuStateFrame, State.rip), rsi);
// Back to the loop top now
jmp(LoopTop);
}
{
// Signal return handler
SignalHandlerReturnAddress = getCurr<uint64_t>();
ud2();
}
{
// RT Signal return handler
SignalHandlerReturnAddressRT = getCurr<uint64_t>();
ud2();
}
{
// Guest SIGILL handler
// Needs to be distinct from the SignalHandlerReturnAddress
GuestSignal_SIGILL = getCurr<uint64_t>();
ud2();
}
{
// Guest SIGTRAP handler
// Needs to be distinct from the SignalHandlerReturnAddress
GuestSignal_SIGTRAP = getCurr<uint64_t>();
// ud2 = SIGILL
// int3 = SIGTRAP
// hlt = SIGSEGV
int3();
}
{
// Guest SIGSEGV handler
// Needs to be distinct from the SignalHandlerReturnAddress
GuestSignal_SIGSEGV = getCurr<uint64_t>();
// ud2 = SIGILL
// int3 = SIGTRAP
// hlt = SIGSEGV
hlt();
}
{
IntCallbackReturnAddress = getCurr<uint64_t>();
// using CallbackReturn = FEX_NAKED void(*)(FEXCore::Core::InternalThreadState *Thread, volatile void *Host_RSP);
// rdi = thread
// rsi = rsp
mov(rsp, rsi);
// Now jump back to the thunk
// XXX: XMM?
add(rsp, 8);
pop(r15);
pop(r14);
pop(r13);
pop(r12);
pop(rbp);
pop(rbx);
ret();
}
ready();
Start = reinterpret_cast<uint64_t>(getCode());
End = Start + getSize();
if (CTX->Config.BlockJITNaming()) {
fextl::string Name = fextl::fmt::format("Dispatch_{}", FHU::Syscalls::gettid());
CTX->Symbols.RegisterNamedRegion(reinterpret_cast<void*>(Start), End-Start, Name);
}
if (CTX->Config.GlobalJITNaming()) {
CTX->Symbols.RegisterJITSpace(reinterpret_cast<void*>(Start), End-Start);
}
}
size_t X86Dispatcher::GenerateGDBPauseCheck(uint8_t *CodeBuffer, uint64_t GuestRIP) {
using namespace Xbyak;
using namespace Xbyak::util;
Xbyak::CodeGenerator emit(1, &emit); // actual emit target set with setNewBuffer
emit.setNewBuffer(CodeBuffer, MaxGDBPauseCheckSize);
Label RunBlock;
// If we have a gdb server running then run in a less efficient mode that checks if we need to exit
// This happens when single stepping
static_assert(sizeof(CTX->Config.RunningMode) == 4, "This is expected to be size of 4");
emit.mov(rax, reinterpret_cast<uint64_t>(CTX));
// If the value == 0 then we don't need to stop
emit.cmp(dword [rax + (offsetof(FEXCore::Context::ContextImpl, Config.RunningMode))], 0);
emit.je(RunBlock);
{
// Make sure RIP is syncronized to the context
emit.mov(rax, GuestRIP);
emit.mov(qword STATE_PTR(CpuStateFrame, State.rip), rax);
// Stop the thread
emit.mov(rax, qword STATE_PTR(CpuStateFrame, Pointers.Common.ThreadPauseHandlerSpillSRA));
emit.jmp(rax);
}
emit.L(RunBlock);
emit.ready();
return emit.getSize();
}
X86Dispatcher::~X86Dispatcher() {
FEXCore::Allocator::VirtualFree(top_, MAX_DISPATCHER_CODE_SIZE);
}
void X86Dispatcher::InitThreadPointers(FEXCore::Core::InternalThreadState *Thread) {
// Setup dispatcher specific pointers that need to be accessed from JIT code
{
auto &Common = Thread->CurrentFrame->Pointers.Common;
Common.DispatcherLoopTop = AbsoluteLoopTopAddress;
Common.DispatcherLoopTopFillSRA = AbsoluteLoopTopAddressFillSRA;
Common.ExitFunctionLinker = ExitFunctionLinkerAddress;
Common.ThreadStopHandlerSpillSRA = ThreadStopHandlerAddress;
Common.ThreadPauseHandlerSpillSRA = ThreadPauseHandlerAddress;
Common.GuestSignal_SIGILL = GuestSignal_SIGILL;
Common.GuestSignal_SIGTRAP = GuestSignal_SIGTRAP;
Common.GuestSignal_SIGSEGV = GuestSignal_SIGSEGV;
Common.SignalReturnHandler = SignalHandlerReturnAddress;
Common.SignalReturnHandlerRT = SignalHandlerReturnAddressRT;
}
}
fextl::unique_ptr<Dispatcher> Dispatcher::CreateX86(FEXCore::Context::ContextImpl *CTX, const DispatcherConfig &Config) {
return fextl::make_unique<X86Dispatcher>(CTX, Config);
}
}
FEXCore/Source/Interface/Core/Dispatcher/X86Dispatcher.h
-42
View File
@@ -1,42 +0,0 @@
// SPDX-License-Identifier: MIT
#pragma once
#include <FEXCore/fextl/list.h>
#include <FEXCore/fextl/unordered_map.h>
#include <FEXCore/fextl/unordered_set.h>
#include "Interface/Core/Dispatcher/Dispatcher.h"
#define XBYAK64
#define XBYAK_CUSTOM_ALLOC
#define XBYAK_CUSTOM_MALLOC FEXCore::Allocator::malloc
#define XBYAK_CUSTOM_FREE FEXCore::Allocator::free
#define XBYAK_CUSTOM_SETS
#define XBYAK_STD_UNORDERED_SET fextl::unordered_set
#define XBYAK_STD_UNORDERED_MAP fextl::unordered_map
#define XBYAK_STD_UNORDERED_MULTIMAP fextl::unordered_multimap
#define XBYAK_STD_LIST fextl::list
#define XBYAK_NO_EXCEPTION
#include <xbyak/xbyak.h>
#include <xbyak/xbyak_util.h>
namespace FEXCore::Core {
struct InternalThreadState;
}
#define STATE_PTR(STATE_TYPE, FIELD) \
[STATE + offsetof(FEXCore::Core::STATE_TYPE, FIELD)]
namespace FEXCore::CPU {
class X86Dispatcher final : public Dispatcher, public Xbyak::CodeGenerator {
public:
X86Dispatcher(FEXCore::Context::ContextImpl *ctx, const DispatcherConfig &config);
void InitThreadPointers(FEXCore::Core::InternalThreadState *Thread) override;
size_t GenerateGDBPauseCheck(uint8_t *CodeBuffer, uint64_t GuestRIP) override;
virtual ~X86Dispatcher() override;
};
}
FEXCore/Source/Interface/Core/HostFeatures.cpp
+43 -34
View File
@@ -2,15 +2,28 @@
#include "Interface/Core/CPUID.h"
#include <FEXCore/Core/HostFeatures.h>
#if defined(_M_ARM_64) || defined(VIXL_SIMULATOR)
#include "aarch64/assembler-aarch64.h"
#include "aarch64/cpu-aarch64.h"
#include "aarch64/disasm-aarch64.h"
#include "aarch64/assembler-aarch64.h"
#endif
#ifdef _M_X86_64
#include "Interface/Core/Dispatcher/X86Dispatcher.h"
#define XBYAK64
#define XBYAK_CUSTOM_ALLOC
#define XBYAK_CUSTOM_MALLOC FEXCore::Allocator::malloc
#define XBYAK_CUSTOM_FREE FEXCore::Allocator::free
#define XBYAK_CUSTOM_SETS
#define XBYAK_STD_UNORDERED_SET fextl::unordered_set
#define XBYAK_STD_UNORDERED_MAP fextl::unordered_map
#define XBYAK_STD_UNORDERED_MULTIMAP fextl::unordered_multimap
#define XBYAK_STD_LIST fextl::list
#define XBYAK_NO_EXCEPTION
#include <FEXCore/fextl/list.h>
#include <FEXCore/fextl/unordered_map.h>
#include <FEXCore/fextl/unordered_set.h>
#include <xbyak/xbyak.h>
#include <xbyak/xbyak_util.h>
#endif
namespace FEXCore {
@@ -41,7 +54,6 @@ static void SetFPCR(uint64_t Value) {
__asm ("msr FPCR, %[Value]"
:: [Value] "r" (Value));
}
#else
static uint32_t GetDCZID() {
// Return unsupported
@@ -190,17 +202,15 @@ static void OverrideFeatures(HostFeatures *Features) {
}
HostFeatures::HostFeatures() {
#if defined(_M_ARM_64) || defined(VIXL_SIMULATOR)
#ifdef VIXL_SIMULATOR
auto Features = vixl::CPUFeatures::All();
#else
#ifndef _WIN32
#elif !defined(_WIN32)
auto Features = vixl::CPUFeatures::InferFromOS();
#else
// Need to use ID registers in WINE.
auto Features = vixl::CPUFeatures::InferFromIDRegisters();
#endif
#endif
SupportsAES = Features.Has(vixl::CPUFeatures::Feature::kAES);
SupportsCRC = Features.Has(vixl::CPUFeatures::Feature::kCRC32);
SupportsAtomics = Features.Has(vixl::CPUFeatures::Feature::kAtomics);
@@ -266,22 +276,36 @@ HostFeatures::HostFeatures() {
SetFPCR(OriginalFPCR);
#endif
#ifdef VIXL_SIMULATOR
// simulator doesn't support dc(ZVA)
SupportsCLZERO = false;
#else
// Check if we can support cacheline clears
uint32_t DCZID = GetDCZID();
if ((DCZID & DCZID_DZP_MASK) == 0) {
uint32_t DCZID_Log2 = DCZID & DCZID_BS_MASK;
uint32_t DCZID_Bytes = (1 << DCZID_Log2) * sizeof(uint32_t);
// If the DC ZVA size matches the emulated cache line size
// This means we can use the instruction
SupportsCLZERO = DCZID_Bytes == CPUIDEmu::CACHELINE_SIZE;
}
#endif
#if defined(_M_X86_64) && !defined(VIXL_SIMULATOR)
Xbyak::util::Cpu Features{};
SupportsAES = Features.has(Xbyak::util::Cpu::tAESNI);
SupportsCRC = Features.has(Xbyak::util::Cpu::tSSE42);
SupportsRAND = Features.has(Xbyak::util::Cpu::tRDRAND) && Features.has(Xbyak::util::Cpu::tRDSEED);
Xbyak::util::Cpu X86Features{};
SupportsAES = X86Features.has(Xbyak::util::Cpu::tAESNI);
SupportsCRC = X86Features.has(Xbyak::util::Cpu::tSSE42);
SupportsRAND = X86Features.has(Xbyak::util::Cpu::tRDRAND) && X86Features.has(Xbyak::util::Cpu::tRDSEED);
SupportsRCPC = true;
SupportsTSOImm9 = true;
Supports3DNow = Features.has(Xbyak::util::Cpu::t3DN) && Features.has(Xbyak::util::Cpu::tE3DN);
SupportsSSE4A = Features.has(Xbyak::util::Cpu::tSSE4a);
Supports3DNow = X86Features.has(Xbyak::util::Cpu::t3DN) && X86Features.has(Xbyak::util::Cpu::tE3DN);
SupportsSSE4A = X86Features.has(Xbyak::util::Cpu::tSSE4a);
SupportsAVX = true;
SupportsSHA = Features.has(Xbyak::util::Cpu::tSHA);
SupportsBMI1 = Features.has(Xbyak::util::Cpu::tBMI1);
SupportsBMI2 = Features.has(Xbyak::util::Cpu::tBMI2);
SupportsCLWB = Features.has(Xbyak::util::Cpu::tCLWB);
SupportsPMULL_128Bit = Features.has(Xbyak::util::Cpu::tPCLMULQDQ);
SupportsSHA = X86Features.has(Xbyak::util::Cpu::tSHA);
SupportsBMI1 = X86Features.has(Xbyak::util::Cpu::tBMI1);
SupportsBMI2 = X86Features.has(Xbyak::util::Cpu::tBMI2);
SupportsCLWB = X86Features.has(Xbyak::util::Cpu::tCLWB);
SupportsPMULL_128Bit = X86Features.has(Xbyak::util::Cpu::tPCLMULQDQ);
// xbyak doesn't know how to check for CLZero
// First ensure we support a new enough extended CPUID function range
@@ -297,21 +321,6 @@ HostFeatures::HostFeatures() {
SupportsFlushInputsToZero = true;
SupportsFloatExceptions = true;
#endif
#ifdef VIXL_SIMULATOR
// simulator doesn't support dc(ZVA)
SupportsCLZERO = false;
#else
// Check if we can support cacheline clears
uint32_t DCZID = GetDCZID();
if ((DCZID & DCZID_DZP_MASK) == 0) {
uint32_t DCZID_Log2 = DCZID & DCZID_BS_MASK;
uint32_t DCZID_Bytes = (1 << DCZID_Log2) * sizeof(uint32_t);
// If the DC ZVA size matches the emulated cache line size
// This means we can use the instruction
SupportsCLZERO = DCZID_Bytes == CPUIDEmu::CACHELINE_SIZE;
}
#endif
OverrideFeatures(this);
}
}
FEXCore/Source/Interface/Core/JIT/x86_64/ALUOps.cpp
-1633
View File
File diff suppressed because it is too large. Load diff
FEXCore/Source/Interface/Core/JIT/x86_64/AtomicOps.cpp
-755
View File
@@ -1,755 +0,0 @@
// SPDX-License-Identifier: MIT
/*
$info$
tags: backend|x86-64
$end_info$
*/
#include "Interface/Core/JIT/x86_64/JITClass.h"
#include "Interface/Core/Dispatcher/X86Dispatcher.h"
#include <FEXCore/IR/IR.h>
#include <FEXCore/Utils/LogManager.h>
#include <array>
#include <stdint.h>
#include <utility>
namespace FEXCore::CPU {
#define DEF_OP(x) void X86JITCore::Op_##x(IR::IROp_Header *IROp, IR::NodeID Node)
DEF_OP(CASPair) {
auto Op = IROp->C<IR::IROp_CAS>();
// DataSrc = *Src1
// if (DataSrc == Src3) { *Src1 == Src2; } Src2 = DataSrc
// This will write to memory! Careful!
// Third operand must be a calculated guest memory address
//OrderedNode *CASResult = _CAS(Src3, Src2, Src1);
auto Dst = GetSrcPair<RA_64>(Node);
auto Expected = GetSrcPair<RA_64>(Op->Expected.ID());
auto Desired = GetSrcPair<RA_64>(Op->Desired.ID());
auto MemSrc = GetSrc<RA_64>(Op->Addr.ID());
Xbyak::Reg MemReg = MemSrc;
mov(rax, Expected.first);
mov(rdx, Expected.second);
mov(rbx, Desired.first);
mov(rcx, Desired.second);
// RDI(Or Source) now contains pointer
// RDX:RAX contains our expected value
// RCX:RBX contains our desired
lock();
switch (IROp->ElementSize) {
case 4: {
cmpxchg8b(dword [MemReg]);
// EDX:EAX now contains the result
mov(Dst.first.cvt32(), eax);
mov(Dst.second.cvt32(), edx);
break;
}
case 8: {
cmpxchg16b(qword [MemReg]);
// RDX:RAX now contains the result
mov(Dst.first, rax);
mov(Dst.second, rdx);
break;
}
default: LOGMAN_MSG_A_FMT("Unsupported: {}", IROp->ElementSize);
}
}
DEF_OP(CAS) {
auto Op = IROp->C<IR::IROp_CAS>();
uint8_t OpSize = IROp->Size;
// DataSrc = *Src1
// if (DataSrc == Src3) { *Src1 == Src2; } Src2 = DataSrc
// This will write to memory! Careful!
// Third operand must be a calculated guest memory address
//OrderedNode *CASResult = _CAS(Src3, Src2, Src1);
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Addr.ID());
mov(rax, GetSrc<RA_64>(Op->Expected.ID()));
// RCX now contains pointer
// RAX contains our expected value
// RDX contains our desired
lock();
switch (OpSize) {
case 1: {
cmpxchg(byte [MemReg], GetSrc<RA_8>(Op->Desired.ID()));
movzx(GetDst<RA_64>(Node), al);
break;
}
case 2: {
cmpxchg(word [MemReg], GetSrc<RA_16>(Op->Desired.ID()));
movzx(GetDst<RA_64>(Node), ax);
break;
}
case 4: {
cmpxchg(dword [MemReg], GetSrc<RA_32>(Op->Desired.ID()));
// RAX now contains the result
mov (GetDst<RA_64>(Node), eax);
break;
}
case 8: {
cmpxchg(qword [MemReg], GetSrc<RA_64>(Op->Desired.ID()));
// RAX now contains the result
mov (GetDst<RA_64>(Node), rax);
break;
}
default: LOGMAN_MSG_A_FMT("Unsupported: {}", OpSize);
}
}
DEF_OP(AtomicAdd) {
auto Op = IROp->C<IR::IROp_AtomicAdd>();
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Addr.ID());
lock();
switch (IROp->Size) {
case 1:
add(byte [MemReg], GetSrc<RA_8>(Op->Value.ID()));
break;
case 2:
add(word [MemReg], GetSrc<RA_16>(Op->Value.ID()));
break;
case 4:
add(dword [MemReg], GetSrc<RA_32>(Op->Value.ID()));
break;
case 8:
add(qword [MemReg], GetSrc<RA_64>(Op->Value.ID()));
break;
default: LOGMAN_MSG_A_FMT("Unhandled AtomicAdd size: {}", IROp->Size);
}
}
DEF_OP(AtomicSub) {
auto Op = IROp->C<IR::IROp_AtomicSub>();
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Addr.ID());
lock();
switch (IROp->Size) {
case 1:
sub(byte [MemReg], GetSrc<RA_8>(Op->Value.ID()));
break;
case 2:
sub(word [MemReg], GetSrc<RA_16>(Op->Value.ID()));
break;
case 4:
sub(dword [MemReg], GetSrc<RA_32>(Op->Value.ID()));
break;
case 8:
sub(qword [MemReg], GetSrc<RA_64>(Op->Value.ID()));
break;
default: LOGMAN_MSG_A_FMT("Unhandled AtomicAdd size: {}", IROp->Size);
}
}
DEF_OP(AtomicAnd) {
auto Op = IROp->C<IR::IROp_AtomicAnd>();
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Addr.ID());
lock();
switch (IROp->Size) {
case 1:
and_(byte [MemReg], GetSrc<RA_8>(Op->Value.ID()));
break;
case 2:
and_(word [MemReg], GetSrc<RA_16>(Op->Value.ID()));
break;
case 4:
and_(dword [MemReg], GetSrc<RA_32>(Op->Value.ID()));
break;
case 8:
and_(qword [MemReg], GetSrc<RA_64>(Op->Value.ID()));
break;
default: LOGMAN_MSG_A_FMT("Unhandled AtomicAdd size: {}", IROp->Size);
}
}
DEF_OP(AtomicOr) {
auto Op = IROp->C<IR::IROp_AtomicOr>();
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Addr.ID());
lock();
switch (IROp->Size) {
case 1:
or_(byte [MemReg], GetSrc<RA_8>(Op->Value.ID()));
break;
case 2:
or_(word [MemReg], GetSrc<RA_16>(Op->Value.ID()));
break;
case 4:
or_(dword [MemReg], GetSrc<RA_32>(Op->Value.ID()));
break;
case 8:
or_(qword [MemReg], GetSrc<RA_64>(Op->Value.ID()));
break;
default: LOGMAN_MSG_A_FMT("Unhandled AtomicAdd size: {}", IROp->Size);
}
}
DEF_OP(AtomicXor) {
auto Op = IROp->C<IR::IROp_AtomicXor>();
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Addr.ID());
lock();
switch (IROp->Size) {
case 1:
xor_(byte [MemReg], GetSrc<RA_8>(Op->Value.ID()));
break;
case 2:
xor_(word [MemReg], GetSrc<RA_16>(Op->Value.ID()));
break;
case 4:
xor_(dword [MemReg], GetSrc<RA_32>(Op->Value.ID()));
break;
case 8:
xor_(qword [MemReg], GetSrc<RA_64>(Op->Value.ID()));
break;
default: LOGMAN_MSG_A_FMT("Unhandled AtomicAdd size: {}", IROp->Size);
}
}
DEF_OP(AtomicSwap) {
auto Op = IROp->C<IR::IROp_AtomicSwap>();
Xbyak::Reg MemReg = rax;
mov(MemReg, GetSrc<RA_64>(Op->Addr.ID()));
switch (IROp->Size) {
case 1:
movzx(GetDst<RA_64>(Node), GetSrc<RA_8>(Op->Value.ID()));
lock();
xchg(byte [MemReg], GetDst<RA_8>(Node));
break;
case 2:
movzx(GetDst<RA_64>(Node), GetSrc<RA_16>(Op->Value.ID()));
lock();
xchg(word [MemReg], GetDst<RA_16>(Node));
break;
case 4:
mov(GetDst<RA_64>(Node), GetSrc<RA_32>(Op->Value.ID()));
lock();
xchg(dword [MemReg], GetDst<RA_32>(Node));
break;
case 8:
mov(GetDst<RA_64>(Node), GetSrc<RA_64>(Op->Value.ID()));
lock();
xchg(qword [MemReg], GetDst<RA_64>(Node));
break;
default: LOGMAN_MSG_A_FMT("Unhandled AtomicSwap size: {}", IROp->Size);
}
}
DEF_OP(AtomicFetchAdd) {
auto Op = IROp->C<IR::IROp_AtomicFetchAdd>();
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Addr.ID());
switch (IROp->Size) {
case 1:
movzx(rcx, GetSrc<RA_8>(Op->Value.ID()));
lock();
xadd(byte [MemReg], cl);
movzx(GetDst<RA_32>(Node), cl);
break;
case 2:
movzx(rcx, GetSrc<RA_16>(Op->Value.ID()));
lock();
xadd(word [MemReg], cx);
movzx(GetDst<RA_32>(Node), cx);
break;
case 4:
mov(ecx, GetSrc<RA_32>(Op->Value.ID()));
lock();
xadd(dword [MemReg], ecx);
mov(GetDst<RA_64>(Node), ecx);
break;
case 8:
mov(rcx, GetSrc<RA_64>(Op->Value.ID()));
lock();
xadd(qword [MemReg], rcx);
mov(GetDst<RA_64>(Node), rcx);
break;
default: LOGMAN_MSG_A_FMT("Unhandled AtomicFetchAdd size: {}", IROp->Size);
}
}
DEF_OP(AtomicFetchSub) {
auto Op = IROp->C<IR::IROp_AtomicFetchSub>();
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Addr.ID());
switch (IROp->Size) {
case 1:
mov(cl, GetSrc<RA_8>(Op->Value.ID()));
neg(cl);
lock();
xadd(byte [MemReg], cl);
movzx(GetDst<RA_32>(Node), cl);
break;
case 2:
mov(cx, GetSrc<RA_16>(Op->Value.ID()));
neg(cx);
lock();
xadd(word [MemReg], cx);
movzx(GetDst<RA_32>(Node), cx);
break;
case 4:
mov(ecx, GetSrc<RA_32>(Op->Value.ID()));
neg(ecx);
lock();
xadd(dword [MemReg], ecx);
mov(GetDst<RA_32>(Node), ecx);
break;
case 8:
mov(rcx, GetSrc<RA_64>(Op->Value.ID()));
neg(rcx);
lock();
xadd(qword [MemReg], rcx);
mov(GetDst<RA_64>(Node), rcx);
break;
default: LOGMAN_MSG_A_FMT("Unhandled AtomicFetchSub size: {}", IROp->Size);
}
}
DEF_OP(AtomicFetchAnd) {
auto Op = IROp->C<IR::IROp_AtomicFetchAnd>();
// TMP1 = rax
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Addr.ID());
switch (IROp->Size) {
case 1: {
mov(TMP1.cvt8(), byte [MemReg]);
Label Loop;
L(Loop);
mov(TMP2.cvt8(), TMP1.cvt8());
mov(TMP3.cvt8(), TMP1.cvt8());
and_(TMP2.cvt8(), GetSrc<RA_8>(Op->Value.ID()));
// Updates RAX with the value from memory
lock(); cmpxchg(byte [MemReg], TMP2.cvt8());
jne(Loop);
// Result is the previous value from memory, which is currently in TMP3
movzx(GetDst<RA_64>(Node), TMP3.cvt8());
break;
}
case 2: {
mov(TMP1.cvt16(), word [MemReg]);
Label Loop;
L(Loop);
mov(TMP2.cvt16(), TMP1.cvt16());
mov(TMP3.cvt16(), TMP1.cvt16());
and_(TMP2.cvt16(), GetSrc<RA_16>(Op->Value.ID()));
// Updates RAX with the value from memory
lock(); cmpxchg(word [MemReg], TMP2.cvt16());
jne(Loop);
// Result is the previous value from memory, which is currently in TMP3
movzx(GetDst<RA_64>(Node), TMP3.cvt16());
break;
}
case 4: {
mov(TMP1.cvt32(), dword [MemReg]);
Label Loop;
L(Loop);
mov(TMP2.cvt32(), TMP1.cvt32());
mov(TMP3.cvt32(), TMP1.cvt32());
and_(TMP2.cvt32(), GetSrc<RA_32>(Op->Value.ID()));
// Updates RAX with the value from memory
lock(); cmpxchg(dword [MemReg], TMP2.cvt32());
jne(Loop);
// Result is the previous value from memory, which is currently in TMP3
mov(GetDst<RA_32>(Node), TMP3.cvt32());
break;
}
case 8: {
mov(TMP1.cvt64(), qword [MemReg]);
Label Loop;
L(Loop);
mov(TMP2.cvt64(), TMP1.cvt64());
mov(TMP3.cvt64(), TMP1.cvt64());
and_(TMP2.cvt64(), GetSrc<RA_64>(Op->Value.ID()));
// Updates RAX with the value from memory
lock(); cmpxchg(qword [MemReg], TMP2.cvt64());
jne(Loop);
// Result is the previous value from memory, which is currently in TMP3
mov(GetDst<RA_64>(Node), TMP3.cvt64());
break;
}
default: LOGMAN_MSG_A_FMT("Unhandled AtomicFetchAnd size: {}", IROp->Size);
}
}
DEF_OP(AtomicFetchCLR) {
auto Op = IROp->C<IR::IROp_AtomicFetchCLR>();
// TMP1 = rax
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Addr.ID());
switch (IROp->Size) {
case 1: {
mov(TMP1.cvt8(), byte [MemReg]);
Label Loop;
L(Loop);
mov(TMP2.cvt8(), TMP1.cvt8());
mov(TMP3.cvt8(), TMP1.cvt8());
mov(TMP4.cvt8(), GetSrc<RA_8>(Op->Value.ID()));
not_(TMP4.cvt8());
and_(TMP2.cvt8(), TMP4.cvt8());
// Updates RAX with the value from memory
lock(); cmpxchg(byte [MemReg], TMP2.cvt8());
jne(Loop);
// Result is the previous value from memory, which is currently in TMP3
movzx(GetDst<RA_64>(Node), TMP3.cvt8());
break;
}
case 2: {
mov(TMP1.cvt16(), word [MemReg]);
Label Loop;
L(Loop);
mov(TMP2.cvt16(), TMP1.cvt16());
mov(TMP3.cvt16(), TMP1.cvt16());
mov(TMP4.cvt16(), GetSrc<RA_16>(Op->Value.ID()));
not_(TMP4.cvt16());
and_(TMP2.cvt16(), TMP4.cvt16());
// Updates RAX with the value from memory
lock(); cmpxchg(word [MemReg], TMP2.cvt16());
jne(Loop);
// Result is the previous value from memory, which is currently in TMP3
movzx(GetDst<RA_64>(Node), TMP3.cvt16());
break;
}
case 4: {
mov(TMP1.cvt32(), dword [MemReg]);
Label Loop;
L(Loop);
mov(TMP2.cvt32(), TMP1.cvt32());
mov(TMP3.cvt32(), TMP1.cvt32());
mov(TMP4.cvt32(), GetSrc<RA_32>(Op->Value.ID()));
not_(TMP4.cvt32());
and_(TMP2.cvt32(), TMP4.cvt32());
// Updates RAX with the value from memory
lock(); cmpxchg(dword [MemReg], TMP2.cvt32());
jne(Loop);
// Result is the previous value from memory, which is currently in TMP3
mov(GetDst<RA_32>(Node), TMP3.cvt32());
break;
}
case 8: {
mov(TMP1.cvt64(), qword [MemReg]);
Label Loop;
L(Loop);
mov(TMP2.cvt64(), TMP1.cvt64());
mov(TMP3.cvt64(), TMP1.cvt64());
mov(TMP4.cvt64(), GetSrc<RA_64>(Op->Value.ID()));
not_(TMP4.cvt64());
and_(TMP2.cvt64(), TMP4.cvt64());
// Updates RAX with the value from memory
lock(); cmpxchg(qword [MemReg], TMP2.cvt64());
jne(Loop);
// Result is the previous value from memory, which is currently in TMP3
mov(GetDst<RA_64>(Node), TMP3.cvt64());
break;
}
default: LOGMAN_MSG_A_FMT("Unhandled AtomicFetchAnd size: {}", IROp->Size);
}
}
DEF_OP(AtomicFetchOr) {
auto Op = IROp->C<IR::IROp_AtomicFetchOr>();
// TMP1 = rax
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Addr.ID());
switch (IROp->Size) {
case 1: {
mov(TMP1.cvt8(), byte [MemReg]);
Label Loop;
L(Loop);
mov(TMP2.cvt8(), TMP1.cvt8());
mov(TMP3.cvt8(), TMP1.cvt8());
or_(TMP2.cvt8(), GetSrc<RA_8>(Op->Value.ID()));
// Updates RAX with the value from memory
lock(); cmpxchg(byte [MemReg], TMP2.cvt8());
jne(Loop);
// Result is the previous value from memory, which is currently in TMP3
movzx(GetDst<RA_64>(Node), TMP3.cvt8());
break;
}
case 2: {
mov(TMP1.cvt16(), word [MemReg]);
Label Loop;
L(Loop);
mov(TMP2.cvt16(), TMP1.cvt16());
mov(TMP3.cvt16(), TMP1.cvt16());
or_(TMP2.cvt16(), GetSrc<RA_16>(Op->Value.ID()));
// Updates RAX with the value from memory
lock(); cmpxchg(word [MemReg], TMP2.cvt16());
jne(Loop);
// Result is the previous value from memory, which is currently in TMP3
movzx(GetDst<RA_64>(Node), TMP3.cvt16());
break;
}
case 4: {
mov(TMP1.cvt32(), dword [MemReg]);
Label Loop;
L(Loop);
mov(TMP2.cvt32(), TMP1.cvt32());
mov(TMP3.cvt32(), TMP1.cvt32());
or_(TMP2.cvt32(), GetSrc<RA_32>(Op->Value.ID()));
// Updates RAX with the value from memory
lock(); cmpxchg(dword [MemReg], TMP2.cvt32());
jne(Loop);
// Result is the previous value from memory, which is currently in TMP3
mov(GetDst<RA_32>(Node), TMP3.cvt32());
break;
}
case 8: {
mov(TMP1.cvt64(), qword [MemReg]);
Label Loop;
L(Loop);
mov(TMP2.cvt64(), TMP1.cvt64());
mov(TMP3.cvt64(), TMP1.cvt64());
or_(TMP2.cvt64(), GetSrc<RA_64>(Op->Value.ID()));
// Updates RAX with the value from memory
lock(); cmpxchg(qword [MemReg], TMP2.cvt64());
jne(Loop);
// Result is the previous value from memory, which is currently in TMP3
mov(GetDst<RA_64>(Node), TMP3.cvt64());
break;
}
default: LOGMAN_MSG_A_FMT("Unhandled AtomicFetchOr size: {}", IROp->Size);
}
}
DEF_OP(AtomicFetchXor) {
auto Op = IROp->C<IR::IROp_AtomicFetchXor>();
// TMP1 = rax
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Addr.ID());
switch (IROp->Size) {
case 1: {
mov(TMP1.cvt8(), byte [MemReg]);
Label Loop;
L(Loop);
mov(TMP2.cvt8(), TMP1.cvt8());
mov(TMP3.cvt8(), TMP1.cvt8());
xor_(TMP2.cvt8(), GetSrc<RA_8>(Op->Value.ID()));
// Updates RAX with the value from memory
lock(); cmpxchg(byte [MemReg], TMP2.cvt8());
jne(Loop);
// Result is the previous value from memory, which is currently in TMP3
movzx(GetDst<RA_64>(Node), TMP3.cvt8());
break;
}
case 2: {
mov(TMP1.cvt16(), word [MemReg]);
Label Loop;
L(Loop);
mov(TMP2.cvt16(), TMP1.cvt16());
mov(TMP3.cvt16(), TMP1.cvt16());
xor_(TMP2.cvt16(), GetSrc<RA_16>(Op->Value.ID()));
// Updates RAX with the value from memory
lock(); cmpxchg(word [MemReg], TMP2.cvt16());
jne(Loop);
// Result is the previous value from memory, which is currently in TMP3
movzx(GetDst<RA_64>(Node), TMP3.cvt16());
break;
}
case 4: {
mov(TMP1.cvt32(), dword [MemReg]);
Label Loop;
L(Loop);
mov(TMP2.cvt32(), TMP1.cvt32());
mov(TMP3.cvt32(), TMP1.cvt32());
xor_(TMP2.cvt32(), GetSrc<RA_32>(Op->Value.ID()));
// Updates RAX with the value from memory
lock(); cmpxchg(dword [MemReg], TMP2.cvt32());
jne(Loop);
// Result is the previous value from memory, which is currently in TMP3
mov(GetDst<RA_32>(Node), TMP3.cvt32());
break;
}
case 8: {
mov(TMP1.cvt64(), qword [MemReg]);
Label Loop;
L(Loop);
mov(TMP2.cvt64(), TMP1.cvt64());
mov(TMP3.cvt64(), TMP1.cvt64());
xor_(TMP2.cvt64(), GetSrc<RA_64>(Op->Value.ID()));
// Updates RAX with the value from memory
lock(); cmpxchg(qword [MemReg], TMP2.cvt64());
jne(Loop);
// Result is the previous value from memory, which is currently in TMP3
mov(GetDst<RA_64>(Node), TMP3.cvt64());
break;
}
default: LOGMAN_MSG_A_FMT("Unhandled AtomicFetchXor size: {}", IROp->Size);
}
}
DEF_OP(AtomicFetchNeg) {
auto Op = IROp->C<IR::IROp_AtomicFetchNeg>();
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Addr.ID());
switch (IROp->Size) {
case 1: {
mov(TMP1.cvt8(), byte [MemReg]);
Label Loop;
L(Loop);
mov(TMP2.cvt8(), TMP1.cvt8());
mov(TMP3.cvt8(), TMP1.cvt8());
neg(TMP2.cvt8());
// Updates RAX with the value from memory
lock(); cmpxchg(byte [MemReg], TMP2.cvt8());
jne(Loop);
// Result is the previous value from memory, which is currently in TMP3
movzx(GetDst<RA_64>(Node), TMP3.cvt8());
break;
}
case 2: {
mov(TMP1.cvt16(), word [MemReg]);
Label Loop;
L(Loop);
mov(TMP2.cvt16(), TMP1.cvt16());
mov(TMP3.cvt16(), TMP1.cvt16());
neg(TMP2.cvt16());
// Updates RAX with the value from memory
lock(); cmpxchg(word [MemReg], TMP2.cvt16());
jne(Loop);
// Result is the previous value from memory, which is currently in TMP3
movzx(GetDst<RA_64>(Node), TMP3.cvt16());
break;
}
case 4: {
mov(TMP1.cvt32(), dword [MemReg]);
Label Loop;
L(Loop);
mov(TMP2.cvt32(), TMP1.cvt32());
mov(TMP3.cvt32(), TMP1.cvt32());
neg(TMP2.cvt32());
// Updates RAX with the value from memory
lock(); cmpxchg(dword [MemReg], TMP2.cvt32());
jne(Loop);
// Result is the previous value from memory, which is currently in TMP3
mov(GetDst<RA_32>(Node), TMP3.cvt32());
break;
}
case 8: {
mov(TMP1.cvt64(), qword [MemReg]);
Label Loop;
L(Loop);
mov(TMP2.cvt64(), TMP1.cvt64());
mov(TMP3.cvt64(), TMP1.cvt64());
neg(TMP2.cvt64());
// Updates RAX with the value from memory
lock(); cmpxchg(qword [MemReg], TMP2.cvt64());
jne(Loop);
// Result is the previous value from memory, which is currently in TMP3
mov(GetDst<RA_64>(Node), TMP3.cvt64());
break;
}
default: LOGMAN_MSG_A_FMT("Unhandled AtomicFetchNeg size: {}", IROp->Size);
}
}
DEF_OP(TelemetrySetValue) {
#ifndef FEX_DISABLE_TELEMETRY
auto Op = IROp->C<IR::IROp_TelemetrySetValue>();
auto Src = GetSrc<RA_32>(Op->Value.ID());
xor_(TMP1, TMP1);
mov(TMP2, qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.TelemetryValueAddresses[Op->TelemetryValueIndex])]);
test(Src, Src);
setne(TMP1.cvt8());
lock(); or_(qword [TMP2], TMP1);
#endif
}
#undef DEF_OP
void X86JITCore::RegisterAtomicHandlers() {
#define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &X86JITCore::Op_##x
REGISTER_OP(CASPAIR, CASPair);
REGISTER_OP(CAS, CAS);
REGISTER_OP(ATOMICADD, AtomicAdd);
REGISTER_OP(ATOMICSUB, AtomicSub);
REGISTER_OP(ATOMICAND, AtomicAnd);
REGISTER_OP(ATOMICOR, AtomicOr);
REGISTER_OP(ATOMICXOR, AtomicXor);
REGISTER_OP(ATOMICSWAP, AtomicSwap);
REGISTER_OP(ATOMICFETCHADD, AtomicFetchAdd);
REGISTER_OP(ATOMICFETCHSUB, AtomicFetchSub);
REGISTER_OP(ATOMICFETCHAND, AtomicFetchAnd);
REGISTER_OP(ATOMICFETCHCLR, AtomicFetchCLR);
REGISTER_OP(ATOMICFETCHOR, AtomicFetchOr);
REGISTER_OP(ATOMICFETCHXOR, AtomicFetchXor);
REGISTER_OP(ATOMICFETCHNEG, AtomicFetchNeg);
REGISTER_OP(TELEMETRYSETVALUE, TelemetrySetValue);
#undef REGISTER_OP
}
}
FEXCore/Source/Interface/Core/JIT/x86_64/BranchOps.cpp
-362
View File
@@ -1,362 +0,0 @@
// SPDX-License-Identifier: MIT
/*
$info$
tags: backend|x86-64
$end_info$
*/
#include "Interface/Context/Context.h"
#include "Interface/Core/CPUID.h"
#include "Interface/Core/Dispatcher/Dispatcher.h"
#include "Interface/Core/Dispatcher/X86Dispatcher.h"
#include "Interface/Core/LookupCache.h"
#include "Interface/Core/JIT/x86_64/JITClass.h"
#include "Interface/HLE/Thunks/Thunks.h"
#include <FEXCore/Core/CPUID.h>
#include <FEXCore/Core/CoreState.h>
#include <FEXCore/Core/X86Enums.h>
#include <FEXCore/Debug/InternalThreadState.h>
#include <FEXCore/HLE/SyscallHandler.h>
#include <FEXCore/IR/IR.h>
#include <FEXCore/Utils/LogManager.h>
#include <array>
#include <memory>
#include <stddef.h>
#include <stdint.h>
#include <unordered_map>
#include <utility>
namespace FEXCore::CPU {
#define DEF_OP(x) void X86JITCore::Op_##x(IR::IROp_Header *IROp, IR::NodeID Node)
DEF_OP(CallbackReturn) {
// Adjust the stack first for a regular return
if (SpillSlots) {
add(rsp, SpillSlots * MaxSpillSlotSize); // + 8 to consume return address
}
// Make sure to adjust the refcounter so we don't clear the cache now
sub(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, SignalHandlerRefCounter)], 1);
// We need to adjust an additional 8 bytes to get back to the original "misaligned" RSP state
add(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RSP])], 8);
// Now jump back to the thunk
// XXX: XMM?
add(rsp, 8);
pop(r15);
pop(r14);
pop(r13);
pop(r12);
pop(rbp);
pop(rbx);
ret();
}
DEF_OP(ExitFunction) {
Label FullLookup;
auto Op = IROp->C<IR::IROp_ExitFunction>();
if (SpillSlots) {
add(rsp, SpillSlots * MaxSpillSlotSize);
}
uint64_t NewRIP;
if (IsInlineConstant(Op->NewRIP, &NewRIP) || IsInlineEntrypointOffset(Op->NewRIP, &NewRIP)) {
Label l_BranchHost;
Label l_BranchGuest;
lea(rax, ptr[rip + l_BranchHost]);
jmp(qword[rax]);
L(l_BranchHost);
//FEX_TODO(this is not per thread)
dq(ThreadState->CurrentFrame->Pointers.Common.ExitFunctionLinker);
L(l_BranchGuest);
dq(NewRIP);
} else {
Xbyak::Reg RipReg = GetSrc<RA_64>(Op->NewRIP.ID());
// L1 Cache
mov(rcx, qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.L1Pointer)]);
mov(rax, RipReg);
and_(rax, LookupCache::L1_ENTRIES_MASK);
shl(rax, 4);
Xbyak::RegExp LookupBase = rcx + rax;
cmp(qword[LookupBase + 8], RipReg);
jne(FullLookup);
jmp(qword[LookupBase + 0]);
L(FullLookup);
mov(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, State.rip)], RipReg);
jmp(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.DispatcherLoopTop)]);
}
#ifdef BLOCKSTATS
ExitBlock();
#endif
}
DEF_OP(Jump) {
const auto Op = IROp->C<IR::IROp_Jump>();
const auto Target = Op->TargetBlock.ID();
PendingTargetLabel = &JumpTargets.try_emplace(Target).first->second;
}
#define GRCMP(Node) (Op->CompareSize == 4 ? GetSrc<RA_32>(Node) : GetSrc<RA_64>(Node))
DEF_OP(CondJump) {
auto Op = IROp->C<IR::IROp_CondJump>();
Label *TrueTargetLabel = &JumpTargets.try_emplace(Op->TrueBlock.ID()).first->second;
if (IsGPR(Op->Cmp1.ID())) {
uint64_t Const;
if (IsInlineConstant(Op->Cmp2, &Const)) {
cmp(GRCMP(Op->Cmp1.ID()), Const);
} else {
cmp(GRCMP(Op->Cmp1.ID()), GRCMP(Op->Cmp2.ID()));
}
} else if (IsFPR(Op->Cmp1.ID())) {
if (Op->CompareSize == 4) {
ucomiss(GetSrc(Op->Cmp1.ID()), GetSrc(Op->Cmp2.ID()));
} else {
ucomisd(GetSrc(Op->Cmp1.ID()), GetSrc(Op->Cmp2.ID()));
}
}
auto [_, __, JCC] = GetCC(Op->Cond);
(this->*JCC)(*TrueTargetLabel, T_NEAR);
PendingTargetLabel = &JumpTargets.try_emplace(Op->FalseBlock.ID()).first->second;
}
DEF_OP(Syscall) {
auto Op = IROp->C<IR::IROp_Syscall>();
// XXX: This is very terrible, but I don't care for right now
FEXCore::IR::SyscallFlags Flags = Op->Flags;
auto NumPush = RA64.size();
for (auto &Reg : RA64)
push(Reg);
// Syscall ABI for x86-64
// this: rdi
// Thread: rsi
// ArgPointer: rdx (Stack)
//
// Result: RAX
// These are pushed in reverse order because stacks
for (uint32_t i = FEXCore::HLE::SyscallArguments::MAX_ARGS; i > 0; --i) {
if (Op->Header.Args[i - 1].IsInvalid()) continue;
push(GetSrc<RA_64>(Op->Header.Args[i - 1].ID()));
++NumPush;
}
mov(rsi, STATE); // Move thread in to rsi
mov(rdi, qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.SyscallHandlerObj)]);
mov(rdx, rsp);
if (NumPush & 1)
sub(rsp, 8); // Align
// {rdi, rsi, rdx}
call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.SyscallHandlerFunc)]);
if (NumPush & 1)
add(rsp, 8); // Align
// Reload arguments just in case they are sill live after the fact
for (uint32_t i = 0; i < FEXCore::HLE::SyscallArguments::MAX_ARGS; ++i) {
if (Op->Header.Args[i].IsInvalid()) continue;
pop(GetSrc<RA_64>(Op->Header.Args[i].ID()));
}
for (uint32_t i = RA64.size(); i > 0; --i)
pop(RA64[i - 1]);
if ((Flags & FEXCore::IR::SyscallFlags::NORETURNEDRESULT) != FEXCore::IR::SyscallFlags::NORETURNEDRESULT) {
// Move result to its destination register.
// Only if `NORETURNEDRESULT` wasn't set, otherwise we might overwrite the CPUState refilled with `FillStaticRegs`
mov (GetDst<RA_64>(Node), rax);
}
}
DEF_OP(Thunk) {
auto Op = IROp->C<IR::IROp_Thunk>();
auto NumPush = RA64.size();
for (auto &Reg : RA64)
push(Reg);
if (NumPush & 1)
sub(rsp, 8); // Align
mov(rdi, GetSrc<RA_64>(Op->ArgPtr.ID()));
auto thunkFn = static_cast<Context::ContextImpl*>(ThreadState->CTX)->ThunkHandler->LookupThunk(Op->ThunkNameHash);
mov(rax, reinterpret_cast<uintptr_t>(thunkFn));
call(rax);
if (NumPush & 1)
add(rsp, 8); // Align
for (uint32_t i = RA64.size(); i > 0; --i)
pop(RA64[i - 1]);
}
DEF_OP(ValidateCode) {
auto Op = IROp->C<IR::IROp_ValidateCode>();
const auto* OldCode = (const uint8_t*)&Op->CodeOriginalLow;
int len = Op->CodeLength;
int idx = 0;
xor_(GetDst<RA_64>(Node), GetDst<RA_64>(Node));
mov(rax, Entry + Op->Offset);
mov(rbx, 1);
while (len >= 4) {
cmp(dword[rax + idx], *(const uint32_t*)(OldCode + idx));
cmovne(GetDst<RA_64>(Node), rbx);
len-=4;
idx+=4;
}
while (len >= 2) {
mov(rcx, *(const uint16_t*)(OldCode + idx));
cmp(word[rax + idx], cx);
cmovne(GetDst<RA_64>(Node), rbx);
len-=2;
idx+=2;
}
while (len >= 1) {
cmp(byte[rax + idx], *(const uint8_t*)(OldCode + idx));
cmovne(GetDst<RA_64>(Node), rbx);
len-=1;
idx+=1;
}
}
DEF_OP(ThreadRemoveCodeEntry) {
auto NumPush = RA64.size();
for (auto &Reg : RA64)
push(Reg);
if (NumPush & 1)
sub(rsp, 8); // Align
mov(rdi, STATE);
mov(rax, Entry); // imm64 move
mov(rsi, rax);
call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.ThreadRemoveCodeEntryFromJIT)]);
if (NumPush & 1)
add(rsp, 8); // Align
for (uint32_t i = RA64.size(); i > 0; --i)
pop(RA64[i - 1]);
}
DEF_OP(CPUID) {
auto Op = IROp->C<IR::IROp_CPUID>();
for (auto &Reg : RA64)
push(Reg);
// CPUID ABI
// this: rdi
// Function: rsi
//
// Result: RAX, RDX. 4xi32
// rsi can be in the source registers, so copy argument to edx first
mov (edx, GetSrc<RA_32>(Op->Leaf.ID()));
mov (esi, GetSrc<RA_32>(Op->Function.ID()));
mov (rdi, qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.CPUIDObj)]);
auto NumPush = RA64.size();
if (NumPush & 1)
sub(rsp, 8); // Align
// {rdi, rsi, rdx}
call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.CPUIDFunction)]);
if (NumPush & 1)
add(rsp, 8); // Align
for (uint32_t i = RA64.size(); i > 0; --i)
pop(RA64[i - 1]);
auto Dst = GetSrcPair<RA_64>(Node);
mov(Dst.first, rax);
mov(Dst.second, rdx);
}
DEF_OP(XGETBV) {
auto Op = IROp->C<IR::IROp_XGetBV>();
for (auto &Reg : RA64)
push(Reg);
// CPUID ABI
// this: rdi
// Function: rsi
//
// Result: RAX, RDX. 4xi32
// rsi can be in the source registers, so copy argument to edx first
mov (esi, GetSrc<RA_32>(Op->Function.ID()));
mov (rdi, qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.CPUIDObj)]);
auto NumPush = RA64.size();
if (NumPush & 1)
sub(rsp, 8); // Align
// {rdi, rsi, rdx}
call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.XCRFunction)]);
if (NumPush & 1)
add(rsp, 8); // Align
for (uint32_t i = RA64.size(); i > 0; --i)
pop(RA64[i - 1]);
auto Dst = GetSrcPair<RA_64>(Node);
mov(Dst.first.cvt32(), eax);
mov(Dst.second, rax);
shr(Dst.second, 32);
}
#undef DEF_OP
void X86JITCore::RegisterBranchHandlers() {
#define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &X86JITCore::Op_##x
REGISTER_OP(CALLBACKRETURN, CallbackReturn);
REGISTER_OP(EXITFUNCTION, ExitFunction);
REGISTER_OP(JUMP, Jump);
REGISTER_OP(CONDJUMP, CondJump);
REGISTER_OP(SYSCALL, Syscall);
REGISTER_OP(THUNK, Thunk);
REGISTER_OP(VALIDATECODE, ValidateCode);
REGISTER_OP(THREADREMOVECODEENTRY, ThreadRemoveCodeEntry);
REGISTER_OP(CPUID, CPUID);
REGISTER_OP(XGETBV, XGETBV);
#undef REGISTER_OP
}
}
FEXCore/Source/Interface/Core/JIT/x86_64/ConversionOps.cpp
-459
View File
@@ -1,459 +0,0 @@
// SPDX-License-Identifier: MIT
/*
$info$
tags: backend|x86-64
$end_info$
*/
#include "Interface/Core/JIT/x86_64/JITClass.h"
#include "Interface/Core/Dispatcher/X86Dispatcher.h"
#include <FEXCore/IR/IR.h>
#include <FEXCore/Utils/LogManager.h>
#include <array>
#include <stdint.h>
namespace FEXCore::CPU {
#define DEF_OP(x) void X86JITCore::Op_##x(IR::IROp_Header *IROp, IR::NodeID Node)
DEF_OP(VInsGPR) {
const auto Op = IROp->C<IR::IROp_VInsGPR>();
const auto OpSize = IROp->Size;
const auto Dst = GetDst(Node);
const auto DestVector = GetSrc(Op->DestVector.ID());
const auto DestIdx = Op->DestIdx;
const auto ElementSize = Op->Header.ElementSize;
const auto ElementSizeBits = ElementSize * 8;
const auto Offset = ElementSizeBits * DestIdx;
constexpr auto SSEBitSize = Core::CPUState::XMM_SSE_REG_SIZE * 8;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto InUpperLane = Offset >= SSEBitSize;
if (InUpperLane && !Is256Bit) {
LOGMAN_MSG_A_FMT("Attempt to access upper 128-bit lane in 128-bit operation! Offset={}",
Offset);
return;
}
if (Is256Bit) {
vmovapd(ToYMM(Dst), ToYMM(DestVector));
} else {
vmovapd(Dst, DestVector);
}
const auto Insert = [&](const Xbyak::Xmm& reg, int index) {
switch (ElementSize) {
case 1: {
if (InUpperLane) {
index -= 16;
}
pinsrb(reg, GetSrc<RA_32>(Op->Src.ID()), index);
break;
}
case 2: {
if (InUpperLane) {
index -= 8;
}
pinsrw(reg, GetSrc<RA_32>(Op->Src.ID()), index);
break;
}
case 4: {
if (InUpperLane) {
index -= 4;
}
pinsrd(reg, GetSrc<RA_32>(Op->Src.ID()), index);
break;
}
case 8: {
if (InUpperLane) {
index -= 2;
}
pinsrq(reg, GetSrc<RA_64>(Op->Src.ID()), index);
break;
}
default:
LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize);
break;
}
};
if (InUpperLane) {
vextracti128(xmm15, ToYMM(Dst), 1);
Insert(xmm15, DestIdx);
vinserti128(ToYMM(Dst), ToYMM(Dst), xmm15, 1);
} else {
Insert(Dst, DestIdx);
}
}
DEF_OP(VCastFromGPR) {
auto Op = IROp->C<IR::IROp_VCastFromGPR>();
switch (Op->Header.ElementSize) {
case 1:
movzx(rax, GetSrc<RA_8>(Op->Src.ID()));
vmovq(GetDst(Node), rax);
break;
case 2:
movzx(rax, GetSrc<RA_16>(Op->Src.ID()));
vmovq(GetDst(Node), rax);
break;
case 4:
vmovd(GetDst(Node), GetSrc<RA_32>(Op->Src.ID()).cvt32());
break;
case 8:
vmovq(GetDst(Node), GetSrc<RA_64>(Op->Src.ID()).cvt64());
break;
default: LOGMAN_MSG_A_FMT("Unknown VCastFromGPR element size: {}", Op->Header.ElementSize);
}
}
DEF_OP(VDupFromGPR) {
const auto Op = IROp->C<IR::IROp_VDupFromGPR>();
const auto OpSize = IROp->Size;
const auto ElementSize = IROp->ElementSize;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto Dst = GetDst(Node);
const auto Src = GetSrc<RA_64>(Op->Src.ID()).cvt64();
vmovq(Dst, Src);
switch (ElementSize) {
case 1:
if (Is256Bit) {
vpbroadcastb(ToYMM(Dst), Dst);
} else {
vpbroadcastb(Dst, Dst);
}
break;
case 2:
if (Is256Bit) {
vpbroadcastw(ToYMM(Dst), Dst);
} else {
vpbroadcastw(Dst, Dst);
}
break;
case 4:
if (Is256Bit) {
vpbroadcastd(ToYMM(Dst), Dst);
} else {
vpbroadcastd(Dst, Dst);
}
break;
case 8:
if (Is256Bit) {
vpbroadcastq(ToYMM(Dst), Dst);
} else {
vpbroadcastq(Dst, Dst);
}
break;
default:
LOGMAN_MSG_A_FMT("Unhandled element size: {}", ElementSize);
return;
}
}
DEF_OP(Float_FromGPR_S) {
const auto Op = IROp->C<IR::IROp_Float_FromGPR_S>();
const uint16_t ElementSize = Op->Header.ElementSize;
const uint16_t Conv = (ElementSize << 8) | Op->SrcElementSize;
switch (Conv) {
case 0x0404: { // Float <- int32_t
cvtsi2ss(GetDst(Node), GetSrc<RA_32>(Op->Src.ID()));
break;
}
case 0x0408: { // Float <- int64_t
cvtsi2ss(GetDst(Node), GetSrc<RA_64>(Op->Src.ID()));
break;
}
case 0x0804: { // Double <- int32_t
cvtsi2sd(GetDst(Node), GetSrc<RA_32>(Op->Src.ID()));
break;
}
case 0x0808: { // Double <- int64_t
cvtsi2sd(GetDst(Node), GetSrc<RA_64>(Op->Src.ID()));
break;
}
default:
LOGMAN_MSG_A_FMT("Unhandled conversion mask: Mask=0x{:04x}, ElementSize={}, SrcElementSize={}",
Conv, ElementSize, Op->SrcElementSize);
break;
}
}
DEF_OP(Float_FToF) {
auto Op = IROp->C<IR::IROp_Float_FToF>();
const uint16_t Conv = (Op->Header.ElementSize << 8) | Op->SrcElementSize;
switch (Conv) {
case 0x0804: { // Double <- Float
cvtss2sd(GetDst(Node), GetSrc(Op->Scalar.ID()));
break;
}
case 0x0408: { // Float <- Double
cvtsd2ss(GetDst(Node), GetSrc(Op->Scalar.ID()));
break;
}
default: LOGMAN_MSG_A_FMT("Unknown Float_FToF sizes: 0x{:x}", Conv);
}
}
DEF_OP(Vector_SToF) {
const auto Op = IROp->C<IR::IROp_Vector_SToF>();
const auto OpSize = IROp->Size;
const auto ElementSize = Op->Header.ElementSize;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto Dst = GetDst(Node);
const auto Vector = GetSrc(Op->Vector.ID());
if (OpSize == ElementSize) {
if (ElementSize == 8) {
vmovq(TMP1, Vector);
cvtsi2sd(Dst, TMP1);
}
else if (ElementSize == 4) {
vmovd(TMP1.cvt32(), Vector);
cvtsi2ss(Dst, TMP1.cvt32());
}
else {
LOGMAN_MSG_A_FMT("Unknown Vector_SToF element size: {}", ElementSize);
}
}
else {
switch (ElementSize) {
case 4:
if (Is256Bit) {
vcvtdq2ps(ToYMM(Dst), ToYMM(Vector));
} else {
vcvtdq2ps(Dst, Vector);
}
break;
case 8:
// This operation is a bit disgusting in x86
// There is no vector form of this instruction until AVX512VL + AVX512DQ (vcvtqq2pd)
// 1) First extract the top 64bits
// 2) Do a scalar conversion on each
// 3) Make sure to merge them together at the end
pextrq(rax, Vector, 1);
pextrq(rcx, Vector, 0);
cvtsi2sd(Dst, rcx);
cvtsi2sd(xmm15, rax);
if (Is256Bit) {
movlhps(Dst, xmm15);
vextracti128(xmm15, ToYMM(Vector), 1);
pextrq(rax, xmm15, 1);
pextrq(rcx, xmm15, 0);
cvtsi2sd(xmm15, rcx);
cvtsi2sd(xmm14, rax);
movlhps(xmm15, xmm14);
vinserti128(ToYMM(Dst), ToYMM(Dst), xmm15, 1);
} else {
vmovlhps(Dst, Dst, xmm15);
}
break;
default:
LOGMAN_MSG_A_FMT("Unknown Vector_SToF element size: {}", ElementSize);
break;
}
}
}
DEF_OP(Vector_FToZS) {
const auto Op = IROp->C<IR::IROp_Vector_FToZS>();
const auto OpSize = IROp->Size;
const auto ElementSize = Op->Header.ElementSize;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto Dst = GetDst(Node);
const auto Vector = GetSrc(Op->Vector.ID());
if (OpSize == ElementSize) {
if (ElementSize == 8) {
cvttss2si(Dst, Vector);
}
else if (ElementSize == 4) {
cvttss2si(Dst.cvt32(), Vector);
}
else {
LOGMAN_MSG_A_FMT("Unknown Vector_FToZS element size: {}", ElementSize);
}
}
else {
switch (ElementSize) {
case 4:
if (Is256Bit) {
vcvttps2dq(ToYMM(Dst), ToYMM(Vector));
} else {
vcvttps2dq(Dst, Vector);
}
break;
case 8:
if (Is256Bit) {
vcvttpd2dq(ToYMM(Dst), ToYMM(Vector));
} else {
vcvttpd2dq(Dst, Vector);
}
break;
default:
LOGMAN_MSG_A_FMT("Unknown Vector_FToZS element size: {}", ElementSize);
break;
}
}
}
DEF_OP(Vector_FToS) {
const auto Op = IROp->C<IR::IROp_Vector_FToS>();
const auto OpSize = IROp->Size;
const auto ElementSize = Op->Header.ElementSize;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto Dst = GetDst(Node);
const auto Vector = GetSrc(Op->Vector.ID());
if (OpSize == ElementSize) {
if (ElementSize == 8) {
cvtss2si(Dst, Vector);
}
else if (ElementSize == 4) {
cvtss2si(Dst.cvt32(), Vector);
}
else {
LOGMAN_MSG_A_FMT("Unknown Vector_FToS element size: {}", ElementSize);
}
}
else {
switch (ElementSize) {
case 4:
if (Is256Bit) {
vcvtps2dq(ToYMM(Dst), ToYMM(Vector));
} else {
vcvtps2dq(Dst, Vector);
}
break;
case 8:
if (Is256Bit) {
vcvtpd2dq(ToYMM(Dst), ToYMM(Vector));
} else {
vcvtpd2dq(Dst, Vector);
}
break;
default:
LOGMAN_MSG_A_FMT("Unknown Vector_FToS element size: {}", ElementSize);
break;
}
}
}
DEF_OP(Vector_FToF) {
const auto Op = IROp->C<IR::IROp_Vector_FToF>();
const auto OpSize = IROp->Size;
const auto ElementSize = Op->Header.ElementSize;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto Conv = (ElementSize << 8) | Op->SrcElementSize;
const auto Dst = GetDst(Node);
const auto Vector = GetSrc(Op->Vector.ID());
switch (Conv) {
case 0x0804: { // Double <- Float
if (Is256Bit) {
vcvtps2pd(ToYMM(Dst), Vector);
} else {
vcvtps2pd(Dst, Vector);
}
break;
}
case 0x0408: { // Float <- Double
if (Is256Bit) {
vcvtpd2ps(Dst, ToYMM(Vector));
} else {
vcvtpd2ps(Dst, Vector);
}
break;
}
default:
LOGMAN_MSG_A_FMT("Unknown Vector_FToF conversion type : 0x{:04x}", Conv);
break;
}
}
DEF_OP(Vector_FToI) {
const auto Op = IROp->C<IR::IROp_Vector_FToI>();
const auto OpSize = IROp->Size;
const uint8_t RoundMode = [Op] {
switch (Op->Round) {
case FEXCore::IR::Round_Nearest.Val:
return 0b0000'0'0'00;
case FEXCore::IR::Round_Negative_Infinity.Val:
return 0b0000'0'0'01;
case FEXCore::IR::Round_Positive_Infinity.Val:
return 0b0000'0'0'10;
case FEXCore::IR::Round_Towards_Zero.Val:
return 0b0000'0'0'11;
case FEXCore::IR::Round_Host.Val:
return 0b0000'0'1'00;
default:
LOGMAN_MSG_A_FMT("Unhandled rounding mode");
return 0;
}
}();
const auto ElementSize = Op->Header.ElementSize;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto Dst = GetDst(Node);
const auto Vector = GetSrc(Op->Vector.ID());
switch (ElementSize) {
case 4:
if (Is256Bit) {
vroundps(ToYMM(Dst), ToYMM(Vector), RoundMode);
} else {
vroundps(Dst, Vector, RoundMode);
}
break;
case 8:
if (Is256Bit) {
vroundpd(ToYMM(Dst), ToYMM(Vector), RoundMode);
} else {
vroundpd(Dst, Vector, RoundMode);
}
break;
default:
LOGMAN_MSG_A_FMT("Unhandled element size: {}", ElementSize);
break;
}
}
#undef DEF_OP
void X86JITCore::RegisterConversionHandlers() {
#define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &X86JITCore::Op_##x
REGISTER_OP(VINSGPR, VInsGPR);
REGISTER_OP(VCASTFROMGPR, VCastFromGPR);
REGISTER_OP(VDUPFROMGPR, VDupFromGPR);
REGISTER_OP(FLOAT_FROMGPR_S, Float_FromGPR_S);
REGISTER_OP(FLOAT_FTOF, Float_FToF);
REGISTER_OP(VECTOR_STOF, Vector_SToF);
REGISTER_OP(VECTOR_FTOZS, Vector_FToZS);
REGISTER_OP(VECTOR_FTOS, Vector_FToS);
REGISTER_OP(VECTOR_FTOF, Vector_FToF);
REGISTER_OP(VECTOR_FTOI, Vector_FToI);
#undef REGISTER_OP
}
}
FEXCore/Source/Interface/Core/JIT/x86_64/EncryptionOps.cpp
-161
View File
@@ -1,161 +0,0 @@
// SPDX-License-Identifier: MIT
/*
$info$
tags: backend|x86-64
$end_info$
*/
#include "Interface/Core/JIT/x86_64/JITClass.h"
#include "Interface/Core/Dispatcher/X86Dispatcher.h"
#include <FEXCore/IR/IR.h>
#include <array>
#include <stdint.h>
namespace FEXCore::CPU {
#define DEF_OP(x) void X86JITCore::Op_##x(IR::IROp_Header *IROp, IR::NodeID Node)
DEF_OP(AESImc) {
auto Op = IROp->C<IR::IROp_VAESImc>();
vaesimc(GetDst(Node), GetSrc(Op->Vector.ID()));
}
DEF_OP(AESEnc) {
const auto Op = IROp->C<IR::IROp_VAESEnc>();
const auto OpSize = IROp->Size;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto Dst = GetDst(Node);
const auto Key = GetSrc(Op->Key.ID());
const auto State = GetSrc(Op->State.ID());
if (Is256Bit) {
vaesenc(ToYMM(Dst), ToYMM(State), ToYMM(Key));
} else {
vaesenc(Dst, State, Key);
}
}
DEF_OP(AESEncLast) {
const auto Op = IROp->C<IR::IROp_VAESEncLast>();
const auto OpSize = IROp->Size;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto Dst = GetDst(Node);
const auto Key = GetSrc(Op->Key.ID());
const auto State = GetSrc(Op->State.ID());
if (Is256Bit) {
vaesenclast(ToYMM(Dst), ToYMM(State), ToYMM(Key));
} else {
vaesenclast(Dst, State, Key);
}
}
DEF_OP(AESDec) {
const auto Op = IROp->C<IR::IROp_VAESDec>();
const auto OpSize = IROp->Size;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto Dst = GetDst(Node);
const auto Key = GetSrc(Op->Key.ID());
const auto State = GetSrc(Op->State.ID());
if (Is256Bit) {
vaesdec(ToYMM(Dst), ToYMM(State), ToYMM(Key));
} else {
vaesdec(Dst, State, Key);
}
}
DEF_OP(AESDecLast) {
const auto Op = IROp->C<IR::IROp_VAESDecLast>();
const auto OpSize = IROp->Size;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto Dst = GetDst(Node);
const auto Key = GetSrc(Op->Key.ID());
const auto State = GetSrc(Op->State.ID());
if (Is256Bit) {
vaesdeclast(ToYMM(Dst), ToYMM(State), ToYMM(Key));
} else {
vaesdeclast(Dst, State, Key);
}
}
DEF_OP(AESKeyGenAssist) {
auto Op = IROp->C<IR::IROp_VAESKeyGenAssist>();
vaeskeygenassist(GetDst(Node), GetSrc(Op->Src.ID()), Op->RCON);
}
DEF_OP(CRC32) {
auto Op = IROp->C<IR::IROp_CRC32>();
switch (IROp->Size) {
case 4:
mov(TMP1, GetSrc<RA_32>(Op->Src2.ID()));
mov(GetDst<RA_32>(Node), GetSrc<RA_32>(Op->Src1.ID()));
break;
case 8:
mov(TMP1, GetSrc<RA_64>(Op->Src2.ID()));
mov(GetDst<RA_64>(Node), GetSrc<RA_64>(Op->Src1.ID()));
break;
default: LOGMAN_MSG_A_FMT("Unknown CRC32 size: {}", IROp->Size);
}
switch (Op->SrcSize) {
case 1:
crc32(GetDst<RA_32>(Node).cvt32(), TMP1.cvt8());
break;
case 2:
crc32(GetDst<RA_32>(Node).cvt32(), TMP1.cvt16());
break;
case 4:
crc32(GetDst<RA_32>(Node).cvt32(), TMP1.cvt32());
break;
case 8:
crc32(GetDst<RA_64>(Node).cvt64(), TMP1.cvt64());
break;
}
}
DEF_OP(PCLMUL) {
const auto Op = IROp->C<IR::IROp_PCLMUL>();
const auto OpSize = IROp->Size;
const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
const auto Dst = GetDst(Node);
const auto Src1 = GetSrc(Op->Src1.ID());
const auto Src2 = GetSrc(Op->Src2.ID());
switch (Op->Selector) {
case 0b00000000:
case 0b00000001:
case 0b00010000:
case 0b00010001:
if (Is256Bit) {
vpclmulqdq(ToYMM(Dst), ToYMM(Src1), ToYMM(Src2), Op->Selector);
} else {
vpclmulqdq(Dst, Src1, Src2, Op->Selector);
}
break;
default:
LOGMAN_MSG_A_FMT("Unknown PCLMUL selector: {}", Op->Selector);
break;
}
}
#undef DEF_OP
void X86JITCore::RegisterEncryptionHandlers() {
#define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &X86JITCore::Op_##x
REGISTER_OP(VAESIMC, AESImc);
REGISTER_OP(VAESENC, AESEnc);
REGISTER_OP(VAESENCLAST, AESEncLast);
REGISTER_OP(VAESDEC, AESDec);
REGISTER_OP(VAESDECLAST, AESDecLast);
REGISTER_OP(VAESKEYGENASSIST, AESKeyGenAssist);
REGISTER_OP(CRC32, CRC32);
REGISTER_OP(PCLMUL, PCLMUL);
#undef REGISTER_OP
}
}
FEXCore/Source/Interface/Core/JIT/x86_64/FlagOps.cpp
-35
View File
@@ -1,35 +0,0 @@
// SPDX-License-Identifier: MIT
/*
$info$
tags: backend|x86-64
$end_info$
*/
#include "Interface/Core/JIT/x86_64/JITClass.h"
#include "Interface/Core/Dispatcher/X86Dispatcher.h"
#include <FEXCore/IR/IR.h>
#include <array>
#include <stdint.h>
namespace FEXCore::CPU {
#define DEF_OP(x) void X86JITCore::Op_##x(IR::IROp_Header *IROp, IR::NodeID Node)
DEF_OP(GetHostFlag) {
auto Op = IROp->C<IR::IROp_GetHostFlag>();
mov(rax, GetSrc<RA_64>(Op->Value.ID()));
shr(rax, Op->Flag);
and_(rax, 1);
mov(GetDst<RA_64>(Node), rax);
}
#undef DEF_OP
void X86JITCore::RegisterFlagHandlers() {
#define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &X86JITCore::Op_##x
REGISTER_OP(GETHOSTFLAG, GetHostFlag);
#undef REGISTER_OP
}
}
FEXCore/Source/Interface/Core/JIT/x86_64/JIT.cpp
-884
View File
@@ -1,884 +0,0 @@
// SPDX-License-Identifier: MIT
/*
$info$
tags: backend|x86-64
desc: Main glue logic of the x86-64 splatter backend
$end_info$
*/
#include "Interface/Context/Context.h"
#include "Interface/Core/LookupCache.h"
#include "Interface/Core/Dispatcher/Dispatcher.h"
#include "Interface/Core/Dispatcher/X86Dispatcher.h"
#include "Interface/Core/Interpreter/InterpreterOps.h"
#include "Interface/Core/JIT/x86_64/JITClass.h"
#include "Interface/IR/PassManager.h"
#include "Interface/IR/Passes/RegisterAllocationPass.h"
#include "Utils/MemberFunctionToPointer.h"
#include <FEXCore/Core/CPUBackend.h>
#include <FEXCore/Core/CoreState.h>
#include <FEXCore/Debug/InternalThreadState.h>
#include <FEXCore/IR/IR.h>
#include <FEXCore/IR/IntrusiveIRList.h>
#include <FEXCore/IR/RegisterAllocationData.h>
#include <FEXCore/Utils/Allocator.h>
#include <FEXCore/Utils/EnumUtils.h>
#include <FEXCore/Utils/LogManager.h>
#include <FEXCore/Utils/Profiler.h>
#include <FEXCore/fextl/sstream.h>
#include <algorithm>
#include <array>
#include <memory>
#include <stddef.h>
#include <stdint.h>
#include <signal.h>
#include <tuple>
#include <unordered_map>
#include <utility>
// #define DEBUG_RA 1
// #define DEBUG_CYCLES
static constexpr size_t INITIAL_CODE_SIZE = 1024 * 1024 * 16;
static constexpr size_t MAX_CODE_SIZE = 1024 * 1024 * 256;
namespace {
static void PrintValue(uint64_t Value) {
LogMan::Msg::DFmt("Value: 0x{:x}", Value);
}
static void PrintVectorValue(uint64_t Value, uint64_t ValueUpper) {
LogMan::Msg::DFmt("Value: 0x{:016x}'{:016x}", ValueUpper, Value);
}
}
namespace FEXCore::CPU {
void X86JITCore::PushRegs() {
const auto AVXRegSize = Core::CPUState::XMM_AVX_REG_SIZE;
sub(rsp, AVXRegSize * RAXMM_x.size());
for (size_t i = 0; i < RAXMM_x.size(); ++i) {
vmovups(ptr[rsp + i * AVXRegSize], ToYMM(RAXMM_x[i]));
}
for (const auto &Reg : RA64) {
push(Reg);
}
const auto NumPush = RA64.size();
if ((NumPush & 1) != 0) {
// Align
sub(rsp, 8);
}
}
void X86JITCore::PopRegs() {
const auto AVXRegSize = Core::CPUState::XMM_AVX_REG_SIZE;
const auto NumPush = RA64.size();
if ((NumPush & 1) != 0) {
// Align
add(rsp, 8);
}
for (uint32_t i = RA64.size(); i > 0; --i) {
pop(RA64[i - 1]);
}
for (size_t i = 0; i < RAXMM_x.size(); ++i) {
vmovups(ToYMM(RAXMM_x[i]), ptr[rsp + i * AVXRegSize]);
}
add(rsp, AVXRegSize * RAXMM_x.size());
}
void X86JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
FallbackInfo Info;
if (!InterpreterOps::GetFallbackHandler(IROp, &Info)) {
#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
LOGMAN_MSG_A_FMT("Unhandled IR Op: {}", FEXCore::IR::GetName(IROp->Op));
#endif
} else {
switch(Info.ABI) {
case FABI_F80_I16_F32:{
PushRegs();
movss(xmm0, GetSrc(IROp->Args[0].ID()));
mov(rdi, word [STATE + offsetof(FEXCore::Core::CPUState, FCW)]);
call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]);
PopRegs();
pxor(GetDst(Node), GetDst(Node));
movq(GetDst(Node), rax);
pinsrw(GetDst(Node), edx, 4);
}
break;
case FABI_F80_I16_F64:{
PushRegs();
movsd(xmm0, GetSrc(IROp->Args[0].ID()));
mov(rdi, word [STATE + offsetof(FEXCore::Core::CPUState, FCW)]);
call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]);
PopRegs();
pxor(GetDst(Node), GetDst(Node));
movq(GetDst(Node), rax);
pinsrw(GetDst(Node), edx, 4);
}
break;
case FABI_F80_I16_I16:
case FABI_F80_I16_I32: {
PushRegs();
mov(rdi, word [STATE + offsetof(FEXCore::Core::CPUState, FCW)]);
if (Info.ABI == FABI_F80_I16_I16) {
movsx(rsi, GetSrc<RA_32>(IROp->Args[0].ID()).cvt16());
}
else {
mov(esi, GetSrc<RA_32>(IROp->Args[0].ID()));
}
call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]);
PopRegs();
pxor(GetDst(Node), GetDst(Node));
movq(GetDst(Node), rax);
pinsrw(GetDst(Node), edx, 4);
}
break;
case FABI_F32_I16_F80:{
PushRegs();
mov(rdi, word [STATE + offsetof(FEXCore::Core::CPUState, FCW)]);
movq(rsi, GetSrc(IROp->Args[0].ID()));
pextrq(rdx, GetSrc(IROp->Args[0].ID()), 1);
call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]);
PopRegs();
movss(GetDst(Node), xmm0);
}
break;
case FABI_F64_I16_F80:{
PushRegs();
mov(rdi, word [STATE + offsetof(FEXCore::Core::CPUState, FCW)]);
movq(rsi, GetSrc(IROp->Args[0].ID()));
pextrq(rdx, GetSrc(IROp->Args[0].ID()), 1);
call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]);
PopRegs();
movsd(GetDst(Node), xmm0);
}
break;
case FABI_F64_I16_F64: {
PushRegs();
mov(rdi, word [STATE + offsetof(FEXCore::Core::CPUState, FCW)]);
movsd(xmm0, GetSrc(IROp->Args[0].ID()));
call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]);
PopRegs();
movsd(GetDst(Node), xmm0);
}
break;
case FABI_F64_I16_F64_F64: {
PushRegs();
mov(rdi, word [STATE + offsetof(FEXCore::Core::CPUState, FCW)]);
movsd(xmm0, GetSrc(IROp->Args[0].ID()));
movsd(xmm1, GetSrc(IROp->Args[1].ID()));
call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]);
PopRegs();
movsd(GetDst(Node), xmm0);
}
break;
case FABI_I16_I16_F80:{
PushRegs();
mov(rdi, word [STATE + offsetof(FEXCore::Core::CPUState, FCW)]);
movq(rsi, GetSrc(IROp->Args[0].ID()));
pextrq(rdx, GetSrc(IROp->Args[0].ID()), 1);
call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]);
PopRegs();
movsx(GetDst<RA_64>(Node), ax);
}
break;
case FABI_I32_I16_F80:{
PushRegs();
mov(rdi, word [STATE + offsetof(FEXCore::Core::CPUState, FCW)]);
movq(rsi, GetSrc(IROp->Args[0].ID()));
pextrq(rdx, GetSrc(IROp->Args[0].ID()), 1);
call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]);
PopRegs();
mov(GetDst<RA_32>(Node), eax);
}
break;
case FABI_I64_I16_F80:{
PushRegs();
mov(rdi, word [STATE + offsetof(FEXCore::Core::CPUState, FCW)]);
movq(rsi, GetSrc(IROp->Args[0].ID()));
pextrq(rdx, GetSrc(IROp->Args[0].ID()), 1);
call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]);
PopRegs();
mov(GetDst<RA_64>(Node), rax);
}
break;
case FABI_I64_I16_F80_F80:{
PushRegs();
mov(rdi, word [STATE + offsetof(FEXCore::Core::CPUState, FCW)]);
movq(rsi, GetSrc(IROp->Args[0].ID()));
pextrq(rdx, GetSrc(IROp->Args[0].ID()), 1);
movq(rcx, GetSrc(IROp->Args[1].ID()));
pextrq(r8, GetSrc(IROp->Args[1].ID()), 1);
call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]);
PopRegs();
mov(GetDst<RA_64>(Node), rax);
}
break;
case FABI_F80_I16_F80:{
PushRegs();
mov(rdi, word [STATE + offsetof(FEXCore::Core::CPUState, FCW)]);
movq(rsi, GetSrc(IROp->Args[0].ID()));
pextrq(rdx, GetSrc(IROp->Args[0].ID()), 1);
call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]);
PopRegs();
pxor(GetDst(Node), GetDst(Node));
movq(GetDst(Node), rax);
pinsrw(GetDst(Node), edx, 4);
}
break;
case FABI_F80_I16_F80_F80:{
PushRegs();
mov(rdi, word [STATE + offsetof(FEXCore::Core::CPUState, FCW)]);
movq(rsi, GetSrc(IROp->Args[0].ID()));
pextrq(rdx, GetSrc(IROp->Args[0].ID()), 1);
movq(rcx, GetSrc(IROp->Args[1].ID()));
pextrq(r8, GetSrc(IROp->Args[1].ID()), 1);
call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]);
PopRegs();
pxor(GetDst(Node), GetDst(Node));
movq(GetDst(Node), rax);
pinsrw(GetDst(Node), edx, 4);
}
break;
case FABI_I32_I64_I64_I128_I128_I16: {
PushRegs();
const auto Op = IROp->C<IR::IROp_VPCMPESTRX>();
const auto Control = Op->Control;
const auto LHS = GetSrc(Op->LHS.ID());
const auto RHS = GetSrc(Op->RHS.ID());
const auto SrcRAX = GetSrc<RA_64>(Op->RAX.ID());
const auto SrcRDX = GetSrc<RA_64>(Op->RDX.ID());
mov(rdi, SrcRAX);
mov(rsi, SrcRDX);
movq(rdx, LHS);
pextrq(rcx, LHS, 1);
movq(r8, RHS);
pextrq(r9, RHS, 1);
sub(rsp, 16);
mov(dword [rsp], Control);
call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]);
add(rsp, 16);
PopRegs();
mov(GetDst<RA_32>(Node), rax);
break;
}
case FABI_I32_I128_I128_I16: {
PushRegs();
const auto Op = IROp->C<IR::IROp_VPCMPISTRX>();
const auto LHS = GetSrc(Op->LHS.ID());
const auto RHS = GetSrc(Op->RHS.ID());
const auto Control = Op->Control;
movq(rdi, LHS);
pextrq(rsi, LHS, 1);
movq(rdx, RHS);
pextrq(rcx, RHS, 1);
mov(r8, Control);
call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]);
PopRegs();
mov(GetDst<RA_32>(Node), rax);
break;
}
case FABI_UNKNOWN:
default:
#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
LOGMAN_MSG_A_FMT("Unhandled IR Fallback ABI: {} {}",
IR::GetName(IROp->Op), ToUnderlying(Info.ABI));
#endif
break;
}
}
}
static uint64_t X86JITCore_ExitFunctionLink(FEXCore::Core::CpuStateFrame *Frame, uint64_t *record) {
auto Thread = Frame->Thread;
auto GuestRip = record[1];
auto HostCode = Thread->LookupCache->FindBlock(GuestRip);
if (!HostCode) {
Thread->CurrentFrame->State.rip = GuestRip;
return Frame->Pointers.Common.DispatcherLoopTop;
}
auto LinkerAddress = Frame->Pointers.Common.ExitFunctionLinker;
Thread->LookupCache->AddBlockLink(GuestRip, (uintptr_t)record, [record, LinkerAddress]{
// undo the link
record[0] = LinkerAddress;
});
record[0] = HostCode;
return HostCode;
}
void X86JITCore::Op_NoOp(IR::IROp_Header *IROp, IR::NodeID Node) {
}
X86JITCore::X86JITCore(FEXCore::Context::ContextImpl *ctx, FEXCore::Core::InternalThreadState *Thread)
: CPUBackend(Thread, INITIAL_CODE_SIZE, MAX_CODE_SIZE)
, CodeGenerator(0, this, nullptr) // this is not used here
, CTX {ctx} {
RAPass = Thread->PassManager->GetPass<IR::RegisterAllocationPass>("RA");
RAPass->AllocateRegisterSet(RegisterClasses);
RAPass->AddRegisters(FEXCore::IR::GPRClass, NumGPRs);
RAPass->AddRegisters(FEXCore::IR::FPRClass, NumXMMs);
RAPass->AddRegisters(FEXCore::IR::GPRPairClass, NumGPRPairs);
for (uint32_t i = 0; i < NumGPRPairs; ++i) {
RAPass->AddRegisterConflict(FEXCore::IR::GPRClass, i * 2, FEXCore::IR::GPRPairClass, i);
RAPass->AddRegisterConflict(FEXCore::IR::GPRClass, i * 2 + 1, FEXCore::IR::GPRPairClass, i);
}
for (uint32_t i = 0; i < FEXCore::IR::IROps::OP_LAST + 1; ++i) {
OpHandlers[i] = &X86JITCore::Op_Unhandled;
}
RegisterALUHandlers();
RegisterAtomicHandlers();
RegisterBranchHandlers();
RegisterConversionHandlers();
RegisterFlagHandlers();
RegisterMemoryHandlers();
RegisterMiscHandlers();
RegisterMoveHandlers();
RegisterVectorHandlers();
RegisterEncryptionHandlers();
{
auto &Common = ThreadState->CurrentFrame->Pointers.Common;
Common.PrintValue = reinterpret_cast<uint64_t>(PrintValue);
Common.PrintVectorValue = reinterpret_cast<uint64_t>(PrintVectorValue);
Common.ThreadRemoveCodeEntryFromJIT = reinterpret_cast<uintptr_t>(&Context::ContextImpl::ThreadRemoveCodeEntryFromJit);
Common.CPUIDObj = reinterpret_cast<uint64_t>(&CTX->CPUID);
{
FEXCore::Utils::MemberFunctionToPointerCast PMF(&FEXCore::CPUIDEmu::RunFunction);
Common.CPUIDFunction = PMF.GetConvertedPointer();
}
{
FEXCore::Utils::MemberFunctionToPointerCast PMF(&FEXCore::CPUIDEmu::RunXCRFunction);
Common.XCRFunction = PMF.GetConvertedPointer();
}
Common.SyscallHandlerObj = reinterpret_cast<uint64_t>(CTX->SyscallHandler);
Common.SyscallHandlerFunc = reinterpret_cast<uint64_t>(FEXCore::Context::HandleSyscall);
Common.ExitFunctionLink = reinterpret_cast<uintptr_t>(&Context::ContextImpl::ThreadExitFunctionLink<X86JITCore_ExitFunctionLink>);
// Fill in the fallback handlers
InterpreterOps::FillFallbackIndexPointers(Common.FallbackHandlerPointers);
}
// Must be done after Dispatcher init
ClearCache();
}
X86JITCore::~X86JITCore() {
}
void X86JITCore::EmitDetectionString() {
const char JITString[] = "FEXJIT::X86JITCore::";
for (char c : JITString) {
db(c);
}
}
void X86JITCore::ClearCache() {
auto CodeBuffer = GetEmptyCodeBuffer();
setNewBuffer(CodeBuffer->Ptr, CodeBuffer->Size);
EmitDetectionString();
}
IR::PhysicalRegister X86JITCore::GetPhys(IR::NodeID Node) const {
auto PhyReg = RAData->GetNodeRegister(Node);
LOGMAN_THROW_AA_FMT(PhyReg.Raw != 255, "Couldn't Allocate register for node: ssa{}. Class: {}", Node, PhyReg.Class);
return PhyReg;
}
bool X86JITCore::IsFPR(IR::NodeID Node) const {
return RAData->GetNodeRegister(Node).Class == IR::FPRClass;
}
bool X86JITCore::IsGPR(IR::NodeID Node) const {
return RAData->GetNodeRegister(Node).Class == IR::GPRClass;
}
bool X86JITCore::IsGPRPair(IR::NodeID Node) const {
return RAData->GetNodeRegister(Node).Class == IR::GPRPairClass;
}
template<uint8_t RAType>
Xbyak::Reg X86JITCore::GetSrc(IR::NodeID Node) const {
// rax, rcx, rdx, rsi, r8, r9,
// r10
// Callee Saved
// rbx, rbp, r12, r13, r14, r15
auto PhyReg = GetPhys(Node);
if constexpr (RAType == RA_64)
return RA64[PhyReg.Reg].cvt64();
else if constexpr (RAType == RA_XMM)
return RAXMM[PhyReg.Reg];
else if constexpr (RAType == RA_32)
return RA64[PhyReg.Reg].cvt32();
else if constexpr (RAType == RA_16)
return RA64[PhyReg.Reg].cvt16();
else if constexpr (RAType == RA_8)
return RA64[PhyReg.Reg].cvt8();
}
template
Xbyak::Reg X86JITCore::GetSrc<X86JITCore::RA_64>(IR::NodeID Node) const;
template
Xbyak::Reg X86JITCore::GetSrc<X86JITCore::RA_32>(IR::NodeID Node) const;
template
Xbyak::Reg X86JITCore::GetSrc<X86JITCore::RA_16>(IR::NodeID Node) const;
template
Xbyak::Reg X86JITCore::GetSrc<X86JITCore::RA_8>(IR::NodeID Node) const;
Xbyak::Xmm X86JITCore::GetSrc(IR::NodeID Node) const {
auto PhyReg = GetPhys(Node);
return RAXMM_x[PhyReg.Reg];
}
template<uint8_t RAType>
Xbyak::Reg X86JITCore::GetDst(IR::NodeID Node) const {
auto PhyReg = GetPhys(Node);
if constexpr (RAType == RA_64)
return RA64[PhyReg.Reg].cvt64();
else if constexpr (RAType == RA_XMM)
return RAXMM[PhyReg.Reg];
else if constexpr (RAType == RA_32)
return RA64[PhyReg.Reg].cvt32();
else if constexpr (RAType == RA_16)
return RA64[PhyReg.Reg].cvt16();
else if constexpr (RAType == RA_8)
return RA64[PhyReg.Reg].cvt8();
}
template
Xbyak::Reg X86JITCore::GetDst<X86JITCore::RA_64>(IR::NodeID Node) const;
template
Xbyak::Reg X86JITCore::GetDst<X86JITCore::RA_32>(IR::NodeID Node) const;
template
Xbyak::Reg X86JITCore::GetDst<X86JITCore::RA_16>(IR::NodeID Node) const;
template
Xbyak::Reg X86JITCore::GetDst<X86JITCore::RA_8>(IR::NodeID Node) const;
template<uint8_t RAType>
std::pair<Xbyak::Reg, Xbyak::Reg> X86JITCore::GetSrcPair(IR::NodeID Node) const {
auto PhyReg = GetPhys(Node);
if constexpr (RAType == RA_64)
return RA64Pair[PhyReg.Reg];
else if constexpr (RAType == RA_32)
return {RA64Pair[PhyReg.Reg].first.cvt32(), RA64Pair[PhyReg.Reg].second.cvt32()};
}
template
std::pair<Xbyak::Reg, Xbyak::Reg> X86JITCore::GetSrcPair<X86JITCore::RA_64>(IR::NodeID Node) const;
template
std::pair<Xbyak::Reg, Xbyak::Reg> X86JITCore::GetSrcPair<X86JITCore::RA_32>(IR::NodeID Node) const;
Xbyak::Xmm X86JITCore::GetDst(IR::NodeID Node) const {
auto PhyReg = GetPhys(Node);
return RAXMM_x[PhyReg.Reg];
}
bool X86JITCore::IsInlineConstant(const IR::OrderedNodeWrapper& WNode, uint64_t* Value) const {
auto OpHeader = IR->GetOp<IR::IROp_Header>(WNode);
if (OpHeader->Op == IR::IROps::OP_INLINECONSTANT) {
auto Op = OpHeader->C<IR::IROp_InlineConstant>();
if (Value) {
*Value = Op->Constant;
}
return true;
} else {
return false;
}
}
bool X86JITCore::IsInlineEntrypointOffset(const IR::OrderedNodeWrapper& WNode, uint64_t* Value) const {
auto OpHeader = IR->GetOp<IR::IROp_Header>(WNode);
if (OpHeader->Op == IR::IROps::OP_INLINEENTRYPOINTOFFSET) {
auto Op = OpHeader->C<IR::IROp_InlineEntrypointOffset>();
if (Value) {
uint64_t Mask = ~0ULL;
uint8_t OpSize = OpHeader->Size;
if (OpSize == 4) {
Mask = 0xFFFF'FFFFULL;
}
*Value = (Entry + Op->Offset) & Mask;
}
return true;
} else {
return false;
}
}
std::tuple<X86JITCore::SetCC, X86JITCore::CMovCC, X86JITCore::JCC> X86JITCore::GetCC(IR::CondClassType cond) {
switch (cond.Val) {
case FEXCore::IR::COND_EQ: return { &CodeGenerator::sete , &CodeGenerator::cmove , &CodeGenerator::je };
case FEXCore::IR::COND_NEQ: return { &CodeGenerator::setne, &CodeGenerator::cmovne, &CodeGenerator::jne };
case FEXCore::IR::COND_SGE: return { &CodeGenerator::setge, &CodeGenerator::cmovge, &CodeGenerator::jge };
case FEXCore::IR::COND_SLT: return { &CodeGenerator::setl , &CodeGenerator::cmovl , &CodeGenerator::jl };
case FEXCore::IR::COND_SGT: return { &CodeGenerator::setg , &CodeGenerator::cmovg , &CodeGenerator::jg };
case FEXCore::IR::COND_SLE: return { &CodeGenerator::setle, &CodeGenerator::cmovle, &CodeGenerator::jle };
case FEXCore::IR::COND_UGE: return { &CodeGenerator::setae, &CodeGenerator::cmovae, &CodeGenerator::jae };
case FEXCore::IR::COND_ULT: return { &CodeGenerator::setb , &CodeGenerator::cmovb , &CodeGenerator::jb };
case FEXCore::IR::COND_UGT: return { &CodeGenerator::seta , &CodeGenerator::cmova , &CodeGenerator::ja };
case FEXCore::IR::COND_ULE: return { &CodeGenerator::setna, &CodeGenerator::cmovna, &CodeGenerator::jna };
case FEXCore::IR::COND_FLU: return { &CodeGenerator::setb , &CodeGenerator::cmovb , &CodeGenerator::jb };
case FEXCore::IR::COND_FGE: return { &CodeGenerator::setae, &CodeGenerator::cmovae, &CodeGenerator::jae };
case FEXCore::IR::COND_FLEU: return { &CodeGenerator::setna, &CodeGenerator::cmovna, &CodeGenerator::jna };
case FEXCore::IR::COND_FGT: return { &CodeGenerator::seta , &CodeGenerator::cmova , &CodeGenerator::ja };
case FEXCore::IR::COND_FU: return { &CodeGenerator::setp , &CodeGenerator::cmovp , &CodeGenerator::jp };
case FEXCore::IR::COND_FNU: return { &CodeGenerator::setnp, &CodeGenerator::cmovnp, &CodeGenerator::jnp };
case FEXCore::IR::COND_MI:
case FEXCore::IR::COND_PL:
case FEXCore::IR::COND_VS:
case FEXCore::IR::COND_VC:
default:
LOGMAN_MSG_A_FMT("Unsupported compare type");
break;
}
// Hope for the best
return { &CodeGenerator::sete , &CodeGenerator::cmove , &CodeGenerator::je };
}
CPUBackend::CompiledCode X86JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IRListView const *IR, [[maybe_unused]] FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData, bool GDBEnabled) {
FEXCORE_PROFILE_SCOPED("x86::CompileCode");
JumpTargets.clear();
uint32_t SSACount = IR->GetSSACount();
this->Entry = Entry;
this->RAData = RAData;
this->DebugData = DebugData;
// Fairly excessive buffer range to make sure we don't overflow
uint32_t BufferRange = SSACount * 16 + GDBEnabled * Dispatcher::MaxGDBPauseCheckSize;
if ((getSize() + BufferRange) > CurrentCodeBuffer->Size) {
CTX->ClearCodeCache(ThreadState);
}
CodeData.BlockBegin = getCurr<uint8_t*>();
// Put the code header at the start of the data block.
Label JITCodeHeaderLabel{};
L(JITCodeHeaderLabel);
JITCodeHeader *CodeHeader = getCurr<JITCodeHeader *>();
setSize(getSize() + sizeof(JITCodeHeader));
CodeData.BlockEntry = getCurr<uint8_t*>();
// Get the address of the JITCodeHeader and store in to the core state.
// Only two instructions, so very low overhead.
lea(TMP1, ptr [rip + JITCodeHeaderLabel]);
mov(qword [STATE + offsetof(FEXCore::Core::CPUState, InlineJITBlockHeader)], TMP1);
CursorEntry = getSize();
this->IR = IR;
if (GDBEnabled) {
auto GDBSize = CTX->Dispatcher->GenerateGDBPauseCheck(CodeData.BlockBegin, Entry);
setSize(getSize() + GDBSize);
}
LOGMAN_THROW_AA_FMT(RAData != nullptr, "Needs RA");
SpillSlots = RAData->SpillSlots();
if (SpillSlots) {
sub(rsp, SpillSlots * MaxSpillSlotSize);
}
#ifdef BLOCKSTATS
BlockSamplingData::BlockData *SamplingData = CTX->BlockData->GetBlockData(Entry);
if (GetSamplingData) {
mov(rcx, reinterpret_cast<uintptr_t>(SamplingData));
rdtsc();
shl(rdx, 32);
or_(rax, rdx);
mov(qword [rcx + offsetof(BlockSamplingData::BlockData, Start)], rax);
}
auto ExitBlock = [&]() {
if (GetSamplingData) {
mov(rcx, reinterpret_cast<uintptr_t>(SamplingData));
// Get time
rdtsc();
shl(rdx, 32);
or_(rax, rdx);
// Calculate time spent in block
mov(rdx, qword [rcx + offsetof(BlockSamplingData::BlockData, Start)]);
sub(rax, rdx);
// Add time to total time
add(qword [rcx + offsetof(BlockSamplingData::BlockData, TotalTime)], rax);
// Increment call count
inc(qword [rcx + offsetof(BlockSamplingData::BlockData, TotalCalls)]);
// Calculate min
mov(rdx, qword [rcx + offsetof(BlockSamplingData::BlockData, Min)]);
cmp(rdx, rax);
cmova(rdx, rax);
mov(qword [rcx + offsetof(BlockSamplingData::BlockData, Min)], rdx);
// Calculate max
mov(rdx, qword [rcx + offsetof(BlockSamplingData::BlockData, Max)]);
cmp(rdx, rax);
cmovb(rdx, rax);
mov(qword [rcx + offsetof(BlockSamplingData::BlockData, Max)], rdx);
}
};
#endif
PendingTargetLabel = nullptr;
for (auto [BlockNode, BlockHeader] : IR->GetBlocks()) {
using namespace FEXCore::IR;
#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
auto BlockIROp = BlockHeader->CW<IROp_CodeBlock>();
LOGMAN_THROW_AA_FMT(BlockIROp->Header.Op == IR::OP_CODEBLOCK, "IR type failed to be a code block");
#endif
auto BlockStartHostCode = getCurr<uint8_t *>();
{
const auto Node = IR->GetID(BlockNode);
const auto IsTarget = JumpTargets.try_emplace(Node).first;
// if there is a pending branch, and it is not fall-through
if (PendingTargetLabel && PendingTargetLabel != &IsTarget->second) {
jmp(*PendingTargetLabel, T_NEAR);
}
PendingTargetLabel = nullptr;
L(IsTarget->second);
}
for (auto [CodeNode, IROp] : IR->GetCode(BlockNode)) {
#ifdef DEBUG_RA
if (IROp->Op != IR::OP_BEGINBLOCK &&
IROp->Op != IR::OP_CONDJUMP &&
IROp->Op != IR::OP_JUMP) {
fextl::stringstream Inst;
auto Name = FEXCore::IR::GetName(IROp->Op);
if (IROp->HasDest) {
uint64_t PhysReg = RAPass->GetNodeRegister(Node);
if (PhysReg >= GPRPairBase)
Inst << "\tPair" << GetPhys(Node) << " = " << Name << " ";
else if (PhysReg >= XMMBase)
Inst << "\tXMM" << GetPhys(Node) << " = " << Name << " ";
else
Inst << "\tReg" << GetPhys(Node) << " = " << Name << " ";
}
else {
Inst << "\t" << Name << " ";
}
const uint8_t NumArgs = IR::GetArgs(IROp->Op);
for (uint8_t i = 0; i < NumArgs; ++i) {
const auto ArgNode = IROp->Args[i].ID();
const uint64_t PhysReg = RAPass->GetNodeRegister(ArgNode);
if (PhysReg >= GPRPairBase)
Inst << "Pair" << GetPhys(ArgNode) << (i + 1 == NumArgs ? "" : ", ");
else if (PhysReg >= XMMBase)
Inst << "XMM" << GetPhys(ArgNode) << (i + 1 == NumArgs ? "" : ", ");
else
Inst << "Reg" << GetPhys(ArgNode) << (i + 1 == NumArgs ? "" : ", ");
}
LogMan::Msg::DFmt("{}", Inst.str());
}
#endif
const auto ID = IR->GetID(CodeNode);
// Execute handler
OpHandler Handler = OpHandlers[IROp->Op];
(this->*Handler)(IROp, ID);
}
if (DebugData) {
DebugData->Subblocks.push_back({
static_cast<uint32_t>(BlockStartHostCode - CodeData.BlockBegin),
static_cast<uint32_t>(getCurr<uint8_t *>() - BlockStartHostCode)
});
}
}
// Make sure last branch is generated. It certainly can't be eliminated here.
if (PendingTargetLabel)
{
jmp(*PendingTargetLabel, T_NEAR);
}
PendingTargetLabel = nullptr;
// Add the JitCodeTail
auto JITBlockTailLocation = getCurr<uint8_t *>();
auto JITBlockTail = getCurr<JITCodeTail*>();
setSize(getSize() + sizeof(JITCodeTail));
auto JITRIPEntriesLocation = getCurr<uint8_t *>();
auto JITRIPEntries = getCurr<JITRIPReconstructEntries*>();
setSize(getSize() + sizeof(JITRIPReconstructEntries) * DebugData->GuestOpcodes.size());
// Put the block's RIP entry in the tail.
// This will be used for RIP reconstruction in the future.
// TODO: This needs to be a data RIP relocation once code caching works.
// Current relocation code doesn't support this feature yet.
JITBlockTail->RIP = Entry;
{
// Store the RIP entries.
JITBlockTail->NumberOfRIPEntries = DebugData->GuestOpcodes.size();
JITBlockTail->OffsetToRIPEntries = JITRIPEntriesLocation - JITBlockTailLocation;
uintptr_t CurrentRIPOffset = 0;
uint64_t CurrentPCOffset = 0;
for (size_t i = 0; i < DebugData->GuestOpcodes.size(); i++) {
const auto &GuestOpcode = DebugData->GuestOpcodes[i];
auto &RIPEntry = JITRIPEntries[i];
RIPEntry.HostPCOffset = GuestOpcode.HostEntryOffset - CurrentPCOffset;
RIPEntry.GuestRIPOffset = GuestOpcode.GuestEntryOffset - CurrentRIPOffset;
CurrentPCOffset = GuestOpcode.HostEntryOffset;
CurrentRIPOffset = GuestOpcode.GuestEntryOffset;
}
}
CodeHeader->OffsetToBlockTail = JITBlockTailLocation - CodeData.BlockBegin;
CodeData.Size = getCurr<uint8_t*>() - CodeData.BlockBegin;
JITBlockTail->Size = CodeData.Size;
this->IR = nullptr;
ready();
if (DebugData) {
DebugData->HostCodeSize = CodeData.Size;
DebugData->Relocations = &Relocations;
}
return CodeData;
}
fextl::unique_ptr<CPUBackend> CreateX86JITCore(FEXCore::Context::ContextImpl *ctx, FEXCore::Core::InternalThreadState *Thread) {
return fextl::make_unique<X86JITCore>(ctx, Thread);
}
CPUBackendFeatures GetX86JITBackendFeatures() {
return CPUBackendFeatures { };
}
}
FEXCore/Source/Interface/Core/JIT/x86_64/JITClass.h
-492
View File
@@ -1,492 +0,0 @@
// SPDX-License-Identifier: MIT
/*
$info$
tags: backend|x86-64
$end_info$
*/
#pragma once
#include <FEXCore/IR/RegisterAllocationData.h>
#include "Interface/Core/BlockSamplingData.h"
#include "Interface/Core/Dispatcher/Dispatcher.h"
#include "Interface/Core/Dispatcher/X86Dispatcher.h"
#include "Interface/Core/ObjectCache/Relocations.h"
using namespace Xbyak;
#include <FEXCore/Core/CoreState.h>
#include <FEXCore/Core/CPUBackend.h>
#include <FEXCore/IR/IR.h>
#include <FEXCore/IR/IntrusiveIRList.h>
#include <FEXCore/Utils/MathUtils.h>
#include <FEXCore/fextl/string.h>
#include <FEXCore/fextl/unordered_map.h>
#include <FEXCore/fextl/vector.h>
#include "Interface/IR/Passes/RegisterAllocationPass.h"
#include <tuple>
namespace FEXCore::CPU {
// Temp registers
// rax, rcx, rdx, rsi, r8, r9,
// r10, r11
//
// Callee Saved
// rbx, rbp, r12, r13, r14, r15
//
// 1St Argument: rdi <ThreadState>
// XMM:
// All temp
#define STATE r14
#define TMP1 rax
#define TMP2 rcx
#define TMP3 rdx
#define TMP4 rdi
#define TMP5 rbx
using namespace Xbyak::util;
const std::array<Xbyak::Reg, 9> RA64 = { rsi, r8, r9, r10, r11, rbp, r12, r13, r15 };
const std::array<std::pair<Xbyak::Reg, Xbyak::Reg>, 4> RA64Pair = {{ {rsi, r8}, {r9, r10}, {r11, rbp}, {r12, r13} }};
const std::array<Xbyak::Reg, 11> RAXMM = { xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11};
const std::array<Xbyak::Xmm, 11> RAXMM_x = { xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11};
class X86JITCore final : public CPUBackend, public Xbyak::CodeGenerator {
public:
explicit X86JITCore(FEXCore::Context::ContextImpl *ctx,
FEXCore::Core::InternalThreadState *Thread);
~X86JITCore() override;
[[nodiscard]] fextl::string GetName() override { return "JIT"; }
[[nodiscard]] CPUBackend::CompiledCode CompileCode(uint64_t Entry,
FEXCore::IR::IRListView const *IR,
FEXCore::Core::DebugData *DebugData,
FEXCore::IR::RegisterAllocationData *RAData, bool GDBEnabled) override;
[[nodiscard]] void *MapRegion(void* HostPtr, uint64_t, uint64_t) override { return HostPtr; }
[[nodiscard]] bool NeedsOpDispatch() override { return true; }
void ClearCache() override;
void ClearRelocations() override { Relocations.clear(); }
private:
/**
* @name Relocations
* @{ */
uint64_t GetNamedSymbolLiteral(FEXCore::CPU::RelocNamedSymbolLiteral::NamedSymbol Op);
void LoadConstantWithPadding(Xbyak::Reg Reg, uint64_t Constant);
/**
* @brief A literal pair relocation object for named symbol literals
*/
struct NamedSymbolLiteralPair {
Label Offset;
Relocation MoveABI{};
};
/**
* @brief Inserts a thunk relocation
*
* @param Reg - The GPR to move the thunk handler in to
* @param Sum - The hash of the thunk
*/
void InsertNamedThunkRelocation(Xbyak::Reg Reg, const IR::SHA256Sum &Sum);
/**
* @brief Inserts a guest GPR move relocation
*
* @param Reg - The GPR to move the guest RIP in to
* @param Constant - The guest RIP that will be relocated
*/
void InsertGuestRIPMove(Xbyak::Reg Reg, uint64_t Constant);
/**
* @brief Inserts a named symbol as a literal in memory
*
* Need to use `PlaceNamedSymbolLiteral` with the return value to place the literal in the desired location
*
* @param Op The named symbol to place
*
* @return A temporary `NamedSymbolLiteralPair`
*/
NamedSymbolLiteralPair InsertNamedSymbolLiteral(FEXCore::CPU::RelocNamedSymbolLiteral::NamedSymbol Op);
/**
* @brief Place the named symbol literal relocation in memory
*
* @param Lit - Which literal to place
*/
void PlaceNamedSymbolLiteral(NamedSymbolLiteralPair &Lit);
fextl::vector<FEXCore::CPU::Relocation> Relocations;
///< Relocation code loading
bool ApplyRelocations(uint64_t GuestEntry, uint64_t CodeEntry, uint64_t CursorEntry, size_t NumRelocations, const char* EntryRelocations);
/**
* @brief Current guest RIP entrypoint
*/
uint64_t CursorEntry{};
/** @} */
Label* PendingTargetLabel{};
FEXCore::Context::ContextImpl *CTX;
FEXCore::IR::IRListView const *IR;
uint64_t Entry;
CPUBackend::CompiledCode CodeData{};
fextl::unordered_map<IR::NodeID, Label> JumpTargets;
Xbyak::util::Cpu Features{};
bool MemoryDebug = false;
/**
* @name Register Allocation
* @{ */
constexpr static uint32_t NumGPRs = RA64.size(); // 4 is the minimum required for GPR ops
constexpr static uint32_t NumXMMs = RAXMM.size();
constexpr static uint32_t NumGPRPairs = RA64Pair.size();
constexpr static uint32_t RegisterClasses = 6;
constexpr static uint64_t GPRBase = (0ULL << 32);
constexpr static uint64_t XMMBase = (1ULL << 32);
constexpr static uint64_t GPRPairBase = (2ULL << 32);
/** @} */
constexpr static uint8_t RA_8 = 0;
constexpr static uint8_t RA_16 = 1;
constexpr static uint8_t RA_32 = 2;
constexpr static uint8_t RA_64 = 3;
constexpr static uint8_t RA_XMM = 4;
[[nodiscard]] IR::PhysicalRegister GetPhys(IR::NodeID Node) const;
[[nodiscard]] bool IsFPR(IR::NodeID Node) const;
[[nodiscard]] bool IsGPR(IR::NodeID Node) const;
[[nodiscard]] bool IsGPRPair(IR::NodeID Node) const;
template<uint8_t RAType>
[[nodiscard]] Xbyak::Reg GetSrc(IR::NodeID Node) const;
template<uint8_t RAType>
[[nodiscard]] std::pair<Xbyak::Reg, Xbyak::Reg> GetSrcPair(IR::NodeID Node) const;
template<uint8_t RAType>
[[nodiscard]] Xbyak::Reg GetDst(IR::NodeID Node) const;
[[nodiscard]] Xbyak::Xmm GetSrc(IR::NodeID Node) const;
[[nodiscard]] Xbyak::Xmm GetDst(IR::NodeID Node) const;
[[nodiscard]] static Xbyak::Ymm ToYMM(const Xbyak::Xmm& xmm) {
return Xbyak::Ymm{xmm.getIdx()};
}
[[nodiscard]] Xbyak::RegExp GenerateModRM(Xbyak::Reg Base, IR::OrderedNodeWrapper Offset,
IR::MemOffsetType OffsetType, uint8_t OffsetScale) const;
[[nodiscard]] bool IsInlineConstant(const IR::OrderedNodeWrapper& Node, uint64_t* Value = nullptr) const;
[[nodiscard]] bool IsInlineEntrypointOffset(const IR::OrderedNodeWrapper& WNode, uint64_t* Value) const;
IR::RegisterAllocationPass *RAPass;
FEXCore::IR::RegisterAllocationData *RAData;
FEXCore::Core::DebugData *DebugData;
#ifdef BLOCKSTATS
bool GetSamplingData {true};
#endif
static uint64_t ExitFunctionLink(FEXCore::Core::CpuStateFrame *Frame, uint64_t *record);
// This is purely a debugging aid for developers to see if they are in JIT code space when inspecting raw memory
void EmitDetectionString();
uint32_t SpillSlots{};
using SetCC = void (X86JITCore::*)(const Operand& op);
using CMovCC = void (X86JITCore::*)(const Reg& reg, const Operand& op);
using JCC = void (X86JITCore::*)(const Label& label, LabelType type);
std::tuple<SetCC, CMovCC, JCC> GetCC(IR::CondClassType cond);
using OpHandler = void (X86JITCore::*)(IR::IROp_Header *IROp, IR::NodeID Node);
std::array<OpHandler, IR::IROps::OP_LAST + 1> OpHandlers {};
void RegisterALUHandlers();
void RegisterAtomicHandlers();
void RegisterBranchHandlers();
void RegisterConversionHandlers();
void RegisterFlagHandlers();
void RegisterMemoryHandlers();
void RegisterMiscHandlers();
void RegisterMoveHandlers();
void RegisterVectorHandlers();
void RegisterEncryptionHandlers();
void PushRegs();
void PopRegs();
#define DEF_OP(x) void Op_##x(IR::IROp_Header *IROp, IR::NodeID Node)
///< Unhandled handler
DEF_OP(Unhandled);
///< No-op Handler
DEF_OP(NoOp);
///< ALU Ops
DEF_OP(TruncElementPair);
DEF_OP(Constant);
DEF_OP(EntrypointOffset);
DEF_OP(InlineConstant);
DEF_OP(InlineEntrypointOffset);
DEF_OP(CycleCounter);
DEF_OP(Add);
DEF_OP(AddNZCV);
DEF_OP(TestNZ);
DEF_OP(Sub);
DEF_OP(SubNZCV);
DEF_OP(Neg);
DEF_OP(Abs);
DEF_OP(Mul);
DEF_OP(UMul);
DEF_OP(Div);
DEF_OP(UDiv);
DEF_OP(Rem);
DEF_OP(URem);
DEF_OP(MulH);
DEF_OP(UMulH);
DEF_OP(Or);
DEF_OP(Orlshl);
DEF_OP(Orlshr);
DEF_OP(And);
DEF_OP(Andn);
DEF_OP(Xor);
DEF_OP(Lshl);
DEF_OP(Lshr);
DEF_OP(Ashr);
DEF_OP(Rol);
DEF_OP(Ror);
DEF_OP(Extr);
DEF_OP(PDep);
DEF_OP(PExt);
DEF_OP(LDiv);
DEF_OP(LUDiv);
DEF_OP(LRem);
DEF_OP(LURem);
DEF_OP(Zext);
DEF_OP(Not);
DEF_OP(Popcount);
DEF_OP(FindLSB);
DEF_OP(FindMSB);
DEF_OP(FindTrailingZeroes);
DEF_OP(CountLeadingZeroes);
DEF_OP(Rev);
DEF_OP(Bfi);
DEF_OP(Bfxil);
DEF_OP(Bfe);
DEF_OP(Sbfe);
DEF_OP(Select);
DEF_OP(VExtractToGPR);
DEF_OP(Float_ToGPR_ZS);
DEF_OP(Float_ToGPR_S);
DEF_OP(FCmp);
DEF_OP(F80Cmp);
///< Atomic ops
DEF_OP(CASPair);
DEF_OP(CAS);
DEF_OP(AtomicAdd);
DEF_OP(AtomicSub);
DEF_OP(AtomicAnd);
DEF_OP(AtomicOr);
DEF_OP(AtomicXor);
DEF_OP(AtomicSwap);
DEF_OP(AtomicFetchAdd);
DEF_OP(AtomicFetchSub);
DEF_OP(AtomicFetchAnd);
DEF_OP(AtomicFetchCLR);
DEF_OP(AtomicFetchOr);
DEF_OP(AtomicFetchXor);
DEF_OP(AtomicFetchNeg);
DEF_OP(TelemetrySetValue);
///< Branch ops
DEF_OP(CallbackReturn);
DEF_OP(ExitFunction);
DEF_OP(Jump);
DEF_OP(CondJump);
DEF_OP(Syscall);
DEF_OP(Thunk);
DEF_OP(ValidateCode);
DEF_OP(ThreadRemoveCodeEntry);
DEF_OP(CPUID);
DEF_OP(XGETBV);
///< Conversion ops
DEF_OP(VInsGPR);
DEF_OP(VCastFromGPR);
DEF_OP(VDupFromGPR);
DEF_OP(Float_FromGPR_S);
DEF_OP(Float_FToF);
DEF_OP(Vector_UToF);
DEF_OP(Vector_SToF);
DEF_OP(Vector_FToZS);
DEF_OP(Vector_FToS);
DEF_OP(Vector_FToF);
DEF_OP(Vector_FToI);
///< Flag ops
DEF_OP(GetHostFlag);
///< Memory ops
DEF_OP(LoadContext);
DEF_OP(StoreContext);
DEF_OP(LoadRegister);
DEF_OP(StoreRegister);
DEF_OP(LoadContextIndexed);
DEF_OP(StoreContextIndexed);
DEF_OP(SpillRegister);
DEF_OP(FillRegister);
DEF_OP(LoadFlag);
DEF_OP(StoreFlag);
DEF_OP(LoadMem);
DEF_OP(StoreMem);
DEF_OP(VLoadVectorMasked);
DEF_OP(VStoreVectorMasked);
DEF_OP(VLoadVectorElement);
DEF_OP(VStoreVectorElement);
DEF_OP(VBroadcastFromMem);
DEF_OP(Push);
DEF_OP(MemSet);
DEF_OP(MemCpy);
DEF_OP(CacheLineClear);
DEF_OP(CacheLineClean);
DEF_OP(CacheLineZero);
///< Misc ops
DEF_OP(GuestOpcode);
DEF_OP(Fence);
DEF_OP(Break);
DEF_OP(Print);
DEF_OP(GetRoundingMode);
DEF_OP(SetRoundingMode);
DEF_OP(ProcessorID);
DEF_OP(RDRAND);
DEF_OP(Yield);
///< Move ops
DEF_OP(ExtractElementPair);
DEF_OP(CreateElementPair);
///< Vector ops
DEF_OP(VectorZero);
DEF_OP(VectorImm);
DEF_OP(LoadNamedVectorConstant);
DEF_OP(LoadNamedVectorIndexedConstant);
DEF_OP(VMov);
DEF_OP(VAnd);
DEF_OP(VBic);
DEF_OP(VOr);
DEF_OP(VXor);
DEF_OP(VAdd);
DEF_OP(VSub);
DEF_OP(VUQAdd);
DEF_OP(VUQSub);
DEF_OP(VSQAdd);
DEF_OP(VSQSub);
DEF_OP(VAddP);
DEF_OP(VAddV);
DEF_OP(VUMinV);
DEF_OP(VURAvg);
DEF_OP(VAbs);
DEF_OP(VPopcount);
DEF_OP(VFAdd);
DEF_OP(VFAddP);
DEF_OP(VFSub);
DEF_OP(VFMul);
DEF_OP(VFDiv);
DEF_OP(VFMin);
DEF_OP(VFMax);
DEF_OP(VFRecp);
DEF_OP(VFSqrt);
DEF_OP(VFRSqrt);
DEF_OP(VNeg);
DEF_OP(VFNeg);
DEF_OP(VNot);
DEF_OP(VUMin);
DEF_OP(VSMin);
DEF_OP(VUMax);
DEF_OP(VSMax);
DEF_OP(VZip);
DEF_OP(VZip2);
DEF_OP(VUnZip);
DEF_OP(VUnZip2);
DEF_OP(VTrn);
DEF_OP(VTrn2);
DEF_OP(VBSL);
DEF_OP(VCMPEQ);
DEF_OP(VCMPEQZ);
DEF_OP(VCMPGT);
DEF_OP(VCMPGTZ);
DEF_OP(VCMPLTZ);
DEF_OP(VFCMPEQ);
DEF_OP(VFCMPNEQ);
DEF_OP(VFCMPLT);
DEF_OP(VFCMPGT);
DEF_OP(VFCMPLE);
DEF_OP(VFCMPORD);
DEF_OP(VFCMPUNO);
DEF_OP(VUShl);
DEF_OP(VUShr);
DEF_OP(VSShr);
DEF_OP(VUShlS);
DEF_OP(VUShrS);
DEF_OP(VSShrS);
DEF_OP(VInsElement);
DEF_OP(VDupElement);
DEF_OP(VExtr);
DEF_OP(VUShrI);
DEF_OP(VSShrI);
DEF_OP(VShlI);
DEF_OP(VUShrNI);
DEF_OP(VUShrNI2);
DEF_OP(VSXTL);
DEF_OP(VSXTL2);
DEF_OP(VUXTL);
DEF_OP(VUXTL2);
DEF_OP(VSQXTN);
DEF_OP(VSQXTN2);
DEF_OP(VSQXTNPair);
DEF_OP(VSQXTUN);
DEF_OP(VSQXTUN2);
DEF_OP(VSQXTUNPair);
DEF_OP(VMul);
DEF_OP(VUMull);
DEF_OP(VSMull);
DEF_OP(VUMull2);
DEF_OP(VSMull2);
DEF_OP(VUMulH);
DEF_OP(VSMulH);
DEF_OP(VUABDL);
DEF_OP(VUABDL2);
DEF_OP(VTBL1);
DEF_OP(VRev32);
DEF_OP(VRev64);
///< Encryption ops
DEF_OP(AESImc);
DEF_OP(AESEnc);
DEF_OP(AESEncLast);
DEF_OP(AESDec);
DEF_OP(AESDecLast);
DEF_OP(AESKeyGenAssist);
DEF_OP(CRC32);
DEF_OP(PCLMUL);
#undef DEF_OP
};
}
FEXCore/Source/Interface/Core/JIT/x86_64/MemoryOps.cpp
-1301
View File
File diff suppressed because it is too large. Load diff
FEXCore/Source/Interface/Core/JIT/x86_64/MiscOps.cpp
-190
View File
@@ -1,190 +0,0 @@
// SPDX-License-Identifier: MIT
/*
$info$
tags: backend|x86-64
$end_info$
*/
#include "Interface/Context/Context.h"
#include "Interface/Core/Dispatcher/Dispatcher.h"
#include "Interface/Core/Dispatcher/X86Dispatcher.h"
#include "Interface/Core/JIT/x86_64/JITClass.h"
#include "FEXCore/Debug/InternalThreadState.h"
#include <FEXCore/Core/CoreState.h>
#include <FEXCore/Utils/LogManager.h>
#include <FEXCore/IR/IR.h>
#include <array>
#include <stddef.h>
#include <stdint.h>
namespace FEXCore::CPU {
#define DEF_OP(x) void X86JITCore::Op_##x(IR::IROp_Header *IROp, IR::NodeID Node)
DEF_OP(GuestOpcode) {
auto Op = IROp->C<IR::IROp_GuestOpcode>();
// metadata
DebugData->GuestOpcodes.push_back({Op->GuestEntryOffset, getCurr<uint8_t*>() - CodeData.BlockBegin});
}
DEF_OP(Fence) {
auto Op = IROp->C<IR::IROp_Fence>();
switch (Op->Fence) {
case IR::Fence_Load.Val:
lfence();
break;
case IR::Fence_LoadStore.Val:
mfence();
break;
case IR::Fence_Store.Val:
sfence();
break;
default: LOGMAN_MSG_A_FMT("Unknown Fence: {}", Op->Fence); break;
}
}
#ifndef _WIN32
DEF_OP(Break) {
auto Op = IROp->C<IR::IROp_Break>();
if (SpillSlots) {
add(rsp, SpillSlots * MaxSpillSlotSize);
}
Core::CpuStateFrame::SynchronousFaultDataStruct State = {
.FaultToTopAndGeneratedException = 1,
.Signal = Op->Reason.Signal,
.TrapNo = Op->Reason.TrapNumber,
.si_code = Op->Reason.si_code,
.err_code = Op->Reason.ErrorRegister,
};
uint64_t Constant{};
memcpy(&Constant, &State, sizeof(State));
mov(TMP1, Constant);
mov(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, SynchronousFaultData)], TMP1);
switch (Op->Reason.Signal) {
case SIGILL:
jmp(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.GuestSignal_SIGILL)]);
break;
case SIGTRAP:
jmp(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.GuestSignal_SIGTRAP)]);
break;
case SIGSEGV:
jmp(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.GuestSignal_SIGSEGV)]);
break;
default:
jmp(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.GuestSignal_SIGTRAP)]);
break;
}
}
#else
DEF_OP(Break) {
ERROR_AND_DIE_FMT("Unsupported");
}
#endif
DEF_OP(GetRoundingMode) {
auto Dst = GetDst<RA_32>(Node);
sub(rsp, 4);
// Only stores to memory
stmxcsr(dword [rsp]);
mov(Dst, dword [rsp]);
add(rsp, 4);
shr(Dst, 13);
}
DEF_OP(SetRoundingMode) {
auto Op = IROp->C<IR::IROp_SetRoundingMode>();
auto Src = GetSrc<RA_32>(Op->RoundMode.ID());
// Load old mxcsr
// Only stores to memory
sub(rsp, 4);
stmxcsr(dword [rsp]);
mov(TMP1.cvt32(), dword [rsp]);
// Insert the new rounding mode
and_(TMP1.cvt32(), ~(0b111 << 13));
mov(TMP2.cvt32(), Src);
shl(TMP2.cvt32(), 13);
or_(TMP1.cvt32(), TMP2.cvt32());
// Store it to mxcsr
// Only loads from memory
mov(dword [rsp], TMP1.cvt32());
ldmxcsr(dword [rsp]);
add(rsp, 4);
}
DEF_OP(Print) {
auto Op = IROp->C<IR::IROp_Print>();
PushRegs();
if (IsGPR(Op->Value.ID())) {
mov (rdi, GetSrc<RA_64>(Op->Value.ID()));
call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.PrintValue)]);
}
else {
pextrq(rdi, GetSrc(Op->Value.ID()), 0);
pextrq(rsi, GetSrc(Op->Value.ID()), 1);
call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.PrintVectorValue)]);
}
PopRegs();
}
DEF_OP(ProcessorID) {
// Cyclecounter in EDX:EAX
// IA32_TSC_AUX in ECX
rdtscp();
mov (GetDst<RA_32>(Node), ecx);
}
DEF_OP(RDRAND) {
auto Op = IROp->C<IR::IROp_RDRAND>();
auto Dst = GetSrcPair<RA_64>(Node);
if (Op->GetReseeded) {
rdrand(Dst.first);
}
else {
rdseed(Dst.first);
}
// In the case of RDRAND or RDSEED returning a valid number then CF = 1, else 0
mov (Dst.second, 0);
setc(Dst.second.cvt8());
}
DEF_OP(Yield) {
pause();
}
#undef DEF_OP
void X86JITCore::RegisterMiscHandlers() {
#define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &X86JITCore::Op_##x
REGISTER_OP(DUMMY, NoOp);
REGISTER_OP(IRHEADER, NoOp);
REGISTER_OP(CODEBLOCK, NoOp);
REGISTER_OP(BEGINBLOCK, NoOp);
REGISTER_OP(ENDBLOCK, NoOp);
REGISTER_OP(GUESTOPCODE, GuestOpcode);
REGISTER_OP(FENCE, Fence);
REGISTER_OP(BREAK, Break);
REGISTER_OP(PRINT, Print);
REGISTER_OP(GETROUNDINGMODE, GetRoundingMode);
REGISTER_OP(SETROUNDINGMODE, SetRoundingMode);
REGISTER_OP(INVALIDATEFLAGS, NoOp);
REGISTER_OP(PROCESSORID, ProcessorID);
REGISTER_OP(RDRAND, RDRAND);
REGISTER_OP(YIELD, Yield);
#undef REGISTER_OP
}
}
FEXCore/Source/Interface/Core/JIT/x86_64/MoveOps.cpp
-85
View File
@@ -1,85 +0,0 @@
// SPDX-License-Identifier: MIT
/*
$info$
tags: backend|x86-64
$end_info$
*/
#include "Interface/Core/JIT/x86_64/JITClass.h"
#include "Interface/Core/Dispatcher/X86Dispatcher.h"
#include <FEXCore/Utils/LogManager.h>
#include <FEXCore/IR/IR.h>
#include <array>
#include <stdint.h>
#include <utility>
namespace FEXCore::CPU {
#define DEF_OP(x) void X86JITCore::Op_##x(IR::IROp_Header *IROp, IR::NodeID Node)
DEF_OP(ExtractElementPair) {
auto Op = IROp->C<IR::IROp_ExtractElementPair>();
switch (Op->Header.Size) {
case 4: {
auto Src = GetSrcPair<RA_32>(Op->Pair.ID());
std::array<Xbyak::Reg, 2> Regs = {Src.first, Src.second};
mov (GetDst<RA_32>(Node), Regs[Op->Element]);
break;
}
case 8: {
auto Src = GetSrcPair<RA_64>(Op->Pair.ID());
std::array<Xbyak::Reg, 2> Regs = {Src.first, Src.second};
mov (GetDst<RA_64>(Node), Regs[Op->Element]);
break;
}
default: LOGMAN_MSG_A_FMT("Unknown Size"); break;
}
}
DEF_OP(CreateElementPair) {
auto Op = IROp->C<IR::IROp_CreateElementPair>();
std::pair<Xbyak::Reg, Xbyak::Reg> Dst;
Xbyak::Reg RegFirst;
Xbyak::Reg RegSecond;
Xbyak::Reg RegTmp;
switch (IROp->ElementSize) {
case 4: {
Dst = GetSrcPair<RA_32>(Node);
RegFirst = GetSrc<RA_32>(Op->Lower.ID());
RegSecond = GetSrc<RA_32>(Op->Upper.ID());
RegTmp = eax;
break;
}
case 8: {
Dst = GetSrcPair<RA_64>(Node);
RegFirst = GetSrc<RA_64>(Op->Lower.ID());
RegSecond = GetSrc<RA_64>(Op->Upper.ID());
RegTmp = rax;
break;
}
default: LOGMAN_MSG_A_FMT("Unknown Size"); break;
}
if (Dst.first != RegSecond) {
mov(Dst.first, RegFirst);
mov(Dst.second, RegSecond);
} else if (Dst.second != RegFirst) {
mov(Dst.second, RegSecond);
mov(Dst.first, RegFirst);
} else {
mov(RegTmp, RegFirst);
mov(Dst.second, RegSecond);
mov(Dst.first, RegTmp);
}
}
#undef DEF_OP
void X86JITCore::RegisterMoveHandlers() {
#define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &X86JITCore::Op_##x
REGISTER_OP(EXTRACTELEMENTPAIR, ExtractElementPair);
REGISTER_OP(CREATEELEMENTPAIR, CreateElementPair);
#undef REGISTER_OP
}
}
FEXCore/Source/Interface/Core/JIT/x86_64/VectorOps.cpp
-5045
View File
File diff suppressed because it is too large. Load diff
FEXCore/Source/Interface/Core/JIT/x86_64/x64Relocations.cpp
-140
View File
@@ -1,140 +0,0 @@
// SPDX-License-Identifier: MIT
/*
$info$
tags: backend|x86-64
desc: relocation logic of the x86-64 splatter backend
$end_info$
*/
#include "Interface/Context/Context.h"
#include "Interface/Core/JIT/x86_64/JITClass.h"
#include "Interface/HLE/Thunks/Thunks.h"
namespace FEXCore::CPU {
uint64_t X86JITCore::GetNamedSymbolLiteral(FEXCore::CPU::RelocNamedSymbolLiteral::NamedSymbol Op) {
switch (Op) {
case FEXCore::CPU::RelocNamedSymbolLiteral::NamedSymbol::SYMBOL_LITERAL_EXITFUNCTION_LINKER:
return ThreadState->CurrentFrame->Pointers.Common.ExitFunctionLinker;
break;
default:
ERROR_AND_DIE_FMT("Unknown named symbol literal: {}", static_cast<uint32_t>(Op));
break;
}
return ~0ULL;
}
void X86JITCore::LoadConstantWithPadding(Xbyak::Reg Reg, uint64_t Constant) {
// The maximum size a move constant can be in bytes
// Need to NOP pad to this size to ensure backpatching is always the same size
// Calculated as:
// [Rex]
// [Mov op]
// [8 byte constant]
//
// All other move types are smaller than this. xbyak will use a NOP slide which is quite quick
constexpr static size_t MAX_MOVE_SIZE = 10;
auto StartingOffset = getSize();
mov(Reg, Constant);
auto MoveSize = getSize() - StartingOffset;
auto NOPPadSize = MAX_MOVE_SIZE - MoveSize;
nop(NOPPadSize);
}
X86JITCore::NamedSymbolLiteralPair X86JITCore::InsertNamedSymbolLiteral(FEXCore::CPU::RelocNamedSymbolLiteral::NamedSymbol Op) {
NamedSymbolLiteralPair Lit {
.MoveABI = {
.NamedSymbolLiteral = {
.Header = {
.Type = FEXCore::CPU::RelocationTypes::RELOC_NAMED_SYMBOL_LITERAL,
},
.Symbol = Op,
.Offset = 0,
},
},
};
return Lit;
}
void X86JITCore::PlaceNamedSymbolLiteral(NamedSymbolLiteralPair &Lit) {
// Offset is the offset from the entrypoint of the block
auto CurrentCursor = getSize();
Lit.MoveABI.NamedSymbolLiteral.Offset = CurrentCursor - CursorEntry;
uint64_t Pointer = GetNamedSymbolLiteral(Lit.MoveABI.NamedSymbolLiteral.Symbol);
L(Lit.Offset);
dq(Pointer);
Relocations.emplace_back(Lit.MoveABI);
}
void X86JITCore::InsertGuestRIPMove(Xbyak::Reg Reg, uint64_t Constant) {
Relocation MoveABI{};
MoveABI.GuestRIPMove.Header.Type = FEXCore::CPU::RelocationTypes::RELOC_GUEST_RIP_MOVE;
// Offset is the offset from the entrypoint of the block
auto CurrentCursor = getSize();
MoveABI.GuestRIPMove.Offset = CurrentCursor - CursorEntry;
MoveABI.GuestRIPMove.GuestRIP = Constant;
MoveABI.GuestRIPMove.RegisterIndex = Reg.getIdx();
if (CTX->Config.CacheObjectCodeCompilation()) {
LoadConstantWithPadding(Reg, Constant);
}
else {
mov(Reg, Constant);
}
Relocations.emplace_back(MoveABI);
}
bool X86JITCore::ApplyRelocations(uint64_t GuestEntry, uint64_t CodeEntry, uint64_t CursorEntry, size_t NumRelocations, const char* EntryRelocations) {
size_t DataIndex{};
for (size_t j = 0; j < NumRelocations; ++j) {
const FEXCore::CPU::Relocation *Reloc = reinterpret_cast<const FEXCore::CPU::Relocation *>(&EntryRelocations[DataIndex]);
LOGMAN_THROW_AA_FMT((DataIndex % alignof(Relocation)) == 0, "Alignment of relocation wasn't adhered to");
switch (Reloc->Header.Type) {
case FEXCore::CPU::RelocationTypes::RELOC_NAMED_SYMBOL_LITERAL: {
uint64_t Pointer = GetNamedSymbolLiteral(Reloc->NamedSymbolLiteral.Symbol);
// Relocation occurs at the cursorEntry + offset relative to that cursor.
setSize(CursorEntry + Reloc->NamedSymbolLiteral.Offset);
// Place the pointer
dq(Pointer);
DataIndex += sizeof(Reloc->NamedSymbolLiteral);
break;
}
case FEXCore::CPU::RelocationTypes::RELOC_NAMED_THUNK_MOVE: {
uint64_t Pointer = reinterpret_cast<uint64_t>(CTX->ThunkHandler->LookupThunk(Reloc->NamedThunkMove.Symbol));
if (Pointer == ~0ULL) {
return false;
}
// Relocation occurs at the cursorEntry + offset relative to that cursor.
setSize(CursorEntry + Reloc->NamedThunkMove.Offset);
LoadConstantWithPadding(Xbyak::Reg64(Reloc->NamedThunkMove.RegisterIndex), Pointer);
DataIndex += sizeof(Reloc->NamedThunkMove);
break;
}
case FEXCore::CPU::RelocationTypes::RELOC_GUEST_RIP_MOVE:
// XXX: Reenable once the JIT Object Cache is upstream
// XXX: Should spin the relocation list, create a list of guest RIP moves, and ask for them all once, reduces lock contention.
uint64_t Pointer = ~0ULL; // EmitterCTX->JITObjectCache->FindRelocatedRIP(Reloc->GuestRIPMove.GuestRIP);
if (Pointer == ~0ULL) {
return false;
}
// Relocation occurs at the cursorEntry + offset relative to that cursor.
setSize(CursorEntry + Reloc->GuestRIPMove.Offset);
LoadConstantWithPadding(Xbyak::Reg64(Reloc->GuestRIPMove.RegisterIndex), Pointer);
DataIndex += sizeof(Reloc->GuestRIPMove);
break;
}
}
return true;
}
}
FEXCore/Source/Interface/IR/Passes/ConstProp.cpp
-13
View File
@@ -7,13 +7,11 @@ $end_info$
*/
#if JIT_ARM64
//aarch64 heuristics
#include "aarch64/assembler-aarch64.h"
#include "aarch64/cpu-aarch64.h"
#include "aarch64/disasm-aarch64.h"
#include "aarch64/assembler-aarch64.h"
#endif
#include "Interface/IR/PassManager.h"
@@ -59,23 +57,12 @@ static bool HasConsecutiveBits(uint64_t imm, unsigned width) {
return ((imm ^ (imm >> 1)) & ((1ULL << (width - 1)) - 1)) == 0;
}
#if JIT_ARM64
//aarch64 heuristics
static bool IsImmLogical(uint64_t imm, unsigned width) { if (width < 32) width = 32; return vixl::aarch64::Assembler::IsImmLogical(imm, width); }
static bool IsImmAddSub(uint64_t imm) { return vixl::aarch64::Assembler::IsImmAddSub(imm); }
static bool IsMemoryScale(uint64_t Scale, uint8_t AccessSize) {
return Scale == AccessSize;
}
#elif JIT_X86_64
// very lazy heuristics
static bool IsImmLogical(uint64_t imm, unsigned width) { return imm < 0x8000'0000; }
static bool IsImmAddSub(uint64_t imm) { return imm < 0x8000'0000; }
static bool IsMemoryScale(uint64_t Scale, uint8_t AccessSize) {
return Scale == 1 || Scale == 2 || Scale == 4 || Scale == 8;
}
#else
#error No inline constant heuristics for this target
#endif
static bool IsImmMemory(uint64_t imm, uint8_t AccessSize) {
if ( ((int64_t)imm >= -255) && ((int64_t)imm <= 256) )
unittests/32Bit_ASM/CMakeLists.txt
+11 -7
View File
@@ -48,14 +48,18 @@ foreach(ASM_SRC ${ASM_SOURCES})
list(APPEND ASM_DEPENDS "${OUTPUT_NAME};${OUTPUT_CONFIG_NAME}")
# Format is "<Test Arguments>" "<Test Name>"
set(TEST_ARGS
"--no-silent -g -c irjit -n 1 --no-multiblock" "jit_1" "jit"
"--no-silent -g -c irjit -n 500 --no-multiblock" "jit_500" "jit"
"--no-silent -g -c irjit -n 500 --multiblock" "jit_500_m" "jit"
)
set(TEST_ARGS)
if (_M_ARM_64 OR ENABLE_VIXL_SIMULATOR)
list(APPEND TEST_ARGS
"--no-silent -g -c irjit -n 1 --no-multiblock" "jit_1" "jit"
"--no-silent -g -c irjit -n 500 --no-multiblock" "jit_500" "jit"
"--no-silent -g -c irjit -n 500 --multiblock" "jit_500_m" "jit"
)
endif()
if (_M_X86_64)
if (ENABLE_VIXL_SIMULATOR)
set(CPU_CLASS Simulator)
elseif (_M_X86_64)
list(APPEND TEST_ARGS
"--no-silent -g -c host" "host" "host"
)
unittests/ASM/CMakeLists.txt
+13 -11
View File
@@ -49,20 +49,22 @@ foreach(ASM_SRC ${ASM_SOURCES})
list(APPEND ASM_DEPENDS "${OUTPUT_NAME};${OUTPUT_CONFIG_NAME}")
# Format is "<Test Arguments>" "<Test Name>" "<Test Type>"
set(TEST_ARGS
"--no-silent -g -c irjit -n 1 --no-multiblock" "jit_1" "jit"
"--no-silent -g -c irjit -n 500 --no-multiblock" "jit_500" "jit"
"--no-silent -g -c irjit -n 500 --multiblock" "jit_500_m" "jit"
)
set(TEST_ARGS)
if (_M_ARM_64 OR ENABLE_VIXL_SIMULATOR)
list(APPEND TEST_ARGS
"--no-silent -g -c irjit -n 1 --no-multiblock" "jit_1" "jit"
"--no-silent -g -c irjit -n 500 --no-multiblock" "jit_500" "jit"
"--no-silent -g -c irjit -n 500 --multiblock" "jit_500_m" "jit"
)
endif()
if (ENABLE_VIXL_SIMULATOR)
set(CPU_CLASS Simulator)
else()
if (_M_X86_64)
list(APPEND TEST_ARGS
"--no-silent -g -c host" "host" "host"
)
endif()
elseif (_M_X86_64)
list(APPEND TEST_ARGS
"--no-silent -g -c host" "host" "host"
)
endif()
if (NOT MINGW_BUILD)
unittests/CMakeLists.txt
+1 -1
View File
@@ -19,7 +19,7 @@ endif()
add_subdirectory(ASM/)
add_subdirectory(32Bit_ASM/)
if (ENABLE_VIXL_DISASSEMBLER AND (ENABLE_JIT_ARM64 OR CMAKE_SYSTEM_PROCESSOR MATCHES "^aarch64|^arm64|^armv8\.*") AND NOT ENABLE_JIT_X86_64)
if (ENABLE_VIXL_DISASSEMBLER AND (ENABLE_JIT_ARM64 OR CMAKE_SYSTEM_PROCESSOR MATCHES "^aarch64|^arm64|^armv8\.*"))
# Tests are only valid to run if the vixl disassembler is enabled and the active JIT is the ARM64 JIT.
add_subdirectory(InstructionCountCI/)
endif()