diff --git a/.github/workflows/ccpp.yml b/.github/workflows/ccpp.yml index a9dde32bd..4776f35e5 100644 --- a/.github/workflows/ccpp.yml +++ b/.github/workflows/ccpp.yml @@ -21,7 +21,7 @@ jobs: runs-on: ${{ matrix.arch }} strategy: matrix: - arch: [[self-hosted, x64], [self-hosted, ARMv8.0], [self-hosted, ARMv8.2], [self-hosted, ARMv8.4]] + arch: [[self-hosted, ARMv8.0], [self-hosted, ARMv8.2], [self-hosted, ARMv8.4]] fail-fast: false steps: diff --git a/.github/workflows/glibc_fault.yml b/.github/workflows/glibc_fault.yml index 549ab074e..1b4c84aa5 100644 --- a/.github/workflows/glibc_fault.yml +++ b/.github/workflows/glibc_fault.yml @@ -28,8 +28,7 @@ jobs: runs-on: ${{ matrix.arch }} strategy: matrix: - # Run on an x86 device and any ARM runner. - arch: [[self-hosted, x64], [self-hosted, ARM64]] + arch: [[self-hosted, ARM64]] fail-fast: false steps: diff --git a/.github/workflows/hostrunner.yml b/.github/workflows/hostrunner.yml new file mode 100644 index 000000000..2e9149db1 --- /dev/null +++ b/.github/workflows/hostrunner.yml @@ -0,0 +1,107 @@ +name: Hostrunner tests + +on: + push: + branches: + - main + pull_request: + branches: + - main + +env: + # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.) + BUILD_TYPE: Release + CC: clang + CXX: clang++ + FEX_ENABLEAVX: 1 + +jobs: + build: + runs-on: ${{ matrix.arch }} + strategy: + matrix: + arch: [[self-hosted, x64]] + fail-fast: false + + steps: + - uses: actions/checkout@v3 + + - name: Set runner label + run: echo "runner_label=${{ matrix.arch[1] }}" >> $GITHUB_ENV + + - name: Set rootfs paths + run: | + echo "FEX_ROOTFS_MOUNT=/mnt/AutoNFS/rootfs/" >> $GITHUB_ENV + echo "FEX_ROOTFS_PATH=$HOME/Rootfs/" >> $GITHUB_ENV + echo "FEX_ROOTFS=$HOME/Rootfs/" >> $GITHUB_ENV + echo "ROOTFS=$HOME/Rootfs/" >> $GITHUB_ENV + + - name: Update RootFS cache + # Use a bash shell so we can use the same syntax for environment variable + # access regardless of the host operating system + shell: bash + run: $GITHUB_WORKSPACE/Scripts/CI_FetchRootFS.py + + - name : submodule checkout + # Need to update submodules + run: | + git submodule sync --recursive + git submodule update --init --depth 1 + + - name: Clean Build Environment + run: rm -Rf ${{runner.workspace}}/build + + - name: Create Build Environment + # Some projects don't allow in-source building, so create a separate build directory + # We'll use this as our working directory for all subsequent commands + run: cmake -E make_directory ${{runner.workspace}}/build + + - name: Configure CMake + # Use a bash shell so we can use the same syntax for environment variable + # access regardless of the host operating system + shell: bash + working-directory: ${{runner.workspace}}/build + # Note the current convention is to use the -S and -B options here to specify source + # and build directories, but this is only available with CMake 3.13 and higher. + # The CMake binaries on the Github Actions machines are (as of this writing) 3.12 + run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -G Ninja -DENABLE_LTO=False -DENABLE_ASSERTIONS=True -DENABLE_X86_HOST_DEBUG=True + + - name: Build + working-directory: ${{runner.workspace}}/build + shell: bash + # Execute the build. You can specify a specific target with "--target " + run: cmake --build . --config $BUILD_TYPE + + - name: ASM Tests + working-directory: ${{runner.workspace}}/build + shell: bash + # Execute the unit tests + run: cmake --build . --config $BUILD_TYPE --target asm_tests + + - name: ASM Test Results move + if: ${{ always() }} + shell: bash + working-directory: ${{runner.workspace}}/build + run: mv ${{runner.workspace}}/build/Testing/Temporary/LastTest.log ${{runner.workspace}}/build/Testing/Temporary/LastTest_ASM.log || true + + - name: Truncate test results + if: ${{ always() }} + shell: bash + working-directory: ${{runner.workspace}}/build + # Cap out the log files at 20M in case something crash spins and dumps fault text + # ASM tests get quite close to 10MB + run: truncate --size=<20M ${{runner.workspace}}/build/Testing/Temporary/LastTest_*.log || true + + - name: Set runner name + if: ${{ always() }} + run: echo "runner_name=$(hostname)" >> $GITHUB_ENV + + - name: Upload results + if: ${{ always() }} + uses: 'actions/upload-artifact@v3' + timeout-minutes: 1 + with: + name: Results-${{ env.runner_name }} + path: ${{runner.workspace}}/build/Testing/Temporary/LastTest_*.log + retention-days: 3 + diff --git a/.github/workflows/mingw_build.yml b/.github/workflows/mingw_build.yml index b8e289578..9958da43d 100644 --- a/.github/workflows/mingw_build.yml +++ b/.github/workflows/mingw_build.yml @@ -17,7 +17,7 @@ jobs: runs-on: ${{ matrix.arch }} strategy: matrix: - arch: [[self-hosted, x64, mingw], [self-hosted, ARM64, mingw]] + arch: [[self-hosted, ARM64, mingw]] fail-fast: false steps: diff --git a/CMakeLists.txt b/CMakeLists.txt index 8150e25e2..58196db55 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -112,14 +112,6 @@ else() endif() if (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64") - option(ENABLE_X86_HOST_DEBUG "Enables compiling on x86_64 host" FALSE) - if (NOT ENABLE_X86_HOST_DEBUG) - message(FATAL_ERROR - " Be warned: FEX isn't optimized for x86_64 hosts!\n" - " Support for x86_64 hosts is only for debugging and convenience!\n" - " Don't expect amazing performance or optimal code generation!\n" - " Pass -DENABLE_X86_HOST_DEBUG=True to bypass this message!") - endif() set(_M_X86_64 1) add_definitions(-D_M_X86_64=1) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcx16") diff --git a/FEXCore/CMakeLists.txt b/FEXCore/CMakeLists.txt index 2dc062499..0b453bcc8 100644 --- a/FEXCore/CMakeLists.txt +++ b/FEXCore/CMakeLists.txt @@ -13,15 +13,6 @@ if (CMAKE_SYSTEM_PROCESSOR MATCHES "^aarch64|^arm64|^armv8\.*") set(_M_ARM_64 1) endif() -if (ENABLE_VIXL_SIMULATOR) - # If the vixl simulator is enabled then we are using the ARM64 JIT - option(ENABLE_JIT_X86_64 "Enable the x86_64 JIT" FALSE) - option(ENABLE_JIT_ARM64 "Enable the ARM64 JIT" TRUE) -else() - option(ENABLE_JIT_X86_64 "Enable the x86_64 JIT" ${_M_X86_64}) - option(ENABLE_JIT_ARM64 "Enable the ARM64 JIT" ${_M_ARM_64}) -endif() - option(ENABLE_CLANG_FORMAT "Run clang format over the source" FALSE) set(CMAKE_POSITION_INDEPENDENT_CODE ON) diff --git a/FEXCore/Source/CMakeLists.txt b/FEXCore/Source/CMakeLists.txt index d0eb2d2f4..376fefc02 100644 --- a/FEXCore/Source/CMakeLists.txt +++ b/FEXCore/Source/CMakeLists.txt @@ -107,9 +107,20 @@ set (SRCS Interface/Core/X86HelperGen.cpp Interface/Core/ArchHelpers/Arm64Emitter.cpp Interface/Core/Dispatcher/Dispatcher.cpp - Interface/Core/Dispatcher/X86Dispatcher.cpp Interface/Core/Dispatcher/Arm64Dispatcher.cpp Interface/Core/Interpreter/Fallbacks/InterpreterFallbacks.cpp + Interface/Core/JIT/Arm64/JIT.cpp + Interface/Core/JIT/Arm64/ALUOps.cpp + Interface/Core/JIT/Arm64/AtomicOps.cpp + Interface/Core/JIT/Arm64/BranchOps.cpp + Interface/Core/JIT/Arm64/ConversionOps.cpp + Interface/Core/JIT/Arm64/EncryptionOps.cpp + Interface/Core/JIT/Arm64/FlagOps.cpp + Interface/Core/JIT/Arm64/MemoryOps.cpp + Interface/Core/JIT/Arm64/MiscOps.cpp + Interface/Core/JIT/Arm64/MoveOps.cpp + Interface/Core/JIT/Arm64/VectorOps.cpp + Interface/Core/JIT/Arm64/Arm64Relocations.cpp Interface/Core/X86Tables/BaseTables.cpp Interface/Core/X86Tables/DDDTables.cpp Interface/Core/X86Tables/EVEXTables.cpp @@ -159,7 +170,7 @@ if (ENABLE_GLIBC_ALLOCATOR_HOOK_FAULT) Utils/AllocatorOverride.cpp) endif() -set(DEFINES -DTHREAD_LOCAL=_Thread_local) +set(DEFINES -DTHREAD_LOCAL=_Thread_local -DJIT_ARM64) if (_M_X86_64) list(APPEND DEFINES -D_M_X86_64=1) @@ -178,42 +189,6 @@ if (ENABLE_VIXL_DISASSEMBLER) list(APPEND DEFINES -DVIXL_DISASSEMBLER=1) endif() -if (ENABLE_JIT_X86_64) - list(APPEND SRCS - Interface/Core/JIT/x86_64/JIT.cpp - Interface/Core/JIT/x86_64/ALUOps.cpp - Interface/Core/JIT/x86_64/AtomicOps.cpp - Interface/Core/JIT/x86_64/BranchOps.cpp - Interface/Core/JIT/x86_64/ConversionOps.cpp - Interface/Core/JIT/x86_64/EncryptionOps.cpp - Interface/Core/JIT/x86_64/FlagOps.cpp - Interface/Core/JIT/x86_64/MemoryOps.cpp - Interface/Core/JIT/x86_64/MiscOps.cpp - Interface/Core/JIT/x86_64/MoveOps.cpp - Interface/Core/JIT/x86_64/VectorOps.cpp - Interface/Core/JIT/x86_64/x64Relocations.cpp - ) - list(APPEND DEFINES -DJIT_X86_64) -endif() - -if (ENABLE_JIT_ARM64) - list(APPEND DEFINES -DJIT_ARM64) - list(APPEND SRCS - Interface/Core/JIT/Arm64/JIT.cpp - Interface/Core/JIT/Arm64/ALUOps.cpp - Interface/Core/JIT/Arm64/AtomicOps.cpp - Interface/Core/JIT/Arm64/BranchOps.cpp - Interface/Core/JIT/Arm64/ConversionOps.cpp - Interface/Core/JIT/Arm64/EncryptionOps.cpp - Interface/Core/JIT/Arm64/FlagOps.cpp - Interface/Core/JIT/Arm64/MemoryOps.cpp - Interface/Core/JIT/Arm64/MiscOps.cpp - Interface/Core/JIT/Arm64/MoveOps.cpp - Interface/Core/JIT/Arm64/VectorOps.cpp - Interface/Core/JIT/Arm64/Arm64Relocations.cpp - ) -endif() - if (_M_ARM_64 AND HAS_CLANG_PRESERVE_ALL) list(APPEND DEFINES "-DFEXCORE_PRESERVE_ALL_ATTR=__attribute__((preserve_all));-DFEXCORE_HAS_PRESERVE_ALL_ATTR=1") else() diff --git a/FEXCore/Source/Interface/Core/CPUID.cpp b/FEXCore/Source/Interface/Core/CPUID.cpp index 5fb817f97..f968391fc 100644 --- a/FEXCore/Source/Interface/Core/CPUID.cpp +++ b/FEXCore/Source/Interface/Core/CPUID.cpp @@ -21,9 +21,6 @@ $end_info$ #include "git_version.h" #include -#ifdef _M_X86_64 -#include "Interface/Core/Dispatcher/X86Dispatcher.h" -#endif namespace FEXCore { namespace ProductNames { @@ -67,7 +64,6 @@ namespace ProductNames { static const char ARM_Firestorm[] = "Apple Firestorm"; static const char ARM_Icestorm[] = "Apple Icestorm"; #else - static const char UNKNOWN[] = "Unknown CPU"; #endif } @@ -343,33 +339,10 @@ void CPUIDEmu::SetupHostHybridFlag() { #else static uint32_t GetCycleCounterFrequency() { - uint32_t data[4]; - Xbyak::util::Cpu::getCpuid(0, data); - if (data[0] >= 0x15) { - Xbyak::util::Cpu::getCpuid(0x15, data); - - if (data[0] && data[1] && data[2]) { - return data[2] * data[1] / data[0]; - } - } return 0; } void CPUIDEmu::SetupHostHybridFlag() { - uint32_t data[4]; - Xbyak::util::Cpu::getCpuid(0, data); - if (data[0] >= 0x7) { - Xbyak::util::Cpu::getCpuid(0x7, data); - // Bit 15 of edx claims hybrid CPU - Hybrid = (data[3] & (1U << 15)) != 0; - } - - size_t CPUs = FEXCore::CPUInfo::CalculateNumberOfCPUs(); - PerCPUData.resize(CPUs); - for (size_t i = 0; i < CPUs; ++i) { - PerCPUData[i].IsBig = true; - PerCPUData[i].ProductName = ProductNames::UNKNOWN; - } } #endif diff --git a/FEXCore/Source/Interface/Core/Core.cpp b/FEXCore/Source/Interface/Core/Core.cpp index 091455cb2..25a7505b6 100644 --- a/FEXCore/Source/Interface/Core/Core.cpp +++ b/FEXCore/Source/Interface/Core/Core.cpp @@ -308,13 +308,7 @@ namespace FEXCore::Context { // Initialize the CPU core signal handlers & DispatcherConfig switch (Config.Core) { case FEXCore::Config::CONFIG_IRJIT: -#if (_M_X86_64 && JIT_X86_64) - BackendFeatures = FEXCore::CPU::GetX86JITBackendFeatures(); -#elif (_M_ARM_64 && JIT_ARM64) || defined(VIXL_SIMULATOR) BackendFeatures = FEXCore::CPU::GetArm64JITBackendFeatures(); -#else - ERROR_AND_DIE_FMT("FEXCore has been compiled without a viable JIT core"); -#endif break; case FEXCore::Config::CONFIG_CUSTOM: // Do nothing @@ -325,14 +319,7 @@ namespace FEXCore::Context { } DispatcherConfig.StaticRegisterAllocation = Config.StaticRegisterAllocation && BackendFeatures.SupportsStaticRegisterAllocation; - -#if JIT_ARM64 Dispatcher = FEXCore::CPU::Dispatcher::CreateArm64(this, DispatcherConfig); -#elif JIT_X86_64 - Dispatcher = FEXCore::CPU::Dispatcher::CreateX86(this, DispatcherConfig); -#else - ERROR_AND_DIE_FMT("FEXCore has been compiled with an unknown target"); -#endif // Set up the SignalDelegator config since core is initialized. FEXCore::SignalDelegator::SignalDelegatorConfig SignalConfig { @@ -667,15 +654,7 @@ namespace FEXCore::Context { switch (Config.Core) { case FEXCore::Config::CONFIG_IRJIT: Thread->PassManager->InsertRegisterAllocationPass(DoSRA, HostFeatures.SupportsAVX); - -#if (_M_X86_64 && JIT_X86_64) - Thread->CPUBackend = FEXCore::CPU::CreateX86JITCore(this, Thread); -#elif (_M_ARM_64 && JIT_ARM64) || defined(VIXL_SIMULATOR) Thread->CPUBackend = FEXCore::CPU::CreateArm64JITCore(this, Thread); -#else - ERROR_AND_DIE_FMT("FEXCore has been compiled without a viable JIT core"); -#endif - break; case FEXCore::Config::CONFIG_CUSTOM: Thread->CPUBackend = CustomCPUFactory(this, Thread); diff --git a/FEXCore/Source/Interface/Core/Dispatcher/X86Dispatcher.cpp b/FEXCore/Source/Interface/Core/Dispatcher/X86Dispatcher.cpp deleted file mode 100644 index 3f4f5756c..000000000 --- a/FEXCore/Source/Interface/Core/Dispatcher/X86Dispatcher.cpp +++ /dev/null @@ -1,409 +0,0 @@ -// SPDX-License-Identifier: MIT -#include "FEXCore/Utils/AllocatorHooks.h" -#include "Interface/Core/LookupCache.h" - -#include "Interface/Core/Dispatcher/X86Dispatcher.h" - -#include "Interface/Core/X86HelperGen.h" -#include "Interface/Context/Context.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -namespace FEXCore::CPU { -static constexpr size_t MAX_DISPATCHER_CODE_SIZE = 4096; -#define STATE r14 - -X86Dispatcher::X86Dispatcher(FEXCore::Context::ContextImpl *ctx, const DispatcherConfig &config) - : Dispatcher(ctx, config) - , Xbyak::CodeGenerator(MAX_DISPATCHER_CODE_SIZE, - FEXCore::Allocator::VirtualAlloc(MAX_DISPATCHER_CODE_SIZE, true), - nullptr) { - - LOGMAN_THROW_AA_FMT(!config.StaticRegisterAllocation, "X86 dispatcher does not support SRA"); - - using namespace Xbyak; - using namespace Xbyak::util; - DispatchPtr = getCurr(); - - // Temp registers - // rax, rcx, rdx, rsi, r8, r9, - // r10, r11 - // - // Callee Saved - // rbx, rbp, r12, r13, r14, r15 - // - // 1St Argument: rdi - // XMM: - // All temp - - // while (true) { - // Ptr = FindBlock(RIP) - // if (!Ptr) - // Ptr = CTX->CompileBlock(RIP); - // - // if (Ptr) - // Ptr(); - // else - // { - // Ptr = FallbackCore->CompileBlock() - // if (Ptr) - // Ptr() - // else { - // ShouldStop = true; - // } - // } - // } - // Bunch of exit state stuff - - // x86-64 ABI has the stack aligned when /call/ happens - // Which means the destination has a misaligned stack at that point - push(rbx); - push(rbp); - push(r12); - push(r13); - push(r14); - push(r15); - sub(rsp, 8); - - mov(STATE, rdi); - - // Save this stack pointer so we can cleanly shutdown the emulation with a long jump - // regardless of where we were in the stack - mov(qword STATE_PTR(CpuStateFrame, ReturningStackLocation), rsp); - - Label LoopTop; - Label FullLookup; - Label NoBlock; - Label ExitBlock; - Label ThreadPauseHandler; - - L(LoopTop); - AbsoluteLoopTopAddressFillSRA = AbsoluteLoopTopAddress = getCurr(); - - { - // Load our RIP - mov(rdx, qword STATE_PTR(CPUState, rip)); - - // L1 Cache - mov(r13, qword STATE_PTR(CpuStateFrame, Pointers.Common.L1Pointer)); - mov(rax, rdx); - - and_(rax, LookupCache::L1_ENTRIES_MASK); - shl(rax, 4); - cmp(qword[r13 + rax + offsetof(FEXCore::LookupCache::LookupCacheEntry, GuestCode)], rdx); - jne(FullLookup); - - jmp(qword[r13 + rax + offsetof(FEXCore::LookupCache::LookupCacheEntry, HostCode)]); - - L(FullLookup); - mov(r13, qword STATE_PTR(CpuStateFrame, Pointers.Common.L2Pointer)); - - // Full lookup - uint64_t VirtualMemorySize = CTX->Config.VirtualMemSize; - mov(rax, rdx); - mov(rbx, VirtualMemorySize - 1); - and_(rax, rbx); - shr(rax, 12); - - // Load page pointer - mov(rdi, qword [r13 + rax * 8]); - - cmp(rdi, 0); - je(NoBlock); - - mov (rax, rdx); - and_(rax, 0x0FFF); - - shl(rax, (int)log2(sizeof(FEXCore::LookupCache::LookupCacheEntry))); - - // check for aliasing - mov(rcx, qword [rdi + rax + 8]); - cmp(rcx, rdx); - jne(NoBlock); - - // Load the block pointer - mov(rax, qword [rdi + rax]); - - cmp(rax, 0); - je(NoBlock); - - // Update L1 - mov(r13, qword STATE_PTR(CpuStateFrame, Pointers.Common.L1Pointer)); - mov(rcx, rdx); - and_(rcx, LookupCache::L1_ENTRIES_MASK); - shl(rcx, 1); - mov(qword[r13 + rcx*8 + 8], rdx); - mov(qword[r13 + rcx*8 + 0], rax); - - // Real block if we made it here - jmp(rax); - } - - { - L(ExitBlock); - ThreadStopHandlerAddress = getCurr(); - - add(rsp, 8); - - pop(r15); - pop(r14); - pop(r13); - pop(r12); - pop(rbp); - pop(rbx); - - ret(); - } - - // Block creation - { - L(NoBlock); - - inc(qword [STATE + offsetof(FEXCore::Core::CPUState, DeferredSignalRefCount)]); - - // {rdi, rsi, rdx} - mov(rdi, reinterpret_cast(CTX)); - mov(rsi, STATE); - mov(rax, GetCompileBlockPtr()); - - call(rax); - - dec(qword [STATE + offsetof(FEXCore::Core::CPUState, DeferredSignalRefCount)]); - - Label AfterStore; - // Skip the deferred fault address if the refcount isn't zero - jne(AfterStore); - mov(rax, qword [STATE + offsetof(FEXCore::Core::CPUState, DeferredSignalFaultAddress)]); - mov(rax, qword [rax]); - - L(AfterStore); - - // rdx already contains RIP here - jmp(LoopTop); - } - - { - ExitFunctionLinkerAddress = getCurr(); - inc(qword [STATE + offsetof(FEXCore::Core::CPUState, DeferredSignalRefCount)]); - - // {rdi, rsi} - mov(rdi, STATE); - mov(rsi, rax); // rax is set at the block end - - call(qword STATE_PTR(CpuStateFrame, Pointers.Common.ExitFunctionLink)); - - dec(qword [STATE + offsetof(FEXCore::Core::CPUState, DeferredSignalRefCount)]); - - Label AfterStore; - // Skip the deferred fault address if the refcount isn't zero - jne(AfterStore); - mov(rbx, qword [STATE + offsetof(FEXCore::Core::CPUState, DeferredSignalFaultAddress)]); - mov(qword [rbx], rbx); - - L(AfterStore); - - jmp(rax); - } - - { - // Pause handler - ThreadPauseHandlerAddress = getCurr(); - L(ThreadPauseHandler); - - mov(rdi, reinterpret_cast(CTX)); - mov(rsi, STATE); - mov(rax, reinterpret_cast(SleepThread)); - - call(rax); - - // XXX: Unsupported atm - PauseReturnInstruction = getCurr(); - ud2(); - } - - { - CallbackPtr = getCurr(); - - push(rbx); - push(rbp); - push(r12); - push(r13); - push(r14); - push(r15); - sub(rsp, 8); - - // First thing we need to move the thread state pointer back in to our register - mov(STATE, rdi); - // XXX: XMM? - - // Make sure to adjust the refcounter so we don't clear the cache now - add(qword STATE_PTR(CpuStateFrame, SignalHandlerRefCounter), 1); - - // Now push the callback return trampoline to the guest stack - // Guest will be misaligned because calling a thunk won't correct the guest's stack once we call the callback from the host - mov(rax, CTX->X86CodeGen.CallbackReturn); - - // Store the trampoline to the guest stack - // Guest stack is now correctly misaligned after a regular call instruction - sub(qword STATE_PTR(CpuStateFrame, State.gregs[X86State::REG_RSP]), 16); - mov(rbx, qword STATE_PTR(CpuStateFrame, State.gregs[X86State::REG_RSP])); - mov(qword [rbx], rax); - - // Store RIP to the context state - mov(qword STATE_PTR(CpuStateFrame, State.rip), rsi); - - // Back to the loop top now - jmp(LoopTop); - } - - { - // Signal return handler - SignalHandlerReturnAddress = getCurr(); - ud2(); - } - - { - // RT Signal return handler - SignalHandlerReturnAddressRT = getCurr(); - ud2(); - } - - { - // Guest SIGILL handler - // Needs to be distinct from the SignalHandlerReturnAddress - GuestSignal_SIGILL = getCurr(); - ud2(); - } - - { - // Guest SIGTRAP handler - // Needs to be distinct from the SignalHandlerReturnAddress - GuestSignal_SIGTRAP = getCurr(); - - // ud2 = SIGILL - // int3 = SIGTRAP - // hlt = SIGSEGV - int3(); - } - - { - // Guest SIGSEGV handler - // Needs to be distinct from the SignalHandlerReturnAddress - GuestSignal_SIGSEGV = getCurr(); - - // ud2 = SIGILL - // int3 = SIGTRAP - // hlt = SIGSEGV - hlt(); - } - - { - IntCallbackReturnAddress = getCurr(); -// using CallbackReturn = FEX_NAKED void(*)(FEXCore::Core::InternalThreadState *Thread, volatile void *Host_RSP); - - // rdi = thread - // rsi = rsp - - mov(rsp, rsi); - - // Now jump back to the thunk - // XXX: XMM? - add(rsp, 8); - - pop(r15); - pop(r14); - pop(r13); - pop(r12); - pop(rbp); - pop(rbx); - - ret(); - } - ready(); - - Start = reinterpret_cast(getCode()); - End = Start + getSize(); - - if (CTX->Config.BlockJITNaming()) { - fextl::string Name = fextl::fmt::format("Dispatch_{}", FHU::Syscalls::gettid()); - CTX->Symbols.RegisterNamedRegion(reinterpret_cast(Start), End-Start, Name); - } - if (CTX->Config.GlobalJITNaming()) { - CTX->Symbols.RegisterJITSpace(reinterpret_cast(Start), End-Start); - } - -} - -size_t X86Dispatcher::GenerateGDBPauseCheck(uint8_t *CodeBuffer, uint64_t GuestRIP) { - using namespace Xbyak; - using namespace Xbyak::util; - - Xbyak::CodeGenerator emit(1, &emit); // actual emit target set with setNewBuffer - emit.setNewBuffer(CodeBuffer, MaxGDBPauseCheckSize); - - Label RunBlock; - - // If we have a gdb server running then run in a less efficient mode that checks if we need to exit - // This happens when single stepping - static_assert(sizeof(CTX->Config.RunningMode) == 4, "This is expected to be size of 4"); - emit.mov(rax, reinterpret_cast(CTX)); - - // If the value == 0 then we don't need to stop - emit.cmp(dword [rax + (offsetof(FEXCore::Context::ContextImpl, Config.RunningMode))], 0); - emit.je(RunBlock); - { - // Make sure RIP is syncronized to the context - emit.mov(rax, GuestRIP); - emit.mov(qword STATE_PTR(CpuStateFrame, State.rip), rax); - - // Stop the thread - emit.mov(rax, qword STATE_PTR(CpuStateFrame, Pointers.Common.ThreadPauseHandlerSpillSRA)); - emit.jmp(rax); - } - - emit.L(RunBlock); - - emit.ready(); - - return emit.getSize(); -} - -X86Dispatcher::~X86Dispatcher() { - FEXCore::Allocator::VirtualFree(top_, MAX_DISPATCHER_CODE_SIZE); -} - -void X86Dispatcher::InitThreadPointers(FEXCore::Core::InternalThreadState *Thread) { - // Setup dispatcher specific pointers that need to be accessed from JIT code - { - auto &Common = Thread->CurrentFrame->Pointers.Common; - - Common.DispatcherLoopTop = AbsoluteLoopTopAddress; - Common.DispatcherLoopTopFillSRA = AbsoluteLoopTopAddressFillSRA; - Common.ExitFunctionLinker = ExitFunctionLinkerAddress; - Common.ThreadStopHandlerSpillSRA = ThreadStopHandlerAddress; - Common.ThreadPauseHandlerSpillSRA = ThreadPauseHandlerAddress; - Common.GuestSignal_SIGILL = GuestSignal_SIGILL; - Common.GuestSignal_SIGTRAP = GuestSignal_SIGTRAP; - Common.GuestSignal_SIGSEGV = GuestSignal_SIGSEGV; - Common.SignalReturnHandler = SignalHandlerReturnAddress; - Common.SignalReturnHandlerRT = SignalHandlerReturnAddressRT; - } -} - -fextl::unique_ptr Dispatcher::CreateX86(FEXCore::Context::ContextImpl *CTX, const DispatcherConfig &Config) { - return fextl::make_unique(CTX, Config); -} - -} diff --git a/FEXCore/Source/Interface/Core/Dispatcher/X86Dispatcher.h b/FEXCore/Source/Interface/Core/Dispatcher/X86Dispatcher.h deleted file mode 100644 index eb6ba2af1..000000000 --- a/FEXCore/Source/Interface/Core/Dispatcher/X86Dispatcher.h +++ /dev/null @@ -1,42 +0,0 @@ -// SPDX-License-Identifier: MIT -#pragma once - -#include -#include -#include - -#include "Interface/Core/Dispatcher/Dispatcher.h" - -#define XBYAK64 -#define XBYAK_CUSTOM_ALLOC -#define XBYAK_CUSTOM_MALLOC FEXCore::Allocator::malloc -#define XBYAK_CUSTOM_FREE FEXCore::Allocator::free -#define XBYAK_CUSTOM_SETS -#define XBYAK_STD_UNORDERED_SET fextl::unordered_set -#define XBYAK_STD_UNORDERED_MAP fextl::unordered_map -#define XBYAK_STD_UNORDERED_MULTIMAP fextl::unordered_multimap -#define XBYAK_STD_LIST fextl::list -#define XBYAK_NO_EXCEPTION - -#include -#include - -namespace FEXCore::Core { - struct InternalThreadState; -} - -#define STATE_PTR(STATE_TYPE, FIELD) \ - [STATE + offsetof(FEXCore::Core::STATE_TYPE, FIELD)] - -namespace FEXCore::CPU { - -class X86Dispatcher final : public Dispatcher, public Xbyak::CodeGenerator { - public: - X86Dispatcher(FEXCore::Context::ContextImpl *ctx, const DispatcherConfig &config); - void InitThreadPointers(FEXCore::Core::InternalThreadState *Thread) override; - size_t GenerateGDBPauseCheck(uint8_t *CodeBuffer, uint64_t GuestRIP) override; - - virtual ~X86Dispatcher() override; -}; - -} diff --git a/FEXCore/Source/Interface/Core/HostFeatures.cpp b/FEXCore/Source/Interface/Core/HostFeatures.cpp index 51c2b1bb8..3a0922716 100644 --- a/FEXCore/Source/Interface/Core/HostFeatures.cpp +++ b/FEXCore/Source/Interface/Core/HostFeatures.cpp @@ -2,15 +2,28 @@ #include "Interface/Core/CPUID.h" #include -#if defined(_M_ARM_64) || defined(VIXL_SIMULATOR) #include "aarch64/assembler-aarch64.h" #include "aarch64/cpu-aarch64.h" #include "aarch64/disasm-aarch64.h" #include "aarch64/assembler-aarch64.h" -#endif #ifdef _M_X86_64 -#include "Interface/Core/Dispatcher/X86Dispatcher.h" +#define XBYAK64 +#define XBYAK_CUSTOM_ALLOC +#define XBYAK_CUSTOM_MALLOC FEXCore::Allocator::malloc +#define XBYAK_CUSTOM_FREE FEXCore::Allocator::free +#define XBYAK_CUSTOM_SETS +#define XBYAK_STD_UNORDERED_SET fextl::unordered_set +#define XBYAK_STD_UNORDERED_MAP fextl::unordered_map +#define XBYAK_STD_UNORDERED_MULTIMAP fextl::unordered_multimap +#define XBYAK_STD_LIST fextl::list +#define XBYAK_NO_EXCEPTION +#include +#include +#include + +#include +#include #endif namespace FEXCore { @@ -41,7 +54,6 @@ static void SetFPCR(uint64_t Value) { __asm ("msr FPCR, %[Value]" :: [Value] "r" (Value)); } - #else static uint32_t GetDCZID() { // Return unsupported @@ -190,17 +202,15 @@ static void OverrideFeatures(HostFeatures *Features) { } HostFeatures::HostFeatures() { -#if defined(_M_ARM_64) || defined(VIXL_SIMULATOR) #ifdef VIXL_SIMULATOR auto Features = vixl::CPUFeatures::All(); -#else -#ifndef _WIN32 +#elif !defined(_WIN32) auto Features = vixl::CPUFeatures::InferFromOS(); #else // Need to use ID registers in WINE. auto Features = vixl::CPUFeatures::InferFromIDRegisters(); #endif -#endif + SupportsAES = Features.Has(vixl::CPUFeatures::Feature::kAES); SupportsCRC = Features.Has(vixl::CPUFeatures::Feature::kCRC32); SupportsAtomics = Features.Has(vixl::CPUFeatures::Feature::kAtomics); @@ -266,22 +276,36 @@ HostFeatures::HostFeatures() { SetFPCR(OriginalFPCR); #endif +#ifdef VIXL_SIMULATOR + // simulator doesn't support dc(ZVA) + SupportsCLZERO = false; +#else + // Check if we can support cacheline clears + uint32_t DCZID = GetDCZID(); + if ((DCZID & DCZID_DZP_MASK) == 0) { + uint32_t DCZID_Log2 = DCZID & DCZID_BS_MASK; + uint32_t DCZID_Bytes = (1 << DCZID_Log2) * sizeof(uint32_t); + // If the DC ZVA size matches the emulated cache line size + // This means we can use the instruction + SupportsCLZERO = DCZID_Bytes == CPUIDEmu::CACHELINE_SIZE; + } #endif + #if defined(_M_X86_64) && !defined(VIXL_SIMULATOR) - Xbyak::util::Cpu Features{}; - SupportsAES = Features.has(Xbyak::util::Cpu::tAESNI); - SupportsCRC = Features.has(Xbyak::util::Cpu::tSSE42); - SupportsRAND = Features.has(Xbyak::util::Cpu::tRDRAND) && Features.has(Xbyak::util::Cpu::tRDSEED); + Xbyak::util::Cpu X86Features{}; + SupportsAES = X86Features.has(Xbyak::util::Cpu::tAESNI); + SupportsCRC = X86Features.has(Xbyak::util::Cpu::tSSE42); + SupportsRAND = X86Features.has(Xbyak::util::Cpu::tRDRAND) && X86Features.has(Xbyak::util::Cpu::tRDSEED); SupportsRCPC = true; SupportsTSOImm9 = true; - Supports3DNow = Features.has(Xbyak::util::Cpu::t3DN) && Features.has(Xbyak::util::Cpu::tE3DN); - SupportsSSE4A = Features.has(Xbyak::util::Cpu::tSSE4a); + Supports3DNow = X86Features.has(Xbyak::util::Cpu::t3DN) && X86Features.has(Xbyak::util::Cpu::tE3DN); + SupportsSSE4A = X86Features.has(Xbyak::util::Cpu::tSSE4a); SupportsAVX = true; - SupportsSHA = Features.has(Xbyak::util::Cpu::tSHA); - SupportsBMI1 = Features.has(Xbyak::util::Cpu::tBMI1); - SupportsBMI2 = Features.has(Xbyak::util::Cpu::tBMI2); - SupportsCLWB = Features.has(Xbyak::util::Cpu::tCLWB); - SupportsPMULL_128Bit = Features.has(Xbyak::util::Cpu::tPCLMULQDQ); + SupportsSHA = X86Features.has(Xbyak::util::Cpu::tSHA); + SupportsBMI1 = X86Features.has(Xbyak::util::Cpu::tBMI1); + SupportsBMI2 = X86Features.has(Xbyak::util::Cpu::tBMI2); + SupportsCLWB = X86Features.has(Xbyak::util::Cpu::tCLWB); + SupportsPMULL_128Bit = X86Features.has(Xbyak::util::Cpu::tPCLMULQDQ); // xbyak doesn't know how to check for CLZero // First ensure we support a new enough extended CPUID function range @@ -297,21 +321,6 @@ HostFeatures::HostFeatures() { SupportsFlushInputsToZero = true; SupportsFloatExceptions = true; #endif - -#ifdef VIXL_SIMULATOR - // simulator doesn't support dc(ZVA) - SupportsCLZERO = false; -#else - // Check if we can support cacheline clears - uint32_t DCZID = GetDCZID(); - if ((DCZID & DCZID_DZP_MASK) == 0) { - uint32_t DCZID_Log2 = DCZID & DCZID_BS_MASK; - uint32_t DCZID_Bytes = (1 << DCZID_Log2) * sizeof(uint32_t); - // If the DC ZVA size matches the emulated cache line size - // This means we can use the instruction - SupportsCLZERO = DCZID_Bytes == CPUIDEmu::CACHELINE_SIZE; - } -#endif OverrideFeatures(this); } } diff --git a/FEXCore/Source/Interface/Core/JIT/x86_64/ALUOps.cpp b/FEXCore/Source/Interface/Core/JIT/x86_64/ALUOps.cpp deleted file mode 100644 index 2a62e323a..000000000 --- a/FEXCore/Source/Interface/Core/JIT/x86_64/ALUOps.cpp +++ /dev/null @@ -1,1633 +0,0 @@ -// SPDX-License-Identifier: MIT -/* -$info$ -tags: backend|x86-64 -$end_info$ -*/ - -#include "Interface/Core/JIT/x86_64/JITClass.h" -#include "Interface/Core/Dispatcher/X86Dispatcher.h" - -#include -#include - -#include -#include -#include - -namespace FEXCore::CPU { - -#define GRS(Node) (IROp->Size <= 4 ? GetSrc(Node) : GetSrc(Node)) -#define GRD(Node) (IROp->Size <= 4 ? GetDst(Node) : GetDst(Node)) -#define GRCMP(Node) (Op->CompareSize == 4 ? GetSrc(Node) : GetSrc(Node)) - -#define DEF_OP(x) void X86JITCore::Op_##x(IR::IROp_Header *IROp, IR::NodeID Node) -DEF_OP(TruncElementPair) { - auto Op = IROp->C(); - - switch (IROp->Size) { - case 4: { - auto Dst = GetSrcPair(Node); - auto Src = GetSrcPair(Op->Pair.ID()); - mov(Dst.first, Src.first); - mov(Dst.second, Src.second); - break; - } - default: LOGMAN_MSG_A_FMT("Unhandled Truncation size: {}", IROp->Size); break; - } -} - -DEF_OP(Constant) { - auto Op = IROp->C(); - mov(GetDst(Node), Op->Constant); -} - -DEF_OP(EntrypointOffset) { - auto Op = IROp->C(); - - auto Constant = Entry + Op->Offset; - uint64_t Mask = ~0ULL; - uint8_t OpSize = IROp->Size; - if (OpSize == 4) { - Mask = 0xFFFF'FFFFULL; - } - - mov(GetDst(Node), Constant & Mask); -} - -DEF_OP(InlineConstant) { - //nop -} - -DEF_OP(InlineEntrypointOffset) { - //nop -} - -DEF_OP(CycleCounter) { -#ifdef DEBUG_CYCLES - mov (GetDst(Node), 0); -#else - rdtsc(); - shl(rdx, 32); - or_(rax, rdx); - mov (GetDst(Node), rax); -#endif -} - -DEF_OP(Add) { - auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; - - mov(rax, GetSrc(Op->Src1.ID())); - - uint64_t Const; - if (IsInlineConstant(Op->Src2, &Const)) { - switch (OpSize) { - case 4: - add(eax, Const); - break; - case 8: - add(rax, Const); - break; - default: LOGMAN_MSG_A_FMT("Unhandled Add size: {}", OpSize); - break; - } - } else { - switch (OpSize) { - case 4: - add(eax, GetSrc(Op->Src2.ID())); - break; - case 8: - add(rax, GetSrc(Op->Src2.ID())); - break; - default: LOGMAN_MSG_A_FMT("Unhandled Add size: {}", OpSize); - break; - } - } - mov(GetDst(Node), rax); -} - -DEF_OP(AddNZCV) { - auto Op = IROp->C(); - const uint8_t OpSize = Op->Size; - - // Results returned in Arm64 NZCV format - // N = Sign bit - // Z = Is Zero - // C = Carry occured (Unsigned result can't fit within resulting register) - // V = Overflow occured (Signed result can't fit in to resulting register) - - Xbyak::Reg Src2 = TMP2; - uint64_t Const; - if (IsInlineConstant(Op->Src2, &Const)) { - mov(Src2, Const); - } - else { - Src2 = GetSrc(Op->Src2.ID()); - } - - switch (OpSize) { - case 4: - mov(TMP1.cvt32(), GetSrc(Op->Src1.ID())); - add(TMP1.cvt32(), Src2.cvt32()); - break; - case 8: - mov(TMP1.cvt64(), GetSrc(Op->Src1.ID())); - add(TMP1.cvt64(), Src2.cvt64()); - break; - default: LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, OpSize); - break; - } - - mov(TMP1, 0); - mov(TMP2, 0); - mov(TMP3, 0); - mov(TMP4, 0); - sets(TMP1.cvt8()); - setz(TMP2.cvt8()); - setc(TMP3.cvt8()); - seto(TMP4.cvt8()); - // Flags NZCV in Tmps 1,2,3,4 respectively - shl(TMP1, 31); - shl(TMP2, 30); - shl(TMP3, 29); - shl(TMP4, 28); - or_(TMP1, TMP2); - or_(TMP1, TMP3); - or_(TMP1, TMP4); - mov(GetDst(Node), TMP1); -} - -DEF_OP(TestNZ) { - auto Op = IROp->C(); - const uint8_t OpSize = Op->Size; - - // Results returned in Arm64 NZCV format - // N = Sign bit - // Z = Is Zero - // CV = 00 - switch (OpSize) { - case 4: - mov(TMP1.cvt32(), GetSrc(Op->Src1.ID())); - shr(TMP1.cvt32(), OpSize * 8 - 1); - shl(TMP1.cvt32(), 31); - cmp(GetSrc(Op->Src1.ID()), 0); - mov(GetDst(Node), 0); - sete(GetDst(Node).cvt8()); - shl(GetDst(Node), 30); - or_(GetDst(Node), TMP1.cvt32()); - break; - case 8: - mov(TMP1, GetSrc(Op->Src1.ID())); - shr(TMP1, OpSize * 8 - 1); - shl(TMP1, 31); - cmp(GetSrc(Op->Src1.ID()), 0); - mov(GetDst(Node), 0); - sete(GetDst(Node).cvt8()); - shl(GetDst(Node), 30); - or_(GetDst(Node), TMP1); - break; - default: LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, OpSize); - break; - } -} - -DEF_OP(Sub) { - auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; - - mov(rax, GetSrc(Op->Src1.ID())); - - uint64_t Const; - if (IsInlineConstant(Op->Src2, &Const)) { - switch (OpSize) { - case 4: - sub(eax, Const); - break; - case 8: - sub(rax, Const); - break; - default: LOGMAN_MSG_A_FMT("Unhandled Sub size: {}", OpSize); - break; - } - } else { - switch (OpSize) { - case 4: - sub(eax, GetSrc(Op->Src2.ID())); - break; - case 8: - sub(rax, GetSrc(Op->Src2.ID())); - break; - default: LOGMAN_MSG_A_FMT("Unhandled Sub size: {}", OpSize); - break; - } - } - mov(GetDst(Node), rax); -} - -DEF_OP(SubNZCV) { - auto Op = IROp->C(); - const uint8_t OpSize = Op->Size; - - // Results returned in Arm64 NZCV format - // N = Sign bit - // Z = Is Zero - // C = Carry occured (Unsigned result can't fit within resulting register) - // V = Overflow occured (Signed result can't fit in to resulting register) - - Xbyak::Reg Src1 = TMP1; - Xbyak::Reg Src2 = TMP2; - uint64_t Const; - if (IsInlineConstant(Op->Src1, &Const)) { - mov(Src1, Const); - } - else { - Src1 = GetSrc(Op->Src1.ID()); - } - - if (IsInlineConstant(Op->Src2, &Const)) { - mov(Src2, Const); - } - else { - Src2 = GetSrc(Op->Src2.ID()); - } - - switch (OpSize) { - case 4: - cmp(Src1.cvt32(), Src2.cvt32()); - break; - case 8: - cmp(Src1.cvt64(), Src2.cvt64()); - break; - default: LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, OpSize); - break; - } - - if (!Op->InvertCarry) { - cmc(); - } - - mov(TMP1, 0); - mov(TMP2, 0); - mov(TMP3, 0); - mov(TMP4, 0); - sets(TMP1.cvt8()); - setz(TMP2.cvt8()); - setc(TMP3.cvt8()); - seto(TMP4.cvt8()); - // Flags NZCV in Tmps 1,2,3,4 respectively - shl(TMP1, 31); - shl(TMP2, 30); - shl(TMP3, 29); - shl(TMP4, 28); - or_(TMP1, TMP2); - or_(TMP1, TMP3); - or_(TMP1, TMP4); - mov(GetDst(Node), TMP1); -} - - -DEF_OP(Neg) { - auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; - - Xbyak::Reg Src; - Xbyak::Reg Dst; - switch (OpSize) { - case 4: - Src = GetSrc(Op->Src.ID()); - Dst = GetDst(Node); - break; - case 8: - Src = GetSrc(Op->Src.ID()); - Dst = GetDst(Node); - break; - default: LOGMAN_MSG_A_FMT("Unhandled Neg size: {}", OpSize); - break; - } - mov(Dst, Src); - neg(Dst); -} - -DEF_OP(Abs) { - auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; - - Xbyak::Reg Src; - Xbyak::Reg Dst; - switch (OpSize) { - case 4: - Src = GetSrc(Op->Src.ID()); - Dst = GetDst(Node); - break; - case 8: - Src = GetSrc(Op->Src.ID()); - Dst = GetDst(Node); - break; - default: LOGMAN_MSG_A_FMT("Unhandled Abs size: {}", OpSize); - break; - } - mov(Dst, Src); - mov(TMP1, Src); - neg(Dst); - cmovs(Dst, TMP1); -} - -DEF_OP(Mul) { - auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; - - auto Dst = GetDst(Node); - - switch (OpSize) { - case 4: - movsxd(rax, GetSrc(Op->Src1.ID())); - imul(eax, GetSrc(Op->Src2.ID())); - mov(Dst.cvt32(), eax); - break; - case 8: - mov(rax, GetSrc(Op->Src1.ID())); - imul(rax, GetSrc(Op->Src2.ID())); - mov(Dst, rax); - break; - default: LOGMAN_MSG_A_FMT("Unknown Mul size: {}", OpSize); - } -} - -DEF_OP(UMul) { - auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; - - switch (OpSize) { - case 4: - mov(rax, GetSrc(Op->Src1.ID())); - mul(GetSrc(Op->Src2.ID())); - mov(GetDst(Node), rax); - break; - case 8: - mov(rax, GetSrc(Op->Src1.ID())); - mul(GetSrc(Op->Src2.ID())); - mov(GetDst(Node), rax); - break; - default: LOGMAN_MSG_A_FMT("Unknown UMul size: {}", OpSize); - } -} - -DEF_OP(Div) { - auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; - - // Each source is OpSize in size - // So you can have up to a 128bit divide from x86-64 - auto Size = OpSize; - switch (Size) { - case 1: { - movsx(ax, GetSrc(Op->Src1.ID())); - idiv(GetSrc(Op->Src2.ID())); - movsx(GetDst(Node), al); - break; - } - case 2: { - mov (ax, GetSrc(Op->Src1.ID())); - cwd(); - idiv(GetSrc(Op->Src2.ID())); - movsx(GetDst(Node), ax); - break; - } - case 4: { - mov (eax, GetSrc(Op->Src1.ID())); - cdq(); - idiv(GetSrc(Op->Src2.ID())); - movsxd(GetDst(Node).cvt64(), eax); - break; - } - case 8: { - mov (rax, GetSrc(Op->Src1.ID())); - cqo(); - idiv(GetSrc(Op->Src2.ID())); - mov(GetDst(Node), rax); - break; - } - default: LOGMAN_MSG_A_FMT("Unknown DIV Size: {}", Size); break; - } -} - -DEF_OP(UDiv) { - auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; - - // Each source is OpSize in size - // So you can have up to a 128bit divide from x86-64 - switch (OpSize) { - case 1: { - mov (al, GetSrc(Op->Src1.ID())); - mov (edx, 0); - mov (cl, GetSrc(Op->Src2.ID())); - div(cl); - movzx(GetDst(Node), al); - break; - } - case 2: { - mov (ax, GetSrc(Op->Src1.ID())); - mov (edx, 0); - mov (cx, GetSrc(Op->Src2.ID())); - div(cx); - movzx(GetDst(Node), ax); - break; - } - case 4: { - mov (eax, GetSrc(Op->Src1.ID())); - mov (edx, 0); - mov (ecx, GetSrc(Op->Src2.ID())); - div(ecx); - mov(GetDst(Node), eax); - break; - } - case 8: { - mov (rax, GetSrc(Op->Src1.ID())); - mov (rdx, 0); - mov (rcx, GetSrc(Op->Src2.ID())); - div(rcx); - mov(GetDst(Node), rax); - break; - } - default: LOGMAN_MSG_A_FMT("Unknown UDIV OpSize: {}", OpSize); break; - } -} - -DEF_OP(Rem) { - auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; - - switch (OpSize) { - case 1: { - movsx(ax, GetSrc(Op->Src1.ID())); - idiv(GetSrc(Op->Src2.ID())); - mov(al, ah); - movsx(GetDst(Node), al); - break; - } - case 2: { - mov (ax, GetSrc(Op->Src1.ID())); - cwd(); - idiv(GetSrc(Op->Src2.ID())); - movsx(GetDst(Node), dx); - break; - } - case 4: { - mov (eax, GetSrc(Op->Src1.ID())); - cdq(); - idiv(GetSrc(Op->Src2.ID())); - movsxd(GetDst(Node).cvt64(), edx); - break; - } - case 8: { - mov (rax, GetSrc(Op->Src1.ID())); - cqo(); - idiv(GetSrc(Op->Src2.ID())); - mov(GetDst(Node), rdx); - break; - } - default: LOGMAN_MSG_A_FMT("Unknown Rem Size: {}", OpSize); break; - } -} - -DEF_OP(URem) { - auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; - - // Each source is OpSize in size - // So you can have up to a 128bit divide from x86-64 - switch (OpSize) { - case 1: { - mov (al, GetSrc(Op->Src1.ID())); - mov (edx, 0); - mov (cl, GetSrc(Op->Src2.ID())); - div(cl); - movzx(GetDst(Node), ah); - break; - } - case 2: { - mov (ax, GetSrc(Op->Src1.ID())); - mov (edx, 0); - mov (cx, GetSrc(Op->Src2.ID())); - div(cx); - movzx(GetDst(Node), dx); - break; - } - case 4: { - mov (eax, GetSrc(Op->Src1.ID())); - mov (edx, 0); - mov (ecx, GetSrc(Op->Src2.ID())); - div(ecx); - mov(GetDst(Node), edx); - break; - } - case 8: { - mov (rax, GetSrc(Op->Src1.ID())); - mov (rdx, 0); - mov (rcx, GetSrc(Op->Src2.ID())); - div(rcx); - mov(GetDst(Node), rdx); - break; - } - default: LOGMAN_MSG_A_FMT("Unknown URem OpSize: {}", OpSize); break; - } -} - -DEF_OP(MulH) { - auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; - - switch (OpSize) { - case 4: - movsxd(rax, GetSrc(Op->Src1.ID())); - imul(GetSrc(Op->Src2.ID())); - movsxd(GetDst(Node).cvt64(), edx); - break; - case 8: - mov(rax, GetSrc(Op->Src1.ID())); - imul(GetSrc(Op->Src2.ID())); - mov(GetDst(Node), rdx); - break; - default: LOGMAN_MSG_A_FMT("Unknown MulH size: {}", OpSize); - } -} - -DEF_OP(UMulH) { - auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; - - switch (OpSize) { - case 4: - mov(rax, GetSrc(Op->Src1.ID())); - mul(GetSrc(Op->Src2.ID())); - mov(GetDst(Node), rdx); - break; - case 8: - mov(rax, GetSrc(Op->Src1.ID())); - mul(GetSrc(Op->Src2.ID())); - mov(GetDst(Node), rdx); - break; - default: LOGMAN_MSG_A_FMT("Unknown UMulH size: {}", OpSize); - } -} - -DEF_OP(Or) { - auto Op = IROp->C(); - auto Dst = GetDst(Node); - mov(rax, GetSrc(Op->Src1.ID())); - - uint64_t Const; - if (IsInlineConstant(Op->Src2, &Const)) { - or_(rax, Const); - } else { - or_(rax, GetSrc(Op->Src2.ID())); - } - mov(Dst, rax); -} - -DEF_OP(Orlshl) { - auto Op = IROp->C(); - auto Dst = GetDst(Node); - const auto BitShift = Op->BitShift; - - uint64_t Const; - if (IsInlineConstant(Op->Src2, &Const)) { - if (IROp->Size == 8) { - mov(Dst, GetSrc(Op->Src1.ID())); - or_(Dst, Const << BitShift); - } - else { - mov(Dst.cvt32(), GetSrc(Op->Src1.ID())); - or_(Dst.cvt32(), Const << BitShift); - } - } else { - if (IROp->Size == 8) { - mov(TMP2, GetSrc(Op->Src2.ID())); - mov(Dst, GetSrc(Op->Src1.ID())); - shl(TMP2, BitShift); - or_(Dst, TMP2); - } - else { - mov(TMP2.cvt32(), GetSrc(Op->Src2.ID())); - mov(Dst.cvt32(), GetSrc(Op->Src1.ID())); - shl(TMP2.cvt32(), BitShift); - or_(Dst.cvt32(), TMP2.cvt32()); - } - } -} - -DEF_OP(Orlshr) { - auto Op = IROp->C(); - auto Dst = GetDst(Node); - const auto BitShift = Op->BitShift; - - uint64_t Const; - if (IsInlineConstant(Op->Src2, &Const)) { - if (IROp->Size == 8) { - mov(Dst, GetSrc(Op->Src1.ID())); - or_(Dst, Const >> BitShift); - } - else { - mov(Dst.cvt32(), GetSrc(Op->Src1.ID())); - or_(Dst.cvt32(), Const >> BitShift); - } - } else { - if (IROp->Size == 8) { - mov(TMP2, GetSrc(Op->Src2.ID())); - mov(Dst, GetSrc(Op->Src1.ID())); - shr(TMP2, BitShift); - or_(Dst, TMP2); - } - else { - mov(TMP2.cvt32(), GetSrc(Op->Src2.ID())); - mov(Dst.cvt32(), GetSrc(Op->Src1.ID())); - shr(TMP2.cvt32(), BitShift); - or_(Dst.cvt32(), TMP2.cvt32()); - } - } -} - -DEF_OP(And) { - auto Op = IROp->C(); - auto Dst = GetDst(Node); - mov(rax, GetSrc(Op->Src1.ID())); - uint64_t Const; - if (IsInlineConstant(Op->Src2, &Const)) { - and_(rax, Const); - } else { - and_(rax, GetSrc(Op->Src2.ID())); - } - mov(Dst, rax); -} - -DEF_OP(Andn) { - auto Op = IROp->C(); - const auto& Lhs = Op->Src1; - const auto& Rhs = Op->Src2; - auto Dst = GRD(Node); - - uint64_t Const{}; - if (IsInlineConstant(Rhs, &Const)) { - mov(Dst, GRS(Lhs.ID())); - and_(Dst, ~Const); - } else { - const auto Temp = IROp->Size <= 4 ? Xbyak::Reg{rax.cvt32()} : Xbyak::Reg{rax}; - mov(Temp, GRS(Rhs.ID())); - not_(Temp); - and_(Temp, GRS(Lhs.ID())); - mov(Dst, Temp); - } -} - -DEF_OP(Xor) { - auto Op = IROp->C(); - auto Dst = GetDst(Node); - mov(rax, GetSrc(Op->Src1.ID())); - uint64_t Const; - if (IsInlineConstant(Op->Src2, &Const)) { - xor_(rax, Const); - } else { - xor_(rax, GetSrc(Op->Src2.ID())); - } - mov(Dst, rax); -} - -DEF_OP(Lshl) { - auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; - const uint8_t Mask = OpSize * 8 - 1; - - uint64_t Const; - if (IsInlineConstant(Op->Src2, &Const)) { - Const &= Mask; - switch (OpSize) { - case 4: - mov(GetDst(Node), GetSrc(Op->Src1.ID())); - shl(GetDst(Node), Const); - break; - case 8: - mov(GetDst(Node), GetSrc(Op->Src1.ID())); - shl(GetDst(Node), Const); - break; - default: LOGMAN_MSG_A_FMT("Unknown LSHL Size: {}\n", OpSize); break; - }; - } else { - mov(rcx, GetSrc(Op->Src2.ID())); - and_(rcx, Mask); - - switch (OpSize) { - case 4: - mov(GetDst(Node), GetSrc(Op->Src1.ID())); - shl(GetDst(Node), cl); - break; - case 8: - mov(GetDst(Node), GetSrc(Op->Src1.ID())); - shl(GetDst(Node), cl); - break; - default: LOGMAN_MSG_A_FMT("Unknown LSHL Size: {}\n", OpSize); break; - }; - } -} - -DEF_OP(Lshr) { - auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; - const uint8_t Mask = OpSize * 8 - 1; - - uint64_t Const; - if (IsInlineConstant(Op->Src2, &Const)) { - Const &= Mask; - - switch (OpSize) { - case 1: - movzx(GetDst(Node), GetSrc(Op->Src1.ID())); - shr(GetDst(Node).cvt8(), Const); - break; - case 2: - movzx(GetDst(Node), GetSrc(Op->Src1.ID())); - shr(GetDst(Node).cvt16(), Const); - break; - case 4: - mov(GetDst(Node), GetSrc(Op->Src1.ID())); - shr(GetDst(Node), Const); - break; - case 8: - mov(GetDst(Node), GetSrc(Op->Src1.ID())); - shr(GetDst(Node), Const); - break; - default: LOGMAN_MSG_A_FMT("Unknown Size: {}\n", OpSize); break; - }; - - } else { - mov (rcx, GetSrc(Op->Src2.ID())); - and_(rcx, Mask); - - switch (OpSize) { - case 1: - movzx(GetDst(Node), GetSrc(Op->Src1.ID())); - shr(GetDst(Node).cvt8(), cl); - break; - case 2: - movzx(GetDst(Node), GetSrc(Op->Src1.ID())); - shr(GetDst(Node).cvt16(), cl); - break; - case 4: - mov(GetDst(Node), GetSrc(Op->Src1.ID())); - shr(GetDst(Node), cl); - break; - case 8: - mov(GetDst(Node), GetSrc(Op->Src1.ID())); - shr(GetDst(Node), cl); - break; - default: LOGMAN_MSG_A_FMT("Unknown Size: {}\n", OpSize); break; - }; - } -} - -DEF_OP(Ashr) { - auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; - const uint8_t Mask = OpSize * 8 - 1; - - uint64_t Const; - if (IsInlineConstant(Op->Src2, &Const)) { - Const &= Mask; - - switch (OpSize) { - case 1: - movsx(rax, GetSrc(Op->Src1.ID())); - sar(al, Const); - movzx(GetDst(Node), al); - break; - case 2: - movsx(rax, GetSrc(Op->Src1.ID())); - sar(ax, Const); - movzx(GetDst(Node), ax); - break; - case 4: - mov(GetDst(Node), GetSrc(Op->Src1.ID())); - sar(GetDst(Node), Const); - break; - case 8: - mov(GetDst(Node), GetSrc(Op->Src1.ID())); - sar(GetDst(Node), Const); - break; - default: LOGMAN_MSG_A_FMT("Unknown ASHR Size: {}\n", OpSize); break; - }; - - } else { - mov (rcx, GetSrc(Op->Src2.ID())); - and_(rcx, Mask); - switch (OpSize) { - case 1: - movsx(rax, GetSrc(Op->Src1.ID())); - sar(al, cl); - movzx(GetDst(Node), al); - break; - case 2: - movsx(rax, GetSrc(Op->Src1.ID())); - sar(ax, cl); - movzx(GetDst(Node), ax); - break; - case 4: - mov(GetDst(Node), GetSrc(Op->Src1.ID())); - sar(GetDst(Node), cl); - break; - case 8: - mov(GetDst(Node), GetSrc(Op->Src1.ID())); - sar(GetDst(Node), cl); - break; - default: LOGMAN_MSG_A_FMT("Unknown ASHR Size: {}\n", OpSize); break; - }; - } -} - -DEF_OP(Ror) { - auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; - const uint8_t Mask = OpSize * 8 - 1; - - uint64_t Const; - if (IsInlineConstant(Op->Src2, &Const)) { - Const &= Mask; - switch (OpSize) { - case 4: { - mov(eax, GetSrc(Op->Src1.ID())); - ror(eax, Const); - break; - } - case 8: { - mov(rax, GetSrc(Op->Src1.ID())); - ror(rax, Const); - break; - } - default: LOGMAN_MSG_A_FMT("Unknown ROR Size: {}\n", OpSize); break; - } - } else { - mov (rcx, GetSrc(Op->Src2.ID())); - and_(rcx, Mask); - switch (OpSize) { - case 4: { - mov(eax, GetSrc(Op->Src1.ID())); - ror(eax, cl); - break; - } - case 8: { - mov(rax, GetSrc(Op->Src1.ID())); - ror(rax, cl); - break; - } - default: LOGMAN_MSG_A_FMT("Unknown ROR Size: {}\n", OpSize); break; - } - } - mov(GetDst(Node), rax); -} - -DEF_OP(Extr) { - auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; - - switch (OpSize) { - case 4: { - mov(eax, GetSrc(Op->Upper.ID())); - mov(ecx, GetSrc(Op->Lower.ID())); - shrd(ecx, eax, Op->LSB); - mov(GetDst(Node), ecx); - break; - } - case 8: { - mov(rax, GetSrc(Op->Upper.ID())); - mov(rcx, GetSrc(Op->Lower.ID())); - shrd(rcx, rax, Op->LSB); - mov(GetDst(Node), rcx); - break; - } - } -} - -DEF_OP(PDep) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto Input = GRS(Op->Input.ID()); - const auto Mask = GRS(Op->Mask.ID()); - const auto Dest = GRD(Node); - - if (OpSize == 4) { - pdep(Dest.cvt32(), Input.cvt32(), Mask.cvt32()); - } else { - pdep(Dest.cvt64(), Input.cvt64(), Mask.cvt64()); - } -} - -DEF_OP(PExt) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto Input = GRS(Op->Input.ID()); - const auto Mask = GRS(Op->Mask.ID()); - const auto Dest = GRD(Node); - - if (OpSize == 4) { - pext(Dest.cvt32(), Input.cvt32(), Mask.cvt32()); - } else { - pext(Dest.cvt64(), Input.cvt64(), Mask.cvt64()); - } -} - -DEF_OP(LDiv) { - auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; - - // Each source is OpSize in size - // So you can have up to a 128bit divide from x86-64 - switch (OpSize) { - case 2: { - mov(eax, GetSrc(Op->Lower.ID())); - mov(edx, GetSrc(Op->Upper.ID())); - idiv(GetSrc(Op->Divisor.ID())); - movsx(GetDst(Node), ax); - break; - } - case 4: { - mov(eax, GetSrc(Op->Lower.ID())); - mov(edx, GetSrc(Op->Upper.ID())); - idiv(GetSrc(Op->Divisor.ID())); - movsxd(GetDst(Node).cvt64(), eax); - break; - } - case 8: { - mov(rax, GetSrc(Op->Lower.ID())); - mov(rdx, GetSrc(Op->Upper.ID())); - idiv(GetSrc(Op->Divisor.ID())); - mov(GetDst(Node), rax); - break; - } - default: LOGMAN_MSG_A_FMT("Unknown LDIV OpSize: {}", OpSize); break; - } -} - -DEF_OP(LUDiv) { - auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; - - // Each source is OpSize in size - // So you can have up to a 128bit divide from x86-64 - switch (OpSize) { - case 2: { - mov (ax, GetSrc(Op->Lower.ID())); - mov (dx, GetSrc(Op->Upper.ID())); - div(GetSrc(Op->Divisor.ID())); - movzx(GetDst(Node), ax); - break; - } - case 4: { - mov (eax, GetSrc(Op->Lower.ID())); - mov (edx, GetSrc(Op->Upper.ID())); - div(GetSrc(Op->Divisor.ID())); - mov(GetDst(Node), rax); - break; - } - case 8: { - mov (rax, GetSrc(Op->Lower.ID())); - mov (rdx, GetSrc(Op->Upper.ID())); - div(GetSrc(Op->Divisor.ID())); - mov(GetDst(Node), rax); - break; - } - default: LOGMAN_MSG_A_FMT("Unknown LUDIV OpSize: {}", OpSize); break; - } -} - -DEF_OP(LRem) { - auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; - - // Each source is OpSize in size - // So you can have up to a 128bit divide from x86-64 - switch (OpSize) { - case 2: { - mov(ax, GetSrc(Op->Lower.ID())); - mov(dx, GetSrc(Op->Upper.ID())); - idiv(GetSrc(Op->Divisor.ID())); - movsx(GetDst(Node), dx); - break; - } - case 4: { - mov(eax, GetSrc(Op->Lower.ID())); - mov(edx, GetSrc(Op->Upper.ID())); - idiv(GetSrc(Op->Divisor.ID())); - mov(GetDst(Node), rdx); - break; - } - case 8: { - mov(rax, GetSrc(Op->Lower.ID())); - mov(rdx, GetSrc(Op->Upper.ID())); - idiv(GetSrc(Op->Divisor.ID())); - mov(GetDst(Node), rdx); - break; - } - default: LOGMAN_MSG_A_FMT("Unknown LREM OpSize: {}", OpSize); break; - } -} - -DEF_OP(LURem) { - auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; - - // Each source is OpSize in size - // So you can have up to a 128bit divide from x86-64 - switch (OpSize) { - case 2: { - mov (ax, GetSrc(Op->Lower.ID())); - mov (dx, GetSrc(Op->Upper.ID())); - div(GetSrc(Op->Divisor.ID())); - movzx(GetDst(Node), dx); - break; - } - case 4: { - mov (eax, GetSrc(Op->Lower.ID())); - mov (edx, GetSrc(Op->Upper.ID())); - div(GetSrc(Op->Divisor.ID())); - mov(GetDst(Node), rdx); - break; - } - case 8: { - mov (rax, GetSrc(Op->Lower.ID())); - mov (rdx, GetSrc(Op->Upper.ID())); - div(GetSrc(Op->Divisor.ID())); - mov(GetDst(Node), rdx); - break; - } - default: LOGMAN_MSG_A_FMT("Unknown LUREM OpSize: {}", OpSize); break; - } -} - - -DEF_OP(Not) { - auto Op = IROp->C(); - auto Dst = GetDst(Node); - mov(Dst, GetSrc(Op->Src.ID())); - not_(Dst); -} - -DEF_OP(Popcount) { - auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; - - auto Dst64 = GetDst(Node); - - switch (OpSize) { - case 1: - movzx(GetDst(Node), GetSrc(Op->Src.ID())); - popcnt(Dst64, Dst64); - break; - case 2: { - movzx(GetDst(Node), GetSrc(Op->Src.ID())); - popcnt(Dst64, Dst64); - break; - } - case 4: - popcnt(GetDst(Node), GetSrc(Op->Src.ID())); - break; - case 8: - popcnt(GetDst(Node), GetSrc(Op->Src.ID())); - break; - } -} - -DEF_OP(FindLSB) { - auto Op = IROp->C(); - - bsf(rcx, GetSrc(Op->Src.ID())); - mov(rax, 0x40); - cmovz(rcx, rax); - xor_(rax, rax); - cmp(GetSrc(Op->Src.ID()), 1); - sbb(rax, rax); - or_(rax, rcx); - mov (GetDst(Node), rax); -} - -DEF_OP(FindMSB) { - auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; - - switch (OpSize) { - case 2: - bsr(GetDst(Node), GetSrc(Op->Src.ID())); - movzx(GetDst(Node), GetDst(Node)); - break; - case 4: - bsr(GetDst(Node), GetSrc(Op->Src.ID())); - break; - case 8: - bsr(GetDst(Node), GetSrc(Op->Src.ID())); - break; - default: LOGMAN_MSG_A_FMT("Unknown FindMSB OpSize: {}", OpSize); - } -} - -DEF_OP(FindTrailingZeroes) { - auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; - - switch (OpSize) { - case 2: - bsf(GetDst(Node), GetSrc(Op->Src.ID())); - mov(ax, 0x10); - cmovz(GetDst(Node), ax); - break; - case 4: - bsf(GetDst(Node), GetSrc(Op->Src.ID())); - mov(eax, 0x20); - cmovz(GetDst(Node), eax); - break; - case 8: - bsf(GetDst(Node), GetSrc(Op->Src.ID())); - mov(rax, 0x40); - cmovz(GetDst(Node), rax); - break; - default: LOGMAN_MSG_A_FMT("Unknown FindTrailingZeroes size: {}", OpSize); break; - } -} - -DEF_OP(CountLeadingZeroes) { - auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; - - if (Features.has(Xbyak::util::Cpu::tLZCNT)) { - switch (OpSize) { - case 2: { - lzcnt(GetDst(Node), GetSrc(Op->Src.ID())); - movzx(GetDst(Node), GetDst(Node)); - break; - } - case 4: { - lzcnt(GetDst(Node), GetSrc(Op->Src.ID())); - break; - } - case 8: { - lzcnt(GetDst(Node), GetSrc(Op->Src.ID())); - break; - } - default: LOGMAN_MSG_A_FMT("Unknown CountLeadingZeroes size: {}", OpSize); break; - } - } - else { - switch (OpSize) { - case 2: { - test(GetSrc(Op->Src.ID()), GetSrc(Op->Src.ID())); - mov(eax, 0x10); - Label Skip; - je(Skip); - bsr(ax, GetSrc(Op->Src.ID())); - xor_(ax, 0xF); - movzx(eax, ax); - L(Skip); - mov(GetDst(Node), eax); - break; - } - case 4: { - test(GetSrc(Op->Src.ID()), GetSrc(Op->Src.ID())); - mov(eax, 0x20); - Label Skip; - je(Skip); - bsr(eax, GetSrc(Op->Src.ID())); - xor_(eax, 0x1F); - L(Skip); - mov(GetDst(Node), eax); - break; - } - case 8: { - test(GetSrc(Op->Src.ID()), GetSrc(Op->Src.ID())); - mov(rax, 0x40); - Label Skip; - je(Skip); - bsr(rax, GetSrc(Op->Src.ID())); - xor_(rax, 0x3F); - L(Skip); - mov(GetDst(Node), rax); - break; - } - default: LOGMAN_MSG_A_FMT("Unknown CountLeadingZeroes size: {}", OpSize); break; - } - } -} - -DEF_OP(Rev) { - auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; - - switch (OpSize) { - case 2: - mov (GetDst(Node), GetSrc(Op->Src.ID())); - rol(GetDst(Node), 8); - break; - case 4: - mov (GetDst(Node), GetSrc(Op->Src.ID())); - bswap(GetDst(Node).cvt32()); - break; - case 8: - mov (GetDst(Node), GetSrc(Op->Src.ID())); - bswap(GetDst(Node).cvt64()); - break; - default: LOGMAN_MSG_A_FMT("Unknown REV size: {}", OpSize); break; - } -} - -DEF_OP(Bfi) { - auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; - - auto Dst = GetDst(Node); - - uint64_t SourceMask = (1ULL << Op->Width) - 1; - if (Op->Width == 64) { - SourceMask = ~0ULL; - } - const uint64_t DestMask = ~(SourceMask << Op->lsb); - - mov(TMP1, GetSrc(Op->Src.ID())); - mov(Dst, GetSrc(Op->Dest.ID())); - - mov(TMP2, DestMask); - and_(Dst, TMP2); - mov(TMP2, SourceMask); - and_(TMP1, TMP2); - shl(TMP1, Op->lsb); - or_(Dst, TMP1); - - if (OpSize != 8) { - mov(rcx, uint64_t((1ULL << (OpSize * 8)) - 1)); - and_(Dst, rcx); - } -} - -DEF_OP(Bfxil) { - auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; - - auto Dst = GetDst(Node); - - uint64_t SourceMask = (1ULL << Op->Width) - 1; - if (Op->Width == 64) { - SourceMask = ~0ULL; - } - const uint64_t DestMask = ~SourceMask; - - mov(TMP1, GetSrc(Op->Src.ID())); - shr(TMP1, Op->lsb); - mov(Dst, GetSrc(Op->Dest.ID())); - - mov(TMP2, DestMask); - and_(Dst, TMP2); - mov(TMP2, SourceMask); - and_(TMP1, TMP2); - or_(Dst, TMP1); - - if (OpSize != 8) { - mov(rcx, uint64_t((1ULL << (OpSize * 8)) - 1)); - and_(Dst, rcx); - } -} - -DEF_OP(Bfe) { - auto Op = IROp->C(); - LOGMAN_THROW_AA_FMT(IROp->Size <= 8, "OpSize is too large for BFE: {}", IROp->Size); - - auto Dst = GetDst(Node); - - // Special cases for fast extends - if (Op->lsb == 0) { - switch (Op->Width / 8) { - case 1: - movzx(Dst, GetSrc(Op->Src.ID())); - return; - case 2: - movzx(Dst, GetSrc(Op->Src.ID())); - return; - case 4: - mov(Dst.cvt32(), GetSrc(Op->Src.ID())); - return; - case 8: - mov(Dst, GetSrc(Op->Src.ID())); - return; - default: - // Need to use slower general case - break; - } - } - - mov(Dst, GetSrc(Op->Src.ID())); - - if (Op->lsb != 0) - shr(Dst, Op->lsb); - - if (Op->Width != 64) { - mov(rcx, uint64_t((1ULL << Op->Width) - 1)); - and_(Dst, rcx); - } -} - -DEF_OP(Sbfe) { - auto Op = IROp->C(); - auto Dst = GetDst(Node); - - // Special cases for fast signed extends - if (Op->lsb == 0) { - switch (Op->Width / 8) { - case 1: - movsx(Dst, GetSrc(Op->Src.ID())); - return; - case 2: - movsx(Dst, GetSrc(Op->Src.ID())); - return; - case 4: - movsxd(Dst.cvt64(), GetSrc(Op->Src.ID())); - return; - case 8: - mov(Dst, GetSrc(Op->Src.ID())); - return; - default: - // Need to use slower general case - break; - } - } - - // Slightly slower general case - { - uint64_t ShiftLeftAmount = (64 - (Op->Width + Op->lsb)); - uint64_t ShiftRightAmount = ShiftLeftAmount + Op->lsb; - mov(Dst, GetSrc(Op->Src.ID())); - shl(Dst, ShiftLeftAmount); - sar(Dst, ShiftRightAmount); - } -} - -DEF_OP(Select) { - auto Op = IROp->C(); - auto Dst = GRD(Node); - - if (IsGPR(Op->Cmp1.ID())) { - uint64_t Const; - if (IsInlineConstant(Op->Cmp2, &Const)) { - cmp(GRCMP(Op->Cmp1.ID()), Const); - } else { - cmp(GRCMP(Op->Cmp1.ID()), GRCMP(Op->Cmp2.ID())); - } - } - else if (IsGPRPair(Op->Cmp1.ID())) { - if (Op->CompareSize == 4) { - const auto Src1 = GetSrcPair(Op->Cmp1.ID()); - const auto Src2 = GetSrcPair(Op->Cmp2.ID()); - mov (TMP1.cvt32(), Src1.first); - mov (TMP2.cvt32(), Src1.second); - xor_(TMP1.cvt32(), Src2.first); - xor_(TMP2.cvt32(), Src2.second); - or_(TMP1.cvt32(), TMP2.cvt32()); - } - else { - const auto Src1 = GetSrcPair(Op->Cmp1.ID()); - const auto Src2 = GetSrcPair(Op->Cmp2.ID()); - mov (TMP1, Src1.first); - mov (TMP2, Src1.second); - xor_(TMP1, Src2.first); - xor_(TMP2, Src2.second); - or_(TMP1, TMP2); - } - } - else if (IsFPR(Op->Cmp1.ID())) { - if (Op->CompareSize == 4) - ucomiss(GetSrc(Op->Cmp1.ID()), GetSrc(Op->Cmp2.ID())); - else - ucomisd(GetSrc(Op->Cmp1.ID()), GetSrc(Op->Cmp2.ID())); - } - - uint64_t const_true, const_false; - bool is_const_true = IsInlineConstant(Op->TrueVal, &const_true); - bool is_const_false = IsInlineConstant(Op->FalseVal, &const_false); - - auto [SetCC, CMovCC, _] = GetCC(Op->Cond); - - if (is_const_true || is_const_false) { - if (is_const_false != true || is_const_true != true || const_true != 1 || const_false != 0) { - LOGMAN_MSG_A_FMT("Select: Unsupported compare inline parameters"); - } - (this->*SetCC)(al); - movzx(Dst, al); - } else { - mov(rax, GetSrc(Op->FalseVal.ID())); - (this->*CMovCC)(rax, GetSrc(Op->TrueVal.ID())); - mov (Dst, rax.changeBit(IROp->Size * 8)); - } -} - -DEF_OP(VExtractToGPR) { - const auto Op = IROp->C(); - - constexpr auto SSERegSize = Core::CPUState::XMM_SSE_REG_SIZE; - constexpr auto SSEBitSize = SSERegSize * 8; - - const auto ElementSize = Op->Header.ElementSize; - const auto ElementSizeBits = ElementSize * 8; - const auto Offset = ElementSizeBits * Op->Index; - - const auto Is256Bit = Offset >= SSEBitSize; - - const auto Vector = GetSrc(Op->Vector.ID()); - - switch (ElementSize) { - case 1: { - if (Is256Bit) { - vextracti128(xmm15, ToYMM(Vector), 1); - pextrb(GetDst(Node), xmm15, Op->Index - 16); - } else { - pextrb(GetDst(Node), Vector, Op->Index); - } - break; - } - case 2: { - if (Is256Bit) { - vextracti128(xmm15, ToYMM(Vector), 1); - pextrw(GetDst(Node), xmm15, Op->Index - 8); - } else { - pextrw(GetDst(Node), Vector, Op->Index); - } - break; - } - case 4: { - if (Is256Bit) { - vextracti128(xmm15, ToYMM(Vector), 1); - pextrd(GetDst(Node), xmm15, Op->Index - 4); - } else { - pextrd(GetDst(Node), Vector, Op->Index); - } - break; - } - case 8: { - if (Is256Bit) { - vextracti128(xmm15, ToYMM(Vector), 1); - pextrq(GetDst(Node), xmm15, Op->Index - 2); - } else { - pextrq(GetDst(Node), Vector, Op->Index); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } -} - -DEF_OP(Float_ToGPR_ZS) { - auto Op = IROp->C(); - const uint16_t Conv = (IROp->Size << 8) | Op->SrcElementSize; - - switch (Conv) { - case 0x0804: // int64_t <- float - cvttss2si(GetDst(Node), GetSrc(Op->Scalar.ID())); - break; - case 0x0808: // int64_t <- double - cvttsd2si(GetDst(Node), GetSrc(Op->Scalar.ID())); - break; - case 0x0404: // int32_t <- float - cvttss2si(GetDst(Node), GetSrc(Op->Scalar.ID())); - break; - case 0x0408: // int32_t <- double - cvttsd2si(GetDst(Node), GetSrc(Op->Scalar.ID())); - break; - } -} - -DEF_OP(Float_ToGPR_S) { - auto Op = IROp->C(); - const uint16_t Conv = (IROp->Size << 8) | Op->SrcElementSize; - - switch (Conv) { - case 0x0804: // int64_t <- float - cvtss2si(GetDst(Node), GetSrc(Op->Scalar.ID())); - break; - case 0x0808: // int64_t <- double - cvtsd2si(GetDst(Node), GetSrc(Op->Scalar.ID())); - break; - case 0x0404: // int32_t <- float - cvtss2si(GetDst(Node), GetSrc(Op->Scalar.ID())); - break; - case 0x0408: // int32_t <- double - cvtsd2si(GetDst(Node), GetSrc(Op->Scalar.ID())); - break; - } -} - -DEF_OP(FCmp) { - auto Op = IROp->C(); - - if (Op->Flags & (1 << IR::FCMP_FLAG_UNORDERED)) { - if (Op->ElementSize == 4) { - ucomiss(GetSrc(Op->Scalar1.ID()), GetSrc(Op->Scalar2.ID())); - } - else { - ucomisd(GetSrc(Op->Scalar1.ID()), GetSrc(Op->Scalar2.ID())); - } - } - else { - if (Op->ElementSize == 4) { - comiss(GetSrc(Op->Scalar1.ID()), GetSrc(Op->Scalar2.ID())); - } - else { - comisd(GetSrc(Op->Scalar1.ID()), GetSrc(Op->Scalar2.ID())); - } - } - mov (rdx, 0); - - lahf(); - if (Op->Flags & (1 << IR::FCMP_FLAG_LT)) { - sahf(); - mov(rcx, 0); - setb(cl); - shl(rcx, IR::FCMP_FLAG_LT); - or_(rdx, rcx); - } - if (Op->Flags & (1 << IR::FCMP_FLAG_UNORDERED)) { - sahf(); - mov(rcx, 0); - setp(cl); - shl(rcx, IR::FCMP_FLAG_UNORDERED); - or_(rdx, rcx); - } - if (Op->Flags & (1 << IR::FCMP_FLAG_EQ)) { - sahf(); - mov(rcx, 0); - setz(cl); - shl(rcx, IR::FCMP_FLAG_EQ); - or_(rdx, rcx); - } - mov (GetDst(Node), rdx); -} - -#undef DEF_OP - -void X86JITCore::RegisterALUHandlers() { -#define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &X86JITCore::Op_##x - REGISTER_OP(TRUNCELEMENTPAIR, TruncElementPair); - REGISTER_OP(CONSTANT, Constant); - REGISTER_OP(ENTRYPOINTOFFSET, EntrypointOffset); - REGISTER_OP(INLINECONSTANT, InlineConstant); - REGISTER_OP(INLINEENTRYPOINTOFFSET, InlineEntrypointOffset); - REGISTER_OP(CYCLECOUNTER, CycleCounter); - REGISTER_OP(ADD, Add); - REGISTER_OP(ADDNZCV, AddNZCV); - REGISTER_OP(TESTNZ, TestNZ); - REGISTER_OP(SUB, Sub); - REGISTER_OP(SUBNZCV, SubNZCV); - REGISTER_OP(NEG, Neg); - REGISTER_OP(ABS, Abs); - REGISTER_OP(MUL, Mul); - REGISTER_OP(UMUL, UMul); - REGISTER_OP(DIV, Div); - REGISTER_OP(UDIV, UDiv); - REGISTER_OP(REM, Rem); - REGISTER_OP(UREM, URem); - REGISTER_OP(MULH, MulH); - REGISTER_OP(UMULH, UMulH); - REGISTER_OP(OR, Or); - REGISTER_OP(ORLSHL, Orlshl); - REGISTER_OP(ORLSHR, Orlshr); - REGISTER_OP(AND, And); - REGISTER_OP(ANDN, Andn); - REGISTER_OP(XOR, Xor); - REGISTER_OP(LSHL, Lshl); - REGISTER_OP(LSHR, Lshr); - REGISTER_OP(ASHR, Ashr); - REGISTER_OP(ROR, Ror); - REGISTER_OP(EXTR, Extr); - REGISTER_OP(PDEP, PDep); - REGISTER_OP(PEXT, PExt); - REGISTER_OP(LDIV, LDiv); - REGISTER_OP(LUDIV, LUDiv); - REGISTER_OP(LREM, LRem); - REGISTER_OP(LUREM, LURem); - REGISTER_OP(NOT, Not); - REGISTER_OP(POPCOUNT, Popcount); - REGISTER_OP(FINDLSB, FindLSB); - REGISTER_OP(FINDMSB, FindMSB); - REGISTER_OP(FINDTRAILINGZEROES, FindTrailingZeroes); - REGISTER_OP(COUNTLEADINGZEROES, CountLeadingZeroes); - REGISTER_OP(REV, Rev); - REGISTER_OP(BFI, Bfi); - REGISTER_OP(BFXIL, Bfxil); - REGISTER_OP(BFE, Bfe); - REGISTER_OP(SBFE, Sbfe); - REGISTER_OP(SELECT, Select); - REGISTER_OP(VEXTRACTTOGPR, VExtractToGPR); - REGISTER_OP(FLOAT_TOGPR_ZS, Float_ToGPR_ZS); - REGISTER_OP(FLOAT_TOGPR_S, Float_ToGPR_S); - REGISTER_OP(FCMP, FCmp); -#undef REGISTER_OP -} - - -} diff --git a/FEXCore/Source/Interface/Core/JIT/x86_64/AtomicOps.cpp b/FEXCore/Source/Interface/Core/JIT/x86_64/AtomicOps.cpp deleted file mode 100644 index 140053544..000000000 --- a/FEXCore/Source/Interface/Core/JIT/x86_64/AtomicOps.cpp +++ /dev/null @@ -1,755 +0,0 @@ -// SPDX-License-Identifier: MIT -/* -$info$ -tags: backend|x86-64 -$end_info$ -*/ - -#include "Interface/Core/JIT/x86_64/JITClass.h" -#include "Interface/Core/Dispatcher/X86Dispatcher.h" - -#include -#include - -#include -#include -#include - -namespace FEXCore::CPU { -#define DEF_OP(x) void X86JITCore::Op_##x(IR::IROp_Header *IROp, IR::NodeID Node) -DEF_OP(CASPair) { - auto Op = IROp->C(); - - // DataSrc = *Src1 - // if (DataSrc == Src3) { *Src1 == Src2; } Src2 = DataSrc - // This will write to memory! Careful! - // Third operand must be a calculated guest memory address - //OrderedNode *CASResult = _CAS(Src3, Src2, Src1); - auto Dst = GetSrcPair(Node); - auto Expected = GetSrcPair(Op->Expected.ID()); - auto Desired = GetSrcPair(Op->Desired.ID()); - auto MemSrc = GetSrc(Op->Addr.ID()); - - Xbyak::Reg MemReg = MemSrc; - - mov(rax, Expected.first); - mov(rdx, Expected.second); - - mov(rbx, Desired.first); - mov(rcx, Desired.second); - - // RDI(Or Source) now contains pointer - // RDX:RAX contains our expected value - // RCX:RBX contains our desired - - lock(); - - switch (IROp->ElementSize) { - case 4: { - cmpxchg8b(dword [MemReg]); - // EDX:EAX now contains the result - mov(Dst.first.cvt32(), eax); - mov(Dst.second.cvt32(), edx); - break; - } - case 8: { - cmpxchg16b(qword [MemReg]); - // RDX:RAX now contains the result - mov(Dst.first, rax); - mov(Dst.second, rdx); - break; - } - default: LOGMAN_MSG_A_FMT("Unsupported: {}", IROp->ElementSize); - } -} - -DEF_OP(CAS) { - auto Op = IROp->C(); - uint8_t OpSize = IROp->Size; - - // DataSrc = *Src1 - // if (DataSrc == Src3) { *Src1 == Src2; } Src2 = DataSrc - // This will write to memory! Careful! - // Third operand must be a calculated guest memory address - //OrderedNode *CASResult = _CAS(Src3, Src2, Src1); - - Xbyak::Reg MemReg = GetSrc(Op->Addr.ID()); - - mov(rax, GetSrc(Op->Expected.ID())); - - // RCX now contains pointer - // RAX contains our expected value - // RDX contains our desired - - lock(); - switch (OpSize) { - case 1: { - cmpxchg(byte [MemReg], GetSrc(Op->Desired.ID())); - movzx(GetDst(Node), al); - break; - } - case 2: { - cmpxchg(word [MemReg], GetSrc(Op->Desired.ID())); - movzx(GetDst(Node), ax); - break; - } - case 4: { - cmpxchg(dword [MemReg], GetSrc(Op->Desired.ID())); - // RAX now contains the result - mov (GetDst(Node), eax); - break; - } - case 8: { - cmpxchg(qword [MemReg], GetSrc(Op->Desired.ID())); - // RAX now contains the result - mov (GetDst(Node), rax); - break; - } - default: LOGMAN_MSG_A_FMT("Unsupported: {}", OpSize); - } -} - -DEF_OP(AtomicAdd) { - auto Op = IROp->C(); - - Xbyak::Reg MemReg = GetSrc(Op->Addr.ID()); - - lock(); - switch (IROp->Size) { - case 1: - add(byte [MemReg], GetSrc(Op->Value.ID())); - break; - case 2: - add(word [MemReg], GetSrc(Op->Value.ID())); - break; - case 4: - add(dword [MemReg], GetSrc(Op->Value.ID())); - break; - case 8: - add(qword [MemReg], GetSrc(Op->Value.ID())); - break; - default: LOGMAN_MSG_A_FMT("Unhandled AtomicAdd size: {}", IROp->Size); - } -} - -DEF_OP(AtomicSub) { - auto Op = IROp->C(); - - Xbyak::Reg MemReg = GetSrc(Op->Addr.ID()); - lock(); - switch (IROp->Size) { - case 1: - sub(byte [MemReg], GetSrc(Op->Value.ID())); - break; - case 2: - sub(word [MemReg], GetSrc(Op->Value.ID())); - break; - case 4: - sub(dword [MemReg], GetSrc(Op->Value.ID())); - break; - case 8: - sub(qword [MemReg], GetSrc(Op->Value.ID())); - break; - default: LOGMAN_MSG_A_FMT("Unhandled AtomicAdd size: {}", IROp->Size); - } -} - -DEF_OP(AtomicAnd) { - auto Op = IROp->C(); - - Xbyak::Reg MemReg = GetSrc(Op->Addr.ID()); - lock(); - switch (IROp->Size) { - case 1: - and_(byte [MemReg], GetSrc(Op->Value.ID())); - break; - case 2: - and_(word [MemReg], GetSrc(Op->Value.ID())); - break; - case 4: - and_(dword [MemReg], GetSrc(Op->Value.ID())); - break; - case 8: - and_(qword [MemReg], GetSrc(Op->Value.ID())); - break; - default: LOGMAN_MSG_A_FMT("Unhandled AtomicAdd size: {}", IROp->Size); - } -} - -DEF_OP(AtomicOr) { - auto Op = IROp->C(); - - Xbyak::Reg MemReg = GetSrc(Op->Addr.ID()); - lock(); - switch (IROp->Size) { - case 1: - or_(byte [MemReg], GetSrc(Op->Value.ID())); - break; - case 2: - or_(word [MemReg], GetSrc(Op->Value.ID())); - break; - case 4: - or_(dword [MemReg], GetSrc(Op->Value.ID())); - break; - case 8: - or_(qword [MemReg], GetSrc(Op->Value.ID())); - break; - default: LOGMAN_MSG_A_FMT("Unhandled AtomicAdd size: {}", IROp->Size); - } -} - -DEF_OP(AtomicXor) { - auto Op = IROp->C(); - - Xbyak::Reg MemReg = GetSrc(Op->Addr.ID()); - lock(); - switch (IROp->Size) { - case 1: - xor_(byte [MemReg], GetSrc(Op->Value.ID())); - break; - case 2: - xor_(word [MemReg], GetSrc(Op->Value.ID())); - break; - case 4: - xor_(dword [MemReg], GetSrc(Op->Value.ID())); - break; - case 8: - xor_(qword [MemReg], GetSrc(Op->Value.ID())); - break; - default: LOGMAN_MSG_A_FMT("Unhandled AtomicAdd size: {}", IROp->Size); - } -} - -DEF_OP(AtomicSwap) { - auto Op = IROp->C(); - - Xbyak::Reg MemReg = rax; - mov(MemReg, GetSrc(Op->Addr.ID())); - - switch (IROp->Size) { - case 1: - movzx(GetDst(Node), GetSrc(Op->Value.ID())); - lock(); - xchg(byte [MemReg], GetDst(Node)); - break; - case 2: - movzx(GetDst(Node), GetSrc(Op->Value.ID())); - lock(); - xchg(word [MemReg], GetDst(Node)); - break; - case 4: - mov(GetDst(Node), GetSrc(Op->Value.ID())); - lock(); - xchg(dword [MemReg], GetDst(Node)); - break; - case 8: - mov(GetDst(Node), GetSrc(Op->Value.ID())); - lock(); - xchg(qword [MemReg], GetDst(Node)); - break; - default: LOGMAN_MSG_A_FMT("Unhandled AtomicSwap size: {}", IROp->Size); - } -} - -DEF_OP(AtomicFetchAdd) { - auto Op = IROp->C(); - - Xbyak::Reg MemReg = GetSrc(Op->Addr.ID()); - switch (IROp->Size) { - case 1: - movzx(rcx, GetSrc(Op->Value.ID())); - lock(); - xadd(byte [MemReg], cl); - movzx(GetDst(Node), cl); - break; - case 2: - movzx(rcx, GetSrc(Op->Value.ID())); - lock(); - xadd(word [MemReg], cx); - movzx(GetDst(Node), cx); - break; - case 4: - mov(ecx, GetSrc(Op->Value.ID())); - lock(); - xadd(dword [MemReg], ecx); - mov(GetDst(Node), ecx); - break; - case 8: - mov(rcx, GetSrc(Op->Value.ID())); - lock(); - xadd(qword [MemReg], rcx); - mov(GetDst(Node), rcx); - break; - default: LOGMAN_MSG_A_FMT("Unhandled AtomicFetchAdd size: {}", IROp->Size); - } -} - -DEF_OP(AtomicFetchSub) { - auto Op = IROp->C(); - - Xbyak::Reg MemReg = GetSrc(Op->Addr.ID()); - switch (IROp->Size) { - case 1: - mov(cl, GetSrc(Op->Value.ID())); - neg(cl); - lock(); - xadd(byte [MemReg], cl); - movzx(GetDst(Node), cl); - break; - case 2: - mov(cx, GetSrc(Op->Value.ID())); - neg(cx); - lock(); - xadd(word [MemReg], cx); - movzx(GetDst(Node), cx); - break; - case 4: - mov(ecx, GetSrc(Op->Value.ID())); - neg(ecx); - lock(); - xadd(dword [MemReg], ecx); - mov(GetDst(Node), ecx); - break; - case 8: - mov(rcx, GetSrc(Op->Value.ID())); - neg(rcx); - lock(); - xadd(qword [MemReg], rcx); - mov(GetDst(Node), rcx); - break; - default: LOGMAN_MSG_A_FMT("Unhandled AtomicFetchSub size: {}", IROp->Size); - } -} - -DEF_OP(AtomicFetchAnd) { - auto Op = IROp->C(); - - // TMP1 = rax - Xbyak::Reg MemReg = GetSrc(Op->Addr.ID()); - - switch (IROp->Size) { - case 1: { - mov(TMP1.cvt8(), byte [MemReg]); - - Label Loop; - L(Loop); - mov(TMP2.cvt8(), TMP1.cvt8()); - mov(TMP3.cvt8(), TMP1.cvt8()); - and_(TMP2.cvt8(), GetSrc(Op->Value.ID())); - - // Updates RAX with the value from memory - lock(); cmpxchg(byte [MemReg], TMP2.cvt8()); - jne(Loop); - // Result is the previous value from memory, which is currently in TMP3 - movzx(GetDst(Node), TMP3.cvt8()); - break; - } - case 2: { - mov(TMP1.cvt16(), word [MemReg]); - - Label Loop; - L(Loop); - mov(TMP2.cvt16(), TMP1.cvt16()); - mov(TMP3.cvt16(), TMP1.cvt16()); - and_(TMP2.cvt16(), GetSrc(Op->Value.ID())); - - // Updates RAX with the value from memory - lock(); cmpxchg(word [MemReg], TMP2.cvt16()); - jne(Loop); - - // Result is the previous value from memory, which is currently in TMP3 - movzx(GetDst(Node), TMP3.cvt16()); - break; - } - case 4: { - mov(TMP1.cvt32(), dword [MemReg]); - - Label Loop; - L(Loop); - mov(TMP2.cvt32(), TMP1.cvt32()); - mov(TMP3.cvt32(), TMP1.cvt32()); - and_(TMP2.cvt32(), GetSrc(Op->Value.ID())); - - // Updates RAX with the value from memory - lock(); cmpxchg(dword [MemReg], TMP2.cvt32()); - jne(Loop); - - // Result is the previous value from memory, which is currently in TMP3 - mov(GetDst(Node), TMP3.cvt32()); - break; - } - case 8: { - mov(TMP1.cvt64(), qword [MemReg]); - - Label Loop; - L(Loop); - mov(TMP2.cvt64(), TMP1.cvt64()); - mov(TMP3.cvt64(), TMP1.cvt64()); - and_(TMP2.cvt64(), GetSrc(Op->Value.ID())); - - // Updates RAX with the value from memory - lock(); cmpxchg(qword [MemReg], TMP2.cvt64()); - jne(Loop); - - // Result is the previous value from memory, which is currently in TMP3 - mov(GetDst(Node), TMP3.cvt64()); - break; - } - default: LOGMAN_MSG_A_FMT("Unhandled AtomicFetchAnd size: {}", IROp->Size); - } -} - -DEF_OP(AtomicFetchCLR) { - auto Op = IROp->C(); - - // TMP1 = rax - Xbyak::Reg MemReg = GetSrc(Op->Addr.ID()); - - switch (IROp->Size) { - case 1: { - mov(TMP1.cvt8(), byte [MemReg]); - - Label Loop; - L(Loop); - mov(TMP2.cvt8(), TMP1.cvt8()); - mov(TMP3.cvt8(), TMP1.cvt8()); - mov(TMP4.cvt8(), GetSrc(Op->Value.ID())); - not_(TMP4.cvt8()); - and_(TMP2.cvt8(), TMP4.cvt8()); - - // Updates RAX with the value from memory - lock(); cmpxchg(byte [MemReg], TMP2.cvt8()); - jne(Loop); - // Result is the previous value from memory, which is currently in TMP3 - movzx(GetDst(Node), TMP3.cvt8()); - break; - } - case 2: { - mov(TMP1.cvt16(), word [MemReg]); - - Label Loop; - L(Loop); - mov(TMP2.cvt16(), TMP1.cvt16()); - mov(TMP3.cvt16(), TMP1.cvt16()); - mov(TMP4.cvt16(), GetSrc(Op->Value.ID())); - not_(TMP4.cvt16()); - and_(TMP2.cvt16(), TMP4.cvt16()); - - // Updates RAX with the value from memory - lock(); cmpxchg(word [MemReg], TMP2.cvt16()); - jne(Loop); - - // Result is the previous value from memory, which is currently in TMP3 - movzx(GetDst(Node), TMP3.cvt16()); - break; - } - case 4: { - mov(TMP1.cvt32(), dword [MemReg]); - - Label Loop; - L(Loop); - mov(TMP2.cvt32(), TMP1.cvt32()); - mov(TMP3.cvt32(), TMP1.cvt32()); - mov(TMP4.cvt32(), GetSrc(Op->Value.ID())); - not_(TMP4.cvt32()); - and_(TMP2.cvt32(), TMP4.cvt32()); - - // Updates RAX with the value from memory - lock(); cmpxchg(dword [MemReg], TMP2.cvt32()); - jne(Loop); - - // Result is the previous value from memory, which is currently in TMP3 - mov(GetDst(Node), TMP3.cvt32()); - break; - } - case 8: { - mov(TMP1.cvt64(), qword [MemReg]); - - Label Loop; - L(Loop); - mov(TMP2.cvt64(), TMP1.cvt64()); - mov(TMP3.cvt64(), TMP1.cvt64()); - mov(TMP4.cvt64(), GetSrc(Op->Value.ID())); - not_(TMP4.cvt64()); - and_(TMP2.cvt64(), TMP4.cvt64()); - - // Updates RAX with the value from memory - lock(); cmpxchg(qword [MemReg], TMP2.cvt64()); - jne(Loop); - - // Result is the previous value from memory, which is currently in TMP3 - mov(GetDst(Node), TMP3.cvt64()); - break; - } - default: LOGMAN_MSG_A_FMT("Unhandled AtomicFetchAnd size: {}", IROp->Size); - } -} - -DEF_OP(AtomicFetchOr) { - auto Op = IROp->C(); - - // TMP1 = rax - Xbyak::Reg MemReg = GetSrc(Op->Addr.ID()); - switch (IROp->Size) { - case 1: { - mov(TMP1.cvt8(), byte [MemReg]); - - Label Loop; - L(Loop); - mov(TMP2.cvt8(), TMP1.cvt8()); - mov(TMP3.cvt8(), TMP1.cvt8()); - or_(TMP2.cvt8(), GetSrc(Op->Value.ID())); - - // Updates RAX with the value from memory - lock(); cmpxchg(byte [MemReg], TMP2.cvt8()); - jne(Loop); - // Result is the previous value from memory, which is currently in TMP3 - movzx(GetDst(Node), TMP3.cvt8()); - break; - } - case 2: { - mov(TMP1.cvt16(), word [MemReg]); - - Label Loop; - L(Loop); - mov(TMP2.cvt16(), TMP1.cvt16()); - mov(TMP3.cvt16(), TMP1.cvt16()); - or_(TMP2.cvt16(), GetSrc(Op->Value.ID())); - - // Updates RAX with the value from memory - lock(); cmpxchg(word [MemReg], TMP2.cvt16()); - jne(Loop); - - // Result is the previous value from memory, which is currently in TMP3 - movzx(GetDst(Node), TMP3.cvt16()); - break; - } - case 4: { - mov(TMP1.cvt32(), dword [MemReg]); - - Label Loop; - L(Loop); - mov(TMP2.cvt32(), TMP1.cvt32()); - mov(TMP3.cvt32(), TMP1.cvt32()); - or_(TMP2.cvt32(), GetSrc(Op->Value.ID())); - - // Updates RAX with the value from memory - lock(); cmpxchg(dword [MemReg], TMP2.cvt32()); - jne(Loop); - - // Result is the previous value from memory, which is currently in TMP3 - mov(GetDst(Node), TMP3.cvt32()); - break; - } - case 8: { - mov(TMP1.cvt64(), qword [MemReg]); - - Label Loop; - L(Loop); - mov(TMP2.cvt64(), TMP1.cvt64()); - mov(TMP3.cvt64(), TMP1.cvt64()); - or_(TMP2.cvt64(), GetSrc(Op->Value.ID())); - - // Updates RAX with the value from memory - lock(); cmpxchg(qword [MemReg], TMP2.cvt64()); - jne(Loop); - - // Result is the previous value from memory, which is currently in TMP3 - mov(GetDst(Node), TMP3.cvt64()); - break; - } - default: LOGMAN_MSG_A_FMT("Unhandled AtomicFetchOr size: {}", IROp->Size); - } -} - -DEF_OP(AtomicFetchXor) { - auto Op = IROp->C(); - - // TMP1 = rax - Xbyak::Reg MemReg = GetSrc(Op->Addr.ID()); - switch (IROp->Size) { - case 1: { - mov(TMP1.cvt8(), byte [MemReg]); - - Label Loop; - L(Loop); - mov(TMP2.cvt8(), TMP1.cvt8()); - mov(TMP3.cvt8(), TMP1.cvt8()); - xor_(TMP2.cvt8(), GetSrc(Op->Value.ID())); - - // Updates RAX with the value from memory - lock(); cmpxchg(byte [MemReg], TMP2.cvt8()); - jne(Loop); - // Result is the previous value from memory, which is currently in TMP3 - movzx(GetDst(Node), TMP3.cvt8()); - break; - } - case 2: { - mov(TMP1.cvt16(), word [MemReg]); - - Label Loop; - L(Loop); - mov(TMP2.cvt16(), TMP1.cvt16()); - mov(TMP3.cvt16(), TMP1.cvt16()); - xor_(TMP2.cvt16(), GetSrc(Op->Value.ID())); - - // Updates RAX with the value from memory - lock(); cmpxchg(word [MemReg], TMP2.cvt16()); - jne(Loop); - - // Result is the previous value from memory, which is currently in TMP3 - movzx(GetDst(Node), TMP3.cvt16()); - break; - } - case 4: { - mov(TMP1.cvt32(), dword [MemReg]); - - Label Loop; - L(Loop); - mov(TMP2.cvt32(), TMP1.cvt32()); - mov(TMP3.cvt32(), TMP1.cvt32()); - xor_(TMP2.cvt32(), GetSrc(Op->Value.ID())); - - // Updates RAX with the value from memory - lock(); cmpxchg(dword [MemReg], TMP2.cvt32()); - jne(Loop); - - // Result is the previous value from memory, which is currently in TMP3 - mov(GetDst(Node), TMP3.cvt32()); - break; - } - case 8: { - mov(TMP1.cvt64(), qword [MemReg]); - - Label Loop; - L(Loop); - mov(TMP2.cvt64(), TMP1.cvt64()); - mov(TMP3.cvt64(), TMP1.cvt64()); - xor_(TMP2.cvt64(), GetSrc(Op->Value.ID())); - - // Updates RAX with the value from memory - lock(); cmpxchg(qword [MemReg], TMP2.cvt64()); - jne(Loop); - - // Result is the previous value from memory, which is currently in TMP3 - mov(GetDst(Node), TMP3.cvt64()); - break; - } - default: LOGMAN_MSG_A_FMT("Unhandled AtomicFetchXor size: {}", IROp->Size); - } -} - -DEF_OP(AtomicFetchNeg) { - auto Op = IROp->C(); - - Xbyak::Reg MemReg = GetSrc(Op->Addr.ID()); - switch (IROp->Size) { - case 1: { - mov(TMP1.cvt8(), byte [MemReg]); - - Label Loop; - L(Loop); - mov(TMP2.cvt8(), TMP1.cvt8()); - mov(TMP3.cvt8(), TMP1.cvt8()); - neg(TMP2.cvt8()); - - // Updates RAX with the value from memory - lock(); cmpxchg(byte [MemReg], TMP2.cvt8()); - jne(Loop); - // Result is the previous value from memory, which is currently in TMP3 - movzx(GetDst(Node), TMP3.cvt8()); - break; - } - case 2: { - mov(TMP1.cvt16(), word [MemReg]); - - Label Loop; - L(Loop); - mov(TMP2.cvt16(), TMP1.cvt16()); - mov(TMP3.cvt16(), TMP1.cvt16()); - neg(TMP2.cvt16()); - - // Updates RAX with the value from memory - lock(); cmpxchg(word [MemReg], TMP2.cvt16()); - jne(Loop); - - // Result is the previous value from memory, which is currently in TMP3 - movzx(GetDst(Node), TMP3.cvt16()); - break; - } - case 4: { - mov(TMP1.cvt32(), dword [MemReg]); - - Label Loop; - L(Loop); - mov(TMP2.cvt32(), TMP1.cvt32()); - mov(TMP3.cvt32(), TMP1.cvt32()); - neg(TMP2.cvt32()); - - // Updates RAX with the value from memory - lock(); cmpxchg(dword [MemReg], TMP2.cvt32()); - jne(Loop); - - // Result is the previous value from memory, which is currently in TMP3 - mov(GetDst(Node), TMP3.cvt32()); - break; - } - case 8: { - mov(TMP1.cvt64(), qword [MemReg]); - - Label Loop; - L(Loop); - mov(TMP2.cvt64(), TMP1.cvt64()); - mov(TMP3.cvt64(), TMP1.cvt64()); - neg(TMP2.cvt64()); - - // Updates RAX with the value from memory - lock(); cmpxchg(qword [MemReg], TMP2.cvt64()); - jne(Loop); - - // Result is the previous value from memory, which is currently in TMP3 - mov(GetDst(Node), TMP3.cvt64()); - break; - } - default: LOGMAN_MSG_A_FMT("Unhandled AtomicFetchNeg size: {}", IROp->Size); - } -} - -DEF_OP(TelemetrySetValue) { -#ifndef FEX_DISABLE_TELEMETRY - auto Op = IROp->C(); - auto Src = GetSrc(Op->Value.ID()); - - xor_(TMP1, TMP1); - mov(TMP2, qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.TelemetryValueAddresses[Op->TelemetryValueIndex])]); - test(Src, Src); - setne(TMP1.cvt8()); - lock(); or_(qword [TMP2], TMP1); -#endif -} - -#undef DEF_OP -void X86JITCore::RegisterAtomicHandlers() { -#define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &X86JITCore::Op_##x - REGISTER_OP(CASPAIR, CASPair); - REGISTER_OP(CAS, CAS); - REGISTER_OP(ATOMICADD, AtomicAdd); - REGISTER_OP(ATOMICSUB, AtomicSub); - REGISTER_OP(ATOMICAND, AtomicAnd); - REGISTER_OP(ATOMICOR, AtomicOr); - REGISTER_OP(ATOMICXOR, AtomicXor); - REGISTER_OP(ATOMICSWAP, AtomicSwap); - REGISTER_OP(ATOMICFETCHADD, AtomicFetchAdd); - REGISTER_OP(ATOMICFETCHSUB, AtomicFetchSub); - REGISTER_OP(ATOMICFETCHAND, AtomicFetchAnd); - REGISTER_OP(ATOMICFETCHCLR, AtomicFetchCLR); - - REGISTER_OP(ATOMICFETCHOR, AtomicFetchOr); - REGISTER_OP(ATOMICFETCHXOR, AtomicFetchXor); - REGISTER_OP(ATOMICFETCHNEG, AtomicFetchNeg); - REGISTER_OP(TELEMETRYSETVALUE, TelemetrySetValue); - -#undef REGISTER_OP -} -} - diff --git a/FEXCore/Source/Interface/Core/JIT/x86_64/BranchOps.cpp b/FEXCore/Source/Interface/Core/JIT/x86_64/BranchOps.cpp deleted file mode 100644 index 48b219399..000000000 --- a/FEXCore/Source/Interface/Core/JIT/x86_64/BranchOps.cpp +++ /dev/null @@ -1,362 +0,0 @@ -// SPDX-License-Identifier: MIT -/* -$info$ -tags: backend|x86-64 -$end_info$ -*/ - -#include "Interface/Context/Context.h" -#include "Interface/Core/CPUID.h" -#include "Interface/Core/Dispatcher/Dispatcher.h" -#include "Interface/Core/Dispatcher/X86Dispatcher.h" -#include "Interface/Core/LookupCache.h" -#include "Interface/Core/JIT/x86_64/JITClass.h" -#include "Interface/HLE/Thunks/Thunks.h" - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -namespace FEXCore::CPU { -#define DEF_OP(x) void X86JITCore::Op_##x(IR::IROp_Header *IROp, IR::NodeID Node) - -DEF_OP(CallbackReturn) { - // Adjust the stack first for a regular return - if (SpillSlots) { - add(rsp, SpillSlots * MaxSpillSlotSize); // + 8 to consume return address - } - - // Make sure to adjust the refcounter so we don't clear the cache now - sub(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, SignalHandlerRefCounter)], 1); - - // We need to adjust an additional 8 bytes to get back to the original "misaligned" RSP state - add(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RSP])], 8); - - // Now jump back to the thunk - // XXX: XMM? - add(rsp, 8); - - pop(r15); - pop(r14); - pop(r13); - pop(r12); - pop(rbp); - pop(rbx); - - ret(); -} - -DEF_OP(ExitFunction) { - Label FullLookup; - auto Op = IROp->C(); - - - if (SpillSlots) { - add(rsp, SpillSlots * MaxSpillSlotSize); - } - - uint64_t NewRIP; - - if (IsInlineConstant(Op->NewRIP, &NewRIP) || IsInlineEntrypointOffset(Op->NewRIP, &NewRIP)) { - Label l_BranchHost; - Label l_BranchGuest; - - lea(rax, ptr[rip + l_BranchHost]); - jmp(qword[rax]); - - L(l_BranchHost); - //FEX_TODO(this is not per thread) - dq(ThreadState->CurrentFrame->Pointers.Common.ExitFunctionLinker); - L(l_BranchGuest); - dq(NewRIP); - } else { - Xbyak::Reg RipReg = GetSrc(Op->NewRIP.ID()); - - // L1 Cache - mov(rcx, qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.L1Pointer)]); - - mov(rax, RipReg); - - and_(rax, LookupCache::L1_ENTRIES_MASK); - shl(rax, 4); - - Xbyak::RegExp LookupBase = rcx + rax; - - cmp(qword[LookupBase + 8], RipReg); - jne(FullLookup); - jmp(qword[LookupBase + 0]); - - L(FullLookup); - mov(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, State.rip)], RipReg); - jmp(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.DispatcherLoopTop)]); - } - -#ifdef BLOCKSTATS - ExitBlock(); -#endif -} - -DEF_OP(Jump) { - const auto Op = IROp->C(); - const auto Target = Op->TargetBlock.ID(); - - PendingTargetLabel = &JumpTargets.try_emplace(Target).first->second; -} - -#define GRCMP(Node) (Op->CompareSize == 4 ? GetSrc(Node) : GetSrc(Node)) - -DEF_OP(CondJump) { - auto Op = IROp->C(); - - Label *TrueTargetLabel = &JumpTargets.try_emplace(Op->TrueBlock.ID()).first->second; - - if (IsGPR(Op->Cmp1.ID())) { - uint64_t Const; - if (IsInlineConstant(Op->Cmp2, &Const)) { - cmp(GRCMP(Op->Cmp1.ID()), Const); - } else { - cmp(GRCMP(Op->Cmp1.ID()), GRCMP(Op->Cmp2.ID())); - } - } else if (IsFPR(Op->Cmp1.ID())) { - if (Op->CompareSize == 4) { - ucomiss(GetSrc(Op->Cmp1.ID()), GetSrc(Op->Cmp2.ID())); - } else { - ucomisd(GetSrc(Op->Cmp1.ID()), GetSrc(Op->Cmp2.ID())); - } - } - - auto [_, __, JCC] = GetCC(Op->Cond); - - (this->*JCC)(*TrueTargetLabel, T_NEAR); - - PendingTargetLabel = &JumpTargets.try_emplace(Op->FalseBlock.ID()).first->second; -} - -DEF_OP(Syscall) { - auto Op = IROp->C(); - // XXX: This is very terrible, but I don't care for right now - - FEXCore::IR::SyscallFlags Flags = Op->Flags; - auto NumPush = RA64.size(); - - for (auto &Reg : RA64) - push(Reg); - - // Syscall ABI for x86-64 - // this: rdi - // Thread: rsi - // ArgPointer: rdx (Stack) - // - // Result: RAX - - // These are pushed in reverse order because stacks - for (uint32_t i = FEXCore::HLE::SyscallArguments::MAX_ARGS; i > 0; --i) { - if (Op->Header.Args[i - 1].IsInvalid()) continue; - push(GetSrc(Op->Header.Args[i - 1].ID())); - ++NumPush; - } - - mov(rsi, STATE); // Move thread in to rsi - mov(rdi, qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.SyscallHandlerObj)]); - mov(rdx, rsp); - - if (NumPush & 1) - sub(rsp, 8); // Align - // {rdi, rsi, rdx} - call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.SyscallHandlerFunc)]); - - if (NumPush & 1) - add(rsp, 8); // Align - - // Reload arguments just in case they are sill live after the fact - for (uint32_t i = 0; i < FEXCore::HLE::SyscallArguments::MAX_ARGS; ++i) { - if (Op->Header.Args[i].IsInvalid()) continue; - pop(GetSrc(Op->Header.Args[i].ID())); - } - - for (uint32_t i = RA64.size(); i > 0; --i) - pop(RA64[i - 1]); - - if ((Flags & FEXCore::IR::SyscallFlags::NORETURNEDRESULT) != FEXCore::IR::SyscallFlags::NORETURNEDRESULT) { - // Move result to its destination register. - // Only if `NORETURNEDRESULT` wasn't set, otherwise we might overwrite the CPUState refilled with `FillStaticRegs` - mov (GetDst(Node), rax); - } -} - -DEF_OP(Thunk) { - auto Op = IROp->C(); - - auto NumPush = RA64.size(); - - for (auto &Reg : RA64) - push(Reg); - - if (NumPush & 1) - sub(rsp, 8); // Align - - mov(rdi, GetSrc(Op->ArgPtr.ID())); - - auto thunkFn = static_cast(ThreadState->CTX)->ThunkHandler->LookupThunk(Op->ThunkNameHash); - - mov(rax, reinterpret_cast(thunkFn)); - call(rax); - - if (NumPush & 1) - add(rsp, 8); // Align - - for (uint32_t i = RA64.size(); i > 0; --i) - pop(RA64[i - 1]); -} - -DEF_OP(ValidateCode) { - auto Op = IROp->C(); - const auto* OldCode = (const uint8_t*)&Op->CodeOriginalLow; - int len = Op->CodeLength; - int idx = 0; - - xor_(GetDst(Node), GetDst(Node)); - mov(rax, Entry + Op->Offset); - mov(rbx, 1); - while (len >= 4) { - cmp(dword[rax + idx], *(const uint32_t*)(OldCode + idx)); - cmovne(GetDst(Node), rbx); - len-=4; - idx+=4; - } - while (len >= 2) { - mov(rcx, *(const uint16_t*)(OldCode + idx)); - cmp(word[rax + idx], cx); - cmovne(GetDst(Node), rbx); - len-=2; - idx+=2; - } - while (len >= 1) { - cmp(byte[rax + idx], *(const uint8_t*)(OldCode + idx)); - cmovne(GetDst(Node), rbx); - len-=1; - idx+=1; - } -} - -DEF_OP(ThreadRemoveCodeEntry) { - auto NumPush = RA64.size(); - - for (auto &Reg : RA64) - push(Reg); - - if (NumPush & 1) - sub(rsp, 8); // Align - - mov(rdi, STATE); - mov(rax, Entry); // imm64 move - mov(rsi, rax); - - call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.ThreadRemoveCodeEntryFromJIT)]); - - if (NumPush & 1) - add(rsp, 8); // Align - - for (uint32_t i = RA64.size(); i > 0; --i) - pop(RA64[i - 1]); -} - -DEF_OP(CPUID) { - auto Op = IROp->C(); - - for (auto &Reg : RA64) - push(Reg); - - // CPUID ABI - // this: rdi - // Function: rsi - // - // Result: RAX, RDX. 4xi32 - - // rsi can be in the source registers, so copy argument to edx first - mov (edx, GetSrc(Op->Leaf.ID())); - mov (esi, GetSrc(Op->Function.ID())); - mov (rdi, qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.CPUIDObj)]); - - auto NumPush = RA64.size(); - - if (NumPush & 1) - sub(rsp, 8); // Align - - // {rdi, rsi, rdx} - call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.CPUIDFunction)]); - - if (NumPush & 1) - add(rsp, 8); // Align - - for (uint32_t i = RA64.size(); i > 0; --i) - pop(RA64[i - 1]); - - auto Dst = GetSrcPair(Node); - mov(Dst.first, rax); - mov(Dst.second, rdx); -} - -DEF_OP(XGETBV) { - auto Op = IROp->C(); - - for (auto &Reg : RA64) - push(Reg); - - // CPUID ABI - // this: rdi - // Function: rsi - // - // Result: RAX, RDX. 4xi32 - - // rsi can be in the source registers, so copy argument to edx first - mov (esi, GetSrc(Op->Function.ID())); - mov (rdi, qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.CPUIDObj)]); - - auto NumPush = RA64.size(); - - if (NumPush & 1) - sub(rsp, 8); // Align - - // {rdi, rsi, rdx} - call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.XCRFunction)]); - - if (NumPush & 1) - add(rsp, 8); // Align - - for (uint32_t i = RA64.size(); i > 0; --i) - pop(RA64[i - 1]); - - auto Dst = GetSrcPair(Node); - mov(Dst.first.cvt32(), eax); - mov(Dst.second, rax); - shr(Dst.second, 32); -} - -#undef DEF_OP -void X86JITCore::RegisterBranchHandlers() { -#define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &X86JITCore::Op_##x - REGISTER_OP(CALLBACKRETURN, CallbackReturn); - REGISTER_OP(EXITFUNCTION, ExitFunction); - REGISTER_OP(JUMP, Jump); - REGISTER_OP(CONDJUMP, CondJump); - REGISTER_OP(SYSCALL, Syscall); - REGISTER_OP(THUNK, Thunk); - REGISTER_OP(VALIDATECODE, ValidateCode); - REGISTER_OP(THREADREMOVECODEENTRY, ThreadRemoveCodeEntry); - REGISTER_OP(CPUID, CPUID); - REGISTER_OP(XGETBV, XGETBV); -#undef REGISTER_OP -} -} diff --git a/FEXCore/Source/Interface/Core/JIT/x86_64/ConversionOps.cpp b/FEXCore/Source/Interface/Core/JIT/x86_64/ConversionOps.cpp deleted file mode 100644 index 26fadc9d5..000000000 --- a/FEXCore/Source/Interface/Core/JIT/x86_64/ConversionOps.cpp +++ /dev/null @@ -1,459 +0,0 @@ -// SPDX-License-Identifier: MIT -/* -$info$ -tags: backend|x86-64 -$end_info$ -*/ - -#include "Interface/Core/JIT/x86_64/JITClass.h" -#include "Interface/Core/Dispatcher/X86Dispatcher.h" -#include -#include - -#include -#include - -namespace FEXCore::CPU { - -#define DEF_OP(x) void X86JITCore::Op_##x(IR::IROp_Header *IROp, IR::NodeID Node) -DEF_OP(VInsGPR) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto Dst = GetDst(Node); - const auto DestVector = GetSrc(Op->DestVector.ID()); - - const auto DestIdx = Op->DestIdx; - const auto ElementSize = Op->Header.ElementSize; - const auto ElementSizeBits = ElementSize * 8; - const auto Offset = ElementSizeBits * DestIdx; - - constexpr auto SSEBitSize = Core::CPUState::XMM_SSE_REG_SIZE * 8; - - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - const auto InUpperLane = Offset >= SSEBitSize; - - if (InUpperLane && !Is256Bit) { - LOGMAN_MSG_A_FMT("Attempt to access upper 128-bit lane in 128-bit operation! Offset={}", - Offset); - return; - } - - if (Is256Bit) { - vmovapd(ToYMM(Dst), ToYMM(DestVector)); - } else { - vmovapd(Dst, DestVector); - } - - const auto Insert = [&](const Xbyak::Xmm& reg, int index) { - switch (ElementSize) { - case 1: { - if (InUpperLane) { - index -= 16; - } - pinsrb(reg, GetSrc(Op->Src.ID()), index); - break; - } - case 2: { - if (InUpperLane) { - index -= 8; - } - pinsrw(reg, GetSrc(Op->Src.ID()), index); - break; - } - case 4: { - if (InUpperLane) { - index -= 4; - } - pinsrd(reg, GetSrc(Op->Src.ID()), index); - break; - } - case 8: { - if (InUpperLane) { - index -= 2; - } - pinsrq(reg, GetSrc(Op->Src.ID()), index); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } - }; - - if (InUpperLane) { - vextracti128(xmm15, ToYMM(Dst), 1); - Insert(xmm15, DestIdx); - vinserti128(ToYMM(Dst), ToYMM(Dst), xmm15, 1); - } else { - Insert(Dst, DestIdx); - } -} - -DEF_OP(VCastFromGPR) { - auto Op = IROp->C(); - switch (Op->Header.ElementSize) { - case 1: - movzx(rax, GetSrc(Op->Src.ID())); - vmovq(GetDst(Node), rax); - break; - case 2: - movzx(rax, GetSrc(Op->Src.ID())); - vmovq(GetDst(Node), rax); - break; - case 4: - vmovd(GetDst(Node), GetSrc(Op->Src.ID()).cvt32()); - break; - case 8: - vmovq(GetDst(Node), GetSrc(Op->Src.ID()).cvt64()); - break; - default: LOGMAN_MSG_A_FMT("Unknown VCastFromGPR element size: {}", Op->Header.ElementSize); - } -} - -DEF_OP(VDupFromGPR) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = IROp->ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Src = GetSrc(Op->Src.ID()).cvt64(); - - vmovq(Dst, Src); - - switch (ElementSize) { - case 1: - if (Is256Bit) { - vpbroadcastb(ToYMM(Dst), Dst); - } else { - vpbroadcastb(Dst, Dst); - } - break; - case 2: - if (Is256Bit) { - vpbroadcastw(ToYMM(Dst), Dst); - } else { - vpbroadcastw(Dst, Dst); - } - break; - case 4: - if (Is256Bit) { - vpbroadcastd(ToYMM(Dst), Dst); - } else { - vpbroadcastd(Dst, Dst); - } - break; - case 8: - if (Is256Bit) { - vpbroadcastq(ToYMM(Dst), Dst); - } else { - vpbroadcastq(Dst, Dst); - } - break; - default: - LOGMAN_MSG_A_FMT("Unhandled element size: {}", ElementSize); - return; - } -} - -DEF_OP(Float_FromGPR_S) { - const auto Op = IROp->C(); - - const uint16_t ElementSize = Op->Header.ElementSize; - const uint16_t Conv = (ElementSize << 8) | Op->SrcElementSize; - - switch (Conv) { - case 0x0404: { // Float <- int32_t - cvtsi2ss(GetDst(Node), GetSrc(Op->Src.ID())); - break; - } - case 0x0408: { // Float <- int64_t - cvtsi2ss(GetDst(Node), GetSrc(Op->Src.ID())); - break; - } - case 0x0804: { // Double <- int32_t - cvtsi2sd(GetDst(Node), GetSrc(Op->Src.ID())); - break; - } - case 0x0808: { // Double <- int64_t - cvtsi2sd(GetDst(Node), GetSrc(Op->Src.ID())); - break; - } - default: - LOGMAN_MSG_A_FMT("Unhandled conversion mask: Mask=0x{:04x}, ElementSize={}, SrcElementSize={}", - Conv, ElementSize, Op->SrcElementSize); - break; - } -} - -DEF_OP(Float_FToF) { - auto Op = IROp->C(); - const uint16_t Conv = (Op->Header.ElementSize << 8) | Op->SrcElementSize; - - switch (Conv) { - case 0x0804: { // Double <- Float - cvtss2sd(GetDst(Node), GetSrc(Op->Scalar.ID())); - break; - } - case 0x0408: { // Float <- Double - cvtsd2ss(GetDst(Node), GetSrc(Op->Scalar.ID())); - break; - } - default: LOGMAN_MSG_A_FMT("Unknown Float_FToF sizes: 0x{:x}", Conv); - } -} - -DEF_OP(Vector_SToF) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector = GetSrc(Op->Vector.ID()); - - if (OpSize == ElementSize) { - if (ElementSize == 8) { - vmovq(TMP1, Vector); - cvtsi2sd(Dst, TMP1); - } - else if (ElementSize == 4) { - vmovd(TMP1.cvt32(), Vector); - cvtsi2ss(Dst, TMP1.cvt32()); - } - else { - LOGMAN_MSG_A_FMT("Unknown Vector_SToF element size: {}", ElementSize); - } - } - else { - switch (ElementSize) { - case 4: - if (Is256Bit) { - vcvtdq2ps(ToYMM(Dst), ToYMM(Vector)); - } else { - vcvtdq2ps(Dst, Vector); - } - break; - case 8: - // This operation is a bit disgusting in x86 - // There is no vector form of this instruction until AVX512VL + AVX512DQ (vcvtqq2pd) - // 1) First extract the top 64bits - // 2) Do a scalar conversion on each - // 3) Make sure to merge them together at the end - pextrq(rax, Vector, 1); - pextrq(rcx, Vector, 0); - cvtsi2sd(Dst, rcx); - cvtsi2sd(xmm15, rax); - if (Is256Bit) { - movlhps(Dst, xmm15); - vextracti128(xmm15, ToYMM(Vector), 1); - - pextrq(rax, xmm15, 1); - pextrq(rcx, xmm15, 0); - cvtsi2sd(xmm15, rcx); - cvtsi2sd(xmm14, rax); - movlhps(xmm15, xmm14); - - vinserti128(ToYMM(Dst), ToYMM(Dst), xmm15, 1); - } else { - vmovlhps(Dst, Dst, xmm15); - } - break; - default: - LOGMAN_MSG_A_FMT("Unknown Vector_SToF element size: {}", ElementSize); - break; - } - } -} - -DEF_OP(Vector_FToZS) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector = GetSrc(Op->Vector.ID()); - - if (OpSize == ElementSize) { - if (ElementSize == 8) { - cvttss2si(Dst, Vector); - } - else if (ElementSize == 4) { - cvttss2si(Dst.cvt32(), Vector); - } - else { - LOGMAN_MSG_A_FMT("Unknown Vector_FToZS element size: {}", ElementSize); - } - } - else { - switch (ElementSize) { - case 4: - if (Is256Bit) { - vcvttps2dq(ToYMM(Dst), ToYMM(Vector)); - } else { - vcvttps2dq(Dst, Vector); - } - break; - case 8: - if (Is256Bit) { - vcvttpd2dq(ToYMM(Dst), ToYMM(Vector)); - } else { - vcvttpd2dq(Dst, Vector); - } - break; - default: - LOGMAN_MSG_A_FMT("Unknown Vector_FToZS element size: {}", ElementSize); - break; - } - } -} - -DEF_OP(Vector_FToS) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector = GetSrc(Op->Vector.ID()); - - if (OpSize == ElementSize) { - if (ElementSize == 8) { - cvtss2si(Dst, Vector); - } - else if (ElementSize == 4) { - cvtss2si(Dst.cvt32(), Vector); - } - else { - LOGMAN_MSG_A_FMT("Unknown Vector_FToS element size: {}", ElementSize); - } - } - else { - switch (ElementSize) { - case 4: - if (Is256Bit) { - vcvtps2dq(ToYMM(Dst), ToYMM(Vector)); - } else { - vcvtps2dq(Dst, Vector); - } - break; - case 8: - if (Is256Bit) { - vcvtpd2dq(ToYMM(Dst), ToYMM(Vector)); - } else { - vcvtpd2dq(Dst, Vector); - } - break; - default: - LOGMAN_MSG_A_FMT("Unknown Vector_FToS element size: {}", ElementSize); - break; - } - } -} - -DEF_OP(Vector_FToF) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - const auto Conv = (ElementSize << 8) | Op->SrcElementSize; - - const auto Dst = GetDst(Node); - const auto Vector = GetSrc(Op->Vector.ID()); - - switch (Conv) { - case 0x0804: { // Double <- Float - if (Is256Bit) { - vcvtps2pd(ToYMM(Dst), Vector); - } else { - vcvtps2pd(Dst, Vector); - } - break; - } - case 0x0408: { // Float <- Double - if (Is256Bit) { - vcvtpd2ps(Dst, ToYMM(Vector)); - } else { - vcvtpd2ps(Dst, Vector); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Vector_FToF conversion type : 0x{:04x}", Conv); - break; - } -} - -DEF_OP(Vector_FToI) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const uint8_t RoundMode = [Op] { - switch (Op->Round) { - case FEXCore::IR::Round_Nearest.Val: - return 0b0000'0'0'00; - case FEXCore::IR::Round_Negative_Infinity.Val: - return 0b0000'0'0'01; - case FEXCore::IR::Round_Positive_Infinity.Val: - return 0b0000'0'0'10; - case FEXCore::IR::Round_Towards_Zero.Val: - return 0b0000'0'0'11; - case FEXCore::IR::Round_Host.Val: - return 0b0000'0'1'00; - default: - LOGMAN_MSG_A_FMT("Unhandled rounding mode"); - return 0; - } - }(); - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector = GetSrc(Op->Vector.ID()); - - switch (ElementSize) { - case 4: - if (Is256Bit) { - vroundps(ToYMM(Dst), ToYMM(Vector), RoundMode); - } else { - vroundps(Dst, Vector, RoundMode); - } - break; - case 8: - if (Is256Bit) { - vroundpd(ToYMM(Dst), ToYMM(Vector), RoundMode); - } else { - vroundpd(Dst, Vector, RoundMode); - } - break; - default: - LOGMAN_MSG_A_FMT("Unhandled element size: {}", ElementSize); - break; - } -} - -#undef DEF_OP -void X86JITCore::RegisterConversionHandlers() { -#define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &X86JITCore::Op_##x - REGISTER_OP(VINSGPR, VInsGPR); - REGISTER_OP(VCASTFROMGPR, VCastFromGPR); - REGISTER_OP(VDUPFROMGPR, VDupFromGPR); - REGISTER_OP(FLOAT_FROMGPR_S, Float_FromGPR_S); - REGISTER_OP(FLOAT_FTOF, Float_FToF); - REGISTER_OP(VECTOR_STOF, Vector_SToF); - REGISTER_OP(VECTOR_FTOZS, Vector_FToZS); - REGISTER_OP(VECTOR_FTOS, Vector_FToS); - REGISTER_OP(VECTOR_FTOF, Vector_FToF); - REGISTER_OP(VECTOR_FTOI, Vector_FToI); -#undef REGISTER_OP -} -} - diff --git a/FEXCore/Source/Interface/Core/JIT/x86_64/EncryptionOps.cpp b/FEXCore/Source/Interface/Core/JIT/x86_64/EncryptionOps.cpp deleted file mode 100644 index 4b424572f..000000000 --- a/FEXCore/Source/Interface/Core/JIT/x86_64/EncryptionOps.cpp +++ /dev/null @@ -1,161 +0,0 @@ -// SPDX-License-Identifier: MIT -/* -$info$ -tags: backend|x86-64 -$end_info$ -*/ - -#include "Interface/Core/JIT/x86_64/JITClass.h" -#include "Interface/Core/Dispatcher/X86Dispatcher.h" -#include - -#include -#include - -namespace FEXCore::CPU { -#define DEF_OP(x) void X86JITCore::Op_##x(IR::IROp_Header *IROp, IR::NodeID Node) - -DEF_OP(AESImc) { - auto Op = IROp->C(); - vaesimc(GetDst(Node), GetSrc(Op->Vector.ID())); -} - -DEF_OP(AESEnc) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Key = GetSrc(Op->Key.ID()); - const auto State = GetSrc(Op->State.ID()); - - if (Is256Bit) { - vaesenc(ToYMM(Dst), ToYMM(State), ToYMM(Key)); - } else { - vaesenc(Dst, State, Key); - } -} - -DEF_OP(AESEncLast) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Key = GetSrc(Op->Key.ID()); - const auto State = GetSrc(Op->State.ID()); - - if (Is256Bit) { - vaesenclast(ToYMM(Dst), ToYMM(State), ToYMM(Key)); - } else { - vaesenclast(Dst, State, Key); - } -} - -DEF_OP(AESDec) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Key = GetSrc(Op->Key.ID()); - const auto State = GetSrc(Op->State.ID()); - - if (Is256Bit) { - vaesdec(ToYMM(Dst), ToYMM(State), ToYMM(Key)); - } else { - vaesdec(Dst, State, Key); - } -} - -DEF_OP(AESDecLast) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Key = GetSrc(Op->Key.ID()); - const auto State = GetSrc(Op->State.ID()); - - if (Is256Bit) { - vaesdeclast(ToYMM(Dst), ToYMM(State), ToYMM(Key)); - } else { - vaesdeclast(Dst, State, Key); - } -} - -DEF_OP(AESKeyGenAssist) { - auto Op = IROp->C(); - vaeskeygenassist(GetDst(Node), GetSrc(Op->Src.ID()), Op->RCON); -} - -DEF_OP(CRC32) { - auto Op = IROp->C(); - switch (IROp->Size) { - case 4: - mov(TMP1, GetSrc(Op->Src2.ID())); - mov(GetDst(Node), GetSrc(Op->Src1.ID())); - break; - case 8: - mov(TMP1, GetSrc(Op->Src2.ID())); - mov(GetDst(Node), GetSrc(Op->Src1.ID())); - break; - default: LOGMAN_MSG_A_FMT("Unknown CRC32 size: {}", IROp->Size); - } - - switch (Op->SrcSize) { - case 1: - crc32(GetDst(Node).cvt32(), TMP1.cvt8()); - break; - case 2: - crc32(GetDst(Node).cvt32(), TMP1.cvt16()); - break; - case 4: - crc32(GetDst(Node).cvt32(), TMP1.cvt32()); - break; - case 8: - crc32(GetDst(Node).cvt64(), TMP1.cvt64()); - break; - } -} - -DEF_OP(PCLMUL) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Src1 = GetSrc(Op->Src1.ID()); - const auto Src2 = GetSrc(Op->Src2.ID()); - - switch (Op->Selector) { - case 0b00000000: - case 0b00000001: - case 0b00010000: - case 0b00010001: - if (Is256Bit) { - vpclmulqdq(ToYMM(Dst), ToYMM(Src1), ToYMM(Src2), Op->Selector); - } else { - vpclmulqdq(Dst, Src1, Src2, Op->Selector); - } - break; - default: - LOGMAN_MSG_A_FMT("Unknown PCLMUL selector: {}", Op->Selector); - break; - } -} - -#undef DEF_OP -void X86JITCore::RegisterEncryptionHandlers() { -#define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &X86JITCore::Op_##x - REGISTER_OP(VAESIMC, AESImc); - REGISTER_OP(VAESENC, AESEnc); - REGISTER_OP(VAESENCLAST, AESEncLast); - REGISTER_OP(VAESDEC, AESDec); - REGISTER_OP(VAESDECLAST, AESDecLast); - REGISTER_OP(VAESKEYGENASSIST, AESKeyGenAssist); - REGISTER_OP(CRC32, CRC32); - REGISTER_OP(PCLMUL, PCLMUL); -#undef REGISTER_OP -} -} diff --git a/FEXCore/Source/Interface/Core/JIT/x86_64/FlagOps.cpp b/FEXCore/Source/Interface/Core/JIT/x86_64/FlagOps.cpp deleted file mode 100644 index d8b34be52..000000000 --- a/FEXCore/Source/Interface/Core/JIT/x86_64/FlagOps.cpp +++ /dev/null @@ -1,35 +0,0 @@ -// SPDX-License-Identifier: MIT -/* -$info$ -tags: backend|x86-64 -$end_info$ -*/ - -#include "Interface/Core/JIT/x86_64/JITClass.h" -#include "Interface/Core/Dispatcher/X86Dispatcher.h" - -#include - -#include -#include - -namespace FEXCore::CPU { - -#define DEF_OP(x) void X86JITCore::Op_##x(IR::IROp_Header *IROp, IR::NodeID Node) -DEF_OP(GetHostFlag) { - auto Op = IROp->C(); - - mov(rax, GetSrc(Op->Value.ID())); - shr(rax, Op->Flag); - and_(rax, 1); - mov(GetDst(Node), rax); -} - -#undef DEF_OP -void X86JITCore::RegisterFlagHandlers() { -#define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &X86JITCore::Op_##x - REGISTER_OP(GETHOSTFLAG, GetHostFlag); -#undef REGISTER_OP -} -} - diff --git a/FEXCore/Source/Interface/Core/JIT/x86_64/JIT.cpp b/FEXCore/Source/Interface/Core/JIT/x86_64/JIT.cpp deleted file mode 100644 index 75cb39601..000000000 --- a/FEXCore/Source/Interface/Core/JIT/x86_64/JIT.cpp +++ /dev/null @@ -1,884 +0,0 @@ -// SPDX-License-Identifier: MIT -/* -$info$ -tags: backend|x86-64 -desc: Main glue logic of the x86-64 splatter backend -$end_info$ -*/ - -#include "Interface/Context/Context.h" -#include "Interface/Core/LookupCache.h" - -#include "Interface/Core/Dispatcher/Dispatcher.h" -#include "Interface/Core/Dispatcher/X86Dispatcher.h" -#include "Interface/Core/Interpreter/InterpreterOps.h" -#include "Interface/Core/JIT/x86_64/JITClass.h" -#include "Interface/IR/PassManager.h" -#include "Interface/IR/Passes/RegisterAllocationPass.h" - -#include "Utils/MemberFunctionToPointer.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// #define DEBUG_RA 1 -// #define DEBUG_CYCLES - -static constexpr size_t INITIAL_CODE_SIZE = 1024 * 1024 * 16; -static constexpr size_t MAX_CODE_SIZE = 1024 * 1024 * 256; - -namespace { -static void PrintValue(uint64_t Value) { - LogMan::Msg::DFmt("Value: 0x{:x}", Value); -} - -static void PrintVectorValue(uint64_t Value, uint64_t ValueUpper) { - LogMan::Msg::DFmt("Value: 0x{:016x}'{:016x}", ValueUpper, Value); -} -} - -namespace FEXCore::CPU { - -void X86JITCore::PushRegs() { - const auto AVXRegSize = Core::CPUState::XMM_AVX_REG_SIZE; - - sub(rsp, AVXRegSize * RAXMM_x.size()); - for (size_t i = 0; i < RAXMM_x.size(); ++i) { - vmovups(ptr[rsp + i * AVXRegSize], ToYMM(RAXMM_x[i])); - } - - for (const auto &Reg : RA64) { - push(Reg); - } - - const auto NumPush = RA64.size(); - if ((NumPush & 1) != 0) { - // Align - sub(rsp, 8); - } -} - -void X86JITCore::PopRegs() { - const auto AVXRegSize = Core::CPUState::XMM_AVX_REG_SIZE; - const auto NumPush = RA64.size(); - - if ((NumPush & 1) != 0) { - // Align - add(rsp, 8); - } - - for (uint32_t i = RA64.size(); i > 0; --i) { - pop(RA64[i - 1]); - } - - for (size_t i = 0; i < RAXMM_x.size(); ++i) { - vmovups(ToYMM(RAXMM_x[i]), ptr[rsp + i * AVXRegSize]); - } - - add(rsp, AVXRegSize * RAXMM_x.size()); -} - -void X86JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) { - FallbackInfo Info; - if (!InterpreterOps::GetFallbackHandler(IROp, &Info)) { -#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED - LOGMAN_MSG_A_FMT("Unhandled IR Op: {}", FEXCore::IR::GetName(IROp->Op)); -#endif - } else { - switch(Info.ABI) { - case FABI_F80_I16_F32:{ - PushRegs(); - - movss(xmm0, GetSrc(IROp->Args[0].ID())); - mov(rdi, word [STATE + offsetof(FEXCore::Core::CPUState, FCW)]); - call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]); - - PopRegs(); - - pxor(GetDst(Node), GetDst(Node)); - movq(GetDst(Node), rax); - pinsrw(GetDst(Node), edx, 4); - } - break; - - case FABI_F80_I16_F64:{ - PushRegs(); - - movsd(xmm0, GetSrc(IROp->Args[0].ID())); - mov(rdi, word [STATE + offsetof(FEXCore::Core::CPUState, FCW)]); - call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]); - - PopRegs(); - - pxor(GetDst(Node), GetDst(Node)); - movq(GetDst(Node), rax); - pinsrw(GetDst(Node), edx, 4); - } - break; - - case FABI_F80_I16_I16: - case FABI_F80_I16_I32: { - PushRegs(); - - mov(rdi, word [STATE + offsetof(FEXCore::Core::CPUState, FCW)]); - - if (Info.ABI == FABI_F80_I16_I16) { - movsx(rsi, GetSrc(IROp->Args[0].ID()).cvt16()); - } - else { - mov(esi, GetSrc(IROp->Args[0].ID())); - } - call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]); - - PopRegs(); - - pxor(GetDst(Node), GetDst(Node)); - movq(GetDst(Node), rax); - pinsrw(GetDst(Node), edx, 4); - } - break; - - case FABI_F32_I16_F80:{ - PushRegs(); - - mov(rdi, word [STATE + offsetof(FEXCore::Core::CPUState, FCW)]); - movq(rsi, GetSrc(IROp->Args[0].ID())); - pextrq(rdx, GetSrc(IROp->Args[0].ID()), 1); - - call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]); - - PopRegs(); - - movss(GetDst(Node), xmm0); - } - break; - - case FABI_F64_I16_F80:{ - PushRegs(); - - mov(rdi, word [STATE + offsetof(FEXCore::Core::CPUState, FCW)]); - movq(rsi, GetSrc(IROp->Args[0].ID())); - pextrq(rdx, GetSrc(IROp->Args[0].ID()), 1); - - call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]); - - PopRegs(); - - movsd(GetDst(Node), xmm0); - } - break; - - case FABI_F64_I16_F64: { - PushRegs(); - - mov(rdi, word [STATE + offsetof(FEXCore::Core::CPUState, FCW)]); - movsd(xmm0, GetSrc(IROp->Args[0].ID())); - - call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]); - - PopRegs(); - - movsd(GetDst(Node), xmm0); - } - break; - - case FABI_F64_I16_F64_F64: { - PushRegs(); - - mov(rdi, word [STATE + offsetof(FEXCore::Core::CPUState, FCW)]); - movsd(xmm0, GetSrc(IROp->Args[0].ID())); - movsd(xmm1, GetSrc(IROp->Args[1].ID())); - - call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]); - - PopRegs(); - - movsd(GetDst(Node), xmm0); - } - break; - - case FABI_I16_I16_F80:{ - PushRegs(); - - mov(rdi, word [STATE + offsetof(FEXCore::Core::CPUState, FCW)]); - movq(rsi, GetSrc(IROp->Args[0].ID())); - pextrq(rdx, GetSrc(IROp->Args[0].ID()), 1); - - call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]); - - PopRegs(); - - movsx(GetDst(Node), ax); - } - break; - case FABI_I32_I16_F80:{ - PushRegs(); - - mov(rdi, word [STATE + offsetof(FEXCore::Core::CPUState, FCW)]); - movq(rsi, GetSrc(IROp->Args[0].ID())); - pextrq(rdx, GetSrc(IROp->Args[0].ID()), 1); - - call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]); - - PopRegs(); - - mov(GetDst(Node), eax); - } - break; - case FABI_I64_I16_F80:{ - PushRegs(); - - mov(rdi, word [STATE + offsetof(FEXCore::Core::CPUState, FCW)]); - movq(rsi, GetSrc(IROp->Args[0].ID())); - pextrq(rdx, GetSrc(IROp->Args[0].ID()), 1); - - call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]); - - PopRegs(); - - mov(GetDst(Node), rax); - } - break; - case FABI_I64_I16_F80_F80:{ - PushRegs(); - - mov(rdi, word [STATE + offsetof(FEXCore::Core::CPUState, FCW)]); - movq(rsi, GetSrc(IROp->Args[0].ID())); - pextrq(rdx, GetSrc(IROp->Args[0].ID()), 1); - - movq(rcx, GetSrc(IROp->Args[1].ID())); - pextrq(r8, GetSrc(IROp->Args[1].ID()), 1); - - call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]); - - PopRegs(); - - mov(GetDst(Node), rax); - } - break; - case FABI_F80_I16_F80:{ - PushRegs(); - - mov(rdi, word [STATE + offsetof(FEXCore::Core::CPUState, FCW)]); - movq(rsi, GetSrc(IROp->Args[0].ID())); - pextrq(rdx, GetSrc(IROp->Args[0].ID()), 1); - - call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]); - - PopRegs(); - - pxor(GetDst(Node), GetDst(Node)); - movq(GetDst(Node), rax); - pinsrw(GetDst(Node), edx, 4); - } - break; - case FABI_F80_I16_F80_F80:{ - PushRegs(); - - mov(rdi, word [STATE + offsetof(FEXCore::Core::CPUState, FCW)]); - movq(rsi, GetSrc(IROp->Args[0].ID())); - pextrq(rdx, GetSrc(IROp->Args[0].ID()), 1); - - movq(rcx, GetSrc(IROp->Args[1].ID())); - pextrq(r8, GetSrc(IROp->Args[1].ID()), 1); - - call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]); - - PopRegs(); - - pxor(GetDst(Node), GetDst(Node)); - movq(GetDst(Node), rax); - pinsrw(GetDst(Node), edx, 4); - } - break; - - case FABI_I32_I64_I64_I128_I128_I16: { - PushRegs(); - - const auto Op = IROp->C(); - const auto Control = Op->Control; - - const auto LHS = GetSrc(Op->LHS.ID()); - const auto RHS = GetSrc(Op->RHS.ID()); - const auto SrcRAX = GetSrc(Op->RAX.ID()); - const auto SrcRDX = GetSrc(Op->RDX.ID()); - - mov(rdi, SrcRAX); - mov(rsi, SrcRDX); - - movq(rdx, LHS); - pextrq(rcx, LHS, 1); - - movq(r8, RHS); - pextrq(r9, RHS, 1); - - sub(rsp, 16); - mov(dword [rsp], Control); - - call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]); - - add(rsp, 16); - PopRegs(); - - mov(GetDst(Node), rax); - break; - } - - case FABI_I32_I128_I128_I16: { - PushRegs(); - - const auto Op = IROp->C(); - - const auto LHS = GetSrc(Op->LHS.ID()); - const auto RHS = GetSrc(Op->RHS.ID()); - const auto Control = Op->Control; - - movq(rdi, LHS); - pextrq(rsi, LHS, 1); - - movq(rdx, RHS); - pextrq(rcx, RHS, 1); - - mov(r8, Control); - - call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]); - - PopRegs(); - - mov(GetDst(Node), rax); - break; - } - - case FABI_UNKNOWN: - default: -#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED - LOGMAN_MSG_A_FMT("Unhandled IR Fallback ABI: {} {}", - IR::GetName(IROp->Op), ToUnderlying(Info.ABI)); -#endif - break; - } - } -} - -static uint64_t X86JITCore_ExitFunctionLink(FEXCore::Core::CpuStateFrame *Frame, uint64_t *record) { - auto Thread = Frame->Thread; - auto GuestRip = record[1]; - - auto HostCode = Thread->LookupCache->FindBlock(GuestRip); - - if (!HostCode) { - Thread->CurrentFrame->State.rip = GuestRip; - return Frame->Pointers.Common.DispatcherLoopTop; - } - - auto LinkerAddress = Frame->Pointers.Common.ExitFunctionLinker; - Thread->LookupCache->AddBlockLink(GuestRip, (uintptr_t)record, [record, LinkerAddress]{ - // undo the link - record[0] = LinkerAddress; - }); - - record[0] = HostCode; - return HostCode; -} - -void X86JITCore::Op_NoOp(IR::IROp_Header *IROp, IR::NodeID Node) { -} - -X86JITCore::X86JITCore(FEXCore::Context::ContextImpl *ctx, FEXCore::Core::InternalThreadState *Thread) - : CPUBackend(Thread, INITIAL_CODE_SIZE, MAX_CODE_SIZE) - , CodeGenerator(0, this, nullptr) // this is not used here - , CTX {ctx} { - - RAPass = Thread->PassManager->GetPass("RA"); - - RAPass->AllocateRegisterSet(RegisterClasses); - RAPass->AddRegisters(FEXCore::IR::GPRClass, NumGPRs); - RAPass->AddRegisters(FEXCore::IR::FPRClass, NumXMMs); - RAPass->AddRegisters(FEXCore::IR::GPRPairClass, NumGPRPairs); - - for (uint32_t i = 0; i < NumGPRPairs; ++i) { - RAPass->AddRegisterConflict(FEXCore::IR::GPRClass, i * 2, FEXCore::IR::GPRPairClass, i); - RAPass->AddRegisterConflict(FEXCore::IR::GPRClass, i * 2 + 1, FEXCore::IR::GPRPairClass, i); - } - - for (uint32_t i = 0; i < FEXCore::IR::IROps::OP_LAST + 1; ++i) { - OpHandlers[i] = &X86JITCore::Op_Unhandled; - } - - RegisterALUHandlers(); - RegisterAtomicHandlers(); - RegisterBranchHandlers(); - RegisterConversionHandlers(); - RegisterFlagHandlers(); - RegisterMemoryHandlers(); - RegisterMiscHandlers(); - RegisterMoveHandlers(); - RegisterVectorHandlers(); - RegisterEncryptionHandlers(); - - { - auto &Common = ThreadState->CurrentFrame->Pointers.Common; - - Common.PrintValue = reinterpret_cast(PrintValue); - Common.PrintVectorValue = reinterpret_cast(PrintVectorValue); - Common.ThreadRemoveCodeEntryFromJIT = reinterpret_cast(&Context::ContextImpl::ThreadRemoveCodeEntryFromJit); - Common.CPUIDObj = reinterpret_cast(&CTX->CPUID); - - { - FEXCore::Utils::MemberFunctionToPointerCast PMF(&FEXCore::CPUIDEmu::RunFunction); - Common.CPUIDFunction = PMF.GetConvertedPointer(); - } - - { - FEXCore::Utils::MemberFunctionToPointerCast PMF(&FEXCore::CPUIDEmu::RunXCRFunction); - Common.XCRFunction = PMF.GetConvertedPointer(); - } - - Common.SyscallHandlerObj = reinterpret_cast(CTX->SyscallHandler); - Common.SyscallHandlerFunc = reinterpret_cast(FEXCore::Context::HandleSyscall); - Common.ExitFunctionLink = reinterpret_cast(&Context::ContextImpl::ThreadExitFunctionLink); - - // Fill in the fallback handlers - InterpreterOps::FillFallbackIndexPointers(Common.FallbackHandlerPointers); - } - - // Must be done after Dispatcher init - ClearCache(); -} - -X86JITCore::~X86JITCore() { - -} - -void X86JITCore::EmitDetectionString() { - const char JITString[] = "FEXJIT::X86JITCore::"; - for (char c : JITString) { - db(c); - } -} - -void X86JITCore::ClearCache() { - auto CodeBuffer = GetEmptyCodeBuffer(); - setNewBuffer(CodeBuffer->Ptr, CodeBuffer->Size); - EmitDetectionString(); -} - -IR::PhysicalRegister X86JITCore::GetPhys(IR::NodeID Node) const { - auto PhyReg = RAData->GetNodeRegister(Node); - - LOGMAN_THROW_AA_FMT(PhyReg.Raw != 255, "Couldn't Allocate register for node: ssa{}. Class: {}", Node, PhyReg.Class); - - return PhyReg; -} - -bool X86JITCore::IsFPR(IR::NodeID Node) const { - return RAData->GetNodeRegister(Node).Class == IR::FPRClass; -} - -bool X86JITCore::IsGPR(IR::NodeID Node) const { - return RAData->GetNodeRegister(Node).Class == IR::GPRClass; -} - -bool X86JITCore::IsGPRPair(IR::NodeID Node) const { - return RAData->GetNodeRegister(Node).Class == IR::GPRPairClass; -} - -template -Xbyak::Reg X86JITCore::GetSrc(IR::NodeID Node) const { - // rax, rcx, rdx, rsi, r8, r9, - // r10 - // Callee Saved - // rbx, rbp, r12, r13, r14, r15 - auto PhyReg = GetPhys(Node); - if constexpr (RAType == RA_64) - return RA64[PhyReg.Reg].cvt64(); - else if constexpr (RAType == RA_XMM) - return RAXMM[PhyReg.Reg]; - else if constexpr (RAType == RA_32) - return RA64[PhyReg.Reg].cvt32(); - else if constexpr (RAType == RA_16) - return RA64[PhyReg.Reg].cvt16(); - else if constexpr (RAType == RA_8) - return RA64[PhyReg.Reg].cvt8(); -} - -template -Xbyak::Reg X86JITCore::GetSrc(IR::NodeID Node) const; - -template -Xbyak::Reg X86JITCore::GetSrc(IR::NodeID Node) const; - -template -Xbyak::Reg X86JITCore::GetSrc(IR::NodeID Node) const; - -template -Xbyak::Reg X86JITCore::GetSrc(IR::NodeID Node) const; - -Xbyak::Xmm X86JITCore::GetSrc(IR::NodeID Node) const { - auto PhyReg = GetPhys(Node); - return RAXMM_x[PhyReg.Reg]; -} - -template -Xbyak::Reg X86JITCore::GetDst(IR::NodeID Node) const { - auto PhyReg = GetPhys(Node); - if constexpr (RAType == RA_64) - return RA64[PhyReg.Reg].cvt64(); - else if constexpr (RAType == RA_XMM) - return RAXMM[PhyReg.Reg]; - else if constexpr (RAType == RA_32) - return RA64[PhyReg.Reg].cvt32(); - else if constexpr (RAType == RA_16) - return RA64[PhyReg.Reg].cvt16(); - else if constexpr (RAType == RA_8) - return RA64[PhyReg.Reg].cvt8(); -} - -template -Xbyak::Reg X86JITCore::GetDst(IR::NodeID Node) const; - -template -Xbyak::Reg X86JITCore::GetDst(IR::NodeID Node) const; - -template -Xbyak::Reg X86JITCore::GetDst(IR::NodeID Node) const; - -template -Xbyak::Reg X86JITCore::GetDst(IR::NodeID Node) const; - -template -std::pair X86JITCore::GetSrcPair(IR::NodeID Node) const { - auto PhyReg = GetPhys(Node); - if constexpr (RAType == RA_64) - return RA64Pair[PhyReg.Reg]; - else if constexpr (RAType == RA_32) - return {RA64Pair[PhyReg.Reg].first.cvt32(), RA64Pair[PhyReg.Reg].second.cvt32()}; -} - -template -std::pair X86JITCore::GetSrcPair(IR::NodeID Node) const; - -template -std::pair X86JITCore::GetSrcPair(IR::NodeID Node) const; - -Xbyak::Xmm X86JITCore::GetDst(IR::NodeID Node) const { - auto PhyReg = GetPhys(Node); - return RAXMM_x[PhyReg.Reg]; -} - -bool X86JITCore::IsInlineConstant(const IR::OrderedNodeWrapper& WNode, uint64_t* Value) const { - auto OpHeader = IR->GetOp(WNode); - - if (OpHeader->Op == IR::IROps::OP_INLINECONSTANT) { - auto Op = OpHeader->C(); - if (Value) { - *Value = Op->Constant; - } - return true; - } else { - return false; - } -} - -bool X86JITCore::IsInlineEntrypointOffset(const IR::OrderedNodeWrapper& WNode, uint64_t* Value) const { - auto OpHeader = IR->GetOp(WNode); - - if (OpHeader->Op == IR::IROps::OP_INLINEENTRYPOINTOFFSET) { - auto Op = OpHeader->C(); - if (Value) { - uint64_t Mask = ~0ULL; - uint8_t OpSize = OpHeader->Size; - if (OpSize == 4) { - Mask = 0xFFFF'FFFFULL; - } - *Value = (Entry + Op->Offset) & Mask; - } - return true; - } else { - return false; - } -} - -std::tuple X86JITCore::GetCC(IR::CondClassType cond) { - switch (cond.Val) { - case FEXCore::IR::COND_EQ: return { &CodeGenerator::sete , &CodeGenerator::cmove , &CodeGenerator::je }; - case FEXCore::IR::COND_NEQ: return { &CodeGenerator::setne, &CodeGenerator::cmovne, &CodeGenerator::jne }; - case FEXCore::IR::COND_SGE: return { &CodeGenerator::setge, &CodeGenerator::cmovge, &CodeGenerator::jge }; - case FEXCore::IR::COND_SLT: return { &CodeGenerator::setl , &CodeGenerator::cmovl , &CodeGenerator::jl }; - case FEXCore::IR::COND_SGT: return { &CodeGenerator::setg , &CodeGenerator::cmovg , &CodeGenerator::jg }; - case FEXCore::IR::COND_SLE: return { &CodeGenerator::setle, &CodeGenerator::cmovle, &CodeGenerator::jle }; - case FEXCore::IR::COND_UGE: return { &CodeGenerator::setae, &CodeGenerator::cmovae, &CodeGenerator::jae }; - case FEXCore::IR::COND_ULT: return { &CodeGenerator::setb , &CodeGenerator::cmovb , &CodeGenerator::jb }; - case FEXCore::IR::COND_UGT: return { &CodeGenerator::seta , &CodeGenerator::cmova , &CodeGenerator::ja }; - case FEXCore::IR::COND_ULE: return { &CodeGenerator::setna, &CodeGenerator::cmovna, &CodeGenerator::jna }; - - case FEXCore::IR::COND_FLU: return { &CodeGenerator::setb , &CodeGenerator::cmovb , &CodeGenerator::jb }; - case FEXCore::IR::COND_FGE: return { &CodeGenerator::setae, &CodeGenerator::cmovae, &CodeGenerator::jae }; - case FEXCore::IR::COND_FLEU: return { &CodeGenerator::setna, &CodeGenerator::cmovna, &CodeGenerator::jna }; - case FEXCore::IR::COND_FGT: return { &CodeGenerator::seta , &CodeGenerator::cmova , &CodeGenerator::ja }; - case FEXCore::IR::COND_FU: return { &CodeGenerator::setp , &CodeGenerator::cmovp , &CodeGenerator::jp }; - case FEXCore::IR::COND_FNU: return { &CodeGenerator::setnp, &CodeGenerator::cmovnp, &CodeGenerator::jnp }; - - case FEXCore::IR::COND_MI: - case FEXCore::IR::COND_PL: - case FEXCore::IR::COND_VS: - case FEXCore::IR::COND_VC: - default: - LOGMAN_MSG_A_FMT("Unsupported compare type"); - break; - } - - // Hope for the best - return { &CodeGenerator::sete , &CodeGenerator::cmove , &CodeGenerator::je }; -} - -CPUBackend::CompiledCode X86JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IRListView const *IR, [[maybe_unused]] FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData, bool GDBEnabled) { - - FEXCORE_PROFILE_SCOPED("x86::CompileCode"); - JumpTargets.clear(); - uint32_t SSACount = IR->GetSSACount(); - - this->Entry = Entry; - this->RAData = RAData; - this->DebugData = DebugData; - - // Fairly excessive buffer range to make sure we don't overflow - uint32_t BufferRange = SSACount * 16 + GDBEnabled * Dispatcher::MaxGDBPauseCheckSize; - if ((getSize() + BufferRange) > CurrentCodeBuffer->Size) { - CTX->ClearCodeCache(ThreadState); - } - - CodeData.BlockBegin = getCurr(); - - // Put the code header at the start of the data block. - Label JITCodeHeaderLabel{}; - L(JITCodeHeaderLabel); - - JITCodeHeader *CodeHeader = getCurr(); - setSize(getSize() + sizeof(JITCodeHeader)); - - CodeData.BlockEntry = getCurr(); - - // Get the address of the JITCodeHeader and store in to the core state. - // Only two instructions, so very low overhead. - lea(TMP1, ptr [rip + JITCodeHeaderLabel]); - mov(qword [STATE + offsetof(FEXCore::Core::CPUState, InlineJITBlockHeader)], TMP1); - - CursorEntry = getSize(); - this->IR = IR; - - if (GDBEnabled) { - auto GDBSize = CTX->Dispatcher->GenerateGDBPauseCheck(CodeData.BlockBegin, Entry); - setSize(getSize() + GDBSize); - } - - LOGMAN_THROW_AA_FMT(RAData != nullptr, "Needs RA"); - - SpillSlots = RAData->SpillSlots(); - - if (SpillSlots) { - sub(rsp, SpillSlots * MaxSpillSlotSize); - } - -#ifdef BLOCKSTATS - BlockSamplingData::BlockData *SamplingData = CTX->BlockData->GetBlockData(Entry); - if (GetSamplingData) { - mov(rcx, reinterpret_cast(SamplingData)); - rdtsc(); - shl(rdx, 32); - or_(rax, rdx); - mov(qword [rcx + offsetof(BlockSamplingData::BlockData, Start)], rax); - } - - auto ExitBlock = [&]() { - if (GetSamplingData) { - mov(rcx, reinterpret_cast(SamplingData)); - // Get time - rdtsc(); - shl(rdx, 32); - or_(rax, rdx); - - // Calculate time spent in block - mov(rdx, qword [rcx + offsetof(BlockSamplingData::BlockData, Start)]); - sub(rax, rdx); - - // Add time to total time - add(qword [rcx + offsetof(BlockSamplingData::BlockData, TotalTime)], rax); - - // Increment call count - inc(qword [rcx + offsetof(BlockSamplingData::BlockData, TotalCalls)]); - - // Calculate min - mov(rdx, qword [rcx + offsetof(BlockSamplingData::BlockData, Min)]); - cmp(rdx, rax); - cmova(rdx, rax); - mov(qword [rcx + offsetof(BlockSamplingData::BlockData, Min)], rdx); - - // Calculate max - mov(rdx, qword [rcx + offsetof(BlockSamplingData::BlockData, Max)]); - cmp(rdx, rax); - cmovb(rdx, rax); - mov(qword [rcx + offsetof(BlockSamplingData::BlockData, Max)], rdx); - } - }; -#endif - - PendingTargetLabel = nullptr; - - for (auto [BlockNode, BlockHeader] : IR->GetBlocks()) { - using namespace FEXCore::IR; -#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED - auto BlockIROp = BlockHeader->CW(); - LOGMAN_THROW_AA_FMT(BlockIROp->Header.Op == IR::OP_CODEBLOCK, "IR type failed to be a code block"); -#endif - - auto BlockStartHostCode = getCurr(); - { - const auto Node = IR->GetID(BlockNode); - const auto IsTarget = JumpTargets.try_emplace(Node).first; - - // if there is a pending branch, and it is not fall-through - if (PendingTargetLabel && PendingTargetLabel != &IsTarget->second) { - jmp(*PendingTargetLabel, T_NEAR); - } - PendingTargetLabel = nullptr; - - L(IsTarget->second); - } - - for (auto [CodeNode, IROp] : IR->GetCode(BlockNode)) { - #ifdef DEBUG_RA - if (IROp->Op != IR::OP_BEGINBLOCK && - IROp->Op != IR::OP_CONDJUMP && - IROp->Op != IR::OP_JUMP) { - fextl::stringstream Inst; - auto Name = FEXCore::IR::GetName(IROp->Op); - - if (IROp->HasDest) { - uint64_t PhysReg = RAPass->GetNodeRegister(Node); - if (PhysReg >= GPRPairBase) - Inst << "\tPair" << GetPhys(Node) << " = " << Name << " "; - else if (PhysReg >= XMMBase) - Inst << "\tXMM" << GetPhys(Node) << " = " << Name << " "; - else - Inst << "\tReg" << GetPhys(Node) << " = " << Name << " "; - } - else { - Inst << "\t" << Name << " "; - } - - const uint8_t NumArgs = IR::GetArgs(IROp->Op); - for (uint8_t i = 0; i < NumArgs; ++i) { - const auto ArgNode = IROp->Args[i].ID(); - const uint64_t PhysReg = RAPass->GetNodeRegister(ArgNode); - if (PhysReg >= GPRPairBase) - Inst << "Pair" << GetPhys(ArgNode) << (i + 1 == NumArgs ? "" : ", "); - else if (PhysReg >= XMMBase) - Inst << "XMM" << GetPhys(ArgNode) << (i + 1 == NumArgs ? "" : ", "); - else - Inst << "Reg" << GetPhys(ArgNode) << (i + 1 == NumArgs ? "" : ", "); - } - - LogMan::Msg::DFmt("{}", Inst.str()); - } - #endif - const auto ID = IR->GetID(CodeNode); - - // Execute handler - OpHandler Handler = OpHandlers[IROp->Op]; - (this->*Handler)(IROp, ID); - } - - if (DebugData) { - DebugData->Subblocks.push_back({ - static_cast(BlockStartHostCode - CodeData.BlockBegin), - static_cast(getCurr() - BlockStartHostCode) - }); - } - } - - // Make sure last branch is generated. It certainly can't be eliminated here. - if (PendingTargetLabel) - { - jmp(*PendingTargetLabel, T_NEAR); - } - PendingTargetLabel = nullptr; - - // Add the JitCodeTail - auto JITBlockTailLocation = getCurr(); - auto JITBlockTail = getCurr(); - setSize(getSize() + sizeof(JITCodeTail)); - - auto JITRIPEntriesLocation = getCurr(); - auto JITRIPEntries = getCurr(); - - setSize(getSize() + sizeof(JITRIPReconstructEntries) * DebugData->GuestOpcodes.size()); - - // Put the block's RIP entry in the tail. - // This will be used for RIP reconstruction in the future. - // TODO: This needs to be a data RIP relocation once code caching works. - // Current relocation code doesn't support this feature yet. - JITBlockTail->RIP = Entry; - - { - // Store the RIP entries. - JITBlockTail->NumberOfRIPEntries = DebugData->GuestOpcodes.size(); - JITBlockTail->OffsetToRIPEntries = JITRIPEntriesLocation - JITBlockTailLocation; - uintptr_t CurrentRIPOffset = 0; - uint64_t CurrentPCOffset = 0; - for (size_t i = 0; i < DebugData->GuestOpcodes.size(); i++) { - const auto &GuestOpcode = DebugData->GuestOpcodes[i]; - auto &RIPEntry = JITRIPEntries[i]; - RIPEntry.HostPCOffset = GuestOpcode.HostEntryOffset - CurrentPCOffset; - RIPEntry.GuestRIPOffset = GuestOpcode.GuestEntryOffset - CurrentRIPOffset; - CurrentPCOffset = GuestOpcode.HostEntryOffset; - CurrentRIPOffset = GuestOpcode.GuestEntryOffset; - } - } - - CodeHeader->OffsetToBlockTail = JITBlockTailLocation - CodeData.BlockBegin; - - CodeData.Size = getCurr() - CodeData.BlockBegin; - - JITBlockTail->Size = CodeData.Size; - - this->IR = nullptr; - - ready(); - - if (DebugData) { - DebugData->HostCodeSize = CodeData.Size; - DebugData->Relocations = &Relocations; - } - - return CodeData; -} - -fextl::unique_ptr CreateX86JITCore(FEXCore::Context::ContextImpl *ctx, FEXCore::Core::InternalThreadState *Thread) { - return fextl::make_unique(ctx, Thread); -} - -CPUBackendFeatures GetX86JITBackendFeatures() { - return CPUBackendFeatures { }; -} - -} diff --git a/FEXCore/Source/Interface/Core/JIT/x86_64/JITClass.h b/FEXCore/Source/Interface/Core/JIT/x86_64/JITClass.h deleted file mode 100644 index 429537d87..000000000 --- a/FEXCore/Source/Interface/Core/JIT/x86_64/JITClass.h +++ /dev/null @@ -1,492 +0,0 @@ -// SPDX-License-Identifier: MIT -/* -$info$ -tags: backend|x86-64 -$end_info$ -*/ - -#pragma once - -#include -#include "Interface/Core/BlockSamplingData.h" -#include "Interface/Core/Dispatcher/Dispatcher.h" -#include "Interface/Core/Dispatcher/X86Dispatcher.h" -#include "Interface/Core/ObjectCache/Relocations.h" - -using namespace Xbyak; - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "Interface/IR/Passes/RegisterAllocationPass.h" - -#include - -namespace FEXCore::CPU { - -// Temp registers -// rax, rcx, rdx, rsi, r8, r9, -// r10, r11 -// -// Callee Saved -// rbx, rbp, r12, r13, r14, r15 -// -// 1St Argument: rdi -// XMM: -// All temp -#define STATE r14 -#define TMP1 rax -#define TMP2 rcx -#define TMP3 rdx -#define TMP4 rdi -#define TMP5 rbx -using namespace Xbyak::util; -const std::array RA64 = { rsi, r8, r9, r10, r11, rbp, r12, r13, r15 }; -const std::array, 4> RA64Pair = {{ {rsi, r8}, {r9, r10}, {r11, rbp}, {r12, r13} }}; -const std::array RAXMM = { xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11}; -const std::array RAXMM_x = { xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11}; - -class X86JITCore final : public CPUBackend, public Xbyak::CodeGenerator { -public: - explicit X86JITCore(FEXCore::Context::ContextImpl *ctx, - FEXCore::Core::InternalThreadState *Thread); - ~X86JITCore() override; - - [[nodiscard]] fextl::string GetName() override { return "JIT"; } - - [[nodiscard]] CPUBackend::CompiledCode CompileCode(uint64_t Entry, - FEXCore::IR::IRListView const *IR, - FEXCore::Core::DebugData *DebugData, - FEXCore::IR::RegisterAllocationData *RAData, bool GDBEnabled) override; - - [[nodiscard]] void *MapRegion(void* HostPtr, uint64_t, uint64_t) override { return HostPtr; } - - [[nodiscard]] bool NeedsOpDispatch() override { return true; } - - void ClearCache() override; - - void ClearRelocations() override { Relocations.clear(); } - -private: - - /** - * @name Relocations - * @{ */ - uint64_t GetNamedSymbolLiteral(FEXCore::CPU::RelocNamedSymbolLiteral::NamedSymbol Op); - void LoadConstantWithPadding(Xbyak::Reg Reg, uint64_t Constant); - - /** - * @brief A literal pair relocation object for named symbol literals - */ - struct NamedSymbolLiteralPair { - Label Offset; - Relocation MoveABI{}; - }; - - /** - * @brief Inserts a thunk relocation - * - * @param Reg - The GPR to move the thunk handler in to - * @param Sum - The hash of the thunk - */ - void InsertNamedThunkRelocation(Xbyak::Reg Reg, const IR::SHA256Sum &Sum); - - /** - * @brief Inserts a guest GPR move relocation - * - * @param Reg - The GPR to move the guest RIP in to - * @param Constant - The guest RIP that will be relocated - */ - void InsertGuestRIPMove(Xbyak::Reg Reg, uint64_t Constant); - - /** - * @brief Inserts a named symbol as a literal in memory - * - * Need to use `PlaceNamedSymbolLiteral` with the return value to place the literal in the desired location - * - * @param Op The named symbol to place - * - * @return A temporary `NamedSymbolLiteralPair` - */ - NamedSymbolLiteralPair InsertNamedSymbolLiteral(FEXCore::CPU::RelocNamedSymbolLiteral::NamedSymbol Op); - - /** - * @brief Place the named symbol literal relocation in memory - * - * @param Lit - Which literal to place - */ - - void PlaceNamedSymbolLiteral(NamedSymbolLiteralPair &Lit); - - fextl::vector Relocations; - - ///< Relocation code loading - bool ApplyRelocations(uint64_t GuestEntry, uint64_t CodeEntry, uint64_t CursorEntry, size_t NumRelocations, const char* EntryRelocations); - - /** - * @brief Current guest RIP entrypoint - */ - uint64_t CursorEntry{}; - /** @} */ - - Label* PendingTargetLabel{}; - FEXCore::Context::ContextImpl *CTX; - FEXCore::IR::IRListView const *IR; - uint64_t Entry; - CPUBackend::CompiledCode CodeData{}; - - fextl::unordered_map JumpTargets; - Xbyak::util::Cpu Features{}; - - bool MemoryDebug = false; - - /** - * @name Register Allocation - * @{ */ - constexpr static uint32_t NumGPRs = RA64.size(); // 4 is the minimum required for GPR ops - constexpr static uint32_t NumXMMs = RAXMM.size(); - constexpr static uint32_t NumGPRPairs = RA64Pair.size(); - constexpr static uint32_t RegisterClasses = 6; - - constexpr static uint64_t GPRBase = (0ULL << 32); - constexpr static uint64_t XMMBase = (1ULL << 32); - constexpr static uint64_t GPRPairBase = (2ULL << 32); - - /** @} */ - - constexpr static uint8_t RA_8 = 0; - constexpr static uint8_t RA_16 = 1; - constexpr static uint8_t RA_32 = 2; - constexpr static uint8_t RA_64 = 3; - constexpr static uint8_t RA_XMM = 4; - - [[nodiscard]] IR::PhysicalRegister GetPhys(IR::NodeID Node) const; - - [[nodiscard]] bool IsFPR(IR::NodeID Node) const; - [[nodiscard]] bool IsGPR(IR::NodeID Node) const; - [[nodiscard]] bool IsGPRPair(IR::NodeID Node) const; - - template - [[nodiscard]] Xbyak::Reg GetSrc(IR::NodeID Node) const; - template - [[nodiscard]] std::pair GetSrcPair(IR::NodeID Node) const; - - template - [[nodiscard]] Xbyak::Reg GetDst(IR::NodeID Node) const; - - [[nodiscard]] Xbyak::Xmm GetSrc(IR::NodeID Node) const; - [[nodiscard]] Xbyak::Xmm GetDst(IR::NodeID Node) const; - - [[nodiscard]] static Xbyak::Ymm ToYMM(const Xbyak::Xmm& xmm) { - return Xbyak::Ymm{xmm.getIdx()}; - } - - [[nodiscard]] Xbyak::RegExp GenerateModRM(Xbyak::Reg Base, IR::OrderedNodeWrapper Offset, - IR::MemOffsetType OffsetType, uint8_t OffsetScale) const; - - [[nodiscard]] bool IsInlineConstant(const IR::OrderedNodeWrapper& Node, uint64_t* Value = nullptr) const; - [[nodiscard]] bool IsInlineEntrypointOffset(const IR::OrderedNodeWrapper& WNode, uint64_t* Value) const; - - IR::RegisterAllocationPass *RAPass; - FEXCore::IR::RegisterAllocationData *RAData; - FEXCore::Core::DebugData *DebugData; - -#ifdef BLOCKSTATS - bool GetSamplingData {true}; -#endif - - static uint64_t ExitFunctionLink(FEXCore::Core::CpuStateFrame *Frame, uint64_t *record); - - // This is purely a debugging aid for developers to see if they are in JIT code space when inspecting raw memory - void EmitDetectionString(); - - uint32_t SpillSlots{}; - - using SetCC = void (X86JITCore::*)(const Operand& op); - using CMovCC = void (X86JITCore::*)(const Reg& reg, const Operand& op); - using JCC = void (X86JITCore::*)(const Label& label, LabelType type); - - std::tuple GetCC(IR::CondClassType cond); - - using OpHandler = void (X86JITCore::*)(IR::IROp_Header *IROp, IR::NodeID Node); - std::array OpHandlers {}; - void RegisterALUHandlers(); - void RegisterAtomicHandlers(); - void RegisterBranchHandlers(); - void RegisterConversionHandlers(); - void RegisterFlagHandlers(); - void RegisterMemoryHandlers(); - void RegisterMiscHandlers(); - void RegisterMoveHandlers(); - void RegisterVectorHandlers(); - void RegisterEncryptionHandlers(); - - void PushRegs(); - void PopRegs(); -#define DEF_OP(x) void Op_##x(IR::IROp_Header *IROp, IR::NodeID Node) - - ///< Unhandled handler - DEF_OP(Unhandled); - - ///< No-op Handler - DEF_OP(NoOp); - - ///< ALU Ops - DEF_OP(TruncElementPair); - DEF_OP(Constant); - DEF_OP(EntrypointOffset); - DEF_OP(InlineConstant); - DEF_OP(InlineEntrypointOffset); - DEF_OP(CycleCounter); - DEF_OP(Add); - DEF_OP(AddNZCV); - DEF_OP(TestNZ); - DEF_OP(Sub); - DEF_OP(SubNZCV); - DEF_OP(Neg); - DEF_OP(Abs); - DEF_OP(Mul); - DEF_OP(UMul); - DEF_OP(Div); - DEF_OP(UDiv); - DEF_OP(Rem); - DEF_OP(URem); - DEF_OP(MulH); - DEF_OP(UMulH); - DEF_OP(Or); - DEF_OP(Orlshl); - DEF_OP(Orlshr); - DEF_OP(And); - DEF_OP(Andn); - DEF_OP(Xor); - DEF_OP(Lshl); - DEF_OP(Lshr); - DEF_OP(Ashr); - DEF_OP(Rol); - DEF_OP(Ror); - DEF_OP(Extr); - DEF_OP(PDep); - DEF_OP(PExt); - DEF_OP(LDiv); - DEF_OP(LUDiv); - DEF_OP(LRem); - DEF_OP(LURem); - DEF_OP(Zext); - DEF_OP(Not); - DEF_OP(Popcount); - DEF_OP(FindLSB); - DEF_OP(FindMSB); - DEF_OP(FindTrailingZeroes); - DEF_OP(CountLeadingZeroes); - DEF_OP(Rev); - DEF_OP(Bfi); - DEF_OP(Bfxil); - DEF_OP(Bfe); - DEF_OP(Sbfe); - DEF_OP(Select); - DEF_OP(VExtractToGPR); - DEF_OP(Float_ToGPR_ZS); - DEF_OP(Float_ToGPR_S); - DEF_OP(FCmp); - DEF_OP(F80Cmp); - - ///< Atomic ops - DEF_OP(CASPair); - DEF_OP(CAS); - DEF_OP(AtomicAdd); - DEF_OP(AtomicSub); - DEF_OP(AtomicAnd); - DEF_OP(AtomicOr); - DEF_OP(AtomicXor); - DEF_OP(AtomicSwap); - DEF_OP(AtomicFetchAdd); - DEF_OP(AtomicFetchSub); - DEF_OP(AtomicFetchAnd); - DEF_OP(AtomicFetchCLR); - DEF_OP(AtomicFetchOr); - DEF_OP(AtomicFetchXor); - DEF_OP(AtomicFetchNeg); - DEF_OP(TelemetrySetValue); - - ///< Branch ops - DEF_OP(CallbackReturn); - DEF_OP(ExitFunction); - DEF_OP(Jump); - DEF_OP(CondJump); - DEF_OP(Syscall); - DEF_OP(Thunk); - DEF_OP(ValidateCode); - DEF_OP(ThreadRemoveCodeEntry); - DEF_OP(CPUID); - DEF_OP(XGETBV); - - ///< Conversion ops - DEF_OP(VInsGPR); - DEF_OP(VCastFromGPR); - DEF_OP(VDupFromGPR); - DEF_OP(Float_FromGPR_S); - DEF_OP(Float_FToF); - DEF_OP(Vector_UToF); - DEF_OP(Vector_SToF); - DEF_OP(Vector_FToZS); - DEF_OP(Vector_FToS); - DEF_OP(Vector_FToF); - DEF_OP(Vector_FToI); - - ///< Flag ops - DEF_OP(GetHostFlag); - - ///< Memory ops - DEF_OP(LoadContext); - DEF_OP(StoreContext); - DEF_OP(LoadRegister); - DEF_OP(StoreRegister); - DEF_OP(LoadContextIndexed); - DEF_OP(StoreContextIndexed); - DEF_OP(SpillRegister); - DEF_OP(FillRegister); - DEF_OP(LoadFlag); - DEF_OP(StoreFlag); - DEF_OP(LoadMem); - DEF_OP(StoreMem); - DEF_OP(VLoadVectorMasked); - DEF_OP(VStoreVectorMasked); - DEF_OP(VLoadVectorElement); - DEF_OP(VStoreVectorElement); - DEF_OP(VBroadcastFromMem); - DEF_OP(Push); - DEF_OP(MemSet); - DEF_OP(MemCpy); - DEF_OP(CacheLineClear); - DEF_OP(CacheLineClean); - DEF_OP(CacheLineZero); - - ///< Misc ops - DEF_OP(GuestOpcode); - DEF_OP(Fence); - DEF_OP(Break); - DEF_OP(Print); - DEF_OP(GetRoundingMode); - DEF_OP(SetRoundingMode); - DEF_OP(ProcessorID); - DEF_OP(RDRAND); - DEF_OP(Yield); - - ///< Move ops - DEF_OP(ExtractElementPair); - DEF_OP(CreateElementPair); - - ///< Vector ops - DEF_OP(VectorZero); - DEF_OP(VectorImm); - DEF_OP(LoadNamedVectorConstant); - DEF_OP(LoadNamedVectorIndexedConstant); - DEF_OP(VMov); - DEF_OP(VAnd); - DEF_OP(VBic); - DEF_OP(VOr); - DEF_OP(VXor); - DEF_OP(VAdd); - DEF_OP(VSub); - DEF_OP(VUQAdd); - DEF_OP(VUQSub); - DEF_OP(VSQAdd); - DEF_OP(VSQSub); - DEF_OP(VAddP); - DEF_OP(VAddV); - DEF_OP(VUMinV); - DEF_OP(VURAvg); - DEF_OP(VAbs); - DEF_OP(VPopcount); - DEF_OP(VFAdd); - DEF_OP(VFAddP); - DEF_OP(VFSub); - DEF_OP(VFMul); - DEF_OP(VFDiv); - DEF_OP(VFMin); - DEF_OP(VFMax); - DEF_OP(VFRecp); - DEF_OP(VFSqrt); - DEF_OP(VFRSqrt); - DEF_OP(VNeg); - DEF_OP(VFNeg); - DEF_OP(VNot); - DEF_OP(VUMin); - DEF_OP(VSMin); - DEF_OP(VUMax); - DEF_OP(VSMax); - DEF_OP(VZip); - DEF_OP(VZip2); - DEF_OP(VUnZip); - DEF_OP(VUnZip2); - DEF_OP(VTrn); - DEF_OP(VTrn2); - DEF_OP(VBSL); - DEF_OP(VCMPEQ); - DEF_OP(VCMPEQZ); - DEF_OP(VCMPGT); - DEF_OP(VCMPGTZ); - DEF_OP(VCMPLTZ); - DEF_OP(VFCMPEQ); - DEF_OP(VFCMPNEQ); - DEF_OP(VFCMPLT); - DEF_OP(VFCMPGT); - DEF_OP(VFCMPLE); - DEF_OP(VFCMPORD); - DEF_OP(VFCMPUNO); - DEF_OP(VUShl); - DEF_OP(VUShr); - DEF_OP(VSShr); - DEF_OP(VUShlS); - DEF_OP(VUShrS); - DEF_OP(VSShrS); - DEF_OP(VInsElement); - DEF_OP(VDupElement); - DEF_OP(VExtr); - DEF_OP(VUShrI); - DEF_OP(VSShrI); - DEF_OP(VShlI); - DEF_OP(VUShrNI); - DEF_OP(VUShrNI2); - DEF_OP(VSXTL); - DEF_OP(VSXTL2); - DEF_OP(VUXTL); - DEF_OP(VUXTL2); - DEF_OP(VSQXTN); - DEF_OP(VSQXTN2); - DEF_OP(VSQXTNPair); - DEF_OP(VSQXTUN); - DEF_OP(VSQXTUN2); - DEF_OP(VSQXTUNPair); - DEF_OP(VMul); - DEF_OP(VUMull); - DEF_OP(VSMull); - DEF_OP(VUMull2); - DEF_OP(VSMull2); - DEF_OP(VUMulH); - DEF_OP(VSMulH); - DEF_OP(VUABDL); - DEF_OP(VUABDL2); - DEF_OP(VTBL1); - DEF_OP(VRev32); - DEF_OP(VRev64); - - ///< Encryption ops - DEF_OP(AESImc); - DEF_OP(AESEnc); - DEF_OP(AESEncLast); - DEF_OP(AESDec); - DEF_OP(AESDecLast); - DEF_OP(AESKeyGenAssist); - DEF_OP(CRC32); - DEF_OP(PCLMUL); -#undef DEF_OP -}; - -} diff --git a/FEXCore/Source/Interface/Core/JIT/x86_64/MemoryOps.cpp b/FEXCore/Source/Interface/Core/JIT/x86_64/MemoryOps.cpp deleted file mode 100644 index 1d774e1bb..000000000 --- a/FEXCore/Source/Interface/Core/JIT/x86_64/MemoryOps.cpp +++ /dev/null @@ -1,1301 +0,0 @@ -// SPDX-License-Identifier: MIT -/* -$info$ -tags: backend|x86-64 -$end_info$ -*/ - -#include "Interface/Core/CPUID.h" -#include "Interface/Core/JIT/x86_64/JITClass.h" -#include "Interface/Core/Dispatcher/X86Dispatcher.h" -#include -#include -#include - -#include -#include -#include - -namespace FEXCore::CPU { - -#define DEF_OP(x) void X86JITCore::Op_##x(IR::IROp_Header *IROp, IR::NodeID Node) - -DEF_OP(LoadContext) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - if (Op->Class == IR::GPRClass) { - switch (OpSize) { - case 1: { - movzx(GetDst(Node), byte [STATE + Op->Offset]); - break; - } - case 2: { - movzx(GetDst(Node), word [STATE + Op->Offset]); - break; - } - case 4: { - mov(GetDst(Node), dword [STATE + Op->Offset]); - break; - } - case 8: { - mov(GetDst(Node), qword [STATE + Op->Offset]); - break; - } - case 16: { - LOGMAN_MSG_A_FMT("Invalid GPR load of size 16"); - break; - } - default: - LOGMAN_MSG_A_FMT("Unhandled LoadContext size: {}", OpSize); - break; - } - } - else { - const auto Dst = GetDst(Node); - - switch (OpSize) { - case 1: { - movzx(rax, byte [STATE + Op->Offset]); - vmovq(Dst, rax); - break; - } - case 2: { - movzx(rax, word [STATE + Op->Offset]); - vmovq(Dst, rax); - break; - } - case 4: { - vmovd(Dst, dword [STATE + Op->Offset]); - break; - } - case 8: { - vmovq(Dst, qword [STATE + Op->Offset]); - break; - } - case 16: { - if (Op->Offset % 16 == 0) { - vmovaps(Dst, xword [STATE + Op->Offset]); - } else { - vmovups(Dst, xword [STATE + Op->Offset]); - } - break; - } - case 32: { - vmovups(ToYMM(Dst), yword [STATE + Op->Offset]); - break; - } - default: - LOGMAN_MSG_A_FMT("Unhandled LoadContext size: {}", OpSize); - break; - } - } -} - -DEF_OP(StoreContext) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - if (Op->Class == IR::GPRClass) { - switch (OpSize) { - case 1: { - mov(byte [STATE + Op->Offset], GetSrc(Op->Value.ID())); - break; - } - case 2: { - mov(word [STATE + Op->Offset], GetSrc(Op->Value.ID())); - break; - } - case 4: { - mov(dword [STATE + Op->Offset], GetSrc(Op->Value.ID())); - break; - } - case 8: { - mov(qword [STATE + Op->Offset], GetSrc(Op->Value.ID())); - break; - } - case 16: { - LOGMAN_MSG_A_FMT("Invalid store size of 16"); - break; - } - default: - LOGMAN_MSG_A_FMT("Unhandled StoreContext size: {}", OpSize); - break; - } - } - else { - const auto Value = GetSrc(Op->Value.ID()); - - switch (OpSize) { - case 1: { - pextrb(byte [STATE + Op->Offset], Value, 0); - break; - } - case 2: { - pextrw(word [STATE + Op->Offset], Value, 0); - break; - } - case 4: { - vmovd(dword [STATE + Op->Offset], Value); - break; - } - case 8: { - vmovq(qword [STATE + Op->Offset], Value); - break; - } - case 16: { - if (Op->Offset % 16 == 0) { - vmovaps(xword [STATE + Op->Offset], Value); - } else { - vmovups(xword [STATE + Op->Offset], Value); - } - break; - } - case 32: { - vmovups(yword [STATE + Op->Offset], ToYMM(Value)); - break; - } - default: - LOGMAN_MSG_A_FMT("Unhandled StoreContext size: {}", OpSize); - break; - } - } -} - -DEF_OP(LoadRegister) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - if (Op->Class == IR::GPRClass) { - switch (OpSize) { - case 1: { - movzx(GetSrc(Node), byte [STATE + Op->Offset]); - break; - } - case 2: { - movzx(GetSrc(Node), word [STATE + Op->Offset]); - break; - } - case 4: { - mov(GetSrc(Node), dword [STATE + Op->Offset]); - break; - } - case 8: { - mov(GetSrc(Node), qword [STATE + Op->Offset]); - break; - } - default: - LOGMAN_MSG_A_FMT("Unhandled LoadRegister size: {}", OpSize); - break; - } - } - else { - const auto Dst = GetSrc(Node); - - switch (OpSize) { - case 1: { - movzx(rax, byte [STATE + Op->Offset]); - vmovq(Dst, rax); - break; - } - case 2: { - movzx(rax, word [STATE + Op->Offset]); - vmovq(Dst, rax); - break; - } - case 4: { - vmovd(Dst, dword [STATE + Op->Offset]); - break; - } - case 8: { - vmovq(Dst, qword [STATE + Op->Offset]); - break; - } - case 16: { - if (Op->Offset % 16 == 0) { - vmovaps(Dst, xword [STATE + Op->Offset]); - } else { - vmovups(Dst, xword [STATE + Op->Offset]); - } - break; - } - case 32: { - vmovups(ToYMM(Dst), yword [STATE + Op->Offset]); - break; - } - default: - LOGMAN_MSG_A_FMT("Unhandled LoadRegister size: {}", OpSize); - break; - } - } -} - -DEF_OP(StoreRegister) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - if (Op->Class == IR::GPRClass) { - const auto regOffs = Op->Offset & 7; - - switch (OpSize) { - case 4: - LOGMAN_THROW_AA_FMT(regOffs == 0, "unexpected regOffs"); - mov(dword [STATE + Op->Offset], GetSrc(Op->Value.ID())); - break; - - case 8: - LOGMAN_THROW_AA_FMT(regOffs == 0, "unexpected regOffs"); - mov(qword [STATE + Op->Offset], GetSrc(Op->Value.ID())); - break; - - default: - LOGMAN_MSG_A_FMT("Unhandled StoreRegister GPR size: {}", OpSize); - break; - } - } else if (Op->Class == IR::FPRClass) { - const auto Value = GetSrc(Op->Value.ID()); - switch (OpSize) { - case 16: { - if (Op->Offset % 16 == 0) { - vmovaps(xword [STATE + Op->Offset], Value); - } else { - vmovups(xword [STATE + Op->Offset], Value); - } - break; - } - case 32: { - vmovups(yword [STATE + Op->Offset], ToYMM(Value)); - break; - } - default: - LOGMAN_MSG_A_FMT("Unhandled StoreContext size: {}", OpSize); - break; - } - } else { - LOGMAN_THROW_AA_FMT(false, "Unhandled Op->Class {}", Op->Class); - } -} - -DEF_OP(LoadContextIndexed) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const Reg Index = GetSrc(Op->Index.ID()); - - if (Op->Class == IR::GPRClass) { - switch (Op->Stride) { - case 1: - case 2: - case 4: - case 8: { - lea(rax, dword [STATE + Op->BaseOffset]); - switch (OpSize) { - case 1: - movzx(GetDst(Node), byte [rax + Index * Op->Stride]); - break; - case 2: - movzx(GetDst(Node), word [rax + Index * Op->Stride]); - break; - case 4: - mov(GetDst(Node), dword [rax + Index * Op->Stride]); - break; - case 8: - mov(GetDst(Node), qword [rax + Index * Op->Stride]); - break; - default: - LOGMAN_MSG_A_FMT("Unhandled LoadContextIndexed size: {}", OpSize); - break; - } - break; - } - case 16: - LOGMAN_MSG_A_FMT("Invalid Class load of size 16"); - break; - default: - LOGMAN_MSG_A_FMT("Unhandled LoadContextIndexed stride: {}", Op->Stride); - break; - } - } - else { - switch (Op->Stride) { - case 1: - case 2: - case 4: - case 8: { - const auto Dst = GetDst(Node); - - lea(rax, dword [STATE + Op->BaseOffset]); - switch (OpSize) { - case 1: - movzx(eax, byte [rax + Index * Op->Stride]); - vmovd(Dst, eax); - break; - case 2: - movzx(eax, word [rax + Index * Op->Stride]); - vmovd(Dst, eax); - break; - case 4: - vmovd(Dst, dword [rax + Index * Op->Stride]); - break; - case 8: - vmovq(Dst, qword [rax + Index * Op->Stride]); - break; - default: - LOGMAN_MSG_A_FMT("Unhandled LoadContextIndexed size: {}", OpSize); - break; - } - break; - } - case 16: - case 32: { - const auto Dst = GetDst(Node); - const auto Shift = Op->Stride == 16 ? 4 : 5; - - mov(rax, Index); - shl(rax, Shift); - lea(rax, dword [rax + Op->BaseOffset]); - switch (OpSize) { - case 1: - pinsrb(Dst, byte [STATE + rax], 0); - break; - case 2: - pinsrw(Dst, word [STATE + rax], 0); - break; - case 4: - vmovd(Dst, dword [STATE + rax]); - break; - case 8: - vmovq(Dst, qword [STATE + rax]); - break; - case 16: - if (Op->BaseOffset % 16 == 0) { - vmovaps(Dst, xword [STATE + rax]); - } else { - vmovups(Dst, xword [STATE + rax]); - } - break; - case 32: - vmovups(ToYMM(Dst), yword [STATE + rax]); - break; - default: - LOGMAN_MSG_A_FMT("Unhandled LoadContextIndexed size: {}", OpSize); - break; - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unhandled LoadContextIndexed stride: {}", Op->Stride); - break; - } - } -} - -DEF_OP(StoreContextIndexed) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const Reg Index = GetSrc(Op->Index.ID()); - - if (Op->Class == IR::GPRClass) { - const auto Value = GetSrc(Op->Value.ID()); - lea(rax, dword [STATE + Op->BaseOffset]); - - switch (Op->Stride) { - case 1: - case 2: - case 4: - case 8: { - if (!(OpSize == 1 || OpSize == 2 || OpSize == 4 || OpSize == 8)) { - LOGMAN_MSG_A_FMT("Unhandled StoreContextIndexed size: {}", OpSize); - } - mov(AddressFrame(OpSize * 8) [rax + Index * Op->Stride], Value); - break; - } - default: - LOGMAN_MSG_A_FMT("Unhandled StoreContextIndexed stride: {}", Op->Stride); - break; - } - } - else { - const auto Value = GetSrc(Op->Value.ID()); - switch (Op->Stride) { - case 1: - case 2: - case 4: - case 8: { - lea(rax, dword [STATE + Op->BaseOffset]); - switch (OpSize) { - case 1: - pextrb(AddressFrame(OpSize * 8) [rax + Index * Op->Stride], Value, 0); - break; - case 2: - pextrw(AddressFrame(OpSize * 8) [rax + Index * Op->Stride], Value, 0); - break; - case 4: - vmovd(AddressFrame(OpSize * 8) [rax + Index * Op->Stride], Value); - break; - case 8: - vmovq(AddressFrame(OpSize * 8) [rax + Index * Op->Stride], Value); - break; - default: - LOGMAN_MSG_A_FMT("Unhandled StoreContextIndexed size: {}", OpSize); - break; - } - break; - } - case 16: - case 32: { - const auto Shift = Op->Stride == 16 ? 4 : 5; - - mov(rax, Index); - shl(rax, Shift); - lea(rax, dword [rax + Op->BaseOffset]); - switch (OpSize) { - case 1: - pextrb(AddressFrame(OpSize * 8) [STATE + rax], Value, 0); - break; - case 2: - pextrw(AddressFrame(OpSize * 8) [STATE + rax], Value, 0); - break; - case 4: - vmovd(AddressFrame(OpSize * 8) [STATE + rax], Value); - break; - case 8: - vmovq(AddressFrame(OpSize * 8) [STATE + rax], Value); - break; - case 16: - if (Op->BaseOffset % 16 == 0) { - vmovaps(xword [STATE + rax], Value); - } else { - vmovups(xword [STATE + rax], Value); - } - break; - case 32: - vmovups(yword [STATE + rax], ToYMM(Value)); - break; - default: - LOGMAN_MSG_A_FMT("Unhandled StoreContextIndexed size: {}", OpSize); - break; - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unhandled StoreContextIndexed stride: {}", Op->Stride); - break; - } - } -} - -DEF_OP(SpillRegister) { - const auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; - const uint32_t SlotOffset = Op->Slot * MaxSpillSlotSize; - - if (Op->Class == FEXCore::IR::GPRClass) { - switch (OpSize) { - case 1: { - mov(byte [rsp + SlotOffset], GetSrc(Op->Value.ID())); - break; - } - case 2: { - mov(word [rsp + SlotOffset], GetSrc(Op->Value.ID())); - break; - } - case 4: { - mov(dword [rsp + SlotOffset], GetSrc(Op->Value.ID())); - break; - } - case 8: { - mov(qword [rsp + SlotOffset], GetSrc(Op->Value.ID())); - break; - } - default: - LOGMAN_MSG_A_FMT("Unhandled SpillRegister size: {}", OpSize); - break; - } - } else if (Op->Class == FEXCore::IR::FPRClass) { - const auto Src = GetSrc(Op->Value.ID()); - - switch (OpSize) { - case 4: { - movss(dword [rsp + SlotOffset], Src); - break; - } - case 8: { - movsd(qword [rsp + SlotOffset], Src); - break; - } - case 16: { - movaps(xword [rsp + SlotOffset], Src); - break; - } - case 32: { - vmovups(yword [rsp + SlotOffset], ToYMM(Src)); - break; - } - default: - LOGMAN_MSG_A_FMT("Unhandled SpillRegister size: {}", OpSize); - break; - } - } else { - LOGMAN_MSG_A_FMT("Unhandled SpillRegister class: {}", Op->Class.Val); - } -} - -DEF_OP(FillRegister) { - const auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; - const uint32_t SlotOffset = Op->Slot * MaxSpillSlotSize; - - if (Op->Class == FEXCore::IR::GPRClass) { - switch (OpSize) { - case 1: { - movzx(GetDst(Node), byte [rsp + SlotOffset]); - break; - } - case 2: { - movzx(GetDst(Node), word [rsp + SlotOffset]); - break; - } - case 4: { - mov(GetDst(Node), dword [rsp + SlotOffset]); - break; - } - case 8: { - mov(GetDst(Node), qword [rsp + SlotOffset]); - break; - } - default: - LOGMAN_MSG_A_FMT("Unhandled FillRegister size: {}", OpSize); - break; - } - } else if (Op->Class == FEXCore::IR::FPRClass) { - const auto Dst = GetDst(Node); - - switch (OpSize) { - case 4: { - vmovss(Dst, dword [rsp + SlotOffset]); - break; - } - case 8: { - vmovsd(Dst, qword [rsp + SlotOffset]); - break; - } - case 16: { - vmovaps(Dst, xword [rsp + SlotOffset]); - break; - } - case 32: { - vmovups(ToYMM(Dst), yword [rsp + SlotOffset]); - break; - } - default: - LOGMAN_MSG_A_FMT("Unhandled FillRegister size: {}", OpSize); - break; - } - } else { - LOGMAN_MSG_A_FMT("Unhandled FillRegister class: {}", Op->Class.Val); - } -} - -DEF_OP(LoadFlag) { - auto Op = IROp->C(); - - auto Dst = GetDst(Node); - - if (Op->Flag == 24 /* NZCV */) - mov(Dst.cvt32(), dword [STATE + (offsetof(FEXCore::Core::CPUState, flags[0]) + Op->Flag)]); - else - movzx(Dst, byte [STATE + (offsetof(FEXCore::Core::CPUState, flags[0]) + Op->Flag)]); -} - -DEF_OP(StoreFlag) { - auto Op = IROp->C(); - - mov (rax, GetSrc(Op->Value.ID())); - - if (Op->Flag == 24 /* NZCV */) - mov(dword [STATE + (offsetof(FEXCore::Core::CPUState, flags[0]) + Op->Flag)], eax); - else - mov(byte [STATE + (offsetof(FEXCore::Core::CPUState, flags[0]) + Op->Flag)], al); -} - -Xbyak::RegExp X86JITCore::GenerateModRM(Xbyak::Reg Base, IR::OrderedNodeWrapper Offset, IR::MemOffsetType OffsetType, uint8_t OffsetScale) const { - if (Offset.IsInvalid()) { - return Base; - } else { - if (OffsetScale != 1 && OffsetScale != 2 && OffsetScale != 4 && OffsetScale != 8) { - LOGMAN_MSG_A_FMT("Unhandled GenerateModRM OffsetScale: {}", OffsetScale); - } - - if (OffsetType != IR::MEM_OFFSET_SXTX) { - LOGMAN_MSG_A_FMT("Unhandled GenerateModRM OffsetType: {}", OffsetType.Val); - } - - uint64_t Const; - if (IsInlineConstant(Offset, &Const)) { - return Base + Const; - } else { - auto MemOffset = GetSrc(Offset.ID()); - - return Base + MemOffset * OffsetScale; - } - } -} - -DEF_OP(LoadMem) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const Xbyak::Reg MemReg = GetSrc(Op->Addr.ID()); - const auto MemPtr = GenerateModRM(MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale); - - if (Op->Class == IR::GPRClass) { - const auto Dst = GetDst(Node); - - switch (OpSize) { - case 1: { - movzx(Dst, byte [MemPtr]); - break; - } - case 2: { - movzx(Dst, word [MemPtr]); - break; - } - case 4: { - mov(Dst.cvt32(), dword [MemPtr]); - break; - } - case 8: { - mov(Dst, qword [MemPtr]); - break; - } - default: - LOGMAN_MSG_A_FMT("Unhandled LoadMem size: {}", OpSize); - break; - } - } - else - { - const auto Dst = GetDst(Node); - - switch (OpSize) { - case 1: { - movzx(eax, byte [MemPtr]); - vmovd(Dst, eax); - break; - } - case 2: { - movzx(eax, word [MemPtr]); - vmovd(Dst, eax); - break; - } - case 4: { - vmovd(Dst, dword [MemPtr]); - break; - } - case 8: { - vmovq(Dst, qword [MemPtr]); - break; - } - case 16: { - vmovups(Dst, xword [MemPtr]); - if (MemoryDebug) { - movq(rcx, Dst); - } - break; - } - case 32: { - vmovups(ToYMM(Dst), yword [MemPtr]); - if (MemoryDebug) { - movq(rcx, Dst); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unhandled LoadMem size: {}", OpSize); - break; - } - } -} - -DEF_OP(StoreMem) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const Xbyak::Reg MemReg = GetSrc(Op->Addr.ID()); - const auto MemPtr = GenerateModRM(MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale); - - if (Op->Class == IR::GPRClass) { - switch (OpSize) { - case 1: - mov(byte [MemPtr], GetSrc(Op->Value.ID())); - break; - case 2: - mov(word [MemPtr], GetSrc(Op->Value.ID())); - break; - case 4: - mov(dword [MemPtr], GetSrc(Op->Value.ID())); - break; - case 8: - mov(qword [MemPtr], GetSrc(Op->Value.ID())); - break; - default: - LOGMAN_MSG_A_FMT("Unhandled StoreMem size: {}", OpSize); - break; - } - } - else { - const auto Value = GetSrc(Op->Value.ID()); - - switch (OpSize) { - case 1: - pextrb(byte [MemPtr], Value, 0); - break; - case 2: - pextrw(word [MemPtr], Value, 0); - break; - case 4: - vmovd(dword [MemPtr], Value); - break; - case 8: - vmovq(qword [MemPtr], Value); - break; - case 16: - vmovups(xword [MemPtr], Value); - break; - case 32: - vmovups(yword [MemPtr], ToYMM(Value)); - break; - default: - LOGMAN_MSG_A_FMT("Unhandled StoreMem size: {}", OpSize); - break; - } - } -} - -DEF_OP(VLoadVectorMasked) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - const auto ElementSize = IROp->ElementSize; - - const auto Dst = GetDst(Node); - const auto Mask = GetSrc(Op->Mask.ID()); - - const Xbyak::Reg MemReg = GetSrc(Op->Addr.ID()); - const auto MemPtr = GenerateModRM(MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale); - - switch (ElementSize) { - case 4: { - if (Is256Bit) { - vmaskmovps(ToYMM(Dst), ToYMM(Mask), yword [MemPtr]); - } else { - vmaskmovps(Dst, Mask, xword [MemPtr]); - } - return; - } - case 8: { - if (Is256Bit) { - vmaskmovpd(ToYMM(Dst), ToYMM(Mask), yword [MemPtr]); - } else { - vmaskmovpd(Dst, Mask, xword [MemPtr]); - } - return; - } - default: - LOGMAN_MSG_A_FMT("Unhandled VLoadVectorMasked element size: {}", ElementSize); - return; - } -} -DEF_OP(VStoreVectorMasked) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - const auto ElementSize = IROp->ElementSize; - - const auto Data = GetDst(Op->Data.ID()); - const auto Mask = GetSrc(Op->Mask.ID()); - - const Xbyak::Reg MemReg = GetSrc(Op->Addr.ID()); - const auto MemPtr = GenerateModRM(MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale); - - switch (ElementSize) { - case 4: { - if (Is256Bit) { - vmaskmovps(yword [MemPtr], ToYMM(Mask), ToYMM(Data)); - } else { - vmaskmovps(xword [MemPtr], Mask, Data); - } - return; - } - case 8: { - if (Is256Bit) { - vmaskmovpd(yword [MemPtr], ToYMM(Mask), ToYMM(Data)); - } else { - vmaskmovpd(xword [MemPtr], Mask, Data); - } - return; - } - default: - LOGMAN_MSG_A_FMT("Unhandled VStoreVectorMasked element size: {}", ElementSize); - return; - } -} -DEF_OP(VLoadVectorElement) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - const auto ElementSize = IROp->ElementSize; - - const auto Dst = GetDst(Node); - const auto DstSrc = GetSrc(Op->DstSrc.ID()); - const Xbyak::Reg MemReg = GetSrc(Op->Addr.ID()); - - if (Is256Bit) { - LOGMAN_MSG_A_FMT("Unsupported 256-bit VLoadVectorElement"); - } else { - switch (ElementSize) { - case 1: - vpinsrb(Dst, DstSrc, byte [MemReg], Op->Index); - break; - case 2: - vpinsrw(Dst, DstSrc, word [MemReg], Op->Index); - break; - case 4: - vpinsrd(Dst, DstSrc, dword [MemReg], Op->Index); - break; - case 8: - vpinsrq(Dst, DstSrc, qword [MemReg], Op->Index); - break; - case 16: - // Normal load - vmovups(Dst, xword [MemReg]); - break; - default: - LOGMAN_MSG_A_FMT("Unhandled VLoadVectorElement size: {}", ElementSize); - return; - } - } -} -DEF_OP(VStoreVectorElement) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - const auto ElementSize = IROp->ElementSize; - - const auto Value = GetSrc(Op->Value.ID()); - const Xbyak::Reg MemReg = GetSrc(Op->Addr.ID()); - - if (Is256Bit) { - LOGMAN_MSG_A_FMT("Unsupported 256-bit VStoreVectorElement"); - } else { - switch (ElementSize) { - case 1: - pextrb(byte [MemReg], Value, Op->Index); - break; - case 2: - pextrw(word [MemReg], Value, Op->Index); - break; - case 4: - pextrd(dword [MemReg], Value, Op->Index); - break; - case 8: - pextrq(qword [MemReg], Value, Op->Index); - break; - case 16: - // Normal store - vmovups(xword [MemReg], Value); - break; - default: - LOGMAN_MSG_A_FMT("Unhandled VStoreVectorElement size: {}", ElementSize); - return; - } - } -} - -DEF_OP(VBroadcastFromMem) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - const auto ElementSize = IROp->ElementSize; - - const auto Dst = GetDst(Node); - - const Xbyak::Reg MemReg = GetSrc(Op->Address.ID()); - - if (Is256Bit) { - const auto DstYMM = ToYMM(Dst); - - switch (ElementSize) { - case 1: - vpbroadcastb(DstYMM, byte [MemReg]); - break; - case 2: - vpbroadcastw(DstYMM, word [MemReg]); - break; - case 4: - vpbroadcastd(DstYMM, dword [MemReg]); - break; - case 8: - vpbroadcastq(DstYMM, qword [MemReg]); - break; - case 16: - vbroadcasti128(DstYMM, xword [MemReg]); - break; - default: - LOGMAN_MSG_A_FMT("Unhandled VBroadcastFromMem element size: {}", ElementSize); - return; - } - } else { - switch (ElementSize) { - case 1: - vpbroadcastb(Dst, byte [MemReg]); - break; - case 2: - vpbroadcastw(Dst, word [MemReg]); - break; - case 4: - vpbroadcastd(Dst, dword [MemReg]); - break; - case 8: - vpbroadcastq(Dst, qword [MemReg]); - break; - case 16: - vmovaps(Dst, xword [MemReg]); - break; - default: - LOGMAN_MSG_A_FMT("Unhandled VBroadcastFromMem element size: {}", ElementSize); - return; - } - } -} - -DEF_OP(Push) { - const auto Op = IROp->C(); - const auto ValueSize = Op->ValueSize; - - const Xbyak::Reg Value = GetSrc(Op->Value.ID()); - const Xbyak::Reg AddrSrc = GetSrc(Op->Addr.ID()); - const auto Dst = GetSrc(Node); - - switch (ValueSize) { - case 1: - mov(byte [AddrSrc - ValueSize], Value.cvt8()); - break; - case 2: - mov(word [AddrSrc - ValueSize], Value.cvt16()); - break; - case 4: - mov(dword [AddrSrc - ValueSize], Value.cvt32()); - break; - case 8: - mov(qword [AddrSrc - ValueSize], Value); - break; - default: - LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, ValueSize); - break; - } - - lea(Dst, qword [AddrSrc - ValueSize]); -} - -DEF_OP(MemSet) { - const auto Op = IROp->C(); - - const int32_t Size = Op->Size; - const auto MemReg = GetSrc(Op->Addr.ID()); - const auto Value = GetSrc(Op->Value.ID()); - const auto Length = GetSrc(Op->Length.ID()); - const auto Direction = GetSrc(Op->Direction.ID()); - const auto Dst = GetSrc(Node); - - // If Direction == 0 then: - // MemReg is incremented (by size) - // else: - // MemReg is decremented (by size) - // - // Counter is decremented regardless. - - // TMP1 = rax - // TMP2 = rcx - // TMP4 = rdi - // That leaves us with TMP3 and TMP5 - mov(rax, Value); - mov(rcx, Length); - mov(rdi, MemReg); - - if (!Op->Prefix.IsInvalid()) { - add(rdi, GetSrc(Op->Prefix.ID())); - } - - { - mov(TMP3, Length); - auto CalculateDest = [&]() { - mov(Dst, MemReg); - switch (Size) { - case 1: - break; - case 2: - shl(TMP3, 1); - break; - case 4: - shl(TMP3, 2); - break; - case 8: - shl(TMP3, 3); - break; - default: - LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, Size); - break; - } - }; - - Label AfterDir; - Label BackwardDir; - - cmp(Direction, 0); - jne(BackwardDir); - // Incrementing DF flag. - cld(); - CalculateDest(); - add(Dst, TMP3); - jmp(AfterDir); - - L(BackwardDir); - // Decrementing DF flag. - std(); - CalculateDest(); - sub(Dst, TMP3); - - L(AfterDir); - } - - switch (Size) { - case 1: - rep(); stosb(); - break; - case 2: - rep(); stosw(); - break; - case 4: - rep(); stosd(); - break; - case 8: - rep(); stosq(); - break; - default: - LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, Size); - break; - } - // Ensure we set DF back to zero. Required by the ABI. - cld(); -} - -DEF_OP(MemCpy) { - const auto Op = IROp->C(); - - const int32_t Size = Op->Size; - const auto MemRegDest = GetSrc(Op->AddrDest.ID()); - const auto MemRegSrc = GetSrc(Op->AddrSrc.ID()); - - const auto Length = GetSrc(Op->Length.ID()); - const auto Direction = GetSrc(Op->Direction.ID()); - - // If Direction == 0 then: - // MemRegDest is incremented (by size) - // MemRegSrc is incremented (by size) - // else: - // MemRegDest is decremented (by size) - // MemRegSrc is decremented (by size) - // - // Counter is decremented regardless. - - // TMP1 = Length - // TMP2 = Dest - // TMP3 = Src - // TMP4 = Temp value - mov(TMP1, Length); - mov(TMP2, MemRegDest); - mov(TMP3, MemRegSrc); - if (!Op->PrefixDest.IsInvalid()) { - add(TMP2, GetSrc(Op->PrefixDest.ID())); - } - if (!Op->PrefixSrc.IsInvalid()) { - add(TMP3, GetSrc(Op->PrefixSrc.ID())); - } - - auto Dst = GetSrcPair(Node); - Label Done; - Label BackwardImpl; - cmp(Direction, 0); - jne(BackwardImpl); - - // Emit forward direction memcpy then backward direction memcpy. - for (int32_t Direction : { 1, -1 }) { - Label DoneInternal; - Label AgainInternal; - - L(AgainInternal); - cmp(TMP1, 0); - je(DoneInternal); - - { - switch (Size) { - case 1: - movzx(TMP4, byte [TMP3]); - mov(byte [TMP2], TMP4.cvt8()); - break; - case 2: - movzx(TMP4, word [TMP3]); - mov(word [TMP2], TMP4.cvt16()); - break; - case 4: - mov(TMP4.cvt32(), dword [TMP3]); - mov(dword [TMP2], TMP4.cvt32()); - break; - case 8: - mov(TMP4, qword [TMP3]); - mov(qword [TMP2], TMP4); - break; - default: - LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, Size); - break; - } - } - - if (Direction == 1) { - // Incrementing pointers - add(TMP2, Size); - add(TMP3, Size); - } - else { - // Decrementing pointers - sub(TMP2, Size); - sub(TMP3, Size); - } - - // Decrement counter by one - sub(TMP1, 1); - - jmp(AgainInternal); - L(DoneInternal); - - // Pointer math using source pointers and length. - mov(TMP3, Length); - switch (Size) { - case 1: - break; - case 2: - shl(TMP3, 1); - break; - case 4: - shl(TMP3, 2); - break; - case 8: - shl(TMP3, 3); - break; - default: - LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, Size); - break; - } - - // Needs to use temporaries just in case of overwrite - mov(TMP1, MemRegDest); - mov(TMP2, MemRegSrc); - - mov(Dst.first, TMP1); - mov(Dst.second, TMP2); - - if (Direction == 1) { - // Incrementing pointers - add(Dst.first, TMP3); - add(Dst.second, TMP3); - - jmp(Done); - L(BackwardImpl); - } - else { - // Decrementing pointers - sub(Dst.first, TMP3); - sub(Dst.second, TMP3); - } - } - - L(Done); -} - -DEF_OP(CacheLineClear) { - auto Op = IROp->C(); - - Xbyak::Reg MemReg = GetSrc(Op->Addr.ID()); - - if (Op->Serialize) { - clflush(ptr [MemReg]); - } - else { - clflushopt(ptr [MemReg]); - } -} - -DEF_OP(CacheLineClean) { - auto Op = IROp->C(); - - Xbyak::Reg MemReg = GetSrc(Op->Addr.ID()); - clwb(ptr [MemReg]); -} - -DEF_OP(CacheLineZero) { - auto Op = IROp->C(); - - Xbyak::Reg MemReg = GetSrc(Op->Addr.ID()); - - // Align by cacheline - mov (TMP1, CPUIDEmu::CACHELINE_SIZE - 1); - andn(TMP1, TMP1, MemReg.cvt64()); - xor_(TMP2, TMP2); - - using DataType = uint64_t; - // 64-byte cache line zero - for (size_t i = 0; i < CPUIDEmu::CACHELINE_SIZE; i += sizeof(DataType)) { - mov (qword [TMP1 + i], TMP2); - } -} - -#undef DEF_OP -void X86JITCore::RegisterMemoryHandlers() { -#define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &X86JITCore::Op_##x - REGISTER_OP(LOADCONTEXT, LoadContext); - REGISTER_OP(STORECONTEXT, StoreContext); - REGISTER_OP(LOADREGISTER, LoadRegister); - REGISTER_OP(STOREREGISTER, StoreRegister); - REGISTER_OP(LOADCONTEXTINDEXED, LoadContextIndexed); - REGISTER_OP(STORECONTEXTINDEXED, StoreContextIndexed); - REGISTER_OP(SPILLREGISTER, SpillRegister); - REGISTER_OP(FILLREGISTER, FillRegister); - REGISTER_OP(LOADFLAG, LoadFlag); - REGISTER_OP(STOREFLAG, StoreFlag); - REGISTER_OP(LOADMEM, LoadMem); - REGISTER_OP(STOREMEM, StoreMem); - REGISTER_OP(LOADMEMTSO, LoadMem); - REGISTER_OP(STOREMEMTSO, StoreMem); - REGISTER_OP(VLOADVECTORMASKED, VLoadVectorMasked); - REGISTER_OP(VSTOREVECTORMASKED, VStoreVectorMasked); - REGISTER_OP(VLOADVECTORELEMENT, VLoadVectorElement); - REGISTER_OP(VSTOREVECTORELEMENT, VStoreVectorElement); - REGISTER_OP(VBROADCASTFROMMEM, VBroadcastFromMem); - REGISTER_OP(PUSH, Push); - REGISTER_OP(MEMSET, MemSet); - REGISTER_OP(MEMCPY, MemCpy); - REGISTER_OP(CACHELINECLEAR, CacheLineClear); - REGISTER_OP(CACHELINECLEAN, CacheLineClean); - REGISTER_OP(CACHELINEZERO, CacheLineZero); -#undef REGISTER_OP -} -} - diff --git a/FEXCore/Source/Interface/Core/JIT/x86_64/MiscOps.cpp b/FEXCore/Source/Interface/Core/JIT/x86_64/MiscOps.cpp deleted file mode 100644 index 71884a69c..000000000 --- a/FEXCore/Source/Interface/Core/JIT/x86_64/MiscOps.cpp +++ /dev/null @@ -1,190 +0,0 @@ -// SPDX-License-Identifier: MIT -/* -$info$ -tags: backend|x86-64 -$end_info$ -*/ - -#include "Interface/Context/Context.h" -#include "Interface/Core/Dispatcher/Dispatcher.h" -#include "Interface/Core/Dispatcher/X86Dispatcher.h" -#include "Interface/Core/JIT/x86_64/JITClass.h" -#include "FEXCore/Debug/InternalThreadState.h" - -#include -#include -#include - -#include -#include -#include - -namespace FEXCore::CPU { -#define DEF_OP(x) void X86JITCore::Op_##x(IR::IROp_Header *IROp, IR::NodeID Node) - -DEF_OP(GuestOpcode) { - auto Op = IROp->C(); - // metadata - DebugData->GuestOpcodes.push_back({Op->GuestEntryOffset, getCurr() - CodeData.BlockBegin}); -} - -DEF_OP(Fence) { - auto Op = IROp->C(); - switch (Op->Fence) { - case IR::Fence_Load.Val: - lfence(); - break; - case IR::Fence_LoadStore.Val: - mfence(); - break; - case IR::Fence_Store.Val: - sfence(); - break; - default: LOGMAN_MSG_A_FMT("Unknown Fence: {}", Op->Fence); break; - } -} - -#ifndef _WIN32 -DEF_OP(Break) { - auto Op = IROp->C(); - - if (SpillSlots) { - add(rsp, SpillSlots * MaxSpillSlotSize); - } - - Core::CpuStateFrame::SynchronousFaultDataStruct State = { - .FaultToTopAndGeneratedException = 1, - .Signal = Op->Reason.Signal, - .TrapNo = Op->Reason.TrapNumber, - .si_code = Op->Reason.si_code, - .err_code = Op->Reason.ErrorRegister, - }; - - uint64_t Constant{}; - memcpy(&Constant, &State, sizeof(State)); - - mov(TMP1, Constant); - mov(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, SynchronousFaultData)], TMP1); - - switch (Op->Reason.Signal) { - case SIGILL: - jmp(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.GuestSignal_SIGILL)]); - break; - case SIGTRAP: - jmp(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.GuestSignal_SIGTRAP)]); - break; - case SIGSEGV: - jmp(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.GuestSignal_SIGSEGV)]); - break; - default: - jmp(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.GuestSignal_SIGTRAP)]); - break; - } -} -#else -DEF_OP(Break) { - ERROR_AND_DIE_FMT("Unsupported"); -} -#endif - -DEF_OP(GetRoundingMode) { - auto Dst = GetDst(Node); - sub(rsp, 4); - // Only stores to memory - stmxcsr(dword [rsp]); - mov(Dst, dword [rsp]); - add(rsp, 4); - shr(Dst, 13); -} - -DEF_OP(SetRoundingMode) { - auto Op = IROp->C(); - auto Src = GetSrc(Op->RoundMode.ID()); - - // Load old mxcsr - // Only stores to memory - sub(rsp, 4); - stmxcsr(dword [rsp]); - mov(TMP1.cvt32(), dword [rsp]); - - // Insert the new rounding mode - and_(TMP1.cvt32(), ~(0b111 << 13)); - mov(TMP2.cvt32(), Src); - shl(TMP2.cvt32(), 13); - or_(TMP1.cvt32(), TMP2.cvt32()); - - // Store it to mxcsr - // Only loads from memory - mov(dword [rsp], TMP1.cvt32()); - ldmxcsr(dword [rsp]); - add(rsp, 4); -} - -DEF_OP(Print) { - auto Op = IROp->C(); - - PushRegs(); - if (IsGPR(Op->Value.ID())) { - mov (rdi, GetSrc(Op->Value.ID())); - call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.PrintValue)]); - } - else { - pextrq(rdi, GetSrc(Op->Value.ID()), 0); - pextrq(rsi, GetSrc(Op->Value.ID()), 1); - - call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.PrintVectorValue)]); - } - - PopRegs(); -} - -DEF_OP(ProcessorID) { - // Cyclecounter in EDX:EAX - // IA32_TSC_AUX in ECX - rdtscp(); - mov (GetDst(Node), ecx); -} - -DEF_OP(RDRAND) { - auto Op = IROp->C(); - - auto Dst = GetSrcPair(Node); - - if (Op->GetReseeded) { - rdrand(Dst.first); - } - else { - rdseed(Dst.first); - } - - // In the case of RDRAND or RDSEED returning a valid number then CF = 1, else 0 - mov (Dst.second, 0); - setc(Dst.second.cvt8()); -} - -DEF_OP(Yield) { - pause(); -} - -#undef DEF_OP -void X86JITCore::RegisterMiscHandlers() { -#define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &X86JITCore::Op_##x - REGISTER_OP(DUMMY, NoOp); - REGISTER_OP(IRHEADER, NoOp); - REGISTER_OP(CODEBLOCK, NoOp); - REGISTER_OP(BEGINBLOCK, NoOp); - REGISTER_OP(ENDBLOCK, NoOp); - REGISTER_OP(GUESTOPCODE, GuestOpcode); - REGISTER_OP(FENCE, Fence); - REGISTER_OP(BREAK, Break); - REGISTER_OP(PRINT, Print); - REGISTER_OP(GETROUNDINGMODE, GetRoundingMode); - REGISTER_OP(SETROUNDINGMODE, SetRoundingMode); - REGISTER_OP(INVALIDATEFLAGS, NoOp); - REGISTER_OP(PROCESSORID, ProcessorID); - REGISTER_OP(RDRAND, RDRAND); - REGISTER_OP(YIELD, Yield); -#undef REGISTER_OP -} -} - diff --git a/FEXCore/Source/Interface/Core/JIT/x86_64/MoveOps.cpp b/FEXCore/Source/Interface/Core/JIT/x86_64/MoveOps.cpp deleted file mode 100644 index dacee3473..000000000 --- a/FEXCore/Source/Interface/Core/JIT/x86_64/MoveOps.cpp +++ /dev/null @@ -1,85 +0,0 @@ -// SPDX-License-Identifier: MIT -/* -$info$ -tags: backend|x86-64 -$end_info$ -*/ - -#include "Interface/Core/JIT/x86_64/JITClass.h" -#include "Interface/Core/Dispatcher/X86Dispatcher.h" -#include -#include - -#include -#include -#include - -namespace FEXCore::CPU { - -#define DEF_OP(x) void X86JITCore::Op_##x(IR::IROp_Header *IROp, IR::NodeID Node) -DEF_OP(ExtractElementPair) { - auto Op = IROp->C(); - switch (Op->Header.Size) { - case 4: { - auto Src = GetSrcPair(Op->Pair.ID()); - std::array Regs = {Src.first, Src.second}; - mov (GetDst(Node), Regs[Op->Element]); - break; - } - case 8: { - auto Src = GetSrcPair(Op->Pair.ID()); - std::array Regs = {Src.first, Src.second}; - mov (GetDst(Node), Regs[Op->Element]); - break; - } - default: LOGMAN_MSG_A_FMT("Unknown Size"); break; - } -} - -DEF_OP(CreateElementPair) { - auto Op = IROp->C(); - std::pair Dst; - Xbyak::Reg RegFirst; - Xbyak::Reg RegSecond; - Xbyak::Reg RegTmp; - - switch (IROp->ElementSize) { - case 4: { - Dst = GetSrcPair(Node); - RegFirst = GetSrc(Op->Lower.ID()); - RegSecond = GetSrc(Op->Upper.ID()); - RegTmp = eax; - break; - } - case 8: { - Dst = GetSrcPair(Node); - RegFirst = GetSrc(Op->Lower.ID()); - RegSecond = GetSrc(Op->Upper.ID()); - RegTmp = rax; - break; - } - default: LOGMAN_MSG_A_FMT("Unknown Size"); break; - } - - if (Dst.first != RegSecond) { - mov(Dst.first, RegFirst); - mov(Dst.second, RegSecond); - } else if (Dst.second != RegFirst) { - mov(Dst.second, RegSecond); - mov(Dst.first, RegFirst); - } else { - mov(RegTmp, RegFirst); - mov(Dst.second, RegSecond); - mov(Dst.first, RegTmp); - } -} - -#undef DEF_OP -void X86JITCore::RegisterMoveHandlers() { -#define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &X86JITCore::Op_##x - REGISTER_OP(EXTRACTELEMENTPAIR, ExtractElementPair); - REGISTER_OP(CREATEELEMENTPAIR, CreateElementPair); -#undef REGISTER_OP -} -} - diff --git a/FEXCore/Source/Interface/Core/JIT/x86_64/VectorOps.cpp b/FEXCore/Source/Interface/Core/JIT/x86_64/VectorOps.cpp deleted file mode 100644 index 9f32d546e..000000000 --- a/FEXCore/Source/Interface/Core/JIT/x86_64/VectorOps.cpp +++ /dev/null @@ -1,5045 +0,0 @@ -// SPDX-License-Identifier: MIT -/* -$info$ -tags: backend|x86-64 -$end_info$ -*/ - -#include "Interface/Core/JIT/x86_64/JITClass.h" -#include "Interface/Core/Dispatcher/X86Dispatcher.h" -#include -#include - -#include -#include -#include - -namespace FEXCore::CPU { - -#define DEF_OP(x) void X86JITCore::Op_##x(IR::IROp_Header *IROp, IR::NodeID Node) -DEF_OP(VectorZero) { - auto Dst = GetDst(Node); - vpxor(Dst, Dst, Dst); -} - -DEF_OP(VectorImm) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto Is128Bit = OpSize == Core::CPUState::XMM_SSE_REG_SIZE; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const uint64_t Imm = Op->Immediate; - - const auto Dst = GetDst(Node); - - uint64_t Element{}; - switch (Op->Header.ElementSize) { - case 1: - Element = - (Imm << (0 * 8)) | - (Imm << (1 * 8)) | - (Imm << (2 * 8)) | - (Imm << (3 * 8)) | - (Imm << (4 * 8)) | - (Imm << (5 * 8)) | - (Imm << (6 * 8)) | - (Imm << (7 * 8)); - break; - case 2: - Element = - (Imm << (0 * 16)) | - (Imm << (1 * 16)) | - (Imm << (2 * 16)) | - (Imm << (3 * 16)); - break; - case 4: - Element = - (Imm << (0 * 32)) | - (Imm << (1 * 32)); - break; - case 8: - Element = Imm; - break; - } - - mov(TMP1, Element); - vmovq(Dst, TMP1); - - if (Is256Bit) { - vpbroadcastq(ToYMM(Dst), Dst); - } else if (Is128Bit) { - vpbroadcastq(Dst, Dst); - } -} - -DEF_OP(LoadNamedVectorConstant) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - const auto Dst = GetDst(Node); - - switch (Op->Constant) { - case FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_ZERO: - vpxor(Dst, Dst, Dst); - return; - default: - // Intentionally doing nothing. - break; - } - - mov(TMP1, qword STATE_PTR(CpuStateFrame, Pointers.Common.NamedVectorConstantPointers[Op->Constant])); - - switch (OpSize) { - case 1: - movzx(eax, byte [TMP1]); - vmovd(Dst, eax); - break; - case 2: - movzx(eax, word [TMP1]); - vmovd(Dst, eax); - break; - case 4: - vmovd(Dst, dword [TMP1]); - break; - case 8: - vmovq(Dst, dword [TMP1]); - break; - case 16: - vmovups(Dst, xword [TMP1]); - break; - case 32: - vmovups(ToYMM(Dst), yword [TMP1]); - break; - } -} - -DEF_OP(LoadNamedVectorIndexedConstant) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - mov(TMP1, qword STATE_PTR(CpuStateFrame, Pointers.Common.IndexedNamedVectorConstantPointers[Op->Constant])); - - const auto Dst = GetDst(Node); - - switch (OpSize) { - case 1: - movzx(eax, byte [TMP1 + Op->Index]); - vmovd(Dst, eax); - break; - case 2: - movzx(eax, word [TMP1 + Op->Index]); - vmovd(Dst, eax); - break; - case 4: - vmovd(Dst, dword [TMP1 + Op->Index]); - break; - case 8: - vmovq(Dst, dword [TMP1 + Op->Index]); - break; - case 16: - vmovups(Dst, xword [TMP1 + Op->Index]); - break; - case 32: - vmovups(ToYMM(Dst), yword [TMP1 + Op->Index]); - break; - } -} - -DEF_OP(VMov) { - auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; - - const auto Dst = GetDst(Node); - const auto Source = GetSrc(Op->Source.ID()); - - switch (OpSize) { - case 1: { - vpxor(xmm15, xmm15, xmm15); - pextrb(eax, Source, 0); - pinsrb(xmm15, eax, 0); - vmovapd(Dst, xmm15); - break; - } - case 2: { - vpxor(xmm15, xmm15, xmm15); - pextrw(eax, Source, 0); - pinsrw(xmm15, eax, 0); - vmovapd(Dst, xmm15); - break; - } - case 4: { - vpxor(xmm15, xmm15, xmm15); - pextrd(eax, Source, 0); - pinsrd(xmm15, eax, 0); - vmovapd(Dst, xmm15); - break; - } - case 8: { - vmovq(Dst, Source); - break; - } - case 16: { - vmovaps(Dst, Source); - break; - } - case 32: { - vmovaps(ToYMM(Dst), ToYMM(Source)); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Op Size: {}", OpSize); - break; - } -} - -DEF_OP(VAnd) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector1 = GetSrc(Op->Vector1.ID()); - const auto Vector2 = GetSrc(Op->Vector2.ID()); - - if (Is256Bit) { - vpand(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpand(Dst, Vector1, Vector2); - } -} - -DEF_OP(VBic) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector1 = GetSrc(Op->Vector1.ID()); - const auto Vector2 = GetSrc(Op->Vector2.ID()); - - // This doesn't map directly to ARM - if (Is256Bit) { - vpcmpeqd(ymm15, ymm15, ymm15); - vpxor(ymm15, ToYMM(Vector2), ymm15); - vpand(ToYMM(Dst), ToYMM(Vector1), ymm15); - } else { - vpcmpeqd(xmm15, xmm15, xmm15); - vpxor(xmm15, Vector2, xmm15); - vpand(Dst, Vector1, xmm15); - } -} - -DEF_OP(VOr) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector1 = GetSrc(Op->Vector1.ID()); - const auto Vector2 = GetSrc(Op->Vector2.ID()); - - if (Is256Bit) { - vpor(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpor(Dst, Vector1, Vector2); - } -} - -DEF_OP(VXor) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector1 = GetSrc(Op->Vector1.ID()); - const auto Vector2 = GetSrc(Op->Vector2.ID()); - - if (Is256Bit) { - vpxor(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpxor(Dst, Vector1, Vector2); - } -} - -DEF_OP(VAdd) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector1 = GetSrc(Op->Vector1.ID()); - const auto Vector2 = GetSrc(Op->Vector2.ID()); - - switch (ElementSize) { - case 1: { - if (Is256Bit) { - vpaddb(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpaddb(Dst, Vector1, Vector2); - } - break; - } - case 2: { - if (Is256Bit) { - vpaddw(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpaddw(Dst, Vector1, Vector2); - } - break; - } - case 4: { - if (Is256Bit) { - vpaddd(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpaddd(Dst, Vector1, Vector2); - } - break; - } - case 8: { - if (Is256Bit) { - vpaddq(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpaddq(Dst, Vector1, Vector2); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } -} - -DEF_OP(VSub) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector1 = GetSrc(Op->Vector1.ID()); - const auto Vector2 = GetSrc(Op->Vector2.ID()); - - switch (ElementSize) { - case 1: { - if (Is256Bit) { - vpsubb(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpsubb(Dst, Vector1, Vector2); - } - break; - } - case 2: { - if (Is256Bit) { - vpsubw(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpsubw(Dst, Vector1, Vector2); - } - break; - } - case 4: { - if (Is256Bit) { - vpsubd(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpsubd(Dst, Vector1, Vector2); - } - break; - } - case 8: { - if (Is256Bit) { - vpsubq(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpsubq(Dst, Vector1, Vector2); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } -} - -DEF_OP(VUQAdd) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector1 = GetSrc(Op->Vector1.ID()); - const auto Vector2 = GetSrc(Op->Vector2.ID()); - - switch (ElementSize) { - case 1: { - if (Is256Bit) { - vpaddusb(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpaddusb(Dst, Vector1, Vector2); - } - break; - } - case 2: { - if (Is256Bit) { - vpaddusw(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpaddusw(Dst, Vector1, Vector2); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } -} - -DEF_OP(VUQSub) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector1 = GetSrc(Op->Vector1.ID()); - const auto Vector2 = GetSrc(Op->Vector2.ID()); - - switch (ElementSize) { - case 1: { - if (Is256Bit) { - vpsubusb(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpsubusb(Dst, Vector1, Vector2); - } - break; - } - case 2: { - if (Is256Bit) { - vpsubusw(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpsubusw(Dst, Vector1, Vector2); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } -} - -DEF_OP(VSQAdd) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector1 = GetSrc(Op->Vector1.ID()); - const auto Vector2 = GetSrc(Op->Vector2.ID()); - - switch (ElementSize) { - case 1: { - if (Is256Bit) { - vpaddsb(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpaddsb(Dst, Vector1, Vector2); - } - break; - } - case 2: { - if (Is256Bit) { - vpaddsw(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpaddsw(Dst, Vector1, Vector2); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } -} - -DEF_OP(VSQSub) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector1 = GetSrc(Op->Vector1.ID()); - const auto Vector2 = GetSrc(Op->Vector2.ID()); - - switch (ElementSize) { - case 1: { - if (Is256Bit) { - vpsubsb(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpsubsb(Dst, Vector1, Vector2); - } - break; - } - case 2: { - if (Is256Bit) { - vpsubsw(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpsubsw(Dst, Vector1, Vector2); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } -} - -DEF_OP(VAddP) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - const auto ElementSize = Op->Header.ElementSize; - - const auto Dst = GetDst(Node); - const auto VectorLower = GetSrc(Op->VectorLower.ID()); - const auto VectorUpper = GetSrc(Op->VectorUpper.ID()); - - if (OpSize == 8) { - // Can't handle this natively without dropping to MMX - // Emulate - vpxor(xmm14, xmm14, xmm14); - movq(xmm15, VectorLower); - vshufpd(xmm15, xmm15, VectorUpper, 0b00); - vpaddw(Dst, xmm15, xmm14); - switch (ElementSize) { - case 1: - vpunpcklbw(xmm0, xmm15, xmm14); - vpunpckhbw(xmm12, xmm15, xmm14); - - vpunpcklbw(xmm15, xmm0, xmm12); - vpunpckhbw(xmm14, xmm0, xmm12); - - vpunpcklbw(xmm0, xmm15, xmm14); - vpunpckhbw(xmm12, xmm15, xmm14); - - vpunpcklbw(xmm15, xmm0, xmm12); - vpunpckhbw(xmm14, xmm0, xmm12); - - vpaddb(Dst, xmm15, xmm14); - break; - case 2: - vphaddw(Dst, xmm15, xmm14); - break; - case 4: - vphaddd(Dst, xmm15, xmm14); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } - } - else { - const auto DstYMM = ToYMM(Dst); - const auto VectorLowerYMM = ToYMM(VectorLower); - const auto VectorUpperYMM = ToYMM(VectorUpper); - - // To behave like ADDP, we need to swap the second and third elements around - // in the 256-bit case. ADDP operates as if both vectors are concatenated - // together and runs down the length of it adding pairs as it goes, whereas - // VPHADDW/D operates on both individual halves of the entire register. - switch (ElementSize) { - case 1: - if (Is256Bit) { - vmovdqu(ymm15, VectorLowerYMM); - vmovdqu(ymm14, VectorUpperYMM); - - vpunpcklbw(ymm0, ymm15, ymm14); - vpunpckhbw(ymm12, ymm15, ymm14); - - vpunpcklbw(ymm15, ymm0, ymm12); - vpunpckhbw(ymm14, ymm0, ymm12); - - vpunpcklbw(ymm0, ymm15, ymm14); - vpunpckhbw(ymm12, ymm15, ymm14); - - vpunpcklbw(ymm15, ymm0, ymm12); - vpunpckhbw(ymm14, ymm0, ymm12); - - vpaddb(DstYMM, ymm15, ymm14); - vpermq(DstYMM, DstYMM, 0b11'01'10'00); - } else { - vmovdqu(xmm15, VectorLower); - vmovdqu(xmm14, VectorUpper); - - vpunpcklbw(xmm0, xmm15, xmm14); - vpunpckhbw(xmm12, xmm15, xmm14); - - vpunpcklbw(xmm15, xmm0, xmm12); - vpunpckhbw(xmm14, xmm0, xmm12); - - vpunpcklbw(xmm0, xmm15, xmm14); - vpunpckhbw(xmm12, xmm15, xmm14); - - vpunpcklbw(xmm15, xmm0, xmm12); - vpunpckhbw(xmm14, xmm0, xmm12); - - vpaddb(Dst, xmm15, xmm14); - } - break; - case 2: - if (Is256Bit) { - vphaddw(DstYMM, VectorLowerYMM, VectorUpperYMM); - vpermq(DstYMM, DstYMM, 0b11'01'10'00); - } else { - vphaddw(Dst, VectorLower, VectorUpper); - } - break; - case 4: - if (Is256Bit) { - vphaddd(DstYMM, VectorLowerYMM, VectorUpperYMM); - vpermq(DstYMM, DstYMM, 0b11'01'10'00); - } else { - vphaddd(Dst, VectorLower, VectorUpper); - } - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } - } -} - -DEF_OP(VAddV) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto Src = GetSrc(Op->Vector.ID()); - const auto Dest = GetDst(Node); - - const auto ElementSize = Op->Header.ElementSize; - const auto Elements = OpSize / ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - vpxor(xmm15, xmm15, xmm15); - - switch (ElementSize) { - case 2: { - const auto HorizontalAdd = [this, Elements](const Xbyak::Xmm& dst, Xbyak::Xmm src, const Xbyak::Xmm& tmp) { - for (int i = Elements; i > 1; i >>= 1) { - vphaddw(dst, src, dst); - src = dst; - } - pextrw(eax, dst, 0); - pinsrw(tmp, eax, 0); - }; - - if (Is256Bit) { - vpxor(xmm13, xmm13, xmm13); - vextracti128(xmm14, ToYMM(Src), 1); - - HorizontalAdd(Dest, Src, xmm15); - HorizontalAdd(Dest, xmm14, xmm13); - - vpaddw(Dest, xmm13, xmm15); - } else { - HorizontalAdd(Dest, Src, xmm15); - vmovaps(Dest, xmm15); - } - break; - } - case 4: { - const auto HorizontalAdd = [this, Elements](const Xbyak::Xmm& dst, Xbyak::Xmm src, const Xbyak::Xmm& tmp) { - for (int i = Elements; i > 1; i >>= 1) { - vphaddd(dst, src, dst); - src = dst; - } - pextrd(eax, dst, 0); - pinsrd(tmp, eax, 0); - }; - - if (Is256Bit) { - vpxor(xmm13, xmm13, xmm13); - vextracti128(xmm14, ToYMM(Src), 1); - - HorizontalAdd(Dest, Src, xmm15); - HorizontalAdd(Dest, xmm14, xmm13); - - vpaddd(Dest, xmm13, xmm15); - } else { - HorizontalAdd(Dest, Src, xmm15); - vmovaps(Dest, xmm15); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } -} - -DEF_OP(VUMinV) { - auto Op = IROp->C(); - - const auto OpSize = Op->Header.Size; - const auto ElementSize = Op->Header.ElementSize; - - LOGMAN_THROW_AA_FMT(OpSize == 16 || OpSize == 32, - "Can't handle a vector of size: {}", OpSize); - - const auto Src = GetSrc(Op->Vector.ID()); - const auto Dest = GetDst(Node); - - switch (ElementSize) { - case 2: { - if (OpSize == 32) { - // vphminposuw, unfortunately, doesn't traverse the entire - // 256-bit ymm register (it just sets the top 128 bits to zero) - // so we pull the top 128 bits down into its own vector, - // get the horizontal minimum in both vectors, then - // take the minimum between the two. - vextractf128(xmm15, ToYMM(Src), 1); - vphminposuw(xmm15, xmm15); - vphminposuw(xmm14, Src); - vpminuw(Dest, xmm14, xmm15); - } else { - vphminposuw(Dest, Src); - } - - // Extract the upper bits which are zero, overwriting position - pextrw(eax, Dest, 2); - pinsrw(Dest, eax, 1); - break; - } - case 4: { - mov(TMP1, ~0U); - const auto Elements = OpSize / ElementSize; - for (size_t i = 0; i < Elements; ++i) { - pextrd(TMP2.cvt32(), Src, i); - cmp(TMP1.cvt32(), TMP2.cvt32()); - cmovnbe(TMP1.cvt32(), TMP2.cvt32()); - } - vpxor(Dest, Dest, Dest); - pinsrd(Dest, TMP1.cvt32(), 0); - break; - } - case 8: { - mov(TMP1, ~0ULL); - const auto Elements = OpSize / ElementSize; - for (size_t i = 0; i < Elements; ++i) { - pextrq(TMP2, Src, i); - cmp(TMP1, TMP2); - cmovnbe(TMP1, TMP2); - } - vpxor(Dest, Dest, Dest); - pinsrq(Dest, TMP1, 0); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } -} - -DEF_OP(VURAvg) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector1 = GetSrc(Op->Vector1.ID()); - const auto Vector2 = GetSrc(Op->Vector2.ID()); - - switch (ElementSize) { - case 1: { - if (Is256Bit) { - vpavgb(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpavgb(Dst, Vector1, Vector2); - } - break; - } - case 2: { - if (Is256Bit) { - vpavgw(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpavgw(Dst, Vector1, Vector2); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } -} - -DEF_OP(VAbs) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Src = GetSrc(Op->Vector.ID()); - - switch (ElementSize) { - case 1: { - if (Is256Bit) { - vpabsb(ToYMM(Dst), ToYMM(Src)); - } else { - vpabsb(Dst, Src); - } - break; - } - case 2: { - if (Is256Bit) { - vpabsw(ToYMM(Dst), ToYMM(Src)); - } else { - vpabsw(Dst, Src); - } - break; - } - case 4: { - if (Is256Bit) { - vpabsd(ToYMM(Dst), ToYMM(Src)); - } else { - vpabsd(Dst, Src); - } - break; - } - case 8: { - if (Is256Bit) { - vpabsq(ToYMM(Dst), ToYMM(Src)); - } else { - vpabsq(Dst, Src); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } -} - -// This only supports 8bit popcount on 8byte to 32byte registers -DEF_OP(VPopcount) { - const auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; - const bool Is256Bit = OpSize == 32; - - const auto Src = GetSrc(Op->Vector.ID()); - const auto Dest = GetDst(Node); - - const uint8_t ElementSize = Op->Header.ElementSize; - const uint8_t Elements = OpSize / ElementSize; - - vpxor(xmm15, xmm15, xmm15); - - // This is disgustingly bad on x86-64 but we only need it for compatibility - // NOTE: If, in the distant future, we ever use AVX-512, consider - // using vpopcnt{b, d, q, w} to shorten all of this down to one - // instruction. - switch (ElementSize) { - case 1: { - const uint8_t NumElements = Is256Bit ? Elements / 2 - : Elements; - - const auto VectorPopcount = [this, NumElements](const Xbyak::Xmm& src, const Xbyak::Xmm& dst) { - for (size_t i = 0; i < NumElements; ++i) { - pextrb(eax, src, i); - popcnt(eax, eax); - pinsrb(dst, eax, i); - } - }; - - // Bottom 128 bits - VectorPopcount(Src, xmm15); - movaps(Dest, xmm15); - - // Now do the top 128 bits, if necessary - if (Is256Bit) { - vextracti128(xmm14, ToYMM(Src), 1); - VectorPopcount(xmm14, xmm14); - vinserti128(ToYMM(Dest), ToYMM(Dest), xmm14, 1); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } -} - -DEF_OP(VFAdd) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - const auto IsScalar = ElementSize == OpSize; - - const auto Dst = GetDst(Node); - const auto Vector1 = GetSrc(Op->Vector1.ID()); - const auto Vector2 = GetSrc(Op->Vector2.ID()); - - if (IsScalar) { - switch (ElementSize) { - case 4: { - vaddss(Dst, Vector1, Vector2); - break; - } - case 8: { - vaddsd(Dst, Vector1, Vector2); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } - } else { - switch (ElementSize) { - case 4: { - if (Is256Bit) { - vaddps(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vaddps(Dst, Vector1, Vector2); - } - break; - } - case 8: { - if (Is256Bit) { - vaddpd(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vaddpd(Dst, Vector1, Vector2); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } - } -} - -DEF_OP(VFAddP) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto VectorLower = GetSrc(Op->VectorLower.ID()); - const auto VectorUpper = GetSrc(Op->VectorUpper.ID()); - - // To behave like FADDP, we need to swap the second and third elements around - // in the 256-bit case. FADDP operates as if both vectors are concatenated - // together and runs down the length of it adding pairs as it goes, whereas - // VHADDPS operates on both individual halves of the entire register. - switch (ElementSize) { - case 4: - if (Is256Bit) { - vhaddps(ToYMM(Dst), ToYMM(VectorLower), ToYMM(VectorUpper)); - vpermq(ToYMM(Dst), ToYMM(Dst), 0b11'01'10'00); - } else { - vhaddps(Dst, VectorLower, VectorUpper); - } - break; - case 8: - if (Is256Bit) { - vhaddpd(ToYMM(Dst), ToYMM(VectorLower), ToYMM(VectorUpper)); - vpermq(ToYMM(Dst), ToYMM(Dst), 0b11'01'10'00); - } else { - vhaddpd(Dst, VectorLower, VectorUpper); - } - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } -} - -DEF_OP(VFSub) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - const auto IsScalar = ElementSize == OpSize; - - const auto Dst = GetDst(Node); - const auto Vector1 = GetSrc(Op->Vector1.ID()); - const auto Vector2 = GetSrc(Op->Vector2.ID()); - - if (IsScalar) { - switch (ElementSize) { - case 4: { - vsubss(Dst, Vector1, Vector2); - break; - } - case 8: { - vsubsd(Dst, Vector1, Vector2); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } - } else { - switch (ElementSize) { - case 4: { - if (Is256Bit) { - vsubps(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vsubps(Dst, Vector1, Vector2); - } - break; - } - case 8: { - if (Is256Bit) { - vsubpd(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vsubpd(Dst, Vector1, Vector2); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } - } -} - -DEF_OP(VFMul) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - const auto IsScalar = ElementSize == OpSize; - - const auto Dst = GetDst(Node); - const auto Vector1 = GetSrc(Op->Vector1.ID()); - const auto Vector2 = GetSrc(Op->Vector2.ID()); - - if (IsScalar) { - switch (ElementSize) { - case 4: { - vmulss(Dst, Vector1, Vector2); - break; - } - case 8: { - vmulsd(Dst, Vector1, Vector2); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } - } else { - switch (ElementSize) { - case 4: { - if (Is256Bit) { - vmulps(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vmulps(Dst, Vector1, Vector2); - } - break; - } - case 8: { - if (Is256Bit) { - vmulpd(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vmulpd(Dst, Vector1, Vector2); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } - } -} - -DEF_OP(VFDiv) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - const auto IsScalar = ElementSize == OpSize; - - const auto Dst = GetDst(Node); - const auto Vector1 = GetSrc(Op->Vector1.ID()); - const auto Vector2 = GetSrc(Op->Vector2.ID()); - - if (IsScalar) { - switch (ElementSize) { - case 4: { - vdivss(Dst, Vector1, Vector2); - break; - } - case 8: { - vdivsd(Dst, Vector1, Vector2); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } - } else { - switch (ElementSize) { - case 4: { - if (Is256Bit) { - vdivps(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vdivps(Dst, Vector1, Vector2); - } - break; - } - case 8: { - if (Is256Bit) { - vdivpd(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vdivpd(Dst, Vector1, Vector2); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } - } -} - -DEF_OP(VFMin) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - const auto IsScalar = ElementSize == OpSize; - - const auto Dst = GetDst(Node); - const auto Vector1 = GetSrc(Op->Vector1.ID()); - const auto Vector2 = GetSrc(Op->Vector2.ID()); - - if (IsScalar) { - switch (ElementSize) { - case 4: { - vminss(Dst, Vector1, Vector2); - break; - } - case 8: { - vminsd(Dst, Vector1, Vector2); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } - } else { - switch (ElementSize) { - case 4: { - if (Is256Bit) { - vminps(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vminps(Dst, Vector1, Vector2); - } - break; - } - case 8: { - if (Is256Bit) { - vminpd(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vminpd(Dst, Vector1, Vector2); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } - } -} - -DEF_OP(VFMax) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - const auto IsScalar = ElementSize == OpSize; - - const auto Dst = GetDst(Node); - const auto Vector1 = GetSrc(Op->Vector1.ID()); - const auto Vector2 = GetSrc(Op->Vector2.ID()); - - if (IsScalar) { - switch (ElementSize) { - case 4: { - vmaxss(Dst, Vector1, Vector2); - break; - } - case 8: { - vmaxsd(Dst, Vector1, Vector2); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } - } else { - switch (ElementSize) { - case 4: { - if (Is256Bit) { - vmaxps(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vmaxps(Dst, Vector1, Vector2); - } - break; - } - case 8: { - if (Is256Bit) { - vmaxpd(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vmaxpd(Dst, Vector1, Vector2); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } - } -} - -DEF_OP(VFRecp) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - const auto IsScalar = ElementSize == OpSize; - - const auto Dst = GetDst(Node); - const auto Vector = GetSrc(Op->Vector.ID()); - - if (IsScalar) { - switch (ElementSize) { - case 4: { - mov(eax, 0x3f800000); // 1.0f - vmovd(xmm15, eax); - vdivss(Dst, xmm15, Vector); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } - } else { - switch (ElementSize) { - case 4: { - mov(eax, 0x3f800000); // 1.0f - vmovd(xmm15, eax); - if (Is256Bit) { - vbroadcastss(ymm15, xmm15); - vdivps(ToYMM(Dst), ymm15, ToYMM(Vector)); - } else { - vbroadcastss(xmm15, xmm15); - vdivps(Dst, xmm15, Vector); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } - } -} - -DEF_OP(VFSqrt) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - const auto IsScalar = ElementSize == OpSize; - - const auto Dst = GetDst(Node); - const auto Vector = GetSrc(Op->Vector.ID()); - - if (IsScalar) { - switch (ElementSize) { - case 4: { - vsqrtss(Dst, Vector, Vector); - break; - } - case 8: { - vsqrtsd(Dst, Vector, Vector); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } - } else { - switch (ElementSize) { - case 4: { - if (Is256Bit) { - vsqrtps(ToYMM(Dst), ToYMM(Vector)); - } else { - vsqrtps(Dst, Vector); - } - break; - } - case 8: { - if (Is256Bit) { - vsqrtpd(ToYMM(Dst), ToYMM(Vector)); - } else { - vsqrtpd(Dst, Vector); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } - } -} - -DEF_OP(VFRSqrt) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - const auto IsScalar = ElementSize == OpSize; - - const auto Dst = GetDst(Node); - const auto Vector = GetSrc(Op->Vector.ID()); - - if (IsScalar) { - switch (ElementSize) { - case 4: { - mov(eax, 0x3f800000); // 1.0f - sqrtss(xmm15, Vector); - vmovd(Dst, eax); - divss(Dst, xmm15); - break; - } - case 8: { - mov(eax, 0x3f800000); // 1.0f - sqrtsd(xmm15, Vector); - vmovd(Dst, eax); - divsd(Dst, xmm15); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } - } else { - switch (ElementSize) { - case 4: { - mov(rax, 0x3f800000); // 1.0f - if (Is256Bit) { - vsqrtps(ymm15, ToYMM(Vector)); - vmovd(Dst, eax); - vbroadcastss(ToYMM(Dst), Dst); - vdivps(ToYMM(Dst), ymm15); - } else { - vsqrtps(xmm15, Vector); - vmovd(Dst, eax); - vbroadcastss(Dst, Dst); - vdivps(Dst, xmm15); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } - } -} - -DEF_OP(VNeg) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector = GetSrc(Op->Vector.ID()); - - vpxor(xmm15, xmm15, xmm15); - switch (ElementSize) { - case 1: { - if (Is256Bit) { - vpsubb(ToYMM(Dst), ymm15, ToYMM(Vector)); - } else { - vpsubb(Dst, xmm15, Vector); - } - break; - } - case 2: { - if (Is256Bit) { - vpsubw(ToYMM(Dst), ymm15, ToYMM(Vector)); - } else { - vpsubw(Dst, xmm15, Vector); - } - break; - } - case 4: { - if (Is256Bit) { - vpsubd(ToYMM(Dst), ymm15, ToYMM(Vector)); - } else { - vpsubd(Dst, xmm15, Vector); - } - break; - } - case 8: { - if (Is256Bit) { - vpsubq(ToYMM(Dst), ymm15, ToYMM(Vector)); - } else { - vpsubq(Dst, xmm15, Vector); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } -} - -DEF_OP(VFNeg) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector = GetSrc(Op->Vector.ID()); - - switch (ElementSize) { - case 2: { - mov(rax, 0x80008000); - vmovd(xmm15, eax); - if (Is256Bit) { - vbroadcastss(ymm15, xmm15); - vxorps(ToYMM(Dst), ymm15, ToYMM(Vector)); - } else { - vbroadcastss(xmm15, xmm15); - vxorps(Dst, xmm15, Vector); - } - break; - } - case 4: { - mov(rax, 0x80000000); - vmovd(xmm15, eax); - if (Is256Bit) { - vbroadcastss(ymm15, xmm15); - vxorps(ToYMM(Dst), ymm15, ToYMM(Vector)); - } else { - vbroadcastss(xmm15, xmm15); - vxorps(Dst, xmm15, Vector); - } - break; - } - case 8: { - mov(rax, 0x8000000000000000ULL); - vmovq(xmm15, rax); - if (Is256Bit) { - vpbroadcastq(ymm15, xmm15); - vxorpd(ToYMM(Dst), ymm15, ToYMM(Vector)); - } else { - vpbroadcastq(xmm15, xmm15); - vxorpd(Dst, xmm15, Vector); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } -} - -DEF_OP(VNot) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector = GetSrc(Op->Vector.ID()); - - if (Is256Bit) { - vpcmpeqd(ymm15, ymm15, ymm15); - vpxor(ToYMM(Dst), ymm15, ToYMM(Vector)); - } else { - vpcmpeqd(xmm15, xmm15, xmm15); - vpxor(Dst, xmm15, Vector); - } -} - -DEF_OP(VUMin) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - const auto IsScalar = OpSize == ElementSize; - - const auto Dst = GetDst(Node); - const auto Vector1 = GetSrc(Op->Vector1.ID()); - const auto Vector2 = GetSrc(Op->Vector2.ID()); - - if (IsScalar) { - switch (ElementSize) { - case 8: { - // This isn't very nice on x86 until AVX-512 - pextrq(TMP1, Vector1, 0); - pextrq(TMP2, Vector2, 0); - cmp(TMP1, TMP2); - cmovb(TMP2, TMP1); - pinsrq(Dst, TMP2, 0); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } - } else { - switch (ElementSize) { - case 1: { - if (Is256Bit) { - vpminub(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpminub(Dst, Vector1, Vector2); - } - break; - } - case 2: { - if (Is256Bit) { - vpminuw(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpminuw(Dst, Vector1, Vector2); - } - break; - } - case 4: { - if (Is256Bit) { - vpminud(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpminud(Dst, Vector1, Vector2); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } - } -} - -DEF_OP(VSMin) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector1 = GetSrc(Op->Vector1.ID()); - const auto Vector2 = GetSrc(Op->Vector2.ID()); - - switch (ElementSize) { - case 1: { - if (Is256Bit) { - vpminsb(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpminsb(Dst, Vector1, Vector2); - } - break; - } - case 2: { - if (Is256Bit) { - vpminsw(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpminsw(Dst, Vector1, Vector2); - } - break; - } - case 4: { - if (Is256Bit) { - vpminsd(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpminsd(Dst, Vector1, Vector2); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } -} - -DEF_OP(VUMax) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector1 = GetSrc(Op->Vector1.ID()); - const auto Vector2 = GetSrc(Op->Vector2.ID()); - - switch (ElementSize) { - case 1: { - if (Is256Bit) { - vpmaxub(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpmaxub(Dst, Vector1, Vector2); - } - break; - } - case 2: { - if (Is256Bit) { - vpmaxuw(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpmaxuw(Dst, Vector1, Vector2); - } - break; - } - case 4: { - if (Is256Bit) { - vpmaxud(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpmaxud(Dst, Vector1, Vector2); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } -} - -DEF_OP(VSMax) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector1 = GetSrc(Op->Vector1.ID()); - const auto Vector2 = GetSrc(Op->Vector2.ID()); - - switch (ElementSize) { - case 1: { - if (Is256Bit) { - vpmaxsb(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpmaxsb(Dst, Vector1, Vector2); - } - break; - } - case 2: { - if (Is256Bit) { - vpmaxsw(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpmaxsw(Dst, Vector1, Vector2); - } - break; - } - case 4: { - if (Is256Bit) { - vpmaxsd(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpmaxsd(Dst, Vector1, Vector2); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } -} - -DEF_OP(VZip) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto VectorLower = GetSrc(Op->VectorLower.ID()); - const auto VectorUpper = GetSrc(Op->VectorUpper.ID()); - - if (Is256Bit) { - switch (ElementSize) { - case 1: { - vpunpcklbw(xmm15, VectorLower, VectorUpper); - vpunpckhbw(xmm14, VectorLower, VectorUpper); - break; - } - case 2: { - vpunpcklwd(xmm15, VectorLower, VectorUpper); - vpunpckhwd(xmm14, VectorLower, VectorUpper); - break; - } - case 4: { - vpunpckldq(xmm15, VectorLower, VectorUpper); - vpunpckhdq(xmm14, VectorLower, VectorUpper); - break; - } - case 8: { - vpunpcklqdq(xmm15, VectorLower, VectorUpper); - vpunpckhqdq(xmm14, VectorLower, VectorUpper); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } - - vinserti128(ymm15, ymm15, xmm14, 1); - vmovapd(ToYMM(Dst), ymm15); - } else { - switch (ElementSize) { - case 1: { - vpunpcklbw(Dst, VectorLower, VectorUpper); - break; - } - case 2: { - vpunpcklwd(Dst, VectorLower, VectorUpper); - break; - } - case 4: { - vpunpckldq(Dst, VectorLower, VectorUpper); - break; - } - case 8: { - vpunpcklqdq(Dst, VectorLower, VectorUpper); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } - } -} - -DEF_OP(VZip2) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto VectorLower = GetSrc(Op->VectorLower.ID()); - const auto VectorUpper = GetSrc(Op->VectorUpper.ID()); - - if (Is256Bit) { - vextracti128(xmm15, ToYMM(VectorLower), 1); - vextracti128(xmm14, ToYMM(VectorUpper), 1); - - switch (ElementSize) { - case 1: { - vpunpcklbw(Dst, xmm15, xmm14); - vpunpckhbw(xmm13, xmm15, xmm14); - break; - } - case 2: { - vpunpcklwd(Dst, xmm15, xmm14); - vpunpckhwd(xmm13, xmm15, xmm14); - break; - } - case 4: { - vpunpckldq(Dst, xmm15, xmm14); - vpunpckhdq(xmm13, xmm15, xmm14); - break; - } - case 8: { - vpunpcklqdq(Dst, xmm15, xmm14); - vpunpckhqdq(xmm13, xmm15, xmm14); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } - - vinserti128(ToYMM(Dst), ToYMM(Dst), xmm13, 1); - } else { - if (OpSize == 8) { - vpslldq(xmm15, VectorLower, 4); - vpslldq(xmm14, VectorUpper, 4); - switch (ElementSize) { - case 1: { - vpunpckhbw(Dst, xmm15, xmm14); - break; - } - case 2: { - vpunpckhwd(Dst, xmm15, xmm14); - break; - } - case 4: { - vpunpckhdq(Dst, xmm15, xmm14); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } - } else { - switch (ElementSize) { - case 1: { - vpunpckhbw(Dst, VectorLower, VectorUpper); - break; - } - case 2: { - vpunpckhwd(Dst, VectorLower, VectorUpper); - break; - } - case 4: { - vpunpckhdq(Dst, VectorLower, VectorUpper); - break; - } - case 8: { - vpunpckhqdq(Dst, VectorLower, VectorUpper); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } - } - } -} - -DEF_OP(VUnZip) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto VectorLower = GetSrc(Op->VectorLower.ID()); - const auto VectorUpper = GetSrc(Op->VectorUpper.ID()); - - if (OpSize == 8) { - switch (ElementSize) { - case 1: { - mov(rax, 0x80'80'80'80'06'04'02'00); // Lower - mov(rcx, 0x80'80'80'80'80'80'80'80); // Upper - vmovq(xmm15, rax); - pinsrq(xmm15, rcx, 1); - vpshufb(xmm14, VectorLower, xmm15); - vpshufb(xmm13, VectorUpper, xmm15); - vpunpckldq(Dst, xmm14, xmm13); - break; - } - case 2: { - mov(rax, 0x80'80'80'80'05'04'01'00); // Lower - mov(rcx, 0x80'80'80'80'80'80'80'80); // Upper - vmovq(xmm15, rax); - pinsrq(xmm15, rcx, 1); - vpshufb(xmm14, VectorLower, xmm15); - vpshufb(xmm13, VectorUpper, xmm15); - vpunpckldq(Dst, xmm14, xmm13); - break; - } - case 4: { - insertps(xmm15, VectorLower, 0b00'00'1110); - vinsertps(Dst, xmm15, VectorUpper, 0b00'01'1100); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } - } - else { - switch (ElementSize) { - case 1: { - // Shuffle low bits - mov(rax, 0x0E'0C'0A'08'06'04'02'00); // Lower - mov(rcx, 0x80'80'80'80'80'80'80'80); // Upper - vmovq(xmm15, rax); - pinsrq(xmm15, rcx, 1); - if (Is256Bit) { - vinserti128(ymm15, ymm15, xmm15, 1); - vpshufb(ymm14, ToYMM(VectorLower), ymm15); - vpshufb(ymm13, ToYMM(VectorUpper), ymm15); - vpunpcklqdq(ToYMM(Dst), ymm14, ymm13); - vpermq(ToYMM(Dst), ToYMM(Dst), 0b11'01'10'00); - } else { - vpshufb(xmm14, VectorLower, xmm15); - vpshufb(xmm13, VectorUpper, xmm15); - // movlhps back to combine - vmovlhps(Dst, xmm14, xmm13); - } - break; - } - case 2: { - // Shuffle low bits - mov(rax, 0x0D'0C'09'08'05'04'01'00); // Lower - mov(rcx, 0x80'80'80'80'80'80'80'80); // Upper - vmovq(xmm15, rax); - pinsrq(xmm15, rcx, 1); - if (Is256Bit) { - vinserti128(ymm15, ymm15, xmm15, 1); - vpshufb(ymm14, ToYMM(VectorLower), ymm15); - vpshufb(ymm13, ToYMM(VectorUpper), ymm15); - vpunpcklqdq(ToYMM(Dst), ymm14, ymm13); - vpermq(ToYMM(Dst), ToYMM(Dst), 0b11'01'10'00); - } else { - vpshufb(xmm14, VectorLower, xmm15); - vpshufb(xmm13, VectorUpper, xmm15); - // movlhps back to combine - vmovlhps(Dst, xmm14, xmm13); - } - break; - } - case 4: { - if (Is256Bit) { - vshufps(ToYMM(Dst), ToYMM(VectorLower), ToYMM(VectorUpper), 0b10'00'10'00); - vpermq(ToYMM(Dst), ToYMM(Dst), 0b11'01'10'00); - } else { - vshufps(Dst, VectorLower, VectorUpper, 0b10'00'10'00); - } - break; - } - case 8: { - if (Is256Bit) { - vshufpd(ToYMM(Dst), ToYMM(VectorLower), ToYMM(VectorUpper), 0b0'0); - vpermq(ToYMM(Dst), ToYMM(Dst), 0b11'01'10'00); - } else { - vshufpd(Dst, VectorLower, VectorUpper, 0b0'0); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } - } -} - -DEF_OP(VUnZip2) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto VectorLower = GetSrc(Op->VectorLower.ID()); - const auto VectorUpper = GetSrc(Op->VectorUpper.ID()); - - if (OpSize == 8) { - switch (ElementSize) { - case 1: { - mov(rax, 0x80'80'80'80'07'05'03'01); // Lower - mov(rcx, 0x80'80'80'80'80'80'80'80); // Upper - vmovq(xmm15, rax); - pinsrq(xmm15, rcx, 1); - vpshufb(xmm14, VectorLower, xmm15); - vpshufb(xmm13, VectorUpper, xmm15); - vpunpckldq(Dst, xmm14, xmm13); - break; - } - case 2: { - mov(rax, 0x80'80'80'80'07'06'03'02); // Lower - mov(rcx, 0x80'80'80'80'80'80'80'80); // Upper - vmovq(xmm15, rax); - pinsrq(xmm15, rcx, 1); - vpshufb(xmm14, VectorLower, xmm15); - vpshufb(xmm13, VectorUpper, xmm15); - vpunpckldq(Dst, xmm14, xmm13); - break; - } - case 4: { - insertps(xmm15, VectorLower, 0b01'00'1110); - vinsertps(Dst, xmm15, VectorUpper, 0b01'01'1100); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } - } - else { - switch (ElementSize) { - case 1: { - // Shuffle low bits - mov(rax, 0x0F'0D'0B'09'07'05'03'01); // Lower - mov(rcx, 0x80'80'80'80'80'80'80'80); // Upper - vmovq(xmm15, rax); - pinsrq(xmm15, rcx, 1); - if (Is256Bit) { - vinserti128(ymm15, ymm15, xmm15, 1); - vpshufb(ymm14, ToYMM(VectorLower), ymm15); - vpshufb(ymm13, ToYMM(VectorUpper), ymm15); - vpunpcklqdq(ToYMM(Dst), ymm14, ymm13); - vpermq(ToYMM(Dst), ToYMM(Dst), 0b11'01'10'00); - } else { - vpshufb(xmm14, VectorLower, xmm15); - vpshufb(xmm13, VectorUpper, xmm15); - // movlhps back to combine - vmovlhps(Dst, xmm14, xmm13); - } - break; - } - case 2: { - // Shuffle low bits - mov(rax, 0x0F'0E'0B'0A'07'06'03'02); // Lower - mov(rcx, 0x80'80'80'80'80'80'80'80); // Upper - vmovq(xmm15, rax); - pinsrq(xmm15, rcx, 1); - if (Is256Bit) { - vinserti128(ymm15, ymm15, xmm15, 1); - vpshufb(ymm14, ToYMM(VectorLower), ymm15); - vpshufb(ymm13, ToYMM(VectorUpper), ymm15); - vpunpcklqdq(ToYMM(Dst), ymm14, ymm13); - vpermq(ToYMM(Dst), ToYMM(Dst), 0b11'01'10'00); - } else { - vpshufb(xmm14, VectorLower, xmm15); - vpshufb(xmm13, VectorUpper, xmm15); - // movlhps back to combine - vmovlhps(Dst, xmm14, xmm13); - } - break; - } - case 4: { - if (Is256Bit) { - vshufps(ToYMM(Dst), ToYMM(VectorLower), ToYMM(VectorUpper), 0b11'01'11'01); - vpermq(ToYMM(Dst), ToYMM(Dst), 0b11'01'10'00); - } else { - vshufps(Dst, VectorLower, VectorUpper, 0b11'01'11'01); - } - break; - } - case 8: { - if (Is256Bit) { - vshufpd(ToYMM(Dst), ToYMM(VectorLower), ToYMM(VectorUpper), 0b11'11); - vpermq(ToYMM(Dst), ToYMM(Dst), 0b11'01'10'00); - } else { - vshufpd(Dst, VectorLower, VectorUpper, 0b1'1); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } - } -} - -DEF_OP(VTrn) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto VectorLower = GetSrc(Op->VectorLower.ID()); - const auto VectorUpper = GetSrc(Op->VectorUpper.ID()); - - const auto LoadPshufbReg = [&](Xbyak::Xmm reg, uint64_t lower) { - mov(rax, lower); - mov(rcx, 0x80'80'80'80'80'80'80'80); - vmovq(reg, rax); - pinsrq(reg, rcx, 1); - }; - - switch (ElementSize) { - case 1: { - LoadPshufbReg(xmm15, 0x0E'0C'0A'08'06'04'02'00); - - if (Is256Bit) { - vinserti128(ymm15, ymm15, xmm15, 1); - - vpshufb(ymm14, ToYMM(VectorLower), ymm15); - vpshufb(ymm13, ToYMM(VectorUpper), ymm15); - - vpunpcklbw(ToYMM(Dst), ymm14, ymm13); - } else { - vpshufb(xmm14, VectorLower, xmm15); - vpshufb(xmm13, VectorUpper, xmm15); - vpunpcklbw(Dst, xmm14, xmm13); - } - break; - } - case 2: { - LoadPshufbReg(xmm15, 0x0D'0C'09'08'05'04'01'00); - - if (Is256Bit) { - vinserti128(ymm15, ymm15, xmm15, 1); - - vpshufb(ymm14, ToYMM(VectorLower), ymm15); - vpshufb(ymm13, ToYMM(VectorUpper), ymm15); - - vpunpcklwd(ToYMM(Dst), ymm14, ymm13); - } else { - vpshufb(xmm14, VectorLower, xmm15); - vpshufb(xmm13, VectorUpper, xmm15); - vpunpcklwd(Dst, xmm14, xmm13); - } - break; - } - case 4: { - LoadPshufbReg(xmm15, 0x0B'0A'09'08'03'02'01'00); - - if (Is256Bit) { - vinserti128(ymm15, ymm15, xmm15, 1); - - vpshufb(ymm14, ToYMM(VectorLower), ymm15); - vpshufb(ymm13, ToYMM(VectorUpper), ymm15); - - vpunpckldq(ToYMM(Dst), ymm14, ymm13); - } else { - vpshufb(xmm14, VectorLower, xmm15); - vpshufb(xmm13, VectorUpper, xmm15); - vpunpckldq(Dst, xmm14, xmm13); - } - break; - } - case 8: { - LoadPshufbReg(xmm15, 0x07'06'05'04'03'02'01'00); - - if (Is256Bit) { - vinserti128(ymm15, ymm15, xmm15, 1); - - vpshufb(ymm14, ToYMM(VectorLower), ymm15); - vpshufb(ymm13, ToYMM(VectorUpper), ymm15); - - vpunpcklqdq(ToYMM(Dst), ymm14, ymm13); - } else { - vpshufb(xmm14, VectorLower, xmm15); - vpshufb(xmm13, VectorUpper, xmm15); - vpunpcklqdq(Dst, xmm14, xmm13); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } -} - -DEF_OP(VTrn2) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto VectorLower = GetSrc(Op->VectorLower.ID()); - const auto VectorUpper = GetSrc(Op->VectorUpper.ID()); - - const auto LoadPshufbReg = [&](Xbyak::Xmm reg, uint64_t lower) { - mov(rax, lower); - mov(rcx, 0x80'80'80'80'80'80'80'80); - vmovq(reg, rax); - pinsrq(reg, rcx, 1); - }; - - switch (ElementSize) { - case 1: { - LoadPshufbReg(xmm15, 0x0F'0D'0B'09'07'05'03'01); - - if (Is256Bit) { - vinserti128(ymm15, ymm15, xmm15, 1); - - vpshufb(ymm14, ToYMM(VectorLower), ymm15); - vpshufb(ymm13, ToYMM(VectorUpper), ymm15); - - vpunpcklbw(ToYMM(Dst), ymm14, ymm13); - } else { - vpshufb(xmm14, VectorLower, xmm15); - vpshufb(xmm13, VectorUpper, xmm15); - vpunpcklbw(Dst, xmm14, xmm13); - } - break; - } - case 2: { - LoadPshufbReg(xmm15, 0x0F'0E'0B'0A'07'06'03'02); - - if (Is256Bit) { - vinserti128(ymm15, ymm15, xmm15, 1); - - vpshufb(ymm14, ToYMM(VectorLower), ymm15); - vpshufb(ymm13, ToYMM(VectorUpper), ymm15); - - vpunpcklwd(ToYMM(Dst), ymm14, ymm13); - } else { - vpshufb(xmm14, VectorLower, xmm15); - vpshufb(xmm13, VectorUpper, xmm15); - vpunpcklwd(Dst, xmm14, xmm13); - } - break; - } - case 4: { - LoadPshufbReg(xmm15, 0x0F'0E'0D'0C'07'06'05'04); - - if (Is256Bit) { - vinserti128(ymm15, ymm15, xmm15, 1); - - vpshufb(ymm14, ToYMM(VectorLower), ymm15); - vpshufb(ymm13, ToYMM(VectorUpper), ymm15); - - vpunpckldq(ToYMM(Dst), ymm14, ymm13); - } else { - vpshufb(xmm14, VectorLower, xmm15); - vpshufb(xmm13, VectorUpper, xmm15); - vpunpckldq(Dst, xmm14, xmm13); - } - break; - } - case 8: { - LoadPshufbReg(xmm15, 0x0F'0E'0D'0C'0B'0A'09'08); - - if (Is256Bit) { - vinserti128(ymm15, ymm15, xmm15, 1); - - vpshufb(ymm14, ToYMM(VectorLower), ymm15); - vpshufb(ymm13, ToYMM(VectorUpper), ymm15); - - vpunpcklqdq(ToYMM(Dst), ymm14, ymm13); - } else { - vpshufb(xmm14, VectorLower, xmm15); - vpshufb(xmm13, VectorUpper, xmm15); - vpunpcklqdq(Dst, xmm14, xmm13); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } -} - -DEF_OP(VBSL) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto VectorFalse = GetSrc(Op->VectorFalse.ID()); - const auto VectorTrue = GetSrc(Op->VectorTrue.ID()); - const auto VectorMask = GetSrc(Op->VectorMask.ID()); - - if (Is256Bit) { - vpand(ymm0, ToYMM(VectorMask), ToYMM(VectorTrue)); - vpandn(ymm12, ToYMM(VectorMask), ToYMM(VectorFalse)); - vpor(ToYMM(Dst), ymm0, ymm12); - } else { - vpand(xmm0, VectorMask, VectorTrue); - vpandn(xmm12, VectorMask, VectorFalse); - vpor(Dst, xmm0, xmm12); - } -} - -DEF_OP(VCMPEQ) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector1 = GetSrc(Op->Vector1.ID()); - const auto Vector2 = GetSrc(Op->Vector2.ID()); - - switch (ElementSize) { - case 1: - if (Is256Bit) { - vpcmpeqb(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpcmpeqb(Dst, Vector1, Vector2); - } - break; - case 2: - if (Is256Bit) { - vpcmpeqw(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpcmpeqw(Dst, Vector1, Vector2); - } - break; - case 4: - if (Is256Bit) { - vpcmpeqd(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpcmpeqd(Dst, Vector1, Vector2); - } - break; - case 8: - if (Is256Bit) { - vpcmpeqq(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpcmpeqq(Dst, Vector1, Vector2); - } - break; - default: - LOGMAN_MSG_A_FMT("Unsupported element size: {}", ElementSize); - break; - } -} - -DEF_OP(VCMPEQZ) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector = GetSrc(Op->Vector.ID()); - - vpxor(xmm15, xmm15, xmm15); - switch (ElementSize) { - case 1: - if (Is256Bit) { - vpcmpeqb(ToYMM(Dst), ToYMM(Vector), ymm15); - } else { - vpcmpeqb(Dst, Vector, xmm15); - } - break; - case 2: - if (Is256Bit) { - vpcmpeqw(ToYMM(Dst), ToYMM(Vector), ymm15); - } else { - vpcmpeqw(Dst, Vector, xmm15); - } - break; - case 4: - if (Is256Bit) { - vpcmpeqd(ToYMM(Dst), ToYMM(Vector), ymm15); - } else { - vpcmpeqd(Dst, Vector, xmm15); - } - break; - case 8: - if (Is256Bit) { - vpcmpeqq(ToYMM(Dst), ToYMM(Vector), ymm15); - } else { - vpcmpeqq(Dst, Vector, xmm15); - } - break; - default: - LOGMAN_MSG_A_FMT("Unsupported element size: {}", ElementSize); - break; - } -} - -DEF_OP(VCMPGT) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector1 = GetSrc(Op->Vector1.ID()); - const auto Vector2 = GetSrc(Op->Vector2.ID()); - - switch (ElementSize) { - case 1: - if (Is256Bit) { - vpcmpgtb(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpcmpgtb(Dst, Vector1, Vector2); - } - break; - case 2: - if (Is256Bit) { - vpcmpgtw(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpcmpgtw(Dst, Vector1, Vector2); - } - break; - case 4: - if (Is256Bit) { - vpcmpgtd(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpcmpgtd(Dst, Vector1, Vector2); - } - break; - case 8: - if (Is256Bit) { - vpcmpgtq(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpcmpgtq(Dst, Vector1, Vector2); - } - break; - default: - LOGMAN_MSG_A_FMT("Unsupported element size: {}", ElementSize); - break; - } -} - -DEF_OP(VCMPGTZ) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector = GetSrc(Op->Vector.ID()); - - vpxor(xmm15, xmm15, xmm15); - switch (ElementSize) { - case 1: - if (Is256Bit) { - vpcmpgtb(ToYMM(Dst), ToYMM(Vector), ymm15); - } else { - vpcmpgtb(Dst, Vector, xmm15); - } - break; - case 2: - if (Is256Bit) { - vpcmpgtw(ToYMM(Dst), ToYMM(Vector), ymm15); - } else { - vpcmpgtw(Dst, Vector, xmm15); - } - break; - case 4: - if (Is256Bit) { - vpcmpgtd(ToYMM(Dst), ToYMM(Vector), ymm15); - } else { - vpcmpgtd(Dst, Vector, xmm15); - } - break; - case 8: - if (Is256Bit) { - vpcmpgtq(ToYMM(Dst), ToYMM(Vector), ymm15); - } else { - vpcmpgtq(Dst, Vector, xmm15); - } - break; - default: - LOGMAN_MSG_A_FMT("Unsupported element size: {}", ElementSize); - break; - } -} - -DEF_OP(VCMPLTZ) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector = GetSrc(Op->Vector.ID()); - - vpxor(xmm15, xmm15, xmm15); - switch (ElementSize) { - case 1: - if (Is256Bit) { - vpcmpgtb(ToYMM(Dst), ymm15, ToYMM(Vector)); - } else { - vpcmpgtb(Dst, xmm15, Vector); - } - break; - case 2: - if (Is256Bit) { - vpcmpgtw(ToYMM(Dst), ymm15, ToYMM(Vector)); - } else { - vpcmpgtw(Dst, xmm15, Vector); - } - break; - case 4: - if (Is256Bit) { - vpcmpgtd(ToYMM(Dst), ymm15, ToYMM(Vector)); - } else { - vpcmpgtd(Dst, xmm15, Vector); - } - break; - case 8: - if (Is256Bit) { - vpcmpgtq(ToYMM(Dst), ymm15, ToYMM(Vector)); - } else { - vpcmpgtq(Dst, xmm15, Vector); - } - break; - default: - LOGMAN_MSG_A_FMT("Unsupported element size: {}", ElementSize); - break; - } -} - -DEF_OP(VFCMPEQ) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - const auto IsScalar = ElementSize == OpSize; - - const auto Dst = GetDst(Node); - const auto Vector1 = GetSrc(Op->Vector1.ID()); - const auto Vector2 = GetSrc(Op->Vector2.ID()); - - if (IsScalar) { - switch (ElementSize) { - case 4: - vcmpss(Dst, Vector1, Vector2, 0); - break; - case 8: - vcmpsd(Dst, Vector1, Vector2, 0); - break; - default: - LOGMAN_MSG_A_FMT("Unsupported element size: {}", ElementSize); - break; - } - } else { - switch (ElementSize) { - case 4: - if (Is256Bit) { - vcmpps(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2), 0); - } else { - vcmpps(Dst, Vector1, Vector2, 0); - } - break; - case 8: - if (Is256Bit) { - vcmppd(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2), 0); - } else { - vcmppd(Dst, Vector1, Vector2, 0); - } - break; - default: - LOGMAN_MSG_A_FMT("Unsupported element size: {}", ElementSize); - break; - } - } -} - -DEF_OP(VFCMPNEQ) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - const auto IsScalar = ElementSize == OpSize; - - const auto Dst = GetDst(Node); - const auto Vector1 = GetSrc(Op->Vector1.ID()); - const auto Vector2 = GetSrc(Op->Vector2.ID()); - - if (IsScalar) { - switch (ElementSize) { - case 4: - vcmpss(Dst, Vector1, Vector2, 4); - break; - case 8: - vcmpsd(Dst, Vector1, Vector2, 4); - break; - default: - LOGMAN_MSG_A_FMT("Unsupported element size: {}", ElementSize); - break; - } - } else { - switch (ElementSize) { - case 4: - if (Is256Bit) { - vcmpps(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2), 4); - } else { - vcmpps(Dst, Vector1, Vector2, 4); - } - break; - case 8: - if (Is256Bit) { - vcmppd(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2), 4); - } else { - vcmppd(Dst, Vector1, Vector2, 4); - } - break; - default: - LOGMAN_MSG_A_FMT("Unsupported element size: {}", ElementSize); - break; - } - } -} - -DEF_OP(VFCMPLT) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - const auto IsScalar = ElementSize == OpSize; - - const auto Dst = GetDst(Node); - const auto Vector1 = GetSrc(Op->Vector1.ID()); - const auto Vector2 = GetSrc(Op->Vector2.ID()); - - if (IsScalar) { - switch (ElementSize) { - case 4: - vcmpss(Dst, Vector1, Vector2, 1); - break; - case 8: - vcmpsd(Dst, Vector1, Vector2, 1); - break; - default: - LOGMAN_MSG_A_FMT("Unsupported element size: {}", ElementSize); - break; - } - } else { - switch (ElementSize) { - case 4: - if (Is256Bit) { - vcmpps(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2), 1); - } else { - vcmpps(Dst, Vector1, Vector2, 1); - } - break; - case 8: - if (Is256Bit) { - vcmppd(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2), 1); - } else { - vcmppd(Dst, Vector1, Vector2, 1); - } - break; - default: - LOGMAN_MSG_A_FMT("Unsupported element size: {}", ElementSize); - break; - } - } -} - -DEF_OP(VFCMPGT) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - const auto IsScalar = ElementSize == OpSize; - - const auto Dst = GetDst(Node); - const auto Vector1 = GetSrc(Op->Vector1.ID()); - const auto Vector2 = GetSrc(Op->Vector2.ID()); - - if (IsScalar) { - switch (ElementSize) { - case 4: - vcmpss(Dst, Vector2, Vector1, 1); - break; - case 8: - vcmpsd(Dst, Vector2, Vector1, 1); - break; - default: - LOGMAN_MSG_A_FMT("Unsupported element size: {}", ElementSize); - break; - } - } else { - switch (ElementSize) { - case 4: - if (Is256Bit) { - vcmpps(ToYMM(Dst), ToYMM(Vector2), ToYMM(Vector1), 1); - } else { - vcmpps(Dst, Vector2, Vector1, 1); - } - break; - case 8: - if (Is256Bit) { - vcmppd(ToYMM(Dst), ToYMM(Vector2), ToYMM(Vector1), 1); - } else { - vcmppd(Dst, Vector2, Vector1, 1); - } - break; - default: - LOGMAN_MSG_A_FMT("Unsupported element size: {}", ElementSize); - break; - } - } -} - -DEF_OP(VFCMPLE) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - const auto IsScalar = ElementSize == OpSize; - - const auto Dst = GetDst(Node); - const auto Vector1 = GetSrc(Op->Vector1.ID()); - const auto Vector2 = GetSrc(Op->Vector2.ID()); - - if (IsScalar) { - switch (ElementSize) { - case 4: - vcmpss(Dst, Vector1, Vector2, 2); - break; - case 8: - vcmpsd(Dst, Vector1, Vector2, 2); - break; - default: - LOGMAN_MSG_A_FMT("Unsupported element size: {}", ElementSize); - break; - } - } else { - switch (ElementSize) { - case 4: - if (Is256Bit) { - vcmpps(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2), 2); - } else { - vcmpps(Dst, Vector1, Vector2, 2); - } - break; - case 8: - if (Is256Bit) { - vcmppd(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2), 2); - } else { - vcmppd(Dst, Vector1, Vector2, 2); - } - break; - default: - LOGMAN_MSG_A_FMT("Unsupported element size: {}", ElementSize); - break; - } - } -} - -DEF_OP(VFCMPORD) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - const auto IsScalar = ElementSize == OpSize; - - const auto Dst = GetDst(Node); - const auto Vector1 = GetSrc(Op->Vector1.ID()); - const auto Vector2 = GetSrc(Op->Vector2.ID()); - - if (IsScalar) { - switch (ElementSize) { - case 4: - vcmpss(Dst, Vector1, Vector2, 7); - break; - case 8: - vcmpsd(Dst, Vector1, Vector2, 7); - break; - default: - LOGMAN_MSG_A_FMT("Unsupported element size: {}", ElementSize); - break; - } - } else { - switch (ElementSize) { - case 4: - if (Is256Bit) { - vcmpps(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2), 7); - } else { - vcmpps(Dst, Vector1, Vector2, 7); - } - break; - case 8: - if (Is256Bit) { - vcmppd(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2), 7); - } else { - vcmppd(Dst, Vector1, Vector2, 7); - } - break; - default: - LOGMAN_MSG_A_FMT("Unsupported element size: {}", ElementSize); - break; - } - } -} - -DEF_OP(VFCMPUNO) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - const auto IsScalar = ElementSize == OpSize; - - const auto Dst = GetDst(Node); - const auto Vector1 = GetSrc(Op->Vector1.ID()); - const auto Vector2 = GetSrc(Op->Vector2.ID()); - - if (IsScalar) { - switch (ElementSize) { - case 4: - vcmpss(Dst, Vector1, Vector2, 3); - break; - case 8: - vcmpsd(Dst, Vector1, Vector2, 3); - break; - default: - LOGMAN_MSG_A_FMT("Unsupported element size: {}", ElementSize); - break; - } - } else { - switch (ElementSize) { - case 4: - if (Is256Bit) { - vcmpps(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2), 3); - } else { - vcmpps(Dst, Vector1, Vector2, 3); - } - break; - case 8: - if (Is256Bit) { - vcmppd(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2), 3); - } else { - vcmppd(Dst, Vector1, Vector2, 3); - } - break; - default: - LOGMAN_MSG_A_FMT("Unsupported element size: {}", ElementSize); - break; - } - } -} - -DEF_OP(VUShl) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto ElementSize = IROp->ElementSize; - LOGMAN_THROW_AA_FMT(ElementSize == 4 || ElementSize == 8, - "VUShl only supports 32-bit and 64-bit elements"); - - const auto Dst = GetDst(Node); - const auto ShiftVector = GetSrc(Op->ShiftVector.ID()); - const auto Vector = GetSrc(Op->Vector.ID()); - - switch (ElementSize) { - case 4: - if (Is256Bit) { - vpsllvd(ToYMM(Dst), ToYMM(Vector), ToYMM(ShiftVector)); - } else { - vpsllvd(Dst, Vector, ShiftVector); - } - return; - case 8: - if (Is256Bit) { - vpsllvq(ToYMM(Dst), ToYMM(Vector), ToYMM(ShiftVector)); - } else { - vpsllvq(Dst, Vector, ShiftVector); - } - return; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } -} - -DEF_OP(VUShr) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto ElementSize = IROp->ElementSize; - LOGMAN_THROW_AA_FMT(ElementSize == 4 || ElementSize == 8, - "VUShr only supports 32-bit and 64-bit elements"); - - const auto Dst = GetDst(Node); - const auto ShiftVector = GetSrc(Op->ShiftVector.ID()); - const auto Vector = GetSrc(Op->Vector.ID()); - - switch (ElementSize) { - case 4: - if (Is256Bit) { - vpsrlvd(ToYMM(Dst), ToYMM(Vector), ToYMM(ShiftVector)); - } else { - vpsrlvd(Dst, Vector, ShiftVector); - } - return; - case 8: - if (Is256Bit) { - vpsrlvq(ToYMM(Dst), ToYMM(Vector), ToYMM(ShiftVector)); - } else { - vpsrlvq(Dst, Vector, ShiftVector); - } - return; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } -} - -DEF_OP(VSShr) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto ElementSize = IROp->ElementSize; - LOGMAN_THROW_AA_FMT(ElementSize == 4, "VSShr only supports 32-bit elements"); - - const auto Dst = GetDst(Node); - const auto ShiftVector = GetSrc(Op->ShiftVector.ID()); - const auto Vector = GetSrc(Op->Vector.ID()); - - if (Is256Bit) { - vpsravd(ToYMM(Dst), ToYMM(Vector), ToYMM(ShiftVector)); - } else { - vpsravd(Dst, Vector, ShiftVector); - } -} - -DEF_OP(VUShlS) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto ShiftScalar = GetSrc(Op->ShiftScalar.ID()); - const auto Vector = GetSrc(Op->Vector.ID()); - - switch (ElementSize) { - case 2: { - if (Is256Bit) { - vpsllw(ToYMM(Dst), ToYMM(Vector), ShiftScalar); - } else { - vpsllw(Dst, Vector, ShiftScalar); - } - break; - } - case 4: { - if (Is256Bit) { - vpslld(ToYMM(Dst), ToYMM(Vector), ShiftScalar); - } else { - vpslld(Dst, Vector, ShiftScalar); - } - break; - } - case 8: { - if (Is256Bit) { - vpsllq(ToYMM(Dst), ToYMM(Vector), ShiftScalar); - } else { - vpsllq(Dst, Vector, ShiftScalar); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } -} - -DEF_OP(VUShrS) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto ShiftScalar = GetSrc(Op->ShiftScalar.ID()); - const auto Vector = GetSrc(Op->Vector.ID()); - - switch (ElementSize) { - case 2: { - if (Is256Bit) { - vpsrlw(ToYMM(Dst), ToYMM(Vector), ShiftScalar); - } else { - vpsrlw(Dst, Vector, ShiftScalar); - } - break; - } - case 4: { - if (Is256Bit) { - vpsrld(ToYMM(Dst), ToYMM(Vector), ShiftScalar); - } else { - vpsrld(Dst, Vector, ShiftScalar); - } - break; - } - case 8: { - if (Is256Bit) { - vpsrlq(ToYMM(Dst), ToYMM(Vector), ShiftScalar); - } else { - vpsrlq(Dst, Vector, ShiftScalar); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } -} - -DEF_OP(VSShrS) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto ShiftScalar = GetSrc(Op->ShiftScalar.ID()); - const auto Vector = GetSrc(Op->Vector.ID()); - - switch (ElementSize) { - case 2: { - if (Is256Bit) { - vpsraw(ToYMM(Dst), ToYMM(Vector), ShiftScalar); - } else { - vpsraw(Dst, Vector, ShiftScalar); - } - break; - } - case 4: { - if (Is256Bit) { - vpsrad(ToYMM(Dst), ToYMM(Vector), ShiftScalar); - } else { - vpsrad(Dst, Vector, ShiftScalar); - } - break; - } - case 8: // VPSRAQ is only introduced in AVX-512 - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } -} - -DEF_OP(VInsElement) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto DestIdx = Op->DestIdx; - const auto SrcIdx = Op->SrcIdx; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto DestVector = GetSrc(Op->DestVector.ID()); - const auto SrcVector = GetSrc(Op->SrcVector.ID()); - - // Actually performs the insertion from the source vector into the destination vector. - const auto PerformInsertion = [this, ElementSize](const Xbyak::Xmm& Src, uint32_t SrcIndex, - const Xbyak::Xmm& Dest, uint32_t DestIndex) { - switch (ElementSize) { - case 1: { - pextrb(eax, Src, SrcIndex); - pinsrb(Dest, eax, DestIndex); - break; - } - case 2: { - pextrw(eax, Src, SrcIndex); - pinsrw(Dest, eax, DestIndex); - break; - } - case 4: { - pextrd(eax, Src, SrcIndex); - pinsrd(Dest, eax, DestIndex); - break; - } - case 8: { - pextrq(rax, Src, SrcIndex); - pinsrq(Dest, rax, DestIndex); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } - }; - - if (Is256Bit) { - // Whether or not the index is in the upper lane. - const auto IsUpperIdx = [ElementSize](uint32_t Index) { - return Index >= (16U / ElementSize); - }; - - const auto SrcIsUpper = IsUpperIdx(SrcIdx); - const auto DstIsUpper = IsUpperIdx(DestIdx); - - // Sanitizes indices based on whether or not its accessing the upper lane. - const auto SanitizeIndex = [ElementSize](uint32_t Index, bool IsUpper) -> uint32_t { - if (IsUpper) { - return Index - (16U / ElementSize); - } - return Index; - }; - - // Helpers to get the source and destination vectors depending on - // whether or not the top or bottom lane is accessed. Takes a temporary - // to move the extracted lane into. - const auto GetSrcVector = [this, SrcIsUpper, &SrcVector](const Xbyak::Xmm& Tmp) { - if (SrcIsUpper) { - vextracti128(Tmp, ToYMM(SrcVector), 1); - return Tmp; - } else { - return SrcVector; - } - }; - // Always move the dst into a temporary, since regardless of the outcome - // we need to insert it back into the original vector. - const auto GetDstVector = [this, DstIsUpper, &DestVector](const Xbyak::Xmm& Tmp) { - if (DstIsUpper) { - vextracti128(Tmp, ToYMM(DestVector), 1); - return Tmp; - } else { - vmovaps(Tmp, DestVector); - return Tmp; - } - }; - - const auto Is128BitElement = ElementSize == Core::CPUState::XMM_SSE_REG_SIZE; - if (Is128BitElement) { - vextracti128(xmm14, ToYMM(SrcVector), SrcIdx); - vmovapd(ToYMM(Dst), ToYMM(DestVector)); - vinserti128(ToYMM(Dst), ToYMM(Dst), xmm14, DestIdx); - } else { - const auto SrcReg = GetSrcVector(xmm14); - const auto DstReg = GetDstVector(xmm15); - - const auto SanitizedDstIdx = SanitizeIndex(DestIdx, DstIsUpper); - const auto SanitizedSrcIdx = SanitizeIndex(SrcIdx, SrcIsUpper); - - PerformInsertion(SrcReg, SanitizedSrcIdx, DstReg, SanitizedDstIdx); - - vmovapd(ToYMM(Dst), ToYMM(DestVector)); - if (DstIsUpper) { - vinserti128(ToYMM(Dst), ToYMM(Dst), DstReg, 1); - } else { - vinserti128(ToYMM(Dst), ToYMM(Dst), DstReg, 0); - } - } - } else { - vmovapd(xmm15, DestVector); - PerformInsertion(SrcVector, SrcIdx, xmm15, DestIdx); - vmovapd(Dst, xmm15); - } -} - -DEF_OP(VDupElement) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - auto Index = Op->Index; - const auto ElementSize = Op->Header.ElementSize; - const auto ElementSizeBits = ElementSize * 8; - const auto BitOffset = ElementSizeBits * Index; - - constexpr auto AVXRegSize = Core::CPUState::XMM_AVX_REG_SIZE; - constexpr auto AVXBitSize = AVXRegSize * 8; - constexpr auto SSERegSize = Core::CPUState::XMM_SSE_REG_SIZE; - constexpr auto SSEBitSize = SSERegSize * 8; - - const auto Is256Bit = OpSize == AVXRegSize; - const auto IsInUpperLane = BitOffset >= SSEBitSize; - - // Currently we only handle offsets within 256-bit registers at most. - if (BitOffset >= AVXBitSize) { - LOGMAN_MSG_A_FMT("Bit offset for element outside maximum range: Offset={}", - BitOffset); - return; - } - - // If we try to access the upper 128-bit lane on a 128-bit operation - // then we have some invalid values being passed in. - if (!Is256Bit && IsInUpperLane) { - LOGMAN_MSG_A_FMT("Accessing upper 128-bit lane in 128-bit operation: " - "ElementSize={}, Index={}", - ElementSize, Index); - return; - } - - // If we're in the upper lane, we need to adjust the index down, - // since we operate on halves here. - if (IsInUpperLane) { - Index -= 16U / ElementSize; - } - - const auto Dst = GetDst(Node); - const auto Vector = GetSrc(Op->Vector.ID()); - - switch (ElementSize) { - case 1: { - if (Is256Bit && IsInUpperLane) { - vextracti128(xmm15, ToYMM(Vector), 1); - } else { - vmovapd(xmm15, Vector); - } - - // First extract the indexed byte and insert it - // into the first element of the destination. - pextrb(eax, xmm15, Index); - pinsrb(Dst, eax, 0); - - // Now replicate. - if (Is256Bit) { - vpbroadcastb(ToYMM(Dst), Dst); - } else { - vpbroadcastb(Dst, Dst); - } - break; - } - case 2: { - if (Is256Bit && IsInUpperLane) { - vextracti128(xmm15, ToYMM(Vector), 1); - } else { - vmovapd(xmm15, Vector); - } - - // First extract the indexed word and insert it - // into the first element of the destination. - pextrw(eax, xmm15, Index); - pinsrw(Dst, eax, 0); - - if (Is256Bit) { - vpbroadcastw(ToYMM(Dst), Dst); - } else { - vpbroadcastw(Dst, Dst); - } - break; - } - case 4: { - if (IsInUpperLane) { - vextracti128(xmm15, ToYMM(Vector), 1); - vpshufd(Dst, xmm15, - (Index << 0) | - (Index << 2) | - (Index << 4) | - (Index << 6)); - } else { - vpshufd(Dst, Vector, - (Index << 0) | - (Index << 2) | - (Index << 4) | - (Index << 6)); - } - if (Is256Bit) { - vinserti128(ToYMM(Dst), ToYMM(Dst), Dst, 1); - } - break; - } - case 8: { - if (IsInUpperLane) { - vextracti128(xmm15, ToYMM(Vector), 1); - vshufpd(Dst, xmm15, xmm15, - (Index << 0) | - (Index << 1)); - } else { - vshufpd(Dst, Vector, Vector, - (Index << 0) | - (Index << 1)); - } - if (Is256Bit) { - vinserti128(ToYMM(Dst), ToYMM(Dst), Dst, 1); - } - break; - } - case 16: { - LOGMAN_THROW_AA_FMT(Is256Bit, - "Can't perform 128-bit broadcast with 128-bit operation: {}", - OpSize); - - vmovapd(ToYMM(Dst), ToYMM(Vector)); - if (IsInUpperLane) { - vextracti128(xmm15, ToYMM(Dst), 1); - vinserti128(ToYMM(Dst), ToYMM(Dst), xmm15, 0); - } else { - vinserti128(ToYMM(Dst), ToYMM(Dst), Dst, 1); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } -} - - - -DEF_OP(VExtr) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - constexpr auto AVXRegSize = Core::CPUState::XMM_AVX_REG_SIZE; - const auto Is256Bit = OpSize == AVXRegSize; - - const auto Index = Op->Index; - const auto ElementSize = Op->Header.ElementSize; - const auto ByteIndex = uint32_t{Index} * ElementSize; - - const auto Dst = GetDst(Node); - const auto VectorLower = GetSrc(Op->VectorLower.ID()); - const auto VectorUpper = GetSrc(Op->VectorUpper.ID()); - - if (Is256Bit) { - const auto IsUpperVectorZero = ByteIndex >= OpSize; - const auto SanitizedByteIndex = IsUpperVectorZero ? ByteIndex - OpSize - : ByteIndex; - - // Ensure we don't load junk outside of the space we're about to make. - LOGMAN_THROW_AA_FMT(SanitizedByteIndex <= 31, "Invalid VExtr byte index: {}", - SanitizedByteIndex); - - const auto RegsSavedSize = AVXRegSize * 2; - sub(rsp, RegsSavedSize); - - if (IsUpperVectorZero) { - vpxor(xmm15, xmm15, xmm15); - vmovups(ptr[rsp + 0 * AVXRegSize], ToYMM(VectorLower)); - vmovups(ptr[rsp + 1 * AVXRegSize], ymm15); - } else { - vmovups(ptr[rsp + 0 * AVXRegSize], ToYMM(VectorUpper)); - vmovups(ptr[rsp + 1 * AVXRegSize], ToYMM(VectorLower)); - } - - vmovups(ToYMM(Dst), ptr[rsp + SanitizedByteIndex]); - - add(rsp, RegsSavedSize); - } else if (OpSize == 8) { - // No way to do this with 64bit source without dropping to MMX - // So emulate it - vpxor(xmm14, xmm14, xmm14); - movq(xmm15, VectorUpper); - vshufpd(xmm15, xmm15, VectorLower, 0b00); - vpalignr(Dst, xmm14, xmm15, ByteIndex); - } else { - vpalignr(Dst, VectorLower, VectorUpper, ByteIndex); - } -} - -DEF_OP(VUShrI) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto BitShift = Op->BitShift; - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector = GetSrc(Op->Vector.ID()); - - switch (ElementSize) { - case 2: { - if (Is256Bit) { - vpsrlw(ToYMM(Dst), ToYMM(Vector), BitShift); - } else { - vpsrlw(Dst, Vector, BitShift); - } - break; - } - case 4: { - if (Is256Bit) { - vpsrld(ToYMM(Dst), ToYMM(Vector), BitShift); - } else { - vpsrld(Dst, Vector, BitShift); - } - break; - } - case 8: { - if (Is256Bit) { - vpsrlq(ToYMM(Dst), ToYMM(Vector), BitShift); - } else { - vpsrlq(Dst, Vector, BitShift); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } -} - -DEF_OP(VSShrI) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto BitShift = Op->BitShift; - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dest = GetDst(Node); - const auto Vector = GetSrc(Op->Vector.ID()); - - switch (ElementSize) { - case 1: { - // This isn't a native instruction on x86 - const auto PerformShifts = [&](const Xbyak::Xmm& reg) { - for (int i = 0; i < int(Core::CPUState::XMM_SSE_REG_SIZE); ++i) { - pextrb(eax, reg, i); - movsx(eax, al); - sar(al, BitShift); - pinsrb(reg, eax, i); - } - }; - - if (Is256Bit) { - vmovapd(ToYMM(Dest), ToYMM(Vector)); - PerformShifts(Dest); - vextractf128(xmm15, ToYMM(Dest), 1); - PerformShifts(xmm15); - vinsertf128(ToYMM(Dest), ToYMM(Dest), xmm15, 1); - } else { - vmovapd(Dest, Vector); - PerformShifts(Dest); - } - break; - } - case 2: { - if (Is256Bit) { - vpsraw(ToYMM(Dest), ToYMM(Vector), BitShift); - } else { - vpsraw(Dest, Vector, BitShift); - } - break; - } - case 4: { - if (Is256Bit) { - vpsrad(ToYMM(Dest), ToYMM(Vector), BitShift); - } else { - vpsrad(Dest, Vector, BitShift); - } - break; - } - case 8: { - // VPSRAQ is only introduced in AVX-512, so we can't use it. - const auto PerformShifts = [&](const Xbyak::Xmm& reg) { - for (int i = 0; i < 2; ++i) { - pextrq(rax, reg, i); - sar(rax, BitShift); - pinsrq(reg, rax, i); - } - }; - - if (Is256Bit) { - vmovapd(ToYMM(Dest), ToYMM(Vector)); - PerformShifts(Dest); - vextractf128(xmm15, ToYMM(Dest), 1); - PerformShifts(xmm15); - vinsertf128(ToYMM(Dest), ToYMM(Dest), xmm15, 1); - } else { - vmovapd(Dest, Vector); - PerformShifts(Dest); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } -} - -DEF_OP(VShlI) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto BitShift = Op->BitShift; - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector = GetSrc(Op->Vector.ID()); - - switch (ElementSize) { - case 1: { - const auto Mask = 0xFFU >> BitShift; - - mov(rax, Mask); - vmovq(xmm15, rax); - - if (Is256Bit) { - vpsllw(ToYMM(Dst), ToYMM(Vector), BitShift); - vpbroadcastb(ymm15, xmm15); - vpand(ToYMM(Dst), ToYMM(Dst), ymm15); - } else { - vpsllw(Dst, Vector, BitShift); - vpbroadcastb(xmm15, xmm15); - vpand(Dst, Dst, ymm15); - } - break; - } - case 2: { - if (Is256Bit) { - vpsllw(ToYMM(Dst), ToYMM(Vector), BitShift); - } else { - vpsllw(Dst, Vector, BitShift); - } - break; - } - case 4: { - if (Is256Bit) { - vpslld(ToYMM(Dst), ToYMM(Vector), BitShift); - } else { - vpslld(Dst, Vector, BitShift); - } - break; - } - case 8: { - if (Is256Bit) { - vpsllq(ToYMM(Dst), ToYMM(Vector), BitShift); - } else { - vpsllq(Dst, Vector, BitShift); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } -} - -DEF_OP(VUShrNI) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto BitShift = Op->BitShift; - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto DstYMM = ToYMM(Dst); - const auto Vector = GetSrc(Op->Vector.ID()); - - vmovapd(DstYMM, ToYMM(Vector)); - vpxor(xmm15, xmm15, xmm15); - - switch (ElementSize) { - case 1: { - if (Is256Bit) { - vpsrlw(DstYMM, DstYMM, BitShift); - } else { - vpsrlw(Dst, Dst, BitShift); - } - // <8 x i16> -> <8 x i8> - mov(rax, 0x0E'0C'0A'08'06'04'02'00); // Lower - mov(rcx, 0x80'80'80'80'80'80'80'80); // Upper - break; - } - case 2: { - if (Is256Bit) { - vpsrld(DstYMM, DstYMM, BitShift); - } else { - vpsrld(Dst, Dst, BitShift); - } - // <4 x i32> -> <4 x i16> - mov(rax, 0x0D'0C'09'08'05'04'01'00); // Lower - mov(rcx, 0x80'80'80'80'80'80'80'80); // Upper - break; - } - case 4: { - if (Is256Bit) { - vpsrlq(DstYMM, DstYMM, BitShift); - } else { - vpsrlq(Dst, Dst, BitShift); - } - // <2 x i64> -> <2 x i32> - mov(rax, 0x0B'0A'09'08'03'02'01'00); // Lower - mov(rcx, 0x80'80'80'80'80'80'80'80); // Upper - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } - - vmovq(xmm15, rax); - vmovq(xmm14, rcx); - punpcklqdq(xmm15, xmm14); - if (Is256Bit) { - vinserti128(ymm15, ymm15, xmm15, 1); - vpshufb(DstYMM, DstYMM, ymm15); - vextracti128(xmm14, DstYMM, 1); - punpcklqdq(Dst, xmm14); - } else { - vpshufb(Dst, Dst, xmm15); - } -} - -DEF_OP(VUShrNI2) { - // Src1 = Lower results - // Src2 = Upper Results - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto BitShift = Op->BitShift; - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto VectorLower = GetSrc(Op->VectorLower.ID()); - const auto VectorUpper = GetSrc(Op->VectorUpper.ID()); - - vmovapd(ymm13, ToYMM(VectorUpper)); - switch (ElementSize) { - case 1: { - if (Is256Bit) { - vpsrlw(ymm13, ymm13, BitShift); - } else { - vpsrlw(xmm13, xmm13, BitShift); - } - // <8 x i16> -> <8 x i8> - mov(rax, 0x80'80'80'80'80'80'80'80); // Lower - mov(rcx, 0x0E'0C'0A'08'06'04'02'00); // Upper - break; - } - case 2: { - if (Is256Bit) { - vpsrld(ymm13, ymm13, BitShift); - } else { - vpsrld(xmm13, xmm13, BitShift); - } - // <4 x i32> -> <4 x i16> - mov(rax, 0x80'80'80'80'80'80'80'80); // Lower - mov(rcx, 0x0D'0C'09'08'05'04'01'00); // Upper - break; - } - case 4: { - if (Is256Bit) { - vpsrlq(ymm13, ymm13, BitShift); - } else { - vpsrlq(xmm13, xmm13, BitShift); - } - // <2 x i64> -> <2 x i32> - mov(rax, 0x80'80'80'80'80'80'80'80); // Lower - mov(rcx, 0x0B'0A'09'08'03'02'01'00); // Upper - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } - - vmovq(xmm15, rax); - vmovq(xmm14, rcx); - punpcklqdq(xmm15, xmm14); - if (Is256Bit) { - vinserti128(ymm15, ymm15, xmm15, 1); - vpshufb(ymm14, ymm13, ymm15); - vextracti128(xmm12, ymm14, 1); - punpckhqdq(xmm14, xmm12); - - vmovapd(Dst, VectorLower); - vinserti128(ToYMM(Dst), ToYMM(Dst), xmm14, 1); - } else { - vpshufb(xmm14, xmm13, xmm15); - vpor(Dst, xmm14, VectorLower); - } -} - -DEF_OP(VSXTL) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector = GetSrc(Op->Vector.ID()); - - switch (ElementSize) { - case 2: - if (Is256Bit) { - vpmovsxbw(ToYMM(Dst), Vector); - } else { - vpmovsxbw(Dst, Vector); - } - break; - case 4: - if (Is256Bit) { - vpmovsxwd(ToYMM(Dst), Vector); - } else { - vpmovsxwd(Dst, Vector); - } - break; - case 8: - if (Is256Bit) { - vpmovsxdq(ToYMM(Dst), Vector); - } else { - vpmovsxdq(Dst, Vector); - } - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } -} - -DEF_OP(VSXTL2) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector = GetSrc(Op->Vector.ID()); - - if (Is256Bit) { - const auto DstYMM = ToYMM(Dst); - - vextracti128(Dst, ToYMM(Vector), 1); - switch (ElementSize) { - case 2: - vpmovsxbw(DstYMM, Dst); - break; - case 4: - vpmovsxwd(DstYMM, Dst); - break; - case 8: - vpmovsxdq(DstYMM, Dst); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } - } else { - vpsrldq(Dst, Vector, OpSize / 2); - switch (ElementSize) { - case 2: - vpmovsxbw(Dst, Dst); - break; - case 4: - vpmovsxwd(Dst, Dst); - break; - case 8: - vpmovsxdq(Dst, Dst); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } - } -} - -DEF_OP(VUXTL) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector = GetSrc(Op->Vector.ID()); - - switch (ElementSize) { - case 2: - if (Is256Bit) { - vpmovzxbw(ToYMM(Dst), Vector); - } else { - vpmovzxbw(Dst, Vector); - } - break; - case 4: - if (Is256Bit) { - vpmovzxwd(ToYMM(Dst), Vector); - } else { - vpmovzxwd(Dst, Vector); - } - break; - case 8: - if (Is256Bit) { - vpmovzxdq(ToYMM(Dst), Vector); - } else { - vpmovzxdq(Dst, Vector); - } - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } -} - -DEF_OP(VUXTL2) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector = GetSrc(Op->Vector.ID()); - - if (Is256Bit) { - const auto DstYMM = ToYMM(Dst); - - vextracti128(Dst, ToYMM(Vector), 1); - switch (ElementSize) { - case 2: - vpmovzxbw(DstYMM, Dst); - break; - case 4: - vpmovzxwd(DstYMM, Dst); - break; - case 8: - vpmovzxdq(DstYMM, Dst); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } - } else { - vpsrldq(Dst, Vector, OpSize / 2); - switch (ElementSize) { - case 2: - vpmovzxbw(Dst, Dst); - break; - case 4: - vpmovzxwd(Dst, Dst); - break; - case 8: - vpmovzxdq(Dst, Dst); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } - } -} - -DEF_OP(VSQXTN) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector = GetSrc(Op->Vector.ID()); - - if (Is256Bit) { - switch (ElementSize) { - case 1: - vpacksswb(ToYMM(Dst), ToYMM(Vector), ToYMM(Vector)); - break; - case 2: - vpackssdw(ToYMM(Dst), ToYMM(Vector), ToYMM(Vector)); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } - - vextracti128(xmm15, ToYMM(Dst), 1); - pslldq(xmm15, 8); - vmovq(Dst, Dst); - vpor(Dst, Dst, xmm15); - } else { - switch (ElementSize) { - case 1: - vpacksswb(Dst, Vector, Vector); - break; - case 2: - vpackssdw(Dst, Vector, Vector); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } - - // Clear upper 64-bits. - movq(Dst, Dst); - } -} - -DEF_OP(VSQXTN2) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto VectorLower = GetSrc(Op->VectorLower.ID()); - const auto VectorUpper = GetSrc(Op->VectorUpper.ID()); - - // Zero the lower bits - vpxor(xmm15, xmm15, xmm15); - - if (Is256Bit) { - switch (ElementSize) { - case 1: - vpacksswb(ymm15, ToYMM(VectorUpper), ToYMM(VectorUpper)); - break; - case 2: - vpackssdw(ymm15, ToYMM(VectorUpper), ToYMM(VectorUpper)); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } - - vextracti128(xmm14, ymm15, 1); - pslldq(xmm14, 8); - vmovq(xmm15, xmm15); - vpor(xmm15, xmm15, xmm14); - - vmovaps(Dst, VectorLower); - vinserti128(ToYMM(Dst), ToYMM(Dst), xmm15, 1); - } else { - switch (ElementSize) { - case 1: - vpacksswb(xmm15, xmm15, VectorUpper); - break; - case 2: - vpackssdw(xmm15, xmm15, VectorUpper); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } - - if (OpSize == 8) { - psrldq(xmm15, OpSize / 2); - } - vpor(Dst, VectorLower, xmm15); - } -} - -DEF_OP(VSQXTNPair) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto VectorLower = GetSrc(Op->VectorLower.ID()); - const auto VectorUpper = GetSrc(Op->VectorUpper.ID()); - - if (Is256Bit) { - //ud2(); - switch (ElementSize) { - case 1: - vpacksswb(ymm15, ToYMM(VectorLower), ToYMM(VectorUpper)); - break; - case 2: - vpackssdw(ymm15, ToYMM(VectorLower), ToYMM(VectorUpper)); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } - // 256-bit vpack interleaves 128-bit halves - // This needs to be uninterleaved. - // Src1[127:0] > Dest[63:0] - Src1_l - // Src2[127:0] > Dest[127:64] - Src2_l - // Src1[255:128] > Dest[191:128] - Src1_h - // Src2[255:128] > Dest[255:192] - Src2_h - vmovaps(ToYMM(Dst), ymm15); - // Lower bits still in ymm15 - // Extract upper bits - vextracti128(xmm14, ymm15, 1); - - pextrq(TMP1, xmm14, 0); - pinsrq(ToYMM(Dst), TMP1, 1); - - pextrq(TMP1, xmm15, 1); - pinsrq(xmm14, TMP1, 0); - vinserti128(ToYMM(Dst), ToYMM(Dst), xmm14, 1); - } - else if (OpSize == 8) { - vpxor(xmm15, xmm15, xmm15); - vpunpcklqdq(xmm14, VectorLower, VectorUpper); - switch (ElementSize) { - case 1: - vpacksswb(Dst, xmm14, xmm15); - break; - case 2: - vpackssdw(Dst, xmm14, xmm15); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } - } - else { - switch (ElementSize) { - case 1: - vpacksswb(Dst, VectorLower, VectorUpper); - break; - case 2: - vpackssdw(Dst, VectorLower, VectorUpper); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } - } -} - -DEF_OP(VSQXTUN) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector = GetSrc(Op->Vector.ID()); - - if (Is256Bit) { - switch (ElementSize) { - case 1: - vpackuswb(ToYMM(Dst), ToYMM(Vector), ToYMM(Vector)); - break; - case 2: - vpackusdw(ToYMM(Dst), ToYMM(Vector), ToYMM(Vector)); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } - - vextracti128(xmm15, ToYMM(Dst), 1); - pslldq(xmm15, 8); - vmovq(Dst, Dst); - vpor(Dst, Dst, xmm15); - } else { - switch (ElementSize) { - case 1: - vpackuswb(Dst, Vector, Vector); - break; - case 2: - vpackusdw(Dst, Vector, Vector); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } - // Clear upper 64 bits. - movq(Dst, Dst); - } -} - -DEF_OP(VSQXTUN2) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto VectorLower = GetSrc(Op->VectorLower.ID()); - const auto VectorUpper = GetSrc(Op->VectorUpper.ID()); - - // Zero the lower bits - vpxor(xmm15, xmm15, xmm15); - - if (Is256Bit) { - switch (ElementSize) { - case 1: - vpackuswb(ymm15, ToYMM(VectorUpper), ToYMM(VectorUpper)); - break; - case 2: - vpackusdw(ymm15, ToYMM(VectorUpper), ToYMM(VectorUpper)); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } - - vextracti128(xmm14, ymm15, 1); - pslldq(xmm14, 8); - vmovq(xmm15, xmm15); - vpor(xmm15, xmm15, xmm14); - - vmovaps(Dst, VectorLower); - vinserti128(ToYMM(Dst), ToYMM(Dst), xmm15, 1); - } else { - switch (ElementSize) { - case 1: - vpackuswb(xmm15, xmm15, VectorUpper); - break; - case 2: - vpackusdw(xmm15, xmm15, VectorUpper); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } - if (OpSize == 8) { - psrldq(xmm15, OpSize / 2); - } - - vpor(Dst, VectorLower, xmm15); - } -} - -DEF_OP(VSQXTUNPair) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto VectorLower = GetSrc(Op->VectorLower.ID()); - const auto VectorUpper = GetSrc(Op->VectorUpper.ID()); - - if (Is256Bit) { - switch (ElementSize) { - case 1: - vpackuswb(ymm15, ToYMM(VectorLower), ToYMM(VectorUpper)); - break; - case 2: - vpackusdw(ymm15, ToYMM(VectorLower), ToYMM(VectorUpper)); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } - // 256-bit vpack interleaves 128-bit halves - // This needs to be uninterleaved. - // Src1[127:0] > Dest[63:0] - Src1_l - // Src2[127:0] > Dest[127:64] - Src2_l - // Src1[255:128] > Dest[191:128] - Src1_h - // Src2[255:128] > Dest[255:192] - Src2_h - vmovaps(ToYMM(Dst), ymm15); - // Lower bits still in ymm15 - // Extract upper bits - vextracti128(xmm14, ymm15, 1); - - pextrq(TMP1, xmm14, 0); - pinsrq(ToYMM(Dst), TMP1, 1); - - pextrq(TMP1, xmm15, 1); - pinsrq(xmm14, TMP1, 0); - vinserti128(ToYMM(Dst), ToYMM(Dst), xmm14, 1); - } - else if (OpSize == 8) { - vpxor(xmm15, xmm15, xmm15); - vpunpcklqdq(xmm14, VectorLower, VectorUpper); - switch (ElementSize) { - case 1: - vpackuswb(Dst, xmm14, xmm15); - break; - case 2: - vpackusdw(Dst, xmm14, xmm15); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } - } - else { - switch (ElementSize) { - case 1: - vpackuswb(Dst, VectorLower, VectorUpper); - break; - case 2: - vpackusdw(Dst, VectorLower, VectorUpper); - break; - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - return; - } - } -} - -DEF_OP(VMul) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector1 = GetSrc(Op->Vector1.ID()); - const auto Vector2 = GetSrc(Op->Vector2.ID()); - - switch (ElementSize) { - case 2: { - if (Is256Bit) { - vpmullw(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpmullw(Dst, Vector1, Vector2); - } - break; - } - case 4: { - if (Is256Bit) { - vpmulld(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } else { - vpmulld(Dst, Vector1, Vector2); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } -} - -DEF_OP(VUMull) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector1 = GetSrc(Op->Vector1.ID()); - const auto Vector2 = GetSrc(Op->Vector2.ID()); - - switch (ElementSize) { - case 4: { - // IR operation (128-bit): - // [31:00 ] = src1[15:00] * src2[15:00] - // [63:32 ] = src1[31:16] * src2[31:16] - // [95:64 ] = src1[47:32] * src2[47:32] - // [127:96] = src1[63:48] * src2[63:48] - // - vpxor(xmm15, xmm15, xmm15); - vpxor(xmm14, xmm14, xmm14); - - if (Is256Bit) { - vpxor(xmm13, xmm13, xmm13); - vpxor(xmm12, xmm12, xmm12); - - vpunpckhwd(xmm13, Vector1, xmm13); - vpunpckhwd(xmm12, Vector2, xmm12); - } - - vpunpcklwd(xmm15, Vector1, xmm15); - vpunpcklwd(xmm14, Vector2, xmm14); - - if (Is256Bit) { - vinserti128(ymm15, ymm15, xmm13, 1); - vinserti128(ymm14, ymm14, xmm12, 1); - vpmulld(ToYMM(Dst), ymm14, ymm15); - } else { - vpmulld(Dst, xmm14, xmm15); - } - break; - } - case 8: { - // We need to shuffle the data for this one - // x86 PMULUDQ wants the 32bit values in [31:0] and [95:64] - // Which then extends out to [63:0] and [127:64] - vpshufd(xmm14, Vector1, 0b01'01'00'00); - vpshufd(xmm15, Vector2, 0b01'01'00'00); - - if (Is256Bit) { - vpshufd(xmm13, Vector1, 0b11'11'10'10); - vpshufd(xmm12, Vector2, 0b11'11'10'10); - - vinserti128(ymm14, ymm14, xmm13, 1); - vinserti128(ymm15, ymm15, xmm12, 1); - - vpmuludq(ToYMM(Dst), ymm14, ymm15); - } else { - vpmuludq(Dst, xmm14, xmm15); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } -} - -DEF_OP(VSMull) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector1 = GetSrc(Op->Vector1.ID()); - const auto Vector2 = GetSrc(Op->Vector2.ID()); - - switch (ElementSize) { - case 4: { - // IR operation (128-bit): - // [31:00 ] = src1[15:00] * src2[15:00] - // [63:32 ] = src1[31:16] * src2[31:16] - // [95:64 ] = src1[47:32] * src2[47:32] - // [127:96] = src1[63:48] * src2[63:48] - // - vpxor(xmm15, xmm15, xmm15); - vpxor(xmm14, xmm14, xmm14); - - if (Is256Bit) { - vpxor(xmm13, xmm13, xmm13); - vpxor(xmm12, xmm12, xmm12); - - vpunpckhwd(xmm13, Vector1, xmm13); - vpunpckhwd(xmm12, Vector2, xmm12); - } - - vpunpcklwd(xmm15, Vector1, xmm15); - vpunpcklwd(xmm14, Vector2, xmm14); - - if (Is256Bit) { - const auto SignExtend = [this](const Xbyak::Ymm& reg) { - vpslld(reg, reg, 16); - vpsrad(reg, reg, 16); - }; - - vinserti128(ymm15, ymm15, xmm13, 1); - vinserti128(ymm14, ymm14, xmm12, 1); - - SignExtend(ymm15); - SignExtend(ymm14); - - vpmulld(ToYMM(Dst), ymm14, ymm15); - } else { - const auto SignExtend = [this](const Xbyak::Xmm& reg) { - pslld(reg, 16); - psrad(reg, 16); - }; - - SignExtend(xmm15); - SignExtend(xmm14); - - vpmulld(Dst, xmm14, xmm15); - } - break; - } - case 8: { - // We need to shuffle the data for this one - // x86 PMULDQ wants the 32bit values in [31:0] and [95:64] - // Which then extends out to [63:0] and [127:64] - vpshufd(xmm14, Vector1, 0b01'01'00'00); - vpshufd(xmm15, Vector2, 0b01'01'00'00); - - if (Is256Bit) { - vpshufd(xmm13, Vector1, 0b11'11'10'10); - vpshufd(xmm12, Vector2, 0b11'11'10'10); - - vinserti128(ymm14, ymm14, xmm13, 1); - vinserti128(ymm15, ymm15, xmm12, 1); - - vpmuldq(ToYMM(Dst), ymm14, ymm15); - } else { - vpmuldq(Dst, xmm14, xmm15); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } -} - -DEF_OP(VUMull2) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector1 = GetSrc(Op->Vector1.ID()); - const auto Vector2 = GetSrc(Op->Vector2.ID()); - - switch (ElementSize) { - case 4: { - // IR operation (128-bit): - // [31:00 ] = src1[79:64 ] * src2[79:64 ] - // [63:32 ] = src1[95:80 ] * src2[95:80 ] - // [95:64 ] = src1[111:96 ] * src2[111:96 ] - // [127:96] = src1[127:112] * src2[127:112] - // - vpxor(xmm15, xmm15, xmm15); - vpxor(xmm14, xmm14, xmm14); - - if (Is256Bit) { - vextracti128(xmm15, ToYMM(Vector1), 1); - vextracti128(xmm14, ToYMM(Vector2), 1); - - vpxor(xmm12, xmm12, xmm12); - vpxor(xmm13, xmm13, xmm13); - vpxor(Dst, Dst, Dst); - - // Bottom half - vpunpcklwd(xmm13, xmm15, xmm13); - vpunpcklwd(xmm12, xmm14, xmm12); - - // Top half - vpunpckhwd(xmm15, xmm15, Dst); - vpunpckhwd(xmm14, xmm14, Dst); - - // Reinsert - vinserti128(ymm13, ymm13, xmm15, 1); - vinserti128(ymm12, ymm12, xmm14, 1); - - vpmulld(ToYMM(Dst), ymm12, ymm13); - } else { - vpunpckhwd(xmm15, Vector1, xmm15); - vpunpckhwd(xmm14, Vector2, xmm14); - vpmulld(Dst, xmm14, xmm15); - } - break; - } - case 8: { - // IR operation (128-bit): - // [63:00 ] = src1[95:64 ] * src2[95:64 ] - // [127:64] = src1[127:96] * src2[127:96] - // - // x86 vpmuludq - // [63:00 ] = src1[31:0 ] * src2[31:0 ] - // [127:64] = src1[95:64] * src2[95:64] - - if (Is256Bit) { - vextracti128(xmm14, ToYMM(Vector1), 1); - vextracti128(xmm15, ToYMM(Vector2), 1); - - vpshufd(xmm12, xmm14, 0b01'01'00'00); - vpshufd(xmm13, xmm15, 0b01'01'00'00); - - vpshufd(xmm14, xmm14, 0b11'11'10'10); - vpshufd(xmm15, xmm15, 0b11'11'10'10); - - vinserti128(ymm12, ymm12, xmm14, 1); - vinserti128(ymm13, ymm13, xmm15, 1); - - vpmuludq(ToYMM(Dst), ymm12, ymm13); - } else { - vpshufd(xmm14, Vector1, 0b11'11'10'10); - vpshufd(xmm15, Vector2, 0b11'11'10'10); - - vpmuludq(Dst, xmm14, xmm15); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } -} - -DEF_OP(VSMull2) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector1 = GetSrc(Op->Vector1.ID()); - const auto Vector2 = GetSrc(Op->Vector2.ID()); - - switch (ElementSize) { - case 4: { - // IR operation (128-bit): - // [31:00 ] = src1[79:64 ] * src2[79:64 ] - // [63:32 ] = src1[95:80 ] * src2[95:80 ] - // [95:64 ] = src1[111:96 ] * src2[111:96 ] - // [127:96] = src1[127:112] * src2[127:112] - // - vpxor(xmm15, xmm15, xmm15); - vpxor(xmm14, xmm14, xmm14); - - if (Is256Bit) { - const auto SignExtend = [this](const Xbyak::Ymm& reg) { - vpslld(reg, reg, 16); - vpsrad(reg, reg, 16); - }; - - vextracti128(xmm15, ToYMM(Vector1), 1); - vextracti128(xmm14, ToYMM(Vector2), 1); - - vpxor(xmm12, xmm12, xmm12); - vpxor(xmm13, xmm13, xmm13); - vpxor(Dst, Dst, Dst); - - // Bottom half - vpunpcklwd(xmm13, xmm15, xmm13); - vpunpcklwd(xmm12, xmm14, xmm12); - - // Top half - vpunpckhwd(xmm15, xmm15, Dst); - vpunpckhwd(xmm14, xmm14, Dst); - - // Reinsert - vinserti128(ymm13, ymm13, xmm15, 1); - vinserti128(ymm12, ymm12, xmm14, 1); - - SignExtend(ymm13); - SignExtend(ymm12); - - vpmulld(ToYMM(Dst), ymm12, ymm13); - } else { - const auto SignExtend = [this](const Xbyak::Xmm& reg) { - pslld(reg, 16); - psrad(reg, 16); - }; - - vpunpckhwd(xmm15, Vector1, xmm15); - vpunpckhwd(xmm14, Vector2, xmm14); - - SignExtend(xmm15); - SignExtend(xmm14); - - vpmulld(Dst, xmm14, xmm15); - } - break; - } - case 8: { - // IR operation (128-bit): - // [63:00 ] = src1[95:64 ] * src2[95:64 ] - // [127:64] = src1[127:96] * src2[127:96] - // - // x86 vpmuludq - // [63:00 ] = src1[31:0 ] * src2[31:0 ] - // [127:64] = src1[95:64] * src2[95:64] - - if (Is256Bit) { - vextracti128(xmm14, ToYMM(Vector1), 1); - vextracti128(xmm15, ToYMM(Vector2), 1); - - vpshufd(xmm12, xmm14, 0b01'01'00'00); - vpshufd(xmm13, xmm15, 0b01'01'00'00); - - vpshufd(xmm14, xmm14, 0b11'11'10'10); - vpshufd(xmm15, xmm15, 0b11'11'10'10); - - vinserti128(ymm12, ymm12, xmm14, 1); - vinserti128(ymm13, ymm13, xmm15, 1); - - vpmuldq(ToYMM(Dst), ymm12, ymm13); - } else { - vpshufd(xmm14, Vector1, 0b11'11'10'10); - vpshufd(xmm15, Vector2, 0b11'11'10'10); - - vpmuldq(Dst, xmm14, xmm15); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } -} - -DEF_OP(VUABDL) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector1 = GetSrc(Op->Vector1.ID()); - const auto Vector2 = GetSrc(Op->Vector2.ID()); - - switch (ElementSize) { - case 2: { - if (Is256Bit) { - vpmovzxbw(ymm14, Vector1); - vpmovzxbw(ymm15, Vector2); - vpsubw(ToYMM(Dst), ymm14, ymm15); - vpabsw(ToYMM(Dst), ToYMM(Dst)); - } else { - vpmovzxbw(xmm14, Vector1); - vpmovzxbw(xmm15, Vector2); - vpsubw(Dst, xmm14, xmm15); - vpabsw(Dst, Dst); - } - break; - } - case 4: { - if (Is256Bit) { - vpmovzxwd(ymm14, Vector1); - vpmovzxwd(ymm15, Vector2); - vpsubd(ToYMM(Dst), ymm14, ymm15); - vpabsd(ToYMM(Dst), ToYMM(Dst)); - } else { - vpmovzxwd(xmm14, Vector1); - vpmovzxwd(xmm15, Vector2); - vpsubd(Dst, xmm14, xmm15); - vpabsd(Dst, Dst); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } -} - -DEF_OP(VUABDL2) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector1 = GetSrc(Op->Vector1.ID()); - const auto Vector2 = GetSrc(Op->Vector2.ID()); - - switch (ElementSize) { - case 2: { - vpxor(xmm15, xmm15, xmm15); - vpxor(xmm14, xmm14, xmm14); - - if (Is256Bit) { - vextracti128(xmm15, ToYMM(Vector1), 1); - vextracti128(xmm14, ToYMM(Vector2), 1); - - vpxor(xmm12, xmm12, xmm12); - vpxor(xmm13, xmm13, xmm13); - vpxor(Dst, Dst, Dst); - - // Bottom half - vpunpcklbw(xmm13, xmm15, xmm13); - vpunpcklbw(xmm12, xmm14, xmm12); - - // Top half - vpunpckhbw(xmm15, xmm15, Dst); - vpunpckhbw(xmm14, xmm14, Dst); - - // Reinsert - vinserti128(ymm13, ymm13, xmm15, 1); - vinserti128(ymm12, ymm12, xmm14, 1); - - vpsubw(ToYMM(Dst), ymm12, ymm13); - vpabsw(ToYMM(Dst), ToYMM(Dst)); - } else { - vpunpckhbw(xmm15, Vector1, xmm15); - vpunpckhbw(xmm14, Vector2, xmm14); - vpsubw(Dst, xmm14, xmm15); - vpabsw(Dst, Dst); - } - break; - } - case 4: { - vpxor(xmm15, xmm15, xmm15); - vpxor(xmm14, xmm14, xmm14); - - if (Is256Bit) { - vextracti128(xmm15, ToYMM(Vector1), 1); - vextracti128(xmm14, ToYMM(Vector2), 1); - - vpxor(xmm12, xmm12, xmm12); - vpxor(xmm13, xmm13, xmm13); - vpxor(Dst, Dst, Dst); - - // Bottom half - vpunpcklwd(xmm13, xmm15, xmm13); - vpunpcklwd(xmm12, xmm14, xmm12); - - // Top half - vpunpckhwd(xmm15, xmm15, Dst); - vpunpckhwd(xmm14, xmm14, Dst); - - // Reinsert - vinserti128(ymm13, ymm13, xmm15, 1); - vinserti128(ymm12, ymm12, xmm14, 1); - - vpsubd(ToYMM(Dst), ymm12, ymm13); - vpabsd(ToYMM(Dst), ToYMM(Dst)); - } else { - vpunpckhwd(xmm15, Vector1, xmm15); - vpunpckhwd(xmm14, Vector2, xmm14); - vpsubd(Dst, xmm14, xmm15); - vpabsd(Dst, Dst); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } -} - -DEF_OP(VUMulH) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector1 = GetSrc(Op->Vector1.ID()); - const auto Vector2 = GetSrc(Op->Vector2.ID()); - - switch (ElementSize) { - case 2: { - if (Is256Bit) { - vpmulhuw(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } - else { - vpmulhuw(Dst, Vector1, Vector2); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } -} -DEF_OP(VSMulH) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - - const auto Dst = GetDst(Node); - const auto Vector1 = GetSrc(Op->Vector1.ID()); - const auto Vector2 = GetSrc(Op->Vector2.ID()); - - switch (ElementSize) { - case 2: { - if (Is256Bit) { - vpmulhw(ToYMM(Dst), ToYMM(Vector1), ToYMM(Vector2)); - } - else { - vpmulhw(Dst, Vector1, Vector2); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } -} - -DEF_OP(VTBL1) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto Dst = GetDst(Node); - const auto VectorIndices = GetSrc(Op->VectorIndices.ID()); - const auto VectorTable = GetSrc(Op->VectorTable.ID()); - - switch (OpSize) { - case 8: { - vpshufb(Dst, VectorTable, VectorIndices); - vmovq(Dst, Dst); - break; - } - case 16: { - vpshufb(Dst, VectorTable, VectorIndices); - break; - } - case 32: { - // AVX2 vpshufb is a coward and doesn't lane cross, so we - // need to get a little creative. - const auto DstYMM = ToYMM(Dst); - const auto VectorIndicesYMM = ToYMM(VectorIndices); - const auto VectorTableYMM = ToYMM(VectorTable); - - // Permute the bottom lane of the table into a register - // and do the same for the upper lane. - vperm2i128(ymm15, VectorTableYMM, VectorTableYMM, 0b0000'0000); - vperm2i128(ymm14, VectorTableYMM, VectorTableYMM, 0b0001'0001); - - // Shuffle away - vpshufb(ymm15, ymm15, VectorIndicesYMM); - vpshufb(ymm14, ymm14, VectorIndicesYMM); - - // Set up our comparison register - mov(eax, 16); - vmovd(xmm13, eax); - vpbroadcastb(ymm13, xmm13); - - // Now create our control mask - vpcmpgtb(ymm12, ymm13, VectorIndicesYMM); - - // And finally blend everything together - // from the high and low halves depending - // on the control mask. - vpblendvb(DstYMM, ymm14, ymm15, ymm12); - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown OpSize: {}", OpSize); - break; - } -} - -DEF_OP(VRev32) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - - const auto Dst = GetDst(Node); - const auto Vector = GetSrc(Op->Vector.ID()); - - switch (ElementSize) { - case 1: { - mov(rax, 0x04'05'06'07'00'01'02'03); // Lower - vmovq(xmm15, rax); - if (OpSize >= Core::CPUState::XMM_SSE_REG_SIZE) { - // Full 8bit byteswap in each 32-bit element - mov(rcx, 0x0C'0D'0E'0F'08'09'0A'0B); // Upper - pinsrq(xmm15, rcx, 1); - } - else { - // 8byte, upper bits get zero - // Full 8bit byteswap in each 32-bit element - mov(rcx, 0x80'80'80'80'80'80'80'80); // Upper - pinsrq(xmm15, rcx, 1); - } - - if (OpSize == Core::CPUState::XMM_AVX_REG_SIZE) { - // Replicate to the upper lane - vinsertf128(ymm15, ymm15, xmm15, 1); - vpshufb(ToYMM(Dst), ToYMM(Vector), ymm15); - } else { - vpshufb(Dst, Vector, xmm15); - } - break; - } - case 2: { - // Full 16-bit byteswap in each 32-bit element - mov(rax, 0x05'04'07'06'01'00'03'02); // Lower - vmovq(xmm15, rax); - if (OpSize >= Core::CPUState::XMM_SSE_REG_SIZE) { - mov(rcx, 0x0D'0C'0F'0E'09'08'0B'0A); // Upper - pinsrq(xmm15, rcx, 1); - } - else { - // 8byte, upper bits get zero - // Full 8bit byteswap in each 32-bit element - mov(rcx, 0x80'80'80'80'80'80'80'80); // Upper - pinsrq(xmm15, rcx, 1); - } - - if (OpSize == Core::CPUState::XMM_AVX_REG_SIZE) { - // Replicate to the upper lane - vinsertf128(ymm15, ymm15, xmm15, 1); - vpshufb(ToYMM(Dst), ToYMM(Vector), ymm15); - } else { - vpshufb(Dst, Vector, xmm15); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } -} - -DEF_OP(VRev64) { - const auto Op = IROp->C(); - const auto OpSize = IROp->Size; - - const auto ElementSize = Op->Header.ElementSize; - - const auto Dst = GetDst(Node); - const auto Vector = GetSrc(Op->Vector.ID()); - - switch (ElementSize) { - case 1: { - mov(rax, 0x00'01'02'03'04'05'06'07); // Lower - vmovq(xmm15, rax); - if (OpSize >= Core::CPUState::XMM_SSE_REG_SIZE) { - // Full 8bit byteswap in each 64-bit element - mov(rcx, 0x08'09'0A'0B'0C'0D'0E'0F); // Upper - pinsrq(xmm15, rcx, 1); - } - else { - // 8byte, upper bits get zero - // Full 8bit byteswap in each 64-bit element - mov(rcx, 0x80'80'80'80'80'80'80'80); // Upper - pinsrq(xmm15, rcx, 1); - } - - if (OpSize == Core::CPUState::XMM_AVX_REG_SIZE) { - // Replicate to the upper lane - vinsertf128(ymm15, ymm15, xmm15, 1); - vpshufb(ToYMM(Dst), ToYMM(Vector), ymm15); - } else { - vpshufb(Dst, Vector, xmm15); - } - break; - } - case 2: { - // Full 16-bit byteswap in each 64-bit element - mov(rax, 0x01'00'03'02'05'04'07'06); // Lower - vmovq(xmm15, rax); - if (OpSize >= Core::CPUState::XMM_SSE_REG_SIZE) { - mov(rcx, 0x09'08'0B'0A'0D'0C'0F'0E); // Upper - pinsrq(xmm15, rcx, 1); - } - else { - // 8byte, upper bits get zero - // Full 8bit byteswap in each 64-bit element - mov(rcx, 0x80'80'80'80'80'80'80'80); // Upper - pinsrq(xmm15, rcx, 1); - } - - if (OpSize == Core::CPUState::XMM_AVX_REG_SIZE) { - // Replicate to the upper lane - vinsertf128(ymm15, ymm15, xmm15, 1); - vpshufb(ToYMM(Dst), ToYMM(Vector), ymm15); - } else { - vpshufb(Dst, Vector, xmm15); - } - break; - } - case 4: { - if (OpSize == Core::CPUState::XMM_AVX_REG_SIZE) { - // Lower - mov(rax, 0x03'02'01'00'07'06'05'04); - vmovq(xmm15, rax); - - // Upper - mov(rax, 0x0B'0A'09'08'0F'0E'0D'0C); - pinsrq(xmm15, rax, 1); - - // Replicate to upper lane - vinsertf128(ymm15, ymm15, xmm15, 1); - - // And finish it off. - vpshufb(ToYMM(Dst), ToYMM(Vector), ymm15); - } else if (OpSize == Core::CPUState::XMM_SSE_REG_SIZE) { - vpshufd(Dst, - Vector, - (0b01 << 0) | - (0b00 << 2) | - (0b11 << 4) | - (0b10 << 6)); - } else { - vpshufd(Dst, - Vector, - (0b01 << 0) | - (0b00 << 2) | - (0b11 << 4) | // Last two don't matter, will be overwritten with zero - (0b11 << 6)); - - // Zero upper 64-bits - mov(rcx, 0); - pinsrq(Dst, rcx, 1); - } - break; - } - default: - LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); - break; - } -} - -#undef DEF_OP -void X86JITCore::RegisterVectorHandlers() { -#define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &X86JITCore::Op_##x - REGISTER_OP(VECTORZERO, VectorZero); - REGISTER_OP(VECTORIMM, VectorImm); - REGISTER_OP(LOADNAMEDVECTORCONSTANT, LoadNamedVectorConstant); - REGISTER_OP(LOADNAMEDVECTORINDEXEDCONSTANT, LoadNamedVectorIndexedConstant); - REGISTER_OP(VMOV, VMov); - REGISTER_OP(VAND, VAnd); - REGISTER_OP(VBIC, VBic); - REGISTER_OP(VOR, VOr); - REGISTER_OP(VXOR, VXor); - REGISTER_OP(VADD, VAdd); - REGISTER_OP(VSUB, VSub); - REGISTER_OP(VUQADD, VUQAdd); - REGISTER_OP(VUQSUB, VUQSub); - REGISTER_OP(VSQADD, VSQAdd); - REGISTER_OP(VSQSUB, VSQSub); - REGISTER_OP(VADDP, VAddP); - REGISTER_OP(VADDV, VAddV); - REGISTER_OP(VUMINV, VUMinV); - REGISTER_OP(VURAVG, VURAvg); - REGISTER_OP(VABS, VAbs); - REGISTER_OP(VPOPCOUNT, VPopcount); - REGISTER_OP(VFADD, VFAdd); - REGISTER_OP(VFADDP, VFAddP); - REGISTER_OP(VFSUB, VFSub); - REGISTER_OP(VFMUL, VFMul); - REGISTER_OP(VFDIV, VFDiv); - REGISTER_OP(VFMIN, VFMin); - REGISTER_OP(VFMAX, VFMax); - REGISTER_OP(VFRECP, VFRecp); - REGISTER_OP(VFSQRT, VFSqrt); - REGISTER_OP(VFRSQRT, VFRSqrt); - REGISTER_OP(VNEG, VNeg); - REGISTER_OP(VFNEG, VFNeg); - REGISTER_OP(VNOT, VNot); - REGISTER_OP(VUMIN, VUMin); - REGISTER_OP(VSMIN, VSMin); - REGISTER_OP(VUMAX, VUMax); - REGISTER_OP(VSMAX, VSMax); - REGISTER_OP(VZIP, VZip); - REGISTER_OP(VZIP2, VZip2); - REGISTER_OP(VUNZIP, VUnZip); - REGISTER_OP(VUNZIP2, VUnZip2); - REGISTER_OP(VTRN, VTrn); - REGISTER_OP(VTRN2, VTrn2); - REGISTER_OP(VBSL, VBSL); - REGISTER_OP(VCMPEQ, VCMPEQ); - REGISTER_OP(VCMPEQZ, VCMPEQZ); - REGISTER_OP(VCMPGT, VCMPGT); - REGISTER_OP(VCMPGTZ, VCMPGTZ); - REGISTER_OP(VCMPLTZ, VCMPLTZ); - REGISTER_OP(VFCMPEQ, VFCMPEQ); - REGISTER_OP(VFCMPNEQ, VFCMPNEQ); - REGISTER_OP(VFCMPLT, VFCMPLT); - REGISTER_OP(VFCMPGT, VFCMPGT); - REGISTER_OP(VFCMPLE, VFCMPLE); - REGISTER_OP(VFCMPORD, VFCMPORD); - REGISTER_OP(VFCMPUNO, VFCMPUNO); - REGISTER_OP(VUSHL, VUShl); - REGISTER_OP(VUSHR, VUShr); - REGISTER_OP(VSSHR, VSShr); - REGISTER_OP(VUSHLS, VUShlS); - REGISTER_OP(VUSHRS, VUShrS); - REGISTER_OP(VSSHRS, VSShrS); - REGISTER_OP(VUSHRSWIDE, VUShrS); - REGISTER_OP(VSSHRSWIDE, VSShrS); - REGISTER_OP(VUSHLSWIDE, VUShlS); - REGISTER_OP(VINSELEMENT, VInsElement); - REGISTER_OP(VDUPELEMENT, VDupElement); - REGISTER_OP(VEXTR, VExtr); - REGISTER_OP(VUSHRI, VUShrI); - REGISTER_OP(VSSHRI, VSShrI); - REGISTER_OP(VSHLI, VShlI); - REGISTER_OP(VUSHRNI, VUShrNI); - REGISTER_OP(VUSHRNI2, VUShrNI2); - REGISTER_OP(VSXTL, VSXTL); - REGISTER_OP(VSXTL2, VSXTL2); - REGISTER_OP(VUXTL, VUXTL); - REGISTER_OP(VUXTL2, VUXTL2); - REGISTER_OP(VSQXTN, VSQXTN); - REGISTER_OP(VSQXTN2, VSQXTN2); - REGISTER_OP(VSQXTNPAIR, VSQXTNPair); - REGISTER_OP(VSQXTUN, VSQXTUN); - REGISTER_OP(VSQXTUN2, VSQXTUN2); - REGISTER_OP(VSQXTUNPAIR, VSQXTUNPair); - REGISTER_OP(VUMUL, VMul); - REGISTER_OP(VSMUL, VMul); - REGISTER_OP(VUMULL, VUMull); - REGISTER_OP(VSMULL, VSMull); - REGISTER_OP(VUMULL2, VUMull2); - REGISTER_OP(VSMULL2, VSMull2); - REGISTER_OP(VUMULH, VUMulH); - REGISTER_OP(VSMULH, VSMulH); - REGISTER_OP(VUABDL, VUABDL); - REGISTER_OP(VUABDL2, VUABDL2); - REGISTER_OP(VTBL1, VTBL1); - REGISTER_OP(VREV32, VRev32); - REGISTER_OP(VREV64, VRev64); -#undef REGISTER_OP -} -} - diff --git a/FEXCore/Source/Interface/Core/JIT/x86_64/x64Relocations.cpp b/FEXCore/Source/Interface/Core/JIT/x86_64/x64Relocations.cpp deleted file mode 100644 index 08940fc65..000000000 --- a/FEXCore/Source/Interface/Core/JIT/x86_64/x64Relocations.cpp +++ /dev/null @@ -1,140 +0,0 @@ -// SPDX-License-Identifier: MIT -/* -$info$ -tags: backend|x86-64 -desc: relocation logic of the x86-64 splatter backend -$end_info$ -*/ -#include "Interface/Context/Context.h" -#include "Interface/Core/JIT/x86_64/JITClass.h" -#include "Interface/HLE/Thunks/Thunks.h" - -namespace FEXCore::CPU { -uint64_t X86JITCore::GetNamedSymbolLiteral(FEXCore::CPU::RelocNamedSymbolLiteral::NamedSymbol Op) { - switch (Op) { - case FEXCore::CPU::RelocNamedSymbolLiteral::NamedSymbol::SYMBOL_LITERAL_EXITFUNCTION_LINKER: - return ThreadState->CurrentFrame->Pointers.Common.ExitFunctionLinker; - break; - default: - ERROR_AND_DIE_FMT("Unknown named symbol literal: {}", static_cast(Op)); - break; - } - return ~0ULL; - -} - -void X86JITCore::LoadConstantWithPadding(Xbyak::Reg Reg, uint64_t Constant) { - // The maximum size a move constant can be in bytes - // Need to NOP pad to this size to ensure backpatching is always the same size - // Calculated as: - // [Rex] - // [Mov op] - // [8 byte constant] - // - // All other move types are smaller than this. xbyak will use a NOP slide which is quite quick - constexpr static size_t MAX_MOVE_SIZE = 10; - auto StartingOffset = getSize(); - mov(Reg, Constant); - auto MoveSize = getSize() - StartingOffset; - auto NOPPadSize = MAX_MOVE_SIZE - MoveSize; - nop(NOPPadSize); -} - -X86JITCore::NamedSymbolLiteralPair X86JITCore::InsertNamedSymbolLiteral(FEXCore::CPU::RelocNamedSymbolLiteral::NamedSymbol Op) { - NamedSymbolLiteralPair Lit { - .MoveABI = { - .NamedSymbolLiteral = { - .Header = { - .Type = FEXCore::CPU::RelocationTypes::RELOC_NAMED_SYMBOL_LITERAL, - }, - .Symbol = Op, - .Offset = 0, - }, - }, - }; - return Lit; -} - -void X86JITCore::PlaceNamedSymbolLiteral(NamedSymbolLiteralPair &Lit) { - // Offset is the offset from the entrypoint of the block - auto CurrentCursor = getSize(); - Lit.MoveABI.NamedSymbolLiteral.Offset = CurrentCursor - CursorEntry; - - uint64_t Pointer = GetNamedSymbolLiteral(Lit.MoveABI.NamedSymbolLiteral.Symbol); - - L(Lit.Offset); - dq(Pointer); - Relocations.emplace_back(Lit.MoveABI); -} - - -void X86JITCore::InsertGuestRIPMove(Xbyak::Reg Reg, uint64_t Constant) { - Relocation MoveABI{}; - MoveABI.GuestRIPMove.Header.Type = FEXCore::CPU::RelocationTypes::RELOC_GUEST_RIP_MOVE; - - // Offset is the offset from the entrypoint of the block - auto CurrentCursor = getSize(); - MoveABI.GuestRIPMove.Offset = CurrentCursor - CursorEntry; - MoveABI.GuestRIPMove.GuestRIP = Constant; - MoveABI.GuestRIPMove.RegisterIndex = Reg.getIdx(); - - if (CTX->Config.CacheObjectCodeCompilation()) { - LoadConstantWithPadding(Reg, Constant); - } - else { - mov(Reg, Constant); - } - - Relocations.emplace_back(MoveABI); -} - -bool X86JITCore::ApplyRelocations(uint64_t GuestEntry, uint64_t CodeEntry, uint64_t CursorEntry, size_t NumRelocations, const char* EntryRelocations) { - size_t DataIndex{}; - for (size_t j = 0; j < NumRelocations; ++j) { - const FEXCore::CPU::Relocation *Reloc = reinterpret_cast(&EntryRelocations[DataIndex]); - LOGMAN_THROW_AA_FMT((DataIndex % alignof(Relocation)) == 0, "Alignment of relocation wasn't adhered to"); - - switch (Reloc->Header.Type) { - case FEXCore::CPU::RelocationTypes::RELOC_NAMED_SYMBOL_LITERAL: { - uint64_t Pointer = GetNamedSymbolLiteral(Reloc->NamedSymbolLiteral.Symbol); - // Relocation occurs at the cursorEntry + offset relative to that cursor. - setSize(CursorEntry + Reloc->NamedSymbolLiteral.Offset); - - // Place the pointer - dq(Pointer); - - DataIndex += sizeof(Reloc->NamedSymbolLiteral); - break; - } - case FEXCore::CPU::RelocationTypes::RELOC_NAMED_THUNK_MOVE: { - uint64_t Pointer = reinterpret_cast(CTX->ThunkHandler->LookupThunk(Reloc->NamedThunkMove.Symbol)); - if (Pointer == ~0ULL) { - return false; - } - - // Relocation occurs at the cursorEntry + offset relative to that cursor. - setSize(CursorEntry + Reloc->NamedThunkMove.Offset); - LoadConstantWithPadding(Xbyak::Reg64(Reloc->NamedThunkMove.RegisterIndex), Pointer); - DataIndex += sizeof(Reloc->NamedThunkMove); - break; - } - case FEXCore::CPU::RelocationTypes::RELOC_GUEST_RIP_MOVE: - // XXX: Reenable once the JIT Object Cache is upstream - // XXX: Should spin the relocation list, create a list of guest RIP moves, and ask for them all once, reduces lock contention. - uint64_t Pointer = ~0ULL; // EmitterCTX->JITObjectCache->FindRelocatedRIP(Reloc->GuestRIPMove.GuestRIP); - if (Pointer == ~0ULL) { - return false; - } - - // Relocation occurs at the cursorEntry + offset relative to that cursor. - setSize(CursorEntry + Reloc->GuestRIPMove.Offset); - LoadConstantWithPadding(Xbyak::Reg64(Reloc->GuestRIPMove.RegisterIndex), Pointer); - DataIndex += sizeof(Reloc->GuestRIPMove); - break; - } - } - - return true; -} -} - diff --git a/FEXCore/Source/Interface/IR/Passes/ConstProp.cpp b/FEXCore/Source/Interface/IR/Passes/ConstProp.cpp index 11a1a467c..822d84cc1 100644 --- a/FEXCore/Source/Interface/IR/Passes/ConstProp.cpp +++ b/FEXCore/Source/Interface/IR/Passes/ConstProp.cpp @@ -7,13 +7,11 @@ $end_info$ */ -#if JIT_ARM64 //aarch64 heuristics #include "aarch64/assembler-aarch64.h" #include "aarch64/cpu-aarch64.h" #include "aarch64/disasm-aarch64.h" #include "aarch64/assembler-aarch64.h" -#endif #include "Interface/IR/PassManager.h" @@ -59,23 +57,12 @@ static bool HasConsecutiveBits(uint64_t imm, unsigned width) { return ((imm ^ (imm >> 1)) & ((1ULL << (width - 1)) - 1)) == 0; } -#if JIT_ARM64 //aarch64 heuristics static bool IsImmLogical(uint64_t imm, unsigned width) { if (width < 32) width = 32; return vixl::aarch64::Assembler::IsImmLogical(imm, width); } static bool IsImmAddSub(uint64_t imm) { return vixl::aarch64::Assembler::IsImmAddSub(imm); } static bool IsMemoryScale(uint64_t Scale, uint8_t AccessSize) { return Scale == AccessSize; } -#elif JIT_X86_64 -// very lazy heuristics -static bool IsImmLogical(uint64_t imm, unsigned width) { return imm < 0x8000'0000; } -static bool IsImmAddSub(uint64_t imm) { return imm < 0x8000'0000; } -static bool IsMemoryScale(uint64_t Scale, uint8_t AccessSize) { - return Scale == 1 || Scale == 2 || Scale == 4 || Scale == 8; -} -#else -#error No inline constant heuristics for this target -#endif static bool IsImmMemory(uint64_t imm, uint8_t AccessSize) { if ( ((int64_t)imm >= -255) && ((int64_t)imm <= 256) ) diff --git a/unittests/32Bit_ASM/CMakeLists.txt b/unittests/32Bit_ASM/CMakeLists.txt index 0e2d53460..5070afed9 100644 --- a/unittests/32Bit_ASM/CMakeLists.txt +++ b/unittests/32Bit_ASM/CMakeLists.txt @@ -48,14 +48,18 @@ foreach(ASM_SRC ${ASM_SOURCES}) list(APPEND ASM_DEPENDS "${OUTPUT_NAME};${OUTPUT_CONFIG_NAME}") - # Format is "" "" - set(TEST_ARGS - "--no-silent -g -c irjit -n 1 --no-multiblock" "jit_1" "jit" - "--no-silent -g -c irjit -n 500 --no-multiblock" "jit_500" "jit" - "--no-silent -g -c irjit -n 500 --multiblock" "jit_500_m" "jit" - ) + set(TEST_ARGS) + if (_M_ARM_64 OR ENABLE_VIXL_SIMULATOR) + list(APPEND TEST_ARGS + "--no-silent -g -c irjit -n 1 --no-multiblock" "jit_1" "jit" + "--no-silent -g -c irjit -n 500 --no-multiblock" "jit_500" "jit" + "--no-silent -g -c irjit -n 500 --multiblock" "jit_500_m" "jit" + ) + endif() - if (_M_X86_64) + if (ENABLE_VIXL_SIMULATOR) + set(CPU_CLASS Simulator) + elseif (_M_X86_64) list(APPEND TEST_ARGS "--no-silent -g -c host" "host" "host" ) diff --git a/unittests/ASM/CMakeLists.txt b/unittests/ASM/CMakeLists.txt index 2be2dfc64..36de82246 100644 --- a/unittests/ASM/CMakeLists.txt +++ b/unittests/ASM/CMakeLists.txt @@ -49,20 +49,22 @@ foreach(ASM_SRC ${ASM_SOURCES}) list(APPEND ASM_DEPENDS "${OUTPUT_NAME};${OUTPUT_CONFIG_NAME}") # Format is "" "" "" - set(TEST_ARGS - "--no-silent -g -c irjit -n 1 --no-multiblock" "jit_1" "jit" - "--no-silent -g -c irjit -n 500 --no-multiblock" "jit_500" "jit" - "--no-silent -g -c irjit -n 500 --multiblock" "jit_500_m" "jit" - ) + + set(TEST_ARGS) + if (_M_ARM_64 OR ENABLE_VIXL_SIMULATOR) + list(APPEND TEST_ARGS + "--no-silent -g -c irjit -n 1 --no-multiblock" "jit_1" "jit" + "--no-silent -g -c irjit -n 500 --no-multiblock" "jit_500" "jit" + "--no-silent -g -c irjit -n 500 --multiblock" "jit_500_m" "jit" + ) + endif() if (ENABLE_VIXL_SIMULATOR) set(CPU_CLASS Simulator) - else() - if (_M_X86_64) - list(APPEND TEST_ARGS - "--no-silent -g -c host" "host" "host" - ) - endif() + elseif (_M_X86_64) + list(APPEND TEST_ARGS + "--no-silent -g -c host" "host" "host" + ) endif() if (NOT MINGW_BUILD) diff --git a/unittests/CMakeLists.txt b/unittests/CMakeLists.txt index eee4941f2..a8f040a46 100644 --- a/unittests/CMakeLists.txt +++ b/unittests/CMakeLists.txt @@ -19,7 +19,7 @@ endif() add_subdirectory(ASM/) add_subdirectory(32Bit_ASM/) -if (ENABLE_VIXL_DISASSEMBLER AND (ENABLE_JIT_ARM64 OR CMAKE_SYSTEM_PROCESSOR MATCHES "^aarch64|^arm64|^armv8\.*") AND NOT ENABLE_JIT_X86_64) +if (ENABLE_VIXL_DISASSEMBLER AND (ENABLE_JIT_ARM64 OR CMAKE_SYSTEM_PROCESSOR MATCHES "^aarch64|^arm64|^armv8\.*")) # Tests are only valid to run if the vixl disassembler is enabled and the active JIT is the ARM64 JIT. add_subdirectory(InstructionCountCI/) endif()