mirror of
https://github.com/FEX-Emu/FEX.git
synced 2026-10-07 01:00:17 +02:00
Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ea20429351 | ||
|
|
91828efa7a | ||
|
|
cce605d5e0 |
No files matched your search
+2
-2
@@ -32,7 +32,7 @@ AttributeMacros:
|
||||
BinPackArguments: true
|
||||
BinPackParameters: true
|
||||
BitFieldColonSpacing: Both
|
||||
BreakAfterAttributes: Leave
|
||||
BreakAfterAttributes: Always # clang 16 required
|
||||
BreakBeforeBraces: Attach
|
||||
BreakBeforeBinaryOperators: None
|
||||
BreakBeforeInlineASMColon: OnlyMultiline # clang 16 required
|
||||
@@ -60,7 +60,7 @@ IndentRequires: false
|
||||
IndentWidth: 2
|
||||
InsertBraces: true
|
||||
KeepEmptyLinesAtTheStartOfBlocks: true
|
||||
LambdaBodyIndentation: Signature
|
||||
LambdaBodyIndentation: OuterScope
|
||||
LineEnding: LF # clang 16 required
|
||||
MaxEmptyLinesToKeep: 2
|
||||
NamespaceIndentation: Inner
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
# This file is used to ignore files and directories from clang-format
|
||||
|
||||
# Ignore all files in the External directory
|
||||
External/*
|
||||
|
||||
Source/Common/cpp-optparse/*
|
||||
|
||||
# Files with human-indented tables for readability - don't mess with these
|
||||
@@ -7,6 +11,3 @@ FEXCore/Source/Interface/Core/X86Tables/*
|
||||
# Inline headers with list-like content that can't be processed individually
|
||||
Source/Tools/LinuxEmulation/LinuxSyscalls/x*/SyscallsNames.inl
|
||||
Source/Tools/LinuxEmulation/LinuxSyscalls/x*/Ioctl/*.inl
|
||||
|
||||
# Include files in unittests
|
||||
unittests/*ASM/Includes/*.inc
|
||||
@@ -16,9 +16,3 @@
|
||||
|
||||
# Reformat of CodeEmitter inl files
|
||||
8760c593ece92d7e9fa94c40da0368fd367c9cad
|
||||
|
||||
# Whole-tree reformat with clang-format-19
|
||||
5267cde60e7642852d18f20ae8568643bb5293d5
|
||||
|
||||
# Minor reformat with clang-format-19
|
||||
9fdd96af61c969cb5732471223f00eda64b7a069
|
||||
@@ -13,7 +13,6 @@ env:
|
||||
BUILD_TYPE: Release
|
||||
CC: clang
|
||||
CXX: clang++
|
||||
FEX_PORTABLE: 1
|
||||
|
||||
jobs:
|
||||
build_plus_test:
|
||||
@@ -34,6 +33,7 @@ jobs:
|
||||
echo "FEX_ROOTFS_MOUNT=/mnt/AutoNFS/rootfs/" >> $GITHUB_ENV
|
||||
echo "FEX_ROOTFS_PATH=$HOME/Rootfs/" >> $GITHUB_ENV
|
||||
echo "FEX_ROOTFS=$HOME/Rootfs/" >> $GITHUB_ENV
|
||||
echo "ROOTFS=$HOME/Rootfs/" >> $GITHUB_ENV
|
||||
|
||||
- name: Update RootFS cache
|
||||
# Use a bash shell so we can use the same syntax for environment variable
|
||||
@@ -136,9 +136,6 @@ jobs:
|
||||
- name: FEXLinuxTests
|
||||
working-directory: ${{runner.workspace}}/build
|
||||
shell: bash
|
||||
env:
|
||||
# These tests require non-portable install due to thunks.
|
||||
FEX_PORTABLE: 0
|
||||
run: cmake --build . --config $BUILD_TYPE --target fex_linux_tests_all
|
||||
|
||||
- name: FEXLinuxTests Results move
|
||||
|
||||
@@ -20,7 +20,6 @@ env:
|
||||
BUILD_TYPE: Release
|
||||
CC: clang
|
||||
CXX: clang++
|
||||
FEX_PORTABLE: 1
|
||||
|
||||
jobs:
|
||||
glibc_fault_test:
|
||||
@@ -41,6 +40,7 @@ jobs:
|
||||
echo "FEX_ROOTFS_MOUNT=/mnt/AutoNFS/rootfs/" >> $GITHUB_ENV
|
||||
echo "FEX_ROOTFS_PATH=$HOME/Rootfs/" >> $GITHUB_ENV
|
||||
echo "FEX_ROOTFS=$HOME/Rootfs/" >> $GITHUB_ENV
|
||||
echo "ROOTFS=$HOME/Rootfs/" >> $GITHUB_ENV
|
||||
|
||||
- name: Update RootFS cache
|
||||
# Use a bash shell so we can use the same syntax for environment variable
|
||||
|
||||
@@ -13,7 +13,6 @@ env:
|
||||
BUILD_TYPE: Release
|
||||
CC: clang
|
||||
CXX: clang++
|
||||
FEX_PORTABLE: 1
|
||||
|
||||
jobs:
|
||||
hostrunner_tests:
|
||||
@@ -34,6 +33,7 @@ jobs:
|
||||
echo "FEX_ROOTFS_MOUNT=/mnt/AutoNFS/rootfs/" >> $GITHUB_ENV
|
||||
echo "FEX_ROOTFS_PATH=$HOME/Rootfs/" >> $GITHUB_ENV
|
||||
echo "FEX_ROOTFS=$HOME/Rootfs/" >> $GITHUB_ENV
|
||||
echo "ROOTFS=$HOME/Rootfs/" >> $GITHUB_ENV
|
||||
|
||||
- name: Update RootFS cache
|
||||
# Use a bash shell so we can use the same syntax for environment variable
|
||||
|
||||
@@ -33,6 +33,7 @@ jobs:
|
||||
echo "FEX_ROOTFS_MOUNT=/mnt/AutoNFS/rootfs/" >> $GITHUB_ENV
|
||||
echo "FEX_ROOTFS_PATH=$HOME/Rootfs/" >> $GITHUB_ENV
|
||||
echo "FEX_ROOTFS=$HOME/Rootfs/" >> $GITHUB_ENV
|
||||
echo "ROOTFS=$HOME/Rootfs/" >> $GITHUB_ENV
|
||||
|
||||
- name: Update RootFS cache
|
||||
# Use a bash shell so we can use the same syntax for environment variable
|
||||
|
||||
@@ -48,6 +48,7 @@ jobs:
|
||||
echo "FEX_ROOTFS_MOUNT=/mnt/AutoNFS/rootfs/" >> $GITHUB_ENV
|
||||
echo "FEX_ROOTFS_PATH=$HOME/Rootfs/" >> $GITHUB_ENV
|
||||
echo "FEX_ROOTFS=$HOME/Rootfs/" >> $GITHUB_ENV
|
||||
echo "ROOTFS=$HOME/Rootfs/" >> $GITHUB_ENV
|
||||
|
||||
- name: Update RootFS cache
|
||||
# Use a bash shell so we can use the same syntax for environment variable
|
||||
@@ -77,7 +78,7 @@ jobs:
|
||||
# Note the current convention is to use the -S and -B options here to specify source
|
||||
# and build directories, but this is only available with CMake 3.13 and higher.
|
||||
# The CMake binaries on the Github Actions machines are (as of this writing) 3.12
|
||||
run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DCMAKE_TOOLCHAIN_FILE=$GITHUB_WORKSPACE/Data/CMake/toolchain_mingw.cmake -DMINGW_TRIPLE=$MINGW_TRIPLE -G Ninja -DENABLE_LTO=False -DENABLE_ASSERTIONS=True -DENABLE_X86_HOST_DEBUG=True -DBUILD_TESTING=False -DCMAKE_INSTALL_PREFIX=${{runner.workspace}}/build/install
|
||||
run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DCMAKE_TOOLCHAIN_FILE=$GITHUB_WORKSPACE/Data/CMake/toolchain_mingw.cmake -DMINGW_TRIPLE=$MINGW_TRIPLE -G Ninja -DENABLE_LTO=False -DENABLE_ASSERTIONS=True -DENABLE_X86_HOST_DEBUG=True -DBUILD_TESTS=False -DCMAKE_INSTALL_PREFIX=${{runner.workspace}}/build/install
|
||||
|
||||
- name: Build
|
||||
working-directory: ${{runner.workspace}}/build
|
||||
|
||||
@@ -40,8 +40,11 @@ jobs:
|
||||
echo "Formatting files:"
|
||||
echo "$CHANGED_FILES"
|
||||
|
||||
- name: Check git-clang-format-19 exists
|
||||
run: which git-clang-format-19
|
||||
- name: Check for correct clang-format version
|
||||
run: clang-format --version | grep -qF '16.0.6'
|
||||
|
||||
- name: Check git-clang-format-16 exists
|
||||
run: which git-clang-format-16
|
||||
|
||||
- name: Setup Python env
|
||||
uses: actions/setup-python@v4
|
||||
@@ -55,15 +58,19 @@ jobs:
|
||||
|
||||
- name: Run code formatter
|
||||
env:
|
||||
CLANG_FORMAT_PATH: 'git-clang-format-19'
|
||||
CLANG_FORMAT_PATH: 'git-clang-format-16'
|
||||
GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
START_REV: ${{ github.event.pull_request.base.sha }}
|
||||
END_REV: ${{ github.event.pull_request.head.sha }}
|
||||
CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
|
||||
# TODO(pmatos): Once we adopt v18, we should be able
|
||||
# to take advantage of the new --diff_from_common_commit option
|
||||
# explicitly in code-format-helper.py and not have to diff starting at
|
||||
# the merge base.
|
||||
run: |
|
||||
python ./External/code-format-helper/code-format-helper.py \
|
||||
--repo "FEX-emu/FEX" \
|
||||
--issue-number $GITHUB_PR_NUMBER \
|
||||
--start-rev $START_REV \
|
||||
--start-rev $(git merge-base $START_REV $END_REV) \
|
||||
--end-rev $END_REV \
|
||||
--changed-files "$CHANGED_FILES"
|
||||
@@ -1,79 +0,0 @@
|
||||
name: steamrt4 build
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
|
||||
env:
|
||||
BUILD_TYPE: Release
|
||||
CC: clang
|
||||
CXX: clang++
|
||||
|
||||
jobs:
|
||||
steamrt4_build:
|
||||
runs-on: ${{ matrix.arch }}
|
||||
strategy:
|
||||
matrix:
|
||||
arch: [[self-hosted, ARM64, distrobox]]
|
||||
fail-fast: false
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Set runner label
|
||||
run: echo "runner_label=${{ matrix.arch[1] }}" >> $GITHUB_ENV
|
||||
|
||||
- name : submodule checkout
|
||||
run: |
|
||||
git submodule sync --recursive
|
||||
git submodule update --init --depth 1
|
||||
|
||||
- name: Clean Build Environment
|
||||
run: |
|
||||
rm -Rf ${{runner.workspace}}/build
|
||||
cmake -E make_directory ${{runner.workspace}}/build
|
||||
|
||||
# Setup everything required.
|
||||
- name : distrobox setup
|
||||
run: |
|
||||
distrobox create -Y -i registry.gitlab.steamos.cloud/steamrt/steamrt4/sdk/arm64:4.0.20251117.183306 steamrt4 || true
|
||||
distrobox upgrade steamrt4
|
||||
distrobox enter --name steamrt4 -- sudo apt-get install -y \
|
||||
git cmake ninja-build ccache \
|
||||
lld clang \
|
||||
libclang-dev llvm-dev \
|
||||
libstdc++-14-dev-i386-cross libgcc-14-dev-i386-cross \
|
||||
libstdc++-14-dev-amd64-cross libgcc-14-dev-amd64-cross
|
||||
|
||||
- name: Create Build Environment
|
||||
run: distrobox enter --name steamrt4 -- cmake -E make_directory ${{runner.workspace}}/build
|
||||
|
||||
- name: Configure CMake
|
||||
shell: bash
|
||||
working-directory: ${{runner.workspace}}/build
|
||||
run: distrobox enter --name steamrt4 -- cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -G Ninja -DBUILD_STEAM_SUPPORT=True -DENABLE_LTO=True -DENABLE_ASSERTIONS=False -DBUILD_THUNKS=True -DBUILD_FEXCONFIG=False -DBUILD_TESTING=False -DENABLE_CLANG_THUNKS=True -DUSE_LINKER=lld -DCMAKE_INSTALL_PREFIX=/usr
|
||||
|
||||
- name: Build
|
||||
working-directory: ${{runner.workspace}}/build
|
||||
shell: bash
|
||||
run: distrobox enter --name steamrt4 -- cmake --build . --config $BUILD_TYPE
|
||||
|
||||
- name: install
|
||||
working-directory: ${{runner.workspace}}/build
|
||||
shell: bash
|
||||
env:
|
||||
DESTDIR: ${{runner.workspace}}/install
|
||||
run: distrobox enter --name steamrt4 -- cmake --build . --config $BUILD_TYPE -t install
|
||||
- name: Upload libraries
|
||||
uses: 'actions/upload-artifact@v4'
|
||||
timeout-minutes: 1
|
||||
with:
|
||||
overwrite: true
|
||||
name: steamrt4_steampipe_depot
|
||||
path: ${{runner.workspace}}/install/*
|
||||
retention-days: 60
|
||||
compression-level: 9
|
||||
@@ -13,7 +13,6 @@ env:
|
||||
BUILD_TYPE: Release
|
||||
CC: clang
|
||||
CXX: clang++
|
||||
FEX_PORTABLE: 1
|
||||
|
||||
jobs:
|
||||
vixl_simulator:
|
||||
@@ -35,6 +34,7 @@ jobs:
|
||||
echo "FEX_ROOTFS_MOUNT=/mnt/AutoNFS/rootfs/" >> $GITHUB_ENV
|
||||
echo "FEX_ROOTFS_PATH=$HOME/Rootfs/" >> $GITHUB_ENV
|
||||
echo "FEX_ROOTFS=$HOME/Rootfs/" >> $GITHUB_ENV
|
||||
echo "ROOTFS=$HOME/Rootfs/" >> $GITHUB_ENV
|
||||
|
||||
- name: Update RootFS cache
|
||||
# Use a bash shell so we can use the same syntax for environment variable
|
||||
|
||||
@@ -1,88 +0,0 @@
|
||||
name: Wine DLL artifacts
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
|
||||
env:
|
||||
BUILD_TYPE: Release
|
||||
|
||||
jobs:
|
||||
wine_dll_artifacts:
|
||||
runs-on: ${{ matrix.arch }}
|
||||
strategy:
|
||||
matrix:
|
||||
arch: [[self-hosted, ARM64, mingw]]
|
||||
fail-fast: false
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Add MingGW to PATH
|
||||
run: echo "$HOME/llvm-mingw/build/bin/" >> $GITHUB_PATH
|
||||
|
||||
- name : submodule checkout
|
||||
# Need to update submodules
|
||||
run: |
|
||||
git submodule sync --recursive
|
||||
git submodule update --init --depth 1
|
||||
|
||||
- name: Clean install directory
|
||||
run: |
|
||||
rm -Rf ${{runner.workspace}}/build_install
|
||||
mkdir ${{runner.workspace}}/build_install
|
||||
|
||||
- name: Clean Build Environment
|
||||
run: |
|
||||
rm -Rf ${{runner.workspace}}/build_arm64ec
|
||||
rm -Rf ${{runner.workspace}}/build_wow64
|
||||
|
||||
- name: Create Build Environment arm64ec
|
||||
run: |
|
||||
cmake -E make_directory ${{runner.workspace}}/build_arm64ec
|
||||
cmake -E make_directory ${{runner.workspace}}/build_wow64
|
||||
|
||||
- name: Configure CMake arm64ec
|
||||
shell: bash
|
||||
working-directory: ${{runner.workspace}}/build_arm64ec
|
||||
run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DCMAKE_TOOLCHAIN_FILE=$GITHUB_WORKSPACE/Data/CMake/toolchain_mingw.cmake -DMINGW_TRIPLE=arm64ec-w64-mingw32 -DCMAKE_INSTALL_LIBDIR=/usr/lib/wine/aarch64-windows -G Ninja -DENABLE_LTO=False -DENABLE_ASSERTIONS=False -DENABLE_JEMALLOC_GLIBC_ALLOC=False -DCMAKE_INSTALL_PREFIX=/usr -DBUILD_TESTING=False -DCMAKE_INSTALL_PREFIX=/usr
|
||||
|
||||
- name: Configure CMake wow64
|
||||
shell: bash
|
||||
working-directory: ${{runner.workspace}}/build_wow64
|
||||
run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DCMAKE_TOOLCHAIN_FILE=$GITHUB_WORKSPACE/Data/CMake/toolchain_mingw.cmake -DMINGW_TRIPLE=aarch64-w64-mingw32 -DCMAKE_INSTALL_LIBDIR=/usr/lib/wine/aarch64-windows -G Ninja -DENABLE_LTO=False -DENABLE_ASSERTIONS=False -DENABLE_JEMALLOC_GLIBC_ALLOC=False -DCMAKE_INSTALL_PREFIX=/usr -DBUILD_TESTING=False -DCMAKE_INSTALL_PREFIX=/usr
|
||||
|
||||
- name: Build arm64ec
|
||||
working-directory: ${{runner.workspace}}/build_arm64ec
|
||||
shell: bash
|
||||
run: cmake --build . --config $BUILD_TYPE
|
||||
|
||||
- name: install arm64ec
|
||||
working-directory: ${{runner.workspace}}/build_arm64ec
|
||||
shell: bash
|
||||
env:
|
||||
DESTDIR: ${{runner.workspace}}/build_install
|
||||
run: cmake --build . --config $BUILD_TYPE -t install
|
||||
|
||||
- name: Build wow64
|
||||
working-directory: ${{runner.workspace}}/build_wow64
|
||||
shell: bash
|
||||
run: cmake --build . --config $BUILD_TYPE
|
||||
|
||||
- name: install wow64
|
||||
working-directory: ${{runner.workspace}}/build_wow64
|
||||
shell: bash
|
||||
env:
|
||||
DESTDIR: ${{runner.workspace}}/build_install
|
||||
run: cmake --build . --config $BUILD_TYPE -t install
|
||||
|
||||
- name: Upload libraries
|
||||
uses: 'actions/upload-artifact@v4'
|
||||
timeout-minutes: 1
|
||||
with:
|
||||
overwrite: true
|
||||
name: wine_dll_artifacts
|
||||
path: ${{runner.workspace}}/build_install/usr/lib/wine/aarch64-windows/lib*.dll
|
||||
retention-days: 60
|
||||
compression-level: 9
|
||||
@@ -46,6 +46,3 @@
|
||||
[submodule "External/tracy"]
|
||||
path = External/tracy
|
||||
url = https://github.com/wolfpld/tracy
|
||||
[submodule "External/range-v3"]
|
||||
path = External/range-v3
|
||||
url = https://github.com/ericniebler/range-v3.git
|
||||
+196
-211
@@ -1,124 +1,64 @@
|
||||
cmake_minimum_required(VERSION 3.14)
|
||||
project(FEX C CXX ASM)
|
||||
|
||||
include(CheckIncludeFiles)
|
||||
check_include_files("gdb/jit-reader.h" HAVE_GDB_JIT_READER_H)
|
||||
INCLUDE (CheckIncludeFiles)
|
||||
CHECK_INCLUDE_FILES ("gdb/jit-reader.h" HAVE_GDB_JIT_READER_H)
|
||||
|
||||
option(BUILD_FEX_LINUX_TESTS "Build FEXLinuxTests (requires x86 compiler)" FALSE)
|
||||
option(BUILD_TESTS "Build unit tests to ensure sanity" TRUE)
|
||||
option(BUILD_FEX_LINUX_TESTS "Build FEXLinuxTests, requires x86 compiler" FALSE)
|
||||
option(BUILD_THUNKS "Build thunks" FALSE)
|
||||
option(BUILD_FEXCONFIG "Build FEXConfig" TRUE)
|
||||
option(ENABLE_CLANG_THUNKS "Build thunks with clang" TRUE)
|
||||
option(ENABLE_IWYU "Enable the Include What You Use sanitizer" FALSE)
|
||||
option(ENABLE_IWYU "Enables include what you use program" FALSE)
|
||||
option(ENABLE_LTO "Enable LTO with compilation" TRUE)
|
||||
option(ENABLE_XRAY "Enable building with LLVM X-Ray" FALSE)
|
||||
set(USE_LINKER "" CACHE STRING "Path to a custom linker program")
|
||||
option(ENABLE_UBSAN "Enable the Clang Undefined Behavior Sanitizer" FALSE)
|
||||
option(ENABLE_ASAN "Enable the Clang Address Sanitizer" FALSE)
|
||||
option(ENABLE_TSAN "Enable the Clang Thread Sanitizer" FALSE)
|
||||
option(ENABLE_COVERAGE "Enable Code Coverage" FALSE)
|
||||
option(ENABLE_ASSERTIONS "Enable debug assertions" FALSE)
|
||||
option(ENABLE_GDB_SYMBOLS "Enable GDBSymbols integration support" ${HAVE_GDB_JIT_READER_H})
|
||||
option(ENABLE_STRICT_WERROR "Enable stricter -Werror" FALSE)
|
||||
option(ENABLE_WERROR "Enable -Werror" FALSE)
|
||||
option(ENABLE_JEMALLOC "Enable jemalloc allocator" TRUE)
|
||||
option(ENABLE_JEMALLOC_GLIBC_ALLOC "Enable jemalloc glibc allocator" TRUE)
|
||||
option(ENABLE_OFFLINE_TELEMETRY "Enable FEX offline telemetry" TRUE)
|
||||
option(ENABLE_COMPILE_TIME_TRACE "Enable time trace compile option" FALSE)
|
||||
option(ENABLE_LIBCXX "Use LLVM's libc++ instead of the GNU libstdc++" FALSE)
|
||||
option(ENABLE_CCACHE "Enable ccache for build caching" TRUE)
|
||||
option(ENABLE_VIXL_SIMULATOR "Use the VIXL simulator for emulation (only useful for CI testing)" FALSE)
|
||||
option(ENABLE_VIXL_DISASSEMBLER "Enable debug disassembler output with VIXL" FALSE)
|
||||
option(USE_LEGACY_BINFMTMISC "Use legacy method of setting up binfmt_misc" FALSE)
|
||||
option(ENABLE_FEXCORE_PROFILER "Enable FEXCore's timeline profiling capabilities" FALSE)
|
||||
set(FEXCORE_PROFILER_BACKEND "gpuvis" CACHE STRING "Set which backend to use for FEXCore's profiler")
|
||||
set_property(CACHE FEXCORE_PROFILER_BACKEND PROPERTY STRINGS gpuvis tracy)
|
||||
set(USE_LINKER "" CACHE STRING "Allow overriding the linker path directly")
|
||||
option(ENABLE_UBSAN "Enables Clang UBSAN" FALSE)
|
||||
option(ENABLE_ASAN "Enables Clang ASAN" FALSE)
|
||||
option(ENABLE_TSAN "Enables Clang TSAN" FALSE)
|
||||
option(ENABLE_COVERAGE "Enables Coverage" FALSE)
|
||||
option(ENABLE_ASSERTIONS "Enables assertions in build" FALSE)
|
||||
option(ENABLE_GDB_SYMBOLS "Enables GDBSymbols integration support" ${HAVE_GDB_JIT_READER_H})
|
||||
option(ENABLE_STRICT_WERROR "Enables stricter -Werror for CI" FALSE)
|
||||
option(ENABLE_WERROR "Enables -Werror" FALSE)
|
||||
option(ENABLE_JEMALLOC "Enables jemalloc allocator" TRUE)
|
||||
option(ENABLE_JEMALLOC_GLIBC_ALLOC "Enables jemalloc glibc allocator" TRUE)
|
||||
option(ENABLE_OFFLINE_TELEMETRY "Enables FEX offline telemetry" TRUE)
|
||||
option(ENABLE_COMPILE_TIME_TRACE "Enables time trace compile option" FALSE)
|
||||
option(ENABLE_LIBCXX "Enables LLVM libc++" FALSE)
|
||||
option(ENABLE_CCACHE "Enables ccache for compile caching" TRUE)
|
||||
option(ENABLE_VIXL_SIMULATOR "Enable use of VIXL simulator for emulation (only useful for CI testing)" FALSE)
|
||||
option(ENABLE_VIXL_DISASSEMBLER "Enables debug disassembler output with VIXL" FALSE)
|
||||
option(USE_LEGACY_BINFMTMISC "Uses legacy method of setting up binfmt_misc" FALSE)
|
||||
option(ENABLE_FEXCORE_PROFILER "Enables use of the FEXCore timeline profiling capabilities" FALSE)
|
||||
set (FEXCORE_PROFILER_BACKEND "gpuvis" CACHE STRING "Set which backend to use for the FEXCore profiler (gpuvis, tracy)")
|
||||
option(ENABLE_GLIBC_ALLOCATOR_HOOK_FAULT "Enables glibc memory allocation hooking with fault for CI testing")
|
||||
option(USE_PDB_DEBUGINFO "Build debug info in PDB format" FALSE)
|
||||
option(BUILD_STEAM_SUPPORT "Enable Steam integration" FALSE)
|
||||
|
||||
set(X86_32_TOOLCHAIN_FILE "${CMAKE_CURRENT_SOURCE_DIR}/Data/CMake/toolchain_x86_32.cmake" CACHE FILEPATH "Toolchain file for the (cross-)compiler targeting i686")
|
||||
set(X86_64_TOOLCHAIN_FILE "${CMAKE_CURRENT_SOURCE_DIR}/Data/CMake/toolchain_x86_64.cmake" CACHE FILEPATH "Toolchain file for the (cross-)compiler targeting x86_64")
|
||||
set(X86_DEV_ROOTFS "/" CACHE FILEPATH "Path to the sysroot used for cross-compiling for i686 and x86_64")
|
||||
set(DATA_DIRECTORY "" CACHE PATH "Global data directory (override)")
|
||||
set(HOSTLIBS_DATA_DIRECTORY "" CACHE PATH "Global data directory (override)")
|
||||
option(USE_PDB_DEBUGINFO "Builds debug info in PDB format" FALSE)
|
||||
|
||||
set (X86_32_TOOLCHAIN_FILE "${CMAKE_CURRENT_SOURCE_DIR}/Data/CMake/toolchain_x86_32.cmake" CACHE FILEPATH "Toolchain file for the (cross-)compiler targeting i686")
|
||||
set (X86_64_TOOLCHAIN_FILE "${CMAKE_CURRENT_SOURCE_DIR}/Data/CMake/toolchain_x86_64.cmake" CACHE FILEPATH "Toolchain file for the (cross-)compiler targeting x86_64")
|
||||
set (X86_DEV_ROOTFS "/" CACHE FILEPATH "Path to the sysroot used for cross-compiling for i686 and x86_64")
|
||||
set (DATA_DIRECTORY "" CACHE PATH "Global data directory (override)")
|
||||
if (NOT DATA_DIRECTORY)
|
||||
set(DATA_DIRECTORY "${CMAKE_INSTALL_PREFIX}/share/fex-emu")
|
||||
set (DATA_DIRECTORY "${CMAKE_INSTALL_PREFIX}/share/fex-emu")
|
||||
endif()
|
||||
|
||||
include(GNUInstallDirs)
|
||||
if (NOT HOSTLIBS_DATA_DIRECTORY)
|
||||
set(HOSTLIBS_DATA_DIRECTORY "${CMAKE_INSTALL_FULL_LIBDIR}/fex-emu")
|
||||
string(FIND ${CMAKE_BASE_NAME} mingw CONTAINS_MINGW)
|
||||
if (NOT CONTAINS_MINGW EQUAL -1)
|
||||
message (STATUS "Mingw build")
|
||||
set (MINGW_BUILD TRUE)
|
||||
set (ENABLE_JEMALLOC TRUE)
|
||||
set (ENABLE_JEMALLOC_GLIBC_ALLOC FALSE)
|
||||
endif()
|
||||
|
||||
## Platform Checks ##
|
||||
# Only 64-bit Linux and Windows are supported
|
||||
|
||||
# NB: SIZEOF_VOID_P is in bytes, not bits
|
||||
# On 32-bit systems this is set to 4
|
||||
if (NOT CMAKE_SIZEOF_VOID_P EQUAL 8)
|
||||
message(FATAL_ERROR "Unsupported pointer size ${CMAKE_SIZEOF_VOID_P}."
|
||||
" FEX only supports 64-bit (8-byte pointer) systems."
|
||||
" If you believe this is in error, file an issue.")
|
||||
elseif (NOT (WIN32 OR CMAKE_SYSTEM_NAME STREQUAL "Linux"))
|
||||
message(FATAL_ERROR "Unsupported system type ${CMAKE_SYSTEM_NAME}."
|
||||
" FEX only supports Linux and Windows."
|
||||
" If you believe this is in error, file an issue.")
|
||||
endif()
|
||||
|
||||
## Compiler Checks ##
|
||||
# GCC and MSVC are unsupported
|
||||
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
|
||||
message(FATAL_ERROR "FEX doesn't support GCC! Use Clang instead.")
|
||||
elseif (MSVC)
|
||||
message(FATAL_ERROR "FEX doesn't support MSVC! Use Clang on MinGW instead.")
|
||||
elseif (MINGW)
|
||||
message(STATUS "Building for MinGW")
|
||||
set(ENABLE_JEMALLOC TRUE)
|
||||
set(ENABLE_JEMALLOC_GLIBC_ALLOC FALSE)
|
||||
else ()
|
||||
message(STATUS "Clang version ${CMAKE_CXX_COMPILER_VERSION}")
|
||||
set(CLANG_MINIMUM_VERSION 13.0)
|
||||
if (NOT MINGW_BUILD)
|
||||
message (STATUS "Clang version ${CMAKE_CXX_COMPILER_VERSION}")
|
||||
set (CLANG_MINIMUM_VERSION 13.0)
|
||||
if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${CLANG_MINIMUM_VERSION})
|
||||
message(FATAL_ERROR "Clang version too old for FEX. Need at least ${CLANG_MINIMUM_VERSION} but has ${CMAKE_CXX_COMPILER_VERSION}")
|
||||
message (FATAL_ERROR "Clang version too old for FEX. Need at least ${CLANG_MINIMUM_VERSION} but has ${CMAKE_CXX_COMPILER_VERSION}")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
## Architecture Handling ##
|
||||
string(TOLOWER ${CMAKE_SYSTEM_PROCESSOR} processor)
|
||||
if (processor MATCHES "x86|amd64")
|
||||
option(ENABLE_X86_HOST_DEBUG "Enables compiling on x86_64 host" FALSE)
|
||||
if (NOT ENABLE_X86_HOST_DEBUG)
|
||||
message(FATAL_ERROR
|
||||
" FEX doesn't support compiling for x86-64 hosts!"
|
||||
" This is /only/ a supported configuration for FEX CI and nothing else!")
|
||||
else()
|
||||
message(STATUS "x86_64 debug build")
|
||||
endif()
|
||||
|
||||
set(ARCHITECTURE_x86_64 1)
|
||||
add_definitions(-DARCHITECTURE_x86_64=1)
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcx16")
|
||||
elseif (processor MATCHES "^aarch64|^arm64|^armv8\.*")
|
||||
set(ARCHITECTURE_arm64 1)
|
||||
add_definitions(-DARCHITECTURE_arm64=1)
|
||||
|
||||
# arm64ec needs to define both arm64 and arm64ec
|
||||
if (processor MATCHES "^arm64ec")
|
||||
set(ARCHITECTURE_arm64ec 1)
|
||||
add_definitions(-DARCHITECTURE_arm64ec=1)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if (NOT (ARCHITECTURE_arm64 OR ARCHITECTURE_arm64ec OR ARCHITECTURE_x86_64))
|
||||
message(FATAL_ERROR "Unsupported processor type ${processor}."
|
||||
" If you believe this is in error, file an issue.")
|
||||
endif()
|
||||
|
||||
if (BUILD_STEAM_SUPPORT)
|
||||
add_definitions(-DFEX_STEAM_SUPPORT=1)
|
||||
endif()
|
||||
|
||||
if (ENABLE_FEXCORE_PROFILER)
|
||||
add_definitions(-DENABLE_FEXCORE_PROFILER=1)
|
||||
string(TOUPPER "${FEXCORE_PROFILER_BACKEND}" FEXCORE_PROFILER_BACKEND)
|
||||
@@ -138,8 +78,8 @@ if (ENABLE_FEXCORE_PROFILER)
|
||||
add_definitions(-DTRACY_NO_SAMPLING=1)
|
||||
# This pulls in libbacktrace which allocators in global constructors (before FEX can set up its allocator hooks)
|
||||
add_definitions(-DTRACY_NO_CALLSTACK=1)
|
||||
if (MINGW)
|
||||
message(FATAL_ERROR "Tracy profiler not supported on MinGW")
|
||||
if (MINGW_BUILD)
|
||||
message(FATAL_ERROR "Tracy profiler not supported")
|
||||
endif()
|
||||
else()
|
||||
message(FATAL_ERROR "Unknown FEXCore profiler backend ${FEXCORE_PROFILER_BACKEND}")
|
||||
@@ -166,10 +106,9 @@ if(NOT TARGET uninstall)
|
||||
endif()
|
||||
|
||||
# These options are meant for package management
|
||||
set(TUNE_CPU "native" CACHE STRING "Override the CPU the build is tuned for")
|
||||
set(TUNE_ARCH "generic" CACHE STRING "Override the Arch the build is tuned for")
|
||||
set(OVERRIDE_VERSION "detect" CACHE STRING "Override the FEX version")
|
||||
set(OVERRIDE_HASH "detect" CACHE STRING "Override the FEX git hash")
|
||||
set (TUNE_CPU "native" CACHE STRING "Override the CPU the build is tuned for")
|
||||
set (TUNE_ARCH "generic" CACHE STRING "Override the Arch the build is tuned for")
|
||||
set (OVERRIDE_VERSION "detect" CACHE STRING "Override the FEX version in the format of <MMYY>{.<REV>}")
|
||||
|
||||
string(TOUPPER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE)
|
||||
if (CMAKE_BUILD_TYPE MATCHES "DEBUG")
|
||||
@@ -186,6 +125,7 @@ if (ENABLE_GDB_SYMBOLS)
|
||||
add_definitions(-DGDB_SYMBOLS_ENABLED=1)
|
||||
endif()
|
||||
|
||||
|
||||
set(CMAKE_CXX_STANDARD 20)
|
||||
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
|
||||
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/Bin)
|
||||
@@ -195,7 +135,33 @@ cmake_policy(SET CMP0083 NEW) # Follow new PIE policy
|
||||
include(CheckPIESupported)
|
||||
check_pie_supported()
|
||||
|
||||
set(CMAKE_INTERPROCEDURAL_OPTIMIZATION ${ENABLE_LTO})
|
||||
if (ENABLE_LTO)
|
||||
set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE)
|
||||
else()
|
||||
set(CMAKE_INTERPROCEDURAL_OPTIMIZATION FALSE)
|
||||
endif()
|
||||
|
||||
if (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")
|
||||
option(ENABLE_X86_HOST_DEBUG "Enables compiling on x86_64 host" FALSE)
|
||||
if (NOT ENABLE_X86_HOST_DEBUG)
|
||||
message(FATAL_ERROR
|
||||
" FEX-Emu doesn't support compiling for x86-64 hosts!"
|
||||
" This is /only/ a supported configuration for FEX CI and nothing else!")
|
||||
endif()
|
||||
set(_M_X86_64 1)
|
||||
add_definitions(-D_M_X86_64=1)
|
||||
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcx16")
|
||||
endif()
|
||||
|
||||
if (CMAKE_SYSTEM_PROCESSOR MATCHES "^aarch64|^arm64|^armv8\.*")
|
||||
set(_M_ARM_64 1)
|
||||
add_definitions(-D_M_ARM_64=1)
|
||||
endif()
|
||||
|
||||
if (CMAKE_SYSTEM_PROCESSOR MATCHES "^arm64ec")
|
||||
set(_M_ARM_64EC 1)
|
||||
add_definitions(-D_M_ARM_64EC=1)
|
||||
endif()
|
||||
|
||||
include(CheckCXXSourceCompiles)
|
||||
set(CMAKE_REQUIRED_FLAGS "-std=c++11 -Wattributes -Werror=attributes")
|
||||
@@ -211,15 +177,15 @@ check_cxx_source_compiles(
|
||||
HAS_CLANG_PRESERVE_ALL)
|
||||
unset(CMAKE_REQUIRED_FLAGS)
|
||||
if (HAS_CLANG_PRESERVE_ALL)
|
||||
if (MINGW)
|
||||
if (MINGW_BUILD)
|
||||
message(STATUS "Ignoring broken clang::preserve_all support")
|
||||
set(HAS_CLANG_PRESERVE_ALL FALSE)
|
||||
else()
|
||||
message(STATUS "Has clang::preserve_all")
|
||||
endif()
|
||||
endif()
|
||||
endif ()
|
||||
|
||||
if (ARCHITECTURE_arm64 AND HAS_CLANG_PRESERVE_ALL)
|
||||
if (_M_ARM_64 AND HAS_CLANG_PRESERVE_ALL)
|
||||
add_definitions("-DFEX_PRESERVE_ALL_ATTR=__attribute__((preserve_all))" "-DFEX_HAS_PRESERVE_ALL_ATTR=1")
|
||||
else()
|
||||
add_definitions("-DFEX_PRESERVE_ALL_ATTR=" "-DFEX_HAS_PRESERVE_ALL_ATTR=0")
|
||||
@@ -248,7 +214,7 @@ if (ENABLE_COMPILE_TIME_TRACE)
|
||||
link_libraries(-ftime-trace)
|
||||
endif()
|
||||
|
||||
set(PTHREAD_LIB pthread)
|
||||
set (PTHREAD_LIB pthread)
|
||||
|
||||
if (USE_LINKER)
|
||||
message(STATUS "Overriding linker to: ${USE_LINKER}")
|
||||
@@ -300,8 +266,8 @@ if (ENABLE_JEMALLOC_GLIBC_ALLOC)
|
||||
# Required for thunks to work.
|
||||
# All host native libraries will use this allocator, while *most* other FEX internal allocations will use the other jemalloc allocator.
|
||||
add_subdirectory(External/jemalloc_glibc/)
|
||||
elseif (NOT MINGW)
|
||||
message(STATUS
|
||||
elseif (NOT MINGW_BUILD)
|
||||
message (STATUS
|
||||
" jemalloc glibc allocator disabled!\n"
|
||||
" This is not a recommended configuration!\n"
|
||||
" This will very explicitly break thunk execution!\n"
|
||||
@@ -311,8 +277,8 @@ endif()
|
||||
if (ENABLE_JEMALLOC)
|
||||
# The jemalloc subproject that all FEXCore fextl objects allocate through.
|
||||
add_subdirectory(External/jemalloc/)
|
||||
elseif (NOT MINGW)
|
||||
message(STATUS
|
||||
elseif (NOT MINGW_BUILD)
|
||||
message (STATUS
|
||||
" jemalloc disabled!\n"
|
||||
" This is not a recommended configuration!\n"
|
||||
" This will very explicitly break 32-bit application execution!\n"
|
||||
@@ -324,22 +290,15 @@ if (USE_PDB_DEBUGINFO)
|
||||
add_link_options(-g -Wl,--pdb=)
|
||||
endif()
|
||||
|
||||
set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -fno-omit-frame-pointer")
|
||||
set(CMAKE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_LINKER_FLAGS_RELWITHDEBINFO} -fno-omit-frame-pointer")
|
||||
set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -fno-omit-frame-pointer")
|
||||
set (CMAKE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_LINKER_FLAGS_RELWITHDEBINFO} -fno-omit-frame-pointer")
|
||||
|
||||
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -fomit-frame-pointer")
|
||||
set(CMAKE_LINKER_FLAGS_RELEASE "${CMAKE_LINKER_FLAGS_RELEASE} -fomit-frame-pointer")
|
||||
set (CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -fomit-frame-pointer")
|
||||
set (CMAKE_LINKER_FLAGS_RELEASE "${CMAKE_LINKER_FLAGS_RELEASE} -fomit-frame-pointer")
|
||||
|
||||
## Modules ##
|
||||
list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/Data/CMake/)
|
||||
|
||||
include(LinkerGC)
|
||||
|
||||
## Externals ##
|
||||
include_directories(External/robin-map/include/)
|
||||
|
||||
include(CTest)
|
||||
if (BUILD_TESTING OR ENABLE_VIXL_DISASSEMBLER OR ENABLE_VIXL_SIMULATOR)
|
||||
if (BUILD_TESTS OR ENABLE_VIXL_DISASSEMBLER OR ENABLE_VIXL_SIMULATOR)
|
||||
add_subdirectory(External/vixl/)
|
||||
include_directories(SYSTEM External/vixl/src/)
|
||||
endif()
|
||||
@@ -348,16 +307,20 @@ if (ENABLE_FEXCORE_PROFILER AND FEXCORE_PROFILER_BACKEND STREQUAL "TRACY")
|
||||
add_subdirectory(External/tracy)
|
||||
endif()
|
||||
|
||||
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
|
||||
# This means we were attempted to get compiled with GCC
|
||||
message(FATAL_ERROR "FEX doesn't support getting compiled with GCC!")
|
||||
endif()
|
||||
|
||||
find_package(PkgConfig REQUIRED)
|
||||
find_package(Python 3.9 REQUIRED COMPONENTS Interpreter)
|
||||
find_package(Python 3.0 REQUIRED COMPONENTS Interpreter)
|
||||
|
||||
set(BUILD_SHARED_LIBS OFF)
|
||||
|
||||
if (NOT CMAKE_CROSSCOMPILING)
|
||||
find_package(xxhash MODULE QUIET)
|
||||
endif()
|
||||
|
||||
if (NOT TARGET xxHash::xxhash)
|
||||
pkg_search_module(xxhash IMPORTED_TARGET xxhash libxxhash)
|
||||
if (TARGET PkgConfig::xxhash AND NOT CMAKE_CROSSCOMPILING)
|
||||
add_library(xxHash::xxhash ALIAS PkgConfig::xxhash)
|
||||
else()
|
||||
set(XXHASH_BUNDLED_MODE TRUE)
|
||||
set(XXHASH_BUILD_XXHSUM FALSE)
|
||||
add_subdirectory(External/xxhash/cmake_unofficial/)
|
||||
@@ -366,7 +329,7 @@ endif()
|
||||
add_definitions(-Wno-trigraphs)
|
||||
add_definitions(-DGLOBAL_DATA_DIRECTORY="${DATA_DIRECTORY}/")
|
||||
|
||||
if (BUILD_TESTING)
|
||||
if (BUILD_TESTS)
|
||||
find_package(Catch2 3 QUIET)
|
||||
if (NOT Catch2_FOUND)
|
||||
add_subdirectory(External/Catch2/)
|
||||
@@ -376,9 +339,6 @@ if (BUILD_TESTING)
|
||||
endif()
|
||||
|
||||
include(Catch)
|
||||
else ()
|
||||
# Override any previously generated test list to avoid running stale test binaries
|
||||
file(GENERATE OUTPUT CTestTestfile.cmake CONTENT "# No tests since BUILD_TESTING is disabled")
|
||||
endif()
|
||||
|
||||
find_package(fmt QUIET)
|
||||
@@ -388,12 +348,6 @@ if (NOT fmt_FOUND)
|
||||
add_subdirectory(External/fmt/)
|
||||
endif()
|
||||
|
||||
find_package(range-v3 QUIET)
|
||||
if (NOT range-v3_FOUND)
|
||||
add_subdirectory(External/range-v3/)
|
||||
target_compile_definitions(range-v3 INTERFACE RANGES_DISABLE_DEPRECATED_WARNINGS)
|
||||
endif()
|
||||
|
||||
add_subdirectory(External/tiny-json/)
|
||||
include_directories(External/tiny-json/)
|
||||
|
||||
@@ -438,7 +392,7 @@ if (NOT TUNE_ARCH STREQUAL "generic")
|
||||
endif()
|
||||
|
||||
if (TUNE_CPU STREQUAL "native")
|
||||
if(ARCHITECTURE_arm64)
|
||||
if(_M_ARM_64)
|
||||
if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 999999.0)
|
||||
# Clang 12.0 fixed the -mcpu=native bug with mixed big.little implementers
|
||||
# Clang can not currently check for native Apple M1 type in hypervisor. Currently disabled
|
||||
@@ -452,13 +406,6 @@ if (TUNE_CPU STREQUAL "native")
|
||||
|
||||
string(STRIP ${AARCH64_CPU} AARCH64_CPU)
|
||||
|
||||
execute_process(COMMAND python3 "${PROJECT_SOURCE_DIR}/Scripts/NeedDisabledSVE.py"
|
||||
RESULT_VARIABLE NEEDS_SVE_DISABLED)
|
||||
if (NEEDS_SVE_DISABLED)
|
||||
message(STATUS "Platform has bugged SVE. Disabling")
|
||||
set(AARCH64_CPU "cortex-a78")
|
||||
endif()
|
||||
|
||||
check_cxx_compiler_flag("-mcpu=${AARCH64_CPU}" COMPILER_SUPPORTS_CPU_TYPE)
|
||||
if(COMPILER_SUPPORTS_CPU_TYPE)
|
||||
list(APPEND FEX_TUNE_COMPILE_FLAGS "-mcpu=${AARCH64_CPU}")
|
||||
@@ -479,40 +426,6 @@ elseif (NOT TUNE_CPU STREQUAL "none")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
set(GIT_DESCRIBE_STRING "FEX-Unknown")
|
||||
|
||||
if (OVERRIDE_VERSION STREQUAL "detect")
|
||||
find_package(Git)
|
||||
|
||||
if (GIT_FOUND)
|
||||
execute_process(
|
||||
COMMAND ${GIT_EXECUTABLE} describe --abbrev=7
|
||||
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
|
||||
OUTPUT_VARIABLE GIT_DESCRIBE_STRING
|
||||
ERROR_QUIET
|
||||
OUTPUT_STRIP_TRAILING_WHITESPACE)
|
||||
endif()
|
||||
else()
|
||||
set(GIT_DESCRIBE_STRING "${OVERRIDE_VERSION}")
|
||||
endif()
|
||||
|
||||
set(GIT_SHORT_HASH "Unknown")
|
||||
|
||||
if (OVERRIDE_HASH STREQUAL "detect")
|
||||
find_package(Git)
|
||||
|
||||
if (GIT_FOUND)
|
||||
execute_process(
|
||||
COMMAND ${GIT_EXECUTABLE} rev-parse --short=7 HEAD
|
||||
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
|
||||
OUTPUT_VARIABLE GIT_SHORT_HASH
|
||||
ERROR_QUIET
|
||||
OUTPUT_STRIP_TRAILING_WHITESPACE)
|
||||
endif()
|
||||
else()
|
||||
set(GIT_SHORT_HASH "${OVERRIDE_HASH}")
|
||||
endif()
|
||||
|
||||
if (ENABLE_IWYU)
|
||||
find_program(IWYU_EXE "iwyu")
|
||||
if (IWYU_EXE)
|
||||
@@ -523,10 +436,15 @@ endif()
|
||||
|
||||
add_compile_options(-Wall)
|
||||
|
||||
if (BUILD_TESTING)
|
||||
include(CTest)
|
||||
if (BUILD_TESTS)
|
||||
message(STATUS "Unit tests are enabled")
|
||||
if (NOT BUILD_TESTING)
|
||||
# CMake checks this variable before generating CTestTestfile.cmake
|
||||
message(SEND_ERROR "Unit tests require BUILD_TESTING to be enabled")
|
||||
endif()
|
||||
|
||||
set(TEST_JOB_COUNT "" CACHE STRING "Override number of parallel jobs to use while running tests")
|
||||
set (TEST_JOB_COUNT "" CACHE STRING "Override number of parallel jobs to use while running tests")
|
||||
if (TEST_JOB_COUNT)
|
||||
message(STATUS "Running tests with ${TEST_JOB_COUNT} jobs")
|
||||
elseif(CMAKE_VERSION VERSION_LESS "3.29")
|
||||
@@ -541,16 +459,13 @@ add_subdirectory(FEXHeaderUtils/)
|
||||
add_subdirectory(CodeEmitter/)
|
||||
add_subdirectory(FEXCore/)
|
||||
|
||||
if (ARCHITECTURE_arm64 AND NOT MINGW AND NOT BUILD_STEAM_SUPPORT)
|
||||
if (_M_ARM_64 AND NOT MINGW_BUILD)
|
||||
# Binfmt_misc files must be installed prior to Source/ installs
|
||||
add_subdirectory(Data/binfmts/)
|
||||
endif()
|
||||
|
||||
add_subdirectory(Source/)
|
||||
|
||||
if (NOT BUILD_STEAM_SUPPORT)
|
||||
add_subdirectory(Data/AppConfig/)
|
||||
endif()
|
||||
add_subdirectory(Data/AppConfig/)
|
||||
|
||||
# Install the ThunksDB file
|
||||
file(GLOB CONFIG_SOURCES CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/Data/*.json)
|
||||
@@ -558,16 +473,15 @@ file(GLOB CONFIG_SOURCES CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/Data/*.js
|
||||
# Any application configuration json file gets installed
|
||||
foreach(CONFIG_SRC ${CONFIG_SOURCES})
|
||||
install(FILES ${CONFIG_SRC}
|
||||
DESTINATION ${DATA_DIRECTORY}/
|
||||
COMPONENT Runtime)
|
||||
DESTINATION ${DATA_DIRECTORY}/)
|
||||
endforeach()
|
||||
|
||||
if (BUILD_TESTING)
|
||||
if (BUILD_TESTS)
|
||||
add_subdirectory(unittests/)
|
||||
endif()
|
||||
|
||||
if (BUILD_THUNKS)
|
||||
set(FEX_PROJECT_SOURCE_DIR ${PROJECT_SOURCE_DIR})
|
||||
set (FEX_PROJECT_SOURCE_DIR ${PROJECT_SOURCE_DIR})
|
||||
add_subdirectory(ThunkLibs/Generator)
|
||||
|
||||
# Thunk targets for both host libraries and IDE integration
|
||||
@@ -594,7 +508,8 @@ if (BUILD_THUNKS)
|
||||
"-DX86_DEV_ROOTFS=${X86_DEV_ROOTFS}"
|
||||
INSTALL_COMMAND ""
|
||||
BUILD_ALWAYS ON
|
||||
DEPENDS thunkgen)
|
||||
DEPENDS thunkgen
|
||||
)
|
||||
|
||||
ExternalProject_Add(guest-libs-32
|
||||
PREFIX guest-libs-32
|
||||
@@ -612,36 +527,106 @@ if (BUILD_THUNKS)
|
||||
"-DX86_DEV_ROOTFS=${X86_DEV_ROOTFS}"
|
||||
INSTALL_COMMAND ""
|
||||
BUILD_ALWAYS ON
|
||||
DEPENDS thunkgen)
|
||||
DEPENDS thunkgen
|
||||
)
|
||||
|
||||
install(
|
||||
CODE "message(\"-- Installing: guest-libs\")"
|
||||
CODE "MESSAGE(\"-- Installing: guest-libs\")"
|
||||
CODE "
|
||||
execute_process(COMMAND ${CMAKE_COMMAND} --build . --target install
|
||||
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/Guest)"
|
||||
EXECUTE_PROCESS(COMMAND ${CMAKE_COMMAND} --build . --target install
|
||||
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/Guest
|
||||
)"
|
||||
DEPENDS guest-libs
|
||||
COMPONENT Runtime)
|
||||
)
|
||||
|
||||
install(
|
||||
CODE "message(\"-- Installing: guest-libs-32\")"
|
||||
CODE "MESSAGE(\"-- Installing: guest-libs-32\")"
|
||||
CODE "
|
||||
execute_process(COMMAND ${CMAKE_COMMAND} --build . --target install
|
||||
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/Guest_32)"
|
||||
EXECUTE_PROCESS(COMMAND ${CMAKE_COMMAND} --build . --target install
|
||||
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/Guest_32
|
||||
)"
|
||||
DEPENDS guest-libs-32
|
||||
COMPONENT Runtime)
|
||||
)
|
||||
|
||||
add_custom_target(uninstall_guest-libs
|
||||
COMMAND ${CMAKE_COMMAND} "--build" "." "--target" "uninstall"
|
||||
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/Guest)
|
||||
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/Guest
|
||||
)
|
||||
|
||||
add_custom_target(uninstall_guest-libs-32
|
||||
COMMAND ${CMAKE_COMMAND} "--build" "." "--target" "uninstall"
|
||||
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/Guest_32)
|
||||
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/Guest_32
|
||||
)
|
||||
|
||||
add_dependencies(uninstall uninstall_guest-libs)
|
||||
add_dependencies(uninstall uninstall_guest-libs-32)
|
||||
endif()
|
||||
|
||||
if (BUILD_STEAM_SUPPORT)
|
||||
add_subdirectory(Source/Steam/)
|
||||
set(FEX_VERSION_MAJOR "0")
|
||||
set(FEX_VERSION_MINOR "0")
|
||||
set(FEX_VERSION_PATCH "0")
|
||||
|
||||
if (OVERRIDE_VERSION STREQUAL "detect")
|
||||
find_package(Git)
|
||||
if (GIT_FOUND)
|
||||
execute_process(
|
||||
COMMAND ${GIT_EXECUTABLE} describe --abbrev=0
|
||||
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
|
||||
OUTPUT_VARIABLE GIT_DESCRIBE_STRING
|
||||
RESULT_VARIABLE GIT_ERROR
|
||||
ERROR_QUIET
|
||||
OUTPUT_STRIP_TRAILING_WHITESPACE
|
||||
)
|
||||
|
||||
if (NOT ${GIT_ERROR} EQUAL 0)
|
||||
# Likely built in a way that doesn't have tags
|
||||
# Setup a version tag that is unknown
|
||||
set(GIT_DESCRIBE_STRING "FEX-0000")
|
||||
endif()
|
||||
endif()
|
||||
else()
|
||||
set(GIT_DESCRIBE_STRING "FEX-${OVERRIDE_VERSION}")
|
||||
endif()
|
||||
|
||||
# Parse the version here
|
||||
# Change something like `FEX-2106.1-76-<hash>` in to a list
|
||||
string(REPLACE "-" ";" DESCRIBE_LIST ${GIT_DESCRIBE_STRING})
|
||||
|
||||
# Extract the `2106.1` element
|
||||
list(GET DESCRIBE_LIST 1 DESCRIBE_LIST)
|
||||
|
||||
# Change `2106.1` in to a list
|
||||
string(REPLACE "." ";" DESCRIBE_LIST ${DESCRIBE_LIST})
|
||||
|
||||
# Calculate list size
|
||||
list(LENGTH DESCRIBE_LIST LIST_SIZE)
|
||||
|
||||
# Pull out the major version
|
||||
list(GET DESCRIBE_LIST 0 FEX_VERSION_MAJOR)
|
||||
|
||||
# Minor version only exists if there is a .1 at the end
|
||||
# eg: 2106 versus 2106.1
|
||||
if (LIST_SIZE GREATER 1)
|
||||
list(GET DESCRIBE_LIST 1 FEX_VERSION_MINOR)
|
||||
endif()
|
||||
|
||||
# Package creation
|
||||
set (CPACK_GENERATOR "DEB")
|
||||
set (CPACK_PACKAGE_NAME fex-emu)
|
||||
set (CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${GIT_DESCRIBE_STRING}_${CMAKE_SYSTEM_PROCESSOR}")
|
||||
set (CPACK_PACKAGE_CONTACT "FEX-Emu Maintainers <team@fex-emu.com>")
|
||||
set (CPACK_PACKAGE_VERSION_MAJOR "${FEX_VERSION_MAJOR}")
|
||||
set (CPACK_PACKAGE_VERSION_MINOR "${FEX_VERSION_MINOR}")
|
||||
set (CPACK_PACKAGE_VERSION_PATCH "${FEX_VERSION_PATCH}")
|
||||
set (CPACK_PACKAGE_DESCRIPTION_FILE "${CMAKE_CURRENT_SOURCE_DIR}/Data/CMake/CPack/Description.txt")
|
||||
|
||||
# Debian defines
|
||||
set (CPACK_DEBIAN_PACKAGE_DEPENDS "libc6, libstdc++6, libepoxy0, libsdl2-2.0-0, libegl1, libx11-6, squashfuse")
|
||||
set (CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/Data/CMake/CPack/postinst;${CMAKE_CURRENT_SOURCE_DIR}/Data/CMake/CPack/prerm;${CMAKE_CURRENT_SOURCE_DIR}/Data/CMake/CPack/triggers")
|
||||
if (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64")
|
||||
# binfmt_misc conflicts with qemu-user-static
|
||||
# We also only install binfmt_misc on aarch64 hosts
|
||||
set (CPACK_DEBIAN_PACKAGE_CONFLICTS "${CPACK_DEBIAN_PACKAGE_CONFLICTS}, qemu-user-static")
|
||||
endif()
|
||||
include (CPack)
|
||||
+1
-1
@@ -129,4 +129,4 @@
|
||||
"variables": []
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -36,31 +36,24 @@ public:
|
||||
DataProcessing_PCRel_Imm(Op, rd, Imm);
|
||||
}
|
||||
|
||||
[[nodiscard]] BranchEncodeSucceeded adr(ARMEmitter::Register rd, const BackwardLabel* Label) {
|
||||
void adr(ARMEmitter::Register rd, const BackwardLabel* Label) {
|
||||
int32_t Imm = static_cast<int32_t>(Label->Location - GetCursorAddress<uint8_t*>());
|
||||
if (IsADRRange(Imm)) {
|
||||
constexpr uint32_t Op = 0b0001'0000 << 24;
|
||||
DataProcessing_PCRel_Imm(Op, rd, Imm);
|
||||
return BranchEncodeSucceeded::Success;
|
||||
}
|
||||
LOGMAN_THROW_A_FMT(IsADRRange(Imm), "Unscaled offset too large");
|
||||
|
||||
// Can't encode.
|
||||
return BranchEncodeSucceeded::Failure;
|
||||
constexpr uint32_t Op = 0b0001'0000 << 24;
|
||||
DataProcessing_PCRel_Imm(Op, rd, Imm);
|
||||
}
|
||||
[[nodiscard]] BranchEncodeSucceeded adr(ARMEmitter::Register rd, ForwardLabel* Label) {
|
||||
void adr(ARMEmitter::Register rd, ForwardLabel* Label) {
|
||||
AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::ADR});
|
||||
constexpr uint32_t Op = 0b0001'0000 << 24;
|
||||
DataProcessing_PCRel_Imm(Op, rd, 0);
|
||||
|
||||
// Forward label doesn't know if it can encode until Bind.
|
||||
return BranchEncodeSucceeded::Success;
|
||||
}
|
||||
|
||||
[[nodiscard]] BranchEncodeSucceeded adr(ARMEmitter::Register rd, BiDirectionalLabel* Label) {
|
||||
void adr(ARMEmitter::Register rd, BiDirectionalLabel* Label) {
|
||||
if (Label->Backward.Location) {
|
||||
return adr(rd, &Label->Backward);
|
||||
adr(rd, &Label->Backward);
|
||||
} else {
|
||||
return adr(rd, &Label->Forward);
|
||||
adr(rd, &Label->Forward);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -69,53 +62,38 @@ public:
|
||||
DataProcessing_PCRel_Imm(Op, rd, Imm);
|
||||
}
|
||||
|
||||
[[nodiscard]] BranchEncodeSucceeded adrp(ARMEmitter::Register rd, const BackwardLabel* Label) {
|
||||
void adrp(ARMEmitter::Register rd, const BackwardLabel* Label) {
|
||||
int64_t Imm = reinterpret_cast<int64_t>(Label->Location) - (GetCursorAddress<int64_t>() & ~0xFFFLL);
|
||||
LOGMAN_THROW_A_FMT(IsADRPRange(Imm) && IsADRPAligned(Imm), "Unscaled offset too large");
|
||||
|
||||
if (IsADRPRange(Imm) && IsADRPAligned(Imm)) {
|
||||
constexpr uint32_t Op = 0b1001'0000 << 24;
|
||||
DataProcessing_PCRel_Imm(Op, rd, Imm);
|
||||
return BranchEncodeSucceeded::Success;
|
||||
}
|
||||
|
||||
// Can't encode.
|
||||
return BranchEncodeSucceeded::Failure;
|
||||
constexpr uint32_t Op = 0b1001'0000 << 24;
|
||||
DataProcessing_PCRel_Imm(Op, rd, Imm);
|
||||
}
|
||||
|
||||
[[nodiscard]] BranchEncodeSucceeded adrp(ARMEmitter::Register rd, ForwardLabel* Label) {
|
||||
void adrp(ARMEmitter::Register rd, ForwardLabel* Label) {
|
||||
AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::ADRP});
|
||||
constexpr uint32_t Op = 0b1001'0000 << 24;
|
||||
DataProcessing_PCRel_Imm(Op, rd, 0);
|
||||
|
||||
// Forward label doesn't know if it can encode until Bind.
|
||||
return BranchEncodeSucceeded::Success;
|
||||
}
|
||||
|
||||
[[nodiscard]] BranchEncodeSucceeded adrp(ARMEmitter::Register rd, BiDirectionalLabel* Label) {
|
||||
void adrp(ARMEmitter::Register rd, BiDirectionalLabel* Label) {
|
||||
if (Label->Backward.Location) {
|
||||
return adrp(rd, &Label->Backward);
|
||||
adrp(rd, &Label->Backward);
|
||||
} else {
|
||||
return adrp(rd, &Label->Forward);
|
||||
adrp(rd, &Label->Forward);
|
||||
}
|
||||
}
|
||||
|
||||
[[nodiscard]] BranchEncodeSucceeded LongAddressGen(ARMEmitter::Register rd, const BackwardLabel* Label) {
|
||||
const auto SLocation = reinterpret_cast<int64_t>(Label->Location);
|
||||
const auto ULocation = std::bit_cast<uint64_t>(SLocation);
|
||||
|
||||
const int64_t Imm = SLocation - (GetCursorAddress<int64_t>());
|
||||
const auto UImm = std::bit_cast<uint64_t>(Imm);
|
||||
|
||||
void LongAddressGen(ARMEmitter::Register rd, const BackwardLabel* Label) {
|
||||
int64_t Imm = reinterpret_cast<int64_t>(Label->Location) - (GetCursorAddress<int64_t>());
|
||||
if (IsADRRange(Imm)) {
|
||||
// If the range is in ADR range then we can just use ADR.
|
||||
return adr(rd, Label);
|
||||
}
|
||||
if (IsADRPRange(Imm)) {
|
||||
const int64_t ADRPImm = (SLocation & ~0xFFFLL) - (GetCursorAddress<int64_t>() & ~0xFFFLL);
|
||||
adr(rd, Label);
|
||||
} else if (IsADRPRange(Imm)) {
|
||||
int64_t ADRPImm = (reinterpret_cast<int64_t>(Label->Location) & ~0xFFFLL) - (GetCursorAddress<int64_t>() & ~0xFFFLL);
|
||||
|
||||
// If the range is in the ADRP range then we can use ADRP.
|
||||
const bool NeedsOffset = !IsADRPAligned(ULocation);
|
||||
const uint64_t AlignedOffset = ULocation & 0xFFFULL;
|
||||
bool NeedsOffset = !IsADRPAligned(reinterpret_cast<uint64_t>(Label->Location));
|
||||
uint64_t AlignedOffset = reinterpret_cast<uint64_t>(Label->Location) & 0xFFFULL;
|
||||
|
||||
// First emit ADRP
|
||||
adrp(rd, ADRPImm >> 12);
|
||||
@@ -124,33 +102,23 @@ public:
|
||||
// Now even an add
|
||||
add(ARMEmitter::Size::i64Bit, rd, rd, AlignedOffset);
|
||||
}
|
||||
|
||||
return BranchEncodeSucceeded::Success;
|
||||
} else {
|
||||
LOGMAN_MSG_A_FMT("Unscaled offset too large");
|
||||
FEX_UNREACHABLE;
|
||||
}
|
||||
|
||||
// Stinky path, we need to load the address as a sequence of movz+movk+movk
|
||||
movz(ARMEmitter::Size::i64Bit, rd, (UImm >> 32) & 0xFFFF, 32);
|
||||
movk(ARMEmitter::Size::i64Bit, rd, (UImm >> 16) & 0xFFFF, 16);
|
||||
movk(ARMEmitter::Size::i64Bit, rd, UImm & 0xFFFF);
|
||||
|
||||
return BranchEncodeSucceeded::Success;
|
||||
}
|
||||
[[nodiscard]] BranchEncodeSucceeded LongAddressGen(ARMEmitter::Register rd, ForwardLabel* Label) {
|
||||
void LongAddressGen(ARMEmitter::Register rd, ForwardLabel* Label) {
|
||||
AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::LONG_ADDRESS_GEN});
|
||||
// Emit a register index and two nops. These will be backpatched.
|
||||
// Emit a register index and a nop. These will be backpatched.
|
||||
dc32(rd.Idx());
|
||||
nop();
|
||||
nop();
|
||||
|
||||
// Forward label doesn't know if it can encode until Bind.
|
||||
return BranchEncodeSucceeded::Success;
|
||||
}
|
||||
|
||||
[[nodiscard]] BranchEncodeSucceeded LongAddressGen(ARMEmitter::Register rd, BiDirectionalLabel* Label) {
|
||||
void LongAddressGen(ARMEmitter::Register rd, BiDirectionalLabel* Label) {
|
||||
if (Label->Backward.Location) {
|
||||
return LongAddressGen(rd, &Label->Backward);
|
||||
LongAddressGen(rd, &Label->Backward);
|
||||
} else {
|
||||
return LongAddressGen(rd, &Label->Forward);
|
||||
LongAddressGen(rd, &Label->Forward);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -206,7 +174,7 @@ public:
|
||||
// Logical immediate
|
||||
void and_(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint64_t Imm) {
|
||||
uint32_t n, immr, imms;
|
||||
const auto IsImm = IsImmLogical(Imm, RegSizeInBits(s), &n, &imms, &immr);
|
||||
[[maybe_unused]] const auto IsImm = IsImmLogical(Imm, RegSizeInBits(s), &n, &imms, &immr);
|
||||
LOGMAN_THROW_A_FMT(IsImm, "Couldn't encode immediate to logical op");
|
||||
and_(s, rd, rn, n, immr, imms);
|
||||
}
|
||||
@@ -217,7 +185,7 @@ public:
|
||||
|
||||
void ands(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint64_t Imm) {
|
||||
uint32_t n, immr, imms;
|
||||
const auto IsImm = IsImmLogical(Imm, RegSizeInBits(s), &n, &imms, &immr);
|
||||
[[maybe_unused]] const auto IsImm = IsImmLogical(Imm, RegSizeInBits(s), &n, &imms, &immr);
|
||||
LOGMAN_THROW_A_FMT(IsImm, "Couldn't encode immediate to logical op");
|
||||
ands(s, rd, rn, n, immr, imms);
|
||||
}
|
||||
@@ -228,14 +196,14 @@ public:
|
||||
|
||||
void orr(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint64_t Imm) {
|
||||
uint32_t n, immr, imms;
|
||||
const auto IsImm = IsImmLogical(Imm, RegSizeInBits(s), &n, &imms, &immr);
|
||||
[[maybe_unused]] const auto IsImm = IsImmLogical(Imm, RegSizeInBits(s), &n, &imms, &immr);
|
||||
LOGMAN_THROW_A_FMT(IsImm, "Couldn't encode immediate to logical op");
|
||||
orr(s, rd, rn, n, immr, imms);
|
||||
}
|
||||
|
||||
void eor(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint64_t Imm) {
|
||||
uint32_t n, immr, imms;
|
||||
const auto IsImm = IsImmLogical(Imm, RegSizeInBits(s), &n, &imms, &immr);
|
||||
[[maybe_unused]] const auto IsImm = IsImmLogical(Imm, RegSizeInBits(s), &n, &imms, &immr);
|
||||
LOGMAN_THROW_A_FMT(IsImm, "Couldn't encode immediate to logical op");
|
||||
eor(s, rd, rn, n, immr, imms);
|
||||
}
|
||||
@@ -365,7 +333,7 @@ public:
|
||||
bfi(s, rd, Reg::zr, lsb, width);
|
||||
}
|
||||
void bfxil(ARMEmitter::Size s, Register rd, Register rn, uint32_t lsb, uint32_t width) {
|
||||
const auto reg_size_bits = RegSizeInBits(s);
|
||||
[[maybe_unused]] const auto reg_size_bits = RegSizeInBits(s);
|
||||
const auto lsb_p_width = lsb + width;
|
||||
|
||||
LOGMAN_THROW_A_FMT(width >= 1, "bfxil needs width >= 1");
|
||||
@@ -894,6 +862,12 @@ public:
|
||||
}
|
||||
|
||||
private:
|
||||
static constexpr Condition InvertCondition(Condition cond) {
|
||||
// These behave as always, so it makes no sense to allow inverting these.
|
||||
LOGMAN_THROW_A_FMT(cond != Condition::CC_AL && cond != Condition::CC_NV, "Cannot invert CC_AL or CC_NV");
|
||||
return static_cast<Condition>(FEXCore::ToUnderlying(cond) ^ 1);
|
||||
}
|
||||
|
||||
void and_(ARMEmitter::Size s, ARMEmitter::Register rd, ARMEmitter::Register rn, uint32_t n, uint32_t immr, uint32_t imms) {
|
||||
constexpr uint32_t Op = 0b001'0010'00 << 22;
|
||||
DataProcessing_Logical_Imm(Op, s, rd, rn, n, immr, imms);
|
||||
@@ -1003,7 +977,7 @@ private:
|
||||
}
|
||||
|
||||
void xbfiz_helper(bool is_signed, ARMEmitter::Size s, Register rd, Register rn, uint32_t lsb, uint32_t width) {
|
||||
const auto lsb_p_width = lsb + width;
|
||||
[[maybe_unused]] const auto lsb_p_width = lsb + width;
|
||||
const auto reg_size_bits = RegSizeInBits(s);
|
||||
|
||||
LOGMAN_THROW_A_FMT(lsb_p_width <= reg_size_bits, "lsb + width ({}) must be <= {}. lsb={}, width={}", lsb_p_width, reg_size_bits, lsb, width);
|
||||
|
||||
+2125
-1444
File diff suppressed because it is too large.
Load diff
@@ -20,31 +20,23 @@ public:
|
||||
constexpr uint32_t Op = 0b0101'010 << 25;
|
||||
Branch_Conditional(Op, 0, 0, Cond, Imm);
|
||||
}
|
||||
[[nodiscard]] BranchEncodeSucceeded b(ARMEmitter::Condition Cond, const BackwardLabel* Label) {
|
||||
void b(ARMEmitter::Condition Cond, const BackwardLabel* Label) {
|
||||
int32_t Imm = static_cast<int32_t>(Label->Location - GetCursorAddress<uint8_t*>());
|
||||
if (Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0)) {
|
||||
constexpr uint32_t Op = 0b0101'010 << 25;
|
||||
Branch_Conditional(Op, 0, 0, Cond, Imm >> 2);
|
||||
return BranchEncodeSucceeded::Success;
|
||||
}
|
||||
|
||||
// Can't encode.
|
||||
return BranchEncodeSucceeded::Failure;
|
||||
LOGMAN_THROW_A_FMT(Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0), "Unscaled offset too large");
|
||||
constexpr uint32_t Op = 0b0101'010 << 25;
|
||||
Branch_Conditional(Op, 0, 0, Cond, Imm >> 2);
|
||||
}
|
||||
[[nodiscard]] BranchEncodeSucceeded b(ARMEmitter::Condition Cond, ForwardLabel* Label) {
|
||||
void b(ARMEmitter::Condition Cond, ForwardLabel* Label) {
|
||||
AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::BC});
|
||||
constexpr uint32_t Op = 0b0101'010 << 25;
|
||||
Branch_Conditional(Op, 0, 0, Cond, 0);
|
||||
|
||||
// Forward label doesn't know if it can encode until Bind.
|
||||
return BranchEncodeSucceeded::Success;
|
||||
}
|
||||
|
||||
[[nodiscard]] BranchEncodeSucceeded b(ARMEmitter::Condition Cond, BiDirectionalLabel* Label) {
|
||||
void b(ARMEmitter::Condition Cond, BiDirectionalLabel* Label) {
|
||||
if (Label->Backward.Location) {
|
||||
return b(Cond, &Label->Backward);
|
||||
b(Cond, &Label->Backward);
|
||||
} else {
|
||||
return b(Cond, &Label->Forward);
|
||||
b(Cond, &Label->Forward);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -53,32 +45,24 @@ public:
|
||||
constexpr uint32_t Op = 0b0101'010 << 25;
|
||||
Branch_Conditional(Op, 0, 1, Cond, Imm);
|
||||
}
|
||||
[[nodiscard]] BranchEncodeSucceeded bc(ARMEmitter::Condition Cond, const BackwardLabel* Label) {
|
||||
void bc(ARMEmitter::Condition Cond, const BackwardLabel* Label) {
|
||||
int32_t Imm = static_cast<int32_t>(Label->Location - GetCursorAddress<uint8_t*>());
|
||||
if (Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0)) {
|
||||
constexpr uint32_t Op = 0b0101'010 << 25;
|
||||
Branch_Conditional(Op, 0, 1, Cond, Imm >> 2);
|
||||
return BranchEncodeSucceeded::Success;
|
||||
}
|
||||
|
||||
// Can't encode.
|
||||
return BranchEncodeSucceeded::Failure;
|
||||
LOGMAN_THROW_A_FMT(Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0), "Unscaled offset too large");
|
||||
constexpr uint32_t Op = 0b0101'010 << 25;
|
||||
Branch_Conditional(Op, 0, 1, Cond, Imm >> 2);
|
||||
}
|
||||
|
||||
[[nodiscard]] BranchEncodeSucceeded bc(ARMEmitter::Condition Cond, ForwardLabel* Label) {
|
||||
void bc(ARMEmitter::Condition Cond, ForwardLabel* Label) {
|
||||
AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::BC});
|
||||
constexpr uint32_t Op = 0b0101'010 << 25;
|
||||
Branch_Conditional(Op, 0, 1, Cond, 0);
|
||||
|
||||
// Forward label doesn't know if it can encode until Bind.
|
||||
return BranchEncodeSucceeded::Success;
|
||||
}
|
||||
|
||||
[[nodiscard]] BranchEncodeSucceeded bc(ARMEmitter::Condition Cond, BiDirectionalLabel* Label) {
|
||||
void bc(ARMEmitter::Condition Cond, BiDirectionalLabel* Label) {
|
||||
if (Label->Backward.Location) {
|
||||
return bc(Cond, &Label->Backward);
|
||||
bc(Cond, &Label->Backward);
|
||||
} else {
|
||||
return bc(Cond, &Label->Forward);
|
||||
bc(Cond, &Label->Forward);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -114,32 +98,25 @@ public:
|
||||
|
||||
UnconditionalBranch(Op, Imm);
|
||||
}
|
||||
[[nodiscard]] BranchEncodeSucceeded b(const BackwardLabel* Label) {
|
||||
void b(const BackwardLabel* Label) {
|
||||
int32_t Imm = static_cast<int32_t>(Label->Location - GetCursorAddress<uint8_t*>());
|
||||
if (Imm >= -134217728 && Imm <= 134217724 && ((Imm & 0b11) == 0)) {
|
||||
constexpr uint32_t Op = 0b0001'01 << 26;
|
||||
UnconditionalBranch(Op, Imm >> 2);
|
||||
return BranchEncodeSucceeded::Success;
|
||||
}
|
||||
LOGMAN_THROW_A_FMT(Imm >= -134217728 && Imm <= 134217724 && ((Imm & 0b11) == 0), "Unscaled offset too large");
|
||||
constexpr uint32_t Op = 0b0001'01 << 26;
|
||||
|
||||
// Can't encode.
|
||||
return BranchEncodeSucceeded::Failure;
|
||||
UnconditionalBranch(Op, Imm >> 2);
|
||||
}
|
||||
[[nodiscard]] BranchEncodeSucceeded b(ForwardLabel* Label) {
|
||||
void b(ForwardLabel* Label) {
|
||||
AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::B});
|
||||
constexpr uint32_t Op = 0b0001'01 << 26;
|
||||
|
||||
UnconditionalBranch(Op, 0);
|
||||
|
||||
// Forward label doesn't know if it can encode until Bind.
|
||||
return BranchEncodeSucceeded::Success;
|
||||
}
|
||||
|
||||
[[nodiscard]] BranchEncodeSucceeded b(BiDirectionalLabel* Label) {
|
||||
void b(BiDirectionalLabel* Label) {
|
||||
if (Label->Backward.Location) {
|
||||
return b(&Label->Backward);
|
||||
b(&Label->Backward);
|
||||
} else {
|
||||
return b(&Label->Forward);
|
||||
b(&Label->Forward);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -149,33 +126,25 @@ public:
|
||||
UnconditionalBranch(Op, Imm);
|
||||
}
|
||||
|
||||
[[nodiscard]] BranchEncodeSucceeded bl(const BackwardLabel* Label) {
|
||||
void bl(const BackwardLabel* Label) {
|
||||
int32_t Imm = static_cast<int32_t>(Label->Location - GetCursorAddress<uint8_t*>());
|
||||
if (Imm >= -134217728 && Imm <= 134217724 && ((Imm & 0b11) == 0)) {
|
||||
constexpr uint32_t Op = 0b1001'01 << 26;
|
||||
UnconditionalBranch(Op, Imm >> 2);
|
||||
LOGMAN_THROW_A_FMT(Imm >= -134217728 && Imm <= 134217724 && ((Imm & 0b11) == 0), "Unscaled offset too large");
|
||||
constexpr uint32_t Op = 0b1001'01 << 26;
|
||||
|
||||
return BranchEncodeSucceeded::Success;
|
||||
}
|
||||
|
||||
// Can't encode.
|
||||
return BranchEncodeSucceeded::Failure;
|
||||
UnconditionalBranch(Op, Imm >> 2);
|
||||
}
|
||||
[[nodiscard]] BranchEncodeSucceeded bl(ForwardLabel* Label) {
|
||||
void bl(ForwardLabel* Label) {
|
||||
AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::B});
|
||||
constexpr uint32_t Op = 0b1001'01 << 26;
|
||||
|
||||
UnconditionalBranch(Op, 0);
|
||||
|
||||
// Forward label doesn't know if it can encode until Bind.
|
||||
return BranchEncodeSucceeded::Success;
|
||||
}
|
||||
|
||||
[[nodiscard]] BranchEncodeSucceeded bl(BiDirectionalLabel* Label) {
|
||||
void bl(BiDirectionalLabel* Label) {
|
||||
if (Label->Backward.Location) {
|
||||
return bl(&Label->Backward);
|
||||
bl(&Label->Backward);
|
||||
} else {
|
||||
return bl(&Label->Forward);
|
||||
bl(&Label->Forward);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -186,35 +155,28 @@ public:
|
||||
CompareAndBranch(Op, s, rt, Imm);
|
||||
}
|
||||
|
||||
[[nodiscard]] BranchEncodeSucceeded cbz(ARMEmitter::Size s, ARMEmitter::Register rt, const BackwardLabel* Label) {
|
||||
void cbz(ARMEmitter::Size s, ARMEmitter::Register rt, const BackwardLabel* Label) {
|
||||
int32_t Imm = static_cast<int32_t>(Label->Location - GetCursorAddress<uint8_t*>());
|
||||
LOGMAN_THROW_A_FMT(Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0), "Unscaled offset too large");
|
||||
|
||||
if (Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0)) {
|
||||
constexpr uint32_t Op = 0b0011'0100 << 24;
|
||||
CompareAndBranch(Op, s, rt, Imm >> 2);
|
||||
return BranchEncodeSucceeded::Success;
|
||||
}
|
||||
constexpr uint32_t Op = 0b0011'0100 << 24;
|
||||
|
||||
// Can't encode.
|
||||
return BranchEncodeSucceeded::Failure;
|
||||
CompareAndBranch(Op, s, rt, Imm >> 2);
|
||||
}
|
||||
|
||||
[[nodiscard]] BranchEncodeSucceeded cbz(ARMEmitter::Size s, ARMEmitter::Register rt, ForwardLabel* Label) {
|
||||
void cbz(ARMEmitter::Size s, ARMEmitter::Register rt, ForwardLabel* Label) {
|
||||
AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::BC});
|
||||
|
||||
constexpr uint32_t Op = 0b0011'0100 << 24;
|
||||
|
||||
CompareAndBranch(Op, s, rt, 0);
|
||||
|
||||
// Forward label doesn't know if it can encode until Bind.
|
||||
return BranchEncodeSucceeded::Success;
|
||||
}
|
||||
|
||||
[[nodiscard]] BranchEncodeSucceeded cbz(ARMEmitter::Size s, ARMEmitter::Register rt, BiDirectionalLabel* Label) {
|
||||
void cbz(ARMEmitter::Size s, ARMEmitter::Register rt, BiDirectionalLabel* Label) {
|
||||
if (Label->Backward.Location) {
|
||||
return cbz(s, rt, &Label->Backward);
|
||||
cbz(s, rt, &Label->Backward);
|
||||
} else {
|
||||
return cbz(s, rt, &Label->Forward);
|
||||
cbz(s, rt, &Label->Forward);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -224,35 +186,28 @@ public:
|
||||
CompareAndBranch(Op, s, rt, Imm);
|
||||
}
|
||||
|
||||
[[nodiscard]] BranchEncodeSucceeded cbnz(ARMEmitter::Size s, ARMEmitter::Register rt, const BackwardLabel* Label) {
|
||||
void cbnz(ARMEmitter::Size s, ARMEmitter::Register rt, const BackwardLabel* Label) {
|
||||
int32_t Imm = static_cast<int32_t>(Label->Location - GetCursorAddress<uint8_t*>());
|
||||
LOGMAN_THROW_A_FMT(Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0), "Unscaled offset too large");
|
||||
|
||||
if (Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0)) {
|
||||
constexpr uint32_t Op = 0b0011'0101 << 24;
|
||||
CompareAndBranch(Op, s, rt, Imm >> 2);
|
||||
return BranchEncodeSucceeded::Success;
|
||||
}
|
||||
constexpr uint32_t Op = 0b0011'0101 << 24;
|
||||
|
||||
// Can't encode.
|
||||
return BranchEncodeSucceeded::Failure;
|
||||
CompareAndBranch(Op, s, rt, Imm >> 2);
|
||||
}
|
||||
|
||||
[[nodiscard]] BranchEncodeSucceeded cbnz(ARMEmitter::Size s, ARMEmitter::Register rt, ForwardLabel* Label) {
|
||||
void cbnz(ARMEmitter::Size s, ARMEmitter::Register rt, ForwardLabel* Label) {
|
||||
AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::BC});
|
||||
|
||||
constexpr uint32_t Op = 0b0011'0101 << 24;
|
||||
|
||||
CompareAndBranch(Op, s, rt, 0);
|
||||
|
||||
// Forward label doesn't know if it can encode until Bind.
|
||||
return BranchEncodeSucceeded::Success;
|
||||
}
|
||||
|
||||
[[nodiscard]] BranchEncodeSucceeded cbnz(ARMEmitter::Size s, ARMEmitter::Register rt, BiDirectionalLabel* Label) {
|
||||
void cbnz(ARMEmitter::Size s, ARMEmitter::Register rt, BiDirectionalLabel* Label) {
|
||||
if (Label->Backward.Location) {
|
||||
return cbnz(s, rt, &Label->Backward);
|
||||
cbnz(s, rt, &Label->Backward);
|
||||
} else {
|
||||
return cbnz(s, rt, &Label->Forward);
|
||||
cbnz(s, rt, &Label->Forward);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -262,35 +217,28 @@ public:
|
||||
|
||||
TestAndBranch(Op, rt, Bit, Imm);
|
||||
}
|
||||
[[nodiscard]] BranchEncodeSucceeded tbz(ARMEmitter::Register rt, uint32_t Bit, const BackwardLabel* Label) {
|
||||
void tbz(ARMEmitter::Register rt, uint32_t Bit, const BackwardLabel* Label) {
|
||||
int32_t Imm = static_cast<int32_t>(Label->Location - GetCursorAddress<uint8_t*>());
|
||||
LOGMAN_THROW_A_FMT(Imm >= -32768 && Imm <= 32764 && ((Imm & 0b11) == 0), "Unscaled offset too large");
|
||||
|
||||
if (Imm >= -32768 && Imm <= 32764 && ((Imm & 0b11) == 0)) {
|
||||
constexpr uint32_t Op = 0b0011'0110 << 24;
|
||||
TestAndBranch(Op, rt, Bit, Imm >> 2);
|
||||
return BranchEncodeSucceeded::Success;
|
||||
}
|
||||
constexpr uint32_t Op = 0b0011'0110 << 24;
|
||||
|
||||
// Can't encode.
|
||||
return BranchEncodeSucceeded::Failure;
|
||||
TestAndBranch(Op, rt, Bit, Imm >> 2);
|
||||
}
|
||||
|
||||
[[nodiscard]] BranchEncodeSucceeded tbz(ARMEmitter::Register rt, uint32_t Bit, ForwardLabel* Label) {
|
||||
void tbz(ARMEmitter::Register rt, uint32_t Bit, ForwardLabel* Label) {
|
||||
AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::TEST_BRANCH});
|
||||
|
||||
constexpr uint32_t Op = 0b0011'0110 << 24;
|
||||
|
||||
TestAndBranch(Op, rt, Bit, 0);
|
||||
|
||||
// Forward label doesn't know if it can encode until Bind.
|
||||
return BranchEncodeSucceeded::Success;
|
||||
}
|
||||
|
||||
[[nodiscard]] BranchEncodeSucceeded tbz(ARMEmitter::Register rt, uint32_t Bit, BiDirectionalLabel* Label) {
|
||||
void tbz(ARMEmitter::Register rt, uint32_t Bit, BiDirectionalLabel* Label) {
|
||||
if (Label->Backward.Location) {
|
||||
return tbz(rt, Bit, &Label->Backward);
|
||||
tbz(rt, Bit, &Label->Backward);
|
||||
} else {
|
||||
return tbz(rt, Bit, &Label->Forward);
|
||||
tbz(rt, Bit, &Label->Forward);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -299,34 +247,27 @@ public:
|
||||
|
||||
TestAndBranch(Op, rt, Bit, Imm);
|
||||
}
|
||||
[[nodiscard]] BranchEncodeSucceeded tbnz(ARMEmitter::Register rt, uint32_t Bit, const BackwardLabel* Label) {
|
||||
void tbnz(ARMEmitter::Register rt, uint32_t Bit, const BackwardLabel* Label) {
|
||||
int32_t Imm = static_cast<int32_t>(Label->Location - GetCursorAddress<uint8_t*>());
|
||||
LOGMAN_THROW_A_FMT(Imm >= -32768 && Imm <= 32764 && ((Imm & 0b11) == 0), "Unscaled offset too large");
|
||||
|
||||
if (Imm >= -32768 && Imm <= 32764 && ((Imm & 0b11) == 0)) {
|
||||
constexpr uint32_t Op = 0b0011'0111 << 24;
|
||||
TestAndBranch(Op, rt, Bit, Imm >> 2);
|
||||
return BranchEncodeSucceeded::Success;
|
||||
}
|
||||
constexpr uint32_t Op = 0b0011'0111 << 24;
|
||||
|
||||
// Can't encode.
|
||||
return BranchEncodeSucceeded::Failure;
|
||||
TestAndBranch(Op, rt, Bit, Imm >> 2);
|
||||
}
|
||||
|
||||
[[nodiscard]] BranchEncodeSucceeded tbnz(ARMEmitter::Register rt, uint32_t Bit, ForwardLabel* Label) {
|
||||
void tbnz(ARMEmitter::Register rt, uint32_t Bit, ForwardLabel* Label) {
|
||||
AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::TEST_BRANCH});
|
||||
constexpr uint32_t Op = 0b0011'0111 << 24;
|
||||
|
||||
TestAndBranch(Op, rt, Bit, 0);
|
||||
|
||||
// Forward label doesn't know if it can encode until Bind.
|
||||
return BranchEncodeSucceeded::Success;
|
||||
}
|
||||
|
||||
[[nodiscard]] BranchEncodeSucceeded tbnz(ARMEmitter::Register rt, uint32_t Bit, BiDirectionalLabel* Label) {
|
||||
void tbnz(ARMEmitter::Register rt, uint32_t Bit, BiDirectionalLabel* Label) {
|
||||
if (Label->Backward.Location) {
|
||||
return tbnz(rt, Bit, &Label->Backward);
|
||||
tbnz(rt, Bit, &Label->Backward);
|
||||
} else {
|
||||
return tbnz(rt, Bit, &Label->Forward);
|
||||
tbnz(rt, Bit, &Label->Forward);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -12,7 +12,6 @@
|
||||
#include <CodeEmitter/Registers.h>
|
||||
|
||||
#include <array>
|
||||
#include <bit>
|
||||
#include <cstdint>
|
||||
#include <utility>
|
||||
#include <type_traits>
|
||||
@@ -87,14 +86,6 @@ constexpr size_t SubRegSizeInBits(SubRegSize size) {
|
||||
return size_t {8} << FEXCore::ToUnderlying(size);
|
||||
}
|
||||
|
||||
// Many floating point operations constrain their element sizes to the
|
||||
// main three float sizes half, single, and double precision. This just
|
||||
// combines all the checks together for brevity.
|
||||
[[nodiscard]]
|
||||
constexpr bool IsStandardFloatSize(SubRegSize size) {
|
||||
return size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit;
|
||||
}
|
||||
|
||||
/* This `ScalarRegSize` enum is used for most scalar float
|
||||
* operations.
|
||||
*
|
||||
@@ -586,15 +577,6 @@ concept IsXOrWRegister = std::is_same_v<T, XRegister> || std::is_same_v<T, WRegi
|
||||
template<typename T>
|
||||
concept IsQOrDRegister = std::is_same_v<T, QRegister> || std::is_same_v<T, DRegister>;
|
||||
|
||||
template<typename T>
|
||||
concept IsLabel = std::is_same_v<T, ARMEmitter::ForwardLabel> || std::is_same_v<T, ARMEmitter::BackwardLabel> ||
|
||||
std::is_same_v<T, ARMEmitter::BiDirectionalLabel> || std::is_same_v<T, ARMEmitter::ForwardLabel::Reference>;
|
||||
|
||||
enum class BranchEncodeSucceeded {
|
||||
Success,
|
||||
Failure,
|
||||
};
|
||||
|
||||
// Whether or not a given set of vector registers are sequential
|
||||
// in increasing order as far as the register file is concerned (modulo its size)
|
||||
//
|
||||
@@ -647,25 +629,19 @@ public:
|
||||
|
||||
// Bind a backward label to an address.
|
||||
// Address that is bound is the current emitter location.
|
||||
[[nodiscard]] bool Bind(BackwardLabel* Label) {
|
||||
void Bind(BackwardLabel* Label) {
|
||||
LOGMAN_THROW_A_FMT(Label->Location == nullptr, "Trying to bind a label twice");
|
||||
Label->Location = GetCursorAddress<uint8_t*>();
|
||||
|
||||
// Always binds because it is only storing a location.
|
||||
return true;
|
||||
}
|
||||
|
||||
[[nodiscard]] bool Bind(const ForwardLabel::Reference* Label) {
|
||||
void Bind(const ForwardLabel::Reference* Label) {
|
||||
uint8_t* CurrentAddress = GetCursorAddress<uint8_t*>();
|
||||
// Patch up the instructions
|
||||
switch (Label->Type) {
|
||||
case ForwardLabel::InstType::ADR: {
|
||||
uint32_t* Instruction = reinterpret_cast<uint32_t*>(Label->Location);
|
||||
int64_t Imm = reinterpret_cast<int64_t>(CurrentAddress) - reinterpret_cast<int64_t>(Instruction);
|
||||
if (!IsADRRange(Imm)) {
|
||||
// Can't bind.
|
||||
return false;
|
||||
}
|
||||
LOGMAN_THROW_A_FMT(IsADRRange(Imm), "Unscaled offset too large");
|
||||
uint32_t InstMask = 0b11 << 29 | 0b1111'1111'1111'1111'111 << 5;
|
||||
uint32_t Offset = static_cast<uint32_t>(Imm) & 0x3F'FFFF;
|
||||
uint32_t Inst = *Instruction & ~InstMask;
|
||||
@@ -677,12 +653,7 @@ public:
|
||||
case ForwardLabel::InstType::ADRP: {
|
||||
uint32_t* Instruction = reinterpret_cast<uint32_t*>(Label->Location);
|
||||
int64_t Imm = reinterpret_cast<int64_t>(CurrentAddress) - reinterpret_cast<int64_t>(Instruction);
|
||||
|
||||
if (!(IsADRPRange(Imm) && IsADRPAligned(Imm))) {
|
||||
// Can't bind.
|
||||
return false;
|
||||
}
|
||||
|
||||
LOGMAN_THROW_A_FMT(IsADRPRange(Imm) && IsADRPAligned(Imm), "Unscaled offset too large");
|
||||
Imm >>= 12;
|
||||
uint32_t InstMask = 0b11 << 29 | 0b1111'1111'1111'1111'111 << 5;
|
||||
uint32_t Offset = static_cast<uint32_t>(Imm) & 0x3F'FFFF;
|
||||
@@ -692,13 +663,11 @@ public:
|
||||
*Instruction = Inst;
|
||||
break;
|
||||
}
|
||||
|
||||
case ForwardLabel::InstType::B: {
|
||||
uint32_t* Instruction = reinterpret_cast<uint32_t*>(Label->Location);
|
||||
int64_t Imm = reinterpret_cast<int64_t>(CurrentAddress) - reinterpret_cast<int64_t>(Instruction);
|
||||
if (!(Imm >= -134217728 && Imm <= 134217724 && ((Imm & 0b11) == 0))) {
|
||||
// Can't bind.
|
||||
return false;
|
||||
}
|
||||
LOGMAN_THROW_A_FMT(Imm >= -134217728 && Imm <= 134217724 && ((Imm & 0b11) == 0), "Unscaled offset too large");
|
||||
Imm >>= 2;
|
||||
uint32_t InstMask = 0x3FF'FFFF;
|
||||
uint32_t Offset = static_cast<uint32_t>(Imm) & InstMask;
|
||||
@@ -708,13 +677,11 @@ public:
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
case ForwardLabel::InstType::TEST_BRANCH: {
|
||||
uint32_t* Instruction = reinterpret_cast<uint32_t*>(Label->Location);
|
||||
int64_t Imm = reinterpret_cast<int64_t>(CurrentAddress) - reinterpret_cast<int64_t>(Instruction);
|
||||
if (!(Imm >= -32768 && Imm <= 32764 && ((Imm & 0b11) == 0))) {
|
||||
// Can't bind.
|
||||
return false;
|
||||
}
|
||||
LOGMAN_THROW_A_FMT(Imm >= -32768 && Imm <= 32764 && ((Imm & 0b11) == 0), "Unscaled offset too large");
|
||||
Imm >>= 2;
|
||||
uint32_t InstMask = 0x3FFF;
|
||||
uint32_t Offset = static_cast<uint32_t>(Imm) & InstMask;
|
||||
@@ -728,10 +695,7 @@ public:
|
||||
case ForwardLabel::InstType::RELATIVE_LOAD: {
|
||||
uint32_t* Instruction = reinterpret_cast<uint32_t*>(Label->Location);
|
||||
int64_t Imm = reinterpret_cast<int64_t>(CurrentAddress) - reinterpret_cast<int64_t>(Instruction);
|
||||
if (!(Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0))) {
|
||||
// Can't bind.
|
||||
return false;
|
||||
}
|
||||
LOGMAN_THROW_A_FMT(Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0), "Unscaled offset too large");
|
||||
Imm >>= 2;
|
||||
uint32_t InstMask = 0x7'FFFF;
|
||||
uint32_t Offset = static_cast<uint32_t>(Imm) & InstMask;
|
||||
@@ -741,44 +705,38 @@ public:
|
||||
break;
|
||||
}
|
||||
case ForwardLabel::InstType::LONG_ADDRESS_GEN: {
|
||||
const auto* Instructions = reinterpret_cast<uint32_t*>(Label->Location);
|
||||
const auto ImmInstOne = reinterpret_cast<int64_t>(CurrentAddress) - reinterpret_cast<int64_t>(&Instructions[0]);
|
||||
const auto ImmInstTwo = reinterpret_cast<int64_t>(CurrentAddress) - reinterpret_cast<int64_t>(&Instructions[1]);
|
||||
const auto ImmInstThree = reinterpret_cast<int64_t>(CurrentAddress) - reinterpret_cast<int64_t>(&Instructions[2]);
|
||||
const auto OriginalOffset = GetCursorOffset();
|
||||
uint32_t* Instructions = reinterpret_cast<uint32_t*>(Label->Location);
|
||||
int64_t ImmInstOne = reinterpret_cast<int64_t>(CurrentAddress) - reinterpret_cast<int64_t>(&Instructions[0]);
|
||||
int64_t ImmInstTwo = reinterpret_cast<int64_t>(CurrentAddress) - reinterpret_cast<int64_t>(&Instructions[1]);
|
||||
auto OriginalOffset = GetCursorOffset();
|
||||
|
||||
const auto InstOffset = GetCursorOffsetFromAddress(Instructions);
|
||||
auto InstOffset = GetCursorOffsetFromAddress(Instructions);
|
||||
SetCursorOffset(InstOffset);
|
||||
|
||||
// We encoded the destination register in to the first instruction space.
|
||||
// Read it back.
|
||||
ARMEmitter::Register DestReg(Instructions[0]);
|
||||
|
||||
if (IsADRRange(ImmInstThree)) {
|
||||
// If within ADR range from the third instruction, then we can emit NOP+NOP+ADR
|
||||
if (IsADRRange(ImmInstTwo)) {
|
||||
// If within ADR range from the second instruction, then we can emit NOP+ADR
|
||||
nop();
|
||||
nop();
|
||||
adr(DestReg, static_cast<uint32_t>(ImmInstThree) & 0x7FFF);
|
||||
} else if (IsADRPRange(ImmInstTwo)) {
|
||||
adr(DestReg, static_cast<uint32_t>(ImmInstTwo) & 0x7FFF);
|
||||
} else if (IsADRPRange(ImmInstOne)) {
|
||||
|
||||
// If within ADRP range from the first instruction, then we are /definitely/ in range for the second instruction.
|
||||
// First check if we are in non-offset range for second instruction.
|
||||
if (IsADRPAligned(reinterpret_cast<uint64_t>(CurrentAddress))) {
|
||||
// We can emit nop + nop + adrp
|
||||
nop();
|
||||
nop();
|
||||
adrp(DestReg, static_cast<uint32_t>(ImmInstThree >> 12) & 0x7FFF);
|
||||
} else {
|
||||
// Not aligned, need nop + adrp + add
|
||||
// We can emit nop + adrp
|
||||
nop();
|
||||
adrp(DestReg, static_cast<uint32_t>(ImmInstTwo >> 12) & 0x7FFF);
|
||||
add(ARMEmitter::Size::i64Bit, DestReg, DestReg, ImmInstTwo & 0xFFF);
|
||||
} else {
|
||||
// Not aligned, need adrp + add
|
||||
adrp(DestReg, static_cast<uint32_t>(ImmInstOne >> 12) & 0x7FFF);
|
||||
add(ARMEmitter::Size::i64Bit, DestReg, DestReg, ImmInstOne & 0xFFF);
|
||||
}
|
||||
} else {
|
||||
// Stinky path, we need to emit a movz+movk+movk sequence.
|
||||
movz(ARMEmitter::Size::i64Bit, DestReg, uint32_t(ImmInstOne >> 32) & 0x7FFF, 32);
|
||||
movk(ARMEmitter::Size::i64Bit, DestReg, uint32_t(ImmInstOne >> 16) & 0xFFFF, 16);
|
||||
movk(ARMEmitter::Size::i64Bit, DestReg, uint32_t(ImmInstOne) & 0xFFFF);
|
||||
LOGMAN_MSG_A_FMT("Unscaled offset is too large");
|
||||
FEX_UNREACHABLE;
|
||||
}
|
||||
|
||||
SetCursorOffset(OriginalOffset);
|
||||
@@ -786,41 +744,27 @@ public:
|
||||
}
|
||||
default: LOGMAN_MSG_A_FMT("Unexpected inst type in label fixup");
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
// Bind a forward label to a location.
|
||||
// This walks all the instructions in the label's vector.
|
||||
// Then backpatching all instructions that have used the label.
|
||||
[[nodiscard]] bool Bind(ForwardLabel* Label) {
|
||||
bool Bound = true;
|
||||
void Bind(ForwardLabel* Label) {
|
||||
if (Label->FirstInst.Location) {
|
||||
Bound &= Bind(&Label->FirstInst);
|
||||
Bind(&Label->FirstInst);
|
||||
}
|
||||
for (auto& Inst : Label->Insts) {
|
||||
Bound &= Bind(&Inst);
|
||||
Bind(&Inst);
|
||||
}
|
||||
|
||||
return Bound;
|
||||
}
|
||||
|
||||
// Bind a bidirectional location to a location.
|
||||
// Binds both forwards and backwards depending on how the label was used.
|
||||
[[nodiscard]] bool Bind(BiDirectionalLabel* Label) {
|
||||
bool Bound = true;
|
||||
void Bind(BiDirectionalLabel* Label) {
|
||||
if (!Label->Backward.Location) {
|
||||
Bound &= Bind(&Label->Backward);
|
||||
Bind(&Label->Backward);
|
||||
}
|
||||
Bound &= Bind(&Label->Forward);
|
||||
|
||||
return Bound;
|
||||
}
|
||||
|
||||
static constexpr Condition InvertCondition(Condition cond) {
|
||||
// These behave as always, so it makes no sense to allow inverting these.
|
||||
LOGMAN_THROW_A_FMT(cond != Condition::CC_AL && cond != Condition::CC_NV, "Cannot invert CC_AL or CC_NV");
|
||||
return static_cast<Condition>(FEXCore::ToUnderlying(cond) ^ 1);
|
||||
Bind(&Label->Forward);
|
||||
}
|
||||
|
||||
#include <CodeEmitter/VixlUtils.inl>
|
||||
|
||||
@@ -60,7 +60,8 @@ public:
|
||||
}
|
||||
|
||||
void fcmla(SubRegSize size, ZRegister zda, PRegisterMerge pv, ZRegister zn, ZRegister zm, Rotation rot) {
|
||||
LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "SubRegSize must be 16-bit, 32-bit, or 64-bit");
|
||||
LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "SubRegSize must be 16-bit, "
|
||||
"32-bit, or 64-bit");
|
||||
LOGMAN_THROW_A_FMT(pv <= PReg::p7.Merging(), "fcmla can only use p0 to p7");
|
||||
|
||||
uint32_t Op = 0b0110'0100'0000'0000'0000'0000'0000'0000;
|
||||
@@ -75,7 +76,8 @@ public:
|
||||
}
|
||||
|
||||
void fcadd(SubRegSize size, ZRegister zd, PRegisterMerge pv, ZRegister zn, ZRegister zm, Rotation rot) {
|
||||
LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "SubRegSize must be 16-bit, 32-bit, or 64-bit");
|
||||
LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "SubRegSize must be 16-bit, "
|
||||
"32-bit, or 64-bit");
|
||||
LOGMAN_THROW_A_FMT(pv <= PReg::p7.Merging(), "fcadd can only use p0 to p7");
|
||||
LOGMAN_THROW_A_FMT(rot == Rotation::ROTATE_90 || rot == Rotation::ROTATE_270, "fcadd rotation may only be 90 or 270 degrees");
|
||||
LOGMAN_THROW_A_FMT(zd == zn, "fcadd zd and zn must be the same register");
|
||||
@@ -813,12 +815,16 @@ public:
|
||||
// SVE Integer Misc - Unpredicated
|
||||
// SVE floating-point trig select coefficient
|
||||
void ftssel(SubRegSize size, ZRegister zd, ZRegister zn, ZRegister zm) {
|
||||
LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "ftssel may only use 16/32/64-bit element sizes");
|
||||
LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "ftssel may only have "
|
||||
"16-bit, 32-bit, or 64-bit "
|
||||
"element sizes");
|
||||
SVEIntegerMiscUnpredicated(0b00, zm.Idx(), FEXCore::ToUnderlying(size), zd, zn);
|
||||
}
|
||||
// SVE floating-point exponential accelerator
|
||||
void fexpa(SubRegSize size, ZRegister zd, ZRegister zn) {
|
||||
LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "fexpa may only use 16/32/64-bit element sizes");
|
||||
LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "fexpa may only have "
|
||||
"16-bit, 32-bit, or 64-bit "
|
||||
"element sizes");
|
||||
SVEIntegerMiscUnpredicated(0b10, 0b00000, FEXCore::ToUnderlying(size), zd, zn);
|
||||
}
|
||||
// SVE constructive prefix (unpredicated)
|
||||
@@ -1497,9 +1503,9 @@ public:
|
||||
}
|
||||
|
||||
// SVE broadcast floating-point immediate (unpredicated)
|
||||
void fdup(SubRegSize size, ZRegister zd, float Value) {
|
||||
LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "Unsupported fmov size");
|
||||
|
||||
void fdup(ARMEmitter::SubRegSize size, ARMEmitter::ZRegister zd, float Value) {
|
||||
LOGMAN_THROW_A_FMT(size == ARMEmitter::SubRegSize::i16Bit || size == ARMEmitter::SubRegSize::i32Bit || size == ARMEmitter::SubRegSize::i64Bit,
|
||||
"Unsupported fmov size");
|
||||
uint32_t Imm {};
|
||||
if (size == SubRegSize::i16Bit) {
|
||||
LOGMAN_MSG_A_FMT("Unsupported");
|
||||
@@ -1512,7 +1518,7 @@ public:
|
||||
|
||||
SVEBroadcastFloatImmUnpredicated(0b00, 0, Imm, size, zd);
|
||||
}
|
||||
void fmov(SubRegSize size, ZRegister zd, float Value) {
|
||||
void fmov(ARMEmitter::SubRegSize size, ARMEmitter::ZRegister zd, float Value) {
|
||||
fdup(size, zd, Value);
|
||||
}
|
||||
|
||||
@@ -1541,7 +1547,7 @@ public:
|
||||
void sqincp(SubRegSize size, XRegister rdn, PRegister pm) {
|
||||
SVEIncDecPredicateCountScalar(0, 1, 0b10, 0b00, size, rdn, pm);
|
||||
}
|
||||
void sqincp(SubRegSize size, XRegister rdn, PRegister pm, WRegister wn) {
|
||||
void sqincp(SubRegSize size, XRegister rdn, PRegister pm, [[maybe_unused]] WRegister wn) {
|
||||
LOGMAN_THROW_A_FMT(rdn.Idx() == wn.Idx(), "rdn and wn must be the same");
|
||||
SVEIncDecPredicateCountScalar(0, 1, 0b00, 0b00, size, rdn, pm);
|
||||
}
|
||||
@@ -1554,7 +1560,7 @@ public:
|
||||
void sqdecp(SubRegSize size, XRegister rdn, PRegister pm) {
|
||||
SVEIncDecPredicateCountScalar(0, 1, 0b10, 0b10, size, rdn, pm);
|
||||
}
|
||||
void sqdecp(SubRegSize size, XRegister rdn, PRegister pm, WRegister wn) {
|
||||
void sqdecp(SubRegSize size, XRegister rdn, PRegister pm, [[maybe_unused]] WRegister wn) {
|
||||
LOGMAN_THROW_A_FMT(rdn.Idx() == wn.Idx(), "rdn and wn must be the same");
|
||||
SVEIncDecPredicateCountScalar(0, 1, 0b00, 0b10, size, rdn, pm);
|
||||
}
|
||||
@@ -3296,7 +3302,7 @@ private:
|
||||
const auto log2_size_bytes = FEXCore::ilog2(size_bytes);
|
||||
|
||||
// We can index up to 512-bit registers with dup
|
||||
const auto max_index = (64U >> log2_size_bytes) - 1;
|
||||
[[maybe_unused]] const auto max_index = (64U >> log2_size_bytes) - 1;
|
||||
LOGMAN_THROW_A_FMT(Index <= max_index, "dup index ({}) too large. Must be within [0, {}].", Index, max_index);
|
||||
|
||||
// imm2:tsz make up a 7 bit wide field, with each increasing element size
|
||||
@@ -3326,7 +3332,7 @@ private:
|
||||
|
||||
uint32_t shift = 0;
|
||||
if (!is_uint8_imm) {
|
||||
const bool is_uint16_imm = (imm >> 16) == 0;
|
||||
[[maybe_unused]] const bool is_uint16_imm = (imm >> 16) == 0;
|
||||
|
||||
LOGMAN_THROW_A_FMT(is_uint16_imm, "Immediate ({}) must be a 16-bit value within [256, 65280]", imm);
|
||||
LOGMAN_THROW_A_FMT((imm % 256) == 0, "Immediate ({}) must be a multiple of 256", imm);
|
||||
@@ -3395,8 +3401,8 @@ private:
|
||||
}
|
||||
|
||||
void SVEBroadcastFloatImmPredicated(SubRegSize size, ZRegister zd, PRegister pg, float value) {
|
||||
LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "Unsupported fcpy/fmov size");
|
||||
|
||||
LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "Unsupported fcpy/fmov "
|
||||
"size");
|
||||
uint32_t imm {};
|
||||
if (size == SubRegSize::i16Bit) {
|
||||
LOGMAN_MSG_A_FMT("Unsupported");
|
||||
@@ -3572,7 +3578,7 @@ private:
|
||||
// SVE2 floating-point pairwise operations
|
||||
void SVEFloatPairwiseArithmetic(uint32_t opc, SubRegSize size, PRegister pg, ZRegister zd, ZRegister zn, ZRegister zm) {
|
||||
LOGMAN_THROW_A_FMT(zd == zn, "zd needs to equal zn");
|
||||
LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "Invalid float size");
|
||||
LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "Invalid float size");
|
||||
LOGMAN_THROW_A_FMT(pg <= PReg::p7, "Can only use p0-p7 as a governing predicate");
|
||||
|
||||
uint32_t Instr = 0b0110'0100'0001'0000'1000'0000'0000'0000;
|
||||
@@ -3586,7 +3592,7 @@ private:
|
||||
|
||||
// SVE floating-point arithmetic (unpredicated)
|
||||
void SVEFloatArithmeticUnpredicated(uint32_t opc, SubRegSize size, ZRegister zm, ZRegister zn, ZRegister zd) {
|
||||
LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "Invalid float size");
|
||||
LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "Invalid float size");
|
||||
|
||||
uint32_t Instr = 0b0110'0101'0000'0000'0000'0000'0000'0000;
|
||||
Instr |= FEXCore::ToUnderlying(size) << 22;
|
||||
@@ -3694,7 +3700,7 @@ private:
|
||||
// SVE floating-point arithmetic (predicated)
|
||||
void SVEFloatArithmeticPredicated(uint32_t opc, SubRegSize size, PRegister pg, ZRegister zd, ZRegister zn, ZRegister zm) {
|
||||
LOGMAN_THROW_A_FMT(zd == zn, "zn needs to equal zd");
|
||||
LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "Invalid float size");
|
||||
LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "Invalid float size");
|
||||
LOGMAN_THROW_A_FMT(pg <= PReg::p7, "Can only use p0-p7 as a governing predicate");
|
||||
|
||||
uint32_t Instr = 0b0110'0101'0000'0000'1000'0000'0000'0000;
|
||||
@@ -3722,7 +3728,9 @@ private:
|
||||
}
|
||||
|
||||
void SVEFPRecursiveReduction(uint32_t opc, SubRegSize size, VRegister vd, PRegister pg, ZRegister zn) {
|
||||
LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "FP reduction operation can only use 16/32/64-bit element sizes");
|
||||
LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "FP reduction operation can "
|
||||
"only use 16-bit, 32-bit, "
|
||||
"or 64-bit element sizes");
|
||||
LOGMAN_THROW_A_FMT(pg <= PReg::p7, "FP reduction operation can only use p0-p7 as a governing predicate");
|
||||
|
||||
uint32_t Instr = 0b0110'0101'0000'0000'0010'0000'0000'0000;
|
||||
@@ -4104,7 +4112,7 @@ private:
|
||||
// 0b111 - I - Current
|
||||
|
||||
LOGMAN_THROW_A_FMT(pg <= PReg::p7, "Can only use p0-p7 as a governing predicate");
|
||||
LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "Unsupported size in {}", __func__);
|
||||
LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "Unsupported size in {}", __func__);
|
||||
|
||||
uint32_t Instr = 0b0110'0101'0000'0000'1010'0000'0000'0000;
|
||||
Instr |= FEXCore::ToUnderlying(size) << 22;
|
||||
@@ -4152,7 +4160,7 @@ private:
|
||||
|
||||
const auto& op_data = mem_op.MetaType.ScalarVectorType;
|
||||
const bool is_scaled = op_data.scale != 0;
|
||||
const auto msize_value = FEXCore::ToUnderlying(msize);
|
||||
[[maybe_unused]] const auto msize_value = FEXCore::ToUnderlying(msize);
|
||||
|
||||
LOGMAN_THROW_A_FMT(op_data.scale == 0 || op_data.scale == msize_value, "scale may only be 0 or {}", msize_value);
|
||||
|
||||
@@ -4266,7 +4274,7 @@ private:
|
||||
const auto msize_value = FEXCore::ToUnderlying(msize);
|
||||
const auto msize_bytes = 1U << msize_value;
|
||||
|
||||
const auto imm_limit = (32U << msize_value) - msize_bytes;
|
||||
[[maybe_unused]] const auto imm_limit = (32U << msize_value) - msize_bytes;
|
||||
const auto imm = mem_op.MetaType.VectorImmType.Imm;
|
||||
const auto imm_to_encode = imm >> msize_value;
|
||||
|
||||
@@ -4332,8 +4340,8 @@ private:
|
||||
LOGMAN_THROW_A_FMT(pg <= PReg::p7, "Can only use p0-p7 as a governing predicate");
|
||||
LOGMAN_THROW_A_FMT((imm % num_regs) == 0, "Offset must be a multiple of {}", num_regs);
|
||||
|
||||
const auto min_offset = -8 * num_regs;
|
||||
const auto max_offset = 7 * num_regs;
|
||||
[[maybe_unused]] const auto min_offset = -8 * num_regs;
|
||||
[[maybe_unused]] const auto max_offset = 7 * num_regs;
|
||||
LOGMAN_THROW_A_FMT(imm >= min_offset && imm <= max_offset,
|
||||
"Invalid load/store offset ({}). Offset must be a multiple of {} and be within [{}, {}]", imm, num_regs, min_offset,
|
||||
max_offset);
|
||||
@@ -4440,8 +4448,8 @@ private:
|
||||
LOGMAN_THROW_A_FMT(pg <= PReg::p7, "Can only use p0-p7 as a governing predicate");
|
||||
|
||||
const auto esize = static_cast<int>(16 << ssz);
|
||||
const auto max_imm = (esize << 3) - esize;
|
||||
const auto min_imm = -(max_imm + esize);
|
||||
[[maybe_unused]] const auto max_imm = (esize << 3) - esize;
|
||||
[[maybe_unused]] const auto min_imm = -(max_imm + esize);
|
||||
|
||||
LOGMAN_THROW_A_FMT((imm % esize) == 0, "imm ({}) must be a multiple of {}", imm, esize);
|
||||
LOGMAN_THROW_A_FMT(imm >= min_imm && imm <= max_imm, "imm ({}) must be within [{}, {}]", imm, min_imm, max_imm);
|
||||
@@ -4485,7 +4493,7 @@ private:
|
||||
const auto msize_value = FEXCore::ToUnderlying(msize);
|
||||
|
||||
const auto data_size_bytes = 1U << msize_value;
|
||||
const auto max_imm = (64U << msize_value) - data_size_bytes;
|
||||
[[maybe_unused]] const auto max_imm = (64U << msize_value) - data_size_bytes;
|
||||
LOGMAN_THROW_A_FMT((imm % data_size_bytes) == 0 && imm <= max_imm, "imm must be a multiple of {} and be within [0, {}]",
|
||||
data_size_bytes, max_imm);
|
||||
|
||||
@@ -4713,7 +4721,7 @@ private:
|
||||
|
||||
void SVEFloatUnary(uint32_t opc, SubRegSize size, PRegister pg, ZRegister zn, ZRegister zd) {
|
||||
LOGMAN_THROW_A_FMT(pg <= PReg::p7, "Can only use p0-p7 as a governing predicate");
|
||||
LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "Unsupported size in {}", __func__);
|
||||
LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "Unsupported size in {}", __func__);
|
||||
|
||||
uint32_t Instr = 0b0110'0101'0000'1100'1010'0000'0000'0000;
|
||||
Instr |= FEXCore::ToUnderlying(size) << 22;
|
||||
@@ -4801,7 +4809,8 @@ private:
|
||||
}
|
||||
|
||||
void SVEFPUnaryOpsUnpredicated(uint32_t opc, SubRegSize size, ZRegister zd, ZRegister zn) {
|
||||
LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "SubRegSize must be 16-bit, 32-bit, or 64-bit");
|
||||
LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "SubRegSize must be 16-bit, "
|
||||
"32-bit, or 64-bit");
|
||||
|
||||
uint32_t Instr = 0b0110'0101'0000'1000'0011'0000'0000'0000;
|
||||
Instr |= FEXCore::ToUnderlying(size) << 22;
|
||||
@@ -4812,7 +4821,8 @@ private:
|
||||
}
|
||||
|
||||
void SVEFPSerialReductionPredicated(uint32_t opc, SubRegSize size, VRegister vd, PRegister pg, VRegister vn, ZRegister zm) {
|
||||
LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "SubRegSize must be 16-bit, 32-bit, or 64-bit");
|
||||
LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "SubRegSize must be 16-bit, "
|
||||
"32-bit, or 64-bit");
|
||||
LOGMAN_THROW_A_FMT(pg <= PReg::p7, "Can only use p0-p7 as a governing predicate");
|
||||
LOGMAN_THROW_A_FMT(vd == vn, "vn must be the same as vd");
|
||||
|
||||
@@ -4826,7 +4836,8 @@ private:
|
||||
}
|
||||
|
||||
void SVEFPCompareWithZero(uint32_t eqlt, uint32_t ne, SubRegSize size, PRegister pd, PRegister pg, ZRegister zn) {
|
||||
LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "SubRegSize must be 16-bit, 32-bit, or 64-bit");
|
||||
LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "SubRegSize must be 16-bit, "
|
||||
"32-bit, or 64-bit");
|
||||
LOGMAN_THROW_A_FMT(pg <= PReg::p7, "Can only use p0-p7 as a governing predicate");
|
||||
|
||||
uint32_t Instr = 0b0110'0101'0001'0000'0010'0000'0000'0000;
|
||||
@@ -4841,7 +4852,8 @@ private:
|
||||
|
||||
void SVEFPMultiplyAdd(uint32_t opc, SubRegSize size, ZRegister zd, PRegister pg, ZRegister zn, ZRegister zm) {
|
||||
// NOTE: opc also includes the op0 bit (bit 15) like op0:opc, since the fields are adjacent
|
||||
LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "SubRegSize must be 16-bit, 32-bit, or 64-bit");
|
||||
LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "SubRegSize must be 16-bit, "
|
||||
"32-bit, or 64-bit");
|
||||
LOGMAN_THROW_A_FMT(pg <= PReg::p7, "Can only use p0-p7 as a governing predicate");
|
||||
|
||||
uint32_t Instr = 0b0110'0101'0010'0000'0000'0000'0000'0000;
|
||||
@@ -4855,13 +4867,14 @@ private:
|
||||
}
|
||||
|
||||
void SVEFPMultiplyAddIndexed(uint32_t op, SubRegSize size, ZRegister zda, ZRegister zn, ZRegister zm, uint32_t index) {
|
||||
LOGMAN_THROW_A_FMT(IsStandardFloatSize(size), "SubRegSize must be 16-bit, 32-bit, or 64-bit");
|
||||
LOGMAN_THROW_A_FMT(size == SubRegSize::i16Bit || size == SubRegSize::i32Bit || size == SubRegSize::i64Bit, "SubRegSize must be 16-bit, "
|
||||
"32-bit, or 64-bit");
|
||||
LOGMAN_THROW_A_FMT((size <= SubRegSize::i32Bit && zm <= ZReg::z7) || (size == SubRegSize::i64Bit && zm <= ZReg::z15),
|
||||
"16-bit and 32-bit indexed variants may only use Zm between z0-z7\n"
|
||||
"64-bit variants may only use Zm between z0-z15");
|
||||
|
||||
const auto Underlying = FEXCore::ToUnderlying(size);
|
||||
const uint32_t IndexMax = (16 / (1U << Underlying)) - 1;
|
||||
[[maybe_unused]] const uint32_t IndexMax = (16 / (1U << Underlying)) - 1;
|
||||
LOGMAN_THROW_A_FMT(index <= IndexMax, "Index must be within 0-{}", IndexMax);
|
||||
|
||||
// Can be bit 20 or 19 depending on whether or not the element size is 64-bit.
|
||||
@@ -5117,15 +5130,14 @@ private:
|
||||
requires (std::is_same_v<T, float> || std::is_same_v<T, double>)
|
||||
using FloatToEquivalentUInt = std::conditional_t<std::is_same_v<T, float>, uint32_t, uint64_t>;
|
||||
|
||||
#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
|
||||
// Determines if a floating-point value is capable of being converted
|
||||
// into an 8-bit immediate. See pseudocode definition of VFPExpandImm
|
||||
// in ARM A-profile reference manual for a general overview of how this was derived.
|
||||
template<typename T>
|
||||
requires (std::is_same_v<T, float> || std::is_same_v<T, double>)
|
||||
[[nodiscard]]
|
||||
[[nodiscard, maybe_unused]]
|
||||
static bool IsValidFPValueForImm8(T value) {
|
||||
const uint64_t bits = std::bit_cast<FloatToEquivalentUInt<T>>(value);
|
||||
const uint64_t bits = FEXCore::BitCast<FloatToEquivalentUInt<T>>(value);
|
||||
const uint64_t datasize_idx = FEXCore::ilog2(sizeof(T)) - 1;
|
||||
|
||||
static constexpr std::array mantissa_masks {
|
||||
@@ -5163,15 +5175,12 @@ private:
|
||||
|
||||
return true;
|
||||
}
|
||||
#endif
|
||||
|
||||
protected:
|
||||
static uint32_t FP32ToImm8(float value) {
|
||||
#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
|
||||
LOGMAN_THROW_A_FMT(IsValidFPValueForImm8(value), "Value ({}) cannot be encoded into an 8-bit immediate", value);
|
||||
#endif
|
||||
|
||||
const auto bits = std::bit_cast<uint32_t>(value);
|
||||
const auto bits = FEXCore::BitCast<uint32_t>(value);
|
||||
const auto sign = (bits & 0x80000000) >> 24;
|
||||
const auto expb2 = (bits & 0x20000000) >> 23;
|
||||
const auto b5_to_0 = (bits >> 19) & 0x3F;
|
||||
@@ -5180,11 +5189,9 @@ protected:
|
||||
}
|
||||
|
||||
static uint32_t FP64ToImm8(double value) {
|
||||
#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
|
||||
LOGMAN_THROW_A_FMT(IsValidFPValueForImm8(value), "Value ({}) cannot be encoded into an 8-bit immediate", value);
|
||||
#endif
|
||||
|
||||
const auto bits = std::bit_cast<uint64_t>(value);
|
||||
const auto bits = FEXCore::BitCast<uint64_t>(value);
|
||||
const auto sign = (bits & 0x80000000'00000000) >> 56;
|
||||
const auto expb2 = (bits & 0x20000000'00000000) >> 55;
|
||||
const auto b5_to_0 = (bits >> 48) & 0x3F;
|
||||
@@ -5208,7 +5215,7 @@ private:
|
||||
uint32_t shift = 0;
|
||||
if (!is_int8_imm) {
|
||||
const int32_t imm16_limit = 32768;
|
||||
const bool is_int16_imm = -imm16_limit <= imm && imm < imm16_limit;
|
||||
[[maybe_unused]] const bool is_int16_imm = -imm16_limit <= imm && imm < imm16_limit;
|
||||
|
||||
LOGMAN_THROW_A_FMT(is_int16_imm, "Immediate ({}) must be a 16-bit value within [-32768, 32512]", imm);
|
||||
LOGMAN_THROW_A_FMT((imm % 256) == 0, "Immediate ({}) must be a multiple of 256", imm);
|
||||
|
||||
@@ -27,19 +27,21 @@ struct EmitterOps : Emitter {
|
||||
public:
|
||||
// Advanced SIMD scalar copy
|
||||
void dup(ScalarRegSize size, VRegister rd, VRegister rn, uint32_t Index) {
|
||||
constexpr uint32_t Op = 0b0101'1110'0000'0000'0000'01 << 10;
|
||||
|
||||
const uint32_t SizeImm = FEXCore::ToUnderlying(size);
|
||||
const uint32_t IndexShift = SizeImm + 1;
|
||||
const uint32_t ElementSize = 1U << SizeImm;
|
||||
const uint32_t MaxIndex = 128U / (ElementSize * 8);
|
||||
[[maybe_unused]] const uint32_t MaxIndex = 128U / (ElementSize * 8);
|
||||
|
||||
LOGMAN_THROW_A_FMT(Index < MaxIndex, "Index too large. Index={}, Max Index: {}", Index, MaxIndex);
|
||||
|
||||
const uint32_t imm5 = (Index << IndexShift) | ElementSize;
|
||||
|
||||
ASIMDScalarCopy(1, 1, imm5, 0b0000, rd, rn);
|
||||
ASIMDScalarCopy(Op, 1, imm5, 0b0000, rd, rn);
|
||||
}
|
||||
|
||||
void mov(ScalarRegSize size, VRegister rd, VRegister rn, uint32_t Index) {
|
||||
void mov(ARMEmitter::ScalarRegSize size, ARMEmitter::VRegister rd, ARMEmitter::VRegister rn, uint32_t Index) {
|
||||
dup(size, rd, rn, Index);
|
||||
}
|
||||
|
||||
@@ -1280,10 +1282,10 @@ public:
|
||||
|
||||
private:
|
||||
// Advanced SIMD scalar copy
|
||||
void ASIMDScalarCopy(uint32_t Q, uint32_t b28, uint32_t imm5, uint32_t imm4, VRegister rd, VRegister rn) {
|
||||
uint32_t Instr = 0b0000'1110'0000'0000'0000'01U << 10;
|
||||
void ASIMDScalarCopy(uint32_t Op, uint32_t Q, uint32_t imm5, uint32_t imm4, ARMEmitter::VRegister rd, ARMEmitter::VRegister rn) {
|
||||
uint32_t Instr = Op;
|
||||
|
||||
Instr |= Q << 30;
|
||||
Instr |= b28 << 28;
|
||||
Instr |= imm5 << 16;
|
||||
Instr |= imm4 << 11;
|
||||
Instr |= Encode_rn(rn);
|
||||
@@ -1381,7 +1383,7 @@ private:
|
||||
void ASIMDScalarXIndexedElement(uint32_t U, ScalarRegSize size, uint32_t opcode, VRegister rm, VRegister rn, VRegister rd, uint32_t index) {
|
||||
LOGMAN_THROW_A_FMT(size != ScalarRegSize::i8Bit, "Scalar size must not be 8-bit");
|
||||
|
||||
const auto invalid_bound = 16U >> FEXCore::ToUnderlying(size);
|
||||
[[maybe_unused]] const auto invalid_bound = 16U >> FEXCore::ToUnderlying(size);
|
||||
LOGMAN_THROW_A_FMT(index < invalid_bound, "Index ({}) must be within [0-{}]", index, invalid_bound - 1);
|
||||
|
||||
uint32_t Instr = 0b0101'1111'0000'0000'0000'0000'0000'0000;
|
||||
|
||||
@@ -41,6 +41,7 @@ static bool IsImmLogical(uint64_t value, unsigned width, unsigned* n = nullptr,
|
||||
[[maybe_unused]] constexpr auto kDRegSize = 64;
|
||||
|
||||
constexpr auto kWRegSize = 32;
|
||||
constexpr auto kXRegSize = 64;
|
||||
|
||||
LOGMAN_THROW_A_FMT((width == kBRegSize) || (width == kHRegSize) || (width == kSRegSize) || (width == kDRegSize), "Unexpected imm size");
|
||||
|
||||
@@ -128,8 +129,8 @@ static bool IsImmLogical(uint64_t value, unsigned width, unsigned* n = nullptr,
|
||||
// Compute the repeat distance d, and set up a bitmask covering the basic
|
||||
// unit of repetition (i.e. a word with the bottom d bits set). Also, in all
|
||||
// of these cases the N bit of the output will be zero.
|
||||
clz_a = std::countl_zero(a);
|
||||
int clz_c = std::countl_zero(c);
|
||||
clz_a = CountLeadingZeros(a, kXRegSize);
|
||||
int clz_c = CountLeadingZeros(c, kXRegSize);
|
||||
d = clz_a - clz_c;
|
||||
mask = ((UINT64_C(1) << d) - 1);
|
||||
out_n = 0;
|
||||
@@ -150,7 +151,7 @@ static bool IsImmLogical(uint64_t value, unsigned width, unsigned* n = nullptr,
|
||||
// of set bits in our word, meaning that we have the trivial case of
|
||||
// d == 64 and only one 'repetition'. Set up all the same variables as in
|
||||
// the general case above, and set the N bit in the output.
|
||||
clz_a = std::countl_zero(a);
|
||||
clz_a = CountLeadingZeros(a, kXRegSize);
|
||||
d = 64;
|
||||
mask = ~UINT64_C(0);
|
||||
out_n = 1;
|
||||
@@ -158,7 +159,7 @@ static bool IsImmLogical(uint64_t value, unsigned width, unsigned* n = nullptr,
|
||||
}
|
||||
|
||||
// If the repeat period d is not a power of two, it can't be encoded.
|
||||
if (!std::has_single_bit(uint32_t(d))) {
|
||||
if (!IsPowerOf2(d)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -178,7 +179,7 @@ static bool IsImmLogical(uint64_t value, unsigned width, unsigned* n = nullptr,
|
||||
static const uint64_t multipliers[] = {
|
||||
0x0000000000000001UL, 0x0000000100000001UL, 0x0001000100010001UL, 0x0101010101010101UL, 0x1111111111111111UL, 0x5555555555555555UL,
|
||||
};
|
||||
uint64_t multiplier = multipliers[std::countl_zero(uint64_t(d)) - 57];
|
||||
uint64_t multiplier = multipliers[CountLeadingZeros(d, kXRegSize) - 57];
|
||||
uint64_t candidate = (b - a) * multiplier;
|
||||
|
||||
if (value != candidate) {
|
||||
@@ -193,7 +194,7 @@ static bool IsImmLogical(uint64_t value, unsigned width, unsigned* n = nullptr,
|
||||
// Count the set bits in our basic stretch. The special case of clz(0) == -1
|
||||
// makes the answer come out right for stretches that reach the very top of
|
||||
// the word (e.g. numbers like 0xffffc00000000000).
|
||||
int clz_b = (b == 0) ? -1 : std::countl_zero(b);
|
||||
int clz_b = (b == 0) ? -1 : CountLeadingZeros(b, kXRegSize);
|
||||
int s = clz_a - clz_b;
|
||||
|
||||
// Decide how many bits to rotate right by, to put the low bit of that basic
|
||||
@@ -223,13 +224,9 @@ static bool IsImmLogical(uint64_t value, unsigned width, unsigned* n = nullptr,
|
||||
// 11110s 2 UInt(s)
|
||||
//
|
||||
// So we 'or' (2 * -d) with our computed s to form imms.
|
||||
if (n != nullptr) {
|
||||
if ((n != NULL) || (imm_s != NULL) || (imm_r != NULL)) {
|
||||
*n = out_n;
|
||||
}
|
||||
if (imm_s != nullptr) {
|
||||
*imm_s = ((2 * -d) | (s - 1)) & 0x3f;
|
||||
}
|
||||
if (imm_r != nullptr) {
|
||||
*imm_r = r;
|
||||
}
|
||||
|
||||
@@ -284,6 +281,11 @@ INT_1_TO_63_LIST(DECLARE_IS_UINT_N)
|
||||
|
||||
private:
|
||||
|
||||
template<typename V>
|
||||
static inline bool IsPowerOf2(V value) {
|
||||
return (value != 0) && ((value & (value - 1)) == 0);
|
||||
}
|
||||
|
||||
// Some compilers dislike negating unsigned integers,
|
||||
// so we provide an equivalent.
|
||||
template<typename T>
|
||||
@@ -296,4 +298,50 @@ static inline uint64_t LowestSetBit(uint64_t value) {
|
||||
return value & UnsignedNegate(value);
|
||||
}
|
||||
|
||||
template<typename V>
|
||||
static inline int CountLeadingZeros(V value, int width = (sizeof(V) * 8)) {
|
||||
#if COMPILER_HAS_BUILTIN_CLZ
|
||||
if (width == 32) {
|
||||
return (value == 0) ? 32 : __builtin_clz(static_cast<unsigned>(value));
|
||||
} else if (width == 64) {
|
||||
return (value == 0) ? 64 : __builtin_clzll(value);
|
||||
}
|
||||
#endif
|
||||
return CountLeadingZerosFallBack(value, width);
|
||||
}
|
||||
|
||||
static inline int CountLeadingZerosFallBack(uint64_t value, int width) {
|
||||
LOGMAN_THROW_A_FMT(IsPowerOf2(width) && (width <= 64), "Invalid width");
|
||||
if (value == 0) {
|
||||
return width;
|
||||
}
|
||||
int count = 0;
|
||||
value = value << (64 - width);
|
||||
if ((value & UINT64_C(0xffffffff00000000)) == 0) {
|
||||
count += 32;
|
||||
value = value << 32;
|
||||
}
|
||||
if ((value & UINT64_C(0xffff000000000000)) == 0) {
|
||||
count += 16;
|
||||
value = value << 16;
|
||||
}
|
||||
if ((value & UINT64_C(0xff00000000000000)) == 0) {
|
||||
count += 8;
|
||||
value = value << 8;
|
||||
}
|
||||
if ((value & UINT64_C(0xf000000000000000)) == 0) {
|
||||
count += 4;
|
||||
value = value << 4;
|
||||
}
|
||||
if ((value & UINT64_C(0xc000000000000000)) == 0) {
|
||||
count += 2;
|
||||
value = value << 2;
|
||||
}
|
||||
if ((value & UINT64_C(0x8000000000000000)) == 0) {
|
||||
count += 1;
|
||||
}
|
||||
count += (value == 0);
|
||||
return count;
|
||||
}
|
||||
|
||||
public:
|
||||
@@ -4,8 +4,7 @@ file(GLOB GEN_CONFIG_SOURCES CONFIGURE_DEPENDS *.json.in)
|
||||
# Any application configuration json file gets installed
|
||||
foreach(CONFIG_SRC ${CONFIG_SOURCES})
|
||||
install(FILES ${CONFIG_SRC}
|
||||
DESTINATION ${DATA_DIRECTORY}/AppConfig/
|
||||
COMPONENT Runtime)
|
||||
DESTINATION ${DATA_DIRECTORY}/AppConfig/)
|
||||
endforeach()
|
||||
|
||||
# Any configuration file json file that needs to be generated
|
||||
@@ -15,10 +14,12 @@ foreach(GEN_CONFIG_SRC ${GEN_CONFIG_SOURCES})
|
||||
get_filename_component(CONFIG_NAME ${GEN_CONFIG_SRC} NAME_WLE)
|
||||
|
||||
# Configure it
|
||||
configure_file(${GEN_CONFIG_SRC} ${CMAKE_BINARY_DIR}/Data/AppConfig/${CONFIG_NAME})
|
||||
configure_file(
|
||||
${GEN_CONFIG_SRC}
|
||||
${CMAKE_BINARY_DIR}/Data/AppConfig/${CONFIG_NAME})
|
||||
|
||||
# Then install the configured json
|
||||
install(FILES ${CMAKE_BINARY_DIR}/Data/AppConfig/${CONFIG_NAME}
|
||||
DESTINATION ${DATA_DIRECTORY}/AppConfig/
|
||||
COMPONENT Runtime)
|
||||
install(
|
||||
FILES ${CMAKE_BINARY_DIR}/Data/AppConfig/${CONFIG_NAME}
|
||||
DESTINATION ${DATA_DIRECTORY}/AppConfig/)
|
||||
endforeach()
|
||||
@@ -0,0 +1,3 @@
|
||||
x86 and x86-64 Linux emulator
|
||||
|
||||
FEX allows you to run x86 applications on ARM64 Linux devices. It offers broad compatibility with both 32-bit and 64-bit binaries, and it can be used alongside Wine/Proton to play Windows games.
|
||||
Executable
+18
@@ -0,0 +1,18 @@
|
||||
#!/bin/sh
|
||||
set -e
|
||||
update_binfmt() {
|
||||
# Check for update-binfmts
|
||||
command -v update-binfmts >/dev/null || return 0
|
||||
|
||||
# Setup binfmt_misc
|
||||
update-binfmts --import FEX-x86
|
||||
update-binfmts --import FEX-x86_64
|
||||
}
|
||||
|
||||
# Install FEXInterpreter hardlink
|
||||
# Needs to be done before setting up binfmt_misc
|
||||
ln -f /usr/bin/FEXLoader /usr/bin/FEXInterpreter
|
||||
|
||||
if [ $(uname -m) = 'aarch64' ]; then
|
||||
update_binfmt
|
||||
fi
|
||||
Executable
+17
@@ -0,0 +1,17 @@
|
||||
#!/bin/sh
|
||||
set -e
|
||||
update_binfmt() {
|
||||
# Check for update-binfmts
|
||||
command -v update-binfmts >/dev/null || return 0
|
||||
|
||||
# Uninstall
|
||||
update-binfmts --unimport FEX-x86
|
||||
update-binfmts --unimport FEX-x86_64
|
||||
}
|
||||
|
||||
if [ $(uname -m) = 'aarch64' ]; then
|
||||
update_binfmt
|
||||
fi
|
||||
|
||||
# Remove FEXInterpreter hardlink
|
||||
unlink /usr/bin/FEXInterpreter
|
||||
@@ -0,0 +1 @@
|
||||
activate-noawait ldconfig
|
||||
@@ -1,18 +0,0 @@
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
include(FindPackageHandleStandardArgs)
|
||||
|
||||
find_package(PkgConfig QUIET)
|
||||
pkg_search_module(xxhash QUIET IMPORTED_TARGET xxhash libxxhash)
|
||||
find_package_handle_standard_args(xxhash
|
||||
REQUIRED_VARS xxhash_LINK_LIBRARIES
|
||||
VERSION_VAR xxhash_VERSION
|
||||
)
|
||||
|
||||
if (xxhash_FOUND AND NOT TARGET xxHash::xxhash)
|
||||
if (TARGET PkgConfig::xxhash)
|
||||
add_library(xxHash::xxhash ALIAS PkgConfig::xxhash)
|
||||
else()
|
||||
add_library(xxHash::xxhash ALIAS xxhash)
|
||||
endif()
|
||||
endif()
|
||||
@@ -1,15 +0,0 @@
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
# This applies some common linker options that reduce code size and linking time in Release mode. Namely:
|
||||
# --gc-sections: Linktime garbage collection, discards unused sections from the final output
|
||||
# --strip-all : Similar to running `strip`, discards the symbol table from the final output
|
||||
# --as-needed : Only includes libraries that are actually needed in the final output.
|
||||
|
||||
macro(LinkerGC target)
|
||||
if (CMAKE_BUILD_TYPE MATCHES "RELEASE")
|
||||
target_link_options(${target} PRIVATE
|
||||
"LINKER:--gc-sections"
|
||||
"LINKER:--strip-all"
|
||||
"LINKER:--as-needed")
|
||||
endif()
|
||||
endmacro()
|
||||
@@ -4,7 +4,6 @@ set(CMAKE_RC_COMPILER ${MINGW_TRIPLE}-windres)
|
||||
set(CMAKE_C_COMPILER ${MINGW_TRIPLE}-clang)
|
||||
set(CMAKE_CXX_COMPILER ${MINGW_TRIPLE}-clang++)
|
||||
set(CMAKE_DLLTOOL ${MINGW_TRIPLE}-dlltool)
|
||||
set(CMAKE_AR ${MINGW_TRIPLE}-ar)
|
||||
|
||||
# Compile everything as static to avoid requiring the MinGW runtime libraries, force page aligned sections so that
|
||||
# debug symbols work correctly, and disable loop alignment to workaround an LLVM bug
|
||||
|
||||
+1
-1
@@ -14,7 +14,7 @@ RUN mkdir build
|
||||
|
||||
ARG CC=clang-13
|
||||
ARG CXX=clang++-13
|
||||
RUN cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Release -DUSE_LINKER=lld -DENABLE_LTO=True -DBUILD_TESTING=False -DENABLE_ASSERTIONS=False -G Ninja .
|
||||
RUN cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Release -DUSE_LINKER=lld -DENABLE_LTO=True -DBUILD_TESTS=False -DENABLE_ASSERTIONS=False -G Ninja .
|
||||
RUN ninja
|
||||
|
||||
WORKDIR /FEX/build
|
||||
|
||||
@@ -3,21 +3,23 @@ function(GenBinFmt Name)
|
||||
get_filename_component(FMT_NAME ${Name} NAME_WE)
|
||||
|
||||
# Configure it
|
||||
configure_file(${Name} ${CMAKE_BINARY_DIR}/Data/binfmts/${FMT_NAME})
|
||||
configure_file(
|
||||
${Name}
|
||||
${CMAKE_BINARY_DIR}/Data/binfmts/${FMT_NAME})
|
||||
|
||||
# Then install the configured binfmt
|
||||
install(FILES ${CMAKE_BINARY_DIR}/Data/binfmts/${FMT_NAME}
|
||||
DESTINATION ${CMAKE_INSTALL_PREFIX}/share/binfmts/
|
||||
COMPONENT Runtime)
|
||||
install(
|
||||
FILES ${CMAKE_BINARY_DIR}/Data/binfmts/${FMT_NAME}
|
||||
DESTINATION ${CMAKE_INSTALL_PREFIX}/share/binfmts/)
|
||||
endfunction()
|
||||
|
||||
if (NOT USE_LEGACY_BINFMTMISC)
|
||||
configure_file(FEX-x86.conf.in ${CMAKE_BINARY_DIR}/Data/binfmts/FEX-x86.conf)
|
||||
configure_file(FEX-x86_64.conf.in ${CMAKE_BINARY_DIR}/Data/binfmts/FEX-x86_64.conf)
|
||||
|
||||
install(FILES ${CMAKE_BINARY_DIR}/Data/binfmts/FEX-x86.conf ${CMAKE_BINARY_DIR}/Data/binfmts/FEX-x86_64.conf
|
||||
DESTINATION ${CMAKE_INSTALL_PREFIX}/lib/binfmt.d/
|
||||
COMPONENT Runtime)
|
||||
install(
|
||||
FILES ${CMAKE_BINARY_DIR}/Data/binfmts/FEX-x86.conf ${CMAKE_BINARY_DIR}/Data/binfmts/FEX-x86_64.conf
|
||||
DESTINATION ${CMAKE_INSTALL_PREFIX}/lib/binfmt.d/)
|
||||
else()
|
||||
GenBinFmt(FEX-x86.in)
|
||||
GenBinFmt(FEX-x86_64.in)
|
||||
|
||||
@@ -1 +1 @@
|
||||
:FEX-x86:M:0:\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x03\x00:\xff\xff\xff\xff\xff\xfe\xfe\x00\x00\x00\x00\xff\xff\xff\xff\xff\xfe\xff\xff\xff:@CMAKE_INSTALL_PREFIX@/bin/FEX:POCF
|
||||
:FEX-x86:M:0:\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x03\x00:\xff\xff\xff\xff\xff\xfe\xfe\x00\x00\x00\x00\xff\xff\xff\xff\xff\xfe\xff\xff\xff:@CMAKE_INSTALL_PREFIX@/bin/FEXInterpreter:POCF
|
||||
@@ -1,5 +1,5 @@
|
||||
package fex
|
||||
interpreter @CMAKE_INSTALL_PREFIX@/bin/FEX
|
||||
interpreter @CMAKE_INSTALL_PREFIX@/bin/FEXInterpreter
|
||||
magic \x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x03\x00
|
||||
offset 0
|
||||
mask \xff\xff\xff\xff\xff\xfe\xfe\x00\x00\x00\x00\xff\xff\xff\xff\xff\xfe\xff\xff\xff
|
||||
|
||||
@@ -1 +1 @@
|
||||
:FEX-x86_64:M:0:\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x3e\x00:\xff\xff\xff\xff\xff\xfe\xfe\x00\x00\x00\x00\xff\xff\xff\xff\xff\xfe\xff\xff\xff:@CMAKE_INSTALL_PREFIX@/bin/FEX:POCF
|
||||
:FEX-x86_64:M:0:\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x3e\x00:\xff\xff\xff\xff\xff\xfe\xfe\x00\x00\x00\x00\xff\xff\xff\xff\xff\xfe\xff\xff\xff:@CMAKE_INSTALL_PREFIX@/bin/FEXInterpreter:POCF
|
||||
@@ -1,5 +1,5 @@
|
||||
package fex
|
||||
interpreter @CMAKE_INSTALL_PREFIX@/bin/FEX
|
||||
interpreter @CMAKE_INSTALL_PREFIX@/bin/FEXInterpreter
|
||||
magic \x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x3e\x00
|
||||
offset 0
|
||||
mask \xff\xff\xff\xff\xff\xfe\xfe\x00\x00\x00\x00\xff\xff\xff\xff\xff\xfe\xff\xff\xff
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
|
||||
let
|
||||
toolchain = pkgs.fetchzip {
|
||||
url = "https://github.com/bylaws/llvm-mingw/releases/download/20250920/llvm-mingw-20250920-ucrt-ubuntu-22.04-aarch64.tar.xz";
|
||||
sha256 = "sha256-LaojKjC8KzY+soW5u6eoDoXE3qtYk9Ejr7M3enTqRAE=";
|
||||
url = "https://github.com/bylaws/llvm-mingw/releases/download/20250305/llvm-mingw-20250305-ucrt-ubuntu-20.04-aarch64.tar.xz";
|
||||
sha256 = "sha256-cA03/ab9O61eO9+S2JzIXD4V0HzTXK5/AYyxW2d73Po=";
|
||||
};
|
||||
|
||||
cmakeToolchainFile = pkgs.substitute {
|
||||
@@ -45,7 +45,7 @@ pkgs.mkShell {
|
||||
fi
|
||||
'';
|
||||
|
||||
# E.g. cmake $FEX_CMAKE_TOOLCHAIN_ARM64EC -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_LTO=False -DBUILD_TESTING=False
|
||||
# E.g. cmake $FEX_CMAKE_TOOLCHAIN_ARM64EC -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_LTO=False -DBUILD_TESTS=False
|
||||
FEX_CMAKE_TOOLCHAIN_ARM64EC = "--toolchain ${cmakeToolchainFile} -DMINGW_TRIPLE=arm64ec-w64-mingw32 -DCMAKE_INSTALL_LIBDIR=/usr/lib/wine/aarch64-windows";
|
||||
FEX_CMAKE_TOOLCHAIN_WOW64 = "--toolchain ${cmakeToolchainFile} -DMINGW_TRIPLE=aarch64-w64-mingw32 -DCMAKE_INSTALL_LIBDIR=/usr/lib/wine/aarch64-windows";
|
||||
FEX_MESON_CROSSFILE = "--cross-file ${mesonCrossFile}";
|
||||
|
||||
@@ -18,4 +18,4 @@ then
|
||||
fi
|
||||
|
||||
set -o xtrace
|
||||
cmake $FEX_CMAKE_TOOLCHAIN_WOW64 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_LTO=False -DBUILD_TESTING=False $@
|
||||
cmake $FEX_CMAKE_TOOLCHAIN_WOW64 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_LTO=False -DBUILD_TESTS=False $@
|
||||
@@ -18,4 +18,4 @@ then
|
||||
fi
|
||||
|
||||
set -o xtrace
|
||||
cmake $FEX_CMAKE_TOOLCHAIN_ARM64EC -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_LTO=False -DBUILD_TESTING=False $@
|
||||
cmake $FEX_CMAKE_TOOLCHAIN_ARM64EC -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_LTO=False -DBUILD_TESTS=False $@
|
||||
@@ -14,4 +14,4 @@ fi
|
||||
rm -rf unittests/FEXLinuxTests
|
||||
|
||||
set -o xtrace
|
||||
cmake . $FEX_CMAKE_TOOLCHAINS -DBUILD_TESTING=ON -DBUILD_FEX_LINUX_TESTS=ON
|
||||
cmake . $FEX_CMAKE_TOOLCHAINS -DBUILD_TESTS=ON -DBUILD_FEX_LINUX_TESTS=ON
|
||||
Vendored
-1
@@ -1 +0,0 @@
|
||||
DisableFormat: true
|
||||
Vendored
+1
-1
Submodule External/Catch2 updated: b3fb4b9fea...8ac8190e49.
Vendored
+3
-2
@@ -1,5 +1,5 @@
|
||||
|
||||
add_library(softfloat_3e STATIC
|
||||
set (SRCS
|
||||
# F80 support
|
||||
src/extF80_add.c
|
||||
src/extF80_div.c
|
||||
@@ -84,7 +84,7 @@ add_library(softfloat_3e STATIC
|
||||
src/s_normSubnormalF32Sig.c
|
||||
src/s_f32UIToCommonNaN.c)
|
||||
|
||||
if (ARCHITECTURE_arm64 AND HAS_CLANG_PRESERVE_ALL)
|
||||
if (_M_ARM_64 AND HAS_CLANG_PRESERVE_ALL)
|
||||
list(APPEND DEFINES "-DFEXCORE_PRESERVE_ALL_ATTR=__attribute__((preserve_all));-DFEXCORE_HAS_PRESERVE_ALL_ATTR=1")
|
||||
else()
|
||||
list(APPEND DEFINES "-DFEXCORE_PRESERVE_ALL_ATTR=;-DFEXCORE_HAS_PRESERVE_ALL_ATTR=0")
|
||||
@@ -92,6 +92,7 @@ endif()
|
||||
|
||||
list(APPEND DEFINES "-DSOFTFLOAT_BUILTIN_CLZ=1;-DINLINE=static inline;-DINLINE_LEVEL=4;-DSOFTFLOAT_FAST_INT64=1;-DSOFTFLOAT_FAST_DIV32TO16=1;-DSOFTFLOAT_FAST_DIV64TO32=1")
|
||||
|
||||
add_library(softfloat_3e STATIC ${SRCS})
|
||||
target_include_directories(softfloat_3e PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include/)
|
||||
target_include_directories(softfloat_3e PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include/SoftFloat-3e/)
|
||||
target_compile_definitions(softfloat_3e PUBLIC ${DEFINES})
|
||||
Vendored
+1
-1
Submodule External/Vulkan-Headers updated: 450bd22322...cacef3039d.
Vendored
+2
-1
@@ -1,4 +1,4 @@
|
||||
add_library(cephes_128bit STATIC
|
||||
set(SRCS_128BIT
|
||||
src/128bit/Impl.cpp
|
||||
src/128bit/atanll.c
|
||||
src/128bit/constll.c
|
||||
@@ -11,6 +11,7 @@ add_library(cephes_128bit STATIC
|
||||
src/128bit/tanll.c)
|
||||
|
||||
# 128-bit library
|
||||
add_library(cephes_128bit STATIC ${SRCS_128BIT})
|
||||
target_link_libraries(cephes_128bit softfloat_3e)
|
||||
target_include_directories(cephes_128bit PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include/)
|
||||
target_compile_options(cephes_128bit PRIVATE -fno-builtin)
|
||||
+8
-3
@@ -169,9 +169,14 @@ View the diff from {self.name} here.
|
||||
|
||||
|
||||
class ClangFormatHelper(FormatHelper):
|
||||
name = "git-clang-format"
|
||||
name = "clang-format"
|
||||
friendly_name = "C/C++ code formatter"
|
||||
|
||||
@property
|
||||
def cformat_wrapper_path(self) -> str:
|
||||
relpath = "../../Scripts/clang-format.py"
|
||||
curpath = os.path.dirname(os.path.abspath(__file__))
|
||||
return os.path.abspath(os.path.normpath(os.path.join(curpath, relpath)))
|
||||
|
||||
@property
|
||||
def instructions(self) -> str:
|
||||
@@ -194,7 +199,7 @@ class ClangFormatHelper(FormatHelper):
|
||||
def clang_fmt_path(self) -> str:
|
||||
if "CLANG_FORMAT_PATH" in os.environ:
|
||||
return os.environ["CLANG_FORMAT_PATH"]
|
||||
return "git-clang-format-19"
|
||||
return "git-clang-format"
|
||||
|
||||
def has_tool(self) -> bool:
|
||||
cmd = [self.clang_fmt_path, "-h"]
|
||||
@@ -212,7 +217,7 @@ class ClangFormatHelper(FormatHelper):
|
||||
|
||||
cf_cmd = [
|
||||
self.clang_fmt_path,
|
||||
"--binary=clang-format-19",
|
||||
f"--binary={self.cformat_wrapper_path}",
|
||||
"--diff",
|
||||
]
|
||||
|
||||
|
||||
+31
-371
@@ -1,392 +1,52 @@
|
||||
#
|
||||
# This file is autogenerated by pip-compile with Python 3.13
|
||||
# This file is autogenerated by pip-compile with Python 3.11
|
||||
# by the following command:
|
||||
#
|
||||
# pip-compile --generate-hashes --output-file=requirements_formatting.txt --strip-extras requirements_formatting.txt.in
|
||||
# pip-compile --output-file=llvm/utils/git/requirements_formatting.txt llvm/utils/git/requirements_formatting.txt.in
|
||||
#
|
||||
black==25.1.0 \
|
||||
--hash=sha256:030b9759066a4ee5e5aca28c3c77f9c64789cdd4de8ac1df642c40b708be6171 \
|
||||
--hash=sha256:055e59b198df7ac0b7efca5ad7ff2516bca343276c466be72eb04a3bcc1f82d7 \
|
||||
--hash=sha256:0e519ecf93120f34243e6b0054db49c00a35f84f195d5bce7e9f5cfc578fc2da \
|
||||
--hash=sha256:172b1dbff09f86ce6f4eb8edf9dede08b1fce58ba194c87d7a4f1a5aa2f5b3c2 \
|
||||
--hash=sha256:1e2978f6df243b155ef5fa7e558a43037c3079093ed5d10fd84c43900f2d8ecc \
|
||||
--hash=sha256:33496d5cd1222ad73391352b4ae8da15253c5de89b93a80b3e2c8d9a19ec2666 \
|
||||
--hash=sha256:3b48735872ec535027d979e8dcb20bf4f70b5ac75a8ea99f127c106a7d7aba9f \
|
||||
--hash=sha256:4b60580e829091e6f9238c848ea6750efed72140b91b048770b64e74fe04908b \
|
||||
--hash=sha256:759e7ec1e050a15f89b770cefbf91ebee8917aac5c20483bc2d80a6c3a04df32 \
|
||||
--hash=sha256:8f0b18a02996a836cc9c9c78e5babec10930862827b1b724ddfe98ccf2f2fe4f \
|
||||
--hash=sha256:95e8176dae143ba9097f351d174fdaf0ccd29efb414b362ae3fd72bf0f710717 \
|
||||
--hash=sha256:96c1c7cd856bba8e20094e36e0f948718dc688dba4a9d78c3adde52b9e6c2299 \
|
||||
--hash=sha256:a1ee0a0c330f7b5130ce0caed9936a904793576ef4d2b98c40835d6a65afa6a0 \
|
||||
--hash=sha256:a22f402b410566e2d1c950708c77ebf5ebd5d0d88a6a2e87c86d9fb48afa0d18 \
|
||||
--hash=sha256:a39337598244de4bae26475f77dda852ea00a93bd4c728e09eacd827ec929df0 \
|
||||
--hash=sha256:afebb7098bfbc70037a053b91ae8437c3857482d3a690fefc03e9ff7aa9a5fd3 \
|
||||
--hash=sha256:bacabb307dca5ebaf9c118d2d2f6903da0d62c9faa82bd21a33eecc319559355 \
|
||||
--hash=sha256:bce2e264d59c91e52d8000d507eb20a9aca4a778731a08cfff7e5ac4a4bb7096 \
|
||||
--hash=sha256:d9e6827d563a2c820772b32ce8a42828dc6790f095f441beef18f96aa6f8294e \
|
||||
--hash=sha256:db8ea9917d6f8fc62abd90d944920d95e73c83a5ee3383493e35d271aca872e9 \
|
||||
--hash=sha256:ea0213189960bda9cf99be5b8c8ce66bb054af5e9e861249cd23471bd7b0b3ba \
|
||||
--hash=sha256:f3df5f1bf91d36002b0a75389ca8663510cf0531cca8aa5c1ef695b46d98655f
|
||||
black==23.9.1
|
||||
# via
|
||||
# -r requirements_formatting.txt.in
|
||||
# -r llvm/utils/git/requirements_formatting.txt.in
|
||||
# darker
|
||||
certifi==2025.7.14 \
|
||||
--hash=sha256:6b31f564a415d79ee77df69d757bb49a5bb53bd9f756cbbe24394ffd6fc1f4b2 \
|
||||
--hash=sha256:8ea99dbdfaaf2ba2f9bac77b9249ef62ec5218e7c2b2e903378ed5fccf765995
|
||||
# via
|
||||
# -r requirements_formatting.txt.in
|
||||
# requests
|
||||
cffi==1.15.1 \
|
||||
--hash=sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5 \
|
||||
--hash=sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef \
|
||||
--hash=sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104 \
|
||||
--hash=sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426 \
|
||||
--hash=sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405 \
|
||||
--hash=sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375 \
|
||||
--hash=sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a \
|
||||
--hash=sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e \
|
||||
--hash=sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc \
|
||||
--hash=sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf \
|
||||
--hash=sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185 \
|
||||
--hash=sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497 \
|
||||
--hash=sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3 \
|
||||
--hash=sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35 \
|
||||
--hash=sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c \
|
||||
--hash=sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83 \
|
||||
--hash=sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21 \
|
||||
--hash=sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca \
|
||||
--hash=sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984 \
|
||||
--hash=sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac \
|
||||
--hash=sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd \
|
||||
--hash=sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee \
|
||||
--hash=sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a \
|
||||
--hash=sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2 \
|
||||
--hash=sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192 \
|
||||
--hash=sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7 \
|
||||
--hash=sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585 \
|
||||
--hash=sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f \
|
||||
--hash=sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e \
|
||||
--hash=sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27 \
|
||||
--hash=sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b \
|
||||
--hash=sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e \
|
||||
--hash=sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e \
|
||||
--hash=sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d \
|
||||
--hash=sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c \
|
||||
--hash=sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415 \
|
||||
--hash=sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82 \
|
||||
--hash=sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02 \
|
||||
--hash=sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314 \
|
||||
--hash=sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325 \
|
||||
--hash=sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c \
|
||||
--hash=sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3 \
|
||||
--hash=sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914 \
|
||||
--hash=sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045 \
|
||||
--hash=sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d \
|
||||
--hash=sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9 \
|
||||
--hash=sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5 \
|
||||
--hash=sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2 \
|
||||
--hash=sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c \
|
||||
--hash=sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3 \
|
||||
--hash=sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2 \
|
||||
--hash=sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8 \
|
||||
--hash=sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d \
|
||||
--hash=sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d \
|
||||
--hash=sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9 \
|
||||
--hash=sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162 \
|
||||
--hash=sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76 \
|
||||
--hash=sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4 \
|
||||
--hash=sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e \
|
||||
--hash=sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9 \
|
||||
--hash=sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6 \
|
||||
--hash=sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b \
|
||||
--hash=sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01 \
|
||||
--hash=sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0
|
||||
certifi==2023.7.22
|
||||
# via requests
|
||||
cffi==1.15.1
|
||||
# via
|
||||
# cryptography
|
||||
# pynacl
|
||||
charset-normalizer==3.2.0 \
|
||||
--hash=sha256:04e57ab9fbf9607b77f7d057974694b4f6b142da9ed4a199859d9d4d5c63fe96 \
|
||||
--hash=sha256:09393e1b2a9461950b1c9a45d5fd251dc7c6f228acab64da1c9c0165d9c7765c \
|
||||
--hash=sha256:0b87549028f680ca955556e3bd57013ab47474c3124dc069faa0b6545b6c9710 \
|
||||
--hash=sha256:1000fba1057b92a65daec275aec30586c3de2401ccdcd41f8a5c1e2c87078706 \
|
||||
--hash=sha256:1249cbbf3d3b04902ff081ffbb33ce3377fa6e4c7356f759f3cd076cc138d020 \
|
||||
--hash=sha256:1920d4ff15ce893210c1f0c0e9d19bfbecb7983c76b33f046c13a8ffbd570252 \
|
||||
--hash=sha256:193cbc708ea3aca45e7221ae58f0fd63f933753a9bfb498a3b474878f12caaad \
|
||||
--hash=sha256:1a100c6d595a7f316f1b6f01d20815d916e75ff98c27a01ae817439ea7726329 \
|
||||
--hash=sha256:1f30b48dd7fa1474554b0b0f3fdfdd4c13b5c737a3c6284d3cdc424ec0ffff3a \
|
||||
--hash=sha256:203f0c8871d5a7987be20c72442488a0b8cfd0f43b7973771640fc593f56321f \
|
||||
--hash=sha256:246de67b99b6851627d945db38147d1b209a899311b1305dd84916f2b88526c6 \
|
||||
--hash=sha256:2dee8e57f052ef5353cf608e0b4c871aee320dd1b87d351c28764fc0ca55f9f4 \
|
||||
--hash=sha256:2efb1bd13885392adfda4614c33d3b68dee4921fd0ac1d3988f8cbb7d589e72a \
|
||||
--hash=sha256:2f4ac36d8e2b4cc1aa71df3dd84ff8efbe3bfb97ac41242fbcfc053c67434f46 \
|
||||
--hash=sha256:3170c9399da12c9dc66366e9d14da8bf7147e1e9d9ea566067bbce7bb74bd9c2 \
|
||||
--hash=sha256:3b1613dd5aee995ec6d4c69f00378bbd07614702a315a2cf6c1d21461fe17c23 \
|
||||
--hash=sha256:3bb3d25a8e6c0aedd251753a79ae98a093c7e7b471faa3aa9a93a81431987ace \
|
||||
--hash=sha256:3bb7fda7260735efe66d5107fb7e6af6a7c04c7fce9b2514e04b7a74b06bf5dd \
|
||||
--hash=sha256:41b25eaa7d15909cf3ac4c96088c1f266a9a93ec44f87f1d13d4a0e86c81b982 \
|
||||
--hash=sha256:45de3f87179c1823e6d9e32156fb14c1927fcc9aba21433f088fdfb555b77c10 \
|
||||
--hash=sha256:46fb8c61d794b78ec7134a715a3e564aafc8f6b5e338417cb19fe9f57a5a9bf2 \
|
||||
--hash=sha256:48021783bdf96e3d6de03a6e39a1171ed5bd7e8bb93fc84cc649d11490f87cea \
|
||||
--hash=sha256:4957669ef390f0e6719db3613ab3a7631e68424604a7b448f079bee145da6e09 \
|
||||
--hash=sha256:5e86d77b090dbddbe78867a0275cb4df08ea195e660f1f7f13435a4649e954e5 \
|
||||
--hash=sha256:6339d047dab2780cc6220f46306628e04d9750f02f983ddb37439ca47ced7149 \
|
||||
--hash=sha256:681eb3d7e02e3c3655d1b16059fbfb605ac464c834a0c629048a30fad2b27489 \
|
||||
--hash=sha256:6c409c0deba34f147f77efaa67b8e4bb83d2f11c8806405f76397ae5b8c0d1c9 \
|
||||
--hash=sha256:7095f6fbfaa55defb6b733cfeb14efaae7a29f0b59d8cf213be4e7ca0b857b80 \
|
||||
--hash=sha256:70c610f6cbe4b9fce272c407dd9d07e33e6bf7b4aa1b7ffb6f6ded8e634e3592 \
|
||||
--hash=sha256:72814c01533f51d68702802d74f77ea026b5ec52793c791e2da806a3844a46c3 \
|
||||
--hash=sha256:7a4826ad2bd6b07ca615c74ab91f32f6c96d08f6fcc3902ceeedaec8cdc3bcd6 \
|
||||
--hash=sha256:7c70087bfee18a42b4040bb9ec1ca15a08242cf5867c58726530bdf3945672ed \
|
||||
--hash=sha256:855eafa5d5a2034b4621c74925d89c5efef61418570e5ef9b37717d9c796419c \
|
||||
--hash=sha256:8700f06d0ce6f128de3ccdbc1acaea1ee264d2caa9ca05daaf492fde7c2a7200 \
|
||||
--hash=sha256:89f1b185a01fe560bc8ae5f619e924407efca2191b56ce749ec84982fc59a32a \
|
||||
--hash=sha256:8b2c760cfc7042b27ebdb4a43a4453bd829a5742503599144d54a032c5dc7e9e \
|
||||
--hash=sha256:8c2f5e83493748286002f9369f3e6607c565a6a90425a3a1fef5ae32a36d749d \
|
||||
--hash=sha256:8e098148dd37b4ce3baca71fb394c81dc5d9c7728c95df695d2dca218edf40e6 \
|
||||
--hash=sha256:94aea8eff76ee6d1cdacb07dd2123a68283cb5569e0250feab1240058f53b623 \
|
||||
--hash=sha256:95eb302ff792e12aba9a8b8f8474ab229a83c103d74a750ec0bd1c1eea32e669 \
|
||||
--hash=sha256:9bd9b3b31adcb054116447ea22caa61a285d92e94d710aa5ec97992ff5eb7cf3 \
|
||||
--hash=sha256:9e608aafdb55eb9f255034709e20d5a83b6d60c054df0802fa9c9883d0a937aa \
|
||||
--hash=sha256:a103b3a7069b62f5d4890ae1b8f0597618f628b286b03d4bc9195230b154bfa9 \
|
||||
--hash=sha256:a386ebe437176aab38c041de1260cd3ea459c6ce5263594399880bbc398225b2 \
|
||||
--hash=sha256:a38856a971c602f98472050165cea2cdc97709240373041b69030be15047691f \
|
||||
--hash=sha256:a401b4598e5d3f4a9a811f3daf42ee2291790c7f9d74b18d75d6e21dda98a1a1 \
|
||||
--hash=sha256:a7647ebdfb9682b7bb97e2a5e7cb6ae735b1c25008a70b906aecca294ee96cf4 \
|
||||
--hash=sha256:aaf63899c94de41fe3cf934601b0f7ccb6b428c6e4eeb80da72c58eab077b19a \
|
||||
--hash=sha256:b0dac0ff919ba34d4df1b6131f59ce95b08b9065233446be7e459f95554c0dc8 \
|
||||
--hash=sha256:baacc6aee0b2ef6f3d308e197b5d7a81c0e70b06beae1f1fcacffdbd124fe0e3 \
|
||||
--hash=sha256:bf420121d4c8dce6b889f0e8e4ec0ca34b7f40186203f06a946fa0276ba54029 \
|
||||
--hash=sha256:c04a46716adde8d927adb9457bbe39cf473e1e2c2f5d0a16ceb837e5d841ad4f \
|
||||
--hash=sha256:c0b21078a4b56965e2b12f247467b234734491897e99c1d51cee628da9786959 \
|
||||
--hash=sha256:c1c76a1743432b4b60ab3358c937a3fe1341c828ae6194108a94c69028247f22 \
|
||||
--hash=sha256:c4983bf937209c57240cff65906b18bb35e64ae872da6a0db937d7b4af845dd7 \
|
||||
--hash=sha256:c4fb39a81950ec280984b3a44f5bd12819953dc5fa3a7e6fa7a80db5ee853952 \
|
||||
--hash=sha256:c57921cda3a80d0f2b8aec7e25c8aa14479ea92b5b51b6876d975d925a2ea346 \
|
||||
--hash=sha256:c8063cf17b19661471ecbdb3df1c84f24ad2e389e326ccaf89e3fb2484d8dd7e \
|
||||
--hash=sha256:ccd16eb18a849fd8dcb23e23380e2f0a354e8daa0c984b8a732d9cfaba3a776d \
|
||||
--hash=sha256:cd6dbe0238f7743d0efe563ab46294f54f9bc8f4b9bcf57c3c666cc5bc9d1299 \
|
||||
--hash=sha256:d62e51710986674142526ab9f78663ca2b0726066ae26b78b22e0f5e571238dd \
|
||||
--hash=sha256:db901e2ac34c931d73054d9797383d0f8009991e723dab15109740a63e7f902a \
|
||||
--hash=sha256:e03b8895a6990c9ab2cdcd0f2fe44088ca1c65ae592b8f795c3294af00a461c3 \
|
||||
--hash=sha256:e1c8a2f4c69e08e89632defbfabec2feb8a8d99edc9f89ce33c4b9e36ab63037 \
|
||||
--hash=sha256:e4b749b9cc6ee664a3300bb3a273c1ca8068c46be705b6c31cf5d276f8628a94 \
|
||||
--hash=sha256:e6a5bf2cba5ae1bb80b154ed68a3cfa2fa00fde979a7f50d6598d3e17d9ac20c \
|
||||
--hash=sha256:e857a2232ba53ae940d3456f7533ce6ca98b81917d47adc3c7fd55dad8fab858 \
|
||||
--hash=sha256:ee4006268ed33370957f55bf2e6f4d263eaf4dc3cfc473d1d90baff6ed36ce4a \
|
||||
--hash=sha256:eef9df1eefada2c09a5e7a40991b9fc6ac6ef20b1372abd48d2794a316dc0449 \
|
||||
--hash=sha256:f058f6963fd82eb143c692cecdc89e075fa0828db2e5b291070485390b2f1c9c \
|
||||
--hash=sha256:f25c229a6ba38a35ae6e25ca1264621cc25d4d38dca2942a7fce0b67a4efe918 \
|
||||
--hash=sha256:f2a1d0fd4242bd8643ce6f98927cf9c04540af6efa92323e9d3124f57727bfc1 \
|
||||
--hash=sha256:f7560358a6811e52e9c4d142d497f1a6e10103d3a6881f18d04dbce3729c0e2c \
|
||||
--hash=sha256:f779d3ad205f108d14e99bb3859aa7dd8e9c68874617c72354d7ecaec2a054ac \
|
||||
--hash=sha256:f87f746ee241d30d6ed93969de31e5ffd09a2961a051e60ae6bddde9ec3583aa
|
||||
charset-normalizer==3.2.0
|
||||
# via requests
|
||||
click==8.1.7 \
|
||||
--hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 \
|
||||
--hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de
|
||||
click==8.1.7
|
||||
# via black
|
||||
cryptography==45.0.5 \
|
||||
--hash=sha256:0027d566d65a38497bc37e0dd7c2f8ceda73597d2ac9ba93810204f56f52ebc7 \
|
||||
--hash=sha256:101ee65078f6dd3e5a028d4f19c07ffa4dd22cce6a20eaa160f8b5219911e7d8 \
|
||||
--hash=sha256:12e55281d993a793b0e883066f590c1ae1e802e3acb67f8b442e721e475e6463 \
|
||||
--hash=sha256:14d96584701a887763384f3c47f0ca7c1cce322aa1c31172680eb596b890ec30 \
|
||||
--hash=sha256:1e1da5accc0c750056c556a93c3e9cb828970206c68867712ca5805e46dc806f \
|
||||
--hash=sha256:206210d03c1193f4e1ff681d22885181d47efa1ab3018766a7b32a7b3d6e6afd \
|
||||
--hash=sha256:2089cc8f70a6e454601525e5bf2779e665d7865af002a5dec8d14e561002e135 \
|
||||
--hash=sha256:3a264aae5f7fbb089dbc01e0242d3b67dffe3e6292e1f5182122bdf58e65215d \
|
||||
--hash=sha256:3af26738f2db354aafe492fb3869e955b12b2ef2e16908c8b9cb928128d42c57 \
|
||||
--hash=sha256:3fcfbefc4a7f332dece7272a88e410f611e79458fab97b5efe14e54fe476f4fd \
|
||||
--hash=sha256:460f8c39ba66af7db0545a8c6f2eabcbc5a5528fc1cf6c3fa9a1e44cec33385e \
|
||||
--hash=sha256:57c816dfbd1659a367831baca4b775b2a5b43c003daf52e9d57e1d30bc2e1b0e \
|
||||
--hash=sha256:5aa1e32983d4443e310f726ee4b071ab7569f58eedfdd65e9675484a4eb67bd1 \
|
||||
--hash=sha256:6ff8728d8d890b3dda5765276d1bc6fb099252915a2cd3aff960c4c195745dd0 \
|
||||
--hash=sha256:7259038202a47fdecee7e62e0fd0b0738b6daa335354396c6ddebdbe1206af2a \
|
||||
--hash=sha256:72e76caa004ab63accdf26023fccd1d087f6d90ec6048ff33ad0445abf7f605a \
|
||||
--hash=sha256:7760c1c2e1a7084153a0f68fab76e754083b126a47d0117c9ed15e69e2103492 \
|
||||
--hash=sha256:8c4a6ff8a30e9e3d38ac0539e9a9e02540ab3f827a3394f8852432f6b0ea152e \
|
||||
--hash=sha256:9024beb59aca9d31d36fcdc1604dd9bbeed0a55bface9f1908df19178e2f116e \
|
||||
--hash=sha256:90cb0a7bb35959f37e23303b7eed0a32280510030daba3f7fdfbb65defde6a97 \
|
||||
--hash=sha256:91098f02ca81579c85f66df8a588c78f331ca19089763d733e34ad359f474174 \
|
||||
--hash=sha256:926c3ea71a6043921050eaa639137e13dbe7b4ab25800932a8498364fc1abec9 \
|
||||
--hash=sha256:982518cd64c54fcada9d7e5cf28eabd3ee76bd03ab18e08a48cad7e8b6f31b18 \
|
||||
--hash=sha256:9b4cf6318915dccfe218e69bbec417fdd7c7185aa7aab139a2c0beb7468c89f0 \
|
||||
--hash=sha256:ad0caded895a00261a5b4aa9af828baede54638754b51955a0ac75576b831b27 \
|
||||
--hash=sha256:b85980d1e345fe769cfc57c57db2b59cff5464ee0c045d52c0df087e926fbe63 \
|
||||
--hash=sha256:b8fa8b0a35a9982a3c60ec79905ba5bb090fc0b9addcfd3dc2dd04267e45f25e \
|
||||
--hash=sha256:b9e38e0a83cd51e07f5a48ff9691cae95a79bea28fe4ded168a8e5c6c77e819d \
|
||||
--hash=sha256:bd4c45986472694e5121084c6ebbd112aa919a25e783b87eb95953c9573906d6 \
|
||||
--hash=sha256:be97d3a19c16a9be00edf79dca949c8fa7eff621763666a145f9f9535a5d7f42 \
|
||||
--hash=sha256:c648025b6840fe62e57107e0a25f604db740e728bd67da4f6f060f03017d5097 \
|
||||
--hash=sha256:d05a38884db2ba215218745f0781775806bde4f32e07b135348355fe8e4991d9 \
|
||||
--hash=sha256:dd420e577921c8c2d31289536c386aaa30140b473835e97f83bc71ea9d2baf2d \
|
||||
--hash=sha256:e357286c1b76403dd384d938f93c46b2b058ed4dfcdce64a770f0537ed3feb6f \
|
||||
--hash=sha256:e6c00130ed423201c5bc5544c23359141660b07999ad82e34e7bb8f882bb78e0 \
|
||||
--hash=sha256:e74d30ec9c7cb2f404af331d5b4099a9b322a8a6b25c4632755c8757345baac5 \
|
||||
--hash=sha256:f3562c2f23c612f2e4a6964a61d942f891d29ee320edb62ff48ffb99f3de9ae8
|
||||
# via
|
||||
# -r requirements_formatting.txt.in
|
||||
# pyjwt
|
||||
darker==2.1.1 \
|
||||
--hash=sha256:a6e6a682c0604e76fe9aec7650e96a944f517563c69b28fcc076db9d957d98ea \
|
||||
--hash=sha256:ead701414c45359fc0312bc285614d3285fc135476d43f3bc08d989ee19d9020
|
||||
# via -r requirements_formatting.txt.in
|
||||
darkgraylib==1.2.1 \
|
||||
--hash=sha256:60c59de69842367ce0c78c32c451fa8e9d29500e681312d9864a7416bcdb7792 \
|
||||
--hash=sha256:a5dd6a2015a470d9047278cdd01a91ccb1d746675f8fd4562b3b5f6b8cbda930
|
||||
# via
|
||||
# darker
|
||||
# graylint
|
||||
deprecated==1.2.14 \
|
||||
--hash=sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c \
|
||||
--hash=sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3
|
||||
cryptography==41.0.3
|
||||
# via pyjwt
|
||||
darker==1.7.2
|
||||
# via -r llvm/utils/git/requirements_formatting.txt.in
|
||||
deprecated==1.2.14
|
||||
# via pygithub
|
||||
graylint==1.1.1 \
|
||||
--hash=sha256:0fd8e02972ca03d0ef2bf0adea76b5343efcd492d7afb5f658f3e3a724f55a36 \
|
||||
--hash=sha256:b7e0eab6c159684dbf5ef84e942c3340f6a6549b02a3d11b1a1763cc4f8f0593
|
||||
# via darker
|
||||
idna==3.10 \
|
||||
--hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 \
|
||||
--hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3
|
||||
# via
|
||||
# -r requirements_formatting.txt.in
|
||||
# requests
|
||||
mypy-extensions==1.0.0 \
|
||||
--hash=sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d \
|
||||
--hash=sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782
|
||||
idna==3.4
|
||||
# via requests
|
||||
mypy-extensions==1.0.0
|
||||
# via black
|
||||
packaging==23.1 \
|
||||
--hash=sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61 \
|
||||
--hash=sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f
|
||||
packaging==23.1
|
||||
# via black
|
||||
pathspec==0.11.2 \
|
||||
--hash=sha256:1d6ed233af05e679efb96b1851550ea95bbb64b7c490b0f5aa52996c11e92a20 \
|
||||
--hash=sha256:e0d8d0ac2f12da61956eb2306b69f9469b42f4deb0f3cb6ed47b9cce9996ced3
|
||||
pathspec==0.11.2
|
||||
# via black
|
||||
platformdirs==3.10.0 \
|
||||
--hash=sha256:b45696dab2d7cc691a3226759c0d3b00c47c8b6e293d96f6436f733303f77f6d \
|
||||
--hash=sha256:d7c24979f292f916dc9cbf8648319032f551ea8c49a4c9bf2fb556a02070ec1d
|
||||
platformdirs==3.10.0
|
||||
# via black
|
||||
pycparser==2.21 \
|
||||
--hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \
|
||||
--hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206
|
||||
pycparser==2.21
|
||||
# via cffi
|
||||
pygithub==2.6.1 \
|
||||
--hash=sha256:6f2fa6d076ccae475f9fc392cc6cdbd54db985d4f69b8833a28397de75ed6ca3 \
|
||||
--hash=sha256:b5c035392991cca63959e9453286b41b54d83bf2de2daa7d7ff7e4312cebf3bf
|
||||
# via -r requirements_formatting.txt.in
|
||||
pyjwt==2.8.0 \
|
||||
--hash=sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de \
|
||||
--hash=sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320
|
||||
pygithub==1.59.1
|
||||
# via -r llvm/utils/git/requirements_formatting.txt.in
|
||||
pyjwt[crypto]==2.8.0
|
||||
# via pygithub
|
||||
pynacl==1.5.0 \
|
||||
--hash=sha256:06b8f6fa7f5de8d5d2f7573fe8c863c051225a27b61e6860fd047b1775807858 \
|
||||
--hash=sha256:0c84947a22519e013607c9be43706dd42513f9e6ae5d39d3613ca1e142fba44d \
|
||||
--hash=sha256:20f42270d27e1b6a29f54032090b972d97f0a1b0948cc52392041ef7831fee93 \
|
||||
--hash=sha256:401002a4aaa07c9414132aaed7f6836ff98f59277a234704ff66878c2ee4a0d1 \
|
||||
--hash=sha256:52cb72a79269189d4e0dc537556f4740f7f0a9ec41c1322598799b0bdad4ef92 \
|
||||
--hash=sha256:61f642bf2378713e2c2e1de73444a3778e5f0a38be6fee0fe532fe30060282ff \
|
||||
--hash=sha256:8ac7448f09ab85811607bdd21ec2464495ac8b7c66d146bf545b0f08fb9220ba \
|
||||
--hash=sha256:a36d4a9dda1f19ce6e03c9a784a2921a4b726b02e1c736600ca9c22029474394 \
|
||||
--hash=sha256:a422368fc821589c228f4c49438a368831cb5bbc0eab5ebe1d7fac9dded6567b \
|
||||
--hash=sha256:e46dae94e34b085175f8abb3b0aaa7da40767865ac82c928eeb9e57e1ea8a543
|
||||
pynacl==1.5.0
|
||||
# via pygithub
|
||||
requests==2.32.4 \
|
||||
--hash=sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c \
|
||||
--hash=sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422
|
||||
# via
|
||||
# -r requirements_formatting.txt.in
|
||||
# pygithub
|
||||
toml==0.10.2 \
|
||||
--hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \
|
||||
--hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f
|
||||
# via
|
||||
# darker
|
||||
# darkgraylib
|
||||
typing-extensions==4.14.1 \
|
||||
--hash=sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36 \
|
||||
--hash=sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76
|
||||
requests==2.31.0
|
||||
# via pygithub
|
||||
urllib3==2.6.0 \
|
||||
--hash=sha256:c90f7a39f716c572c4e3e58509581ebd83f9b59cced005b7db7ad2d22b0db99f \
|
||||
--hash=sha256:cb9bcef5a4b345d5da5d145dc3e30834f58e8018828cbc724d30b4cb7d4d49f1
|
||||
# via
|
||||
# -r requirements_formatting.txt.in
|
||||
# pygithub
|
||||
# requests
|
||||
wrapt==1.15.0 \
|
||||
--hash=sha256:02fce1852f755f44f95af51f69d22e45080102e9d00258053b79367d07af39c0 \
|
||||
--hash=sha256:077ff0d1f9d9e4ce6476c1a924a3332452c1406e59d90a2cf24aeb29eeac9420 \
|
||||
--hash=sha256:078e2a1a86544e644a68422f881c48b84fef6d18f8c7a957ffd3f2e0a74a0d4a \
|
||||
--hash=sha256:0970ddb69bba00670e58955f8019bec4a42d1785db3faa043c33d81de2bf843c \
|
||||
--hash=sha256:1286eb30261894e4c70d124d44b7fd07825340869945c79d05bda53a40caa079 \
|
||||
--hash=sha256:21f6d9a0d5b3a207cdf7acf8e58d7d13d463e639f0c7e01d82cdb671e6cb7923 \
|
||||
--hash=sha256:230ae493696a371f1dbffaad3dafbb742a4d27a0afd2b1aecebe52b740167e7f \
|
||||
--hash=sha256:26458da5653aa5b3d8dc8b24192f574a58984c749401f98fff994d41d3f08da1 \
|
||||
--hash=sha256:2cf56d0e237280baed46f0b5316661da892565ff58309d4d2ed7dba763d984b8 \
|
||||
--hash=sha256:2e51de54d4fb8fb50d6ee8327f9828306a959ae394d3e01a1ba8b2f937747d86 \
|
||||
--hash=sha256:2fbfbca668dd15b744418265a9607baa970c347eefd0db6a518aaf0cfbd153c0 \
|
||||
--hash=sha256:38adf7198f8f154502883242f9fe7333ab05a5b02de7d83aa2d88ea621f13364 \
|
||||
--hash=sha256:3a8564f283394634a7a7054b7983e47dbf39c07712d7b177b37e03f2467a024e \
|
||||
--hash=sha256:3abbe948c3cbde2689370a262a8d04e32ec2dd4f27103669a45c6929bcdbfe7c \
|
||||
--hash=sha256:3bbe623731d03b186b3d6b0d6f51865bf598587c38d6f7b0be2e27414f7f214e \
|
||||
--hash=sha256:40737a081d7497efea35ab9304b829b857f21558acfc7b3272f908d33b0d9d4c \
|
||||
--hash=sha256:41d07d029dd4157ae27beab04d22b8e261eddfc6ecd64ff7000b10dc8b3a5727 \
|
||||
--hash=sha256:46ed616d5fb42f98630ed70c3529541408166c22cdfd4540b88d5f21006b0eff \
|
||||
--hash=sha256:493d389a2b63c88ad56cdc35d0fa5752daac56ca755805b1b0c530f785767d5e \
|
||||
--hash=sha256:4ff0d20f2e670800d3ed2b220d40984162089a6e2c9646fdb09b85e6f9a8fc29 \
|
||||
--hash=sha256:54accd4b8bc202966bafafd16e69da9d5640ff92389d33d28555c5fd4f25ccb7 \
|
||||
--hash=sha256:56374914b132c702aa9aa9959c550004b8847148f95e1b824772d453ac204a72 \
|
||||
--hash=sha256:578383d740457fa790fdf85e6d346fda1416a40549fe8db08e5e9bd281c6a475 \
|
||||
--hash=sha256:58d7a75d731e8c63614222bcb21dd992b4ab01a399f1f09dd82af17bbfc2368a \
|
||||
--hash=sha256:5c5aa28df055697d7c37d2099a7bc09f559d5053c3349b1ad0c39000e611d317 \
|
||||
--hash=sha256:5fc8e02f5984a55d2c653f5fea93531e9836abbd84342c1d1e17abc4a15084c2 \
|
||||
--hash=sha256:63424c681923b9f3bfbc5e3205aafe790904053d42ddcc08542181a30a7a51bd \
|
||||
--hash=sha256:64b1df0f83706b4ef4cfb4fb0e4c2669100fd7ecacfb59e091fad300d4e04640 \
|
||||
--hash=sha256:74934ebd71950e3db69960a7da29204f89624dde411afbfb3b4858c1409b1e98 \
|
||||
--hash=sha256:75669d77bb2c071333417617a235324a1618dba66f82a750362eccbe5b61d248 \
|
||||
--hash=sha256:75760a47c06b5974aa5e01949bf7e66d2af4d08cb8c1d6516af5e39595397f5e \
|
||||
--hash=sha256:76407ab327158c510f44ded207e2f76b657303e17cb7a572ffe2f5a8a48aa04d \
|
||||
--hash=sha256:76e9c727a874b4856d11a32fb0b389afc61ce8aaf281ada613713ddeadd1cfec \
|
||||
--hash=sha256:77d4c1b881076c3ba173484dfa53d3582c1c8ff1f914c6461ab70c8428b796c1 \
|
||||
--hash=sha256:780c82a41dc493b62fc5884fb1d3a3b81106642c5c5c78d6a0d4cbe96d62ba7e \
|
||||
--hash=sha256:7dc0713bf81287a00516ef43137273b23ee414fe41a3c14be10dd95ed98a2df9 \
|
||||
--hash=sha256:7eebcdbe3677e58dd4c0e03b4f2cfa346ed4049687d839adad68cc38bb559c92 \
|
||||
--hash=sha256:896689fddba4f23ef7c718279e42f8834041a21342d95e56922e1c10c0cc7afb \
|
||||
--hash=sha256:96177eb5645b1c6985f5c11d03fc2dbda9ad24ec0f3a46dcce91445747e15094 \
|
||||
--hash=sha256:96e25c8603a155559231c19c0349245eeb4ac0096fe3c1d0be5c47e075bd4f46 \
|
||||
--hash=sha256:9d37ac69edc5614b90516807de32d08cb8e7b12260a285ee330955604ed9dd29 \
|
||||
--hash=sha256:9ed6aa0726b9b60911f4aed8ec5b8dd7bf3491476015819f56473ffaef8959bd \
|
||||
--hash=sha256:a487f72a25904e2b4bbc0817ce7a8de94363bd7e79890510174da9d901c38705 \
|
||||
--hash=sha256:a4cbb9ff5795cd66f0066bdf5947f170f5d63a9274f99bdbca02fd973adcf2a8 \
|
||||
--hash=sha256:a74d56552ddbde46c246b5b89199cb3fd182f9c346c784e1a93e4dc3f5ec9975 \
|
||||
--hash=sha256:a89ce3fd220ff144bd9d54da333ec0de0399b52c9ac3d2ce34b569cf1a5748fb \
|
||||
--hash=sha256:abd52a09d03adf9c763d706df707c343293d5d106aea53483e0ec8d9e310ad5e \
|
||||
--hash=sha256:abd8f36c99512755b8456047b7be10372fca271bf1467a1caa88db991e7c421b \
|
||||
--hash=sha256:af5bd9ccb188f6a5fdda9f1f09d9f4c86cc8a539bd48a0bfdc97723970348418 \
|
||||
--hash=sha256:b02f21c1e2074943312d03d243ac4388319f2456576b2c6023041c4d57cd7019 \
|
||||
--hash=sha256:b06fa97478a5f478fb05e1980980a7cdf2712015493b44d0c87606c1513ed5b1 \
|
||||
--hash=sha256:b0724f05c396b0a4c36a3226c31648385deb6a65d8992644c12a4963c70326ba \
|
||||
--hash=sha256:b130fe77361d6771ecf5a219d8e0817d61b236b7d8b37cc045172e574ed219e6 \
|
||||
--hash=sha256:b56d5519e470d3f2fe4aa7585f0632b060d532d0696c5bdfb5e8319e1d0f69a2 \
|
||||
--hash=sha256:b67b819628e3b748fd3c2192c15fb951f549d0f47c0449af0764d7647302fda3 \
|
||||
--hash=sha256:ba1711cda2d30634a7e452fc79eabcadaffedf241ff206db2ee93dd2c89a60e7 \
|
||||
--hash=sha256:bbeccb1aa40ab88cd29e6c7d8585582c99548f55f9b2581dfc5ba68c59a85752 \
|
||||
--hash=sha256:bd84395aab8e4d36263cd1b9308cd504f6cf713b7d6d3ce25ea55670baec5416 \
|
||||
--hash=sha256:c99f4309f5145b93eca6e35ac1a988f0dc0a7ccf9ccdcd78d3c0adf57224e62f \
|
||||
--hash=sha256:ca1cccf838cd28d5a0883b342474c630ac48cac5df0ee6eacc9c7290f76b11c1 \
|
||||
--hash=sha256:cd525e0e52a5ff16653a3fc9e3dd827981917d34996600bbc34c05d048ca35cc \
|
||||
--hash=sha256:cdb4f085756c96a3af04e6eca7f08b1345e94b53af8921b25c72f096e704e145 \
|
||||
--hash=sha256:ce42618f67741d4697684e501ef02f29e758a123aa2d669e2d964ff734ee00ee \
|
||||
--hash=sha256:d06730c6aed78cee4126234cf2d071e01b44b915e725a6cb439a879ec9754a3a \
|
||||
--hash=sha256:d5fe3e099cf07d0fb5a1e23d399e5d4d1ca3e6dfcbe5c8570ccff3e9208274f7 \
|
||||
--hash=sha256:d6bcbfc99f55655c3d93feb7ef3800bd5bbe963a755687cbf1f490a71fb7794b \
|
||||
--hash=sha256:d787272ed958a05b2c86311d3a4135d3c2aeea4fc655705f074130aa57d71653 \
|
||||
--hash=sha256:e169e957c33576f47e21864cf3fc9ff47c223a4ebca8960079b8bd36cb014fd0 \
|
||||
--hash=sha256:e20076a211cd6f9b44a6be58f7eeafa7ab5720eb796975d0c03f05b47d89eb90 \
|
||||
--hash=sha256:e826aadda3cae59295b95343db8f3d965fb31059da7de01ee8d1c40a60398b29 \
|
||||
--hash=sha256:eef4d64c650f33347c1f9266fa5ae001440b232ad9b98f1f43dfe7a79435c0a6 \
|
||||
--hash=sha256:f2e69b3ed24544b0d3dbe2c5c0ba5153ce50dcebb576fdc4696d52aa22db6034 \
|
||||
--hash=sha256:f87ec75864c37c4c6cb908d282e1969e79763e0d9becdfe9fe5473b7bb1e5f09 \
|
||||
--hash=sha256:fbec11614dba0424ca72f4e8ba3c420dba07b4a7c206c8c8e4e73f2e98f4c559 \
|
||||
--hash=sha256:fd69666217b62fa5d7c6aa88e507493a34dec4fa20c5bd925e4bc12fce586639
|
||||
toml==0.10.2
|
||||
# via darker
|
||||
urllib3==2.0.4
|
||||
# via requests
|
||||
wrapt==1.15.0
|
||||
# via deprecated
|
||||
@@ -1,8 +0,0 @@
|
||||
black~=25.1
|
||||
darker==2.1.1
|
||||
PyGithub==2.6.1
|
||||
cryptography>=43.0.1
|
||||
urllib3>=2.6.0
|
||||
requests>=2.32.4
|
||||
idna>=3.7
|
||||
certifi>=2024.7.4
|
||||
Vendored
+1
-1
Submodule External/drm-headers updated: 3e49836995...0675d2f291.
Vendored
+1
-1
Submodule External/fmt updated: 407c905e45...123913715a.
Vendored
+1
-1
Submodule External/jemalloc updated: 97d986993d...02ca52b5fe.
Vendored
+1
-1
Submodule External/jemalloc_glibc updated: 8436195ad5...404353974e.
Vendored
-1
Submodule External/range-v3 deleted from ca1388fb9d.
Vendored
+1
-1
Submodule External/vixl updated: ed690c9eca...84bc10c107.
Vendored
+1
-1
Submodule External/xxhash updated: e626a72bc2...bbb27a5efb.
+40
-13
@@ -1,16 +1,16 @@
|
||||
cmake_minimum_required(VERSION 3.14)
|
||||
set(PROJECT_NAME FEXCore)
|
||||
set (PROJECT_NAME FEXCore)
|
||||
project(${PROJECT_NAME}
|
||||
VERSION 0.01
|
||||
LANGUAGES CXX)
|
||||
|
||||
if (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")
|
||||
set(ARCHITECTURE_x86_64 1)
|
||||
set(_M_X86_64 1)
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcx16")
|
||||
endif()
|
||||
|
||||
if (CMAKE_SYSTEM_PROCESSOR MATCHES "^aarch64|^arm64|^armv8\.*")
|
||||
set(ARCHITECTURE_arm64 1)
|
||||
set(_M_ARM_64 1)
|
||||
endif()
|
||||
|
||||
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
|
||||
@@ -25,15 +25,44 @@ include(CheckIncludeFileCXX)
|
||||
include(CheckCXXSourceCompiles)
|
||||
|
||||
if (EXISTS ${CMAKE_CURRENT_DIR}/External/vixl/)
|
||||
# Useful to have for freestanding libFEXCore
|
||||
add_subdirectory(External/vixl/)
|
||||
include_directories(External/vixl/src/)
|
||||
# Useful to have for freestanding libFEXCore
|
||||
add_subdirectory(External/vixl/)
|
||||
include_directories(External/vixl/src/)
|
||||
endif()
|
||||
|
||||
set(CMAKE_CXX_STANDARD 20)
|
||||
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
|
||||
|
||||
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/include/git_version.h.in
|
||||
set(GIT_SHORT_HASH "Unknown")
|
||||
set(GIT_DESCRIBE_STRING "FEX-Unknown")
|
||||
|
||||
if (OVERRIDE_VERSION STREQUAL "detect")
|
||||
# Find our git hash
|
||||
find_package(Git)
|
||||
|
||||
if (GIT_FOUND)
|
||||
execute_process(
|
||||
COMMAND ${GIT_EXECUTABLE} rev-parse --short=7 HEAD
|
||||
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
|
||||
OUTPUT_VARIABLE GIT_SHORT_HASH
|
||||
ERROR_QUIET
|
||||
OUTPUT_STRIP_TRAILING_WHITESPACE
|
||||
)
|
||||
execute_process(
|
||||
COMMAND ${GIT_EXECUTABLE} describe --abbrev=7
|
||||
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
|
||||
OUTPUT_VARIABLE GIT_DESCRIBE_STRING
|
||||
ERROR_QUIET
|
||||
OUTPUT_STRIP_TRAILING_WHITESPACE
|
||||
)
|
||||
endif()
|
||||
else()
|
||||
set(GIT_SHORT_HASH "${OVERRIDE_VERSION}")
|
||||
set(GIT_DESCRIBE_STRING "FEX-${OVERRIDE_VERSION}")
|
||||
endif()
|
||||
|
||||
configure_file(
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/include/git_version.h.in
|
||||
${CMAKE_BINARY_DIR}/generated/git_version.h)
|
||||
|
||||
include_directories(${CMAKE_BINARY_DIR}/generated)
|
||||
@@ -45,12 +74,10 @@ add_compile_options($<$<COMPILE_LANGUAGE:CXX>:-fno-strict-aliasing> $<$<COMPILE_
|
||||
|
||||
add_subdirectory(Source/)
|
||||
|
||||
if (NOT BUILD_STEAM_SUPPORT)
|
||||
install (DIRECTORY include/FEXCore ${CMAKE_BINARY_DIR}/include/FEXCore
|
||||
DESTINATION include
|
||||
COMPONENT Development)
|
||||
endif()
|
||||
install (DIRECTORY include/FEXCore ${CMAKE_BINARY_DIR}/include/FEXCore
|
||||
DESTINATION include
|
||||
COMPONENT Development)
|
||||
|
||||
if (BUILD_TESTING)
|
||||
if (BUILD_TESTS)
|
||||
add_subdirectory(unittests/)
|
||||
endif()
|
||||
@@ -118,6 +118,41 @@ def print_man_env_option(name, desc, default, no_json_key):
|
||||
output_man.write("\\fBdefault:\\fR {0}\n".format(default))
|
||||
output_man.write(".Pp\n\n")
|
||||
|
||||
def print_man_options(options):
|
||||
output_man.write(".Sh OPTIONS\n")
|
||||
output_man.write(".Bl -tag -width -indent\n")
|
||||
for op_group, group_vals in options.items():
|
||||
for op_key, op_vals in group_vals.items():
|
||||
short = None
|
||||
long = op_key.lower()
|
||||
|
||||
if ("ShortArg" in op_vals):
|
||||
short = op_vals["ShortArg"]
|
||||
|
||||
default = op_vals["Default"]
|
||||
value_type = op_vals["Type"]
|
||||
|
||||
# Textual default rather than enum based
|
||||
if ("TextDefault" in op_vals):
|
||||
default = op_vals["TextDefault"]
|
||||
|
||||
if (value_type == "str" or value_type == "strarray" or value_type == "strenum"):
|
||||
# Wrap the string argument in quotes
|
||||
default = "'" + default + "'"
|
||||
print_man_option(
|
||||
short,
|
||||
long,
|
||||
op_vals["Desc"],
|
||||
default
|
||||
)
|
||||
if (value_type == "strenum"):
|
||||
Enums = op_vals["Enums"]
|
||||
output_man.write("\\fBAvailable Options:\\fR\n")
|
||||
output_man.write(", ".join(f"{enum_op_val}" for [_, enum_op_val] in Enums.items()))
|
||||
output_man.write("\n.sp\n")
|
||||
|
||||
output_man.write(".El\n")
|
||||
|
||||
def print_man_environment(options):
|
||||
output_man.write(".Sh ENVIRONMENT\n")
|
||||
output_man.write(".Bl -tag -width -indent\n")
|
||||
@@ -159,7 +194,7 @@ def print_man_environment_tail():
|
||||
"By default FEX will look in {$HOME, $XDG_CONFIG_HOME}/.fex-emu/",
|
||||
"This will override the full path",
|
||||
"If FEX_PORTABLE is declared then relative paths are also supported",
|
||||
"For FEX: Relative to the FEX binary",
|
||||
"For FEXInterpreter: Relative to the FEXInterpreter binary",
|
||||
"For WINE: Relative to %LOCALAPPDATA%"
|
||||
],
|
||||
"''", True)
|
||||
@@ -173,7 +208,7 @@ def print_man_environment_tail():
|
||||
"One must be careful with this option as it will override any applications that load with execve as well"
|
||||
"If you need to support applications that execve then use FEX_APP_CONFIG_LOCATION instead"
|
||||
"If FEX_PORTABLE is declared then relative paths are also supported",
|
||||
"For FEX: Relative to the FEX binary",
|
||||
"For FEXInterpreter: Relative to the FEXInterpreter binary",
|
||||
"For WINE: Relative to %LOCALAPPDATA%"
|
||||
],
|
||||
"''", True)
|
||||
@@ -192,34 +227,33 @@ def print_man_environment_tail():
|
||||
"PORTABLE",
|
||||
[
|
||||
"Allows FEX to run without installation. Global locations for configuration and binfmt_misc are ignored.",
|
||||
"For FEX on Linux:",
|
||||
"These files are instead read from <FEXPath>/fex-emu/ by default.",
|
||||
"For FEXInterpreter on Linux:",
|
||||
"These files are instead read from <FEXInterpreterPath>/fex-emu/ by default.",
|
||||
"For Arm64ec/Wow64 WINE builds:",
|
||||
"These files are instead read from $LOCALAPPDATA/fex-emu/ by default.",
|
||||
"For further customization, see FEX_APP_CONFIG_LOCATION and FEX_APP_DATA_LOCATION."
|
||||
],
|
||||
"''", True)
|
||||
|
||||
print_man_env_option(
|
||||
"APP_CACHE_LOCATION",
|
||||
[
|
||||
"Allows the user to override where FEX stores and loads cache files",
|
||||
"By default FEX will look in $XDG_CACHE_HOME/fex-emu/ or $HOME/.cache/fex-emu/",
|
||||
"This will override the full path, trailing forward-slash is expected to exist",
|
||||
],
|
||||
"''", True)
|
||||
|
||||
def print_man_header():
|
||||
header ='''.Dd {0}
|
||||
.Dt FEX
|
||||
.Os Linux
|
||||
.Sh NAME
|
||||
.Nm FEX
|
||||
.Nm FEXLoader
|
||||
.Nm FEXInterpreter
|
||||
.Nm FEXBash
|
||||
.Nd Fast x86-64 and x86 emulation.
|
||||
.Sh SYNOPSIS
|
||||
.Nm
|
||||
.Ar <args> ...
|
||||
.Op options
|
||||
.Op Ar --
|
||||
.Ar Application
|
||||
<args> ...
|
||||
.Pp
|
||||
.Nm FEXInterpreter
|
||||
.Ar Application
|
||||
<args> ...
|
||||
.Pp
|
||||
.Nm FEXBash
|
||||
.Ar <args> ...
|
||||
@@ -327,6 +361,82 @@ def print_config_option(type, group_name, json_name, default_value, short, choic
|
||||
|
||||
output_argloader.write("\n");
|
||||
|
||||
def print_argloader_options(options):
|
||||
output_argloader.write("#ifdef BEFORE_PARSE\n")
|
||||
output_argloader.write("#undef BEFORE_PARSE\n")
|
||||
for op_group, group_vals in options.items():
|
||||
for op_key, op_vals in group_vals.items():
|
||||
default = op_vals["Default"]
|
||||
|
||||
if (op_vals["Type"] == "str" or op_vals["Type"] == "strarray" or op_vals["Type"] == "strenum"):
|
||||
# Wrap the string argument in quotes
|
||||
default = "\"" + default + "\""
|
||||
|
||||
# Textual default rather than enum based
|
||||
if ("TextDefault" in op_vals):
|
||||
default = "\"" + op_vals["TextDefault"] + "\""
|
||||
|
||||
short = None
|
||||
choices = None
|
||||
|
||||
if ("ShortArg" in op_vals):
|
||||
short = op_vals["ShortArg"]
|
||||
if ("Choices" in op_vals):
|
||||
choices = op_vals["Choices"]
|
||||
|
||||
print_config_option(
|
||||
op_vals["Type"],
|
||||
op_group,
|
||||
op_key,
|
||||
default,
|
||||
short,
|
||||
choices,
|
||||
op_vals["Desc"])
|
||||
|
||||
output_argloader.write("\n")
|
||||
output_argloader.write("#endif\n")
|
||||
|
||||
def print_parse_argloader_options(options):
|
||||
output_argloader.write("#ifdef AFTER_PARSE\n")
|
||||
output_argloader.write("#undef AFTER_PARSE\n")
|
||||
for op_group, group_vals in options.items():
|
||||
for op_key, op_vals in group_vals.items():
|
||||
output_argloader.write("if (Options.is_set_by_user(\"{0}\")) {{\n".format(op_key))
|
||||
|
||||
value_type = op_vals["Type"]
|
||||
NeedsString = False
|
||||
conversion_func = "fextl::fmt::format(\"{}\", "
|
||||
if ("ArgumentHandler" in op_vals):
|
||||
NeedsString = True
|
||||
conversion_func = "FEXCore::Config::Handler::{0}(".format(op_vals["ArgumentHandler"])
|
||||
if (value_type == "str"):
|
||||
NeedsString = True
|
||||
conversion_func = "std::move("
|
||||
if (value_type == "bool"):
|
||||
# boolean values need a decimal specifier. Otherwise fmt prints strings.
|
||||
conversion_func = "fextl::fmt::format(\"{:d}\", "
|
||||
|
||||
if (value_type == "strenum"):
|
||||
output_argloader.write("\tfextl::string UserValue = Options[\"{0}\"];\n".format(op_key))
|
||||
output_argloader.write("\tSet(FEXCore::Config::ConfigOption::CONFIG_{}, FEXCore::Config::EnumParser<FEXCore::Config::{}ConfigPair>(FEXCore::Config::{}_EnumPairs, UserValue));\n".format(op_key.upper(), op_key, op_key, op_key))
|
||||
elif (value_type == "strarray"):
|
||||
# these need a bit more help
|
||||
output_argloader.write("\tauto Array = Options.all(\"{0}\");\n".format(op_key))
|
||||
output_argloader.write("\tfor (auto iter = Array.begin(); iter != Array.end(); ++iter) {\n")
|
||||
output_argloader.write("\t\tAppendStrArrayValue(FEXCore::Config::ConfigOption::CONFIG_{0}, *iter);\n".format(op_key.upper()))
|
||||
output_argloader.write("\t}\n")
|
||||
else:
|
||||
if (NeedsString):
|
||||
output_argloader.write("\tfextl::string UserValue = Options[\"{0}\"];\n".format(op_key))
|
||||
else:
|
||||
output_argloader.write("\t{0} UserValue = Options.get(\"{1}\");\n".format(value_type, op_key))
|
||||
|
||||
output_argloader.write("\tSet(FEXCore::Config::ConfigOption::CONFIG_{0}, {1}UserValue));\n".format(op_key.upper(), conversion_func))
|
||||
output_argloader.write("}\n")
|
||||
|
||||
output_argloader.write("#endif\n")
|
||||
|
||||
|
||||
def print_parse_envloader_options(options):
|
||||
output_argloader.write("#ifdef ENVLOADER\n")
|
||||
output_argloader.write("#undef ENVLOADER\n")
|
||||
@@ -337,13 +447,13 @@ def print_parse_envloader_options(options):
|
||||
value_type = op_vals["Type"]
|
||||
if (value_type == "strenum"):
|
||||
output_argloader.write("else if (Key == \"FEX_{0}\") {{\n".format(op_key.upper()))
|
||||
output_argloader.write("\tValue = FEXCore::Config::EnumParser<FEXCore::Config::{}ConfigPair>(FEXCore::Config::{}_EnumPairs, Value_View);\n".format(op_key, op_key))
|
||||
output_argloader.write("Value = FEXCore::Config::EnumParser<FEXCore::Config::{}ConfigPair>(FEXCore::Config::{}_EnumPairs, Value_View);\n".format(op_key, op_key, op_key))
|
||||
output_argloader.write("}\n")
|
||||
|
||||
if ("ArgumentHandler" in op_vals):
|
||||
conversion_func = "FEXCore::Config::Handler::{0}".format(op_vals["ArgumentHandler"])
|
||||
output_argloader.write("else if (Key == \"FEX_{0}\") {{\n".format(op_key.upper()))
|
||||
output_argloader.write("\tValue = {0}(Value_View);\n".format(conversion_func))
|
||||
output_argloader.write("Value = {0}(Value_View);\n".format(conversion_func))
|
||||
output_argloader.write("}\n")
|
||||
output_argloader.write("#endif\n")
|
||||
|
||||
@@ -357,15 +467,15 @@ def print_parse_jsonloader_options(options):
|
||||
value_type = op_vals["Type"]
|
||||
if (value_type == "strenum"):
|
||||
output_argloader.write("else if (KeyName == \"{0}\") {{\n".format(op_key))
|
||||
output_argloader.write("\tSet(KeyOption, FEXCore::Config::EnumParser<FEXCore::Config::{}ConfigPair>(FEXCore::Config::{}_EnumPairs, Value_View));\n".format(op_key, op_key))
|
||||
output_argloader.write("\tSet(KeyOption, FEXCore::Config::EnumParser<FEXCore::Config::{}ConfigPair>(FEXCore::Config::{}_EnumPairs, Value_View));\n".format(op_key, op_key, op_key))
|
||||
output_argloader.write("}\n")
|
||||
elif (value_type == "strarray"):
|
||||
output_argloader.write("else if (KeyName == \"{0}\") {{\n".format(op_key))
|
||||
output_argloader.write("\tAppendStrArrayValue(KeyOption, ConfigString);\n")
|
||||
output_argloader.write("}\n")
|
||||
assert op_key is not None, "No options found in JSONLOADER"
|
||||
output_argloader.write("else {\n")
|
||||
output_argloader.write("\tSet(KeyOption, ConfigString);\n")
|
||||
output_argloader.write("else {{\n".format(op_key))
|
||||
output_argloader.write("Set(KeyOption, ConfigString);\n")
|
||||
output_argloader.write("}\n")
|
||||
|
||||
output_argloader.write("#endif\n")
|
||||
@@ -407,6 +517,41 @@ def print_parse_enum_options(options):
|
||||
|
||||
output_argloader.write("#endif\n")
|
||||
|
||||
def check_for_duplicate_options(options):
|
||||
short_map = []
|
||||
long_map = []
|
||||
|
||||
# Spin through all the items and see if we have a duplicate option
|
||||
for op_group, group_vals in options.items():
|
||||
for op_key, op_vals in group_vals.items():
|
||||
short = None
|
||||
long = op_key.lower()
|
||||
long_invert = None
|
||||
if ("ShortArg" in op_vals):
|
||||
short = op_vals["ShortArg"]
|
||||
if (op_vals["Type"] == "bool"):
|
||||
long_invert = "no-" + long
|
||||
|
||||
# Check for short key duplication
|
||||
if (short != None):
|
||||
if (short in short_map):
|
||||
raise Exception("Short config '{0}' for option '{1}' has duplicate entry!".format(short, op_key))
|
||||
else:
|
||||
short_map.append(short)
|
||||
|
||||
# Check for long key duplication
|
||||
if (long in long_map):
|
||||
raise Exception("Long config '{0}' has duplicate entry!".format(long))
|
||||
else:
|
||||
long_map.append(long)
|
||||
|
||||
# Check for long key duplication
|
||||
if (long_invert != None):
|
||||
if (long_invert in long_map):
|
||||
raise Exception("Long config '{0}' has duplicate entry!".format(long_invert))
|
||||
else:
|
||||
long_map.append(long_invert)
|
||||
|
||||
if (len(sys.argv) < 5):
|
||||
sys.exit()
|
||||
|
||||
@@ -423,6 +568,8 @@ json_object = json.loads(json_text)
|
||||
options = json_object["Options"]
|
||||
unnamed_options = json_object["UnnamedOptions"]
|
||||
|
||||
check_for_duplicate_options(options)
|
||||
|
||||
# Generate config include file
|
||||
output_file = open(output_filename, "w")
|
||||
print_header()
|
||||
@@ -434,6 +581,7 @@ output_file.close()
|
||||
# Generate man file
|
||||
output_man = open(output_man_page, "w")
|
||||
print_man_header()
|
||||
print_man_options(options)
|
||||
print_man_environment(options)
|
||||
print_man_tail()
|
||||
|
||||
@@ -441,6 +589,8 @@ output_man.close()
|
||||
|
||||
# Generate argument loader code
|
||||
output_argloader = open(output_argumentloader_filename, "w")
|
||||
print_argloader_options(options);
|
||||
print_parse_argloader_options(options);
|
||||
|
||||
# Generate environment loader code
|
||||
print_parse_envloader_options(options);
|
||||
|
||||
@@ -58,10 +58,9 @@ class OpDefinition:
|
||||
JITDispatch: bool
|
||||
JITDispatchOverride: str
|
||||
TiedSource: int
|
||||
Inline: list[str]
|
||||
Arguments: list[OpArgument]
|
||||
EmitValidation: list[str]
|
||||
Desc: list[str]
|
||||
Arguments: list
|
||||
EmitValidation: list
|
||||
Desc: list
|
||||
|
||||
def __init__(self):
|
||||
self.Name = None
|
||||
@@ -92,14 +91,19 @@ class OpDefinition:
|
||||
attrs = vars(self)
|
||||
print(", ".join("%s: %s" % item for item in attrs.items()))
|
||||
|
||||
IRTypesToCXX: dict[str, IRType] = {}
|
||||
CXXTypeToIR: dict[str, IRType] = {}
|
||||
IROps: list[OpDefinition] = []
|
||||
IRTypesToCXX = {}
|
||||
CXXTypeToIR = {}
|
||||
IROps = []
|
||||
|
||||
IROpNameSet: set[str] = set()
|
||||
IROpNameMap = {}
|
||||
|
||||
def is_ssa_type(op_type: str):
|
||||
return op_type in {"SSA", "GPR", "GPRPair", "FPR"}
|
||||
def is_ssa_type(type):
|
||||
if (type == "SSA" or
|
||||
type == "GPR" or
|
||||
type == "GPRPair" or
|
||||
type == "FPR"):
|
||||
return True
|
||||
return False
|
||||
|
||||
def parse_irtypes(irtypes):
|
||||
for op_key, op_val in irtypes.items():
|
||||
@@ -214,8 +218,11 @@ def parse_ops(ops):
|
||||
OpArg.DefaultInitializer = DefaultInit[1][:-1]
|
||||
|
||||
# If SSA type then we can generate validation for this op
|
||||
if OpArg.IsSSA and OpArg.Type in {"GPR", "GPRPair", "FPR"}:
|
||||
OpDef.EmitValidation.append(f"GetOpRegClass({ArgName}) == RegClass::Invalid || WalkFindRegClass({ArgName}) == RegClass::{OpArg.Type}")
|
||||
if (OpArg.IsSSA and
|
||||
(OpArg.Type == "GPR" or
|
||||
OpArg.Type == "GPRPair" or
|
||||
OpArg.Type == "FPR")):
|
||||
OpDef.EmitValidation.append(f"GetOpRegClass({ArgName}) == InvalidClass || WalkFindRegClass({ArgName}) == {OpArg.Type}Class")
|
||||
|
||||
OpArg.Name = ArgName
|
||||
OpArg.NameWithPrefix = NameWithPrefix
|
||||
@@ -271,12 +278,6 @@ def parse_ops(ops):
|
||||
if "TiedSource" in op_val:
|
||||
OpDef.TiedSource = op_val["TiedSource"]
|
||||
|
||||
# Pad Inline out to the argument count
|
||||
OpDef.Inline = [''] * len(OpDef.Arguments)
|
||||
if "Inline" in op_val:
|
||||
Value = op_val["Inline"]
|
||||
OpDef.Inline[0:len(Value)] = Value
|
||||
|
||||
# Do some fixups of the data here
|
||||
if len(OpDef.EmitValidation) != 0:
|
||||
for i in range(len(OpDef.EmitValidation)):
|
||||
@@ -288,28 +289,21 @@ def parse_ops(ops):
|
||||
#OpDef.print()
|
||||
|
||||
# Error on duplicate op
|
||||
if OpDef.Name in IROpNameSet:
|
||||
if OpDef.Name in IROpNameMap:
|
||||
ExitError("Duplicate Op defined! {}".format(OpDef.Name))
|
||||
|
||||
IROps.append(OpDef)
|
||||
IROpNameSet.add(OpDef.Name)
|
||||
IROpNameMap[OpDef.Name] = 1
|
||||
|
||||
# Print out enum values
|
||||
def print_enums(enums):
|
||||
def print_enums():
|
||||
output_file.write("#ifdef IROP_ENUM\n")
|
||||
output_file.write("enum IROps : uint16_t {\n")
|
||||
|
||||
for op in IROps:
|
||||
output_file.write("\tOP_{},\n" .format(op.Name.upper()))
|
||||
output_file.write("};\n")
|
||||
|
||||
for name, members in enums.items():
|
||||
output_file.write(f"enum {name} {{\n")
|
||||
for member in members:
|
||||
if member:
|
||||
output_file.write(f"\t{member}\n")
|
||||
else:
|
||||
output_file.write("\n")
|
||||
output_file.write("};\n\n")
|
||||
output_file.write("};\n")
|
||||
|
||||
output_file.write("#undef IROP_ENUM\n")
|
||||
output_file.write("#endif\n\n")
|
||||
@@ -403,16 +397,16 @@ def print_ir_sizes():
|
||||
// Make sure our array maps directly to the IROps enum
|
||||
static_assert(IRSizes[IROps::OP_LAST] == -1ULL);
|
||||
|
||||
[[nodiscard]] inline size_t GetSize(IROps Op) { return IRSizes[Op]; }
|
||||
[[nodiscard, gnu::const]] std::string_view const& GetName(IROps Op);
|
||||
[[nodiscard, gnu::const]] uint8_t GetArgs(IROps Op);
|
||||
[[nodiscard, gnu::const]] uint8_t GetRAArgs(IROps Op);
|
||||
[[nodiscard, gnu::const]] FEXCore::IR::RegClass GetRegClass(IROps Op);
|
||||
[[nodiscard, gnu::const]] bool HasSideEffects(IROps Op);
|
||||
[[nodiscard, gnu::const]] bool ImplicitFlagClobber(IROps Op);
|
||||
[[nodiscard, gnu::const]] bool GetHasDest(IROps Op);
|
||||
[[nodiscard, gnu::const]] bool LoweredX87(IROps Op);
|
||||
[[nodiscard, gnu::const]] int8_t TiedSource(IROps Op);
|
||||
[[maybe_unused, nodiscard]] static size_t GetSize(IROps Op) { return IRSizes[Op]; }
|
||||
[[nodiscard, gnu::const, gnu::visibility("default")]] std::string_view const& GetName(IROps Op);
|
||||
[[nodiscard, gnu::const, gnu::visibility("default")]] uint8_t GetArgs(IROps Op);
|
||||
[[nodiscard, gnu::const, gnu::visibility("default")]] uint8_t GetRAArgs(IROps Op);
|
||||
[[nodiscard, gnu::const, gnu::visibility("default")]] FEXCore::IR::RegisterClassType GetRegClass(IROps Op);
|
||||
[[nodiscard, gnu::const, gnu::visibility("default")]] bool HasSideEffects(IROps Op);
|
||||
[[nodiscard, gnu::const, gnu::visibility("default")]] bool ImplicitFlagClobber(IROps Op);
|
||||
[[nodiscard, gnu::const, gnu::visibility("default")]] bool GetHasDest(IROps Op);
|
||||
[[nodiscard, gnu::const, gnu::visibility("default")]] bool LoweredX87(IROps Op);
|
||||
[[nodiscard, gnu::const, gnu::visibility("default")]] int8_t TiedSource(IROps Op);
|
||||
|
||||
#undef IROP_SIZES
|
||||
#endif
|
||||
@@ -421,29 +415,30 @@ def print_ir_sizes():
|
||||
def print_ir_reg_classes():
|
||||
output_file.write("#ifdef IROP_REG_CLASSES_IMPL\n")
|
||||
|
||||
output_file.write("constexpr std::array<FEXCore::IR::RegClass, IROps::OP_LAST + 1> IRRegClasses = {\n")
|
||||
output_file.write("constexpr std::array<FEXCore::IR::RegisterClassType, IROps::OP_LAST + 1> IRRegClasses = {\n")
|
||||
for op in IROps:
|
||||
if op.Name == "Last":
|
||||
output_file.write("\tRegClass::Invalid,\n")
|
||||
output_file.write("\tFEXCore::IR::InvalidClass,\n")
|
||||
else:
|
||||
if op.HasDest and op.DestType is None:
|
||||
Class = "Invalid"
|
||||
if op.HasDest and op.DestType == None:
|
||||
ExitError("IR op {} has destination with no destination class".format(op.Name))
|
||||
|
||||
if op.HasDest and op.DestType == "SSA": # Special case SSA type
|
||||
output_file.write("\tRegClass::Complex,\n")
|
||||
output_file.write("\tFEXCore::IR::ComplexClass,\n")
|
||||
elif op.HasDest:
|
||||
output_file.write("\tRegClass::{},\n".format(op.DestType))
|
||||
output_file.write("\tFEXCore::IR::{}Class,\n".format(op.DestType))
|
||||
else:
|
||||
# No destination so it has an invalid destination class
|
||||
output_file.write("\tRegClass::Invalid, // No destination\n")
|
||||
output_file.write("\tFEXCore::IR::InvalidClass, // No destination\n")
|
||||
|
||||
|
||||
output_file.write("};\n\n")
|
||||
|
||||
output_file.write("// Make sure our array maps directly to the IROps enum\n")
|
||||
output_file.write("static_assert(IRRegClasses[IROps::OP_LAST] == RegClass::Invalid);\n\n")
|
||||
output_file.write("static_assert(IRRegClasses[IROps::OP_LAST] == FEXCore::IR::InvalidClass);\n\n")
|
||||
|
||||
output_file.write("FEXCore::IR::RegClass GetRegClass(IROps Op) { return IRRegClasses[Op]; }\n\n")
|
||||
output_file.write("FEXCore::IR::RegisterClassType GetRegClass(IROps Op) { return IRRegClasses[Op]; }\n\n")
|
||||
|
||||
output_file.write("#undef IROP_REG_CLASSES_IMPL\n")
|
||||
output_file.write("#endif\n\n")
|
||||
@@ -566,7 +561,9 @@ def print_ir_arg_printer():
|
||||
|
||||
SSAArgNum = 0
|
||||
FirstArg = True
|
||||
for arg in op.Arguments:
|
||||
for i in range(0, len(op.Arguments)):
|
||||
arg = op.Arguments[i]
|
||||
|
||||
# No point printing temporaries that we can't recover
|
||||
if arg.Temporary:
|
||||
continue
|
||||
@@ -591,13 +588,13 @@ def print_ir_arg_printer():
|
||||
output_file.write("#endif\n")
|
||||
|
||||
def print_validation(op):
|
||||
if len(op.EmitValidation) != 0:
|
||||
output_file.write("#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED\n")
|
||||
if op.EmitValidation != None:
|
||||
output_file.write("\t\t#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED\n")
|
||||
|
||||
for Validation in op.EmitValidation:
|
||||
Sanitized = Validation.replace("\"", "\\\"")
|
||||
output_file.write("\t\tLOGMAN_THROW_A_FMT({}, \"{}\");\n".format(Validation, Sanitized))
|
||||
output_file.write("#endif\n")
|
||||
output_file.write("\tLOGMAN_THROW_A_FMT({}, \"{}\");\n".format(Validation, Sanitized))
|
||||
output_file.write("\t\t#endif\n")
|
||||
|
||||
# Print out IR allocator helpers
|
||||
def print_ir_allocator_helpers():
|
||||
@@ -667,7 +664,7 @@ def print_ir_allocator_helpers():
|
||||
output_file.write("\t\treturn HeaderOp->Op;\n")
|
||||
output_file.write("\t}\n\n")
|
||||
|
||||
output_file.write("\tFEXCore::IR::RegClass GetOpRegClass(const OrderedNode *Op) const {\n")
|
||||
output_file.write("\tFEXCore::IR::RegisterClassType GetOpRegClass(const OrderedNode *Op) const {\n")
|
||||
output_file.write("\t\treturn GetRegClass(GetOpType(Op));\n")
|
||||
output_file.write("\t}\n\n")
|
||||
|
||||
@@ -681,21 +678,22 @@ def print_ir_allocator_helpers():
|
||||
output_file.write("\tIRPair<IROp_{}> _{}(" .format(op.Name, op.Name))
|
||||
|
||||
# Output SSA args first
|
||||
for i, arg in enumerate(op.Arguments):
|
||||
LastArg = i == len(op.Arguments) - 1
|
||||
for i in range(0, len(op.Arguments)):
|
||||
arg = op.Arguments[i]
|
||||
LastArg = len(op.Arguments) - i - 1 == 0
|
||||
|
||||
if arg.Temporary:
|
||||
CType = IRTypesToCXX[arg.Type].CXXName
|
||||
output_file.write("{} {}".format(CType, arg.Name))
|
||||
output_file.write("{} {}".format(CType, arg.Name));
|
||||
elif arg.IsSSA:
|
||||
# SSA value
|
||||
output_file.write("OrderedNodeWrapper {}".format(arg.Name))
|
||||
else:
|
||||
# User defined op that is stored
|
||||
CType = IRTypesToCXX[arg.Type].CXXName
|
||||
output_file.write("{} {}".format(CType, arg.Name))
|
||||
output_file.write("{} {}".format(CType, arg.Name));
|
||||
|
||||
if arg.DefaultInitializer:
|
||||
if arg.DefaultInitializer != None:
|
||||
output_file.write(" = {}".format(arg.DefaultInitializer))
|
||||
|
||||
if not LastArg:
|
||||
@@ -753,19 +751,20 @@ def print_ir_allocator_helpers():
|
||||
if op.SSAArgNum:
|
||||
output_file.write("\tIRPair<IROp_{}> _{}(" .format(op.Name, op.Name))
|
||||
|
||||
for i, arg in enumerate(op.Arguments):
|
||||
LastArg = i == len(op.Arguments) - 1
|
||||
for i in range(0, len(op.Arguments)):
|
||||
arg = op.Arguments[i]
|
||||
LastArg = len(op.Arguments) - i - 1 == 0
|
||||
|
||||
if arg.Temporary:
|
||||
CType = IRTypesToCXX[arg.Type].CXXName
|
||||
output_file.write("{} {}".format(CType, arg.Name))
|
||||
output_file.write("{} {}".format(CType, arg.Name));
|
||||
elif arg.IsSSA:
|
||||
output_file.write("OrderedNode *{}".format(arg.Name))
|
||||
else:
|
||||
CType = IRTypesToCXX[arg.Type].CXXName
|
||||
output_file.write("{} {}".format(CType, arg.Name))
|
||||
output_file.write("{} {}".format(CType, arg.Name));
|
||||
|
||||
if arg.DefaultInitializer:
|
||||
if arg.DefaultInitializer != None:
|
||||
output_file.write(" = {}".format(arg.DefaultInitializer))
|
||||
|
||||
if not LastArg:
|
||||
@@ -774,29 +773,9 @@ def print_ir_allocator_helpers():
|
||||
output_file.write(") {\n")
|
||||
output_file.write("\t\tauto ListDataBegin = DualListData.ListBegin();\n")
|
||||
|
||||
idx = 0
|
||||
for arg in op.Arguments:
|
||||
if arg.IsSSA:
|
||||
# Inline an immediate if we can
|
||||
inline = op.Inline[idx]
|
||||
idx += 1
|
||||
|
||||
if inline != '':
|
||||
Sized = "Size" in [x.Name for x in op.Arguments]
|
||||
P = ["Size" if Sized else "OpSize::i64Bit", arg.Name]
|
||||
|
||||
# A few cases need extra info plumbed.
|
||||
if inline == "SubtractZero":
|
||||
P += ["Src2"]
|
||||
elif inline == "Mem":
|
||||
P += ["OffsetType", "OffsetScale"]
|
||||
elif inline == "Memtso":
|
||||
P += ["OffsetType", "OffsetScale", "true /* TSO */"]
|
||||
inline = "Mem"
|
||||
|
||||
output_file.write(f"\t\t{arg.Name} = Inline{inline}({', '.join(P)});\n")
|
||||
|
||||
output_file.write(f"\t\t{arg.Name}->AddUse();\n")
|
||||
output_file.write("\t\t{}->AddUse();\n".format(arg.Name))
|
||||
|
||||
# Insert validation here. This is skipped for the
|
||||
# OrderedNodeWrapper version because validation can depend on
|
||||
@@ -806,15 +785,16 @@ def print_ir_allocator_helpers():
|
||||
print_validation(op)
|
||||
|
||||
output_file.write(f"\t\treturn _{op.Name}(")
|
||||
for i, arg in enumerate(op.Arguments):
|
||||
LastArg = i == len(op.Arguments) - 1
|
||||
for i in range(0, len(op.Arguments)):
|
||||
arg = op.Arguments[i]
|
||||
LastArg = len(op.Arguments) - i - 1 == 0
|
||||
output_file.write(arg.Name)
|
||||
if arg.IsSSA:
|
||||
output_file.write("->Wrapped(ListDataBegin)")
|
||||
if not LastArg:
|
||||
output_file.write(", ")
|
||||
output_file.write(");\n")
|
||||
output_file.write("\t}\n\n")
|
||||
output_file.write(");\n");
|
||||
output_file.write("\t}\n\n");
|
||||
|
||||
output_file.write("#undef IROP_ALLOCATE_HELPERS\n")
|
||||
output_file.write("#endif\n")
|
||||
@@ -845,8 +825,8 @@ def print_ir_dispatcher_dispatch():
|
||||
output_dispatch_file.write("#endif\n")
|
||||
|
||||
|
||||
if len(sys.argv) < 4:
|
||||
ExitError("Insufficient parameters passed to script")
|
||||
if (len(sys.argv) < 4):
|
||||
ExitError()
|
||||
|
||||
output_filename = sys.argv[2]
|
||||
output_dispatcher_filename = sys.argv[3]
|
||||
@@ -858,7 +838,6 @@ json_file.close()
|
||||
json_object = json.loads(json_text)
|
||||
json_object = {k.upper(): v for k, v in json_object.items()}
|
||||
|
||||
enums = json_object["ENUMS"]
|
||||
ops = json_object["OPS"]
|
||||
irtypes = json_object["IRTYPES"]
|
||||
defines = json_object["DEFINES"]
|
||||
@@ -868,7 +847,7 @@ parse_ops(ops)
|
||||
|
||||
output_file = open(output_filename, "w")
|
||||
|
||||
print_enums(enums)
|
||||
print_enums()
|
||||
print_ir_structs(defines)
|
||||
print_ir_sizes()
|
||||
print_ir_reg_classes()
|
||||
|
||||
@@ -1,28 +1,32 @@
|
||||
set(MAN_DIR share/man CACHE PATH "MAN_DIR")
|
||||
include(GNUInstallDirs)
|
||||
set (MAN_DIR share/man CACHE PATH "MAN_DIR")
|
||||
|
||||
set(FEXCORE_BASE_SRCS
|
||||
set (FEXCORE_BASE_SRCS
|
||||
Interface/Config/Config.cpp
|
||||
Utils/Allocator.cpp
|
||||
Utils/FileLoading.cpp
|
||||
Utils/ForcedAssert.cpp
|
||||
Utils/LogManager.cpp
|
||||
Utils/SpinWaitLock.cpp)
|
||||
Utils/SpinWaitLock.cpp
|
||||
)
|
||||
|
||||
if (NOT MINGW)
|
||||
if (NOT MINGW_BUILD)
|
||||
list(APPEND FEXCORE_BASE_SRCS
|
||||
Utils/Allocator/64BitAllocator.cpp)
|
||||
endif()
|
||||
|
||||
set(SRCS
|
||||
set (SRCS
|
||||
Common/JitSymbols.cpp
|
||||
Interface/Context/Context.cpp
|
||||
Interface/Core/LookupCache.cpp
|
||||
Interface/Core/CodeCache.cpp
|
||||
Interface/Core/Core.cpp
|
||||
Interface/Core/CPUBackend.cpp
|
||||
Interface/Core/Addressing.cpp
|
||||
Interface/Core/CPUID.cpp
|
||||
Interface/Core/Frontend.cpp
|
||||
Interface/Core/ObjectCache/JobHandling.cpp
|
||||
Interface/Core/ObjectCache/NamedRegionObjectHandler.cpp
|
||||
Interface/Core/ObjectCache/ObjectCacheService.cpp
|
||||
Interface/Core/OpcodeDispatcher/AVX_128.cpp
|
||||
Interface/Core/OpcodeDispatcher/Crypto.cpp
|
||||
Interface/Core/OpcodeDispatcher/Flags.cpp
|
||||
@@ -30,6 +34,8 @@ set(SRCS
|
||||
Interface/Core/OpcodeDispatcher/X87.cpp
|
||||
Interface/Core/OpcodeDispatcher/X87F64.cpp
|
||||
Interface/Core/OpcodeDispatcher.cpp
|
||||
Interface/Core/X86Tables.cpp
|
||||
Interface/Core/X86HelperGen.cpp
|
||||
Interface/Core/ArchHelpers/Arm64Emitter.cpp
|
||||
Interface/Core/Dispatcher/Dispatcher.cpp
|
||||
Interface/Core/Interpreter/Fallbacks/InterpreterFallbacks.cpp
|
||||
@@ -56,20 +62,22 @@ set(SRCS
|
||||
Interface/Core/X86Tables/VEXTables.cpp
|
||||
Interface/Core/X86Tables/X87Tables.cpp
|
||||
Interface/GDBJIT/GDBJIT.cpp
|
||||
Interface/IR/AOTIR.cpp
|
||||
Interface/IR/IRDumper.cpp
|
||||
Interface/IR/IREmitter.cpp
|
||||
Interface/IR/PassManager.cpp
|
||||
Interface/IR/Passes/ConstProp.cpp
|
||||
Interface/IR/Passes/IRDumperPass.cpp
|
||||
Interface/IR/Passes/IRValidation.cpp
|
||||
Interface/IR/Passes/RedundantFlagCalculationElimination.cpp
|
||||
Interface/IR/Passes/RegisterAllocationPass.cpp
|
||||
Interface/IR/Passes/x87StackOptimizationPass.cpp
|
||||
Utils/LongJump.cpp
|
||||
Utils/Telemetry.cpp
|
||||
Utils/Threads.cpp
|
||||
Utils/Profiler.cpp)
|
||||
Utils/Profiler.cpp
|
||||
)
|
||||
|
||||
if (ARCHITECTURE_arm64)
|
||||
if (_M_ARM_64)
|
||||
list(APPEND SRCS Utils/ArchHelpers/Arm64.cpp)
|
||||
else()
|
||||
list(APPEND SRCS Utils/ArchHelpers/Arm64_stubs.cpp)
|
||||
@@ -82,41 +90,42 @@ endif()
|
||||
|
||||
set(DEFINES -DJIT_ARM64)
|
||||
|
||||
if (ARCHITECTURE_x86_64)
|
||||
list(APPEND DEFINES -DARCHITECTURE_x86_64=1)
|
||||
if (_M_X86_64)
|
||||
list(APPEND DEFINES -D_M_X86_64=1)
|
||||
endif()
|
||||
|
||||
if (ARCHITECTURE_arm64)
|
||||
list(APPEND DEFINES -DARCHITECTURE_arm64=1)
|
||||
if (_M_ARM_64)
|
||||
list(APPEND DEFINES -D_M_ARM_64=1)
|
||||
endif()
|
||||
|
||||
if (ENABLE_VIXL_DISASSEMBLER)
|
||||
list(APPEND DEFINES -DVIXL_DISASSEMBLER=1)
|
||||
endif()
|
||||
|
||||
if (ARCHITECTURE_arm64 AND HAS_CLANG_PRESERVE_ALL)
|
||||
if (_M_ARM_64 AND HAS_CLANG_PRESERVE_ALL)
|
||||
list(APPEND DEFINES "-DFEXCORE_PRESERVE_ALL_ATTR=__attribute__((preserve_all));-DFEXCORE_HAS_PRESERVE_ALL_ATTR=1")
|
||||
else()
|
||||
list(APPEND DEFINES "-DFEXCORE_PRESERVE_ALL_ATTR=;-DFEXCORE_HAS_PRESERVE_ALL_ATTR=0")
|
||||
endif()
|
||||
|
||||
set(LIBS fmt::fmt xxHash::xxhash FEXHeaderUtils CodeEmitter cephes_128bit)
|
||||
set (LIBS fmt::fmt xxHash::xxhash FEXHeaderUtils CodeEmitter cephes_128bit)
|
||||
|
||||
if (ENABLE_VIXL_DISASSEMBLER OR ENABLE_VIXL_SIMULATOR)
|
||||
list(APPEND LIBS vixl)
|
||||
list (APPEND LIBS vixl)
|
||||
endif()
|
||||
|
||||
if (NOT MINGW)
|
||||
list(APPEND LIBS dl)
|
||||
if (NOT MINGW_BUILD)
|
||||
list (APPEND LIBS dl)
|
||||
else()
|
||||
list(APPEND LIBS synchronization)
|
||||
if (ARCHITECTURE_arm64ec)
|
||||
list(APPEND LIBS mincore)
|
||||
list (APPEND LIBS synchronization)
|
||||
if (_M_ARM_64EC)
|
||||
list (APPEND LIBS mincore)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
# Generate config
|
||||
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/Interface/Config/Config.json.in
|
||||
configure_file(
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/Interface/Config/Config.json.in
|
||||
${CMAKE_BINARY_DIR}/generated/Config/Config.json)
|
||||
|
||||
# Generate IR include file
|
||||
@@ -131,10 +140,11 @@ add_custom_command(
|
||||
OUTPUT "${OUTPUT_NAME}" "${OUTPUT_DISPATCHER_NAME}"
|
||||
DEPENDS "${INPUT_NAME}"
|
||||
DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/../Scripts/json_ir_generator.py"
|
||||
COMMAND "python3" "${CMAKE_CURRENT_SOURCE_DIR}/../Scripts/json_ir_generator.py"
|
||||
"${INPUT_NAME}" "${OUTPUT_NAME}" "${OUTPUT_DISPATCHER_NAME}")
|
||||
COMMAND "python3" "${CMAKE_CURRENT_SOURCE_DIR}/../Scripts/json_ir_generator.py" "${INPUT_NAME}" "${OUTPUT_NAME}" "${OUTPUT_DISPATCHER_NAME}"
|
||||
)
|
||||
|
||||
set_source_files_properties(${OUTPUT_NAME} PROPERTIES GENERATED TRUE)
|
||||
set_source_files_properties(${OUTPUT_NAME} PROPERTIES
|
||||
GENERATED TRUE)
|
||||
|
||||
# Generate IR documentation
|
||||
set(OUTPUT_IR_DOC "${CMAKE_BINARY_DIR}/IR.md")
|
||||
@@ -143,10 +153,11 @@ add_custom_command(
|
||||
OUTPUT "${OUTPUT_IR_DOC}"
|
||||
DEPENDS "${INPUT_NAME}"
|
||||
DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/../Scripts/json_ir_doc_generator.py"
|
||||
COMMAND "python3" "${CMAKE_CURRENT_SOURCE_DIR}/../Scripts/json_ir_doc_generator.py"
|
||||
"${INPUT_NAME}" "${OUTPUT_IR_DOC}")
|
||||
COMMAND "python3" "${CMAKE_CURRENT_SOURCE_DIR}/../Scripts/json_ir_doc_generator.py" "${INPUT_NAME}" "${OUTPUT_IR_DOC}"
|
||||
)
|
||||
|
||||
set_source_files_properties(${OUTPUT_IR_NAME} PROPERTIES GENERATED TRUE)
|
||||
set_source_files_properties(${OUTPUT_IR_NAME} PROPERTIES
|
||||
GENERATED TRUE)
|
||||
|
||||
# Create the target
|
||||
add_custom_target(IR_INC
|
||||
@@ -170,12 +181,14 @@ add_custom_command(
|
||||
DEPENDS "${INPUT_CONFIG_NAME}"
|
||||
DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/../Scripts/config_generator.py"
|
||||
COMMAND "python3" "${CMAKE_CURRENT_SOURCE_DIR}/../Scripts/config_generator.py" "${INPUT_CONFIG_NAME}" "${OUTPUT_CONFIG_NAME}" "${OUTPUT_MAN_NAME}"
|
||||
"${OUTPUT_CONFIG_OPTION_NAME}")
|
||||
"${OUTPUT_CONFIG_OPTION_NAME}"
|
||||
)
|
||||
|
||||
add_custom_command(
|
||||
OUTPUT "${OUTPUT_MAN_NAME_COMPRESS}"
|
||||
DEPENDS "${OUTPUT_MAN_NAME}"
|
||||
COMMAND "gzip" "-kf9n" "${OUTPUT_MAN_NAME}")
|
||||
COMMAND "gzip" "-kf9n" "${OUTPUT_MAN_NAME}"
|
||||
)
|
||||
|
||||
set_source_files_properties(${OUTPUT_CONFIG_NAME} PROPERTIES
|
||||
GENERATED TRUE)
|
||||
@@ -194,10 +207,8 @@ add_custom_target(CONFIG_INC
|
||||
DEPENDS "${OUTPUT_MAN_NAME}"
|
||||
DEPENDS "${OUTPUT_MAN_NAME_COMPRESS}")
|
||||
|
||||
if (NOT BUILD_STEAM_SUPPORT)
|
||||
# Install the compressed man page
|
||||
install(FILES ${OUTPUT_MAN_NAME_COMPRESS} COMPONENT Runtime DESTINATION ${MAN_DIR}/man1)
|
||||
endif()
|
||||
# Install the compressed man page
|
||||
install(FILES ${OUTPUT_MAN_NAME_COMPRESS} DESTINATION ${MAN_DIR}/man1)
|
||||
|
||||
# Add in diagnostic colours if the option is available.
|
||||
# Ninja code generator will kill colours if this isn't here
|
||||
@@ -219,7 +230,8 @@ function(AddDefaultOptionsToTarget Name)
|
||||
target_compile_definitions(${Name} PRIVATE ${DEFINES})
|
||||
add_dependencies(${Name} CONFIG_INC IR_INC)
|
||||
|
||||
target_compile_options(${Name} PRIVATE
|
||||
target_compile_options(${Name}
|
||||
PRIVATE
|
||||
-Wall
|
||||
-Werror=cast-qual
|
||||
-Werror=ignored-qualifiers
|
||||
@@ -227,20 +239,31 @@ function(AddDefaultOptionsToTarget Name)
|
||||
|
||||
-Wno-trigraphs
|
||||
-ffunction-sections
|
||||
-fwrapv)
|
||||
-fwrapv
|
||||
)
|
||||
|
||||
if (GCC_COLOR)
|
||||
target_compile_options(${Name} PRIVATE "-fdiagnostics-color=always")
|
||||
target_compile_options(${Name}
|
||||
PRIVATE
|
||||
"-fdiagnostics-color=always")
|
||||
endif()
|
||||
|
||||
if (CLANG_COLOR)
|
||||
target_compile_options(${Name} PRIVATE "-fcolor-diagnostics")
|
||||
target_compile_options(${Name}
|
||||
PRIVATE
|
||||
"-fcolor-diagnostics")
|
||||
endif()
|
||||
|
||||
LinkerGC(${Name})
|
||||
if (CMAKE_BUILD_TYPE MATCHES "RELEASE")
|
||||
target_link_options(${Name}
|
||||
PRIVATE
|
||||
"LINKER:--gc-sections"
|
||||
"LINKER:--strip-all"
|
||||
"LINKER:--as-needed"
|
||||
)
|
||||
endif()
|
||||
endfunction()
|
||||
|
||||
# Build FEXCore_Base static library
|
||||
# Build FEXCore_Config static library
|
||||
add_library(FEXCore_Base STATIC ${FEXCORE_BASE_SRCS})
|
||||
target_link_libraries(FEXCore_Base ${LIBS})
|
||||
AddDefaultOptionsToTarget(FEXCore_Base)
|
||||
@@ -249,34 +272,34 @@ if (ENABLE_FEXCORE_PROFILER AND FEXCORE_PROFILER_BACKEND STREQUAL "TRACY")
|
||||
target_link_libraries(FEXCore_Base TracyClient)
|
||||
endif()
|
||||
|
||||
function(AddObject Name)
|
||||
add_library(${Name} OBJECT ${SRCS})
|
||||
function(AddObject Name Type)
|
||||
add_library(${Name} ${Type} ${SRCS})
|
||||
|
||||
target_link_libraries(${Name} PRIVATE FEXCore_Base)
|
||||
target_link_libraries(${Name} FEXCore_Base)
|
||||
target_compile_options(${Name} PRIVATE ${FEX_TUNE_COMPILE_FLAGS})
|
||||
AddDefaultOptionsToTarget(${Name})
|
||||
|
||||
set_target_properties(${Name} PROPERTIES OUTPUT_NAME FEXCore)
|
||||
endfunction()
|
||||
|
||||
function(AddLibrary Name Type)
|
||||
add_library(${Name} ${Type} $<TARGET_OBJECTS:${PROJECT_NAME}_object>)
|
||||
target_link_libraries(${Name} FEXCore_Base)
|
||||
target_compile_options(${Name} PRIVATE ${FEX_TUNE_COMPILE_FLAGS})
|
||||
set_target_properties(${Name} PROPERTIES OUTPUT_NAME FEXCore)
|
||||
|
||||
# During generation of the import library (dll.a), MinGW needs some extra symbols from libraries
|
||||
# such as fmt, which are propagated by FEXCore_Base. Wonderful.
|
||||
if (MINGW)
|
||||
target_link_libraries(${Name} FEXCore_Base)
|
||||
endif()
|
||||
AddDefaultOptionsToTarget(${Name})
|
||||
endfunction()
|
||||
|
||||
AddObject(${PROJECT_NAME}_object)
|
||||
AddObject(${PROJECT_NAME}_object OBJECT)
|
||||
AddLibrary(${PROJECT_NAME} STATIC)
|
||||
AddLibrary(${PROJECT_NAME}_shared SHARED)
|
||||
|
||||
if (NOT MINGW AND NOT BUILD_STEAM_SUPPORT)
|
||||
install(TARGETS ${PROJECT_NAME}_shared LIBRARY
|
||||
DESTINATION ${CMAKE_INSTALL_LIBDIR}
|
||||
COMPONENT Libraries)
|
||||
if (NOT MINGW_BUILD)
|
||||
install(TARGETS ${PROJECT_NAME}_shared
|
||||
LIBRARY
|
||||
DESTINATION ${CMAKE_INSTALL_LIBDIR}
|
||||
COMPONENT Libraries)
|
||||
endif()
|
||||
|
||||
# Meta-library to link jemalloc libraries enabled in the build configuration.
|
||||
@@ -291,7 +314,7 @@ if (ENABLE_JEMALLOC_GLIBC_ALLOC)
|
||||
target_link_libraries(JemallocLibs INTERFACE FEX_jemalloc_glibc)
|
||||
endif()
|
||||
|
||||
if (NOT MINGW)
|
||||
if (NOT MINGW_BUILD)
|
||||
# Dummy project to use for host tools.
|
||||
# This overrides use of jemalloc in FEXCore with the normal glibc allocator.
|
||||
add_library(JemallocDummy STATIC Utils/AllocatorHooks.cpp)
|
||||
|
||||
@@ -1,19 +1,20 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
#pragma once
|
||||
#include <FEXCore/Utils/TypeDefines.h>
|
||||
|
||||
#include <FEXCore/fextl/memory.h>
|
||||
#include <FEXCore/Debug/InternalThreadState.h>
|
||||
|
||||
#include <chrono>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <memory>
|
||||
#include <string_view>
|
||||
|
||||
namespace FEXCore {
|
||||
// Buffered JIT symbol tracking.
|
||||
struct JITSymbolBuffer {
|
||||
// Maximum buffer size to ensure we are a page in size.
|
||||
constexpr static size_t BUFFER_SIZE = FEXCore::Utils::FEX_PAGE_SIZE - (8 * 2);
|
||||
constexpr static size_t BUFFER_SIZE = 4096 - (8 * 2);
|
||||
// Maximum distance until the end of the buffer to do a write.
|
||||
constexpr static size_t NEEDS_WRITE_DISTANCE = BUFFER_SIZE - 64;
|
||||
// Maximum time threshhold to wait before a buffer write occurs.
|
||||
@@ -28,7 +29,7 @@ struct JITSymbolBuffer {
|
||||
size_t Offset {};
|
||||
char Buffer[BUFFER_SIZE] {};
|
||||
};
|
||||
static_assert(sizeof(JITSymbolBuffer) == FEXCore::Utils::FEX_PAGE_SIZE, "Ensure this is one page in size");
|
||||
static_assert(sizeof(JITSymbolBuffer) == 4096, "Ensure this is one page in size");
|
||||
|
||||
class JITSymbols final {
|
||||
public:
|
||||
|
||||
@@ -4,9 +4,9 @@
|
||||
#include <FEXCore/Utils/LogManager.h>
|
||||
#include <FEXCore/fextl/sstream.h>
|
||||
#include <FEXCore/fextl/string.h>
|
||||
#include <FEXHeaderUtils/BitUtils.h>
|
||||
#include "cephes_128bit.h"
|
||||
|
||||
#include <bit>
|
||||
#include <cmath>
|
||||
#include <cstring>
|
||||
#include <stdint.h>
|
||||
@@ -19,7 +19,7 @@ extern "C" {
|
||||
}
|
||||
|
||||
struct FEX_PACKED X80SoftFloat {
|
||||
#ifdef ARCHITECTURE_x86_64
|
||||
#ifdef _M_X86_64
|
||||
// Define this to push some operations to x87
|
||||
// Only useful to see if precision loss is killing something
|
||||
// #define DEBUG_X86_FLOAT
|
||||
@@ -30,7 +30,7 @@ struct FEX_PACKED X80SoftFloat {
|
||||
#define BIGFLOAT float128_t
|
||||
#define BIGFLOATSIZE 16
|
||||
#endif
|
||||
#elif defined(ARCHITECTURE_arm64)
|
||||
#elif defined(_M_ARM_64)
|
||||
#define BIGFLOAT float128_t
|
||||
#define BIGFLOATSIZE 16
|
||||
#else
|
||||
@@ -157,30 +157,7 @@ struct FEX_PACKED X80SoftFloat {
|
||||
return Result;
|
||||
#else
|
||||
/*
|
||||
* Check for invalid operation cases first - Intel FPREM sets Invalid Operation
|
||||
* for several cases including infinity dividend and zero divisor.
|
||||
*/
|
||||
X80SoftFloat result = 0;
|
||||
if (HandleInfinityOp(state, lhs, result)) {
|
||||
return result;
|
||||
} else if (lhs.Exponent == 0x7FFF && (lhs.Significand & 0x7FFFFFFFFFFFFFFFULL)) { // NaN
|
||||
// propagate NaN
|
||||
state->exceptionFlags |= softfloat_flag_invalid;
|
||||
return lhs;
|
||||
}
|
||||
|
||||
// Check for zero divisor - fprem(x, 0) is invalid operation
|
||||
if (rhs.Exponent == 0 && rhs.Significand == 0) {
|
||||
state->exceptionFlags |= softfloat_flag_invalid;
|
||||
// Return QNaN
|
||||
result.Sign = 0;
|
||||
result.Exponent = 0x7FFF;
|
||||
result.Significand = 0xC000000000000000ULL;
|
||||
return result;
|
||||
}
|
||||
|
||||
/*
|
||||
* FPREM is not an IEEE-754 remainder. From the Intel spec:
|
||||
* FPREM is not an IEEE-754 remainder. From the spec:
|
||||
*
|
||||
* Computes the remainder obtained from dividing the value in the ST(0)
|
||||
* register (the dividend) by the value in the ST(1) register (the divisor
|
||||
@@ -294,11 +271,7 @@ struct FEX_PACKED X80SoftFloat {
|
||||
FCMP(softfloat_state* state, const X80SoftFloat& lhs, const X80SoftFloat& rhs, bool* eq, bool* lt, bool* nan) {
|
||||
*eq = extF80_eq(state, lhs, rhs);
|
||||
*lt = extF80_lt(state, lhs, rhs);
|
||||
|
||||
// Use IEEE 754 semantics: unordered if neither <, =, nor > is true
|
||||
// This is more reliable than custom NaN detection
|
||||
bool gt = !(*eq) && !(*lt) && extF80_le(state, rhs, lhs);
|
||||
*nan = !(*eq) && !(*lt) && !gt;
|
||||
*nan = IsNan(lhs) || IsNan(rhs);
|
||||
}
|
||||
|
||||
FEXCORE_PRESERVE_ALL_ATTR static X80SoftFloat FSCALE(softfloat_state* state, const X80SoftFloat& lhs, const X80SoftFloat& rhs) {
|
||||
@@ -417,11 +390,6 @@ struct FEX_PACKED X80SoftFloat {
|
||||
|
||||
return Result;
|
||||
#else
|
||||
X80SoftFloat result;
|
||||
if (HandleInfinityOp(state, lhs, result)) {
|
||||
return result;
|
||||
}
|
||||
|
||||
BIGFLOAT Src_d = lhs.ToFMax(state);
|
||||
Src_d = FEXCore::cephes_128bit::tanl(Src_d);
|
||||
return X80SoftFloat(state, Src_d);
|
||||
@@ -443,11 +411,6 @@ struct FEX_PACKED X80SoftFloat {
|
||||
|
||||
return Result;
|
||||
#else
|
||||
X80SoftFloat result;
|
||||
if (HandleInfinityOp(state, lhs, result)) {
|
||||
return result;
|
||||
}
|
||||
|
||||
BIGFLOAT Src_d = lhs.ToFMax(state);
|
||||
Src_d = FEXCore::cephes_128bit::sinl(Src_d);
|
||||
return X80SoftFloat(state, Src_d);
|
||||
@@ -469,11 +432,6 @@ struct FEX_PACKED X80SoftFloat {
|
||||
|
||||
return Result;
|
||||
#else
|
||||
X80SoftFloat result;
|
||||
if (HandleInfinityOp(state, lhs, result)) {
|
||||
return result;
|
||||
}
|
||||
|
||||
BIGFLOAT Src_d = lhs.ToFMax(state);
|
||||
Src_d = FEXCore::cephes_128bit::cosl(Src_d);
|
||||
return X80SoftFloat(state, Src_d);
|
||||
@@ -501,12 +459,12 @@ struct FEX_PACKED X80SoftFloat {
|
||||
|
||||
float ToF32(softfloat_state* state) const {
|
||||
const float32_t Result = extF80_to_f32(state, *this);
|
||||
return std::bit_cast<float>(Result);
|
||||
return FEXCore::BitCast<float>(Result);
|
||||
}
|
||||
|
||||
double ToF64(softfloat_state* state) const {
|
||||
const float64_t Result = extF80_to_f64(state, *this);
|
||||
return std::bit_cast<double>(Result);
|
||||
return FEXCore::BitCast<double>(Result);
|
||||
}
|
||||
|
||||
FEXCore::VectorRegType ToVector() const {
|
||||
@@ -518,7 +476,7 @@ struct FEX_PACKED X80SoftFloat {
|
||||
BIGFLOAT ToFMax(softfloat_state* state) const {
|
||||
#if BIGFLOATSIZE == 16
|
||||
const float128_t Result = extF80_to_f128(state, *this);
|
||||
return std::bit_cast<BIGFLOAT>(Result);
|
||||
return FEXCore::BitCast<BIGFLOAT>(Result);
|
||||
#else
|
||||
BIGFLOAT result {};
|
||||
memcpy(&result, this, sizeof(result));
|
||||
@@ -577,18 +535,18 @@ struct FEX_PACKED X80SoftFloat {
|
||||
}
|
||||
|
||||
X80SoftFloat(softfloat_state* state, const float rhs) {
|
||||
*this = f32_to_extF80(state, std::bit_cast<float32_t>(rhs));
|
||||
*this = f32_to_extF80(state, FEXCore::BitCast<float32_t>(rhs));
|
||||
}
|
||||
|
||||
X80SoftFloat(softfloat_state* state, const double rhs) {
|
||||
*this = f64_to_extF80(state, std::bit_cast<float64_t>(rhs));
|
||||
*this = f64_to_extF80(state, FEXCore::BitCast<float64_t>(rhs));
|
||||
}
|
||||
|
||||
X80SoftFloat(softfloat_state* state, BIGFLOAT rhs) {
|
||||
#if BIGFLOATSIZE == 16
|
||||
*this = f128_to_extF80(state, std::bit_cast<float128_t>(rhs));
|
||||
*this = f128_to_extF80(state, FEXCore::BitCast<float128_t>(rhs));
|
||||
#else
|
||||
*this = std::bit_cast<long double>(rhs);
|
||||
*this = FEXCore::BitCast<long double>(rhs);
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -633,20 +591,6 @@ private:
|
||||
static constexpr uint64_t IntegerBit = (1ULL << 63);
|
||||
static constexpr uint64_t Bottom62Significand = ((1ULL << 62) - 1);
|
||||
static constexpr uint32_t ExponentBias = 16383;
|
||||
|
||||
// Helper function to check for infinity and set invalid operation flag.
|
||||
// Returns true if infinity is dealt with, false otherwise.
|
||||
FEXCORE_PRESERVE_ALL_ATTR static bool HandleInfinityOp(softfloat_state* state, const X80SoftFloat& arg, X80SoftFloat& result) {
|
||||
if (arg.Exponent == 0x7FFF && arg.Significand == 0x8000000000000000ULL) {
|
||||
state->exceptionFlags |= softfloat_flag_invalid;
|
||||
// Return QNaN.
|
||||
result.Sign = 0;
|
||||
result.Exponent = 0x7FFF;
|
||||
result.Significand = 0xC000000000000000ULL;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
#ifndef _WIN32
|
||||
|
||||
@@ -2,27 +2,74 @@
|
||||
#pragma once
|
||||
#include <FEXCore/fextl/string.h>
|
||||
|
||||
#include <concepts>
|
||||
#include <cstdint>
|
||||
#include <string_view>
|
||||
#include <optional>
|
||||
|
||||
namespace FEXCore::StrConv {
|
||||
template<std::integral T>
|
||||
bool Conv(std::string_view Value, T* Result) {
|
||||
if constexpr (std::is_signed_v<T>) {
|
||||
*Result = static_cast<T>(std::strtoll(Value.data(), nullptr, 0));
|
||||
} else {
|
||||
*Result = static_cast<T>(std::strtoull(Value.data(), nullptr, 0));
|
||||
}
|
||||
[[maybe_unused]]
|
||||
static bool Conv(std::string_view Value, bool* Result) {
|
||||
*Result = std::strtoull(Value.data(), nullptr, 0);
|
||||
return true;
|
||||
}
|
||||
|
||||
template<typename T, typename = std::enable_if_t<std::is_enum_v<T>, T>>
|
||||
bool Conv(std::string_view Value, T* Result) {
|
||||
*Result = static_cast<T>(std::strtoull(Value.data(), nullptr, 0));
|
||||
[[maybe_unused]]
|
||||
static bool Conv(std::string_view Value, uint8_t* Result) {
|
||||
*Result = std::strtoul(Value.data(), nullptr, 0);
|
||||
return true;
|
||||
}
|
||||
|
||||
inline bool Conv(std::string_view Value, fextl::string* Result) {
|
||||
[[maybe_unused]]
|
||||
static bool Conv(std::string_view Value, int8_t* Result) {
|
||||
*Result = std::strtol(Value.data(), nullptr, 0);
|
||||
return true;
|
||||
}
|
||||
|
||||
[[maybe_unused]]
|
||||
static bool Conv(std::string_view Value, uint16_t* Result) {
|
||||
*Result = std::strtoul(Value.data(), nullptr, 0);
|
||||
return true;
|
||||
}
|
||||
|
||||
[[maybe_unused]]
|
||||
static bool Conv(std::string_view Value, int16_t* Result) {
|
||||
*Result = std::strtol(Value.data(), nullptr, 0);
|
||||
return true;
|
||||
}
|
||||
|
||||
[[maybe_unused]]
|
||||
static bool Conv(std::string_view Value, uint32_t* Result) {
|
||||
*Result = std::strtoul(Value.data(), nullptr, 0);
|
||||
return true;
|
||||
}
|
||||
|
||||
[[maybe_unused]]
|
||||
static bool Conv(std::string_view Value, int32_t* Result) {
|
||||
*Result = std::strtol(Value.data(), nullptr, 0);
|
||||
return true;
|
||||
}
|
||||
|
||||
[[maybe_unused]]
|
||||
static bool Conv(std::string_view Value, uint64_t* Result) {
|
||||
*Result = std::strtoull(Value.data(), nullptr, 0);
|
||||
return true;
|
||||
}
|
||||
|
||||
[[maybe_unused]]
|
||||
static bool Conv(std::string_view Value, int64_t* Result) {
|
||||
*Result = std::strtoll(Value.data(), nullptr, 0);
|
||||
return true;
|
||||
}
|
||||
|
||||
template<typename T, typename = std::enable_if<std::is_enum<T>::value, T>>
|
||||
[[maybe_unused]]
|
||||
static bool Conv(std::string_view Value, T* Result) {
|
||||
*Result = static_cast<T>(std::stoull(Value.data(), nullptr, 0));
|
||||
return true;
|
||||
}
|
||||
|
||||
[[maybe_unused]]
|
||||
static bool Conv(std::string_view Value, fextl::string* Result) {
|
||||
*Result = Value;
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -1,11 +1,9 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
#pragma once
|
||||
|
||||
#ifdef ARCHITECTURE_x86_64
|
||||
#ifdef _M_X86_64
|
||||
#include <xmmintrin.h>
|
||||
#include <immintrin.h>
|
||||
#else
|
||||
#include <cstdint>
|
||||
#endif
|
||||
|
||||
namespace FEXCore {
|
||||
@@ -13,7 +11,7 @@ struct VectorScalarF64Pair {
|
||||
double val[2];
|
||||
};
|
||||
|
||||
#ifdef ARCHITECTURE_arm64
|
||||
#ifdef _M_ARM_64
|
||||
// Can't use uint8x16_t directly from arm_neon.h here.
|
||||
// Overrides softfloat-3e's defines which causes problems.
|
||||
using VectorRegType = __attribute__((neon_vector_type(16))) uint8_t;
|
||||
@@ -25,7 +23,7 @@ static inline VectorRegPairType MakeVectorRegPair(VectorRegType low, VectorRegTy
|
||||
return VectorRegPairType {low, high};
|
||||
}
|
||||
|
||||
#elif defined(ARCHITECTURE_x86_64)
|
||||
#elif defined(_M_X86_64)
|
||||
using VectorRegType = __m128i;
|
||||
using VectorRegPairType = __m256i;
|
||||
|
||||
|
||||
@@ -30,14 +30,14 @@ class Context;
|
||||
}
|
||||
|
||||
namespace FEXCore::Config {
|
||||
namespace detail {
|
||||
namespace DefaultValues {
|
||||
#define P(x) x
|
||||
#define OPT_BASE(type, group, enum, json, default) const P(type) P(enum) = P(default);
|
||||
#define OPT_STR(group, enum, json, default) const std::string_view P(enum) = P(default);
|
||||
#define OPT_STRARRAY(group, enum, json, default) OPT_STR(group, enum, json, default)
|
||||
#define OPT_STRENUM(group, enum, json, default) const uint64_t P(enum) = FEXCore::ToUnderlying(P(default));
|
||||
#include <FEXCore/Config/ConfigValues.inl>
|
||||
} // namespace detail
|
||||
} // namespace DefaultValues
|
||||
|
||||
enum Paths {
|
||||
PATH_DATA_DIR_LOCAL = 0,
|
||||
@@ -134,7 +134,7 @@ public:
|
||||
void Load();
|
||||
|
||||
template<typename T>
|
||||
requires (!std::is_same_v<fextl::string, T> && !std::is_same_v<StringArrayType, T>)
|
||||
requires (!std::is_same_v<fextl::string, T> && !std::is_same_v<DefaultValues::Type::StringArrayType, T>)
|
||||
std::optional<T> GetConv(ConfigOption Option) {
|
||||
const auto it = OptionMap.find(Option);
|
||||
if (it == OptionMap.end()) {
|
||||
@@ -142,7 +142,7 @@ public:
|
||||
}
|
||||
|
||||
const auto& Value = it->second;
|
||||
LOGMAN_THROW_A_FMT(!std::holds_alternative<StringArrayType>(Value), "Tried to get config of invalid type!");
|
||||
LOGMAN_THROW_A_FMT(!std::holds_alternative<DefaultValues::Type::StringArrayType>(Value), "Tried to get config of invalid type!");
|
||||
|
||||
if (std::holds_alternative<T>(Value)) [[likely]] {
|
||||
return std::get<T>(Value);
|
||||
@@ -165,7 +165,7 @@ public:
|
||||
|
||||
private:
|
||||
void MergeConfigMap(const LayerOptions& Options);
|
||||
void MergeEnvironmentVariables(const ConfigOption& Option, const StringArrayType& Value);
|
||||
void MergeEnvironmentVariables(const ConfigOption& Option, const DefaultValues::Type::StringArrayType& Value);
|
||||
};
|
||||
|
||||
void MetaLayer::Load() {
|
||||
@@ -181,7 +181,7 @@ void MetaLayer::Load() {
|
||||
}
|
||||
|
||||
|
||||
void MetaLayer::MergeEnvironmentVariables(const ConfigOption& Option, const StringArrayType& Value) {
|
||||
void MetaLayer::MergeEnvironmentVariables(const ConfigOption& Option, const DefaultValues::Type::StringArrayType& Value) {
|
||||
// Environment variables need a bit of additional work
|
||||
// We want to merge the arrays rather than overwrite entirely
|
||||
auto MetaEnvironment = OptionMap.find(Option);
|
||||
@@ -193,7 +193,7 @@ void MetaLayer::MergeEnvironmentVariables(const ConfigOption& Option, const Stri
|
||||
|
||||
// If an environment variable exists in both current meta and in the incoming layer then the meta layer value is overwritten
|
||||
fextl::unordered_map<fextl::string, fextl::string> LookupMap;
|
||||
const auto AddToMap = [&LookupMap](const StringArrayType& Value) {
|
||||
const auto AddToMap = [&LookupMap](const DefaultValues::Type::StringArrayType& Value) {
|
||||
for (const auto& EnvVar : Value) {
|
||||
const auto ItEq = EnvVar.find_first_of('=');
|
||||
if (ItEq == fextl::string::npos) {
|
||||
@@ -209,7 +209,7 @@ void MetaLayer::MergeEnvironmentVariables(const ConfigOption& Option, const Stri
|
||||
}
|
||||
};
|
||||
|
||||
AddToMap(std::get<StringArrayType>(MetaEnvironment->second));
|
||||
AddToMap(std::get<DefaultValues::Type::StringArrayType>(MetaEnvironment->second));
|
||||
AddToMap(Value);
|
||||
|
||||
// Now with the two layers merged in the map
|
||||
@@ -225,8 +225,8 @@ void MetaLayer::MergeConfigMap(const LayerOptions& Options) {
|
||||
// Insert this layer's options, overlaying previous options that exist here
|
||||
for (auto& it : Options) {
|
||||
if (it.first == FEXCore::Config::ConfigOption::CONFIG_ENV || it.first == FEXCore::Config::ConfigOption::CONFIG_HOSTENV) {
|
||||
LOGMAN_THROW_A_FMT(std::holds_alternative<StringArrayType>(it.second), "Tried to get config of invalid type!");
|
||||
MergeEnvironmentVariables(it.first, std::get<StringArrayType>(it.second));
|
||||
LOGMAN_THROW_A_FMT(std::holds_alternative<DefaultValues::Type::StringArrayType>(it.second), "Tried to get config of invalid type!");
|
||||
MergeEnvironmentVariables(it.first, std::get<DefaultValues::Type::StringArrayType>(it.second));
|
||||
} else {
|
||||
OptionMap.insert_or_assign(it.first, it.second);
|
||||
}
|
||||
@@ -423,7 +423,7 @@ bool Exists(ConfigOption Option) {
|
||||
return Meta->OptionExists(Option);
|
||||
}
|
||||
|
||||
std::optional<StringArrayType*> All(ConfigOption Option) {
|
||||
std::optional<DefaultValues::Type::StringArrayType*> All(ConfigOption Option) {
|
||||
return Meta->All(Option);
|
||||
}
|
||||
|
||||
@@ -436,12 +436,6 @@ std::optional<T> GetConv(ConfigOption Option) {
|
||||
return Meta->GetConv<T>(Option);
|
||||
}
|
||||
|
||||
template std::optional<bool> GetConv(ConfigOption Option);
|
||||
template std::optional<uint8_t> GetConv(ConfigOption Option);
|
||||
template std::optional<int32_t> GetConv(ConfigOption Option);
|
||||
template std::optional<uint32_t> GetConv(ConfigOption Option);
|
||||
template std::optional<uint64_t> GetConv(ConfigOption Option);
|
||||
|
||||
void Set(ConfigOption Option, std::string_view Data) {
|
||||
Meta->Set(Option, Data);
|
||||
}
|
||||
@@ -497,12 +491,13 @@ template Value<uint8_t>::Value(FEXCore::Config::ConfigOption _Option, uint8_t De
|
||||
template Value<uint64_t>::Value(FEXCore::Config::ConfigOption _Option, uint64_t Default);
|
||||
|
||||
template<typename T>
|
||||
void Value<T>::GetListIfExists(FEXCore::Config::ConfigOption Option, StringArrayType* List) {
|
||||
void Value<T>::GetListIfExists(FEXCore::Config::ConfigOption Option, DefaultValues::Type::StringArrayType* List) {
|
||||
auto Value = FEXCore::Config::All(Option);
|
||||
List->clear();
|
||||
if (Value) {
|
||||
*List = **Value;
|
||||
}
|
||||
}
|
||||
template void Value<StringArrayType>::GetListIfExists(FEXCore::Config::ConfigOption Option, StringArrayType* List);
|
||||
template void Value<DefaultValues::Type::StringArrayType>::GetListIfExists(FEXCore::Config::ConfigOption Option,
|
||||
DefaultValues::Type::StringArrayType* List);
|
||||
} // namespace FEXCore::Config
|
||||
@@ -4,6 +4,7 @@
|
||||
"Multiblock": {
|
||||
"Type": "bool",
|
||||
"Default": "true",
|
||||
"ShortArg": "m",
|
||||
"Desc": [
|
||||
"Controls multiblock code compilation",
|
||||
"Can cause long JIT compilation times and stutter"
|
||||
@@ -12,22 +13,20 @@
|
||||
"MaxInst": {
|
||||
"Type": "int32",
|
||||
"Default": "5000",
|
||||
"ShortArg": "n",
|
||||
"Desc": [
|
||||
"Maximum number of instruction to store in a block"
|
||||
]
|
||||
},
|
||||
"EnableCodeCachingWIP": {
|
||||
"Type": "bool",
|
||||
"Default": "false",
|
||||
"CacheObjectCodeCompilation": {
|
||||
"Type": "uint32",
|
||||
"Default": "FEXCore::Config::ConfigObjectCodeHandler::CONFIG_NONE",
|
||||
"TextDefault": "none",
|
||||
"Choices": [ "none", "read", "readwrite" ],
|
||||
"ArgumentHandler": "CacheObjectCodeHandler",
|
||||
"Desc": [
|
||||
"Enable the code caching subsystem"
|
||||
]
|
||||
},
|
||||
"EnableCodeCacheValidation": {
|
||||
"Type": "bool",
|
||||
"Default": "false",
|
||||
"Desc": [
|
||||
"Enable expensive validation when loading code caches"
|
||||
"Cache JIT object code to drive.",
|
||||
"Allows JIT code to be shared between applications"
|
||||
]
|
||||
},
|
||||
"HostFeatures": {
|
||||
@@ -69,13 +68,7 @@
|
||||
"ENABLESVEBITPERM": "enablesvebitperm",
|
||||
"DISABLESVEBITPERM": "disablesvebitperm",
|
||||
"ENABLEPRESERVEALLABI": "enablepreserveallabi",
|
||||
"DISABLEPRESERVEALLABI": "disablepreserveallabi",
|
||||
"ENABLEWFXT": "enablewfxt",
|
||||
"DISABLEWFXT": "disablewfxt",
|
||||
"ENABLE3DNOW": "enable3dnow",
|
||||
"DISABLE3DNOW": "disable3dnow",
|
||||
"ENABLESSE4A": "enablesse4a",
|
||||
"DISABLESSE4A": "disablesse4a"
|
||||
"DISABLEPRESERVEALLABI": "disablepreserveallabi"
|
||||
},
|
||||
"Desc": [
|
||||
"Allows controlling of the CPU features in the JIT.",
|
||||
@@ -96,10 +89,7 @@
|
||||
"\t{enable,disable}crypto: Will force enable or disable crypto extensions even if the host doesn't support it",
|
||||
"\t{enable,disable}rpres: Will force enable or disable rpres even if the host doesn't support it",
|
||||
"\t{enable,disable}svebitperm: Will force enable or disable svebitperm even if the host doesn't support it",
|
||||
"\t{enable,disable}preserveallabi: Will force enable or disable preserve_all abi even if the host doesn't support it",
|
||||
"\t{enable,disable}wfxt: Will force enable or disable wfxt even if the host doesn't support it",
|
||||
"\t{enable,disable}3dnow: Will force enable or disable 3DNow! even if the host doesn't support it",
|
||||
"\t{enable,disable}sse4a: Will force enable or disable SSE4a even if the host doesn't support it"
|
||||
"\t{enable,disable}preserveallabi: Will force enable or disable preserve_all abi even if the host doesn't support it"
|
||||
]
|
||||
},
|
||||
"SmallTSCScale": {
|
||||
@@ -108,19 +98,13 @@
|
||||
"Desc": [
|
||||
"Scales the cycle counter on systems that have low frequencies."
|
||||
]
|
||||
},
|
||||
"CPUFeatureRegisters": {
|
||||
"Type": "str",
|
||||
"Default": "",
|
||||
"Desc": [
|
||||
"Allows overriding cpu feature flags for manual testing"
|
||||
]
|
||||
}
|
||||
},
|
||||
"Emulation": {
|
||||
"RootFS": {
|
||||
"Type": "str",
|
||||
"Default": "",
|
||||
"ShortArg": "R",
|
||||
"Desc": [
|
||||
"Which Root filesystem prefix to use",
|
||||
"This can be a filesystem path",
|
||||
@@ -135,6 +119,7 @@
|
||||
"ThunkHostLibs": {
|
||||
"Type": "str",
|
||||
"Default": "@CMAKE_INSTALL_FULL_LIBDIR@/fex-emu/HostThunks",
|
||||
"ShortArg": "t",
|
||||
"Desc": [
|
||||
"Folder to find the host-side thunking libraries."
|
||||
]
|
||||
@@ -142,6 +127,7 @@
|
||||
"ThunkGuestLibs": {
|
||||
"Type": "str",
|
||||
"Default": "@CMAKE_INSTALL_PREFIX@/share/fex-emu/GuestThunks",
|
||||
"ShortArg": "j",
|
||||
"Desc": [
|
||||
"Folder to find the guest-side thunking libraries."
|
||||
]
|
||||
@@ -149,6 +135,7 @@
|
||||
"ThunkConfig": {
|
||||
"Type": "str",
|
||||
"Default": "",
|
||||
"ShortArg": "k",
|
||||
"Desc": [
|
||||
"A json file specifying where to overlay the thunks.",
|
||||
"This can be a filesystem path",
|
||||
@@ -163,6 +150,7 @@
|
||||
"Env": {
|
||||
"Type": "strarray",
|
||||
"Default": "",
|
||||
"ShortArg": "E",
|
||||
"Desc": [
|
||||
"Adds an environment variable to the emulated environment."
|
||||
]
|
||||
@@ -170,6 +158,7 @@
|
||||
"HostEnv": {
|
||||
"Type": "strarray",
|
||||
"Default": "",
|
||||
"ShortArg": "H",
|
||||
"Desc": [
|
||||
"Adds an environment variable to the host environment.",
|
||||
"This can be useful for setting environment variables that thunks can pick up.",
|
||||
@@ -182,50 +171,13 @@
|
||||
"Desc": [
|
||||
"Allows the user to pass additional arguments to the application"
|
||||
]
|
||||
},
|
||||
"DisableL2Cache": {
|
||||
"Type": "bool",
|
||||
"Default": "false",
|
||||
"Desc": [
|
||||
"Disables FEXCore's JIT L2 cache lookup. Saving memory.",
|
||||
"Can potentially introduce more stutters."
|
||||
]
|
||||
},
|
||||
"DynamicL1Cache": {
|
||||
"Type": "bool",
|
||||
"Default": "false",
|
||||
"Desc": [
|
||||
"Switches FEXCore's JIT L1 cache to be dynamically sized. Saving memory.",
|
||||
"Can potentially introduce more stutters."
|
||||
]
|
||||
},
|
||||
"DynamicL1CacheIncreaseCountHeuristic": {
|
||||
"Type": "uint64",
|
||||
"Default": "250",
|
||||
"Desc": [
|
||||
"Threshold of lookups per second that the L1 dynamic cache should increase its size.",
|
||||
"Lower numbers means more aggressive scaling upward to the maximum size.",
|
||||
"Higher numbers means more conservative scaling, using less memory.",
|
||||
"Can potentially introduce stutters, more likely the higher the number.",
|
||||
"Don't have this number smaller than the decrease count!"
|
||||
]
|
||||
},
|
||||
"DynamicL1CacheDecreaseCountHeuristic": {
|
||||
"Type": "uint64",
|
||||
"Default": "50",
|
||||
"Desc": [
|
||||
"Threshold of lookups per second that the L1 dynamic cache should decrease its size.",
|
||||
"The higher the number, the more aggressively it reduces the L1 cache size.",
|
||||
"Lower numbers means more conservative memory savings.",
|
||||
"Can potentially introduce more stutters, more likely the higher the number.",
|
||||
"Don't have this number larger than the increase count!"
|
||||
]
|
||||
}
|
||||
},
|
||||
"Debug": {
|
||||
"SingleStep": {
|
||||
"Type": "bool",
|
||||
"Default": "false",
|
||||
"ShortArg": "S",
|
||||
"Desc": [
|
||||
"Single stepping configuration."
|
||||
]
|
||||
@@ -233,6 +185,7 @@
|
||||
"GdbServer": {
|
||||
"Type": "bool",
|
||||
"Default": "false",
|
||||
"ShortArg": "G",
|
||||
"Desc": [
|
||||
"Enables the GDB server."
|
||||
]
|
||||
@@ -266,6 +219,7 @@
|
||||
"DumpGPRs": {
|
||||
"Type": "bool",
|
||||
"Default": "false",
|
||||
"ShortArg": "g",
|
||||
"Desc": [
|
||||
"When the test harness ends, print the GPR state."
|
||||
]
|
||||
@@ -273,6 +227,7 @@
|
||||
"O0": {
|
||||
"Type": "bool",
|
||||
"Default": "false",
|
||||
"ShortArg": "O0",
|
||||
"Desc": [
|
||||
"Disables optimizations passes for debugging."
|
||||
]
|
||||
@@ -362,6 +317,7 @@
|
||||
"SilentLog": {
|
||||
"Type": "bool",
|
||||
"Default": "true",
|
||||
"ShortArg": "s",
|
||||
"Desc": [
|
||||
"Disables logging"
|
||||
]
|
||||
@@ -369,9 +325,10 @@
|
||||
"OutputLog": {
|
||||
"Type": "str",
|
||||
"Default": "server",
|
||||
"ShortArg": "o",
|
||||
"Desc": [
|
||||
"File to write FEX output to.",
|
||||
"[stderr, server, <Filename>]"
|
||||
"[stdout, stderr, server, <Filename>]"
|
||||
]
|
||||
},
|
||||
"TelemetryDirectory": {
|
||||
@@ -389,13 +346,6 @@
|
||||
"Enables FEX's low-overhead sampling profile statistics.",
|
||||
"Requires a supported version of Mangohud to see the results"
|
||||
]
|
||||
},
|
||||
"EnableGpuvisProfiling": {
|
||||
"Type": "bool",
|
||||
"Default": "false",
|
||||
"Desc": [
|
||||
"Enables profiling when FEX was built with the gpuvis profiler backend."
|
||||
]
|
||||
}
|
||||
},
|
||||
"Hacks": {
|
||||
@@ -450,11 +400,12 @@
|
||||
"This is required to ensure a split-lock doesn't tear inside the process"
|
||||
]
|
||||
},
|
||||
"KernelUnalignedAtomicBackpatching": {
|
||||
"TSOAutoMigration": {
|
||||
"Type": "bool",
|
||||
"Default": "true",
|
||||
"Desc": [
|
||||
"When the kernel unaligned atomic handler is enabled, use backpatching to reduce kernel context switches."
|
||||
"Automatically enables TSO when shared memory is used.",
|
||||
"Should work without issues in most cases."
|
||||
]
|
||||
},
|
||||
"VolatileMetadata": {
|
||||
@@ -462,7 +413,7 @@
|
||||
"Default": "true",
|
||||
"Desc": [
|
||||
"Use volatile metadata in PE files to inform TSO instructions when available.",
|
||||
"When metadata is unavailable falls back to the currently enabled TSO options."
|
||||
"When metadata is unavailable falls back to the currently enabled TSO options."
|
||||
]
|
||||
},
|
||||
"X87ReducedPrecision": {
|
||||
@@ -472,6 +423,23 @@
|
||||
"Emulates X87 floating point using 64-bit precision. This reduces emulation accuracy and may result in rendering bugs."
|
||||
]
|
||||
},
|
||||
"ABILocalFlags": {
|
||||
"Type": "bool",
|
||||
"Default": "false",
|
||||
"Desc": [
|
||||
"When enabled enables an optimization around flags.",
|
||||
"Assumes flags are not used across cals.",
|
||||
"Hand-written assembly can violate this assumption."
|
||||
]
|
||||
},
|
||||
"ParanoidTSO": {
|
||||
"Type": "bool",
|
||||
"Default": "false",
|
||||
"Desc": [
|
||||
"Makes TSO operations even more strict.",
|
||||
"Forces vector loadstores to also become atomic."
|
||||
]
|
||||
},
|
||||
"StallProcess": {
|
||||
"Type": "bool",
|
||||
"Default": "false",
|
||||
@@ -502,16 +470,32 @@
|
||||
"Desc": [
|
||||
"Contrains the startup sleep to only apply to processes that match this name."
|
||||
]
|
||||
},
|
||||
"MonoHacks": {
|
||||
"Type": "bool",
|
||||
"Default": "true",
|
||||
"Desc": [
|
||||
"Permits a hook-based SMC approach and smaller JIT blocks when mono is detected."
|
||||
]
|
||||
}
|
||||
},
|
||||
"Misc": {
|
||||
"AOTIRCapture": {
|
||||
"Type": "bool",
|
||||
"Default": "false",
|
||||
"Desc": [
|
||||
"Captures IR and generates an AOT IR cache.",
|
||||
"Captures both the loaded executable and libraries it loads."
|
||||
]
|
||||
},
|
||||
"AOTIRGenerate": {
|
||||
"Type": "bool",
|
||||
"Default": "false",
|
||||
"Desc": [
|
||||
"Scans file for executable code and generates an AOT IR cache.",
|
||||
"Does not run the executable."
|
||||
]
|
||||
},
|
||||
"AOTIRLoad": {
|
||||
"Type": "bool",
|
||||
"Default": "false",
|
||||
"Desc": [
|
||||
"Loads an AOT IR cache for the loaded executable."
|
||||
]
|
||||
},
|
||||
"ServerSocketPath": {
|
||||
"Type": "str",
|
||||
"Default": "",
|
||||
@@ -525,32 +509,15 @@
|
||||
"Desc": [
|
||||
"Disables inline syscalls in order to support seccomp handling"
|
||||
]
|
||||
},
|
||||
"ExtendedVolatileMetadata": {
|
||||
"Type": "str",
|
||||
"Default": "",
|
||||
"Desc": [
|
||||
"Configuration provided volatile metadata. Only implemented for WoW64/arm64ec.",
|
||||
"Limited in its use but can be handy.",
|
||||
"Extends on top of what Microsoft has for volatile metadata, but also supported for WoW64.",
|
||||
"Colon delimited modules, then semi-colon delimited instructions, then comma delimited ranges",
|
||||
"Default disables TSO in the module, unless instructions overlap the range",
|
||||
"<module>;<offset begin>-<offset-end>,...;<instruction offset to force TSO>,...:<another>",
|
||||
"examples:",
|
||||
" * Disable TSO for a full module: Just provide the module name:",
|
||||
" `hl2_linux`",
|
||||
" * Disable TSO for a part of the module:",
|
||||
" `hl2_linux;<offset begin>-<offset-end>`",
|
||||
" * Disable TSO for a part of the module, but enable TSO for some instructions within the module",
|
||||
" `hl2_linux;<offset begin>-<offset-end>;<instruction offset>,<instruction offset>`",
|
||||
" * Disable TSO for multiple modules",
|
||||
" `hl2_linux:libsdl2.so`"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"UnnamedOptions": {
|
||||
"Misc": {
|
||||
"IS_INTERPRETER": {
|
||||
"Type": "bool",
|
||||
"Default": "false"
|
||||
},
|
||||
"INTERPRETER_INSTALLED": {
|
||||
"Type": "bool",
|
||||
"Default": "false"
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
#include "Interface/Context/Context.h"
|
||||
#include "Interface/Core/OpcodeDispatcher.h"
|
||||
#include "Interface/Core/Dispatcher/Dispatcher.h"
|
||||
#include "Interface/Core/X86Tables/X86Tables.h"
|
||||
|
||||
#include <FEXCore/Core/CoreState.h>
|
||||
@@ -9,12 +8,18 @@
|
||||
#include <FEXCore/Core/CPUID.h>
|
||||
#include <FEXCore/Core/HostFeatures.h>
|
||||
#include <FEXCore/Core/SignalDelegator.h>
|
||||
#include <FEXCore/HLE/SyscallHandler.h>
|
||||
|
||||
#include <FEXCore/Core/Thunks.h>
|
||||
#include "FEXCore/Debug/InternalThreadState.h"
|
||||
|
||||
#include <string.h>
|
||||
#include <utility>
|
||||
|
||||
namespace FEXCore::Context {
|
||||
void InitializeStaticTables(OperatingMode Mode) {
|
||||
X86Tables::InitializeInfoTables(Mode);
|
||||
IR::InstallOpcodeHandlers(Mode);
|
||||
}
|
||||
|
||||
fextl::unique_ptr<FEXCore::Context::Context> FEXCore::Context::Context::CreateNewContext(const FEXCore::HostFeatures& Features) {
|
||||
return fextl::make_unique<FEXCore::Context::ContextImpl>(Features);
|
||||
}
|
||||
|
||||
@@ -2,54 +2,65 @@
|
||||
#pragma once
|
||||
|
||||
#include "Common/JitSymbols.h"
|
||||
#include "Interface/Core/CPUBackend.h"
|
||||
#include "Interface/Core/CPUID.h"
|
||||
#include <Interface/IR/IntrusiveIRList.h>
|
||||
#include "Interface/Core/X86HelperGen.h"
|
||||
#include "Interface/Core/ObjectCache/ObjectCacheService.h"
|
||||
#include "Interface/Core/Dispatcher/Dispatcher.h"
|
||||
#include "Interface/IR/AOTIR.h"
|
||||
#include <FEXCore/Config/Config.h>
|
||||
#include <FEXCore/Core/Context.h>
|
||||
#include <FEXCore/Core/CoreState.h>
|
||||
#include <FEXCore/Core/HostFeatures.h>
|
||||
#include <FEXCore/Core/SignalDelegator.h>
|
||||
#include <FEXCore/Debug/InternalThreadState.h>
|
||||
#include <FEXCore/IR/IR.h>
|
||||
#include <FEXCore/Utils/CompilerDefs.h>
|
||||
#include <FEXCore/Utils/Event.h>
|
||||
#include <FEXCore/Utils/SignalScopeGuards.h>
|
||||
#include <FEXCore/fextl/memory.h>
|
||||
#include <FEXCore/fextl/set.h>
|
||||
#include <FEXCore/fextl/string.h>
|
||||
#include <FEXCore/fextl/unordered_map.h>
|
||||
#include <FEXCore/fextl/vector.h>
|
||||
#include <FEXHeaderUtils/Syscalls.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#include <atomic>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <mutex>
|
||||
#include <optional>
|
||||
#include <shared_mutex>
|
||||
|
||||
namespace FEXCore {
|
||||
class SignalDelegator;
|
||||
class CodeLoader;
|
||||
class ThunkHandler;
|
||||
struct LookupCacheWriteLockToken;
|
||||
|
||||
namespace Core {
|
||||
struct DebugData;
|
||||
struct InternalThreadState;
|
||||
} // namespace Core
|
||||
namespace CodeSerialize {
|
||||
class CodeObjectSerializeService;
|
||||
}
|
||||
|
||||
namespace CPU {
|
||||
class Arm64JITCore;
|
||||
class Dispatcher;
|
||||
} // namespace CPU
|
||||
|
||||
namespace HLE {
|
||||
class SourcecodeResolver;
|
||||
struct SyscallArguments;
|
||||
class SyscallHandler;
|
||||
class SourcecodeResolver;
|
||||
struct SourcecodeMap;
|
||||
} // namespace HLE
|
||||
} // namespace FEXCore
|
||||
|
||||
namespace FEXCore::IR {
|
||||
struct IRListCopy;
|
||||
class IRListView;
|
||||
namespace Validation {
|
||||
class IRValidation;
|
||||
}
|
||||
} // namespace FEXCore::IR
|
||||
|
||||
namespace FEXCore::Context {
|
||||
struct FEX_PACKED ExitFunctionLinkData {
|
||||
uint64_t HostCode;
|
||||
uint64_t HostBranch;
|
||||
uint64_t GuestRIP;
|
||||
int64_t CallerOffset;
|
||||
};
|
||||
|
||||
struct CustomIRResult {
|
||||
@@ -61,66 +72,10 @@ struct CustomIRResult {
|
||||
, Data(Data) {}
|
||||
};
|
||||
|
||||
using BlockDelinkerFunc = void (*)(FEXCore::Context::ExitFunctionLinkData* Record);
|
||||
using BlockDelinkerFunc = void (*)(FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record);
|
||||
constexpr uint32_t TSC_SCALE_MAXIMUM = 1'000'000'000; ///< 1Ghz
|
||||
|
||||
class CodeCache : public AbstractCodeCache {
|
||||
public:
|
||||
CodeCache(ContextImpl&);
|
||||
~CodeCache();
|
||||
|
||||
ContextImpl& CTX;
|
||||
fextl::unique_ptr<ContextImpl> ValidationCTX;
|
||||
fextl::unique_ptr<Core::InternalThreadState> ValidationThread;
|
||||
FEXCore::Core::CPUState::gdt_segment ValidationGDT[32] {};
|
||||
bool IsGeneratingCache = false;
|
||||
|
||||
FEX_CONFIG_OPT(EnableCodeCaching, ENABLECODECACHINGWIP);
|
||||
FEX_CONFIG_OPT(EnableCodeCacheValidation, ENABLECODECACHEVALIDATION);
|
||||
|
||||
uint64_t ComputeCodeMapId(std::string_view Filename, int FD) override;
|
||||
bool SaveData(Core::InternalThreadState&, int TargetFD, const ExecutableFileSectionInfo&, uint64_t SerializedBaseAddress) override;
|
||||
bool LoadData(Core::InternalThreadState*, std::byte* MappedCacheFile, const ExecutableFileSectionInfo&) override;
|
||||
|
||||
/**
|
||||
* Performs expensive extra validation on the loaded code cache data.
|
||||
*
|
||||
* This kicks off an in-process recompile of all cached blocks and compares
|
||||
* them with the cached data. Differences will be reported as fatal errors,
|
||||
* which can uncover bugs like for example:
|
||||
* - mismatches of the JIT configuration used during cache generation
|
||||
* - hidden position dependencies due to missing FEX relocations
|
||||
* - incorrect instruction padding
|
||||
*/
|
||||
void Validate(const ExecutableFileSectionInfo&, fextl::set<uint64_t> GuestBlocks, const fextl::set<uint64_t>& HostBlocks,
|
||||
std::span<std::byte> CachedCode);
|
||||
|
||||
void InitiateCacheGeneration() override {
|
||||
IsGeneratingCache = true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies a set of FEX relocations to the given code section.
|
||||
*
|
||||
* FEX relocations describe runtime-dependencies of FEX-generated code.
|
||||
* When loading a code cache, they are used to move cached code to the
|
||||
* dynamically chosen base address of the guest binary.
|
||||
*
|
||||
* Conversely, relocations are applied in reverse when writing code caches
|
||||
* to ensure consistency across generation runs.
|
||||
*
|
||||
* Note that FEX relocations are unrelated to ELF/PE relocations.
|
||||
*
|
||||
* @param GuestDelta Guest address offset to apply to RIP-relative data
|
||||
* @param ForStorage True for serializing data (producing deterministic output); false for de-serializing it (resolving dynamic symbols)
|
||||
*
|
||||
* @return Returns true on success
|
||||
*/
|
||||
[[nodiscard]]
|
||||
bool ApplyCodeRelocations(uint64_t GuestDelta, std::span<std::byte> Code, std::span<const CPU::Relocation> Relocations, bool ForStorage);
|
||||
};
|
||||
|
||||
class ContextImpl final : public FEXCore::Context::Context, public CPU::CodeBufferManager {
|
||||
class ContextImpl final : public FEXCore::Context::Context, CPU::CodeBufferManager {
|
||||
public:
|
||||
// Context base class implementation.
|
||||
bool InitCore() override;
|
||||
@@ -134,7 +89,6 @@ public:
|
||||
|
||||
bool IsAddressInCurrentBlock(FEXCore::Core::InternalThreadState* Thread, uint64_t Address, uint64_t Size) override;
|
||||
bool IsCurrentBlockSingleInst(FEXCore::Core::InternalThreadState* Thread) override;
|
||||
uint64_t GetGuestBlockEntry(FEXCore::Core::InternalThreadState* Thread) override;
|
||||
|
||||
uint64_t RestoreRIPFromHostPC(FEXCore::Core::InternalThreadState* Thread, uint64_t HostPC) override;
|
||||
uint32_t ReconstructCompactedEFLAGS(FEXCore::Core::InternalThreadState* Thread, bool WasInJIT, const uint64_t* HostGPRs, uint64_t PSTATE) override;
|
||||
@@ -190,28 +144,35 @@ public:
|
||||
FEXCore::CPUID::XCRResults RunXCRFunction(uint32_t Function) override;
|
||||
FEXCore::CPUID::FunctionResults RunCPUIDFunctionName(uint32_t Function, uint32_t Leaf, uint32_t CPU) override;
|
||||
|
||||
CodeCache& GetCodeCache() override {
|
||||
return CodeCache;
|
||||
FEXCore::IR::AOTIRCacheEntry* LoadAOTIRCacheEntry(const fextl::string& Name) override;
|
||||
void UnloadAOTIRCacheEntry(FEXCore::IR::AOTIRCacheEntry* Entry) override;
|
||||
|
||||
void SetAOTIRLoader(AOTIRLoaderCBFn CacheReader) override {
|
||||
IRCaptureCache.SetAOTIRLoader(std::move(CacheReader));
|
||||
}
|
||||
void SetAOTIRWriter(AOTIRWriterCBFn CacheWriter) override {
|
||||
IRCaptureCache.SetAOTIRWriter(std::move(CacheWriter));
|
||||
}
|
||||
void SetAOTIRRenamer(AOTIRRenamerCBFn CacheRenamer) override {
|
||||
IRCaptureCache.SetAOTIRRenamer(std::move(CacheRenamer));
|
||||
}
|
||||
|
||||
void SetCodeMapWriter(fextl::unique_ptr<CodeMapWriter> Writer) override {
|
||||
CodeMapWriter = std::move(Writer);
|
||||
void FinalizeAOTIRCache() override {
|
||||
IRCaptureCache.FinalizeAOTIRCache();
|
||||
}
|
||||
void WriteFilesWithCode(AOTIRCodeFileWriterFn Writer) override {
|
||||
IRCaptureCache.WriteFilesWithCode(Writer);
|
||||
}
|
||||
|
||||
void FlushAndCloseCodeMap() override {
|
||||
if (CodeMapWriter) {
|
||||
CodeMapWriter.reset();
|
||||
}
|
||||
}
|
||||
|
||||
void OnCodeBufferAllocated(const std::shared_ptr<CPU::CodeBuffer>&) override;
|
||||
void OnCodeBufferAllocated(CPU::CodeBuffer&) override;
|
||||
void ClearCodeCache(FEXCore::Core::InternalThreadState* Thread, bool NewCodeBuffer = true) override;
|
||||
void InvalidateCodeBuffersCodeRange(uint64_t Start, uint64_t Length) override;
|
||||
void InvalidateThreadCachedCodeRange(FEXCore::Core::InternalThreadState* Thread, uint64_t Start, uint64_t Length) override;
|
||||
void InvalidateGuestCodeRange(FEXCore::Core::InternalThreadState* Thread, uint64_t Start, uint64_t Length) override;
|
||||
FEXCore::ForkableSharedMutex& GetCodeInvalidationMutex() override {
|
||||
return CodeInvalidationMutex;
|
||||
}
|
||||
|
||||
void MarkMemoryShared(FEXCore::Core::InternalThreadState* Thread) override;
|
||||
|
||||
void ConfigureAOTGen(FEXCore::Core::InternalThreadState* Thread, fextl::set<uint64_t>* ExternalBranches, uint64_t SectionMaxAddress) override;
|
||||
|
||||
bool IsAddressInCodeBuffer(FEXCore::Core::InternalThreadState* Thread, uintptr_t Address) const override;
|
||||
@@ -226,13 +187,14 @@ public:
|
||||
|
||||
void RemoveForceTSOInformation(uint64_t Address, uint64_t Size) override;
|
||||
|
||||
void MarkMonoDetected() override {
|
||||
MonoDetected = true;
|
||||
}
|
||||
|
||||
void MarkMonoBackpatcherBlock(uint64_t BlockEntry) override;
|
||||
|
||||
public:
|
||||
friend class FEXCore::HLE::SyscallHandler;
|
||||
#ifdef JIT_ARM64
|
||||
friend class FEXCore::CPU::Arm64JITCore;
|
||||
#endif
|
||||
|
||||
friend class FEXCore::IR::Validation::IRValidation;
|
||||
|
||||
struct {
|
||||
uint64_t VirtualMemSize {1ULL << 36};
|
||||
uint64_t TSCScale = 0;
|
||||
@@ -245,8 +207,13 @@ public:
|
||||
FEX_CONFIG_OPT(GdbServer, GDBSERVER);
|
||||
FEX_CONFIG_OPT(Is64BitMode, IS64BIT_MODE);
|
||||
FEX_CONFIG_OPT(TSOEnabled, TSOENABLED);
|
||||
FEX_CONFIG_OPT(TSOAutoMigration, TSOAUTOMIGRATION);
|
||||
FEX_CONFIG_OPT(VectorTSOEnabled, VECTORTSOENABLED);
|
||||
FEX_CONFIG_OPT(MemcpySetTSOEnabled, MEMCPYSETTSOENABLED);
|
||||
FEX_CONFIG_OPT(ABILocalFlags, ABILOCALFLAGS);
|
||||
FEX_CONFIG_OPT(AOTIRCapture, AOTIRCAPTURE);
|
||||
FEX_CONFIG_OPT(AOTIRGenerate, AOTIRGENERATE);
|
||||
FEX_CONFIG_OPT(AOTIRLoad, AOTIRLOAD);
|
||||
FEX_CONFIG_OPT(SMCChecks, SMCCHECKS);
|
||||
FEX_CONFIG_OPT(MaxInstPerBlock, MAXINST);
|
||||
FEX_CONFIG_OPT(RootFSPath, ROOTFS);
|
||||
@@ -254,12 +221,13 @@ public:
|
||||
FEX_CONFIG_OPT(LibraryJITNaming, LIBRARYJITNAMING);
|
||||
FEX_CONFIG_OPT(BlockJITNaming, BLOCKJITNAMING);
|
||||
FEX_CONFIG_OPT(GDBSymbols, GDBSYMBOLS);
|
||||
FEX_CONFIG_OPT(ParanoidTSO, PARANOIDTSO);
|
||||
FEX_CONFIG_OPT(CacheObjectCodeCompilation, CACHEOBJECTCODECOMPILATION);
|
||||
FEX_CONFIG_OPT(x87ReducedPrecision, X87REDUCEDPRECISION);
|
||||
FEX_CONFIG_OPT(DisableTelemetry, DISABLETELEMETRY);
|
||||
FEX_CONFIG_OPT(DisableVixlIndirectCalls, DISABLE_VIXL_INDIRECT_RUNTIME_CALLS);
|
||||
FEX_CONFIG_OPT(SmallTSCScale, SMALLTSCSCALE);
|
||||
FEX_CONFIG_OPT(StrictInProcessSplitLocks, STRICTINPROCESSSPLITLOCKS);
|
||||
FEX_CONFIG_OPT(MonoHacks, MONOHACKS);
|
||||
} Config;
|
||||
|
||||
FEXCore::ForkableSharedMutex CodeInvalidationMutex;
|
||||
@@ -273,21 +241,37 @@ public:
|
||||
FEXCore::HLE::SourcecodeResolver* SourcecodeResolver {};
|
||||
FEXCore::ThunkHandler* ThunkHandler {};
|
||||
fextl::unique_ptr<FEXCore::CPU::Dispatcher> Dispatcher;
|
||||
CodeCache CodeCache;
|
||||
fextl::unique_ptr<CodeMapWriter> CodeMapWriter;
|
||||
|
||||
SignalDelegator* SignalDelegation {};
|
||||
X86GeneratedCode X86CodeGen;
|
||||
|
||||
ContextImpl(const FEXCore::HostFeatures& Features);
|
||||
~ContextImpl();
|
||||
|
||||
static void ThreadRemoveCodeEntryFromJit(FEXCore::Core::CpuStateFrame* Frame, uint64_t GuestRIP);
|
||||
static void ThreadRemoveCodeEntry(FEXCore::Core::InternalThreadState* Thread, uint64_t GuestRIP);
|
||||
|
||||
// This is used as a replacement for the SMC writes in the mono callsite backpatcher that avoids atomic operations
|
||||
// (safe as the invalidation mutex is locked) and manually invalidates the modified range. Allowing SMC to be detected
|
||||
// even if faulting is disabled.
|
||||
static void MonoBackpatcherWrite(FEXCore::Core::CpuStateFrame* Frame, uint8_t Size, uint64_t Address, uint64_t Value);
|
||||
template<auto Fn>
|
||||
static uint64_t ThreadExitFunctionLink(FEXCore::Core::CpuStateFrame* Frame, ExitFunctionLinkData* Record) {
|
||||
auto Thread = Frame->Thread;
|
||||
auto lk = GuardSignalDeferringSection<std::shared_lock>(static_cast<ContextImpl*>(Thread->CTX)->CodeInvalidationMutex, Thread);
|
||||
|
||||
void RemoveCustomIREntrypoint(FEXCore::Core::InternalThreadState* Thread, uintptr_t Entrypoint);
|
||||
return Fn(Frame, Record);
|
||||
}
|
||||
|
||||
// Wrapper which takes CpuStateFrame instead of InternalThreadState and unique_locks CodeInvalidationMutex
|
||||
// Must be called from owning thread
|
||||
static void ThreadRemoveCodeEntryFromJit(FEXCore::Core::CpuStateFrame* Frame, uint64_t GuestRIP) {
|
||||
auto Thread = Frame->Thread;
|
||||
auto lk = GuardSignalDeferringSection(static_cast<ContextImpl*>(Thread->CTX)->CodeInvalidationMutex, Thread);
|
||||
|
||||
// NOTE: Other threads sharing the same CodeBuffer may reference
|
||||
// invalidated data ranges through their L1/L2 caches. This is
|
||||
// not currently a problem since FEX does not repurpose the
|
||||
// invalidated CodeBuffer memory range currently.
|
||||
ThreadRemoveCodeEntry(Thread, GuestRIP);
|
||||
}
|
||||
|
||||
void RemoveCustomIREntrypoint(uintptr_t Entrypoint);
|
||||
|
||||
struct GenerateIRResult {
|
||||
std::optional<IR::IRListView> IRView;
|
||||
@@ -295,28 +279,30 @@ public:
|
||||
uint64_t TotalInstructionsLength;
|
||||
uint64_t StartAddr;
|
||||
uint64_t Length;
|
||||
bool NeedsAddGuestCodeRanges;
|
||||
};
|
||||
[[nodiscard]]
|
||||
GenerateIRResult GenerateIR(FEXCore::Core::InternalThreadState* Thread, uint64_t GuestRIP, bool ExtendedDebugInfo, uint64_t MaxInst);
|
||||
|
||||
struct CompileCodeResult {
|
||||
CPU::CPUBackend::CompiledCode CompiledCode;
|
||||
void* CompiledCode;
|
||||
fextl::unique_ptr<FEXCore::Core::DebugData> DebugData;
|
||||
uint64_t StartAddr;
|
||||
uint64_t Length;
|
||||
bool NeedsAddGuestCodeRanges;
|
||||
};
|
||||
[[nodiscard]]
|
||||
CompileCodeResult CompileCode(FEXCore::Core::InternalThreadState* Thread, uint64_t GuestRIP, uint64_t MaxInst = 0);
|
||||
uintptr_t CompileBlock(FEXCore::Core::CpuStateFrame* Frame, uint64_t GuestRIP, uint64_t MaxInst = 0);
|
||||
uintptr_t CompileSingleStep(FEXCore::Core::CpuStateFrame* Frame, uint64_t GuestRIP);
|
||||
|
||||
IR::OpSize GetGPROpSize() const {
|
||||
return Config.Is64BitMode ? IR::OpSize::i64Bit : IR::OpSize::i32Bit;
|
||||
}
|
||||
|
||||
FEXCore::JITSymbols Symbols;
|
||||
|
||||
FEXCore::Utils::PooledAllocatorVirtual OpDispatcherAllocator {"FEXMem_OpDispatcher"};
|
||||
FEXCore::Utils::PooledAllocatorVirtual FrontendAllocator {"FEXMem_Frontend"};
|
||||
FEXCore::Utils::PooledAllocatorVirtualWithGuard CPUBackendAllocator {"FEXMem_CPUBackend"};
|
||||
FEXCore::Utils::PooledAllocatorVirtual OpDispatcherAllocator;
|
||||
FEXCore::Utils::PooledAllocatorVirtual FrontendAllocator;
|
||||
FEXCore::Utils::PooledAllocatorVirtual CPUBackendAllocator;
|
||||
|
||||
// If Atomic-based TSO emulation is enabled or not.
|
||||
bool IsAtomicTSOEnabled() const {
|
||||
@@ -346,10 +332,6 @@ public:
|
||||
return ExitOnHLT;
|
||||
}
|
||||
|
||||
bool AreMonoHacksActive() const {
|
||||
return Config.MonoHacks && MonoDetected;
|
||||
}
|
||||
|
||||
protected:
|
||||
void UpdateAtomicTSOEmulationConfig() {
|
||||
if (SupportsHardwareTSO) {
|
||||
@@ -357,10 +339,17 @@ protected:
|
||||
AtomicTSOEmulationEnabled = false;
|
||||
VectorAtomicTSOEmulationEnabled = false;
|
||||
MemcpyAtomicTSOEmulationEnabled = false;
|
||||
} else if (Config.ParanoidTSO) {
|
||||
AtomicTSOEmulationEnabled = true;
|
||||
VectorAtomicTSOEmulationEnabled = true;
|
||||
MemcpyAtomicTSOEmulationEnabled = true;
|
||||
} else {
|
||||
AtomicTSOEmulationEnabled = Config.TSOEnabled;
|
||||
VectorAtomicTSOEmulationEnabled = Config.TSOEnabled && Config.VectorTSOEnabled;
|
||||
MemcpyAtomicTSOEmulationEnabled = Config.TSOEnabled && Config.MemcpySetTSOEnabled;
|
||||
// Atomic TSO emulation only enabled if the config option is enabled.
|
||||
AtomicTSOEmulationEnabled = (IsMemoryShared || !Config.TSOAutoMigration) && Config.TSOEnabled;
|
||||
// Atomic vector TSO emulation only enabled if TSO emulation is enabled and also vector TSO is enabled.
|
||||
VectorAtomicTSOEmulationEnabled = (IsMemoryShared || !Config.TSOAutoMigration) && Config.TSOEnabled && Config.VectorTSOEnabled;
|
||||
// Atomic memcpy TSO emulation only enabled if TSO emulation is enabled and also memcpy TSO is enabled.
|
||||
MemcpyAtomicTSOEmulationEnabled = (IsMemoryShared || !Config.TSOAutoMigration) && Config.TSOEnabled && Config.MemcpySetTSOEnabled;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -374,6 +363,10 @@ private:
|
||||
*/
|
||||
void InitializeCompiler(FEXCore::Core::InternalThreadState* Thread);
|
||||
|
||||
IR::AOTIRCaptureCache IRCaptureCache;
|
||||
fextl::unique_ptr<FEXCore::CodeSerialize::CodeObjectSerializeService> CodeObjectCacheService;
|
||||
|
||||
bool IsMemoryShared = false;
|
||||
bool SupportsHardwareTSO = false;
|
||||
bool AtomicTSOEmulationEnabled = true;
|
||||
bool VectorAtomicTSOEmulationEnabled = false;
|
||||
@@ -384,19 +377,8 @@ private:
|
||||
|
||||
std::shared_mutex CustomIRMutex;
|
||||
std::atomic<bool> HasCustomIRHandlers {};
|
||||
struct CustomIRHandlerEntry final {
|
||||
CustomIREntrypointHandler Handler;
|
||||
void* Creator;
|
||||
void* Data;
|
||||
};
|
||||
fextl::unordered_map<uint64_t, CustomIRHandlerEntry> CustomIRHandlers;
|
||||
fextl::unordered_map<uint64_t, std::tuple<CustomIREntrypointHandler, void*, void*>> CustomIRHandlers;
|
||||
IntervalList<uint64_t> ForceTSOValidRanges; // The ranges for which ForceTSOInstructions has populated data
|
||||
fextl::set<uint64_t> ForceTSOInstructions;
|
||||
|
||||
bool MonoDetected = false;
|
||||
std::atomic<uint64_t> MonoBackpatcherBlock;
|
||||
|
||||
std::mutex CodeBufferListLock;
|
||||
fextl::vector<std::weak_ptr<CPU::CodeBuffer>> CodeBufferList;
|
||||
};
|
||||
} // namespace FEXCore::Context
|
||||
@@ -7,11 +7,12 @@
|
||||
|
||||
namespace FEXCore::IR {
|
||||
|
||||
Ref LoadEffectiveAddress(IREmitter* IREmit, const AddressMode& A, IR::OpSize GPRSize, bool AddSegmentBase, bool AllowUpperGarbage) {
|
||||
Ref LoadEffectiveAddress(IREmitter* IREmit, AddressMode A, IR::OpSize GPRSize, bool AddSegmentBase, bool AllowUpperGarbage) {
|
||||
Ref Tmp = A.Base;
|
||||
|
||||
if (A.Offset) {
|
||||
Tmp = Tmp ? IREmit->Add(GPRSize, Tmp, A.Offset) : IREmit->Constant(A.Offset);
|
||||
Ref Offset = IREmit->_Constant(A.Offset);
|
||||
Tmp = Tmp ? IREmit->_Add(GPRSize, Tmp, Offset) : Offset;
|
||||
}
|
||||
|
||||
if (A.Index) {
|
||||
@@ -21,10 +22,10 @@ Ref LoadEffectiveAddress(IREmitter* IREmit, const AddressMode& A, IR::OpSize GPR
|
||||
if (Tmp) {
|
||||
Tmp = IREmit->_AddShift(GPRSize, Tmp, A.Index, ShiftType::LSL, Log2);
|
||||
} else {
|
||||
Tmp = IREmit->_Lshl(GPRSize, A.Index, IREmit->Constant(Log2));
|
||||
Tmp = IREmit->_Lshl(GPRSize, A.Index, IREmit->_Constant(Log2));
|
||||
}
|
||||
} else {
|
||||
Tmp = Tmp ? IREmit->Add(GPRSize, Tmp, A.Index) : A.Index;
|
||||
Tmp = Tmp ? IREmit->_Add(GPRSize, Tmp, A.Index) : A.Index;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -40,28 +41,32 @@ Ref LoadEffectiveAddress(IREmitter* IREmit, const AddressMode& A, IR::OpSize GPR
|
||||
} else if (A.Offset) {
|
||||
uint64_t X = A.Offset;
|
||||
X &= (1ull << Bits) - 1;
|
||||
Tmp = IREmit->Constant(X);
|
||||
Tmp = IREmit->_Constant(X);
|
||||
}
|
||||
}
|
||||
|
||||
if (A.Segment && AddSegmentBase) {
|
||||
Tmp = Tmp ? IREmit->Add(GPRSize, Tmp, A.Segment) : A.Segment;
|
||||
Tmp = Tmp ? IREmit->_Add(GPRSize, Tmp, A.Segment) : A.Segment;
|
||||
}
|
||||
|
||||
return Tmp ?: IREmit->Constant(0);
|
||||
return Tmp ?: IREmit->_Constant(0);
|
||||
}
|
||||
|
||||
AddressMode SelectAddressMode(IREmitter* IREmit, const AddressMode& A, IR::OpSize GPRSize, bool HostSupportsTSOImm9, bool AtomicTSO,
|
||||
bool Vector, IR::OpSize AccessSize) {
|
||||
AddressMode SelectAddressMode(IREmitter* IREmit, AddressMode A, IR::OpSize GPRSize, bool HostSupportsTSOImm9, bool AtomicTSO, bool Vector,
|
||||
IR::OpSize AccessSize) {
|
||||
auto SoftwareAddressCalculation = [IREmit, &A, GPRSize]() -> AddressMode {
|
||||
return {
|
||||
.Base = LoadEffectiveAddress(IREmit, A, GPRSize, true),
|
||||
.Index = IREmit->Invalid(),
|
||||
};
|
||||
};
|
||||
|
||||
const auto Is32Bit = GPRSize == OpSize::i32Bit;
|
||||
const auto GPRSizeMatchesAddrSize = A.AddrSize == GPRSize;
|
||||
const auto OffsetIndexToLargeFor32Bit = Is32Bit && (A.Offset <= -16384 || A.Offset >= 16384);
|
||||
if (!GPRSizeMatchesAddrSize || OffsetIndexToLargeFor32Bit) {
|
||||
// If address size doesn't match GPR size then no optimizations can occur.
|
||||
return {
|
||||
.Base = LoadEffectiveAddress(IREmit, A, GPRSize, true),
|
||||
.Index = IREmit->Invalid(),
|
||||
};
|
||||
return SoftwareAddressCalculation();
|
||||
}
|
||||
|
||||
// Loadstore rules:
|
||||
@@ -95,33 +100,43 @@ AddressMode SelectAddressMode(IREmitter* IREmit, const AddressMode& A, IR::OpSiz
|
||||
const bool OffsetIsSIMM9 = A.Offset && A.Offset >= -256 && A.Offset <= 255;
|
||||
const bool OffsetIsUnsignedScaled = A.Offset > 0 && (A.Offset & (AccessSizeAsImm - 1)) == 0 && (A.Offset / AccessSizeAsImm) <= 4095;
|
||||
|
||||
if ((AtomicTSO && !Vector && HostSupportsTSOImm9 && OffsetIsSIMM9) || (!AtomicTSO && (OffsetIsSIMM9 || OffsetIsUnsignedScaled))) {
|
||||
auto InlineImmOffsetLoadstore = [IREmit, &GPRSize](AddressMode A) -> AddressMode {
|
||||
// Peel off the offset
|
||||
AddressMode B = A;
|
||||
B.Offset = 0;
|
||||
|
||||
return {
|
||||
.Base = LoadEffectiveAddress(IREmit, B, GPRSize, true /* AddSegmentBase */, false),
|
||||
.Index = IREmit->Constant(A.Offset),
|
||||
.IndexType = MemOffsetType::SXTX,
|
||||
.Index = IREmit->_Constant(A.Offset),
|
||||
.IndexType = MEM_OFFSET_SXTX,
|
||||
.IndexScale = 1,
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
auto ScaledRegisterLoadstore = [IREmit, GPRSize](AddressMode A) -> AddressMode {
|
||||
if (A.Index && A.Segment) {
|
||||
A.Base = IREmit->_Add(GPRSize, A.Base, A.Segment);
|
||||
} else if (A.Segment) {
|
||||
A.Index = A.Segment;
|
||||
A.IndexScale = 1;
|
||||
}
|
||||
return A;
|
||||
};
|
||||
|
||||
if (AtomicTSO) {
|
||||
// TODO: LRCPC3 support for vector Imm9.
|
||||
} else if (!Is32Bit && A.Base && (A.Index || A.Segment) && !A.Offset && (A.IndexScale == 1 || A.IndexScale == AccessSizeAsImm)) {
|
||||
AddressMode B = A;
|
||||
|
||||
// ScaledRegisterLoadstore
|
||||
if (B.Index && B.Segment) {
|
||||
B.Base = IREmit->Add(GPRSize, B.Base, B.Segment);
|
||||
} else if (B.Segment) {
|
||||
B.Index = B.Segment;
|
||||
B.IndexScale = 1;
|
||||
if (!Vector) {
|
||||
if (HostSupportsTSOImm9 && OffsetIsSIMM9) {
|
||||
return InlineImmOffsetLoadstore(A);
|
||||
}
|
||||
} else {
|
||||
// TODO: LRCPC3 support for vector Imm9.
|
||||
}
|
||||
} else {
|
||||
if (OffsetIsSIMM9 || OffsetIsUnsignedScaled) {
|
||||
return InlineImmOffsetLoadstore(A);
|
||||
} else if (!Is32Bit && A.Base && (A.Index || A.Segment) && !A.Offset && (A.IndexScale == 1 || A.IndexScale == AccessSizeAsImm)) {
|
||||
return ScaledRegisterLoadstore(A);
|
||||
}
|
||||
|
||||
return B;
|
||||
}
|
||||
|
||||
if (Vector || !AtomicTSO) {
|
||||
@@ -135,8 +150,8 @@ AddressMode SelectAddressMode(IREmitter* IREmit, const AddressMode& A, IR::OpSiz
|
||||
|
||||
return {
|
||||
.Base = LoadEffectiveAddress(IREmit, B, GPRSize, true /* AddSegmentBase */, false),
|
||||
.Index = IREmit->Constant(A.Offset),
|
||||
.IndexType = MemOffsetType::SXTX,
|
||||
.Index = IREmit->_Constant(A.Offset),
|
||||
.IndexType = MEM_OFFSET_SXTX,
|
||||
.IndexScale = 1,
|
||||
};
|
||||
}
|
||||
@@ -144,10 +159,7 @@ AddressMode SelectAddressMode(IREmitter* IREmit, const AddressMode& A, IR::OpSiz
|
||||
}
|
||||
|
||||
// Fallback on software address calculation
|
||||
return {
|
||||
.Base = LoadEffectiveAddress(IREmit, A, GPRSize, true),
|
||||
.Index = IREmit->Invalid(),
|
||||
};
|
||||
return SoftwareAddressCalculation();
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -11,18 +11,17 @@ struct AddressMode {
|
||||
Ref Segment {nullptr};
|
||||
Ref Base {nullptr};
|
||||
Ref Index {nullptr};
|
||||
int64_t Offset = 0;
|
||||
|
||||
MemOffsetType IndexType = MemOffsetType::SXTX;
|
||||
MemOffsetType IndexType = MEM_OFFSET_SXTX;
|
||||
uint8_t IndexScale = 1;
|
||||
int64_t Offset = 0;
|
||||
|
||||
// Size in bytes for the address calculation. 8 for an arm64 hardware mode.
|
||||
IR::OpSize AddrSize;
|
||||
bool NonTSO;
|
||||
};
|
||||
|
||||
Ref LoadEffectiveAddress(IREmitter* IREmit, const AddressMode& A, IR::OpSize GPRSize, bool AddSegmentBase, bool AllowUpperGarbage = false);
|
||||
AddressMode SelectAddressMode(IREmitter* IREmit, const AddressMode& A, IR::OpSize GPRSize, bool HostSupportsTSOImm9, bool AtomicTSO,
|
||||
bool Vector, IR::OpSize AccessSize);
|
||||
Ref LoadEffectiveAddress(IREmitter* IREmit, AddressMode A, IR::OpSize GPRSize, bool AddSegmentBase, bool AllowUpperGarbage = false);
|
||||
AddressMode SelectAddressMode(IREmitter* IREmit, AddressMode A, IR::OpSize GPRSize, bool HostSupportsTSOImm9, bool AtomicTSO, bool Vector,
|
||||
IR::OpSize AccessSize);
|
||||
|
||||
} // namespace FEXCore::IR
|
||||
}; // namespace FEXCore::IR
|
||||
@@ -1,10 +1,10 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
#include "Interface/Core/ArchHelpers/Arm64Emitter.h"
|
||||
#include "FEXCore/Core/X86Enums.h"
|
||||
#include "Interface/Core/Dispatcher/Dispatcher.h"
|
||||
#include "Interface/Context/Context.h"
|
||||
|
||||
#include <FEXCore/Core/CoreState.h>
|
||||
#include <FEXCore/Core/X86Enums.h>
|
||||
#include <FEXCore/Utils/LogManager.h>
|
||||
#include <FEXCore/Utils/MathUtils.h>
|
||||
|
||||
@@ -41,7 +41,7 @@ namespace FEXCore::CPU {
|
||||
// r19-r29 and SP.
|
||||
|
||||
namespace x64 {
|
||||
#ifndef ARCHITECTURE_arm64ec
|
||||
#ifndef _M_ARM_64EC
|
||||
// All but x19 and x29 are caller saved
|
||||
// Note that rax/rdx are rearranged here so we can coalesce cmpxchg.
|
||||
constexpr std::array<ARMEmitter::Register, 18> SRA = {
|
||||
@@ -73,13 +73,13 @@ namespace x64 {
|
||||
ARMEmitter::Reg::r8, ARMEmitter::Reg::r16, ARMEmitter::Reg::r17,
|
||||
};
|
||||
|
||||
constexpr std::array<ARMEmitter::Register, 7> RA = {
|
||||
constexpr std::array<ARMEmitter::Register, 8> RA = {
|
||||
// All these callee saved
|
||||
ARMEmitter::Reg::r20, ARMEmitter::Reg::r21, ARMEmitter::Reg::r22, ARMEmitter::Reg::r23,
|
||||
ARMEmitter::Reg::r24, ARMEmitter::Reg::r30, ARMEmitter::Reg::r18,
|
||||
ARMEmitter::Reg::r24, ARMEmitter::Reg::r25, ARMEmitter::Reg::r30, ARMEmitter::Reg::r18,
|
||||
};
|
||||
|
||||
constexpr unsigned RAPairs = 4;
|
||||
constexpr unsigned RAPairs = 6;
|
||||
|
||||
// Dynamic GPRs
|
||||
constexpr std::array<ARMEmitter::Register, 2> PreserveAll_Dynamic = {
|
||||
@@ -143,18 +143,18 @@ namespace x64 {
|
||||
ARMEmitter::Reg::r4, ARMEmitter::Reg::r5, ARMEmitter::Reg::r8,
|
||||
};
|
||||
|
||||
constexpr std::array<ARMEmitter::Register, 6> RA = {
|
||||
ARMEmitter::Reg::r6, ARMEmitter::Reg::r7, ARMEmitter::Reg::r14, ARMEmitter::Reg::r15, ARMEmitter::Reg::r16, ARMEmitter::Reg::r30,
|
||||
constexpr std::array<ARMEmitter::Register, 7> RA = {
|
||||
ARMEmitter::Reg::r6, ARMEmitter::Reg::r7, ARMEmitter::Reg::r14, ARMEmitter::Reg::r15,
|
||||
ARMEmitter::Reg::r16, ARMEmitter::Reg::r17, ARMEmitter::Reg::r30,
|
||||
};
|
||||
|
||||
constexpr std::array<ARMEmitter::Register, 5> PreserveAll_Dynamic = {ARMEmitter::Reg::r6, ARMEmitter::Reg::r7, ARMEmitter::Reg::r16,
|
||||
ARMEmitter::Reg::r17, ARMEmitter::Reg::r30};
|
||||
constexpr std::array<ARMEmitter::Register, 5> PreserveAll_Dynamic = {
|
||||
ARMEmitter::Reg::r6, ARMEmitter::Reg::r7, ARMEmitter::Reg::r16, ARMEmitter::Reg::r17, ARMEmitter::Reg::r30,
|
||||
};
|
||||
|
||||
constexpr std::array<ARMEmitter::Register, 7> NotPreserved_Dynamic = {ARMEmitter::Reg::r6, ARMEmitter::Reg::r7, ARMEmitter::Reg::r14,
|
||||
ARMEmitter::Reg::r15, ARMEmitter::Reg::r16, ARMEmitter::Reg::r17,
|
||||
ARMEmitter::Reg::r30};
|
||||
constexpr std::array<ARMEmitter::Register, 7> NotPreserved_Dynamic = RA;
|
||||
|
||||
constexpr unsigned RAPairs = 4;
|
||||
constexpr unsigned RAPairs = 6;
|
||||
|
||||
constexpr std::array<ARMEmitter::VRegister, 16> SRAFPR = {
|
||||
ARMEmitter::VReg::v0, ARMEmitter::VReg::v1, ARMEmitter::VReg::v2, ARMEmitter::VReg::v3,
|
||||
@@ -245,12 +245,14 @@ namespace x32 {
|
||||
REG_AF,
|
||||
};
|
||||
|
||||
constexpr std::array<ARMEmitter::Register, 14> RA = {
|
||||
constexpr std::array<ARMEmitter::Register, 15> RA = {
|
||||
// All these callee saved
|
||||
ARMEmitter::Reg::r20,
|
||||
ARMEmitter::Reg::r21,
|
||||
ARMEmitter::Reg::r22,
|
||||
ARMEmitter::Reg::r23,
|
||||
ARMEmitter::Reg::r24,
|
||||
ARMEmitter::Reg::r25,
|
||||
|
||||
// Registers only available on 32-bit
|
||||
// All these are caller saved (except for r19).
|
||||
@@ -263,7 +265,6 @@ namespace x32 {
|
||||
ARMEmitter::Reg::r29,
|
||||
ARMEmitter::Reg::r30,
|
||||
|
||||
ARMEmitter::Reg::r24,
|
||||
ARMEmitter::Reg::r19,
|
||||
};
|
||||
|
||||
@@ -272,7 +273,7 @@ namespace x32 {
|
||||
ARMEmitter::Reg::r16, ARMEmitter::Reg::r17, ARMEmitter::Reg::r30,
|
||||
};
|
||||
|
||||
constexpr unsigned RAPairs = 10;
|
||||
constexpr unsigned RAPairs = 12;
|
||||
|
||||
// All are caller saved
|
||||
constexpr std::array<ARMEmitter::VRegister, 8> SRAFPR = {
|
||||
@@ -369,8 +370,6 @@ Arm64Emitter::Arm64Emitter(FEXCore::Context::ContextImpl* ctx, void* EmissionPtr
|
||||
// Hardcode a 256-bit vector width if we are running in the simulator.
|
||||
// Allow the user to override this.
|
||||
Simulator.SetVectorLengthInBits(ForceSVEWidth() ? ForceSVEWidth() : 256);
|
||||
// FEX doesn't support GCS.
|
||||
Simulator.DisableGCSCheck();
|
||||
#endif
|
||||
#ifdef VIXL_DISASSEMBLER
|
||||
// Only setup the disassembler if enabled.
|
||||
@@ -417,34 +416,18 @@ FEXCore::X86State::X86Reg Arm64Emitter::GetX86RegRelationToARMReg(ARMEmitter::Re
|
||||
return FEXCore::X86State::X86Reg::REG_INVALID;
|
||||
}
|
||||
|
||||
void Arm64Emitter::LoadConstant(ARMEmitter::Size s, ARMEmitter::Register Reg, uint64_t Constant, PadType Pad, int MaxBytes) {
|
||||
bool NOPPad = false;
|
||||
if (Pad == PadType::DOPAD) {
|
||||
NOPPad = true;
|
||||
} else if (Pad == PadType::NOPAD) {
|
||||
NOPPad = false;
|
||||
} else if (Pad == PadType::AUTOPAD) {
|
||||
// Force NOP padding to ensure relocated constants always have enough encoding space available
|
||||
NOPPad = EnableCodeCaching;
|
||||
}
|
||||
|
||||
void Arm64Emitter::LoadConstant(ARMEmitter::Size s, ARMEmitter::Register Reg, uint64_t Constant, bool NOPPad) {
|
||||
bool Is64Bit = s == ARMEmitter::Size::i64Bit;
|
||||
const auto UpperBound = Is64Bit ? 4 : 2;
|
||||
int Segments = MaxBytes ? (MaxBytes / 2) : UpperBound;
|
||||
|
||||
LOGMAN_THROW_A_FMT(MaxBytes >= 0 && MaxBytes <= (UpperBound * 2) && (MaxBytes & 1) == 0,
|
||||
"MaxBytes must be bounded in the range of [0, {}] and 16-bit aligned", UpperBound);
|
||||
// If MaxBytes specified then make sure to sanity check incoming data.
|
||||
LOGMAN_THROW_A_FMT(MaxBytes == 0 || (Constant >> (MaxBytes * 8)) == 0, "MaxBytes provided but data can't fit within provided range.");
|
||||
int Segments = Is64Bit ? 4 : 2;
|
||||
|
||||
if (Is64Bit && ((~Constant) >> 16) == 0) {
|
||||
movn(s, Reg, (~Constant) & 0xFFFF);
|
||||
|
||||
if (NOPPad) {
|
||||
nop();
|
||||
nop();
|
||||
nop();
|
||||
}
|
||||
|
||||
movn(s, Reg, (~Constant) & 0xFFFF);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -452,17 +435,17 @@ void Arm64Emitter::LoadConstant(ARMEmitter::Size s, ARMEmitter::Register Reg, ui
|
||||
// If the upper 32-bits is all zero, we can now switch to a 32-bit move.
|
||||
s = ARMEmitter::Size::i32Bit;
|
||||
Is64Bit = false;
|
||||
Segments = std::min(Segments, 2);
|
||||
Segments = 2;
|
||||
}
|
||||
|
||||
if (!Is64Bit && ((~Constant) & 0xFFFF0000) == 0) {
|
||||
movn(s, Reg.W(), (~Constant) & 0xFFFF);
|
||||
|
||||
if (NOPPad) {
|
||||
nop();
|
||||
nop();
|
||||
nop();
|
||||
}
|
||||
|
||||
movn(s, Reg.W(), (~Constant) & 0xFFFF);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -483,24 +466,24 @@ void Arm64Emitter::LoadConstant(ARMEmitter::Size s, ARMEmitter::Register Reg, ui
|
||||
// `movz` is better than `orr` since hardware will rename or merge if possible when `movz` is used.
|
||||
const auto IsImm = ARMEmitter::Emitter::IsImmLogical(Constant, RegSizeInBits(s));
|
||||
if (IsImm) {
|
||||
orr(s, Reg, ARMEmitter::Reg::zr, Constant);
|
||||
if (NOPPad) {
|
||||
nop();
|
||||
nop();
|
||||
nop();
|
||||
}
|
||||
orr(s, Reg, ARMEmitter::Reg::zr, Constant);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// If we can't handle negatives with the orr, try with movn+movk
|
||||
if (Is64Bit && ((~Constant) >> 32) == 0) {
|
||||
movn(s, Reg, (~Constant) & 0xFFFF);
|
||||
movk(s, Reg, (Constant >> 16) & 0xFFFF, 16);
|
||||
if (NOPPad) {
|
||||
nop();
|
||||
nop();
|
||||
}
|
||||
movn(s, Reg, (~Constant) & 0xFFFF);
|
||||
movk(s, Reg, (Constant >> 16) & 0xFFFF, 16);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -512,7 +495,7 @@ void Arm64Emitter::LoadConstant(ARMEmitter::Size s, ARMEmitter::Register Reg, ui
|
||||
uint64_t AlignedPC = PC & ~0xFFFULL;
|
||||
|
||||
// Offset from aligned PC
|
||||
auto AlignedOffset = std::bit_cast<int64_t>(Constant - AlignedPC);
|
||||
int64_t AlignedOffset = static_cast<int64_t>(Constant) - static_cast<int64_t>(AlignedPC);
|
||||
|
||||
int NumMoves = 0;
|
||||
|
||||
@@ -528,7 +511,7 @@ void Arm64Emitter::LoadConstant(ARMEmitter::Size s, ARMEmitter::Register Reg, ui
|
||||
} else {
|
||||
// If the constant is within 1MB of PC then we can still use ADR to load in a single instruction
|
||||
// 21-bit signed integer here
|
||||
auto SmallOffset = std::bit_cast<int64_t>(Constant - PC);
|
||||
int64_t SmallOffset = static_cast<int64_t>(Constant) - static_cast<int64_t>(PC);
|
||||
if (ARMEmitter::Emitter::IsInt21(SmallOffset)) {
|
||||
adr(Reg, SmallOffset);
|
||||
} else {
|
||||
@@ -711,8 +694,6 @@ void Arm64Emitter::SpillStaticRegs(ARMEmitter::Register TmpReg, bool FPRs, uint3
|
||||
unsigned PFAFSpillMask = GPRSpillMask & PFAFMask;
|
||||
GPRSpillMask &= ~PFAFSpillMask;
|
||||
|
||||
str(REG_CALLRET_SP, STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.callret_sp));
|
||||
|
||||
for (size_t i = 0; i < StaticRegisters.size(); i += 2) {
|
||||
auto Reg1 = StaticRegisters[i];
|
||||
auto Reg2 = StaticRegisters[i + 1];
|
||||
@@ -728,7 +709,7 @@ void Arm64Emitter::SpillStaticRegs(ARMEmitter::Register TmpReg, bool FPRs, uint3
|
||||
// Now handle PF/AF
|
||||
if (PFAFSpillMask) {
|
||||
auto PFOffset = offsetof(FEXCore::Core::CpuStateFrame, State.pf_raw);
|
||||
auto AFOffset = offsetof(FEXCore::Core::CpuStateFrame, State.af_raw);
|
||||
[[maybe_unused]] auto AFOffset = offsetof(FEXCore::Core::CpuStateFrame, State.af_raw);
|
||||
LOGMAN_THROW_A_FMT(PFAFSpillMask == PFAFMask, "PF/AF not spilled together");
|
||||
LOGMAN_THROW_A_FMT(AFOffset == PFOffset + 4, "PF/AF are together");
|
||||
|
||||
@@ -802,14 +783,12 @@ void Arm64Emitter::FillStaticRegs(bool FPRs, uint32_t GPRFillMask, uint32_t FPRF
|
||||
auto TmpReg = *OptionalReg;
|
||||
auto TmpReg2 = *OptionalReg2;
|
||||
|
||||
#ifdef ARCHITECTURE_arm64ec
|
||||
#ifdef _M_ARM_64EC
|
||||
// Load STATE in from the CPU area as x28 is not callee saved in the ARM64EC ABI.
|
||||
ldr(TmpReg.X(), ARMEmitter::Reg::r18, TEB_CPU_AREA_OFFSET);
|
||||
ldr(STATE, TmpReg, CPU_AREA_EMULATOR_DATA_OFFSET);
|
||||
#endif
|
||||
|
||||
ldr(REG_CALLRET_SP, STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.callret_sp));
|
||||
|
||||
// Regardless of what GPRs/FPRs we're filling, we need to fill NZCV since it
|
||||
// is always static and was almost certainly clobbered.
|
||||
//
|
||||
|
||||
@@ -1,38 +1,36 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
#pragma once
|
||||
|
||||
#include <FEXCore/Config/Config.h>
|
||||
#include "FEXCore/Utils/EnumUtils.h"
|
||||
#include "Interface/Core/ObjectCache/Relocations.h"
|
||||
|
||||
#ifdef VIXL_DISASSEMBLER
|
||||
#include <aarch64/disasm-aarch64.h>
|
||||
#include <FEXCore/fextl/memory.h>
|
||||
#include <FEXCore/fextl/vector.h>
|
||||
#endif
|
||||
#ifdef VIXL_SIMULATOR
|
||||
#include <aarch64/simulator-aarch64.h>
|
||||
#include <aarch64/simulator-constants-aarch64.h>
|
||||
#endif
|
||||
|
||||
#include <FEXCore/Core/X86Enums.h>
|
||||
#include <FEXCore/Config/Config.h>
|
||||
#include <FEXCore/fextl/vector.h>
|
||||
#include <CodeEmitter/Emitter.h>
|
||||
#include <CodeEmitter/Registers.h>
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <optional>
|
||||
#include <span>
|
||||
|
||||
namespace FEXCore::Context {
|
||||
class ContextImpl;
|
||||
}
|
||||
namespace FEXCore::X86State {
|
||||
enum X86Reg : uint32_t;
|
||||
}
|
||||
|
||||
namespace FEXCore::CPU {
|
||||
// Contains the address to the currently available CPU state
|
||||
constexpr auto STATE = ARMEmitter::XReg::x28;
|
||||
|
||||
#ifndef ARCHITECTURE_arm64ec
|
||||
#ifndef _M_ARM_64EC
|
||||
// GPR temporaries. Only x3 can be used across spill boundaries
|
||||
// so if these ever need to change, be very careful about that.
|
||||
constexpr auto TMP1 = ARMEmitter::XReg::x0;
|
||||
@@ -45,8 +43,6 @@ constexpr bool TMP_ABIARGS = true;
|
||||
constexpr auto REG_PF = ARMEmitter::Reg::r26;
|
||||
constexpr auto REG_AF = ARMEmitter::Reg::r27;
|
||||
|
||||
constexpr auto REG_CALLRET_SP = ARMEmitter::XReg::x25;
|
||||
|
||||
// Vector temporaries
|
||||
constexpr auto VTMP1 = ARMEmitter::VReg::v0;
|
||||
constexpr auto VTMP2 = ARMEmitter::VReg::v1;
|
||||
@@ -65,8 +61,6 @@ constexpr bool TMP_ABIARGS = false;
|
||||
constexpr auto REG_PF = ARMEmitter::Reg::r9;
|
||||
constexpr auto REG_AF = ARMEmitter::Reg::r24;
|
||||
|
||||
constexpr auto REG_CALLRET_SP = ARMEmitter::XReg::x17;
|
||||
|
||||
// Vector temporaries
|
||||
constexpr auto VTMP1 = ARMEmitter::VReg::v16;
|
||||
constexpr auto VTMP2 = ARMEmitter::VReg::v17;
|
||||
@@ -90,8 +84,7 @@ constexpr uint64_t EC_CODE_BITMAP_MAX_ADDRESS = 1ULL << 47;
|
||||
#endif
|
||||
|
||||
// Will force one single instruction block to be generated first if set when entering the JIT filling SRA.
|
||||
// FillStaticRegs must preserve this
|
||||
constexpr auto ENTRY_FILL_SRA_SINGLE_INST_REG = TMP2;
|
||||
constexpr auto ENTRY_FILL_SRA_SINGLE_INST_REG = TMP1;
|
||||
|
||||
// Predicate to use in the X87 SVE optimization
|
||||
constexpr ARMEmitter::PRegister PRED_X87_SVEOPT = ARMEmitter::PReg::p2;
|
||||
@@ -106,20 +99,9 @@ constexpr ARMEmitter::PRegister PRED_TMP_32B = ARMEmitter::PReg::p7;
|
||||
// This class contains common emitter utility functions that can
|
||||
// be used by both Arm64 JIT and ARM64 Dispatcher
|
||||
class Arm64Emitter : public ARMEmitter::Emitter {
|
||||
public:
|
||||
protected:
|
||||
Arm64Emitter(FEXCore::Context::ContextImpl* ctx, void* EmissionPtr = nullptr, size_t size = 0);
|
||||
|
||||
enum class PadType {
|
||||
// Explicitly does not need padding, even if code-caching is enabled.
|
||||
NOPAD,
|
||||
// Explicitly needs padding, even if code-caching is disabled.
|
||||
DOPAD,
|
||||
// Choose to pad or not depending on if code-caching is enabled.
|
||||
AUTOPAD,
|
||||
};
|
||||
void LoadConstant(ARMEmitter::Size s, ARMEmitter::Register Reg, uint64_t Constant, PadType Pad = PadType::NOPAD, int MaxBytes = 0);
|
||||
|
||||
protected:
|
||||
FEXCore::Context::ContextImpl* EmitterCTX;
|
||||
|
||||
std::span<const ARMEmitter::Register> StaticRegisters {};
|
||||
@@ -129,6 +111,8 @@ protected:
|
||||
std::span<const ARMEmitter::VRegister> GeneralFPRegisters {};
|
||||
uint32_t PairRegisters = 0;
|
||||
|
||||
void LoadConstant(ARMEmitter::Size s, ARMEmitter::Register Reg, uint64_t Constant, bool NOPPad = false);
|
||||
|
||||
void FillSpecialRegs(ARMEmitter::Register TmpReg, ARMEmitter::Register TmpReg2, bool SetFIZ, bool SetPredRegs);
|
||||
|
||||
// Correlate an ARM register back to an x86 register index.
|
||||
@@ -281,8 +265,6 @@ protected:
|
||||
|
||||
FEX_CONFIG_OPT(Disassemble, DISASSEMBLE);
|
||||
#endif
|
||||
|
||||
FEX_CONFIG_OPT(EnableCodeCaching, ENABLECODECACHINGWIP);
|
||||
};
|
||||
|
||||
} // namespace FEXCore::CPU
|
||||
@@ -1,17 +1,14 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
#include "FEXCore/IR/IR.h"
|
||||
#include "FEXCore/Utils/AllocatorHooks.h"
|
||||
#include "Interface/Context/Context.h"
|
||||
#include "Interface/Core/CPUBackend.h"
|
||||
#include "Interface/Core/LookupCache.h"
|
||||
#include "Interface/Core/Dispatcher/Dispatcher.h"
|
||||
|
||||
#include <FEXCore/IR/IR.h>
|
||||
#include <FEXCore/Utils/AllocatorHooks.h>
|
||||
#include <FEXCore/Utils/PrctlUtils.h>
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
#include "LookupCache.h"
|
||||
|
||||
#ifndef _WIN32
|
||||
#include <linux/prctl.h>
|
||||
#include <sys/prctl.h>
|
||||
#endif
|
||||
|
||||
@@ -277,37 +274,37 @@ namespace CPU {
|
||||
: ThreadState(ThreadState)
|
||||
, CodeBuffers(CodeBuffers) {
|
||||
|
||||
auto& Ptrs = ThreadState->CurrentFrame->Pointers;
|
||||
auto& Common = ThreadState->CurrentFrame->Pointers.Common;
|
||||
|
||||
// Initialize named vector constants.
|
||||
for (size_t i = 0; i < FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_CONST_POOL_MAX; ++i) {
|
||||
Ptrs.NamedVectorConstantPointers[i] = reinterpret_cast<uint64_t>(NamedVectorConstants[i]);
|
||||
Common.NamedVectorConstantPointers[i] = reinterpret_cast<uint64_t>(NamedVectorConstants[i]);
|
||||
}
|
||||
|
||||
// Copy named vector constants.
|
||||
memcpy(Ptrs.NamedVectorConstants, NamedVectorConstants, sizeof(NamedVectorConstants));
|
||||
memcpy(Common.NamedVectorConstants, NamedVectorConstants, sizeof(NamedVectorConstants));
|
||||
|
||||
// Initialize Indexed named vector constants.
|
||||
Ptrs.IndexedNamedVectorConstantPointers[FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_PSHUFLW] =
|
||||
Common.IndexedNamedVectorConstantPointers[FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_PSHUFLW] =
|
||||
reinterpret_cast<uint64_t>(PSHUFLW_LUT.data());
|
||||
Ptrs.IndexedNamedVectorConstantPointers[FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_PSHUFHW] =
|
||||
Common.IndexedNamedVectorConstantPointers[FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_PSHUFHW] =
|
||||
reinterpret_cast<uint64_t>(PSHUFHW_LUT.data());
|
||||
Ptrs.IndexedNamedVectorConstantPointers[FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_PSHUFD] =
|
||||
Common.IndexedNamedVectorConstantPointers[FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_PSHUFD] =
|
||||
reinterpret_cast<uint64_t>(PSHUFD_LUT.data());
|
||||
Ptrs.IndexedNamedVectorConstantPointers[FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_SHUFPS] =
|
||||
Common.IndexedNamedVectorConstantPointers[FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_SHUFPS] =
|
||||
reinterpret_cast<uint64_t>(SHUFPS_LUT.data());
|
||||
Ptrs.IndexedNamedVectorConstantPointers[FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_DPPS_MASK] =
|
||||
Common.IndexedNamedVectorConstantPointers[FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_DPPS_MASK] =
|
||||
reinterpret_cast<uint64_t>(DPPS_MASK.data());
|
||||
Ptrs.IndexedNamedVectorConstantPointers[FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_DPPD_MASK] =
|
||||
Common.IndexedNamedVectorConstantPointers[FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_DPPD_MASK] =
|
||||
reinterpret_cast<uint64_t>(DPPD_MASK.data());
|
||||
Ptrs.IndexedNamedVectorConstantPointers[FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_PBLENDW] =
|
||||
Common.IndexedNamedVectorConstantPointers[FEXCore::IR::IndexNamedVectorConstant::INDEXED_NAMED_VECTOR_PBLENDW] =
|
||||
reinterpret_cast<uint64_t>(PBLENDW_LUT.data());
|
||||
|
||||
#ifndef FEX_DISABLE_TELEMETRY
|
||||
// Fill in telemetry values
|
||||
for (size_t i = 0; i < FEXCore::Telemetry::TYPE_LAST; ++i) {
|
||||
auto& Telem = FEXCore::Telemetry::GetTelemetryValue(static_cast<FEXCore::Telemetry::TelemetryType>(i));
|
||||
Ptrs.TelemetryValueAddresses[i] = reinterpret_cast<uint64_t>(&Telem);
|
||||
Common.TelemetryValueAddresses[i] = reinterpret_cast<uint64_t>(&Telem);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
@@ -320,7 +317,7 @@ namespace CPU {
|
||||
// Resize the code buffer and reallocate our code size
|
||||
CurrentCodeBuffer = CodeBuffers.StartLargerCodeBuffer();
|
||||
|
||||
RegisterForSignalHandler(std::move(PrevCodeBuffer));
|
||||
RegisterForSignalHandler(PrevCodeBuffer);
|
||||
return CurrentCodeBuffer.get();
|
||||
}
|
||||
|
||||
@@ -329,13 +326,14 @@ namespace CPU {
|
||||
// We have signal handlers that have generated code
|
||||
// This means that we can not safely clear the code at this point in time
|
||||
// Keep a reference to the old code buffer to delay deallocation
|
||||
SignalHandlerCodeBuffers.push_back(std::move(CodeBuffer));
|
||||
SignalHandlerCodeBuffers.push_back(CodeBuffer);
|
||||
} else {
|
||||
SignalHandlerCodeBuffers.clear();
|
||||
}
|
||||
}
|
||||
|
||||
fextl::shared_ptr<CodeBuffer> CPUBackend::CheckCodeBufferUpdate() {
|
||||
fextl::shared_ptr<CodeBuffer> OldCodeBuffer;
|
||||
auto NewCodeBuffer = CodeBuffers.GetLatest();
|
||||
if (CurrentCodeBuffer != NewCodeBuffer) {
|
||||
RegisterForSignalHandler(CurrentCodeBuffer);
|
||||
@@ -360,8 +358,6 @@ namespace CPU {
|
||||
LogMan::Msg::EFmt("Failed to mprotect last page of code buffer.");
|
||||
}
|
||||
|
||||
FEXCore::Allocator::VirtualName("FEXMemJIT", reinterpret_cast<void*>(Ptr), Size);
|
||||
|
||||
LookupCache = fextl::make_unique<GuestToHostMap>();
|
||||
}
|
||||
|
||||
@@ -400,7 +396,7 @@ namespace CPU {
|
||||
Latest = Buffer;
|
||||
LatestOffset = 0;
|
||||
|
||||
OnCodeBufferAllocated(Buffer);
|
||||
OnCodeBufferAllocated(*Buffer);
|
||||
|
||||
return Buffer;
|
||||
}
|
||||
|
||||
@@ -13,14 +13,9 @@ $end_info$
|
||||
#include <FEXCore/fextl/memory.h>
|
||||
#include <FEXCore/fextl/string.h>
|
||||
#include <FEXCore/fextl/vector.h>
|
||||
#include <FEXCore/fextl/map.h>
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
namespace FEXCore::CPU {
|
||||
union Relocation;
|
||||
}
|
||||
|
||||
namespace FEXCore {
|
||||
|
||||
namespace IR {
|
||||
@@ -81,7 +76,7 @@ namespace CPU {
|
||||
// Protects writes to the latest CodeBuffer and changes to LatestOffset
|
||||
FEXCore::ForkableUniqueMutex CodeBufferWriteMutex;
|
||||
|
||||
virtual void OnCodeBufferAllocated(const std::shared_ptr<CodeBuffer>&) {};
|
||||
virtual void OnCodeBufferAllocated(CodeBuffer&) {};
|
||||
|
||||
private:
|
||||
fextl::shared_ptr<CodeBuffer> Latest;
|
||||
@@ -99,7 +94,15 @@ namespace CPU {
|
||||
struct CompiledCode {
|
||||
// Where this code block begins.
|
||||
uint8_t* BlockBegin;
|
||||
fextl::map<uint64_t, uint8_t*> EntryPoints;
|
||||
/**
|
||||
* The function entrypoint to this codeblock.
|
||||
*
|
||||
* This may or may not equal `BlockBegin` above. Depending on the CPU backend, it may stick data
|
||||
* prior to the BlockEntry.
|
||||
*
|
||||
* Is actually a function pointer of type `void (FEXCore::Core::ThreadState *Thread)`
|
||||
*/
|
||||
uint8_t* BlockEntry;
|
||||
// The total size of the codeblock from [BlockBegin, BlockBegin+Size).
|
||||
size_t Size;
|
||||
};
|
||||
@@ -161,7 +164,18 @@ namespace CPU {
|
||||
virtual CompiledCode CompileCode(uint64_t Entry, uint64_t Size, bool SingleInst, const FEXCore::IR::IRListView* IR,
|
||||
FEXCore::Core::DebugData* DebugData, bool CheckTF) = 0;
|
||||
|
||||
virtual fextl::vector<FEXCore::CPU::Relocation> TakeRelocations(uint64_t GuestBaseAddress) = 0;
|
||||
/**
|
||||
* @brief Relocates a block of code from the JIT code object cache
|
||||
*
|
||||
* @param Entry - RIP of the entry
|
||||
* @param SerializationData - Serialization data referring to the object cache for `Entry`
|
||||
*
|
||||
* @return An executable function pointer relocated from the cache object
|
||||
*/
|
||||
[[nodiscard]]
|
||||
virtual void* RelocateJITObjectCode(uint64_t /* Entry */, const CodeSerialize::CodeObjectFileSection* /* SerializationData */) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
virtual void ClearCache() {}
|
||||
|
||||
|
||||
@@ -14,7 +14,6 @@ $end_info$
|
||||
#include <FEXCore/Core/CPUID.h>
|
||||
#include <FEXCore/Core/HostFeatures.h>
|
||||
#include <FEXCore/Utils/FileLoading.h>
|
||||
#include <FEXCore/Utils/MathUtils.h>
|
||||
#include <FEXCore/fextl/string.h>
|
||||
#include <FEXHeaderUtils/Syscalls.h>
|
||||
|
||||
@@ -24,7 +23,7 @@ $end_info$
|
||||
|
||||
namespace FEXCore {
|
||||
namespace ProductNames {
|
||||
#ifdef ARCHITECTURE_arm64
|
||||
#ifdef _M_ARM_64
|
||||
static const char ARM_UNKNOWN[] = "Unknown ARM CPU";
|
||||
static const char ARM_A57[] = "Cortex-A57";
|
||||
static const char ARM_A72[] = "Cortex-A72";
|
||||
@@ -44,15 +43,12 @@ namespace ProductNames {
|
||||
static const char ARM_A715[] = "Cortex-A715";
|
||||
static const char ARM_A720[] = "Cortex-A720";
|
||||
static const char ARM_A725[] = "Cortex-A725";
|
||||
static const char ARM_C1Pro[] = "C1-Pro";
|
||||
static const char ARM_C1Premium[] = "C1-Premium";
|
||||
static const char ARM_X1[] = "Cortex-X1";
|
||||
static const char ARM_X1C[] = "Cortex-X1C";
|
||||
static const char ARM_X2[] = "Cortex-X2";
|
||||
static const char ARM_X3[] = "Cortex-X3";
|
||||
static const char ARM_X4[] = "Cortex-X4";
|
||||
static const char ARM_X925[] = "Cortex-X925";
|
||||
static const char ARM_C1Ultra[] = "C1-Ultra";
|
||||
static const char ARM_N1[] = "Neoverse N1";
|
||||
static const char ARM_N2[] = "Neoverse N2";
|
||||
static const char ARM_N3[] = "Neoverse N3";
|
||||
@@ -63,7 +59,6 @@ namespace ProductNames {
|
||||
static const char ARM_A65[] = "Cortex-A65";
|
||||
static const char ARM_A510[] = "Cortex-A510";
|
||||
static const char ARM_A520[] = "Cortex-A520";
|
||||
static const char ARM_C1Nano[] = "C1-Nano";
|
||||
|
||||
static const char ARM_Kryo200[] = "Kryo 2xx";
|
||||
static const char ARM_Kryo300[] = "Kryo 3xx";
|
||||
@@ -75,7 +70,6 @@ namespace ProductNames {
|
||||
|
||||
static const char ARM_Denver[] = "Nvidia Denver";
|
||||
static const char ARM_Carmel[] = "Nvidia Carmel";
|
||||
static const char ARM_Olympus[] = "Nvidia Olympus";
|
||||
|
||||
static const char ARM_Firestorm_M1[] = "Apple Firestorm (M1)";
|
||||
static const char ARM_Icestorm_M1[] = "Apple Icestorm (M1)";
|
||||
@@ -89,12 +83,8 @@ namespace ProductNames {
|
||||
static const char ARM_Blizzard_M2Pro[] = "Apple Blizzard (M2 Pro)";
|
||||
static const char ARM_Avalanche_M2Max[] = "Apple Avalanche (M2 Max)";
|
||||
static const char ARM_Blizzard_M2Max[] = "Apple Blizzard (M2 Max)";
|
||||
static const char ARM_AppleSilicon[] = "Apple Silicon";
|
||||
|
||||
static const char ARM_ORYON_1[] = "Oryon-1";
|
||||
static const char ARM_Ampere_1[] = "AmpereOne";
|
||||
static const char ARM_Ampere_1A[] = "AmpereOneA";
|
||||
static const char ARM_Ampere_1B[] = "AmpereOneB";
|
||||
#else
|
||||
#endif
|
||||
} // namespace ProductNames
|
||||
@@ -140,7 +130,7 @@ constexpr uint32_t FAMILY_IDENTIFIER = GenerateFamily(CPUFamily {
|
||||
});
|
||||
#endif
|
||||
|
||||
#ifdef ARCHITECTURE_arm64
|
||||
#ifdef _M_ARM_64
|
||||
uint32_t GetCycleCounterFrequency() {
|
||||
uint64_t Result {};
|
||||
__asm("mrs %[Res], CNTFRQ_EL0" : [Res] "=r"(Result));
|
||||
@@ -180,7 +170,7 @@ void CPUIDEmu::SetupHostHybridFlag() {
|
||||
// CPU priority order
|
||||
// This is mostly arbitrary but will sort by some sort of CPU priority by performance
|
||||
// Relative list so things they will commonly end up in big.little configurations sort of relate
|
||||
static constexpr std::array<CPUMIDR, 66> CPUMIDRs = {{
|
||||
static constexpr std::array<CPUMIDR, 58> CPUMIDRs = {{
|
||||
// Typically big CPU cores
|
||||
{0x51, 0x001, 1, ProductNames::ARM_ORYON_1}, // Qualcomm Oryon-1
|
||||
|
||||
@@ -190,48 +180,39 @@ void CPUIDEmu::SetupHostHybridFlag() {
|
||||
{0x61, 0x029, 1, ProductNames::ARM_Firestorm_M1Max}, // Apple Firestorm (M1 Max)
|
||||
{0x61, 0x025, 1, ProductNames::ARM_Firestorm_M1Pro}, // Apple Firestorm (M1 Pro)
|
||||
{0x61, 0x023, 1, ProductNames::ARM_Firestorm_M1}, // Apple Firestorm (M1)
|
||||
{0x61, 0, 1, ProductNames::ARM_AppleSilicon}, // QEmu Apple Silicon
|
||||
|
||||
{0x41, 0xd8c, 1, ProductNames::ARM_C1Ultra}, // C1-Ultra
|
||||
{0x41, 0xd90, 1, ProductNames::ARM_C1Premium}, // C1-Premium
|
||||
{0x41, 0xd8b, 1, ProductNames::ARM_C1Pro}, // C1-Pro
|
||||
{0x41, 0xd85, 1, ProductNames::ARM_X925}, // X925
|
||||
{0x41, 0xd87, 1, ProductNames::ARM_A725}, // A725
|
||||
{0x41, 0xd84, 1, ProductNames::ARM_V3}, // V3
|
||||
{0x41, 0xd83, 1, ProductNames::ARM_V3AE}, // V3AE
|
||||
{0x41, 0xd8e, 1, ProductNames::ARM_N3}, // N3
|
||||
{0x41, 0xd82, 1, ProductNames::ARM_X4}, // X4
|
||||
{0x41, 0xd81, 1, ProductNames::ARM_A720}, // A720
|
||||
{0x41, 0xd4e, 1, ProductNames::ARM_X3}, // X3
|
||||
{0x41, 0xd4d, 1, ProductNames::ARM_A715}, // A715
|
||||
{0x41, 0xd4f, 1, ProductNames::ARM_V2}, // V2
|
||||
{0x41, 0xd4b, 1, ProductNames::ARM_A78C}, // A78C
|
||||
{0x41, 0xd4a, 1, ProductNames::ARM_E1}, // E1
|
||||
{0x41, 0xd49, 1, ProductNames::ARM_N2}, // N2
|
||||
{0x41, 0xd48, 1, ProductNames::ARM_X2}, // X2
|
||||
{0x41, 0xd47, 1, ProductNames::ARM_A710}, // A710
|
||||
{0x41, 0xd4C, 1, ProductNames::ARM_X1C}, // X1C
|
||||
{0x41, 0xd44, 1, ProductNames::ARM_X1}, // X1
|
||||
{0x41, 0xd42, 1, ProductNames::ARM_A78AE}, // A78AE
|
||||
{0x41, 0xd41, 1, ProductNames::ARM_A78}, // A78
|
||||
{0x41, 0xd40, 1, ProductNames::ARM_V1}, // V1
|
||||
{0x41, 0xd0e, 1, ProductNames::ARM_A76AE}, // A76AE
|
||||
{0x41, 0xd0d, 1, ProductNames::ARM_A77}, // A77
|
||||
{0x41, 0xd0c, 1, ProductNames::ARM_N1}, // N1
|
||||
{0x41, 0xd0b, 1, ProductNames::ARM_A76}, // A76
|
||||
{0x51, 0x804, 1, ProductNames::ARM_Kryo400}, // Kryo 4xx Gold (A76 based)
|
||||
{0x41, 0xd0a, 1, ProductNames::ARM_A75}, // A75
|
||||
{0x51, 0x802, 1, ProductNames::ARM_Kryo300}, // Kryo 3xx Gold (A75 based)
|
||||
{0x41, 0xd09, 1, ProductNames::ARM_A73}, // A73
|
||||
{0x51, 0x800, 1, ProductNames::ARM_Kryo200}, // Kryo 2xx Gold (A73 based)
|
||||
{0x41, 0xd08, 1, ProductNames::ARM_A72}, // A72
|
||||
{0x41, 0xd85, 1, ProductNames::ARM_X925}, // X925
|
||||
{0x41, 0xd87, 1, ProductNames::ARM_A725}, // A725
|
||||
{0x41, 0xd84, 1, ProductNames::ARM_V3}, // V3
|
||||
{0x41, 0xd83, 1, ProductNames::ARM_V3AE}, // V3AE
|
||||
{0x41, 0xd8e, 1, ProductNames::ARM_N3}, // N3
|
||||
{0x41, 0xd82, 1, ProductNames::ARM_X4}, // X4
|
||||
{0x41, 0xd81, 1, ProductNames::ARM_A720}, // A720
|
||||
{0x41, 0xd4e, 1, ProductNames::ARM_X3}, // X3
|
||||
{0x41, 0xd4d, 1, ProductNames::ARM_A715}, // A715
|
||||
{0x41, 0xd4f, 1, ProductNames::ARM_V2}, // V2
|
||||
{0x41, 0xd4b, 1, ProductNames::ARM_A78C}, // A78C
|
||||
{0x41, 0xd4a, 1, ProductNames::ARM_E1}, // E1
|
||||
{0x41, 0xd49, 1, ProductNames::ARM_N2}, // N2
|
||||
{0x41, 0xd48, 1, ProductNames::ARM_X2}, // X2
|
||||
{0x41, 0xd47, 1, ProductNames::ARM_A710}, // A710
|
||||
{0x41, 0xd4C, 1, ProductNames::ARM_X1C}, // X1C
|
||||
{0x41, 0xd44, 1, ProductNames::ARM_X1}, // X1
|
||||
{0x41, 0xd42, 1, ProductNames::ARM_A78AE}, // A78AE
|
||||
{0x41, 0xd41, 1, ProductNames::ARM_A78}, // A78
|
||||
{0x41, 0xd40, 1, ProductNames::ARM_V1}, // V1
|
||||
{0x41, 0xd0e, 1, ProductNames::ARM_A76AE}, // A76AE
|
||||
{0x41, 0xd0d, 1, ProductNames::ARM_A77}, // A77
|
||||
{0x41, 0xd0c, 1, ProductNames::ARM_N1}, // N1
|
||||
{0x41, 0xd0b, 1, ProductNames::ARM_A76}, // A76
|
||||
{0x51, 0x804, 1, ProductNames::ARM_Kryo400}, // Kryo 4xx Gold (A76 based)
|
||||
{0x41, 0xd0a, 1, ProductNames::ARM_A75}, // A75
|
||||
{0x51, 0x802, 1, ProductNames::ARM_Kryo300}, // Kryo 3xx Gold (A75 based)
|
||||
{0x41, 0xd09, 1, ProductNames::ARM_A73}, // A73
|
||||
{0x51, 0x800, 1, ProductNames::ARM_Kryo200}, // Kryo 2xx Gold (A73 based)
|
||||
{0x41, 0xd08, 1, ProductNames::ARM_A72}, // A72
|
||||
|
||||
{0xc0, 0xac3, 1, ProductNames::ARM_Ampere_1}, // AmpereOne
|
||||
{0xc0, 0xac4, 1, ProductNames::ARM_Ampere_1A}, // AmpereOneA
|
||||
{0xc0, 0xac5, 1, ProductNames::ARM_Ampere_1B}, // AmpereOneB
|
||||
|
||||
{0x4e, 0x010, 1, ProductNames::ARM_Olympus}, // Olympus
|
||||
{0x4e, 0x004, 1, ProductNames::ARM_Carmel}, // Carmel
|
||||
{0x4e, 0x004, 1, ProductNames::ARM_Carmel}, // Carmel
|
||||
|
||||
// Denver rated above A57 to match TX2 weirdness
|
||||
{0x4e, 0x003, 1, ProductNames::ARM_Denver}, // Denver
|
||||
@@ -246,7 +227,6 @@ void CPUIDEmu::SetupHostHybridFlag() {
|
||||
{0x61, 0x024, 0, ProductNames::ARM_Icestorm_M1Pro}, // Apple Icestorm (M1 Pro)
|
||||
{0x61, 0x022, 0, ProductNames::ARM_Icestorm_M1}, // Apple Icestorm (M1)
|
||||
|
||||
{0x41, 0xd8a, 1, ProductNames::ARM_C1Nano}, // C1-Nano
|
||||
{0x41, 0xd80, 0, ProductNames::ARM_A520}, // A520
|
||||
{0x41, 0xd46, 0, ProductNames::ARM_A510}, // A510
|
||||
{0x41, 0xd06, 0, ProductNames::ARM_A65}, // A65
|
||||
@@ -444,10 +424,10 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_01h(uint32_t Leaf) const {
|
||||
|
||||
Res.eax = FAMILY_IDENTIFIER;
|
||||
|
||||
Res.ebx = 0 | // Brand index
|
||||
(8 << 8) | // Cache line size in bytes
|
||||
(Cores << 16) | // Number of addressable IDs for the logical cores in the physical CPU
|
||||
(GetCPUID() << 24); // Local APIC ID
|
||||
Res.ebx = 0 | // Brand index
|
||||
(8 << 8) | // Cache line size in bytes
|
||||
(Cores << 16) | // Number of addressable IDs for the logical cores in the physical CPU
|
||||
(0 << 24); // Local APIC ID
|
||||
|
||||
Res.ecx = (1 << 0) | // SSE3
|
||||
(CTX->HostFeatures.SupportsPMULL_128Bit << 1) | // PCLMULQDQ
|
||||
@@ -510,7 +490,7 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_01h(uint32_t Leaf) const {
|
||||
(1 << 25) | // SSE
|
||||
(1 << 26) | // SSE2
|
||||
(0 << 27) | // Self Snoop
|
||||
(0 << 28) | // (HTT) Max APIC IDs reserved field is valid
|
||||
(1 << 28) | // Max APIC IDs reserved field is valid
|
||||
(1 << 29) | // Thermal monitor
|
||||
(0 << 30) | // Reserved
|
||||
(0 << 31); // Pending break enable
|
||||
@@ -646,7 +626,6 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_07h(uint32_t Leaf) const {
|
||||
// Only enable EnhancedREPMOVS if atomic memcpy tso emulation isn't enabled.
|
||||
const uint32_t SupportsEnhancedREPMOVS = CTX->IsMemcpyAtomicTSOEnabled() == false;
|
||||
const uint32_t SupportsVPCLMULQDQ = CTX->HostFeatures.SupportsPMULL_128Bit && SupportsAVX();
|
||||
const uint32_t SupportsWFXT = CTX->HostFeatures.SupportsWFXT;
|
||||
|
||||
// Number of subfunctions
|
||||
Res.eax = 0x0;
|
||||
@@ -666,39 +645,39 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_07h(uint32_t Leaf) const {
|
||||
(1 << 13) | // Deprecates FPU CS and DS
|
||||
(0 << 14) | // Intel MPX
|
||||
(0 << 15) | // Intel Resource Directory Technology Allocation
|
||||
(0 << 16) | // AVX512-F
|
||||
(0 << 17) | // AVX512-DQ
|
||||
(0 << 16) | // Reserved
|
||||
(0 << 17) | // Reserved
|
||||
(CTX->HostFeatures.SupportsRAND << 18) | // RDSEED
|
||||
(1 << 19) | // ADCX and ADOX instructions
|
||||
(0 << 20) | // SMAP Supervisor mode access prevention and CLAC/STAC instructions
|
||||
(0 << 21) | // AVX512-IFMA
|
||||
(0 << 22) | // PCOMMIT (deprecated?)
|
||||
(0 << 21) | // Reserved
|
||||
(0 << 22) | // Reserved
|
||||
(1 << 23) | // CLFLUSHOPT instruction
|
||||
(1 << 24) | // CLWB instruction
|
||||
(0 << 25) | // Intel processor trace
|
||||
(0 << 26) | // AVX512-PF
|
||||
(0 << 27) | // AVX512-ER
|
||||
(0 << 28) | // AVX512-CD
|
||||
(0 << 26) | // Reserved
|
||||
(0 << 27) | // Reserved
|
||||
(0 << 28) | // Reserved
|
||||
(Features.SHA << 29) | // SHA instructions
|
||||
(0 << 30) | // AVX512-BW
|
||||
(0 << 31); // AVX512-VL
|
||||
(0 << 30) | // Reserved
|
||||
(0 << 31); // Reserved
|
||||
|
||||
Res.ecx = (1 << 0) | // PREFETCHWT1
|
||||
(0 << 1) | // AVX512VBMI
|
||||
(0 << 2) | // Usermode instruction prevention
|
||||
(0 << 3) | // Protection keys for user mode pages
|
||||
(0 << 4) | // OS protection keys
|
||||
(SupportsWFXT << 5) | // waitpkg
|
||||
(0 << 6) | // AVX512-VBMI2
|
||||
(0 << 5) | // waitpkg
|
||||
(0 << 6) | // AVX512_VBMI2
|
||||
(0 << 7) | // CET shadow stack
|
||||
(0 << 8) | // GFNI
|
||||
(CTX->HostFeatures.SupportsAES256 << 9) | // VAES
|
||||
(SupportsVPCLMULQDQ << 10) | // VPCLMULQDQ
|
||||
(0 << 11) | // AVX512-VNNI
|
||||
(0 << 12) | // AVX512-BITALG
|
||||
(0 << 11) | // AVX512_VNNI
|
||||
(0 << 12) | // AVX512_BITALG
|
||||
(0 << 13) | // Intel Total Memory Encryption
|
||||
(0 << 14) | // AVX512-VPOPCNTDQ
|
||||
(0 << 15) | // FZM (TDX)
|
||||
(0 << 14) | // AVX512_VPOPCNTDQ
|
||||
(0 << 15) | // Reserved
|
||||
(0 << 16) | // 5 Level page tables
|
||||
(0 << 17) | // MPX MAWAU
|
||||
(0 << 18) | // MPX MAWAU
|
||||
@@ -706,28 +685,28 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_07h(uint32_t Leaf) const {
|
||||
(0 << 20) | // MPX MAWAU
|
||||
(0 << 21) | // MPX MAWAU
|
||||
(1 << 22) | // RDPID Read Processor ID
|
||||
(0 << 23) | // AES Key Locker
|
||||
(1 << 24) | // bus-lock-detect
|
||||
(0 << 23) | // Reserved
|
||||
(0 << 24) | // Reserved
|
||||
(0 << 25) | // CLDEMOTE
|
||||
(0 << 26) | // MPRR (TDX)
|
||||
(0 << 26) | // Reserved
|
||||
(0 << 27) | // MOVDIRI
|
||||
(0 << 28) | // MOVDIR64B
|
||||
(0 << 29) | // ENQCMD
|
||||
(0 << 29) | // Reserved
|
||||
(0 << 30) | // SGX Launch configuration
|
||||
(0 << 31); // PKS
|
||||
(0 << 31); // Reserved
|
||||
|
||||
Res.edx = (0 << 0) | // SGX-TEM (TDX)
|
||||
(0 << 1) | // SGX-KEYS
|
||||
(0 << 2) | // AVX512-4VNNIW
|
||||
(0 << 3) | // AVX512-4FMAPS
|
||||
Res.edx = (0 << 0) | // Reserved
|
||||
(0 << 1) | // Reserved
|
||||
(0 << 2) | // AVX512_4VNNIW
|
||||
(0 << 3) | // AVX512_4FMAPS
|
||||
(1 << 4) | // Fast Short Rep Mov
|
||||
(0 << 5) | // UINTR
|
||||
(0 << 5) | // Reserved
|
||||
(0 << 6) | // Reserved
|
||||
(0 << 7) | // Reserved
|
||||
(0 << 8) | // AVX512-VP2INTERSECT
|
||||
(0 << 8) | // AVX512_VP2INTERSECT
|
||||
(0 << 9) | // SRBDS_CTRL (Special Register Buffer Data Sampling Mitigations)
|
||||
(0 << 10) | // VERW clears CPU buffers
|
||||
(0 << 11) | // rtm-always-abort
|
||||
(0 << 11) | // Reserved
|
||||
(0 << 12) | // Reserved
|
||||
(0 << 13) | // TSX Force Abort (TSX will force abort if attempted)
|
||||
(0 << 14) | // SERIALIZE instruction
|
||||
@@ -739,7 +718,7 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_07h(uint32_t Leaf) const {
|
||||
(0 << 20) | // Intel CET
|
||||
(0 << 21) | // Reserved
|
||||
(0 << 22) | // AMX-BF16 - Tile computation on bfloat16
|
||||
(0 << 23) | // AVX512-FP16 - FP16 AVX512 instructions
|
||||
(0 << 23) | // AVX512_FP16 - FP16 AVX512 instructions
|
||||
(0 << 24) | // AMX-tile - If AMX is implemented
|
||||
(0 << 25) | // AMX-int8 - AMX on 8-bit integers
|
||||
(0 << 26) | // IBRS_IBPB - Speculation control
|
||||
@@ -775,7 +754,7 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_0Dh(uint32_t Leaf) const {
|
||||
// XFeatureSupportedMask[63:32]
|
||||
Res.edx = 0; // Upper 32-bits of XFeatureSupportedMask
|
||||
} else if (Leaf == 1) {
|
||||
Res.eax = (1 << 0) | // XSAVEOPT
|
||||
Res.eax = (0 << 0) | // XSAVEOPT
|
||||
(0 << 1) | // XSAVEC (and XRSTOR)
|
||||
(0 << 2) | // XGETBV - XGETBV with ECX=1 supported
|
||||
(0 << 3); // XSAVES - XSAVES, XRSTORS, and IA32_XSS supported
|
||||
@@ -857,10 +836,10 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_4000_0001h(uint32_t Leaf) con
|
||||
constexpr uint32_t MaximumSubLeafNumber = 2;
|
||||
if (Leaf == 0) {
|
||||
// EAX[3:0] Is the host architecture that FEX is running under
|
||||
#ifdef ARCHITECTURE_x86_64
|
||||
#ifdef _M_X86_64
|
||||
// EAX[3:0] = 1 = x86_64 host architecture
|
||||
Res.eax |= 0b0001;
|
||||
#elif defined(ARCHITECTURE_arm64)
|
||||
#elif defined(_M_ARM_64)
|
||||
// EAX[3:0] = 2 = AArch64 host architecture
|
||||
Res.eax |= 0b0010;
|
||||
#else
|
||||
@@ -912,71 +891,71 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0001h(uint32_t Leaf) con
|
||||
|
||||
Res.eax = FAMILY_IDENTIFIER;
|
||||
|
||||
Res.ecx = (1 << 0) | // LAHF/SAHF
|
||||
(1 << 1) | // 0 = Single core product, 1 = multi core product
|
||||
(0 << 2) | // SVM
|
||||
(1 << 3) | // Extended APIC register space
|
||||
(0 << 4) | // LOCK MOV CR0 means MOV CR8
|
||||
(1 << 5) | // ABM instructions
|
||||
(CTX->HostFeatures.SupportsSSE4a << 6) | // SSE4a
|
||||
(0 << 7) | // Misaligned SSE mode
|
||||
(1 << 8) | // PREFETCHW
|
||||
(0 << 9) | // OS visible workaround support
|
||||
(0 << 10) | // Instruction based sampling support
|
||||
(0 << 11) | // XOP
|
||||
(0 << 12) | // SKINIT
|
||||
(0 << 13) | // Watchdog timer support
|
||||
(0 << 14) | // Reserved
|
||||
(0 << 15) | // Lightweight profiling support
|
||||
(0 << 16) | // FMA4
|
||||
(1 << 17) | // Translation cache extension
|
||||
(0 << 18) | // Reserved
|
||||
(0 << 19) | // Reserved
|
||||
(0 << 20) | // Reserved
|
||||
(0 << 21) | // XOP-TBM
|
||||
(0 << 22) | // Topology extensions support
|
||||
(0 << 23) | // Core performance counter extensions
|
||||
(0 << 24) | // NB performance counter extensions
|
||||
(0 << 25) | // Reserved
|
||||
(0 << 26) | // Data breakpoints extensions
|
||||
(0 << 27) | // Performance TSC
|
||||
(0 << 28) | // L2 perf counter extensions
|
||||
(0 << 29) | // MONITORX
|
||||
(0 << 30) | // Reserved
|
||||
(0 << 31); // Reserved
|
||||
Res.ecx = (1 << 0) | // LAHF/SAHF
|
||||
(1 << 1) | // 0 = Single core product, 1 = multi core product
|
||||
(0 << 2) | // SVM
|
||||
(1 << 3) | // Extended APIC register space
|
||||
(0 << 4) | // LOCK MOV CR0 means MOV CR8
|
||||
(1 << 5) | // ABM instructions
|
||||
(0 << 6) | // SSE4a
|
||||
(0 << 7) | // Misaligned SSE mode
|
||||
(1 << 8) | // PREFETCHW
|
||||
(0 << 9) | // OS visible workaround support
|
||||
(0 << 10) | // Instruction based sampling support
|
||||
(0 << 11) | // XOP
|
||||
(0 << 12) | // SKINIT
|
||||
(0 << 13) | // Watchdog timer support
|
||||
(0 << 14) | // Reserved
|
||||
(0 << 15) | // Lightweight profiling support
|
||||
(0 << 16) | // FMA4
|
||||
(1 << 17) | // Translation cache extension
|
||||
(0 << 18) | // Reserved
|
||||
(0 << 19) | // Reserved
|
||||
(0 << 20) | // Reserved
|
||||
(0 << 21) | // XOP-TBM
|
||||
(0 << 22) | // Topology extensions support
|
||||
(0 << 23) | // Core performance counter extensions
|
||||
(0 << 24) | // NB performance counter extensions
|
||||
(0 << 25) | // Reserved
|
||||
(0 << 26) | // Data breakpoints extensions
|
||||
(0 << 27) | // Performance TSC
|
||||
(0 << 28) | // L2 perf counter extensions
|
||||
(0 << 29) | // MONITORX
|
||||
(0 << 30) | // Reserved
|
||||
(0 << 31); // Reserved
|
||||
|
||||
Res.edx = (1 << 0) | // FPU
|
||||
(1 << 1) | // Virtual mode extensions
|
||||
(1 << 2) | // Debugging extensions
|
||||
(1 << 3) | // Page size extensions
|
||||
(1 << 4) | // TSC
|
||||
(1 << 5) | // MSR support
|
||||
(1 << 6) | // PAE
|
||||
(1 << 7) | // Machine Check Exception
|
||||
(1 << 8) | // CMPXCHG8B
|
||||
(1 << 9) | // APIC
|
||||
(0 << 10) | // Reserved
|
||||
(1 << 11) | // SYSCALL/SYSRET
|
||||
(1 << 12) | // MTRR
|
||||
(1 << 13) | // Page global extension
|
||||
(1 << 14) | // Machine Check architecture
|
||||
(1 << 15) | // CMOV
|
||||
(1 << 16) | // Page attribute table
|
||||
(1 << 17) | // Page-size extensions
|
||||
(0 << 18) | // Reserved
|
||||
(0 << 19) | // Reserved
|
||||
(1 << 20) | // NX
|
||||
(0 << 21) | // Reserved
|
||||
(1 << 22) | // MMXExt
|
||||
(1 << 23) | // MMX
|
||||
(1 << 24) | // FXSAVE/FXRSTOR
|
||||
(1 << 25) | // FXSAVE/FXRSTOR Optimizations
|
||||
(0 << 26) | // 1 gigabit pages
|
||||
(SUPPORTS_RDTSCP << 27) | // RDTSCP
|
||||
(0 << 28) | // Reserved
|
||||
(1 << 29) | // Long Mode
|
||||
(CTX->HostFeatures.Supports3DNow << 30) | // 3DNow! Extensions
|
||||
(CTX->HostFeatures.Supports3DNow << 31); // 3DNow!
|
||||
Res.edx = (1 << 0) | // FPU
|
||||
(1 << 1) | // Virtual mode extensions
|
||||
(1 << 2) | // Debugging extensions
|
||||
(1 << 3) | // Page size extensions
|
||||
(1 << 4) | // TSC
|
||||
(1 << 5) | // MSR support
|
||||
(1 << 6) | // PAE
|
||||
(1 << 7) | // Machine Check Exception
|
||||
(1 << 8) | // CMPXCHG8B
|
||||
(1 << 9) | // APIC
|
||||
(0 << 10) | // Reserved
|
||||
(1 << 11) | // SYSCALL/SYSRET
|
||||
(1 << 12) | // MTRR
|
||||
(1 << 13) | // Page global extension
|
||||
(1 << 14) | // Machine Check architecture
|
||||
(1 << 15) | // CMOV
|
||||
(1 << 16) | // Page attribute table
|
||||
(1 << 17) | // Page-size extensions
|
||||
(0 << 18) | // Reserved
|
||||
(0 << 19) | // Reserved
|
||||
(1 << 20) | // NX
|
||||
(0 << 21) | // Reserved
|
||||
(1 << 22) | // MMXExt
|
||||
(1 << 23) | // MMX
|
||||
(1 << 24) | // FXSAVE/FXRSTOR
|
||||
(1 << 25) | // FXSAVE/FXRSTOR Optimizations
|
||||
(0 << 26) | // 1 gigabit pages
|
||||
(SUPPORTS_RDTSCP << 27) | // RDTSCP
|
||||
(0 << 28) | // Reserved
|
||||
(1 << 29) | // Long Mode
|
||||
(1 << 30) | // 3DNow! Extensions
|
||||
(1 << 31); // 3DNow!
|
||||
return Res;
|
||||
}
|
||||
|
||||
@@ -1097,9 +1076,9 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0008h(uint32_t Leaf) con
|
||||
(CTX->HostFeatures.SupportsCLZERO << 0); // CLZERO support
|
||||
|
||||
uint32_t CoreCount = Cores - 1;
|
||||
Res.ecx = (0 << 16) | // PerfTscSize: Performance timestamp count size
|
||||
(std::bit_ceil(Cores) << 12) | // ApicIdSize: Number of bits in ApicID
|
||||
(CoreCount << 0); // Count count subtract one
|
||||
Res.ecx = (0 << 16) | // PerfTscSize: Performance timestamp count size
|
||||
((uint32_t)std::log2(CoreCount + 1) << 12) | // ApicIdSize: Number of bits in ApicID
|
||||
(CoreCount << 0); // Count count subtract one
|
||||
|
||||
return Res;
|
||||
}
|
||||
@@ -1230,7 +1209,7 @@ CPUIDEmu::CPUIDEmu(const FEXCore::Context::ContextImpl* ctx)
|
||||
|
||||
SetupFeatures();
|
||||
|
||||
#ifdef ARCHITECTURE_arm64
|
||||
#ifdef _M_ARM_64
|
||||
if (SupportsCPUIndexInTPIDRRO) {
|
||||
GetCPUID = GetCPUID_TPIDRRO;
|
||||
}
|
||||
|
||||
@@ -159,7 +159,7 @@ private:
|
||||
|
||||
struct CPUData {
|
||||
const char* ProductName {};
|
||||
#ifdef ARCHITECTURE_arm64
|
||||
#ifdef _M_ARM_64
|
||||
uint32_t MIDR {};
|
||||
#endif
|
||||
bool IsBig {};
|
||||
@@ -277,7 +277,7 @@ private:
|
||||
// 0: Highest function parameter and ID
|
||||
{SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},
|
||||
// 1: Processor info
|
||||
{SupportsConstant::NONCONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},
|
||||
{SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},
|
||||
// 2: Cache and TLB info
|
||||
{SupportsConstant::CONSTANT, NeedsLeafConstant::NOLEAFCONSTANT},
|
||||
// 3: Serial Number(previously), now reserved
|
||||
|
||||
@@ -1,660 +0,0 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
#include "Utils/SpinWaitLock.h"
|
||||
|
||||
#include <Interface/Context/Context.h>
|
||||
#include <Interface/Core/ArchHelpers/Arm64Emitter.h>
|
||||
#include <Interface/Core/Dispatcher/Dispatcher.h>
|
||||
#include <Interface/Core/JIT/DebugData.h>
|
||||
#include <Interface/Core/JIT/Relocations.h>
|
||||
#include <Interface/Core/LookupCache.h>
|
||||
#include <Interface/Core/OpcodeDispatcher.h>
|
||||
#include <Interface/IR/PassManager.h>
|
||||
|
||||
#include <FEXCore/Core/Thunks.h>
|
||||
#include <FEXCore/HLE/SyscallHandler.h>
|
||||
|
||||
#include <FEXHeaderUtils/Filesystem.h>
|
||||
|
||||
#include <git_version.h>
|
||||
|
||||
#include <xxhash.h>
|
||||
|
||||
#include <fstream>
|
||||
|
||||
namespace FEXCore {
|
||||
|
||||
#if __clang_major__ < 16
|
||||
ExecutableFileInfo::ExecutableFileInfo(fextl::unique_ptr<HLE::SourcecodeMap> Map, uint64_t FileId, fextl::string Filename)
|
||||
: SourcecodeMap(std::move(Map))
|
||||
, FileId(FileId)
|
||||
, Filename(Filename) {}
|
||||
#endif
|
||||
|
||||
fextl::string CodeMap::GetBaseFilename(const ExecutableFileInfo& MainExecutable, bool AddNombSuffix) {
|
||||
auto FileId = MainExecutable.FileId;
|
||||
|
||||
std::string_view base_filename = FHU::Filesystem::GetFilename(std::string_view {MainExecutable.Filename});
|
||||
if (FileId != 0xffff'ffff'ffff'ffff) {
|
||||
return fextl::fmt::format("{}-{:016x}{}", base_filename, MainExecutable.FileId, AddNombSuffix ? "-nomb" : "");
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
fextl::map<CodeMapFileId, CodeMap::ParsedContents> CodeMap::ParseCodeMap(std::ifstream& File) {
|
||||
fextl::map<CodeMapFileId, CodeMap::ParsedContents> Ret;
|
||||
while (true) {
|
||||
Entry Entry;
|
||||
File.read(reinterpret_cast<char*>(&Entry), sizeof(Entry));
|
||||
if (!File) {
|
||||
break;
|
||||
}
|
||||
|
||||
if (Entry.FileId == LoadExternalLibrary.FileId && Entry.BlockOffset == LoadExternalLibrary.BlockOffset) {
|
||||
ExternalLibraryInfo Info;
|
||||
File.read(reinterpret_cast<char*>(&Info), sizeof(Info));
|
||||
|
||||
fextl::string Filename;
|
||||
std::getline(File, Filename, '\0');
|
||||
|
||||
// Align to 4-byte boundary
|
||||
char Null[4];
|
||||
File.read(Null, AlignUp(Filename.size() + 1, 4) - Filename.size() - 1);
|
||||
if (!File) {
|
||||
break;
|
||||
}
|
||||
Ret[Info.ExternalFileId].Filename = std::move(Filename);
|
||||
} else if (Entry.FileId == SetExecutableFileId {}.Marker.FileId && Entry.BlockOffset == SetExecutableFileId {}.Marker.BlockOffset) {
|
||||
CodeMapFileId ExecutableFileId;
|
||||
File.read(reinterpret_cast<char*>(&ExecutableFileId), sizeof(ExecutableFileId));
|
||||
if (!File) {
|
||||
break;
|
||||
}
|
||||
Ret[ExecutableFileId].IsExecutable = true;
|
||||
} else {
|
||||
if (!Ret.contains(Entry.FileId)) {
|
||||
LogMan::Msg::EFmt("Code map referenced unknown file id {:016x}", Entry.FileId);
|
||||
} else {
|
||||
Ret[Entry.FileId].Blocks.insert(Entry.BlockOffset);
|
||||
}
|
||||
}
|
||||
|
||||
if (!File) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return Ret;
|
||||
}
|
||||
|
||||
CodeMapWriter::CodeMapWriter(CodeMapOpener& Opener, bool OpenEagerly)
|
||||
: Buffer(4096)
|
||||
, FileOpener(Opener) {
|
||||
if (OpenEagerly) {
|
||||
CodeMapFD = FileOpener.OpenCodeMapFile();
|
||||
}
|
||||
}
|
||||
|
||||
CodeMapWriter::~CodeMapWriter() {
|
||||
if (CodeMapFD.value_or(-1) != -1) {
|
||||
Flush(BufferOffset);
|
||||
close(*CodeMapFD);
|
||||
}
|
||||
}
|
||||
|
||||
bool CodeMapWriter::IsWriteEnabled(const ExecutableFileSectionInfo& Section) {
|
||||
if (CodeMapFD == -1) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// PV libraries can't yet be read by FEXServer, so skip dumping them
|
||||
if (Section.FileInfo.Filename.starts_with("/run/pressure-vessel")) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (CodeMapFD) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Acquire mutex and re-check CodeMapFD to avoid race conditions
|
||||
auto lk = std::unique_lock {Mutex};
|
||||
if (!CodeMapFD) {
|
||||
CodeMapFD = FileOpener.OpenCodeMapFile();
|
||||
}
|
||||
|
||||
return CodeMapFD != -1;
|
||||
}
|
||||
|
||||
void CodeMapWriter::Flush(size_t Offset) {
|
||||
// Acquire exclusive lock and flush circular buffer
|
||||
std::unique_lock Lock {Mutex};
|
||||
Flush(Offset, Lock);
|
||||
}
|
||||
|
||||
void CodeMapWriter::Flush(size_t Offset, std::unique_lock<std::shared_mutex>&) {
|
||||
write(*CodeMapFD, Buffer.data(), Offset);
|
||||
BufferOffset = 0;
|
||||
}
|
||||
|
||||
void CodeMapWriter::AppendBlock(const FEXCore::ExecutableFileSectionInfo& SectionInfo, uint64_t BlockEntry) {
|
||||
if (!IsWriteEnabled(SectionInfo)) {
|
||||
return;
|
||||
}
|
||||
|
||||
BlockEntry -= SectionInfo.FileStartVA;
|
||||
if (BlockEntry > std::numeric_limits<uint32_t>::max()) {
|
||||
ERROR_AND_DIE_FMT("Cannot write code map");
|
||||
}
|
||||
|
||||
// Register new library if not already known
|
||||
bool NewLibraryLoad = false;
|
||||
{
|
||||
// Check prior registration with shared lock
|
||||
std::shared_lock Lock {Mutex};
|
||||
NewLibraryLoad = !KnownFileIds.contains(SectionInfo.FileInfo.FileId);
|
||||
}
|
||||
if (NewLibraryLoad) {
|
||||
// Register to map with exclusive lock
|
||||
std::unique_lock Lock {Mutex};
|
||||
NewLibraryLoad &= KnownFileIds.insert(SectionInfo.FileInfo.FileId).second;
|
||||
}
|
||||
if (NewLibraryLoad) {
|
||||
// Add entry to code map
|
||||
AppendLibraryLoad(SectionInfo.FileInfo);
|
||||
}
|
||||
|
||||
// Register the actual code block
|
||||
CodeMap::Entry DataEntry {SectionInfo.FileInfo.FileId, static_cast<uint32_t>(BlockEntry)};
|
||||
AppendData(std::as_bytes(std::span {&DataEntry, 1}));
|
||||
}
|
||||
|
||||
void CodeMapWriter::AppendLibraryLoad(const FEXCore::ExecutableFileInfo& FileInfo) {
|
||||
// See CodeMap::ExternalLibraryInfo
|
||||
auto ExternalFileId = FileInfo.FileId;
|
||||
auto TotalSize = AlignUp(sizeof(CodeMap::LoadExternalLibrary) + sizeof(ExternalFileId) + FileInfo.Filename.size() + 1, 4);
|
||||
const auto Data = reinterpret_cast<char*>(alloca(TotalSize));
|
||||
auto WritePtr = std::copy_n(reinterpret_cast<const char*>(&CodeMap::LoadExternalLibrary), sizeof(CodeMap::LoadExternalLibrary), Data);
|
||||
WritePtr = std::copy_n(reinterpret_cast<const char*>(&ExternalFileId), sizeof(ExternalFileId), WritePtr);
|
||||
WritePtr = std::copy(FileInfo.Filename.begin(), FileInfo.Filename.end(), WritePtr);
|
||||
std::fill(WritePtr, Data + TotalSize, 0);
|
||||
AppendData(std::as_bytes(std::span {Data, TotalSize}));
|
||||
}
|
||||
|
||||
void CodeMapWriter::AppendSetMainExecutable(const FEXCore::ExecutableFileInfo& FileInfo) {
|
||||
CodeMap::SetExecutableFileId Data {.ExecutableFileId = FileInfo.FileId};
|
||||
AppendData(std::span {reinterpret_cast<const std::byte*>(&Data), sizeof(Data)});
|
||||
}
|
||||
|
||||
void CodeMapWriter::AppendData(std::span<const std::byte> Data) {
|
||||
std::shared_lock Lock {Mutex};
|
||||
auto Offset = BufferOffset.fetch_add(Data.size_bytes());
|
||||
if (Offset + Data.size_bytes() > Buffer.size()) {
|
||||
// Acquire exclusive lock and flush the buffer.
|
||||
// Under heavy pressure, multiple threads may observe an exhausted buffer simultaneously.
|
||||
// The thread with the last in-bounds Offset is responsible for flushing the buffer.
|
||||
Lock.unlock();
|
||||
bool IsResponsibleForFlush = false;
|
||||
{
|
||||
std::unique_lock ExclusiveLock {Mutex};
|
||||
IsResponsibleForFlush = (Offset <= Buffer.size());
|
||||
if (IsResponsibleForFlush) {
|
||||
Flush(Offset, ExclusiveLock);
|
||||
}
|
||||
}
|
||||
if (!IsResponsibleForFlush) {
|
||||
// Wait for the buffer to be flushed on the responsible thread
|
||||
Utils::SpinWaitLock::WaitPred<std::less_equal<>, size_t>(reinterpret_cast<size_t*>(&BufferOffset), Buffer.size());
|
||||
}
|
||||
AppendData(Data);
|
||||
return;
|
||||
}
|
||||
|
||||
memcpy(&Buffer.at(Offset), Data.data(), Data.size_bytes());
|
||||
}
|
||||
|
||||
} // namespace FEXCore
|
||||
|
||||
namespace FEXCore::Context {
|
||||
|
||||
CodeCache::CodeCache(ContextImpl& CTX_)
|
||||
: CTX(CTX_) {}
|
||||
CodeCache::~CodeCache() = default;
|
||||
|
||||
uint64_t CodeCache::ComputeCodeMapId(std::string_view Filename, int FD) {
|
||||
if (Filename.empty()) {
|
||||
return 0xffff'ffff'ffff'ffff;
|
||||
}
|
||||
|
||||
// For now, we just use the file path as an identifier.
|
||||
// TODO: Ensure the hash is unique enough to distinguish executables while remaining independent of the installation location
|
||||
return XXH3_64bits(Filename.data(), Filename.size());
|
||||
}
|
||||
|
||||
struct CodeCacheHeader {
|
||||
std::array<char, 4> Magic = ExpectedMagic;
|
||||
uint32_t FormatVersion = 1;
|
||||
char FEXVersion[8] = {};
|
||||
uint32_t NumBlocks;
|
||||
uint32_t NumCodePages;
|
||||
uint32_t CodeBufferSize;
|
||||
uint32_t NumRelocations;
|
||||
uint64_t SerializedBaseAddress;
|
||||
// TODO: Consider including information from LookupCache.BlockLinks
|
||||
|
||||
static constexpr std::array<char, 4> ExpectedMagic = {'F', 'X', 'C', 'C'};
|
||||
};
|
||||
|
||||
template<typename T>
|
||||
concept OrderedContainer = requires { typename T::key_compare; };
|
||||
|
||||
bool CodeCache::SaveData(Core::InternalThreadState& Thread, int fd, const ExecutableFileSectionInfo& SourceBinary, uint64_t SerializedBaseAddress) {
|
||||
auto CodeBuffer = CTX.GetLatest();
|
||||
auto& LookupCache = *Thread.LookupCache->Shared;
|
||||
auto Relocations = Thread.CPUBackend->TakeRelocations(SourceBinary.FileStartVA);
|
||||
|
||||
// Write file header
|
||||
CodeCacheHeader header {};
|
||||
constexpr std::string_view git_hash = GIT_SHORT_HASH;
|
||||
static_assert(git_hash.size() <= sizeof(header.FEXVersion));
|
||||
std::ranges::copy(git_hash, header.FEXVersion);
|
||||
header.NumBlocks = LookupCache.BlockList.size();
|
||||
header.NumCodePages = LookupCache.CodePages.size();
|
||||
header.CodeBufferSize = CTX.LatestOffset;
|
||||
header.NumRelocations = Relocations.size();
|
||||
header.SerializedBaseAddress = SerializedBaseAddress;
|
||||
::write(fd, &header, sizeof(header));
|
||||
|
||||
// Dump guest<->host block mappings
|
||||
{
|
||||
// Cache contents must be deterministic, so copy the unordered block list and then sort by key
|
||||
static_assert(!OrderedContainer<decltype(LookupCache.BlockList)>, "Already deterministic; drop temporary container");
|
||||
fextl::vector<std::pair<uint64_t, const GuestToHostMap::BlockEntry*>> BlockList;
|
||||
BlockList.reserve(LookupCache.BlockList.size());
|
||||
for (auto& [Guest, BlockEntry] : LookupCache.BlockList) {
|
||||
static_assert(sizeof(Guest) == 8, "Breaking change in code cache data layout");
|
||||
BlockList.emplace_back(Guest, &BlockEntry);
|
||||
}
|
||||
std::ranges::sort(BlockList);
|
||||
|
||||
for (auto [Guest, Host] : BlockList) {
|
||||
static_assert(sizeof(Host->HostCode) == 8, "Breaking change in code cache data layout");
|
||||
static_assert(sizeof(Host->CodePages[0]) == 8, "Breaking change in code cache data layout");
|
||||
|
||||
Guest -= SourceBinary.FileStartVA;
|
||||
::write(fd, &Guest, sizeof(Guest));
|
||||
uint64_t HostCode = Host->HostCode - reinterpret_cast<uintptr_t>(CodeBuffer->Ptr);
|
||||
::write(fd, &HostCode, sizeof(HostCode));
|
||||
uint64_t NumCodePages = Host->CodePages.size();
|
||||
::write(fd, &NumCodePages, sizeof(NumCodePages));
|
||||
LOGMAN_THROW_A_FMT(std::ranges::is_sorted(Host->CodePages), "Code pages aren't sorted");
|
||||
for (auto CodePage : Host->CodePages) {
|
||||
CodePage -= SourceBinary.FileStartVA;
|
||||
::write(fd, &CodePage, sizeof(CodePage));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Dump relocations
|
||||
static_assert(sizeof(Relocations[0]) == 48, "Breaking change in code cache data layout");
|
||||
::write(fd, Relocations.data(), Relocations.size() * sizeof(Relocations[0]));
|
||||
|
||||
// Pad to next page in file so that the CodeBuffer can be mmap'ed into process on load
|
||||
char Zero[64] {};
|
||||
auto Off = lseek(fd, 0, SEEK_CUR);
|
||||
while (Off != AlignUp(Off, Utils::FEX_PAGE_SIZE)) {
|
||||
auto BytesToWrite = std::min(AlignUp(Off, Utils::FEX_PAGE_SIZE) - Off, sizeof(Zero));
|
||||
::write(fd, Zero, BytesToWrite);
|
||||
Off += BytesToWrite;
|
||||
}
|
||||
|
||||
// Dump the host code (relocated for position-independent serialization)
|
||||
std::vector CodeBufferData(reinterpret_cast<std::byte*>(CodeBuffer->Ptr), reinterpret_cast<std::byte*>(CodeBuffer->Ptr) + CTX.LatestOffset);
|
||||
if (!ApplyCodeRelocations(SerializedBaseAddress, CodeBufferData, Relocations, true)) {
|
||||
LOGMAN_THROW_A_FMT(false, "Failed to apply code relocations");
|
||||
return false;
|
||||
}
|
||||
::write(fd, CodeBufferData.data(), CodeBufferData.size());
|
||||
|
||||
// Dump code pages
|
||||
static_assert(OrderedContainer<decltype(LookupCache.CodePages)>, "Non-deterministic data source");
|
||||
for (const auto& [PageIndex, Entrypoints] : LookupCache.CodePages) {
|
||||
uint64_t PageAddr = (PageIndex << 12) - SourceBinary.FileStartVA;
|
||||
::write(fd, &PageAddr, sizeof(PageAddr));
|
||||
uint64_t NumEntrypoints = Entrypoints.size();
|
||||
::write(fd, &NumEntrypoints, sizeof(NumEntrypoints));
|
||||
for (uint64_t Entrypoint : Entrypoints) {
|
||||
Entrypoint -= SourceBinary.FileStartVA;
|
||||
::write(fd, &Entrypoint, sizeof(Entrypoint));
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool CodeCache::LoadData(Core::InternalThreadState* Thread, std::byte* MappedCacheFile, const ExecutableFileSectionInfo& BinarySection) {
|
||||
if (!EnableCodeCaching) {
|
||||
return true;
|
||||
}
|
||||
|
||||
namespace ranges = std::ranges;
|
||||
|
||||
// Read file header
|
||||
CodeCacheHeader header {};
|
||||
::memcpy(&header, MappedCacheFile, sizeof(header));
|
||||
MappedCacheFile += sizeof(header);
|
||||
|
||||
LogMan::Msg::IFmt("Cache load: {:5} blocks; base={:#14x}; off={:#9x}-{:#09x}; {:016x} {}", header.NumBlocks, BinarySection.FileStartVA,
|
||||
BinarySection.BeginVA - BinarySection.FileStartVA, BinarySection.EndVA - BinarySection.FileStartVA,
|
||||
BinarySection.FileInfo.FileId, BinarySection.FileInfo.Filename);
|
||||
|
||||
if (!ranges::equal(header.Magic, header.ExpectedMagic)) {
|
||||
LogMan::Msg::EFmt("Invalid cache file header");
|
||||
return false;
|
||||
}
|
||||
|
||||
char ExpectedVersion[8] = GIT_SHORT_HASH;
|
||||
ranges::fill(ranges::find(ExpectedVersion, 0), std::end(ExpectedVersion), 0);
|
||||
if (!ranges::equal(header.FEXVersion, ExpectedVersion)) {
|
||||
LogMan::Msg::IFmt("Cache generated from old FEX version {}, current is {}; skipping", fmt::join(header.FEXVersion, ""),
|
||||
fmt::join(ExpectedVersion, ""));
|
||||
return false;
|
||||
}
|
||||
|
||||
if (header.NumBlocks == 0) {
|
||||
// Valid caches are never empty
|
||||
LogMan::Msg::IFmt("Code cache empty, aborting");
|
||||
return false;
|
||||
}
|
||||
|
||||
// Read guest<->host block mappings
|
||||
using BlockListEntry = decltype(GuestToHostMap::BlockList)::value_type;
|
||||
fextl::vector<BlockListEntry> BlockList(header.NumBlocks);
|
||||
{
|
||||
for (auto& BlockPtr : BlockList) {
|
||||
::memcpy(&BlockPtr.first, MappedCacheFile, sizeof(BlockPtr.first));
|
||||
MappedCacheFile += sizeof(BlockPtr.first);
|
||||
::memcpy(&BlockPtr.second.HostCode, MappedCacheFile, sizeof(BlockPtr.second.HostCode));
|
||||
MappedCacheFile += sizeof(BlockPtr.second.HostCode);
|
||||
uint64_t NumGuestPages;
|
||||
::memcpy(&NumGuestPages, MappedCacheFile, sizeof(NumGuestPages));
|
||||
MappedCacheFile += sizeof(NumGuestPages);
|
||||
|
||||
BlockPtr.second.CodePages.resize(NumGuestPages);
|
||||
::memcpy(BlockPtr.second.CodePages.data(), MappedCacheFile, std::span {BlockPtr.second.CodePages}.size_bytes());
|
||||
MappedCacheFile += std::span {BlockPtr.second.CodePages}.size_bytes();
|
||||
}
|
||||
|
||||
// Consistency check: VMA regions at the top and end should belong to the same file
|
||||
auto [min_val, max_val] = ranges::minmax_element(BlockList, std::less {}, &decltype(BlockList)::value_type::first);
|
||||
auto MinBound = CTX.SyscallHandler->LookupExecutableFileSection(Thread, min_val->first + BinarySection.FileStartVA);
|
||||
auto MaxBound = CTX.SyscallHandler->LookupExecutableFileSection(Thread, max_val->first + BinarySection.FileStartVA);
|
||||
if (&MinBound->FileInfo != &BinarySection.FileInfo || &MaxBound->FileInfo != &BinarySection.FileInfo) {
|
||||
ERROR_AND_DIE_FMT("Cached blocks offsets {:#x}-{:#x} out of bounds for guest library {} ({:016x} @ {:#x}) while trying to load "
|
||||
"section {:#x}-{:#x}!",
|
||||
min_val->first, max_val->first, BinarySection.FileInfo.Filename, BinarySection.FileInfo.FileId,
|
||||
BinarySection.FileStartVA, BinarySection.BeginVA, BinarySection.EndVA);
|
||||
}
|
||||
|
||||
// Constrain BlockList to the given ExecutableFileSectionInfo
|
||||
LOGMAN_THROW_A_FMT(ranges::is_sorted(BlockList, [](auto& a, auto& b) { return a.first < b.first; }), "Expected sorted block list");
|
||||
auto begin = ranges::lower_bound(BlockList, BinarySection.BeginVA - BinarySection.FileStartVA, std::less {}, &BlockListEntry::first);
|
||||
auto end =
|
||||
ranges::upper_bound(begin, BlockList.end(), BinarySection.EndVA - BinarySection.FileStartVA - 1, std::less {}, &BlockListEntry::first);
|
||||
BlockList.erase(end, BlockList.end());
|
||||
BlockList.erase(BlockList.begin(), begin);
|
||||
if (BlockList.empty()) {
|
||||
// Not an error since there is just no data to load
|
||||
LogMan::Msg::IFmt("No blocks cached in this range, aborting");
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
// Read relocations
|
||||
fextl::vector<FEXCore::CPU::Relocation> Relocations(header.NumRelocations, FEXCore::CPU::Relocation::Default());
|
||||
::memcpy(Relocations.data(), MappedCacheFile, Relocations.size() * sizeof(Relocations[0]));
|
||||
MappedCacheFile += Relocations.size() * sizeof(Relocations[0]);
|
||||
|
||||
// Pad to next page in file, which contains CodeBuffer data
|
||||
MappedCacheFile = reinterpret_cast<std::byte*>(AlignUp(reinterpret_cast<uintptr_t>(MappedCacheFile), Utils::FEX_PAGE_SIZE));
|
||||
|
||||
// Prepare CodeBuffer: Page aligned and big enough to hold all cached data
|
||||
auto Lock = std::unique_lock {CTX.CodeBufferWriteMutex};
|
||||
if (Thread) {
|
||||
if (auto Prev = Thread->CPUBackend->CheckCodeBufferUpdate()) {
|
||||
Allocator::VirtualDontNeed(Thread->CallRetStackBase, FEXCore::Core::InternalThreadState::CALLRET_STACK_SIZE);
|
||||
auto lk = Thread->LookupCache->AcquireWriteLock();
|
||||
Thread->LookupCache->ChangeGuestToHostMapping(*Prev, *CTX.GetLatest()->LookupCache, lk);
|
||||
}
|
||||
}
|
||||
|
||||
auto CodeBuffer = CTX.GetLatest();
|
||||
LOGMAN_THROW_A_FMT(header.CodeBufferSize <= CodeBuffer->Size, "CodeBuffer too small to load code cache");
|
||||
LOGMAN_THROW_A_FMT(reinterpret_cast<uintptr_t>(CodeBuffer->Ptr) % 0x1000 == 0, "Expected CodeBuffer base to be page-aligned");
|
||||
const auto Delta = AlignUp(CTX.LatestOffset, 0x1000) - CTX.LatestOffset;
|
||||
CTX.LatestOffset += Delta;
|
||||
|
||||
while (CTX.LatestOffset + header.CodeBufferSize > CodeBuffer->Size - Utils::FEX_PAGE_SIZE) {
|
||||
if (Thread) {
|
||||
CTX.ClearCodeCache(Thread);
|
||||
CodeBuffer = CTX.GetLatest();
|
||||
LogMan::Msg::IFmt("Increased code buffer size to {} MiB for cache load", CodeBuffer->Size / 1024 / 1024);
|
||||
} else {
|
||||
ERROR_AND_DIE_FMT("Cannot extend codebuffer without thread!");
|
||||
}
|
||||
}
|
||||
|
||||
// Read CodeBuffer data from file. Make sure the destination is page-aligned.
|
||||
// TODO: Only load the data needed for the selected section
|
||||
auto CodeBufferRange = std::as_writable_bytes(std::span {CodeBuffer->Ptr, CodeBuffer->Size}).subspan(CTX.LatestOffset, header.CodeBufferSize);
|
||||
::memcpy(CodeBufferRange.data(), MappedCacheFile, header.CodeBufferSize);
|
||||
MappedCacheFile += header.CodeBufferSize;
|
||||
CTX.LatestOffset += header.CodeBufferSize;
|
||||
|
||||
// Apply FEX relocations
|
||||
auto Ret = ApplyCodeRelocations(BinarySection.FileStartVA, CodeBufferRange, Relocations, false);
|
||||
LOGMAN_THROW_A_FMT(Ret == true, "Failed to apply code cache relocations");
|
||||
|
||||
{
|
||||
auto& LookupCache = *CodeBuffer->LookupCache;
|
||||
auto WriteLock = LookupCache.AcquireWriteLock();
|
||||
|
||||
// Register blocks to LookupCache
|
||||
for (auto& [Guest, Host] : BlockList) {
|
||||
for (auto& CodePage : Host.CodePages) {
|
||||
CodePage += BinarySection.FileStartVA;
|
||||
}
|
||||
auto HostCode = reinterpret_cast<void*>(Host.HostCode + reinterpret_cast<uintptr_t>(CodeBufferRange.data()));
|
||||
LookupCache.AddBlockMapping(Guest + BinarySection.FileStartVA, std::move(Host.CodePages), HostCode, WriteLock);
|
||||
}
|
||||
|
||||
// Register loaded code ranges
|
||||
fextl::vector<uint64_t> Entrypoints;
|
||||
for (uint32_t i = 0; i < header.NumCodePages; ++i) {
|
||||
uint64_t CodePage;
|
||||
memcpy(&CodePage, MappedCacheFile, sizeof(CodePage));
|
||||
CodePage += BinarySection.FileStartVA;
|
||||
MappedCacheFile += sizeof(CodePage);
|
||||
|
||||
uint64_t NumEntrypoints;
|
||||
memcpy(&NumEntrypoints, MappedCacheFile, sizeof(NumEntrypoints));
|
||||
MappedCacheFile += sizeof(NumEntrypoints);
|
||||
|
||||
Entrypoints.resize(NumEntrypoints);
|
||||
memcpy(Entrypoints.data(), MappedCacheFile, NumEntrypoints * sizeof(Entrypoints[0]));
|
||||
MappedCacheFile += NumEntrypoints * sizeof(Entrypoints[0]);
|
||||
for (auto& Entrypoint : Entrypoints) {
|
||||
Entrypoint += BinarySection.FileStartVA;
|
||||
}
|
||||
|
||||
if (LookupCache.AddBlockExecutableRange(Entrypoints, CodePage, FEXCore::Utils::FEX_PAGE_SIZE, WriteLock)) {
|
||||
CTX.SyscallHandler->MarkGuestExecutableRange(Thread, CodePage, FEXCore::Utils::FEX_PAGE_SIZE);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (EnableCodeCacheValidation) {
|
||||
fextl::set<uint64_t> GuestBlocks, HostBlocks;
|
||||
for (auto& [Guest, Host] : BlockList) {
|
||||
GuestBlocks.insert(Guest + BinarySection.FileStartVA);
|
||||
HostBlocks.insert(Host.HostCode);
|
||||
}
|
||||
|
||||
Validate(BinarySection, std::move(GuestBlocks), HostBlocks, CodeBufferRange);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void CodeCache::Validate(const ExecutableFileSectionInfo& Section, fextl::set<uint64_t> GuestBlocks, const fextl::set<uint64_t>& HostBlocks,
|
||||
std::span<std::byte> CachedCode) {
|
||||
LOGMAN_THROW_A_FMT(!HostBlocks.empty(), "Tried to validate without any host blocks");
|
||||
// Skip any cached data before the first host block
|
||||
CachedCode = CachedCode.subspan(*HostBlocks.begin() - sizeof(CPU::CPUBackend::JITCodeHeader));
|
||||
|
||||
if (!ValidationCTX) {
|
||||
ValidationCTX.reset(static_cast<ContextImpl*>(FEXCore::Context::Context::CreateNewContext(CTX.HostFeatures).release()));
|
||||
ValidationCTX->SetSignalDelegator(CTX.SignalDelegation);
|
||||
ValidationCTX->SetSyscallHandler(CTX.SyscallHandler);
|
||||
ValidationCTX->SetThunkHandler(CTX.ThunkHandler);
|
||||
if (!ValidationCTX->InitCore()) {
|
||||
ERROR_AND_DIE_FMT("Failed to create cache load validation context");
|
||||
}
|
||||
|
||||
ValidationThread.reset(ValidationCTX->CreateThread(0, 0, nullptr));
|
||||
|
||||
auto Frame = ValidationThread->CurrentFrame;
|
||||
Frame->State.segment_arrays[FEXCore::Core::CPUState::SEGMENT_ARRAY_INDEX_GDT] = &ValidationGDT[0];
|
||||
Frame->State.segment_arrays[FEXCore::Core::CPUState::SEGMENT_ARRAY_INDEX_LDT] = &ValidationGDT[0];
|
||||
Frame->State.cs_idx = 0;
|
||||
Frame->State.cs_cached = 0;
|
||||
|
||||
if (ValidationCTX->Config.Is64BitMode()) {
|
||||
ValidationGDT[0].L = 1; // L = Long Mode = 64-bit
|
||||
ValidationGDT[0].D = 0; // D = Default Operand Size = Reserved
|
||||
} else {
|
||||
ValidationGDT[0].L = 0; // L = Long Mode = 32-bit
|
||||
ValidationGDT[0].D = 1; // D = Default Operand Size = 32-bit
|
||||
}
|
||||
}
|
||||
|
||||
auto NewCodeBuffer = ValidationCTX->GetLatest();
|
||||
|
||||
std::span<std::byte> CodeBufferRangeRef =
|
||||
std::as_writable_bytes(std::span {NewCodeBuffer->Ptr, NewCodeBuffer->Ptr + NewCodeBuffer->Size}).subspan(0, CachedCode.size_bytes());
|
||||
|
||||
while (!GuestBlocks.empty()) {
|
||||
auto [CompiledBlocks, _, _2, _3, _4] = ValidationCTX->CompileCode(ValidationThread.get(), *GuestBlocks.begin(), 0 /* TODO: Set MaxInst? */);
|
||||
for (auto& Entry : CompiledBlocks.EntryPoints) {
|
||||
GuestBlocks.erase(Entry.first);
|
||||
}
|
||||
}
|
||||
|
||||
// Patch FEX-internal function addresses with values from the main Context to ensure the code blocks are comparable
|
||||
auto NewRelocations = ValidationThread->CPUBackend->TakeRelocations(Section.FileStartVA);
|
||||
NewRelocations.erase(std::remove_if(NewRelocations.begin(), NewRelocations.end(), [](const CPU::Relocation& Reloc) {
|
||||
return Reloc.Header.Type != CPU::RelocationTypes::RELOC_NAMED_SYMBOL_LITERAL && Reloc.Header.Type != CPU::RelocationTypes::RELOC_NAMED_THUNK_MOVE;
|
||||
}));
|
||||
(void)ApplyCodeRelocations(Section.FileStartVA, CodeBufferRangeRef, NewRelocations, false);
|
||||
|
||||
if (ValidationCTX->LatestOffset <= CodeBufferRangeRef.size()) {
|
||||
// Reference compilation produced fewer bytes than our cache, so validation is going to fail.
|
||||
// Make sure we don't output any garbage bytes though.
|
||||
CodeBufferRangeRef = CodeBufferRangeRef.subspan(0, ValidationCTX->LatestOffset);
|
||||
}
|
||||
|
||||
auto [Mismatch, _] = std::mismatch(CodeBufferRangeRef.begin(), CodeBufferRangeRef.end(), CachedCode.begin());
|
||||
if (Mismatch != CodeBufferRangeRef.end()) {
|
||||
// Align down to instruction size
|
||||
auto Idx = AlignDown(std::distance(CodeBufferRangeRef.begin(), Mismatch), 4);
|
||||
|
||||
auto BlockIt = std::prev(HostBlocks.lower_bound(*HostBlocks.begin() + Idx + 1));
|
||||
std::optional<uint64_t> GuestBlockAddr;
|
||||
std::optional<uint64_t> GuestBlockAddrRef;
|
||||
if (BlockIt != HostBlocks.end()) {
|
||||
for (int i : {0, 1}) {
|
||||
std::span Buffer = (i == 0 ? CachedCode : CodeBufferRangeRef);
|
||||
|
||||
// Second instruction is always a constant load for relative offset to the (multi)block start
|
||||
int32_t addr = (*reinterpret_cast<uint32_t*>(&Buffer[*BlockIt - *HostBlocks.begin() + 4]) & 0x3ff'ffe0) << 11;
|
||||
addr >>= 14;
|
||||
auto header = reinterpret_cast<CPU::CPUBackend::JITCodeHeader*>(&Buffer[*BlockIt - *HostBlocks.begin() + 4 + addr]);
|
||||
auto tail = reinterpret_cast<CPU::CPUBackend::JITCodeTail*>(reinterpret_cast<uintptr_t>(header) + header->OffsetToBlockTail);
|
||||
(i == 0 ? GuestBlockAddr : GuestBlockAddrRef) = tail->RIP - Section.FileStartVA;
|
||||
LogMan::Msg::EFmt("Recorded rip {}: {:#x} (offset {:#x})", i, tail->RIP, tail->RIP - Section.FileStartVA);
|
||||
|
||||
if (i == 1) {
|
||||
if (tail->RIP >= Section.BeginVA && tail->RIP < Section.EndVA) {
|
||||
auto [IRView, TotalInstructions, TotalInstructionsLength, StartAddr, Length, _] =
|
||||
ValidationCTX->GenerateIR(ValidationThread.get(), tail->RIP, false, FEXCore::Config::Get_MAXINST());
|
||||
fextl::stringstream ss;
|
||||
FEXCore::IR::Dump(&ss, &*IRView);
|
||||
LogMan::Msg::EFmt("IR:\n{}", ss.str());
|
||||
} else {
|
||||
LogMan::Msg::EFmt("Can't dump IR for out-of-range RIP {:#x}", tail->RIP);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fextl::string GuestBlockInfo = "UNKNOWN";
|
||||
if (GuestBlockAddr) {
|
||||
GuestBlockInfo = fextl::fmt::format("{:#x}", GuestBlockAddr.value());
|
||||
}
|
||||
if (GuestBlockAddr != GuestBlockAddrRef) {
|
||||
GuestBlockInfo += " (MISMATCH)";
|
||||
}
|
||||
ERROR_AND_DIE_FMT("Cache validation failed at offset {:#x}: {:02x} <-> {:02x} (at {} <-> {}, guest block {})", Idx,
|
||||
fmt::join(CachedCode.subspan(Idx, 4), ""), fmt::join(CodeBufferRangeRef.subspan(Idx, 4), ""),
|
||||
fmt::ptr(CachedCode.data()), fmt::ptr(CodeBufferRangeRef.data()), GuestBlockInfo);
|
||||
}
|
||||
|
||||
// Reset Context state for next validation
|
||||
ValidationThread->LookupCache->ClearCache(ValidationThread->LookupCache->AcquireWriteLock());
|
||||
ValidationCTX->LatestOffset = 0;
|
||||
|
||||
LogMan::Msg::IFmt("\tSuccessfully validated cache");
|
||||
}
|
||||
|
||||
bool CodeCache::ApplyCodeRelocations(uint64_t GuestEntry, std::span<std::byte> Code,
|
||||
std::span<const FEXCore::CPU::Relocation> EntryRelocations, bool ForStorage) {
|
||||
CPU::Arm64Emitter Emitter(&CTX, Code.data(), Code.size_bytes());
|
||||
for (size_t j = 0; j < EntryRelocations.size(); ++j) {
|
||||
const FEXCore::CPU::Relocation& Reloc = EntryRelocations[j];
|
||||
Emitter.SetCursorOffset(Reloc.Header.Offset);
|
||||
|
||||
switch (Reloc.Header.Type) {
|
||||
case FEXCore::CPU::RelocationTypes::RELOC_NAMED_SYMBOL_LITERAL: {
|
||||
// Generate a literal so we can place it
|
||||
uint64_t Pointer = ForStorage ? 0 : GetNamedSymbolLiteral(CTX, Reloc.NamedSymbolLiteral.Symbol);
|
||||
Emitter.dc64(Pointer);
|
||||
break;
|
||||
}
|
||||
case FEXCore::CPU::RelocationTypes::RELOC_NAMED_THUNK_MOVE: {
|
||||
uint64_t Pointer = ForStorage ? 0 : reinterpret_cast<uint64_t>(CTX.ThunkHandler->LookupThunk(Reloc.NamedThunkMove.Symbol));
|
||||
if (Pointer == ~0ULL) {
|
||||
return false;
|
||||
}
|
||||
// Pointers are required to fit within 48-bit VA space.
|
||||
Emitter.LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Register(Reloc.NamedThunkMove.RegisterIndex), Pointer,
|
||||
CPU::Arm64Emitter::PadType::DOPAD, 6);
|
||||
break;
|
||||
}
|
||||
case FEXCore::CPU::RelocationTypes::RELOC_GUEST_RIP_LITERAL: {
|
||||
Emitter.dc64(GuestEntry + Reloc.GuestRIP.GuestRIP);
|
||||
break;
|
||||
}
|
||||
case FEXCore::CPU::RelocationTypes::RELOC_GUEST_RIP_MOVE: {
|
||||
uint64_t Pointer = Reloc.GuestRIP.GuestRIP + GuestEntry;
|
||||
// Pointers are required to fit within 48-bit VA space.
|
||||
Emitter.LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Register(Reloc.GuestRIP.RegisterIndex), Pointer,
|
||||
CPU::Arm64Emitter::PadType::DOPAD, 6);
|
||||
break;
|
||||
}
|
||||
|
||||
default: ERROR_AND_DIE_FMT("Unknown relocation type {}", ToUnderlying(Reloc.Header.Type));
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace FEXCore::Context
|
||||
@@ -14,11 +14,11 @@ $end_info$
|
||||
#include "Interface/Core/CPUBackend.h"
|
||||
#include "Interface/Core/CPUID.h"
|
||||
#include "Interface/Core/Frontend.h"
|
||||
#include "Interface/Core/ObjectCache/ObjectCacheService.h"
|
||||
#include "Interface/Core/OpcodeDispatcher.h"
|
||||
#include "Interface/Core/JIT/JITClass.h"
|
||||
#include "Interface/Core/Dispatcher/Dispatcher.h"
|
||||
#include "Interface/Core/X86Tables/X86Tables.h"
|
||||
#include <Interface/GDBJIT/GDBJIT.h>
|
||||
#include "Interface/IR/IR.h"
|
||||
#include "Interface/IR/IREmitter.h"
|
||||
#include "Interface/IR/Passes/RegisterAllocationPass.h"
|
||||
@@ -46,13 +46,13 @@ $end_info$
|
||||
#include "FEXCore/Utils/SignalScopeGuards.h"
|
||||
#include <FEXCore/Utils/Threads.h>
|
||||
#include <FEXCore/Utils/Profiler.h>
|
||||
#include <FEXCore/Utils/SHMStats.h>
|
||||
#include <FEXCore/fextl/fmt.h>
|
||||
#include <FEXCore/fextl/memory.h>
|
||||
#include <FEXCore/fextl/set.h>
|
||||
#include <FEXCore/fextl/sstream.h>
|
||||
#include <FEXCore/fextl/vector.h>
|
||||
#include <FEXHeaderUtils/Syscalls.h>
|
||||
#include <FEXHeaderUtils/TodoDefines.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
@@ -78,7 +78,10 @@ namespace FEXCore::Context {
|
||||
ContextImpl::ContextImpl(const FEXCore::HostFeatures& Features)
|
||||
: HostFeatures {Features}
|
||||
, CPUID {this}
|
||||
, CodeCache {*this} {
|
||||
, IRCaptureCache {this} {
|
||||
if (Config.CacheObjectCodeCompilation() != FEXCore::Config::ConfigObjectCodeHandler::CONFIG_NONE) {
|
||||
CodeObjectCacheService = fextl::make_unique<FEXCore::CodeSerialize::CodeObjectSerializeService>(this);
|
||||
}
|
||||
if (!Config.Is64BitMode()) {
|
||||
// When operating in 32-bit mode, the virtual memory we care about is only the lower 32-bits.
|
||||
Config.VirtualMemSize = 1ULL << 32;
|
||||
@@ -102,6 +105,14 @@ ContextImpl::ContextImpl(const FEXCore::HostFeatures& Features)
|
||||
UpdateAtomicTSOEmulationConfig();
|
||||
}
|
||||
|
||||
ContextImpl::~ContextImpl() {
|
||||
{
|
||||
if (CodeObjectCacheService) {
|
||||
CodeObjectCacheService->Shutdown();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct GetFrameBlockInfoResult {
|
||||
const CPU::CPUBackend::JITCodeHeader* InlineHeader;
|
||||
const CPU::CPUBackend::JITCodeTail* InlineTail;
|
||||
@@ -128,11 +139,6 @@ bool ContextImpl::IsCurrentBlockSingleInst(FEXCore::Core::InternalThreadState* T
|
||||
return InlineTail && InlineTail->SingleInst;
|
||||
}
|
||||
|
||||
uint64_t ContextImpl::GetGuestBlockEntry(FEXCore::Core::InternalThreadState* Thread) {
|
||||
auto [_, InlineTail] = GetFrameBlockInfo(Thread->CurrentFrame);
|
||||
return InlineTail ? InlineTail->RIP : 0;
|
||||
}
|
||||
|
||||
uint64_t ContextImpl::RestoreRIPFromHostPC(FEXCore::Core::InternalThreadState* Thread, uint64_t HostPC) {
|
||||
const auto Frame = Thread->CurrentFrame;
|
||||
const uint64_t BlockBegin = Frame->State.InlineJITBlockHeader;
|
||||
@@ -343,9 +349,36 @@ bool ContextImpl::InitCore() {
|
||||
Dispatcher = FEXCore::CPU::Dispatcher::Create(this);
|
||||
|
||||
// Set up the SignalDelegator config since core is initialized.
|
||||
SignalDelegation->SetConfig(Dispatcher->MakeSignalDelegatorConfig());
|
||||
FEXCore::SignalDelegator::SignalDelegatorConfig SignalConfig {
|
||||
.DispatcherBegin = Dispatcher->Start,
|
||||
.DispatcherEnd = Dispatcher->End,
|
||||
|
||||
#if defined(_WIN32) && !defined(ARCHITECTURE_arm64ec)
|
||||
.AbsoluteLoopTopAddress = Dispatcher->AbsoluteLoopTopAddress,
|
||||
.AbsoluteLoopTopAddressFillSRA = Dispatcher->AbsoluteLoopTopAddressFillSRA,
|
||||
.SignalHandlerReturnAddress = Dispatcher->SignalHandlerReturnAddress,
|
||||
.SignalHandlerReturnAddressRT = Dispatcher->SignalHandlerReturnAddressRT,
|
||||
|
||||
.PauseReturnInstruction = Dispatcher->PauseReturnInstruction,
|
||||
.ThreadPauseHandlerAddressSpillSRA = Dispatcher->ThreadPauseHandlerAddressSpillSRA,
|
||||
.ThreadPauseHandlerAddress = Dispatcher->ThreadPauseHandlerAddress,
|
||||
|
||||
// Stop handlers.
|
||||
.ThreadStopHandlerAddressSpillSRA = Dispatcher->ThreadStopHandlerAddressSpillSRA,
|
||||
.ThreadStopHandlerAddress = Dispatcher->ThreadStopHandlerAddress,
|
||||
|
||||
// SRA information.
|
||||
.SRAGPRCount = Dispatcher->GetSRAGPRCount(),
|
||||
.SRAFPRCount = Dispatcher->GetSRAFPRCount(),
|
||||
};
|
||||
|
||||
Dispatcher->GetSRAGPRMapping(SignalConfig.SRAGPRMapping);
|
||||
Dispatcher->GetSRAFPRMapping(SignalConfig.SRAFPRMapping);
|
||||
|
||||
// Give this configuration to the SignalDelegator.
|
||||
SignalDelegation->SetConfig(SignalConfig);
|
||||
|
||||
#ifndef _WIN32
|
||||
#elif !defined(_M_ARM_64EC)
|
||||
// WOW64 always needs the interrupt fault check to be enabled.
|
||||
Config.NeedsPendingInterruptFaultCheck = true;
|
||||
#endif
|
||||
@@ -363,26 +396,27 @@ void ContextImpl::HandleCallback(FEXCore::Core::InternalThreadState* Thread, uin
|
||||
}
|
||||
|
||||
void ContextImpl::ExecuteThread(FEXCore::Core::InternalThreadState* Thread) {
|
||||
// Update the thread pointer for Thunk return to the latest.
|
||||
Thread->CurrentFrame->Pointers.ThunkCallbackRet = SignalDelegation->GetThunkCallbackRET();
|
||||
|
||||
Dispatcher->ExecuteDispatch(Thread->CurrentFrame);
|
||||
|
||||
if (CodeObjectCacheService) {
|
||||
// Ensure the Code Object Serialization service has fully serialized this thread's data before clearing the cache
|
||||
// Use the thread's object cache ref counter for this
|
||||
CodeSerialize::CodeObjectSerializeService::WaitForEmptyJobQueue(&Thread->ObjectCacheRefCounter);
|
||||
}
|
||||
|
||||
// If it is the parent thread that died then just leave
|
||||
// TODO: This doesn't make sense when the parent thread doesn't outlive its children
|
||||
FEX_TODO("This doesn't make sense when the parent thread doesn't outlive its children");
|
||||
}
|
||||
|
||||
void ContextImpl::InitializeCompiler(FEXCore::Core::InternalThreadState* Thread) {
|
||||
Thread->OpDispatcher = fextl::make_unique<FEXCore::IR::OpDispatchBuilder>(this);
|
||||
Thread->OpDispatcher->SetMultiblock(Config.Multiblock);
|
||||
Thread->LookupCache = fextl::make_unique<FEXCore::LookupCache>(this);
|
||||
Thread->FrontendDecoder = fextl::make_unique<FEXCore::Frontend::Decoder>(Thread);
|
||||
Thread->FrontendDecoder = fextl::make_unique<FEXCore::Frontend::Decoder>(this);
|
||||
Thread->PassManager = fextl::make_unique<FEXCore::IR::PassManager>();
|
||||
|
||||
Thread->CurrentFrame->State.L1Pointer = Thread->LookupCache->GetL1Pointer();
|
||||
Thread->CurrentFrame->State.L1Mask = Thread->LookupCache->GetScaledL1PointerMask();
|
||||
|
||||
Thread->CurrentFrame->Pointers.L2Pointer = Thread->LookupCache->GetPagePointer();
|
||||
Thread->CurrentFrame->Pointers.Common.L1Pointer = Thread->LookupCache->GetL1Pointer();
|
||||
Thread->CurrentFrame->Pointers.Common.L2Pointer = Thread->LookupCache->GetPagePointer();
|
||||
|
||||
Dispatcher->InitThreadPointers(Thread);
|
||||
|
||||
@@ -403,7 +437,6 @@ ContextImpl::CreateThread(uint64_t InitialRIP, uint64_t StackPointer, const FEXC
|
||||
FEXCore::Core::InternalThreadState* Thread = new FEXCore::Core::InternalThreadState {
|
||||
.CTX = this,
|
||||
};
|
||||
FEXCore::Allocator::VirtualName("FEXMem_ThreadState", Thread, sizeof(*Thread));
|
||||
|
||||
Thread->CurrentFrame->State.gregs[X86State::REG_RSP] = StackPointer;
|
||||
Thread->CurrentFrame->State.rip = InitialRIP;
|
||||
@@ -440,10 +473,6 @@ void ContextImpl::UnlockAfterFork(FEXCore::Core::InternalThreadState* LiveThread
|
||||
|
||||
Profiler::PostForkAction(Child);
|
||||
if (Child) {
|
||||
if (CodeMapWriter) {
|
||||
CodeMapWriter->ResetAfterFork();
|
||||
}
|
||||
|
||||
CodeInvalidationMutex.StealAndDropActiveLocks();
|
||||
if (Config.StrictInProcessSplitLocks) {
|
||||
StrictSplitLockMutex = 0;
|
||||
@@ -466,29 +495,28 @@ void ContextImpl::LockBeforeFork(FEXCore::Core::InternalThreadState* Thread) {
|
||||
}
|
||||
#endif
|
||||
|
||||
void ContextImpl::OnCodeBufferAllocated(const fextl::shared_ptr<CPU::CodeBuffer>& Buffer) {
|
||||
void ContextImpl::OnCodeBufferAllocated(CPU::CodeBuffer& Buffer) {
|
||||
if (Config.GlobalJITNaming()) {
|
||||
Symbols.RegisterJITSpace(Buffer->Ptr, Buffer->Size);
|
||||
}
|
||||
|
||||
{
|
||||
std::scoped_lock lk {CodeBufferListLock};
|
||||
CodeBufferList.emplace_back(Buffer);
|
||||
Symbols.RegisterJITSpace(Buffer.Ptr, Buffer.Size);
|
||||
}
|
||||
}
|
||||
|
||||
void ContextImpl::ClearCodeCache(FEXCore::Core::InternalThreadState* Thread, bool NewCodeBuffer) {
|
||||
FEXCORE_PROFILE_INSTANT("ClearCodeCache");
|
||||
|
||||
if (CodeObjectCacheService) {
|
||||
// Ensure the Code Object Serialization service has fully serialized this thread's data before clearing the cache
|
||||
// Use the thread's object cache ref counter for this
|
||||
CodeSerialize::CodeObjectSerializeService::WaitForEmptyJobQueue(&Thread->ObjectCacheRefCounter);
|
||||
}
|
||||
|
||||
if (NewCodeBuffer) {
|
||||
// Allocate new CodeBuffer + L3 LookupCache and clear L1+L2 caches
|
||||
Thread->CPUBackend->ClearCache();
|
||||
} else {
|
||||
// Clear L1+L2 cache of this thread, and clear L3 cache across any threads using it
|
||||
auto lk = Thread->LookupCache->AcquireWriteLock();
|
||||
Thread->LookupCache->ClearCache(lk);
|
||||
Thread->LookupCache->ClearCache();
|
||||
}
|
||||
Allocator::VirtualDontNeed(Thread->CallRetStackBase, FEXCore::Core::InternalThreadState::CALLRET_STACK_SIZE);
|
||||
}
|
||||
|
||||
static void IRDumper(FEXCore::Core::InternalThreadState* Thread, IR::IREmitter* IREmitter, uint64_t GuestRIP) {
|
||||
@@ -517,7 +545,7 @@ ContextImpl::GenerateIR(FEXCore::Core::InternalThreadState* Thread, uint64_t Gue
|
||||
if (Handler != CustomIRHandlers.end()) {
|
||||
TotalInstructions = 1;
|
||||
TotalInstructionsLength = 1;
|
||||
Handler->second.Handler(GuestRIP, Thread->OpDispatcher.get());
|
||||
std::get<0>(Handler->second)(GuestRIP, Thread->OpDispatcher.get());
|
||||
HasCustomIR = true;
|
||||
}
|
||||
}
|
||||
@@ -529,15 +557,19 @@ ContextImpl::GenerateIR(FEXCore::Core::InternalThreadState* Thread, uint64_t Gue
|
||||
bool HadDispatchError {false};
|
||||
bool HadInvalidInst {false};
|
||||
|
||||
Thread->FrontendDecoder->DecodeInstructionsAtEntry(Thread, GuestCode, GuestRIP, MaxInst);
|
||||
Thread->FrontendDecoder->DecodeInstructionsAtEntry(GuestCode, GuestRIP, MaxInst,
|
||||
[Thread](uint64_t BlockEntry, uint64_t Start, uint64_t Length) {
|
||||
if (Thread->LookupCache->AddBlockExecutableRange(BlockEntry, Start, Length)) {
|
||||
static_cast<ContextImpl*>(Thread->CTX)->SyscallHandler->MarkGuestExecutableRange(Thread, Start, Length);
|
||||
}
|
||||
});
|
||||
|
||||
auto BlockInfo = Thread->FrontendDecoder->GetDecodedBlockInfo();
|
||||
auto CodeBlocks = &BlockInfo->Blocks;
|
||||
|
||||
Thread->OpDispatcher->BeginFunction(GuestRIP, CodeBlocks, BlockInfo->TotalInstructionCount, BlockInfo->Is64BitMode,
|
||||
AreMonoHacksActive() && MonoBackpatcherBlock.load(std::memory_order_relaxed) == GuestRIP);
|
||||
Thread->OpDispatcher->BeginFunction(GuestRIP, CodeBlocks, BlockInfo->TotalInstructionCount);
|
||||
|
||||
const auto GPRSize = Thread->OpDispatcher->GetGPROpSize();
|
||||
const auto GPRSize = GetGPROpSize();
|
||||
|
||||
for (size_t j = 0; j < CodeBlocks->size(); ++j) {
|
||||
const FEXCore::Frontend::Decoder::DecodedBlocks& Block = CodeBlocks->at(j);
|
||||
@@ -563,7 +595,7 @@ ContextImpl::GenerateIR(FEXCore::Core::InternalThreadState* Thread, uint64_t Gue
|
||||
|
||||
if (InstsInBlock == 0) {
|
||||
// Special case for an empty instruction block.
|
||||
Thread->OpDispatcher->ExitFunction(Thread->OpDispatcher->_InlineEntrypointOffset(GPRSize, Block.Entry - GuestRIP));
|
||||
Thread->OpDispatcher->ExitFunction(Thread->OpDispatcher->_EntrypointOffset(GPRSize, Block.Entry - GuestRIP));
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < InstsInBlock; ++i) {
|
||||
@@ -593,12 +625,11 @@ ContextImpl::GenerateIR(FEXCore::Core::InternalThreadState* Thread, uint64_t Gue
|
||||
Thread->OpDispatcher->_GuestOpcode(InstAddress - GuestRIP);
|
||||
}
|
||||
|
||||
if (Config.SMCChecks == FEXCore::Config::CONFIG_SMC_FULL || Block.ForceFullSMCDetection) {
|
||||
auto ExistingCodePtr = reinterpret_cast<uint8_t*>(Block.Entry + BlockInstructionsLength);
|
||||
auto InstAddressReg = Thread->OpDispatcher->_EntrypointOffset(GPRSize, InstAddress - GuestRIP);
|
||||
std::array<uint8_t, 0x10> CodeOriginal;
|
||||
memcpy(CodeOriginal.data(), ExistingCodePtr, DecodedInfo->InstSize);
|
||||
auto CodeChanged = Thread->OpDispatcher->_ValidateCode(CodeOriginal, InstAddressReg, DecodedInfo->InstSize);
|
||||
if (Config.SMCChecks == FEXCore::Config::CONFIG_SMC_FULL) {
|
||||
auto ExistingCodePtr = reinterpret_cast<uint64_t*>(Block.Entry + BlockInstructionsLength);
|
||||
|
||||
auto CodeChanged = Thread->OpDispatcher->_ValidateCode(ExistingCodePtr[0], ExistingCodePtr[1],
|
||||
(uintptr_t)ExistingCodePtr - GuestRIP, DecodedInfo->InstSize);
|
||||
|
||||
auto InvalidateCodeCond = Thread->OpDispatcher->CondJump(CodeChanged);
|
||||
|
||||
@@ -608,7 +639,7 @@ ContextImpl::GenerateIR(FEXCore::Core::InternalThreadState* Thread, uint64_t Gue
|
||||
|
||||
Thread->OpDispatcher->SetCurrentCodeBlock(CodeWasChangedBlock);
|
||||
Thread->OpDispatcher->_ThreadRemoveCodeEntry();
|
||||
Thread->OpDispatcher->ExitFunction(Thread->OpDispatcher->_InlineEntrypointOffset(GPRSize, InstAddress - GuestRIP));
|
||||
Thread->OpDispatcher->ExitFunction(Thread->OpDispatcher->_EntrypointOffset(GPRSize, InstAddress - GuestRIP));
|
||||
|
||||
auto NextOpBlock = Thread->OpDispatcher->CreateNewCodeBlockAfter(CurrentBlock);
|
||||
|
||||
@@ -616,21 +647,15 @@ ContextImpl::GenerateIR(FEXCore::Core::InternalThreadState* Thread, uint64_t Gue
|
||||
Thread->OpDispatcher->SetCurrentCodeBlock(NextOpBlock);
|
||||
}
|
||||
|
||||
if (TableInfo && TableInfo->OpcodeDispatcher.OpDispatch) {
|
||||
auto Fn = TableInfo->OpcodeDispatcher.OpDispatch;
|
||||
if (TableInfo && TableInfo->OpcodeDispatcher) {
|
||||
auto Fn = TableInfo->OpcodeDispatcher;
|
||||
Thread->OpDispatcher->ResetHandledLock();
|
||||
Thread->OpDispatcher->ResetDecodeFailure();
|
||||
IR::ForceTSOMode ForceTSO = IR::ForceTSOMode::NoOverride;
|
||||
if (BlockInForceTSOValidRange) {
|
||||
if (InstForceTSOIt != ForceTSOInstructions.end() && *InstForceTSOIt == InstAddress) {
|
||||
ForceTSO = IR::ForceTSOMode::ForceEnabled;
|
||||
} else {
|
||||
ForceTSO = IR::ForceTSOMode::ForceDisabled;
|
||||
}
|
||||
} else if (DecodedInfo->Flags & X86Tables::DecodeFlags::FLAG_FORCE_TSO) {
|
||||
ForceTSO = IR::ForceTSOMode::ForceEnabled;
|
||||
}
|
||||
|
||||
IR::ForceTSOMode ForceTSO =
|
||||
BlockInForceTSOValidRange ?
|
||||
(InstForceTSOIt != ForceTSOInstructions.end() && *InstForceTSOIt == InstAddress ? IR::ForceTSOMode::ForceEnabled :
|
||||
IR::ForceTSOMode::ForceDisabled) :
|
||||
IR::ForceTSOMode::NoOverride;
|
||||
Thread->OpDispatcher->SetForceTSO(ForceTSO);
|
||||
std::invoke(Fn, Thread->OpDispatcher, DecodedInfo);
|
||||
if (Thread->OpDispatcher->HadDecodeFailure()) {
|
||||
@@ -658,12 +683,7 @@ ContextImpl::GenerateIR(FEXCore::Core::InternalThreadState* Thread, uint64_t Gue
|
||||
LogMan::Msg::EFmt("Invalid or Unknown instruction: {} 0x{:x}", TableInfo->Name ?: "UND", Block.Entry - GuestRIP);
|
||||
}
|
||||
|
||||
if (Block.BlockStatus == Frontend::Decoder::DecodedBlockStatus::INVALID_INST ||
|
||||
Block.BlockStatus == Frontend::Decoder::DecodedBlockStatus::BAD_RELOCATION) {
|
||||
Thread->OpDispatcher->InvalidOp(DecodedInfo);
|
||||
} else {
|
||||
Thread->OpDispatcher->NoExecOp(DecodedInfo);
|
||||
}
|
||||
Thread->OpDispatcher->InvalidOp(DecodedInfo);
|
||||
}
|
||||
|
||||
HadInvalidInst = true;
|
||||
@@ -681,8 +701,7 @@ ContextImpl::GenerateIR(FEXCore::Core::InternalThreadState* Thread, uint64_t Gue
|
||||
|
||||
if (NeedsBlockEnd) {
|
||||
// We had some instructions. Early exit
|
||||
Thread->OpDispatcher->ExitFunction(
|
||||
Thread->OpDispatcher->_InlineEntrypointOffset(GPRSize, Block.Entry + BlockInstructionsLength - GuestRIP));
|
||||
Thread->OpDispatcher->ExitFunction(Thread->OpDispatcher->_EntrypointOffset(GPRSize, Block.Entry + BlockInstructionsLength - GuestRIP));
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -720,24 +739,37 @@ ContextImpl::GenerateIR(FEXCore::Core::InternalThreadState* Thread, uint64_t Gue
|
||||
.TotalInstructionsLength = TotalInstructionsLength,
|
||||
.StartAddr = Thread->FrontendDecoder->DecodedMinAddress,
|
||||
.Length = Thread->FrontendDecoder->DecodedMaxAddress - Thread->FrontendDecoder->DecodedMinAddress,
|
||||
.NeedsAddGuestCodeRanges = !HasCustomIR,
|
||||
};
|
||||
}
|
||||
|
||||
ContextImpl::CompileCodeResult ContextImpl::CompileCode(FEXCore::Core::InternalThreadState* Thread, uint64_t GuestRIP, uint64_t MaxInst) {
|
||||
// JIT Code object cache lookup
|
||||
if (CodeObjectCacheService) {
|
||||
auto CodeCacheEntry = CodeObjectCacheService->FetchCodeObjectFromCache(GuestRIP);
|
||||
if (CodeCacheEntry) {
|
||||
auto CompiledCode = Thread->CPUBackend->RelocateJITObjectCode(GuestRIP, CodeCacheEntry);
|
||||
if (CompiledCode) {
|
||||
return {
|
||||
.CompiledCode = CompiledCode,
|
||||
.DebugData = nullptr, // nullptr here ensures that code serialization doesn't occur on from cache read
|
||||
.StartAddr = 0, // Unused
|
||||
.Length = 0, // Unused
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (SourcecodeResolver && Config.GDBSymbols()) {
|
||||
auto MappedSection = SyscallHandler->LookupExecutableFileSection(Thread, GuestRIP);
|
||||
if (MappedSection) {
|
||||
MappedSection->FileInfo.SourcecodeMap =
|
||||
SourcecodeResolver->GenerateMap(MappedSection->FileInfo.Filename, CodeMap::GetBaseFilename(MappedSection->FileInfo, false));
|
||||
auto AOTIRCacheEntry = SyscallHandler->LookupAOTIRCacheEntry(Thread, GuestRIP);
|
||||
if (AOTIRCacheEntry.Entry && !AOTIRCacheEntry.Entry->ContainsCode) {
|
||||
AOTIRCacheEntry.Entry->SourcecodeMap = SourcecodeResolver->GenerateMap(AOTIRCacheEntry.Entry->Filename, AOTIRCacheEntry.Entry->FileId);
|
||||
}
|
||||
}
|
||||
|
||||
// Generate IR + Meta Info
|
||||
auto [IRView, TotalInstructions, TotalInstructionsLength, StartAddr, Length, NeedsAddGuestCodeRanges] =
|
||||
GenerateIR(Thread, GuestRIP, Config.GDBSymbols(), MaxInst);
|
||||
auto [IRView, TotalInstructions, TotalInstructionsLength, StartAddr, Length] = GenerateIR(Thread, GuestRIP, Config.GDBSymbols(), MaxInst);
|
||||
if (!IRView) {
|
||||
return {{}, nullptr, 0, 0, false};
|
||||
return {nullptr, nullptr, 0, 0};
|
||||
}
|
||||
|
||||
// Attempt to get the CPU backend to compile this code
|
||||
@@ -746,13 +778,9 @@ ContextImpl::CompileCodeResult ContextImpl::CompileCode(FEXCore::Core::InternalT
|
||||
// but this would increase lock contention. Redundant frontend runs aren't
|
||||
// as expensive and are easily reverted.
|
||||
if (MaxInst != 1) {
|
||||
if (auto Block = Thread->LookupCache->FindBlock(Thread, GuestRIP)) {
|
||||
if (auto Block = Thread->LookupCache->FindBlock(GuestRIP)) {
|
||||
Thread->OpDispatcher->DelayedDisownBuffer();
|
||||
return {.CompiledCode = {.BlockBegin = reinterpret_cast<uint8_t*>(Block), .EntryPoints = {{GuestRIP, reinterpret_cast<uint8_t*>(Block)}}},
|
||||
.DebugData = nullptr,
|
||||
.StartAddr = 0,
|
||||
.Length = 0,
|
||||
.NeedsAddGuestCodeRanges = false};
|
||||
return {.CompiledCode = reinterpret_cast<uint8_t*>(Block), .DebugData = nullptr, .StartAddr = 0, .Length = 0};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -767,11 +795,13 @@ ContextImpl::CompileCodeResult ContextImpl::CompileCode(FEXCore::Core::InternalT
|
||||
Thread->OpDispatcher->DelayedDisownBuffer();
|
||||
|
||||
return {
|
||||
.CompiledCode = std::move(CompiledCode),
|
||||
// FEX currently throws away the CPUBackend::CompiledCode object other than the entrypoint
|
||||
// In the future with code caching getting wired up, we will pass the rest of the data forward.
|
||||
// TODO: Pass the data forward when code caching is wired up to this.
|
||||
.CompiledCode = CompiledCode.BlockEntry,
|
||||
.DebugData = std::move(DebugData),
|
||||
.StartAddr = StartAddr,
|
||||
.Length = Length,
|
||||
.NeedsAddGuestCodeRanges = NeedsAddGuestCodeRanges,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -787,15 +817,11 @@ uintptr_t ContextImpl::CompileBlock(FEXCore::Core::CpuStateFrame* Frame, uint64_
|
||||
|
||||
// Is the code in the cache?
|
||||
// The backends only check L1 and L2, not L3
|
||||
if (auto HostCode = Thread->LookupCache->FindBlock(Thread, GuestRIP)) {
|
||||
if (auto HostCode = Thread->LookupCache->FindBlock(GuestRIP)) {
|
||||
return HostCode;
|
||||
}
|
||||
|
||||
// Accumulate a JIT count now, as even if another thread raced us, it should count as a compile.
|
||||
FEXCORE_PROFILE_INSTANT_INCREMENT(Thread, AccumulatedJITCount, 1);
|
||||
|
||||
auto [CompiledCode, DebugData, StartAddr, Length, NeedsAddGuestCodeRanges] = CompileCode(Thread, GuestRIP, MaxInst);
|
||||
auto CodePtr = CompiledCode.EntryPoints[GuestRIP];
|
||||
auto [CodePtr, DebugData, StartAddr, Length] = CompileCode(Thread, GuestRIP, MaxInst);
|
||||
if (CodePtr == nullptr) {
|
||||
return 0;
|
||||
} else if (!DebugData) {
|
||||
@@ -805,75 +831,58 @@ uintptr_t ContextImpl::CompileBlock(FEXCore::Core::CpuStateFrame* Frame, uint64_
|
||||
|
||||
// The core managed to compile the code.
|
||||
if (Config.BlockJITNaming()) {
|
||||
auto FragmentBasePtr = CompiledCode.BlockBegin;
|
||||
auto FragmentBasePtr = reinterpret_cast<uint8_t*>(CodePtr);
|
||||
|
||||
auto GuestRIPLookup = SyscallHandler->LookupExecutableFileSection(Thread, GuestRIP);
|
||||
if (DebugData) {
|
||||
auto GuestRIPLookup = SyscallHandler->LookupAOTIRCacheEntry(Thread, GuestRIP);
|
||||
|
||||
if (DebugData->Subblocks.size()) {
|
||||
for (auto& Subblock : DebugData->Subblocks) {
|
||||
auto BlockBasePtr = FragmentBasePtr + Subblock.HostCodeOffset;
|
||||
if (GuestRIPLookup) {
|
||||
Symbols.Register(Thread->SymbolBuffer.get(), BlockBasePtr, CompiledCode.Size, GuestRIPLookup->FileInfo.Filename,
|
||||
GuestRIP - GuestRIPLookup->FileStartVA);
|
||||
} else {
|
||||
Symbols.Register(Thread->SymbolBuffer.get(), BlockBasePtr, GuestRIP, Subblock.HostCodeSize);
|
||||
if (DebugData->Subblocks.size()) {
|
||||
for (auto& Subblock : DebugData->Subblocks) {
|
||||
auto BlockBasePtr = FragmentBasePtr + Subblock.HostCodeOffset;
|
||||
if (GuestRIPLookup.Entry) {
|
||||
Symbols.Register(Thread->SymbolBuffer.get(), BlockBasePtr, DebugData->HostCodeSize, GuestRIPLookup.Entry->Filename,
|
||||
GuestRIP - GuestRIPLookup.VAFileStart);
|
||||
} else {
|
||||
Symbols.Register(Thread->SymbolBuffer.get(), BlockBasePtr, GuestRIP, Subblock.HostCodeSize);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if (GuestRIPLookup) {
|
||||
Symbols.Register(Thread->SymbolBuffer.get(), FragmentBasePtr, CompiledCode.Size, GuestRIPLookup->FileInfo.Filename,
|
||||
GuestRIP - GuestRIPLookup->FileStartVA);
|
||||
} else {
|
||||
Symbols.Register(Thread->SymbolBuffer.get(), FragmentBasePtr, GuestRIP, CompiledCode.Size);
|
||||
if (GuestRIPLookup.Entry) {
|
||||
Symbols.Register(Thread->SymbolBuffer.get(), FragmentBasePtr, DebugData->HostCodeSize, GuestRIPLookup.Entry->Filename,
|
||||
GuestRIP - GuestRIPLookup.VAFileStart);
|
||||
} else {
|
||||
Symbols.Register(Thread->SymbolBuffer.get(), FragmentBasePtr, GuestRIP, DebugData->HostCodeSize);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (Config.LibraryJITNaming() || Config.GDBSymbols()) {
|
||||
auto MappedSection = SyscallHandler->LookupExecutableFileSection(Thread, GuestRIP);
|
||||
if (MappedSection) {
|
||||
if (Config.LibraryJITNaming()) {
|
||||
Symbols.RegisterNamedRegion(Thread->SymbolBuffer.get(), CodePtr, DebugData->HostCodeSize, MappedSection->FileInfo.Filename);
|
||||
}
|
||||
|
||||
if (Config.GDBSymbols()) {
|
||||
GDBJITRegister(MappedSection->FileInfo, MappedSection->FileStartVA, GuestRIP, (uintptr_t)CodePtr, *DebugData);
|
||||
}
|
||||
}
|
||||
// Tell the object cache service to serialize the code if enabled
|
||||
if (CodeObjectCacheService && Config.CacheObjectCodeCompilation == FEXCore::Config::ConfigObjectCodeHandler::CONFIG_READWRITE && DebugData) {
|
||||
CodeObjectCacheService->AsyncAddSerializationJob(
|
||||
fextl::make_unique<CodeSerialize::AsyncJobHandler::SerializationJobData>(CodeSerialize::AsyncJobHandler::SerializationJobData {
|
||||
.GuestRIP = GuestRIP,
|
||||
.GuestCodeLength = Length,
|
||||
.GuestCodeHash = 0,
|
||||
.HostCodeBegin = CodePtr,
|
||||
.HostCodeLength = DebugData->HostCodeSize,
|
||||
.HostCodeHash = 0,
|
||||
.ThreadJobRefCount = &Thread->ObjectCacheRefCounter,
|
||||
.Relocations = std::move(*DebugData->Relocations),
|
||||
}));
|
||||
}
|
||||
|
||||
// Clear any relocations that might have been generated
|
||||
if (!CodeCache.IsGeneratingCache) {
|
||||
Thread->CPUBackend->ClearRelocations();
|
||||
}
|
||||
Thread->CPUBackend->ClearRelocations();
|
||||
|
||||
fextl::vector<uint64_t> CodePages;
|
||||
|
||||
if (NeedsAddGuestCodeRanges) {
|
||||
// Track in the guest to host map all entrypoints for all pages the compiled block touches, if any page didn't previously
|
||||
// contain code, inform the frontend so it can setup SMC detection.
|
||||
auto BlockInfo = Thread->FrontendDecoder->GetDecodedBlockInfo();
|
||||
CodePages.reserve(BlockInfo->CodePages.size());
|
||||
CodePages.insert(CodePages.end(), BlockInfo->CodePages.begin(), BlockInfo->CodePages.end());
|
||||
for (auto CodePage : BlockInfo->CodePages) {
|
||||
if (Thread->LookupCache->AddBlockExecutableRange(Thread, BlockInfo->EntryPoints, CodePage, FEXCore::Utils::FEX_PAGE_SIZE)) {
|
||||
SyscallHandler->MarkGuestExecutableRange(Thread, CodePage, FEXCore::Utils::FEX_PAGE_SIZE);
|
||||
}
|
||||
}
|
||||
if (IRCaptureCache.PostCompileCode(Thread, CodePtr, GuestRIP, StartAddr, Length, {}, DebugData.get(), false)) {
|
||||
// Early exit
|
||||
return (uintptr_t)CodePtr;
|
||||
}
|
||||
|
||||
// Insert to lookup cache
|
||||
|
||||
for (auto [GuestAddr, HostAddr] : CompiledCode.EntryPoints) {
|
||||
Thread->LookupCache->AddBlockMapping(Thread, GuestAddr, CodePages, HostAddr);
|
||||
}
|
||||
|
||||
if (CodeMapWriter) {
|
||||
auto Region = SyscallHandler->LookupExecutableFileSection(Thread, GuestRIP);
|
||||
if (Region && Region->FileStartVA != 0) {
|
||||
CodeMapWriter->AppendBlock(*Region, GuestRIP);
|
||||
}
|
||||
}
|
||||
// Pages containing this block are added via AddBlockExecutableRange before each page gets accessed in the frontend
|
||||
Thread->LookupCache->AddBlockMapping(GuestRIP, CodePtr);
|
||||
|
||||
return (uintptr_t)CodePtr;
|
||||
}
|
||||
@@ -887,8 +896,7 @@ uintptr_t ContextImpl::CompileSingleStep(FEXCore::Core::CpuStateFrame* Frame, ui
|
||||
// Invalidate might take a unique lock on this, to guarantee that during invalidation no code gets compiled
|
||||
auto lk = GuardSignalDeferringSection<std::shared_lock>(CodeInvalidationMutex, Thread);
|
||||
|
||||
auto [CompiledCode, DebugData, StartAddr, Length, _] = CompileCode(Thread, GuestRIP, 1);
|
||||
auto CodePtr = CompiledCode.EntryPoints[GuestRIP];
|
||||
auto [CodePtr, DebugData, StartAddr, Length] = CompileCode(Thread, GuestRIP, 1);
|
||||
if (CodePtr == nullptr) {
|
||||
return 0;
|
||||
}
|
||||
@@ -899,39 +907,46 @@ uintptr_t ContextImpl::CompileSingleStep(FEXCore::Core::CpuStateFrame* Frame, ui
|
||||
return (uintptr_t)CodePtr;
|
||||
}
|
||||
|
||||
void ContextImpl::InvalidateCodeBuffersCodeRange(uint64_t Start, uint64_t Length) {
|
||||
FEXCORE_PROFILE_SCOPED("InvalidateCodeBuffersCodeRange");
|
||||
static void InvalidateGuestThreadCodeRange(FEXCore::Core::InternalThreadState* Thread, uint64_t Start, uint64_t Length) {
|
||||
auto lk = Thread->LookupCache->AcquireLock();
|
||||
|
||||
LOGMAN_THROW_A_FMT(CodeInvalidationMutex.try_lock() == false, "CodeInvalidationMutex needs to be unique_locked here");
|
||||
std::scoped_lock lk {CodeBufferListLock};
|
||||
auto it = CodeBufferList.begin();
|
||||
while (it != CodeBufferList.end()) {
|
||||
if (auto Strong = it->lock()) {
|
||||
Strong->LookupCache->InvalidateRange(Start, Length);
|
||||
it++;
|
||||
} else {
|
||||
it = CodeBufferList.erase(it);
|
||||
auto lower = Thread->LookupCache->CodePages.lower_bound(Start >> 12);
|
||||
auto upper = Thread->LookupCache->CodePages.upper_bound((Start + Length - 1) >> 12);
|
||||
|
||||
for (auto it = lower; it != upper; it++) {
|
||||
for (auto Address : it->second) {
|
||||
ContextImpl::ThreadRemoveCodeEntry(Thread, Address);
|
||||
}
|
||||
it->second.clear();
|
||||
}
|
||||
}
|
||||
|
||||
void ContextImpl::InvalidateGuestCodeRange(FEXCore::Core::InternalThreadState* Thread, uint64_t Start, uint64_t Length) {
|
||||
InvalidateGuestThreadCodeRange(Thread, Start, Length);
|
||||
}
|
||||
|
||||
void ContextImpl::MarkMemoryShared(FEXCore::Core::InternalThreadState* Thread) {
|
||||
if (!Thread) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!IsMemoryShared) {
|
||||
IsMemoryShared = true;
|
||||
UpdateAtomicTSOEmulationConfig();
|
||||
|
||||
if (Config.TSOAutoMigration) {
|
||||
// Only the lookup cache is cleared here, so that old code can keep running until next compilation.
|
||||
// This will leak previously compiled blocks until the CodeBuffer is cleared for some other reason.
|
||||
Thread->LookupCache->ClearCache();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void ContextImpl::InvalidateThreadCachedCodeRange(FEXCore::Core::InternalThreadState* Thread, uint64_t Start, uint64_t Length) {
|
||||
LOGMAN_THROW_A_FMT(CodeInvalidationMutex.try_lock() == false, "CodeInvalidationMutex needs to be unique_locked here");
|
||||
void ContextImpl::ThreadRemoveCodeEntry(FEXCore::Core::InternalThreadState* Thread, uint64_t GuestRIP) {
|
||||
LogMan::Throw::AFmt(static_cast<ContextImpl*>(Thread->CTX)->CodeInvalidationMutex.try_lock() == false, "CodeInvalidationMutex needs to "
|
||||
"be unique_locked here");
|
||||
|
||||
// Ensures now-modified mappings aren't cached as being in their previous non-executable state.
|
||||
// Accessing FrontendDecoder is safe as the thread's code invalidation mutex must be locked here.
|
||||
Thread->FrontendDecoder->ResetExecutableRangeCache();
|
||||
|
||||
if (Thread->LookupCache->InvalidateCacheRange(Start, Length)) {
|
||||
FEXCORE_PROFILE_SCOPED("InvalidateCallRet");
|
||||
|
||||
// This may cause access violations in the thread on Windows as zeroing is not atomic, this is handled by the frontend
|
||||
Allocator::VirtualDontNeed(Thread->CallRetStackBase, FEXCore::Core::InternalThreadState::CALLRET_STACK_SIZE);
|
||||
}
|
||||
}
|
||||
|
||||
void ContextImpl::ThreadRemoveCodeEntryFromJit(FEXCore::Core::CpuStateFrame* Frame, uint64_t GuestRIP) {
|
||||
static_cast<ContextImpl*>(Frame->Thread->CTX)->SyscallHandler->InvalidateGuestCodeRange(Frame->Thread, GuestRIP, 1);
|
||||
Thread->LookupCache->Erase(Thread->CurrentFrame, GuestRIP);
|
||||
}
|
||||
|
||||
std::optional<CustomIRResult>
|
||||
@@ -940,7 +955,7 @@ ContextImpl::AddCustomIREntrypoint(uintptr_t Entrypoint, CustomIREntrypointHandl
|
||||
|
||||
std::unique_lock lk(CustomIRMutex);
|
||||
|
||||
auto InsertedIterator = CustomIRHandlers.emplace(Entrypoint, CustomIRHandlerEntry {Handler, Creator, Data});
|
||||
auto InsertedIterator = CustomIRHandlers.emplace(Entrypoint, std::tuple(Handler, Creator, Data));
|
||||
HasCustomIRHandlers = true;
|
||||
|
||||
if (!InsertedIterator.second) {
|
||||
@@ -964,22 +979,21 @@ void ContextImpl::AddThunkTrampolineIRHandler(uintptr_t Entrypoint, uintptr_t Gu
|
||||
auto Result = AddCustomIREntrypoint(
|
||||
Entrypoint,
|
||||
[this, GuestThunkEntrypoint](uintptr_t Entrypoint, FEXCore::IR::IREmitter* emit) {
|
||||
auto IRHeader = emit->_IRHeader(emit->Invalid(), Entrypoint, 0, 0, 0, 0);
|
||||
auto Block = emit->CreateCodeNode(true, 0);
|
||||
IRHeader.first->Blocks = emit->WrapNode(Block);
|
||||
emit->SetCurrentCodeBlock(Block);
|
||||
auto IRHeader = emit->_IRHeader(emit->Invalid(), Entrypoint, 0, 0, 0, 0);
|
||||
auto Block = emit->CreateCodeNode();
|
||||
IRHeader.first->Blocks = emit->WrapNode(Block);
|
||||
emit->SetCurrentCodeBlock(Block);
|
||||
|
||||
const auto GPRSize = this->Config.Is64BitMode ? IR::OpSize::i64Bit : IR::OpSize::i32Bit;
|
||||
const auto GPRSize = GetGPROpSize();
|
||||
|
||||
// Thunk entry-points don't get cached, don't need to be padded.
|
||||
if (GPRSize == IR::OpSize::i64Bit) {
|
||||
IR::Ref R = emit->_StoreRegister(emit->Constant(Entrypoint), GPRSize);
|
||||
R->Reg = IR::PhysicalRegister(IR::RegClass::GPRFixed, X86State::REG_R11).Raw;
|
||||
} else {
|
||||
emit->_StoreContextFPR(GPRSize, emit->_VCastFromGPR(IR::OpSize::i64Bit, IR::OpSize::i64Bit, emit->Constant(Entrypoint)),
|
||||
offsetof(Core::CPUState, mm[0][0]));
|
||||
}
|
||||
emit->_ExitFunction(IR::OpSize::i64Bit, emit->Constant(GuestThunkEntrypoint), IR::BranchHint::None, emit->Invalid(), emit->Invalid());
|
||||
if (GPRSize == IR::OpSize::i64Bit) {
|
||||
IR::Ref R = emit->_StoreRegister(emit->_Constant(Entrypoint), GPRSize);
|
||||
R->Reg = IR::PhysicalRegister(IR::GPRFixedClass, X86State::REG_R11).Raw;
|
||||
} else {
|
||||
emit->_StoreContext(GPRSize, IR::FPRClass, emit->_VCastFromGPR(IR::OpSize::i64Bit, IR::OpSize::i64Bit, emit->_Constant(Entrypoint)),
|
||||
offsetof(Core::CPUState, mm[0][0]));
|
||||
}
|
||||
emit->_ExitFunction(IR::OpSize::i64Bit, emit->_Constant(GuestThunkEntrypoint));
|
||||
},
|
||||
ThunkHandler, (void*)GuestThunkEntrypoint);
|
||||
|
||||
@@ -998,7 +1012,7 @@ void ContextImpl::AddThunkTrampolineIRHandler(uintptr_t Entrypoint, uintptr_t Gu
|
||||
void ContextImpl::AddForceTSOInformation(const IntervalList<uint64_t>& ValidRanges, fextl::set<uint64_t>&& Instructions) {
|
||||
LogMan::Throw::AFmt(CodeInvalidationMutex.try_lock() == false, "CodeInvalidationMutex needs to be unique_locked here");
|
||||
ForceTSOValidRanges.Insert(ValidRanges);
|
||||
ForceTSOInstructions.merge(std::move(Instructions));
|
||||
ForceTSOInstructions.merge(Instructions);
|
||||
}
|
||||
|
||||
void ContextImpl::RemoveForceTSOInformation(uint64_t Address, uint64_t Size) {
|
||||
@@ -1008,39 +1022,28 @@ void ContextImpl::RemoveForceTSOInformation(uint64_t Address, uint64_t Size) {
|
||||
ForceTSOInstructions.erase(ForceTSOInstructions.lower_bound(Address), ForceTSOInstructions.upper_bound(Address + Size));
|
||||
}
|
||||
|
||||
void ContextImpl::MarkMonoBackpatcherBlock(uint64_t BlockEntry) {
|
||||
MonoBackpatcherBlock.store(BlockEntry, std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
void ContextImpl::RemoveCustomIREntrypoint(FEXCore::Core::InternalThreadState* Thread, uintptr_t Entrypoint) {
|
||||
void ContextImpl::RemoveCustomIREntrypoint(uintptr_t Entrypoint) {
|
||||
LOGMAN_THROW_A_FMT(Config.Is64BitMode || !(Entrypoint >> 32), "64-bit Entrypoint in 32-bit mode {:x}", Entrypoint);
|
||||
|
||||
std::scoped_lock lk(CustomIRMutex);
|
||||
|
||||
InvalidateGuestCodeRange(nullptr, Entrypoint, 1);
|
||||
CustomIRHandlers.erase(Entrypoint);
|
||||
|
||||
HasCustomIRHandlers = !CustomIRHandlers.empty();
|
||||
SyscallHandler->InvalidateGuestCodeRange(Thread, Entrypoint, 1);
|
||||
}
|
||||
|
||||
void ContextImpl::MonoBackpatcherWrite(FEXCore::Core::CpuStateFrame* Frame, uint8_t Size, uint64_t Address, uint64_t Value) {
|
||||
auto Thread = Frame->Thread;
|
||||
auto CTX = static_cast<ContextImpl*>(Thread->CTX);
|
||||
{
|
||||
auto lk = GuardSignalDeferringSection(CTX->CodeInvalidationMutex, Thread);
|
||||
IR::AOTIRCacheEntry* ContextImpl::LoadAOTIRCacheEntry(const fextl::string& filename) {
|
||||
auto rv = IRCaptureCache.LoadAOTIRCacheEntry(filename);
|
||||
return rv;
|
||||
}
|
||||
|
||||
if (Size == 8) {
|
||||
*reinterpret_cast<uint64_t*>(Address) = Value;
|
||||
} else if (Size == 4) {
|
||||
*reinterpret_cast<uint32_t*>(Address) = Value;
|
||||
} else {
|
||||
ERROR_AND_DIE_FMT("Unexpected write size for backpatcher: {}", Size);
|
||||
}
|
||||
}
|
||||
|
||||
CTX->SyscallHandler->InvalidateGuestCodeRange(Thread, Address, Size);
|
||||
void ContextImpl::UnloadAOTIRCacheEntry(IR::AOTIRCacheEntry* Entry) {
|
||||
IRCaptureCache.UnloadAOTIRCacheEntry(Entry);
|
||||
}
|
||||
|
||||
void ContextImpl::ConfigureAOTGen(FEXCore::Core::InternalThreadState* Thread, fextl::set<uint64_t>* ExternalBranches, uint64_t SectionMaxAddress) {
|
||||
Thread->FrontendDecoder->SetExternalBranches(ExternalBranches);
|
||||
Thread->FrontendDecoder->SetSectionMaxAddress(SectionMaxAddress);
|
||||
}
|
||||
} // namespace FEXCore::Context
|
||||
@@ -1,10 +1,10 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
#include "Common/VectorRegType.h"
|
||||
#include "Common/SoftFloat.h"
|
||||
#include "Interface/Context/Context.h"
|
||||
#include "Interface/Core/CPUBackend.h"
|
||||
#include "Interface/Core/Dispatcher/Dispatcher.h"
|
||||
#include "Interface/Core/LookupCache.h"
|
||||
#include "Interface/Core/X86HelperGen.h"
|
||||
#include "Utils/MemberFunctionToPointer.h"
|
||||
|
||||
#include <FEXCore/Config/Config.h>
|
||||
@@ -16,18 +16,14 @@
|
||||
#include <FEXCore/Utils/Event.h>
|
||||
#include <FEXCore/Utils/LogManager.h>
|
||||
#include <FEXCore/Utils/MathUtils.h>
|
||||
#include <FEXHeaderUtils/Syscalls.h>
|
||||
|
||||
#include <CodeEmitter/Emitter.h>
|
||||
|
||||
#ifdef VIXL_SIMULATOR
|
||||
#include <aarch64/simulator-aarch64.h>
|
||||
#endif
|
||||
|
||||
#include <array>
|
||||
#include <bit>
|
||||
#include <atomic>
|
||||
#include <condition_variable>
|
||||
#include <csignal>
|
||||
#include <cstring>
|
||||
#include <signal.h>
|
||||
|
||||
namespace FEXCore::CPU {
|
||||
|
||||
@@ -35,14 +31,12 @@ static void SleepThread(FEXCore::Context::ContextImpl* CTX, FEXCore::Core::CpuSt
|
||||
CTX->SyscallHandler->SleepThread(CTX, Frame);
|
||||
}
|
||||
|
||||
constexpr size_t MAX_DISPATCHER_CODE_SIZE = FEXCore::Utils::FEX_PAGE_SIZE * 4;
|
||||
constexpr size_t MAX_DISPATCHER_CODE_SIZE = 4096 * 4;
|
||||
|
||||
Dispatcher::Dispatcher(FEXCore::Context::ContextImpl* ctx)
|
||||
: Arm64Emitter(ctx, FEXCore::Allocator::VirtualAlloc(MAX_DISPATCHER_CODE_SIZE, true), MAX_DISPATCHER_CODE_SIZE)
|
||||
, CTX {ctx} {
|
||||
EmitDispatcher();
|
||||
|
||||
FEXCore::Allocator::VirtualName("FEXMem_Misc", reinterpret_cast<void*>(GetBufferBase()), MAX_DISPATCHER_CODE_SIZE);
|
||||
}
|
||||
|
||||
Dispatcher::~Dispatcher() {
|
||||
@@ -87,17 +81,14 @@ void Dispatcher::EmitDispatcher() {
|
||||
add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, ARMEmitter::Reg::rsp, 0);
|
||||
str(ARMEmitter::XReg::x0, STATE_PTR(CpuStateFrame, ReturningStackLocation));
|
||||
|
||||
ARMEmitter::ForwardLabel CompileSingleStep;
|
||||
AbsoluteLoopTopAddressFillSRA = GetCursorAddress<uint64_t>();
|
||||
|
||||
FillStaticRegs();
|
||||
ldr(RipReg, STATE_PTR(CpuStateFrame, State.rip));
|
||||
(void)cbnz(ARMEmitter::Size::i32Bit, ENTRY_FILL_SRA_SINGLE_INST_REG, &CompileSingleStep);
|
||||
|
||||
ARMEmitter::BiDirectionalLabel LoopTop {};
|
||||
ARMEmitter::ForwardLabel CompileSingleStep;
|
||||
|
||||
#ifdef ARCHITECTURE_arm64ec
|
||||
(void)b(&LoopTop);
|
||||
#ifdef _M_ARM_64EC
|
||||
b(&LoopTop);
|
||||
|
||||
AbsoluteLoopTopAddressEnterECFillSRA = GetCursorAddress<uint64_t>();
|
||||
ldr(STATE, EC_ENTRY_CPUAREA_REG, CPU_AREA_EMULATOR_DATA_OFFSET);
|
||||
@@ -105,10 +96,10 @@ void Dispatcher::EmitDispatcher() {
|
||||
|
||||
ldr(RipReg, STATE_PTR(CpuStateFrame, State.rip));
|
||||
// Force a single instruction block if ENTRY_FILL_SRA_SINGLE_INST_REG is nonzero entering the JIT, used for inline SMC handling.
|
||||
(void)cbnz(ARMEmitter::Size::i32Bit, ENTRY_FILL_SRA_SINGLE_INST_REG, &CompileSingleStep);
|
||||
cbnz(ARMEmitter::Size::i32Bit, ENTRY_FILL_SRA_SINGLE_INST_REG, &CompileSingleStep);
|
||||
|
||||
// Enter JIT
|
||||
(void)b(&LoopTop);
|
||||
b(&LoopTop);
|
||||
|
||||
AbsoluteLoopTopAddressEnterEC = GetCursorAddress<uint64_t>();
|
||||
// Load ThreadState and write the target PC there
|
||||
@@ -120,121 +111,93 @@ void Dispatcher::EmitDispatcher() {
|
||||
add(ARMEmitter::Size::i64Bit, StaticRegisters[X86State::REG_RSP], ARMEmitter::Reg::rsp, 0);
|
||||
add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, TMP1, 0);
|
||||
|
||||
ldr(REG_CALLRET_SP, STATE_PTR(CpuStateFrame, State.callret_sp));
|
||||
|
||||
FillSpecialRegs(TMP1, TMP2, false, true);
|
||||
|
||||
// As ARM64EC uses this as an entrypoint for both guest calls and host returns, opportunistically try to return
|
||||
// using the call-ret stack to avoid unbalancing it.
|
||||
ldp<ARMEmitter::IndexType::OFFSET>(TMP1, TMP2, REG_CALLRET_SP);
|
||||
// EC_CALL_CHECKER_PC_REG is REG_PF which isn't touched by any of the above
|
||||
sub(ARMEmitter::Size::i64Bit, TMP1, EC_CALL_CHECKER_PC_REG, TMP1);
|
||||
(void)cbnz(ARMEmitter::Size::i64Bit, TMP1, &LoopTop);
|
||||
|
||||
// If the entry at the TOS is for the target address, pop it and return to the JIT code
|
||||
add(ARMEmitter::Size::i64Bit, REG_CALLRET_SP, REG_CALLRET_SP, 0x10);
|
||||
ret(TMP2);
|
||||
|
||||
// Enter JIT
|
||||
#endif
|
||||
|
||||
// We want to ensure that we are 16 byte aligned at the top of this loop
|
||||
Align16B();
|
||||
ARMEmitter::BiDirectionalLabel FullLookup {};
|
||||
ARMEmitter::BiDirectionalLabel CallBlock {};
|
||||
|
||||
(void)Bind(&LoopTop);
|
||||
Bind(&LoopTop);
|
||||
AbsoluteLoopTopAddress = GetCursorAddress<uint64_t>();
|
||||
|
||||
// Load in our RIP
|
||||
ldr(RipReg, STATE_PTR(CpuStateFrame, State.rip));
|
||||
|
||||
#ifdef ARCHITECTURE_arm64ec
|
||||
// Clobbers TMP1/2
|
||||
// Check the EC code bitmap incase we need to exit the JIT to call into native code.
|
||||
ARMEmitter::ForwardLabel l_NotECCode;
|
||||
ldr(TMP1, ARMEmitter::XReg::x18, TEB_PEB_OFFSET);
|
||||
ldr(TMP1, TMP1, PEB_EC_CODE_BITMAP_OFFSET);
|
||||
|
||||
lsr(ARMEmitter::Size::i64Bit, TMP2, RipReg, 15);
|
||||
and_(ARMEmitter::Size::i64Bit, TMP2, TMP2, 0x1fffffffffff8);
|
||||
ldr(TMP1, TMP1, TMP2, ARMEmitter::ExtendedType::LSL_64, 0);
|
||||
lsr(ARMEmitter::Size::i64Bit, TMP2, RipReg, 12);
|
||||
lsrv(ARMEmitter::Size::i64Bit, TMP1, TMP1, TMP2);
|
||||
(void)tbz(TMP1, 0, &l_NotECCode);
|
||||
|
||||
str(REG_CALLRET_SP, STATE_PTR(CpuStateFrame, State.callret_sp));
|
||||
|
||||
add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, StaticRegisters[X86State::REG_RSP], 0);
|
||||
mov(EC_CALL_CHECKER_PC_REG, RipReg);
|
||||
ldr(TMP2, STATE_PTR(CpuStateFrame, Pointers.ExitFunctionEC));
|
||||
br(TMP2);
|
||||
|
||||
(void)Bind(&l_NotECCode);
|
||||
#endif
|
||||
|
||||
ldrb(TMP1, STATE_PTR(CpuStateFrame, State.flags[X86State::RFLAG_TF_RAW_LOC]));
|
||||
(void)cbnz(ARMEmitter::Size::i32Bit, TMP1, &CompileSingleStep);
|
||||
cbnz(ARMEmitter::Size::i32Bit, TMP1, &CompileSingleStep);
|
||||
|
||||
// L1 Cache
|
||||
ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.L1Pointer));
|
||||
|
||||
and_(ARMEmitter::Size::i64Bit, TMP4, RipReg.R(), LookupCache::L1_ENTRIES_MASK);
|
||||
add(ARMEmitter::Size::i64Bit, TMP1, TMP1, TMP4, ARMEmitter::ShiftType::LSL, 4);
|
||||
ldp<ARMEmitter::IndexType::OFFSET>(TMP4, TMP1, TMP1, 0);
|
||||
sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, RipReg);
|
||||
cbnz(ARMEmitter::Size::i64Bit, TMP1, &FullLookup);
|
||||
|
||||
br(TMP4);
|
||||
|
||||
// L1C check failed, do a full lookup
|
||||
Bind(&FullLookup);
|
||||
|
||||
// This is the block cache lookup routine
|
||||
// It matches what is going on it LookupCache.h::FindBlock
|
||||
ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.L2Pointer));
|
||||
|
||||
// Mask the address by the virtual address size so we can check for aliases
|
||||
uint64_t VirtualMemorySize = CTX->Config.VirtualMemSize;
|
||||
if (std::popcount(VirtualMemorySize) == 1) {
|
||||
and_(ARMEmitter::Size::i64Bit, TMP4, RipReg.R(), VirtualMemorySize - 1);
|
||||
} else {
|
||||
LoadConstant(ARMEmitter::Size::i64Bit, TMP4, VirtualMemorySize);
|
||||
and_(ARMEmitter::Size::i64Bit, TMP4, RipReg.R(), TMP4);
|
||||
}
|
||||
|
||||
ARMEmitter::ForwardLabel NoBlock;
|
||||
|
||||
if (DisableL2Cache()) {
|
||||
(void)b(&NoBlock);
|
||||
} else {
|
||||
// This is the block cache lookup routine
|
||||
// It matches what is going on it LookupCache.h::FindBlock
|
||||
ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.L2Pointer));
|
||||
{
|
||||
// Offset the address and add to our page pointer
|
||||
lsr(ARMEmitter::Size::i64Bit, TMP2, TMP4, 12);
|
||||
|
||||
// Mask the address by the virtual address size so we can check for aliases
|
||||
uint64_t VirtualMemorySize = CTX->Config.VirtualMemSize;
|
||||
if (std::popcount(VirtualMemorySize) == 1) {
|
||||
and_(ARMEmitter::Size::i64Bit, TMP4, RipReg.R(), VirtualMemorySize - 1);
|
||||
} else {
|
||||
LoadConstant(ARMEmitter::Size::i64Bit, TMP4, VirtualMemorySize);
|
||||
and_(ARMEmitter::Size::i64Bit, TMP4, RipReg.R(), TMP4);
|
||||
}
|
||||
// Load the pointer from the offset
|
||||
ldr(TMP1, TMP1, TMP2, ARMEmitter::ExtendedType::LSL_64, 3);
|
||||
|
||||
// If page pointer is zero then we have no block
|
||||
cbz(ARMEmitter::Size::i64Bit, TMP1, &NoBlock);
|
||||
|
||||
// Steal the page offset
|
||||
and_(ARMEmitter::Size::i64Bit, TMP2, TMP4, 0x0FFF);
|
||||
|
||||
// Shift the offset by the size of the block cache entry
|
||||
add(TMP1, TMP1, TMP2, ARMEmitter::ShiftType::LSL, (int)log2(sizeof(FEXCore::LookupCache::LookupCacheEntry)));
|
||||
|
||||
// The the full LookupCacheEntry with a single LDP.
|
||||
// Check the guest address first to ensure it maps to the address we are currently at.
|
||||
// This fixes aliasing problems
|
||||
ldp<ARMEmitter::IndexType::OFFSET>(TMP4, TMP2, TMP1, 0);
|
||||
|
||||
// If the guest address doesn't match, Compile the block.
|
||||
sub(TMP2, TMP2, RipReg);
|
||||
cbnz(ARMEmitter::Size::i64Bit, TMP2, &NoBlock);
|
||||
|
||||
// Check the host address to see if it matches, else compile the block.
|
||||
cbz(ARMEmitter::Size::i64Bit, TMP4, &NoBlock);
|
||||
|
||||
// If we've made it here then we have a real compiled block
|
||||
{
|
||||
// Offset the address and add to our page pointer
|
||||
lsr(ARMEmitter::Size::i64Bit, TMP2, TMP4, 12);
|
||||
// update L1 cache
|
||||
ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.L1Pointer));
|
||||
|
||||
// Load the pointer from the offset
|
||||
ldr(TMP1, TMP1, TMP2, ARMEmitter::ExtendedType::LSL_64, 3);
|
||||
and_(ARMEmitter::Size::i64Bit, TMP2, RipReg.R(), LookupCache::L1_ENTRIES_MASK);
|
||||
add(TMP1, TMP1, TMP2, ARMEmitter::ShiftType::LSL, 4);
|
||||
stp<ARMEmitter::IndexType::OFFSET>(TMP4, RipReg, TMP1);
|
||||
|
||||
// If page pointer is zero then we have no block
|
||||
(void)cbz(ARMEmitter::Size::i64Bit, TMP1, &NoBlock);
|
||||
|
||||
// Steal the page offset
|
||||
and_(ARMEmitter::Size::i64Bit, TMP2, TMP4, 0x0FFF);
|
||||
|
||||
// Shift the offset by the size of the block cache entry
|
||||
add(TMP1, TMP1, TMP2, ARMEmitter::ShiftType::LSL, FEXCore::ilog2(sizeof(LookupCache::LookupCacheEntry)));
|
||||
|
||||
// The the full LookupCacheEntry with a single LDP.
|
||||
// Check the guest address first to ensure it maps to the address we are currently at.
|
||||
// This fixes aliasing problems
|
||||
ldp<ARMEmitter::IndexType::OFFSET>(TMP4, TMP2, TMP1, 0);
|
||||
|
||||
// If the guest address doesn't match, Compile the block.
|
||||
sub(TMP2, TMP2, RipReg);
|
||||
(void)cbnz(ARMEmitter::Size::i64Bit, TMP2, &NoBlock);
|
||||
|
||||
// Check the host address to see if it matches, else compile the block.
|
||||
(void)cbz(ARMEmitter::Size::i64Bit, TMP4, &NoBlock);
|
||||
|
||||
// If we've made it here then we have a real compiled block
|
||||
{
|
||||
// update L1 cache
|
||||
ldp<ARMEmitter::IndexType::OFFSET>(TMP1, TMP2, STATE, offsetof(FEXCore::Core::CpuStateFrame, State.L1Pointer));
|
||||
|
||||
// Calculate (tmp1 + ((ripreg & L1_ENTRIES_MASK) << 4)) for the address
|
||||
// L1Mask is pre-shifted.
|
||||
and_(ARMEmitter::Size::i64Bit, TMP2, TMP2, RipReg.R(), ARMEmitter::ShiftType::LSL, FEXCore::ilog2(sizeof(LookupCache::LookupCacheEntry)));
|
||||
add(TMP1, TMP1, TMP2);
|
||||
|
||||
stp<ARMEmitter::IndexType::OFFSET>(TMP4, RipReg, TMP1);
|
||||
|
||||
// Jump to the block
|
||||
br(TMP4);
|
||||
}
|
||||
// Jump to the block
|
||||
br(TMP4);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -259,7 +222,7 @@ void Dispatcher::EmitDispatcher() {
|
||||
str(TMP2, STATE, offsetof(FEXCore::Core::CPUState, DeferredSignalRefCount));
|
||||
#endif
|
||||
|
||||
#ifdef ARCHITECTURE_arm64ec
|
||||
#ifdef _M_ARM_64EC
|
||||
ldr(TMP2, ARMEmitter::XReg::x18, TEB_CPU_AREA_OFFSET);
|
||||
LoadConstant(ARMEmitter::Size::i32Bit, TMP1, 1);
|
||||
strb(TMP1.W(), TMP2, CPU_AREA_IN_SYSCALL_CALLBACK_OFFSET);
|
||||
@@ -267,7 +230,7 @@ void Dispatcher::EmitDispatcher() {
|
||||
|
||||
Body();
|
||||
|
||||
#ifdef ARCHITECTURE_arm64ec
|
||||
#ifdef _M_ARM_64EC
|
||||
ldr(TMP2, ARMEmitter::XReg::x18, TEB_CPU_AREA_OFFSET);
|
||||
strb(ARMEmitter::WReg::zr, TMP2, CPU_AREA_IN_SYSCALL_CALLBACK_OFFSET);
|
||||
#endif
|
||||
@@ -291,7 +254,7 @@ void Dispatcher::EmitDispatcher() {
|
||||
mov(ARMEmitter::XReg::x0, STATE);
|
||||
mov(ARMEmitter::XReg::x1, ARMEmitter::XReg::lr);
|
||||
|
||||
ldr(ARMEmitter::XReg::x2, STATE_PTR(CpuStateFrame, Pointers.ExitFunctionLink));
|
||||
ldr(ARMEmitter::XReg::x2, STATE_PTR(CpuStateFrame, Pointers.Common.ExitFunctionLink));
|
||||
if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {
|
||||
GenerateIndirectRuntimeCall<uintptr_t, void*, void*>(ARMEmitter::Reg::r2);
|
||||
} else {
|
||||
@@ -308,9 +271,37 @@ void Dispatcher::EmitDispatcher() {
|
||||
br(TMP1);
|
||||
}
|
||||
|
||||
#ifdef _M_ARM_64EC
|
||||
// Clobbers TMP1/2
|
||||
auto EmitECExitCheck = [&]() {
|
||||
// Check the EC code bitmap incase we need to exit the JIT to call into native code.
|
||||
ARMEmitter::ForwardLabel l_NotECCode;
|
||||
ldr(TMP1, ARMEmitter::XReg::x18, TEB_PEB_OFFSET);
|
||||
ldr(TMP1, TMP1, PEB_EC_CODE_BITMAP_OFFSET);
|
||||
|
||||
lsr(ARMEmitter::Size::i64Bit, TMP2, RipReg, 15);
|
||||
and_(ARMEmitter::Size::i64Bit, TMP2, TMP2, 0x1fffffffffff8);
|
||||
ldr(TMP1, TMP1, TMP2, ARMEmitter::ExtendedType::LSL_64, 0);
|
||||
lsr(ARMEmitter::Size::i64Bit, TMP2, RipReg, 12);
|
||||
lsrv(ARMEmitter::Size::i64Bit, TMP1, TMP1, TMP2);
|
||||
tbz(TMP1, 0, &l_NotECCode);
|
||||
|
||||
add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, StaticRegisters[X86State::REG_RSP], 0);
|
||||
mov(EC_CALL_CHECKER_PC_REG, RipReg);
|
||||
ldr(TMP2, STATE_PTR(CpuStateFrame, Pointers.Common.ExitFunctionEC));
|
||||
br(TMP2);
|
||||
|
||||
Bind(&l_NotECCode);
|
||||
};
|
||||
#endif
|
||||
|
||||
// Need to create the block
|
||||
{
|
||||
(void)Bind(&NoBlock);
|
||||
Bind(&NoBlock);
|
||||
|
||||
#ifdef _M_ARM_64EC
|
||||
EmitECExitCheck();
|
||||
#endif
|
||||
|
||||
EmitSignalGuardedRegion([&]() {
|
||||
SpillStaticRegs(TMP1);
|
||||
@@ -344,7 +335,11 @@ void Dispatcher::EmitDispatcher() {
|
||||
}
|
||||
|
||||
{
|
||||
(void)Bind(&CompileSingleStep);
|
||||
Bind(&CompileSingleStep);
|
||||
|
||||
#ifdef _M_ARM_64EC
|
||||
EmitECExitCheck();
|
||||
#endif
|
||||
|
||||
EmitSignalGuardedRegion([&]() {
|
||||
SpillStaticRegs(TMP1);
|
||||
@@ -488,7 +483,7 @@ void Dispatcher::EmitDispatcher() {
|
||||
|
||||
// Now push the callback return trampoline to the guest stack
|
||||
// Guest will be misaligned because calling a thunk won't correct the guest's stack once we call the callback from the host
|
||||
ldr(ARMEmitter::XReg::x0, STATE_PTR(CpuStateFrame, Pointers.ThunkCallbackRet));
|
||||
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, CTX->X86CodeGen.CallbackReturn);
|
||||
|
||||
ldr(ARMEmitter::XReg::x2, STATE_PTR(CpuStateFrame, State.gregs[X86State::REG_RSP]));
|
||||
sub(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r2, ARMEmitter::Reg::r2, CTX->Config.Is64BitMode ? 16 : 12);
|
||||
@@ -503,10 +498,9 @@ void Dispatcher::EmitDispatcher() {
|
||||
|
||||
// load static regs
|
||||
FillStaticRegs();
|
||||
stp<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::zr, ARMEmitter::XReg::zr, REG_CALLRET_SP, -0x10);
|
||||
|
||||
// Now go back to the regular dispatcher loop
|
||||
(void)b(&LoopTop);
|
||||
b(&LoopTop);
|
||||
}
|
||||
|
||||
auto EmitLongALUOpHandler = [&](auto R, auto Offset) {
|
||||
@@ -544,8 +538,8 @@ void Dispatcher::EmitDispatcher() {
|
||||
return Address;
|
||||
};
|
||||
|
||||
LUDIVHandlerAddress = EmitLongALUOpHandler(STATE_PTR(CpuStateFrame, Pointers.LUDIV));
|
||||
LDIVHandlerAddress = EmitLongALUOpHandler(STATE_PTR(CpuStateFrame, Pointers.LDIV));
|
||||
LUDIVHandlerAddress = EmitLongALUOpHandler(STATE_PTR(CpuStateFrame, Pointers.AArch64.LUDIV));
|
||||
LDIVHandlerAddress = EmitLongALUOpHandler(STATE_PTR(CpuStateFrame, Pointers.AArch64.LDIV));
|
||||
|
||||
// Interpreter fallbacks
|
||||
{
|
||||
@@ -556,8 +550,8 @@ void Dispatcher::EmitDispatcher() {
|
||||
FABI_F80_I16_I32_PTR,
|
||||
FABI_F32_I16_F80_PTR,
|
||||
FABI_F64_I16_F80_PTR,
|
||||
FABI_F64_F64_PTR,
|
||||
FABI_F64_F64_F64_PTR,
|
||||
FABI_F64_I16_F64_PTR,
|
||||
FABI_F64_I16_F64_F64_PTR,
|
||||
FABI_I16_I16_F80_PTR,
|
||||
FABI_I32_I16_F80_PTR,
|
||||
FABI_I64_I16_F80_PTR,
|
||||
@@ -565,7 +559,7 @@ void Dispatcher::EmitDispatcher() {
|
||||
FABI_F80_I16_F80_PTR,
|
||||
FABI_F80_I16_F80_F80_PTR,
|
||||
FABI_F80x2_I16_F80_PTR,
|
||||
FABI_F64x2_F64_PTR,
|
||||
FABI_F64x2_I16_F64_PTR,
|
||||
FABI_I32_I64_I64_V128_V128_I16,
|
||||
FABI_I32_V128_V128_I16,
|
||||
}};
|
||||
@@ -575,15 +569,14 @@ void Dispatcher::EmitDispatcher() {
|
||||
}
|
||||
}
|
||||
|
||||
(void)Bind(&l_CTX);
|
||||
Bind(&l_CTX);
|
||||
dc64(reinterpret_cast<uintptr_t>(CTX));
|
||||
(void)Bind(&l_Sleep);
|
||||
Bind(&l_Sleep);
|
||||
dc64(reinterpret_cast<uint64_t>(SleepThread));
|
||||
(void)Bind(&l_CompileBlock);
|
||||
Bind(&l_CompileBlock);
|
||||
FEXCore::Utils::MemberFunctionToPointerCast PMFCompileBlock(&FEXCore::Context::ContextImpl::CompileBlock);
|
||||
dc64(PMFCompileBlock.GetConvertedPointer());
|
||||
(void)Bind(&l_CompileSingleStep);
|
||||
|
||||
Bind(&l_CompileSingleStep);
|
||||
FEXCore::Utils::MemberFunctionToPointerCast PMFCompileSingleStep(&FEXCore::Context::ContextImpl::CompileSingleStep);
|
||||
dc64(PMFCompileSingleStep.GetConvertedPointer());
|
||||
|
||||
@@ -614,7 +607,6 @@ void Dispatcher::EmitDispatcher() {
|
||||
#ifdef VIXL_SIMULATOR
|
||||
void Dispatcher::ExecuteDispatch(FEXCore::Core::CpuStateFrame* Frame) {
|
||||
Simulator.WriteXRegister(0, reinterpret_cast<int64_t>(Frame));
|
||||
Simulator.WriteXRegister(1, 0);
|
||||
Simulator.RunFrom(reinterpret_cast< const vixl::aarch64::Instruction*>(DispatchPtr));
|
||||
}
|
||||
|
||||
@@ -765,7 +757,7 @@ uint64_t Dispatcher::GenerateABICall(FallbackABI ABI) {
|
||||
|
||||
ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW));
|
||||
if (!TMP_ABIARGS) {
|
||||
mov(VABI1.Q(), VTMP1.Q());
|
||||
fmov(VABI1.D(), VTMP1.D());
|
||||
}
|
||||
|
||||
mov(ARMEmitter::XReg::x1, STATE);
|
||||
@@ -798,7 +790,7 @@ uint64_t Dispatcher::GenerateABICall(FallbackABI ABI) {
|
||||
|
||||
FillF64Result();
|
||||
} break;
|
||||
case FABI_F64_F64_PTR: {
|
||||
case FABI_F64_I16_F64_PTR: {
|
||||
// Linux Reg/Win32 Reg:
|
||||
// tmp4 (x4/x13): FallbackHandler
|
||||
// x30: return
|
||||
@@ -808,17 +800,18 @@ uint64_t Dispatcher::GenerateABICall(FallbackABI ABI) {
|
||||
if (!TMP_ABIARGS) {
|
||||
fmov(VABI1.D(), VTMP1.D());
|
||||
}
|
||||
mov(ARMEmitter::XReg::x0, STATE);
|
||||
ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW));
|
||||
mov(ARMEmitter::XReg::x1, STATE);
|
||||
|
||||
if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {
|
||||
GenerateIndirectRuntimeCall<double, double, uint64_t>(FallbackPointerReg);
|
||||
GenerateIndirectRuntimeCall<double, uint16_t, double, uint64_t>(FallbackPointerReg);
|
||||
} else {
|
||||
blr(FallbackPointerReg);
|
||||
}
|
||||
|
||||
FillF64Result();
|
||||
} break;
|
||||
case FABI_F64_F64_F64_PTR: {
|
||||
case FABI_F64_I16_F64_F64_PTR: {
|
||||
// Linux Reg/Win32 Reg:
|
||||
// tmp4 (x4/x13): FallbackHandler
|
||||
// x30: return
|
||||
@@ -831,9 +824,10 @@ uint64_t Dispatcher::GenerateABICall(FallbackABI ABI) {
|
||||
fmov(VABI2.D(), VTMP2.D());
|
||||
}
|
||||
|
||||
mov(ARMEmitter::XReg::x0, STATE);
|
||||
ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW));
|
||||
mov(ARMEmitter::XReg::x1, STATE);
|
||||
if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {
|
||||
GenerateIndirectRuntimeCall<double, double, double, uint64_t>(FallbackPointerReg);
|
||||
GenerateIndirectRuntimeCall<double, uint16_t, double, double, uint64_t>(FallbackPointerReg);
|
||||
} else {
|
||||
blr(FallbackPointerReg);
|
||||
}
|
||||
@@ -993,7 +987,7 @@ uint64_t Dispatcher::GenerateABICall(FallbackABI ABI) {
|
||||
|
||||
FillF80x2Result();
|
||||
} break;
|
||||
case FABI_F64x2_F64_PTR: {
|
||||
case FABI_F64x2_I16_F64_PTR: {
|
||||
// Linux Reg/Win32 Reg:
|
||||
// tmp4 (x4/x13): FallbackHandler
|
||||
// x30: return
|
||||
@@ -1002,13 +996,14 @@ uint64_t Dispatcher::GenerateABICall(FallbackABI ABI) {
|
||||
|
||||
SpillForABICall(CTX->HostFeatures.SupportsPreserveAllABI, TMP3, true);
|
||||
|
||||
mov(ARMEmitter::XReg::x0, STATE);
|
||||
ldrh(ARMEmitter::WReg::w0, STATE, offsetof(FEXCore::Core::CPUState, FCW));
|
||||
mov(ARMEmitter::XReg::x1, STATE);
|
||||
if (!TMP_ABIARGS) {
|
||||
fmov(VABI1.D(), VTMP1.D());
|
||||
}
|
||||
|
||||
if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {
|
||||
// GenerateIndirectRuntimeCall<FEXCore::VectorScalarF64Pair, FEXCore::VectorRegType, uint64_t>(FallbackPointerReg);
|
||||
// GenerateIndirectRuntimeCall<FEXCore::VectorScalarF64Pair, uint16_t, FEXCore::VectorRegType, uint64_t>(FallbackPointerReg);
|
||||
} else {
|
||||
blr(FallbackPointerReg);
|
||||
}
|
||||
@@ -1086,75 +1081,30 @@ uint64_t Dispatcher::GenerateABICall(FallbackABI ABI) {
|
||||
void Dispatcher::InitThreadPointers(FEXCore::Core::InternalThreadState* Thread) {
|
||||
// Setup dispatcher specific pointers that need to be accessed from JIT code
|
||||
{
|
||||
auto& Ptrs = Thread->CurrentFrame->Pointers;
|
||||
auto& Common = Thread->CurrentFrame->Pointers.Common;
|
||||
|
||||
Ptrs.DispatcherLoopTop = AbsoluteLoopTopAddress;
|
||||
Ptrs.DispatcherLoopTopFillSRA = AbsoluteLoopTopAddressFillSRA;
|
||||
Ptrs.DispatcherLoopTopEnterEC = AbsoluteLoopTopAddressEnterEC;
|
||||
Ptrs.DispatcherLoopTopEnterECFillSRA = AbsoluteLoopTopAddressEnterECFillSRA;
|
||||
Ptrs.ExitFunctionLinker = ExitFunctionLinkerAddress;
|
||||
Ptrs.ThreadStopHandlerSpillSRA = ThreadStopHandlerAddressSpillSRA;
|
||||
Ptrs.ThreadPauseHandlerSpillSRA = ThreadPauseHandlerAddressSpillSRA;
|
||||
Ptrs.GuestSignal_SIGILL = GuestSignal_SIGILL;
|
||||
Ptrs.GuestSignal_SIGTRAP = GuestSignal_SIGTRAP;
|
||||
Ptrs.GuestSignal_SIGSEGV = GuestSignal_SIGSEGV;
|
||||
Ptrs.SignalReturnHandler = SignalHandlerReturnAddress;
|
||||
Ptrs.SignalReturnHandlerRT = SignalHandlerReturnAddressRT;
|
||||
Ptrs.LUDIVHandler = LUDIVHandlerAddress;
|
||||
Ptrs.LDIVHandler = LDIVHandlerAddress;
|
||||
Common.DispatcherLoopTop = AbsoluteLoopTopAddress;
|
||||
Common.DispatcherLoopTopFillSRA = AbsoluteLoopTopAddressFillSRA;
|
||||
Common.DispatcherLoopTopEnterEC = AbsoluteLoopTopAddressEnterEC;
|
||||
Common.DispatcherLoopTopEnterECFillSRA = AbsoluteLoopTopAddressEnterECFillSRA;
|
||||
Common.ExitFunctionLinker = ExitFunctionLinkerAddress;
|
||||
Common.ThreadStopHandlerSpillSRA = ThreadStopHandlerAddressSpillSRA;
|
||||
Common.ThreadPauseHandlerSpillSRA = ThreadPauseHandlerAddressSpillSRA;
|
||||
Common.GuestSignal_SIGILL = GuestSignal_SIGILL;
|
||||
Common.GuestSignal_SIGTRAP = GuestSignal_SIGTRAP;
|
||||
Common.GuestSignal_SIGSEGV = GuestSignal_SIGSEGV;
|
||||
Common.SignalReturnHandler = SignalHandlerReturnAddress;
|
||||
Common.SignalReturnHandlerRT = SignalHandlerReturnAddressRT;
|
||||
|
||||
auto& AArch64 = Thread->CurrentFrame->Pointers.AArch64;
|
||||
AArch64.LUDIVHandler = LUDIVHandlerAddress;
|
||||
AArch64.LDIVHandler = LDIVHandlerAddress;
|
||||
|
||||
// Fill in the fallback handlers
|
||||
InterpreterOps::FillFallbackIndexPointers(Ptrs.FallbackHandlerPointers, &ABIPointers[0]);
|
||||
InterpreterOps::FillFallbackIndexPointers(Common.FallbackHandlerPointers, &ABIPointers[0]);
|
||||
}
|
||||
}
|
||||
|
||||
SignalDelegatorConfig Dispatcher::MakeSignalDelegatorConfig() const {
|
||||
// PF/AF are the final two SRA registers. We only want GPRs
|
||||
const auto GPRCount = uint16_t(StaticRegisters.size() - 2);
|
||||
const auto FPRCount = uint16_t(StaticFPRegisters.size());
|
||||
|
||||
const auto GetSRAGPRMapping = [GPRCount, this] {
|
||||
SignalDelegatorConfig::SRAIndexMapping Mapping {};
|
||||
for (size_t i = 0; i < GPRCount; ++i) {
|
||||
Mapping[i] = StaticRegisters[i].Idx();
|
||||
}
|
||||
return Mapping;
|
||||
};
|
||||
|
||||
const auto GetSRAFPRMapping = [FPRCount, this] {
|
||||
SignalDelegatorConfig::SRAIndexMapping Mapping {};
|
||||
for (size_t i = 0; i < FPRCount; ++i) {
|
||||
Mapping[i] = StaticFPRegisters[i].Idx();
|
||||
}
|
||||
return Mapping;
|
||||
};
|
||||
|
||||
return FEXCore::SignalDelegatorConfig {
|
||||
.DispatcherBegin = Start,
|
||||
.DispatcherEnd = End,
|
||||
|
||||
.AbsoluteLoopTopAddress = AbsoluteLoopTopAddress,
|
||||
.AbsoluteLoopTopAddressFillSRA = AbsoluteLoopTopAddressFillSRA,
|
||||
.SignalHandlerReturnAddress = SignalHandlerReturnAddress,
|
||||
.SignalHandlerReturnAddressRT = SignalHandlerReturnAddressRT,
|
||||
|
||||
.PauseReturnInstruction = PauseReturnInstruction,
|
||||
.ThreadPauseHandlerAddressSpillSRA = ThreadPauseHandlerAddressSpillSRA,
|
||||
.ThreadPauseHandlerAddress = ThreadPauseHandlerAddress,
|
||||
|
||||
// Stop handlers.
|
||||
.ThreadStopHandlerAddressSpillSRA = ThreadStopHandlerAddressSpillSRA,
|
||||
.ThreadStopHandlerAddress = ThreadStopHandlerAddress,
|
||||
|
||||
// SRA information.
|
||||
.SRAGPRCount = GPRCount,
|
||||
.SRAFPRCount = FPRCount,
|
||||
|
||||
.SRAGPRMapping = GetSRAGPRMapping(),
|
||||
.SRAFPRMapping = GetSRAFPRMapping(),
|
||||
};
|
||||
}
|
||||
|
||||
fextl::unique_ptr<Dispatcher> Dispatcher::Create(FEXCore::Context::ContextImpl* CTX) {
|
||||
return fextl::make_unique<Dispatcher>(CTX);
|
||||
}
|
||||
|
||||
@@ -2,19 +2,25 @@
|
||||
#pragma once
|
||||
|
||||
#include "Interface/Core/ArchHelpers/Arm64Emitter.h"
|
||||
#include "Interface/Core/CPUBackend.h"
|
||||
#include "Interface/Core/Interpreter/InterpreterOps.h"
|
||||
|
||||
#include <FEXCore/Config/Config.h>
|
||||
#include <FEXCore/Core/CoreState.h>
|
||||
#include <FEXCore/fextl/memory.h>
|
||||
|
||||
#include <array>
|
||||
#include <cstddef>
|
||||
#ifdef VIXL_SIMULATOR
|
||||
#include <aarch64/simulator-aarch64.h>
|
||||
#endif
|
||||
|
||||
#include <cstdint>
|
||||
#include <signal.h>
|
||||
#include <stddef.h>
|
||||
#include <stack>
|
||||
#include <tuple>
|
||||
|
||||
namespace FEXCore {
|
||||
struct GuestSigAction;
|
||||
struct SignalDelegatorConfig;
|
||||
} // namespace FEXCore
|
||||
}
|
||||
|
||||
namespace FEXCore::Core {
|
||||
struct CpuStateFrame;
|
||||
@@ -36,36 +42,6 @@ public:
|
||||
Dispatcher(FEXCore::Context::ContextImpl* ctx);
|
||||
~Dispatcher();
|
||||
|
||||
void InitThreadPointers(FEXCore::Core::InternalThreadState* Thread);
|
||||
|
||||
#ifdef VIXL_SIMULATOR
|
||||
void ExecuteDispatch(FEXCore::Core::CpuStateFrame* Frame);
|
||||
void ExecuteJITCallback(FEXCore::Core::CpuStateFrame* Frame, uint64_t RIP);
|
||||
#else
|
||||
void ExecuteDispatch(FEXCore::Core::CpuStateFrame* Frame) {
|
||||
DispatchPtr(Frame, false);
|
||||
}
|
||||
|
||||
void ExecuteJITCallback(FEXCore::Core::CpuStateFrame* Frame, uint64_t RIP) {
|
||||
CallbackPtr(Frame, RIP);
|
||||
}
|
||||
#endif
|
||||
|
||||
uint64_t GetExitFunctionLinkerAddress() const {
|
||||
return ExitFunctionLinkerAddress;
|
||||
}
|
||||
|
||||
SignalDelegatorConfig MakeSignalDelegatorConfig() const;
|
||||
|
||||
protected:
|
||||
FEXCore::Context::ContextImpl* CTX;
|
||||
|
||||
using AsmDispatch = void (*)(FEXCore::Core::CpuStateFrame* Frame, bool SingleInst);
|
||||
using JITCallback = void (*)(FEXCore::Core::CpuStateFrame* Frame, uint64_t RIP);
|
||||
|
||||
AsmDispatch DispatchPtr;
|
||||
JITCallback CallbackPtr;
|
||||
private:
|
||||
/**
|
||||
* @name Dispatch Helper functions
|
||||
* @{ */
|
||||
@@ -83,22 +59,68 @@ private:
|
||||
uint64_t GuestSignal_SIGILL {};
|
||||
uint64_t GuestSignal_SIGTRAP {};
|
||||
uint64_t GuestSignal_SIGSEGV {};
|
||||
uint64_t IntCallbackReturnAddress {};
|
||||
|
||||
uint64_t PauseReturnInstruction {};
|
||||
std::array<uint64_t, FallbackABI::FABI_UNKNOWN> ABIPointers {};
|
||||
|
||||
/** @} */
|
||||
|
||||
uint64_t Start {};
|
||||
uint64_t End {};
|
||||
|
||||
void InitThreadPointers(FEXCore::Core::InternalThreadState* Thread);
|
||||
|
||||
#ifdef VIXL_SIMULATOR
|
||||
void ExecuteDispatch(FEXCore::Core::CpuStateFrame* Frame);
|
||||
void ExecuteJITCallback(FEXCore::Core::CpuStateFrame* Frame, uint64_t RIP);
|
||||
#else
|
||||
void ExecuteDispatch(FEXCore::Core::CpuStateFrame* Frame) {
|
||||
DispatchPtr(Frame);
|
||||
}
|
||||
|
||||
void ExecuteJITCallback(FEXCore::Core::CpuStateFrame* Frame, uint64_t RIP) {
|
||||
CallbackPtr(Frame, RIP);
|
||||
}
|
||||
#endif
|
||||
|
||||
uint16_t GetSRAGPRCount() const {
|
||||
// PF/AF are the final two SRA registers.
|
||||
// Only return the SRA for GPRs.
|
||||
return StaticRegisters.size() - 2;
|
||||
}
|
||||
|
||||
uint16_t GetSRAFPRCount() const {
|
||||
return StaticFPRegisters.size();
|
||||
}
|
||||
|
||||
void GetSRAGPRMapping(uint8_t Mapping[16]) const {
|
||||
for (size_t i = 0; i < StaticRegisters.size() - 2; ++i) {
|
||||
Mapping[i] = StaticRegisters[i].Idx();
|
||||
}
|
||||
}
|
||||
|
||||
void GetSRAFPRMapping(uint8_t Mapping[16]) const {
|
||||
for (size_t i = 0; i < StaticFPRegisters.size(); ++i) {
|
||||
Mapping[i] = StaticFPRegisters[i].Idx();
|
||||
}
|
||||
}
|
||||
|
||||
protected:
|
||||
FEXCore::Context::ContextImpl* CTX;
|
||||
|
||||
using AsmDispatch = void (*)(FEXCore::Core::CpuStateFrame* Frame);
|
||||
using JITCallback = void (*)(FEXCore::Core::CpuStateFrame* Frame, uint64_t RIP);
|
||||
|
||||
AsmDispatch DispatchPtr;
|
||||
JITCallback CallbackPtr;
|
||||
private:
|
||||
// Long division helpers
|
||||
uint64_t LUDIVHandlerAddress {};
|
||||
uint64_t LDIVHandlerAddress {};
|
||||
|
||||
void EmitDispatcher();
|
||||
uint64_t GenerateABICall(FallbackABI ABI);
|
||||
|
||||
FEX_CONFIG_OPT(DisableL2Cache, DISABLEL2CACHE);
|
||||
};
|
||||
|
||||
} // namespace FEXCore::CPU
|
||||
File diff suppressed because it is too large.
Load diff
@@ -2,59 +2,40 @@
|
||||
#pragma once
|
||||
|
||||
#include "Interface/Core/X86Tables/X86Tables.h"
|
||||
#include "Interface/IR/IR.h"
|
||||
|
||||
#include <FEXCore/Config/Config.h>
|
||||
#include <FEXCore/Core/CodeCache.h>
|
||||
#include <FEXCore/Utils/ThreadPoolAllocator.h>
|
||||
#include <FEXCore/HLE/SyscallHandler.h>
|
||||
#include <FEXCore/Utils/Telemetry.h>
|
||||
#include <FEXCore/fextl/set.h>
|
||||
#include <FEXCore/fextl/vector.h>
|
||||
#include <FEXCore/fextl/robin_map.h>
|
||||
|
||||
#include <array>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <optional>
|
||||
#include <stddef.h>
|
||||
|
||||
namespace FEXCore::Context {
|
||||
class ContextImpl;
|
||||
}
|
||||
namespace FEXCore::HLE {
|
||||
enum class SyscallOSABI;
|
||||
}
|
||||
|
||||
namespace FEXCore::Frontend {
|
||||
class Decoder final {
|
||||
public:
|
||||
enum class DecodedBlockStatus {
|
||||
SUCCESS,
|
||||
INVALID_INST,
|
||||
NOEXEC_INST,
|
||||
PARTIAL_DECODE_INST,
|
||||
BAD_RELOCATION,
|
||||
};
|
||||
|
||||
// New Frontend decoding
|
||||
struct DecodedBlocks final {
|
||||
uint64_t Entry {};
|
||||
uint64_t Size {};
|
||||
uint64_t NumInstructions {};
|
||||
FEXCore::X86Tables::DecodedInst* DecodedInstructions;
|
||||
DecodedBlockStatus BlockStatus;
|
||||
bool IsEntryPoint {};
|
||||
bool ForceFullSMCDetection {};
|
||||
bool HasInvalidInstruction {};
|
||||
};
|
||||
|
||||
struct DecodedBlockInformation final {
|
||||
uint64_t TotalInstructionCount;
|
||||
bool Is64BitMode {};
|
||||
fextl::vector<DecodedBlocks> Blocks;
|
||||
fextl::set<uint64_t> EntryPoints;
|
||||
fextl::set<uint64_t> CodePages; // Start addresses of all pages touching the block
|
||||
};
|
||||
|
||||
Decoder(FEXCore::Core::InternalThreadState* Thread);
|
||||
void DecodeInstructionsAtEntry(FEXCore::Core::InternalThreadState* Thread, const uint8_t* InstStream, uint64_t PC, uint64_t MaxInst);
|
||||
Decoder(FEXCore::Context::ContextImpl* ctx);
|
||||
void DecodeInstructionsAtEntry(const uint8_t* InstStream, uint64_t PC, uint64_t MaxInst,
|
||||
std::function<void(uint64_t BlockEntry, uint64_t Start, uint64_t Length)> AddContainedCodePage);
|
||||
|
||||
const DecodedBlockInformation* GetDecodedBlockInfo() const {
|
||||
return &BlockInfo;
|
||||
@@ -63,6 +44,9 @@ public:
|
||||
uint64_t DecodedMinAddress {};
|
||||
uint64_t DecodedMaxAddress {~0ULL};
|
||||
|
||||
void SetSectionMaxAddress(uint64_t v) {
|
||||
SectionMaxAddress = v;
|
||||
}
|
||||
void SetExternalBranches(fextl::set<uint64_t>* v) {
|
||||
ExternalBranches = v;
|
||||
}
|
||||
@@ -71,10 +55,6 @@ public:
|
||||
PoolObject.DelayedDisownBuffer();
|
||||
}
|
||||
|
||||
void ResetExecutableRangeCache() {
|
||||
ExecutableRangeBase = ExecutableRangeEnd = 0;
|
||||
}
|
||||
|
||||
private:
|
||||
// To pass any information from instruction prefixes
|
||||
// down into the actual instruction handling machinery.
|
||||
@@ -84,27 +64,19 @@ private:
|
||||
bool L; // VEX.L bit (if set then 256 bit operation, if unset then scalar or 128-bit operation)
|
||||
};
|
||||
|
||||
FEXCore::Core::InternalThreadState* Thread;
|
||||
FEXCore::Context::ContextImpl* CTX;
|
||||
const FEXCore::HLE::SyscallOSABI OSABI {};
|
||||
|
||||
FEX_CONFIG_OPT(EnableCodeCacheValidation, ENABLECODECACHEVALIDATION);
|
||||
|
||||
bool DecodeInstructionImpl(uint64_t PC);
|
||||
DecodedBlockStatus DecodeInstruction(uint64_t PC);
|
||||
bool DecodeInstruction(uint64_t PC);
|
||||
|
||||
void BranchTargetInMultiblockRange();
|
||||
bool IsBranchMonoTailcall(uint64_t NumInstructions) const;
|
||||
bool InstCanContinue() const;
|
||||
|
||||
void AddBranchTarget(uint64_t Target);
|
||||
|
||||
bool CheckRangeExecutable(uint64_t Address, uint64_t Size);
|
||||
|
||||
uint8_t ReadByte();
|
||||
std::optional<uint8_t> PeekByte(uint8_t Offset);
|
||||
std::pair<uint64_t, bool> ReadData(uint8_t Size);
|
||||
|
||||
uint8_t PeekByte(uint8_t Offset) const;
|
||||
uint64_t ReadData(uint8_t Size);
|
||||
void SkipBytes(uint8_t Size) {
|
||||
InstructionSize += Size;
|
||||
}
|
||||
@@ -112,36 +84,26 @@ private:
|
||||
bool NormalOp(const FEXCore::X86Tables::X86InstInfo* Info, uint16_t Op, DecodedHeader Options = {});
|
||||
bool NormalOpHeader(const FEXCore::X86Tables::X86InstInfo* Info, uint16_t Op);
|
||||
|
||||
void DecodeREXIfValid(int8_t ExpectedOffset = -1);
|
||||
|
||||
static constexpr size_t DefaultDecodedBufferSize = 0x10000;
|
||||
FEXCore::X86Tables::DecodedInst* DecodedBuffer {};
|
||||
Utils::PoolBufferWithTimedRetirement<FEXCore::X86Tables::DecodedInst*, 5000, 500> PoolObject;
|
||||
size_t DecodedSize {};
|
||||
|
||||
uint64_t ExecutableRangeBase {};
|
||||
uint64_t ExecutableRangeEnd {};
|
||||
bool ExecutableRangeWritable {};
|
||||
bool HitNonExecutableRange {};
|
||||
bool HitBadRelocation {};
|
||||
|
||||
const uint8_t* InstStream {};
|
||||
IR::OpSize GetGPROpSize() const {
|
||||
return BlockInfo.Is64BitMode ? IR::OpSize::i64Bit : IR::OpSize::i32Bit;
|
||||
}
|
||||
|
||||
static constexpr size_t MAX_INST_SIZE = 15;
|
||||
uint8_t InstructionSize {};
|
||||
std::array<uint8_t, MAX_INST_SIZE> Instruction;
|
||||
uint8_t LastEscapePrefix {};
|
||||
FEXCore::X86Tables::DecodedInst* DecodeInst;
|
||||
|
||||
// This is for multiblock data tracking
|
||||
bool SymbolAvailable {false};
|
||||
uint64_t EntryPoint {};
|
||||
uint64_t MaxCondBranchForward {};
|
||||
uint64_t MaxCondBranchBackwards {~0ULL};
|
||||
uint64_t SymbolMaxAddress {};
|
||||
uint64_t SymbolMinAddress {~0ULL};
|
||||
uint64_t SectionMaxAddress {~0ULL};
|
||||
uint64_t SectionMinAddress {};
|
||||
uint64_t NextBlockStartAddress {~0ULL};
|
||||
|
||||
DecodedBlockInformation BlockInfo;
|
||||
@@ -150,8 +112,6 @@ private:
|
||||
fextl::set<uint64_t> VisitedBlocks;
|
||||
fextl::set<uint64_t>* ExternalBranches {nullptr};
|
||||
|
||||
const fextl::robin_map<uint32_t, GuestRelocationType>* Relocations {nullptr};
|
||||
|
||||
// ModRM rm decoding
|
||||
using DecodeModRMPtr = void (FEXCore::Frontend::Decoder::*)(X86Tables::DecodedOperand* Operand, X86Tables::ModRMDecoded ModRM);
|
||||
void DecodeModRM_16(X86Tables::DecodedOperand* Operand, X86Tables::ModRMDecoded ModRM);
|
||||
@@ -162,11 +122,6 @@ private:
|
||||
&FEXCore::Frontend::Decoder::DecodeModRM_16,
|
||||
};
|
||||
|
||||
const std::array<X86Tables::X86InstInfo, X86Tables::MAX_X87_TABLE_SIZE>* X87Table;
|
||||
|
||||
const std::array<X86Tables::X86InstInfo, X86Tables::MAX_VEX_TABLE_SIZE>* VEXTable {};
|
||||
const std::array<X86Tables::X86InstInfo, X86Tables::MAX_VEX_GROUP_TABLE_SIZE>* VEXTableGroup {};
|
||||
|
||||
const uint8_t* AdjustAddrForSpecialRegion(const uint8_t* _InstStream, uint64_t EntryPoint, uint64_t RIP);
|
||||
};
|
||||
} // namespace FEXCore::Frontend
|
||||
@@ -6,7 +6,7 @@
|
||||
#include "Interface/IR/IR.h"
|
||||
|
||||
#include <FEXCore/Debug/InternalThreadState.h>
|
||||
#include <FEXCore/Utils/SHMStats.h>
|
||||
#include <FEXCore/Utils/Profiler.h>
|
||||
|
||||
namespace FEXCore::CPU {
|
||||
FEXCORE_PRESERVE_ALL_ATTR static softfloat_state SoftFloatStateFromFCW(uint16_t FCW, bool Force80BitPrecision = false) {
|
||||
@@ -36,48 +36,18 @@ FEXCORE_PRESERVE_ALL_ATTR static softfloat_state SoftFloatStateFromFCW(uint16_t
|
||||
return State;
|
||||
}
|
||||
|
||||
FEXCORE_PRESERVE_ALL_ATTR static void HandleX87Exception(const softfloat_state& State, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
// Check for Invalid Operation exception (bit 0 of X87 status word)
|
||||
if (State.exceptionFlags & softfloat_flag_invalid) {
|
||||
Frame->State.flags[FEXCore::X86State::X87FLAG_IE_LOC] = 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Wrapper for SoftFloat state to handle X87 exceptions
|
||||
class ScopedSoftFloatState {
|
||||
public:
|
||||
FEXCORE_PRESERVE_ALL_ATTR ScopedSoftFloatState(uint16_t FCW, FEXCore::Core::CpuStateFrame* Frame, bool Force80BitPrecision = false)
|
||||
: State(SoftFloatStateFromFCW(FCW, Force80BitPrecision))
|
||||
, Frame(Frame) {}
|
||||
|
||||
FEXCORE_PRESERVE_ALL_ATTR ~ScopedSoftFloatState() {
|
||||
HandleX87Exception(State, Frame);
|
||||
}
|
||||
|
||||
// Disable copy and move to ensure RAII semantics
|
||||
ScopedSoftFloatState(const ScopedSoftFloatState&) = delete;
|
||||
ScopedSoftFloatState& operator=(const ScopedSoftFloatState&) = delete;
|
||||
ScopedSoftFloatState(ScopedSoftFloatState&&) = delete;
|
||||
ScopedSoftFloatState& operator=(ScopedSoftFloatState&&) = delete;
|
||||
|
||||
softfloat_state State;
|
||||
|
||||
private:
|
||||
FEXCore::Core::CpuStateFrame* Frame;
|
||||
};
|
||||
|
||||
template<>
|
||||
struct OpHandlers<IR::OP_F80CVTTO> {
|
||||
FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle4(uint16_t FCW, float src, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
|
||||
ScopedSoftFloatState State {FCW, Frame};
|
||||
return X80SoftFloat(&State.State, src);
|
||||
softfloat_state State = SoftFloatStateFromFCW(FCW);
|
||||
return X80SoftFloat(&State, src);
|
||||
}
|
||||
|
||||
FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle8(uint16_t FCW, double src, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
|
||||
ScopedSoftFloatState State {FCW, Frame};
|
||||
return X80SoftFloat(&State.State, src);
|
||||
softfloat_state State = SoftFloatStateFromFCW(FCW);
|
||||
return X80SoftFloat(&State, src);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -85,12 +55,12 @@ template<>
|
||||
struct OpHandlers<IR::OP_F80CMP> {
|
||||
FEXCORE_PRESERVE_ALL_ATTR static uint64_t handle(uint16_t FCW, VectorRegType Src1, VectorRegType Src2, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
|
||||
ScopedSoftFloatState State {FCW, Frame};
|
||||
softfloat_state State = SoftFloatStateFromFCW(FCW);
|
||||
|
||||
bool eq, lt, nan;
|
||||
uint64_t ResultFlags = 0;
|
||||
|
||||
X80SoftFloat::FCMP(&State.State, Src1, Src2, &eq, <, &nan);
|
||||
X80SoftFloat::FCMP(&State, Src1, Src2, &eq, <, &nan);
|
||||
if (lt) {
|
||||
ResultFlags |= (1 << IR::FCMP_FLAG_LT);
|
||||
}
|
||||
@@ -108,14 +78,14 @@ template<>
|
||||
struct OpHandlers<IR::OP_F80CVT> {
|
||||
FEXCORE_PRESERVE_ALL_ATTR static float handle4(uint16_t FCW, VectorRegType src, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
|
||||
ScopedSoftFloatState State {FCW, Frame};
|
||||
return X80SoftFloat(src).ToF32(&State.State);
|
||||
softfloat_state State = SoftFloatStateFromFCW(FCW);
|
||||
return X80SoftFloat(src).ToF32(&State);
|
||||
}
|
||||
|
||||
FEXCORE_PRESERVE_ALL_ATTR static double handle8(uint16_t FCW, VectorRegType src, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
|
||||
ScopedSoftFloatState State {FCW, Frame};
|
||||
return X80SoftFloat(src).ToF64(&State.State);
|
||||
softfloat_state State = SoftFloatStateFromFCW(FCW);
|
||||
return X80SoftFloat(src).ToF64(&State);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -123,26 +93,26 @@ template<>
|
||||
struct OpHandlers<IR::OP_F80CVTINT> {
|
||||
FEXCORE_PRESERVE_ALL_ATTR static int16_t handle2(uint16_t FCW, VectorRegType src, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
|
||||
ScopedSoftFloatState State {FCW, Frame};
|
||||
return X80SoftFloat(src).ToI16(&State.State);
|
||||
softfloat_state State = SoftFloatStateFromFCW(FCW);
|
||||
return X80SoftFloat(src).ToI16(&State);
|
||||
}
|
||||
|
||||
FEXCORE_PRESERVE_ALL_ATTR static int32_t handle4(uint16_t FCW, VectorRegType src, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
|
||||
ScopedSoftFloatState State {FCW, Frame};
|
||||
return X80SoftFloat(src).ToI32(&State.State);
|
||||
softfloat_state State = SoftFloatStateFromFCW(FCW);
|
||||
return X80SoftFloat(src).ToI32(&State);
|
||||
}
|
||||
|
||||
FEXCORE_PRESERVE_ALL_ATTR static int64_t handle8(uint16_t FCW, VectorRegType src, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
|
||||
ScopedSoftFloatState State {FCW, Frame};
|
||||
return X80SoftFloat(src).ToI64(&State.State);
|
||||
softfloat_state State = SoftFloatStateFromFCW(FCW);
|
||||
return X80SoftFloat(src).ToI64(&State);
|
||||
}
|
||||
|
||||
FEXCORE_PRESERVE_ALL_ATTR static int16_t handle2t(uint16_t FCW, VectorRegType src, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
|
||||
ScopedSoftFloatState State {FCW, Frame};
|
||||
auto rv = extF80_to_i32(&State.State, X80SoftFloat(src), softfloat_round_minMag, false);
|
||||
softfloat_state State = SoftFloatStateFromFCW(FCW);
|
||||
auto rv = extF80_to_i32(&State, X80SoftFloat(src), softfloat_round_minMag, false);
|
||||
|
||||
if (rv > INT16_MAX || rv < INT16_MIN) {
|
||||
///< Indefinite value for 16-bit conversions.
|
||||
@@ -154,14 +124,14 @@ struct OpHandlers<IR::OP_F80CVTINT> {
|
||||
|
||||
FEXCORE_PRESERVE_ALL_ATTR static int32_t handle4t(uint16_t FCW, VectorRegType src, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
|
||||
ScopedSoftFloatState State {FCW, Frame};
|
||||
return extF80_to_i32(&State.State, X80SoftFloat(src), softfloat_round_minMag, false);
|
||||
softfloat_state State = SoftFloatStateFromFCW(FCW);
|
||||
return extF80_to_i32(&State, X80SoftFloat(src), softfloat_round_minMag, false);
|
||||
}
|
||||
|
||||
FEXCORE_PRESERVE_ALL_ATTR static int64_t handle8t(uint16_t FCW, VectorRegType src, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
|
||||
ScopedSoftFloatState State {FCW, Frame};
|
||||
return extF80_to_i64(&State.State, X80SoftFloat(src), softfloat_round_minMag, false);
|
||||
softfloat_state State = SoftFloatStateFromFCW(FCW);
|
||||
return extF80_to_i64(&State, X80SoftFloat(src), softfloat_round_minMag, false);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -182,8 +152,8 @@ template<>
|
||||
struct OpHandlers<IR::OP_F80ROUND> {
|
||||
FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
|
||||
ScopedSoftFloatState State {FCW, Frame, true};
|
||||
return X80SoftFloat::FRNDINT(&State.State, Src1);
|
||||
softfloat_state State = SoftFloatStateFromFCW(FCW, true);
|
||||
return X80SoftFloat::FRNDINT(&State, Src1);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -191,8 +161,8 @@ template<>
|
||||
struct OpHandlers<IR::OP_F80F2XM1> {
|
||||
FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
|
||||
ScopedSoftFloatState State {FCW, Frame, true};
|
||||
return X80SoftFloat::F2XM1(&State.State, Src1);
|
||||
softfloat_state State = SoftFloatStateFromFCW(FCW, true);
|
||||
return X80SoftFloat::F2XM1(&State, Src1);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -200,8 +170,8 @@ template<>
|
||||
struct OpHandlers<IR::OP_F80TAN> {
|
||||
FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
|
||||
ScopedSoftFloatState State {FCW, Frame, true};
|
||||
return X80SoftFloat::FTAN(&State.State, Src1);
|
||||
softfloat_state State = SoftFloatStateFromFCW(FCW, true);
|
||||
return X80SoftFloat::FTAN(&State, Src1);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -209,8 +179,8 @@ template<>
|
||||
struct OpHandlers<IR::OP_F80SQRT> {
|
||||
FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
|
||||
ScopedSoftFloatState State {FCW, Frame};
|
||||
return X80SoftFloat::FSQRT(&State.State, Src1);
|
||||
softfloat_state State = SoftFloatStateFromFCW(FCW);
|
||||
return X80SoftFloat::FSQRT(&State, Src1);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -218,8 +188,8 @@ template<>
|
||||
struct OpHandlers<IR::OP_F80SIN> {
|
||||
FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
|
||||
ScopedSoftFloatState State {FCW, Frame, true};
|
||||
return X80SoftFloat::FSIN(&State.State, Src1);
|
||||
softfloat_state State = SoftFloatStateFromFCW(FCW, true);
|
||||
return X80SoftFloat::FSIN(&State, Src1);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -227,8 +197,8 @@ template<>
|
||||
struct OpHandlers<IR::OP_F80COS> {
|
||||
FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
|
||||
ScopedSoftFloatState State {FCW, Frame, true};
|
||||
return X80SoftFloat::FCOS(&State.State, Src1);
|
||||
softfloat_state State = SoftFloatStateFromFCW(FCW, true);
|
||||
return X80SoftFloat::FCOS(&State, Src1);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -236,8 +206,8 @@ template<>
|
||||
struct OpHandlers<IR::OP_F80SINCOS> {
|
||||
FEXCORE_PRESERVE_ALL_ATTR static VectorRegPairType handle(uint16_t FCW, VectorRegType Src1, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
|
||||
ScopedSoftFloatState State {FCW, Frame, true};
|
||||
return FEXCore::MakeVectorRegPair(X80SoftFloat::FSIN(&State.State, Src1), X80SoftFloat::FCOS(&State.State, Src1));
|
||||
softfloat_state State = SoftFloatStateFromFCW(FCW, true);
|
||||
return FEXCore::MakeVectorRegPair(X80SoftFloat::FSIN(&State, Src1), X80SoftFloat::FCOS(&State, Src1));
|
||||
}
|
||||
};
|
||||
|
||||
@@ -261,8 +231,8 @@ template<>
|
||||
struct OpHandlers<IR::OP_F80ADD> {
|
||||
FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, VectorRegType Src2, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
|
||||
ScopedSoftFloatState State {FCW, Frame};
|
||||
return X80SoftFloat::FADD(&State.State, Src1, Src2);
|
||||
softfloat_state State = SoftFloatStateFromFCW(FCW);
|
||||
return X80SoftFloat::FADD(&State, Src1, Src2);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -270,8 +240,8 @@ template<>
|
||||
struct OpHandlers<IR::OP_F80SUB> {
|
||||
FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, VectorRegType Src2, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
|
||||
ScopedSoftFloatState State {FCW, Frame};
|
||||
return X80SoftFloat::FSUB(&State.State, Src1, Src2);
|
||||
softfloat_state State = SoftFloatStateFromFCW(FCW);
|
||||
return X80SoftFloat::FSUB(&State, Src1, Src2);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -279,8 +249,8 @@ template<>
|
||||
struct OpHandlers<IR::OP_F80MUL> {
|
||||
FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, VectorRegType Src2, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
|
||||
ScopedSoftFloatState State {FCW, Frame};
|
||||
return X80SoftFloat::FMUL(&State.State, Src1, Src2);
|
||||
softfloat_state State = SoftFloatStateFromFCW(FCW);
|
||||
return X80SoftFloat::FMUL(&State, Src1, Src2);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -288,8 +258,8 @@ template<>
|
||||
struct OpHandlers<IR::OP_F80DIV> {
|
||||
FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, VectorRegType Src2, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
|
||||
ScopedSoftFloatState State {FCW, Frame};
|
||||
return X80SoftFloat::FDIV(&State.State, Src1, Src2);
|
||||
softfloat_state State = SoftFloatStateFromFCW(FCW);
|
||||
return X80SoftFloat::FDIV(&State, Src1, Src2);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -297,8 +267,8 @@ template<>
|
||||
struct OpHandlers<IR::OP_F80FYL2X> {
|
||||
FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, VectorRegType Src2, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
|
||||
ScopedSoftFloatState State {FCW, Frame, true};
|
||||
return X80SoftFloat::FYL2X(&State.State, Src1, Src2);
|
||||
softfloat_state State = SoftFloatStateFromFCW(FCW, true);
|
||||
return X80SoftFloat::FYL2X(&State, Src1, Src2);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -306,8 +276,8 @@ template<>
|
||||
struct OpHandlers<IR::OP_F80ATAN> {
|
||||
FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, VectorRegType Src2, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
|
||||
ScopedSoftFloatState State {FCW, Frame, true};
|
||||
return X80SoftFloat::FATAN(&State.State, Src1, Src2);
|
||||
softfloat_state State = SoftFloatStateFromFCW(FCW, true);
|
||||
return X80SoftFloat::FATAN(&State, Src1, Src2);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -315,8 +285,8 @@ template<>
|
||||
struct OpHandlers<IR::OP_F80FPREM1> {
|
||||
FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, VectorRegType Src2, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
|
||||
ScopedSoftFloatState State {FCW, Frame, true};
|
||||
return X80SoftFloat::FREM1(&State.State, Src1, Src2);
|
||||
softfloat_state State = SoftFloatStateFromFCW(FCW, true);
|
||||
return X80SoftFloat::FREM1(&State, Src1, Src2);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -324,8 +294,8 @@ template<>
|
||||
struct OpHandlers<IR::OP_F80FPREM> {
|
||||
FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, VectorRegType Src2, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
|
||||
ScopedSoftFloatState State {FCW, Frame, true};
|
||||
return X80SoftFloat::FREM(&State.State, Src1, Src2);
|
||||
softfloat_state State = SoftFloatStateFromFCW(FCW, true);
|
||||
return X80SoftFloat::FREM(&State, Src1, Src2);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -333,14 +303,14 @@ template<>
|
||||
struct OpHandlers<IR::OP_F80SCALE> {
|
||||
FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1, VectorRegType Src2, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
|
||||
ScopedSoftFloatState State {FCW, Frame, true};
|
||||
return X80SoftFloat::FSCALE(&State.State, Src1, Src2);
|
||||
softfloat_state State = SoftFloatStateFromFCW(FCW, true);
|
||||
return X80SoftFloat::FSCALE(&State, Src1, Src2);
|
||||
}
|
||||
};
|
||||
|
||||
template<>
|
||||
struct OpHandlers<IR::OP_F64SIN> {
|
||||
FEXCORE_PRESERVE_ALL_ATTR static double handle(double src, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PRESERVE_ALL_ATTR static double handle(uint16_t FCW, double src, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
|
||||
return sin(src);
|
||||
}
|
||||
@@ -348,7 +318,7 @@ struct OpHandlers<IR::OP_F64SIN> {
|
||||
|
||||
template<>
|
||||
struct OpHandlers<IR::OP_F64COS> {
|
||||
FEXCORE_PRESERVE_ALL_ATTR static double handle(double src, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PRESERVE_ALL_ATTR static double handle(uint16_t FCW, double src, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
|
||||
return cos(src);
|
||||
}
|
||||
@@ -356,7 +326,7 @@ struct OpHandlers<IR::OP_F64COS> {
|
||||
|
||||
template<>
|
||||
struct OpHandlers<IR::OP_F64SINCOS> {
|
||||
FEXCORE_PRESERVE_ALL_ATTR static VectorScalarF64Pair handle(double src, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PRESERVE_ALL_ATTR static VectorScalarF64Pair handle(uint16_t FCW, double src, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
|
||||
double sin, cos;
|
||||
#ifdef _WIN32
|
||||
@@ -371,7 +341,7 @@ struct OpHandlers<IR::OP_F64SINCOS> {
|
||||
|
||||
template<>
|
||||
struct OpHandlers<IR::OP_F64TAN> {
|
||||
FEXCORE_PRESERVE_ALL_ATTR static double handle(double src, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PRESERVE_ALL_ATTR static double handle(uint16_t FCW, double src, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
|
||||
return tan(src);
|
||||
}
|
||||
@@ -379,7 +349,7 @@ struct OpHandlers<IR::OP_F64TAN> {
|
||||
|
||||
template<>
|
||||
struct OpHandlers<IR::OP_F64F2XM1> {
|
||||
FEXCORE_PRESERVE_ALL_ATTR static double handle(double src, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PRESERVE_ALL_ATTR static double handle(uint16_t FCW, double src, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
|
||||
return exp2(src) - 1.0;
|
||||
}
|
||||
@@ -387,7 +357,7 @@ struct OpHandlers<IR::OP_F64F2XM1> {
|
||||
|
||||
template<>
|
||||
struct OpHandlers<IR::OP_F64ATAN> {
|
||||
FEXCORE_PRESERVE_ALL_ATTR static double handle(double src1, double src2, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PRESERVE_ALL_ATTR static double handle(uint16_t FCW, double src1, double src2, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
|
||||
return atan2(src1, src2);
|
||||
}
|
||||
@@ -395,7 +365,7 @@ struct OpHandlers<IR::OP_F64ATAN> {
|
||||
|
||||
template<>
|
||||
struct OpHandlers<IR::OP_F64FPREM> {
|
||||
FEXCORE_PRESERVE_ALL_ATTR static double handle(double src1, double src2, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PRESERVE_ALL_ATTR static double handle(uint16_t FCW, double src1, double src2, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
|
||||
return fmod(src1, src2);
|
||||
}
|
||||
@@ -403,7 +373,7 @@ struct OpHandlers<IR::OP_F64FPREM> {
|
||||
|
||||
template<>
|
||||
struct OpHandlers<IR::OP_F64FPREM1> {
|
||||
FEXCORE_PRESERVE_ALL_ATTR static double handle(double src1, double src2, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PRESERVE_ALL_ATTR static double handle(uint16_t FCW, double src1, double src2, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
|
||||
return remainder(src1, src2);
|
||||
}
|
||||
@@ -411,7 +381,7 @@ struct OpHandlers<IR::OP_F64FPREM1> {
|
||||
|
||||
template<>
|
||||
struct OpHandlers<IR::OP_F64FYL2X> {
|
||||
FEXCORE_PRESERVE_ALL_ATTR static double handle(double src1, double src2, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PRESERVE_ALL_ATTR static double handle(uint16_t FCW, double src1, double src2, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
|
||||
return src2 * log2(src1);
|
||||
}
|
||||
@@ -419,7 +389,7 @@ struct OpHandlers<IR::OP_F64FYL2X> {
|
||||
|
||||
template<>
|
||||
struct OpHandlers<IR::OP_F64SCALE> {
|
||||
FEXCORE_PRESERVE_ALL_ATTR static double handle(double src1, double src2, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PRESERVE_ALL_ATTR static double handle(uint16_t FCW, double src1, double src2, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
|
||||
if (src1 == 0.0) { // src1 might be +/- zero
|
||||
return src1; // this will return negative or positive zero if when appropriate
|
||||
@@ -434,17 +404,18 @@ struct OpHandlers<IR::OP_F80BCDSTORE> {
|
||||
FEXCORE_PRESERVE_ALL_ATTR static VectorRegType handle(uint16_t FCW, VectorRegType Src1q, FEXCore::Core::CpuStateFrame* Frame) {
|
||||
FEXCORE_PROFILE_INSTANT_INCREMENT(Frame->Thread, AccumulatedFloatFallbackCount, 1);
|
||||
X80SoftFloat Src1 = Src1q;
|
||||
ScopedSoftFloatState State {FCW, Frame};
|
||||
softfloat_state State = SoftFloatStateFromFCW(FCW);
|
||||
bool Negative = Src1.Sign;
|
||||
|
||||
Src1 = X80SoftFloat::FRNDINT(&State.State, Src1);
|
||||
Src1 = X80SoftFloat::FRNDINT(&State, Src1);
|
||||
|
||||
// Clear the Sign bit
|
||||
Src1.Sign = 0;
|
||||
|
||||
uint64_t Tmp = Src1.ToI64(&State.State);
|
||||
uint64_t Tmp = Src1.ToI64(&State);
|
||||
X80SoftFloat Rv;
|
||||
uint8_t* BCD = reinterpret_cast<uint8_t*>(&Rv);
|
||||
memset(BCD, 0, 10);
|
||||
|
||||
for (size_t i = 0; i < 9; ++i) {
|
||||
if (Tmp == 0) {
|
||||
|
||||
@@ -82,22 +82,24 @@ void InterpreterOps::FillFallbackIndexPointers(Core::FallbackABIInfo* Info, uint
|
||||
reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F80SCALE>::handle)};
|
||||
|
||||
// Double Precision Unary
|
||||
Info[Core::OPINDEX_F64SIN] = {ABIHandlers[FABI_F64_F64_PTR], reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F64SIN>::handle)};
|
||||
Info[Core::OPINDEX_F64COS] = {ABIHandlers[FABI_F64_F64_PTR], reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F64COS>::handle)};
|
||||
Info[Core::OPINDEX_F64SINCOS] = {ABIHandlers[FABI_F64x2_F64_PTR],
|
||||
Info[Core::OPINDEX_F64SIN] = {ABIHandlers[FABI_F64_I16_F64_PTR], reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F64SIN>::handle)};
|
||||
Info[Core::OPINDEX_F64COS] = {ABIHandlers[FABI_F64_I16_F64_PTR], reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F64COS>::handle)};
|
||||
Info[Core::OPINDEX_F64SINCOS] = {ABIHandlers[FABI_F64x2_I16_F64_PTR],
|
||||
reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F64SINCOS>::handle)};
|
||||
Info[Core::OPINDEX_F64TAN] = {ABIHandlers[FABI_F64_F64_PTR], reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F64TAN>::handle)};
|
||||
Info[Core::OPINDEX_F64F2XM1] = {ABIHandlers[FABI_F64_F64_PTR], reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F64F2XM1>::handle)};
|
||||
Info[Core::OPINDEX_F64TAN] = {ABIHandlers[FABI_F64_I16_F64_PTR], reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F64TAN>::handle)};
|
||||
Info[Core::OPINDEX_F64F2XM1] = {ABIHandlers[FABI_F64_I16_F64_PTR],
|
||||
reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F64F2XM1>::handle)};
|
||||
|
||||
// Double Precision Binary
|
||||
Info[Core::OPINDEX_F64ATAN] = {ABIHandlers[FABI_F64_F64_F64_PTR], reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F64ATAN>::handle)};
|
||||
Info[Core::OPINDEX_F64FPREM] = {ABIHandlers[FABI_F64_F64_F64_PTR],
|
||||
Info[Core::OPINDEX_F64ATAN] = {ABIHandlers[FABI_F64_I16_F64_F64_PTR],
|
||||
reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F64ATAN>::handle)};
|
||||
Info[Core::OPINDEX_F64FPREM] = {ABIHandlers[FABI_F64_I16_F64_F64_PTR],
|
||||
reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F64FPREM>::handle)};
|
||||
Info[Core::OPINDEX_F64FPREM1] = {ABIHandlers[FABI_F64_F64_F64_PTR],
|
||||
Info[Core::OPINDEX_F64FPREM1] = {ABIHandlers[FABI_F64_I16_F64_F64_PTR],
|
||||
reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F64FPREM1>::handle)};
|
||||
Info[Core::OPINDEX_F64FYL2X] = {ABIHandlers[FABI_F64_F64_F64_PTR],
|
||||
Info[Core::OPINDEX_F64FYL2X] = {ABIHandlers[FABI_F64_I16_F64_F64_PTR],
|
||||
reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F64FYL2X>::handle)};
|
||||
Info[Core::OPINDEX_F64SCALE] = {ABIHandlers[FABI_F64_F64_F64_PTR],
|
||||
Info[Core::OPINDEX_F64SCALE] = {ABIHandlers[FABI_F64_I16_F64_F64_PTR],
|
||||
reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F64SCALE>::handle)};
|
||||
|
||||
// SSE4.2 string instructions
|
||||
@@ -218,21 +220,21 @@ bool InterpreterOps::GetFallbackHandler(const IR::IROp_Header* IROp, FallbackInf
|
||||
return true; \
|
||||
}
|
||||
|
||||
#define COMMON_UNARY_F64_OP(OP) \
|
||||
case IR::OP_F64##OP: { \
|
||||
*Info = {FABI_F64_F64_PTR, Core::OPINDEX_F64##OP}; \
|
||||
return true; \
|
||||
#define COMMON_UNARY_F64_OP(OP) \
|
||||
case IR::OP_F64##OP: { \
|
||||
*Info = {FABI_F64_I16_F64_PTR, Core::OPINDEX_F64##OP}; \
|
||||
return true; \
|
||||
}
|
||||
#define COMMON_UNARYPAIR_F64_OP(OP) \
|
||||
case IR::OP_F64##OP: { \
|
||||
*Info = {FABI_F64x2_F64_PTR, Core::OPINDEX_F64##OP}; \
|
||||
return true; \
|
||||
#define COMMON_UNARYPAIR_F64_OP(OP) \
|
||||
case IR::OP_F64##OP: { \
|
||||
*Info = {FABI_F64x2_I16_F64_PTR, Core::OPINDEX_F64##OP}; \
|
||||
return true; \
|
||||
}
|
||||
|
||||
#define COMMON_BINARY_F64_OP(OP) \
|
||||
case IR::OP_F64##OP: { \
|
||||
*Info = {FABI_F64_F64_F64_PTR, Core::OPINDEX_F64##OP}; \
|
||||
return true; \
|
||||
#define COMMON_BINARY_F64_OP(OP) \
|
||||
case IR::OP_F64##OP: { \
|
||||
*Info = {FABI_F64_I16_F64_F64_PTR, Core::OPINDEX_F64##OP}; \
|
||||
return true; \
|
||||
}
|
||||
|
||||
// Unary
|
||||
|
||||
@@ -2,14 +2,14 @@
|
||||
#include "Interface/Core/Interpreter/Fallbacks/VectorFallbacks.h"
|
||||
#include "Interface/IR/IR.h"
|
||||
|
||||
#ifdef ARCHITECTURE_arm64
|
||||
#ifdef _M_ARM_64
|
||||
#include <arm_neon.h>
|
||||
#endif
|
||||
|
||||
#include <cstring>
|
||||
|
||||
namespace FEXCore::CPU {
|
||||
#ifdef ARCHITECTURE_arm64
|
||||
#ifdef _M_ARM_64
|
||||
FEXCORE_PRESERVE_ALL_ATTR static int32_t GetImplicitLength(FEXCore::VectorRegType data, uint16_t control) {
|
||||
const auto is_using_words = (control & 1) != 0;
|
||||
|
||||
|
||||
@@ -19,8 +19,8 @@ enum FallbackABI {
|
||||
FABI_F80_I16_I32_PTR,
|
||||
FABI_F32_I16_F80_PTR,
|
||||
FABI_F64_I16_F80_PTR,
|
||||
FABI_F64_F64_PTR,
|
||||
FABI_F64_F64_F64_PTR,
|
||||
FABI_F64_I16_F64_PTR,
|
||||
FABI_F64_I16_F64_F64_PTR,
|
||||
FABI_I16_I16_F80_PTR,
|
||||
FABI_I32_I16_F80_PTR,
|
||||
FABI_I64_I16_F80_PTR,
|
||||
@@ -28,7 +28,7 @@ enum FallbackABI {
|
||||
FABI_F80_I16_F80_PTR,
|
||||
FABI_F80_I16_F80_F80_PTR,
|
||||
FABI_F80x2_I16_F80_PTR,
|
||||
FABI_F64x2_F64_PTR,
|
||||
FABI_F64x2_I16_F64_PTR,
|
||||
FABI_I32_I64_I64_V128_V128_I16,
|
||||
FABI_I32_V128_V128_I16,
|
||||
FABI_UNKNOWN,
|
||||
|
||||
@@ -43,28 +43,21 @@ DEF_BINOP_WITH_CONSTANT(Ror, rorv, ror)
|
||||
DEF_OP(Constant) {
|
||||
auto Op = IROp->C<IR::IROp_Constant>();
|
||||
auto Dst = GetReg(Node);
|
||||
|
||||
const auto PadType = [Pad = Op->Pad]() {
|
||||
switch (Pad) {
|
||||
case IR::ConstPad::NoPad: return CPU::Arm64Emitter::PadType::NOPAD;
|
||||
case IR::ConstPad::DoPad: return CPU::Arm64Emitter::PadType::DOPAD;
|
||||
default: return CPU::Arm64Emitter::PadType::AUTOPAD;
|
||||
}
|
||||
}();
|
||||
LoadConstant(ARMEmitter::Size::i64Bit, Dst, Op->Constant, PadType, Op->MaxBytes);
|
||||
LoadConstant(ARMEmitter::Size::i64Bit, Dst, Op->Constant);
|
||||
}
|
||||
|
||||
DEF_OP(EntrypointOffset) {
|
||||
auto Op = IROp->C<IR::IROp_EntrypointOffset>();
|
||||
|
||||
auto Constant = Entry + Op->Offset;
|
||||
auto Dst = GetReg(Node);
|
||||
uint64_t Mask = ~0ULL;
|
||||
const auto OpSize = IROp->Size;
|
||||
if (OpSize == IR::OpSize::i32Bit) {
|
||||
Mask = 0xFFFF'FFFFULL;
|
||||
}
|
||||
|
||||
InsertGuestRIPMove(GetReg(Node), Constant & Mask);
|
||||
LoadConstant(ARMEmitter::Size::i64Bit, Dst, Constant & Mask);
|
||||
}
|
||||
|
||||
DEF_OP(InlineConstant) {
|
||||
@@ -379,7 +372,7 @@ DEF_OP(CondSubNZCV) {
|
||||
DEF_OP(Neg) {
|
||||
auto Op = IROp->C<IR::IROp_Neg>();
|
||||
|
||||
if (Op->Cond == IR::CondClass::AL) {
|
||||
if (Op->Cond == FEXCore::IR::COND_AL) {
|
||||
neg(ConvertSize48(IROp), GetReg(Node), GetReg(Op->Src));
|
||||
} else {
|
||||
cneg(ConvertSize48(IROp), GetReg(Node), GetReg(Op->Src), MapCC(Op->Cond));
|
||||
@@ -522,12 +515,6 @@ DEF_OP(AndWithFlags) {
|
||||
}
|
||||
}
|
||||
|
||||
DEF_OP(AndShift) {
|
||||
auto Op = IROp->C<IR::IROp_XorShift>();
|
||||
|
||||
and_(ConvertSize48(IROp), GetReg(Node), GetReg(Op->Src1), GetReg(Op->Src2), ConvertIRShiftType(Op->Shift), Op->ShiftAmount);
|
||||
}
|
||||
|
||||
DEF_OP(XorShift) {
|
||||
auto Op = IROp->C<IR::IROp_XorShift>();
|
||||
|
||||
@@ -595,7 +582,7 @@ DEF_OP(ShiftFlags) {
|
||||
and_(ARMEmitter::Size::i32Bit, TMP1, Src2, OpSize == IR::OpSize::i64Bit ? 0x3f : 0x1f);
|
||||
|
||||
ARMEmitter::ForwardLabel Done;
|
||||
(void)cbz(EmitSize, TMP1, &Done);
|
||||
cbz(EmitSize, TMP1, &Done);
|
||||
{
|
||||
// PF/SF/ZF/OF
|
||||
if (OpSize >= IR::OpSize::i32Bit) {
|
||||
@@ -659,7 +646,7 @@ DEF_OP(ShiftFlags) {
|
||||
msr(ARMEmitter::SystemRegister::NZCV, TMP2);
|
||||
}
|
||||
}
|
||||
(void)Bind(&Done);
|
||||
Bind(&Done);
|
||||
|
||||
// TODO: Make RA less dumb so this can't happen (e.g. with late-kill).
|
||||
if (PFOutput != PFTemp) {
|
||||
@@ -676,7 +663,7 @@ DEF_OP(RotateFlags) {
|
||||
|
||||
// If shift=0, flags are unaffected. Wrap the whole implementation in a cbz.
|
||||
ARMEmitter::ForwardLabel Done;
|
||||
(void)cbz(EmitSize, Shift, &Done);
|
||||
cbz(EmitSize, Shift, &Done);
|
||||
{
|
||||
// Extract the last bit shifted in to CF
|
||||
const auto BitSize = IR::OpSizeToSize(Op->Size) * 8;
|
||||
@@ -708,7 +695,7 @@ DEF_OP(RotateFlags) {
|
||||
msr(ARMEmitter::SystemRegister::NZCV, TMP3);
|
||||
}
|
||||
}
|
||||
(void)Bind(&Done);
|
||||
Bind(&Done);
|
||||
}
|
||||
|
||||
DEF_OP(Extr) {
|
||||
@@ -774,14 +761,14 @@ DEF_OP(PDep) {
|
||||
// Now, they're copied, so we can start setting Dest (even if it overlaps with
|
||||
// one of them). Handle early exit case
|
||||
mov(EmitSize, Dest, 0);
|
||||
(void)cbz(EmitSize, OrigMask, &Done);
|
||||
cbz(EmitSize, OrigMask, &Done);
|
||||
|
||||
// Setup for first iteration
|
||||
neg(EmitSize, T0, Mask);
|
||||
and_(EmitSize, T0, T0, Mask);
|
||||
|
||||
// Main loop
|
||||
(void)Bind(&NextBit);
|
||||
Bind(&NextBit);
|
||||
sbfx(EmitSize, T1, Input, 0, 1);
|
||||
eor(EmitSize, Mask, Mask, T0);
|
||||
and_(EmitSize, T0, T1, T0);
|
||||
@@ -789,10 +776,10 @@ DEF_OP(PDep) {
|
||||
orr(EmitSize, Dest, Dest, T0);
|
||||
lsr(EmitSize, Input, Input, 1);
|
||||
and_(EmitSize, T0, Mask, T1);
|
||||
(void)cbnz(EmitSize, T0, &NextBit);
|
||||
cbnz(EmitSize, T0, &NextBit);
|
||||
|
||||
// All done with nothing to do.
|
||||
(void)Bind(&Done);
|
||||
Bind(&Done);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -828,27 +815,27 @@ DEF_OP(PExt) {
|
||||
ARMEmitter::BackwardLabel NextBit;
|
||||
ARMEmitter::ForwardLabel Done;
|
||||
|
||||
(void)cbz(EmitSize, Mask, &EarlyExit);
|
||||
cbz(EmitSize, Mask, &EarlyExit);
|
||||
mov(EmitSize, MaskReg, Mask);
|
||||
mov(EmitSize, ValueReg, Input);
|
||||
mov(EmitSize, Dest, ARMEmitter::Reg::zr);
|
||||
|
||||
// Main loop
|
||||
(void)Bind(&NextBit);
|
||||
(void)cbz(EmitSize, MaskReg, &Done);
|
||||
Bind(&NextBit);
|
||||
cbz(EmitSize, MaskReg, &Done);
|
||||
clz(EmitSize, BitReg, MaskReg);
|
||||
lslv(EmitSize, ValueReg, ValueReg, BitReg);
|
||||
lslv(EmitSize, MaskReg, MaskReg, BitReg);
|
||||
extr(EmitSize, Dest, Dest, ValueReg, OpSizeBitsM1);
|
||||
bfc(EmitSize, MaskReg, OpSizeBitsM1, 1);
|
||||
(void)b(&NextBit);
|
||||
b(&NextBit);
|
||||
|
||||
// Early exit
|
||||
(void)Bind(&EarlyExit);
|
||||
Bind(&EarlyExit);
|
||||
mov(EmitSize, Dest, ARMEmitter::Reg::zr);
|
||||
|
||||
// All done with nothing to do.
|
||||
(void)Bind(&Done);
|
||||
Bind(&Done);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -916,7 +903,7 @@ DEF_OP(Div) {
|
||||
eor(EmitSize, TMP1, TMP1, Upper);
|
||||
|
||||
// If the sign bit matches then the result is zero
|
||||
(void)cbz(EmitSize, TMP1, &Only64Bit);
|
||||
cbz(EmitSize, TMP1, &Only64Bit);
|
||||
|
||||
// Long divide
|
||||
{
|
||||
@@ -924,7 +911,7 @@ DEF_OP(Div) {
|
||||
mov(EmitSize, TMP2, Lower);
|
||||
mov(EmitSize, TMP3, Divisor);
|
||||
|
||||
ldr(TMP4, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.LDIVHandler));
|
||||
ldr(TMP4, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.AArch64.LDIVHandler));
|
||||
|
||||
str<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);
|
||||
blr(TMP4);
|
||||
@@ -935,17 +922,17 @@ DEF_OP(Div) {
|
||||
mov(EmitSize, Remainder, TMP2);
|
||||
|
||||
// Skip 64-bit path
|
||||
(void)b(&LongDIVRet);
|
||||
b(&LongDIVRet);
|
||||
}
|
||||
|
||||
(void)Bind(&Only64Bit);
|
||||
Bind(&Only64Bit);
|
||||
// 64-Bit only
|
||||
{
|
||||
sdiv(EmitSize, Quotient, Lower, Divisor);
|
||||
msub(EmitSize, Remainder, Quotient, Divisor, Lower);
|
||||
}
|
||||
|
||||
(void)Bind(&LongDIVRet);
|
||||
Bind(&LongDIVRet);
|
||||
break;
|
||||
}
|
||||
default: LOGMAN_MSG_A_FMT("Unknown DIV Size: {}", OpSize); break;
|
||||
@@ -999,7 +986,7 @@ DEF_OP(UDiv) {
|
||||
|
||||
// Check the upper bits for zero
|
||||
// If the upper bits are zero then we can do a 64-bit divide
|
||||
(void)cbz(EmitSize, Upper, &Only64Bit);
|
||||
cbz(EmitSize, Upper, &Only64Bit);
|
||||
|
||||
// Long divide
|
||||
{
|
||||
@@ -1007,7 +994,7 @@ DEF_OP(UDiv) {
|
||||
mov(EmitSize, TMP2, Lower);
|
||||
mov(EmitSize, TMP3, Divisor);
|
||||
|
||||
ldr(TMP4, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.LUDIVHandler));
|
||||
ldr(TMP4, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.AArch64.LUDIVHandler));
|
||||
|
||||
str<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);
|
||||
blr(TMP4);
|
||||
@@ -1018,17 +1005,17 @@ DEF_OP(UDiv) {
|
||||
mov(EmitSize, Remainder, TMP2);
|
||||
|
||||
// Skip 64-bit path
|
||||
(void)b(&LongDIVRet);
|
||||
b(&LongDIVRet);
|
||||
}
|
||||
|
||||
(void)Bind(&Only64Bit);
|
||||
Bind(&Only64Bit);
|
||||
// 64-Bit only
|
||||
{
|
||||
udiv(EmitSize, Quotient, Lower, Divisor);
|
||||
msub(EmitSize, Remainder, Quotient, Divisor, Lower);
|
||||
}
|
||||
|
||||
(void)Bind(&LongDIVRet);
|
||||
Bind(&LongDIVRet);
|
||||
break;
|
||||
}
|
||||
default: LOGMAN_MSG_A_FMT("Unknown LUDIV Size: {}", OpSize); break;
|
||||
@@ -1051,50 +1038,34 @@ DEF_OP(Popcount) {
|
||||
const auto Dst = GetReg(Node);
|
||||
const auto Src = GetReg(Op->Src);
|
||||
|
||||
if (CTX->HostFeatures.SupportsCSSC) {
|
||||
switch (OpSize) {
|
||||
case IR::OpSize::i8Bit:
|
||||
uxtb(ARMEmitter::Size::i32Bit, Dst, Src);
|
||||
cnt(ARMEmitter::Size::i32Bit, Dst, Dst);
|
||||
break;
|
||||
case IR::OpSize::i16Bit:
|
||||
uxth(ARMEmitter::Size::i32Bit, Dst, Src);
|
||||
cnt(ARMEmitter::Size::i32Bit, Dst, Dst);
|
||||
break;
|
||||
case IR::OpSize::i32Bit: cnt(ARMEmitter::Size::i32Bit, Dst, Src); break;
|
||||
case IR::OpSize::i64Bit: cnt(ARMEmitter::Size::i64Bit, Dst, Src); break;
|
||||
default: LOGMAN_MSG_A_FMT("Unsupported Popcount size: {}", OpSize);
|
||||
}
|
||||
} else {
|
||||
switch (OpSize) {
|
||||
case IR::OpSize::i8Bit:
|
||||
fmov(ARMEmitter::Size::i32Bit, VTMP1.S(), Src);
|
||||
// only use lowest byte
|
||||
cnt(ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D());
|
||||
break;
|
||||
case IR::OpSize::i16Bit:
|
||||
fmov(ARMEmitter::Size::i32Bit, VTMP1.S(), Src);
|
||||
cnt(ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D());
|
||||
// only count two lowest bytes
|
||||
addp(ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D(), VTMP1.D());
|
||||
break;
|
||||
case IR::OpSize::i32Bit:
|
||||
fmov(ARMEmitter::Size::i32Bit, VTMP1.S(), Src);
|
||||
cnt(ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D());
|
||||
// fmov has zero extended, unused bytes are zero
|
||||
addv(ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D());
|
||||
break;
|
||||
case IR::OpSize::i64Bit:
|
||||
fmov(ARMEmitter::Size::i64Bit, VTMP1.D(), Src);
|
||||
cnt(ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D());
|
||||
// fmov has zero extended, unused bytes are zero
|
||||
addv(ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D());
|
||||
break;
|
||||
default: LOGMAN_MSG_A_FMT("Unsupported Popcount size: {}", OpSize);
|
||||
}
|
||||
|
||||
umov<ARMEmitter::SubRegSize::i8Bit>(Dst, VTMP1, 0);
|
||||
switch (OpSize) {
|
||||
case IR::OpSize::i8Bit:
|
||||
fmov(ARMEmitter::Size::i32Bit, VTMP1.S(), Src);
|
||||
// only use lowest byte
|
||||
cnt(ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D());
|
||||
break;
|
||||
case IR::OpSize::i16Bit:
|
||||
fmov(ARMEmitter::Size::i32Bit, VTMP1.S(), Src);
|
||||
cnt(ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D());
|
||||
// only count two lowest bytes
|
||||
addp(ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D(), VTMP1.D());
|
||||
break;
|
||||
case IR::OpSize::i32Bit:
|
||||
fmov(ARMEmitter::Size::i32Bit, VTMP1.S(), Src);
|
||||
cnt(ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D());
|
||||
// fmov has zero extended, unused bytes are zero
|
||||
addv(ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D());
|
||||
break;
|
||||
case IR::OpSize::i64Bit:
|
||||
fmov(ARMEmitter::Size::i64Bit, VTMP1.D(), Src);
|
||||
cnt(ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D());
|
||||
// fmov has zero extended, unused bytes are zero
|
||||
addv(ARMEmitter::SubRegSize::i8Bit, VTMP1.D(), VTMP1.D());
|
||||
break;
|
||||
default: LOGMAN_MSG_A_FMT("Unsupported Popcount size: {}", OpSize);
|
||||
}
|
||||
|
||||
umov<ARMEmitter::SubRegSize::i8Bit>(Dst, VTMP1, 0);
|
||||
}
|
||||
|
||||
DEF_OP(FindLSB) {
|
||||
@@ -1197,19 +1168,6 @@ DEF_OP(Rev) {
|
||||
}
|
||||
}
|
||||
|
||||
DEF_OP(Rbit) {
|
||||
auto Op = IROp->C<IR::IROp_Rbit>();
|
||||
const auto OpSize = IROp->Size;
|
||||
|
||||
LOGMAN_THROW_A_FMT(OpSize == IR::OpSize::i32Bit || OpSize == IR::OpSize::i64Bit, "Unsupported {} size: {}", __func__, OpSize);
|
||||
const auto EmitSize = ConvertSize48(IROp);
|
||||
|
||||
const auto Dst = GetReg(Node);
|
||||
const auto Src = GetReg(Op->Src);
|
||||
|
||||
rbit(EmitSize, Dst, Src);
|
||||
}
|
||||
|
||||
DEF_OP(Bfi) {
|
||||
auto Op = IROp->C<IR::IROp_Bfi>();
|
||||
const auto EmitSize = ConvertSize(IROp);
|
||||
@@ -1293,16 +1251,6 @@ DEF_OP(Sbfe) {
|
||||
sbfx(ConvertSize(IROp), Dst, Src, Op->lsb, Op->Width);
|
||||
}
|
||||
|
||||
DEF_OP(MaskGenerateFromBitWidth) {
|
||||
auto Op = IROp->C<IR::IROp_MaskGenerateFromBitWidth>();
|
||||
auto BitWidth = GetReg(Op->BitWidth);
|
||||
|
||||
LoadConstant(ARMEmitter::Size::i64Bit, TMP1, -1);
|
||||
cmp(ARMEmitter::Size::i64Bit, BitWidth, 0);
|
||||
lslv(ARMEmitter::Size::i64Bit, TMP2, TMP1, BitWidth);
|
||||
csinv(ARMEmitter::Size::i64Bit, GetReg(Node), TMP1, TMP2, ARMEmitter::Condition::CC_EQ);
|
||||
}
|
||||
|
||||
DEF_OP(Select) {
|
||||
auto Op = IROp->C<IR::IROp_Select>();
|
||||
const auto OpSize = IROp->Size;
|
||||
@@ -1399,12 +1347,12 @@ DEF_OP(VExtractToGPR) {
|
||||
const auto Op = IROp->C<IR::IROp_VExtractToGPR>();
|
||||
const auto OpSize = IROp->Size;
|
||||
|
||||
constexpr auto AVXRegBitSize = Core::CPUState::XMM_AVX_REG_SIZE * 8;
|
||||
[[maybe_unused]] constexpr auto AVXRegBitSize = Core::CPUState::XMM_AVX_REG_SIZE * 8;
|
||||
constexpr auto SSERegBitSize = Core::CPUState::XMM_SSE_REG_SIZE * 8;
|
||||
const auto ElementSizeBits = IR::OpSizeAsBits(Op->Header.ElementSize);
|
||||
|
||||
const auto Offset = ElementSizeBits * Op->Index;
|
||||
const auto Is256Bit = Offset >= SSERegBitSize;
|
||||
[[maybe_unused]] const auto Is256Bit = Offset >= SSERegBitSize;
|
||||
LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, "Need SVE256 support in order to use {} with 256-bit operation", __func__);
|
||||
|
||||
const auto Dst = GetReg(Node);
|
||||
|
||||
@@ -11,32 +11,36 @@ $end_info$
|
||||
#include <FEXCore/Core/Thunks.h>
|
||||
|
||||
namespace FEXCore::CPU {
|
||||
uint64_t GetNamedSymbolLiteral(FEXCore::Context::ContextImpl& CTX, FEXCore::CPU::RelocNamedSymbolLiteral::NamedSymbol Op) {
|
||||
|
||||
uint64_t Arm64JITCore::GetNamedSymbolLiteral(FEXCore::CPU::RelocNamedSymbolLiteral::NamedSymbol Op) {
|
||||
switch (Op) {
|
||||
case FEXCore::CPU::RelocNamedSymbolLiteral::NamedSymbol::SYMBOL_LITERAL_EXITFUNCTION_LINKER:
|
||||
return CTX.Dispatcher->GetExitFunctionLinkerAddress();
|
||||
|
||||
default: ERROR_AND_DIE_FMT("Unknown named symbol literal: {}", static_cast<uint32_t>(Op));
|
||||
return ThreadState->CurrentFrame->Pointers.Common.ExitFunctionLinker;
|
||||
break;
|
||||
default: ERROR_AND_DIE_FMT("Unknown named symbol literal: {}", static_cast<uint32_t>(Op)); break;
|
||||
}
|
||||
return ~0ULL;
|
||||
}
|
||||
|
||||
void Arm64JITCore::InsertNamedThunkRelocation(ARMEmitter::Register Reg, const IR::SHA256Sum& Sum) {
|
||||
Relocation MoveABI {};
|
||||
MoveABI.NamedThunkMove.Header = {.Offset = GetCursorOffset(), .Type = FEXCore::CPU::RelocationTypes::RELOC_NAMED_THUNK_MOVE};
|
||||
MoveABI.NamedThunkMove.Header.Type = FEXCore::CPU::RelocationTypes::RELOC_NAMED_THUNK_MOVE;
|
||||
// Offset is the offset from the entrypoint of the block
|
||||
auto CurrentCursor = GetCursorAddress<uint8_t*>();
|
||||
MoveABI.NamedThunkMove.Offset = CurrentCursor - CodeData.BlockBegin;
|
||||
MoveABI.NamedThunkMove.Symbol = Sum;
|
||||
MoveABI.NamedThunkMove.RegisterIndex = Reg.Idx();
|
||||
|
||||
uint64_t Pointer = reinterpret_cast<uint64_t>(EmitterCTX->ThunkHandler->LookupThunk(Sum));
|
||||
|
||||
// Pointers are required to fit within 48-bit VA space.
|
||||
LoadConstant(ARMEmitter::Size::i64Bit, Reg, Pointer, FEXCore::CPU::Arm64Emitter::PadType::AUTOPAD, 6);
|
||||
LoadConstant(ARMEmitter::Size::i64Bit, Reg, Pointer, EmitterCTX->Config.CacheObjectCodeCompilation());
|
||||
Relocations.emplace_back(MoveABI);
|
||||
}
|
||||
|
||||
Arm64JITCore::NamedSymbolLiteralPair Arm64JITCore::InsertNamedSymbolLiteral(FEXCore::CPU::RelocNamedSymbolLiteral::NamedSymbol Op) {
|
||||
uint64_t Pointer = GetNamedSymbolLiteral(*CTX, Op);
|
||||
uint64_t Pointer = GetNamedSymbolLiteral(Op);
|
||||
|
||||
NamedSymbolLiteralPair Lit {
|
||||
Arm64JITCore::NamedSymbolLiteralPair Lit {
|
||||
.Lit = Pointer,
|
||||
.MoveABI =
|
||||
{
|
||||
@@ -44,77 +48,87 @@ Arm64JITCore::NamedSymbolLiteralPair Arm64JITCore::InsertNamedSymbolLiteral(FEXC
|
||||
{
|
||||
.Header =
|
||||
{
|
||||
.Offset = 0, // Set by PlaceNamedSymbolLiteral
|
||||
.Type = FEXCore::CPU::RelocationTypes::RELOC_NAMED_SYMBOL_LITERAL,
|
||||
},
|
||||
.Symbol = Op,
|
||||
.Offset = 0,
|
||||
},
|
||||
},
|
||||
};
|
||||
return Lit;
|
||||
}
|
||||
|
||||
void Arm64JITCore::PlaceNamedSymbolLiteral(NamedSymbolLiteralPair Lit) {
|
||||
switch (Lit.MoveABI.Header.Type) {
|
||||
case RelocationTypes::RELOC_NAMED_SYMBOL_LITERAL:
|
||||
case RelocationTypes::RELOC_GUEST_RIP_LITERAL: {
|
||||
Lit.MoveABI.Header.Offset = GetCursorOffset();
|
||||
break;
|
||||
}
|
||||
void Arm64JITCore::PlaceNamedSymbolLiteral(NamedSymbolLiteralPair& Lit) {
|
||||
// Offset is the offset from the entrypoint of the block
|
||||
auto CurrentCursor = GetCursorAddress<uint8_t*>();
|
||||
Lit.MoveABI.NamedSymbolLiteral.Offset = CurrentCursor - CodeData.BlockBegin;
|
||||
|
||||
default: ERROR_AND_DIE_FMT("Unknown relocation type for {}", __FUNCTION__);
|
||||
}
|
||||
|
||||
BindOrRestart(&Lit.Loc);
|
||||
Bind(&Lit.Loc);
|
||||
dc64(Lit.Lit);
|
||||
Relocations.emplace_back(Lit.MoveABI);
|
||||
}
|
||||
|
||||
auto Arm64JITCore::InsertGuestRIPLiteral(uint64_t GuestRIP) -> NamedSymbolLiteralPair {
|
||||
return {
|
||||
.Lit = GuestRIP,
|
||||
.MoveABI =
|
||||
{
|
||||
.GuestRIP = {.Header =
|
||||
{
|
||||
.Offset = 0, // Set by PlaceNamedSymbolLiteral
|
||||
.Type = FEXCore::CPU::RelocationTypes::RELOC_GUEST_RIP_LITERAL,
|
||||
},
|
||||
// NOTE: Cache serialization will subtract the guest binary base address later to produce consistency results
|
||||
.GuestRIP = GuestRIP},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
void Arm64JITCore::InsertGuestRIPMove(ARMEmitter::Register Reg, uint64_t Constant) {
|
||||
Relocation MoveABI {};
|
||||
MoveABI.GuestRIP.Header = {.Offset = GetCursorOffset(), .Type = FEXCore::CPU::RelocationTypes::RELOC_GUEST_RIP_MOVE};
|
||||
// NOTE: Cache serialization will subtract the guest binary base address later to produce consistency results
|
||||
MoveABI.GuestRIP.GuestRIP = Constant;
|
||||
MoveABI.GuestRIP.RegisterIndex = Reg.Idx();
|
||||
MoveABI.GuestRIPMove.Header.Type = FEXCore::CPU::RelocationTypes::RELOC_GUEST_RIP_MOVE;
|
||||
// Offset is the offset from the entrypoint of the block
|
||||
auto CurrentCursor = GetCursorAddress<uint8_t*>();
|
||||
MoveABI.GuestRIPMove.Offset = CurrentCursor - CodeData.BlockBegin;
|
||||
MoveABI.GuestRIPMove.GuestRIP = Constant;
|
||||
MoveABI.GuestRIPMove.RegisterIndex = Reg.Idx();
|
||||
|
||||
// Pointers are required to fit within 48-bit VA space.
|
||||
// TODO: Force 6-byte `MaxSize`, with sign extension to 64-bit. Current code not smart enough to handle negatives.
|
||||
// 48-bit sign extension works because x86-64 guests only receive 47-bit VA space, with 48-bit being reserved for kernel.
|
||||
// Additional quirk, "canonical" 48-bit pointers on x86-64, sign extend the 48-bit as well (Which is why kernel pointers are negative).
|
||||
LoadConstant(ARMEmitter::Size::i64Bit, Reg, Constant, FEXCore::CPU::Arm64Emitter::PadType::AUTOPAD);
|
||||
LoadConstant(ARMEmitter::Size::i64Bit, Reg, Constant, EmitterCTX->Config.CacheObjectCodeCompilation());
|
||||
Relocations.emplace_back(MoveABI);
|
||||
}
|
||||
|
||||
fextl::vector<FEXCore::CPU::Relocation> Arm64JITCore::TakeRelocations(uint64_t GuestBaseAddress) {
|
||||
// Rebase relocations to library base address
|
||||
for (auto& Relocation : Relocations) {
|
||||
switch (Relocation.Header.Type) {
|
||||
case FEXCore::CPU::RelocationTypes::RELOC_GUEST_RIP_MOVE:
|
||||
case FEXCore::CPU::RelocationTypes::RELOC_GUEST_RIP_LITERAL: {
|
||||
Relocation.GuestRIP.GuestRIP -= GuestBaseAddress;
|
||||
bool Arm64JITCore::ApplyRelocations(uint64_t GuestEntry, uint64_t CodeEntry, uint64_t CursorEntry, size_t NumRelocations,
|
||||
const char* EntryRelocations) {
|
||||
size_t DataIndex {};
|
||||
for (size_t j = 0; j < NumRelocations; ++j) {
|
||||
const FEXCore::CPU::Relocation* Reloc = reinterpret_cast<const FEXCore::CPU::Relocation*>(&EntryRelocations[DataIndex]);
|
||||
LOGMAN_THROW_A_FMT((DataIndex % alignof(Relocation)) == 0, "Alignment of relocation wasn't adhered to");
|
||||
|
||||
switch (Reloc->Header.Type) {
|
||||
case FEXCore::CPU::RelocationTypes::RELOC_NAMED_SYMBOL_LITERAL: {
|
||||
uint64_t Pointer = GetNamedSymbolLiteral(Reloc->NamedSymbolLiteral.Symbol);
|
||||
// Relocation occurs at the cursorEntry + offset relative to that cursor
|
||||
SetCursorOffset(CursorEntry + Reloc->NamedSymbolLiteral.Offset);
|
||||
|
||||
// Generate a literal so we can place it
|
||||
dc64(Pointer);
|
||||
|
||||
DataIndex += sizeof(Reloc->NamedSymbolLiteral);
|
||||
break;
|
||||
}
|
||||
case FEXCore::CPU::RelocationTypes::RELOC_NAMED_THUNK_MOVE: {
|
||||
uint64_t Pointer = reinterpret_cast<uint64_t>(EmitterCTX->ThunkHandler->LookupThunk(Reloc->NamedThunkMove.Symbol));
|
||||
if (Pointer == ~0ULL) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Relocation occurs at the cursorEntry + offset relative to that cursor.
|
||||
SetCursorOffset(CursorEntry + Reloc->NamedThunkMove.Offset);
|
||||
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Register(Reloc->NamedThunkMove.RegisterIndex), Pointer, true);
|
||||
DataIndex += sizeof(Reloc->NamedThunkMove);
|
||||
break;
|
||||
}
|
||||
case FEXCore::CPU::RelocationTypes::RELOC_GUEST_RIP_MOVE: {
|
||||
// XXX: Reenable once the JIT Object Cache is upstream
|
||||
// XXX: Should spin the relocation list, create a list of guest RIP moves, and ask for them all once, reduces lock contention.
|
||||
uint64_t Pointer = ~0ULL; // EmitterCTX->JITObjectCache->FindRelocatedRIP(Reloc->GuestRIPMove.GuestRIP);
|
||||
if (Pointer == ~0ULL) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Relocation occurs at the cursorEntry + offset relative to that cursor.
|
||||
SetCursorOffset(CursorEntry + Reloc->GuestRIPMove.Offset);
|
||||
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Register(Reloc->GuestRIPMove.RegisterIndex), Pointer, true);
|
||||
DataIndex += sizeof(Reloc->GuestRIPMove);
|
||||
break;
|
||||
}
|
||||
default:;
|
||||
}
|
||||
}
|
||||
|
||||
return std::move(Relocations);
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace FEXCore::CPU
|
||||
@@ -62,27 +62,27 @@ DEF_OP(CASPair) {
|
||||
ARMEmitter::BackwardLabel LoopTop;
|
||||
ARMEmitter::ForwardLabel LoopNotExpected;
|
||||
ARMEmitter::ForwardLabel LoopExpected;
|
||||
(void)Bind(&LoopTop);
|
||||
Bind(&LoopTop);
|
||||
|
||||
// This instruction sequence must be synced with HandleCASPAL_Armv8.
|
||||
ldaxp(EmitSize, TMP2, TMP3, MemSrc);
|
||||
cmp(EmitSize, TMP2, Expected0);
|
||||
ccmp(EmitSize, TMP3, Expected1, ARMEmitter::StatusFlags::None, ARMEmitter::Condition::CC_EQ);
|
||||
(void)b(ARMEmitter::Condition::CC_NE, &LoopNotExpected);
|
||||
b(ARMEmitter::Condition::CC_NE, &LoopNotExpected);
|
||||
stlxp(EmitSize, TMP2, Desired0, Desired1, MemSrc);
|
||||
(void)cbnz(EmitSize, TMP2, &LoopTop);
|
||||
cbnz(EmitSize, TMP2, &LoopTop);
|
||||
mov(EmitSize, Dst0, Expected0);
|
||||
mov(EmitSize, Dst1, Expected1);
|
||||
|
||||
(void)b(&LoopExpected);
|
||||
b(&LoopExpected);
|
||||
|
||||
(void)Bind(&LoopNotExpected);
|
||||
Bind(&LoopNotExpected);
|
||||
mov(EmitSize, Dst0, TMP2.R());
|
||||
mov(EmitSize, Dst1, TMP3.R());
|
||||
// exclusive monitor needs to be cleared here
|
||||
// Might have hit the case where ldaxr was hit but stlxr wasn't
|
||||
clrex();
|
||||
(void)Bind(&LoopExpected);
|
||||
Bind(&LoopExpected);
|
||||
|
||||
// Restore
|
||||
msr(ARMEmitter::SystemRegister::NZCV, TMP1);
|
||||
@@ -114,7 +114,7 @@ DEF_OP(CAS) {
|
||||
ARMEmitter::BackwardLabel LoopTop;
|
||||
ARMEmitter::ForwardLabel LoopNotExpected;
|
||||
ARMEmitter::ForwardLabel LoopExpected;
|
||||
(void)Bind(&LoopTop);
|
||||
Bind(&LoopTop);
|
||||
ldaxr(SubEmitSize, TMP2, MemSrc);
|
||||
if (IROp->Size == IR::OpSize::i8Bit) {
|
||||
cmp(EmitSize, TMP2, Expected, ARMEmitter::ExtendedType::UXTB, 0);
|
||||
@@ -123,21 +123,158 @@ DEF_OP(CAS) {
|
||||
} else {
|
||||
cmp(EmitSize, TMP2, Expected);
|
||||
}
|
||||
(void)b(ARMEmitter::Condition::CC_NE, &LoopNotExpected);
|
||||
b(ARMEmitter::Condition::CC_NE, &LoopNotExpected);
|
||||
stlxr(SubEmitSize, TMP3, Desired, MemSrc);
|
||||
(void)cbnz(EmitSize, TMP3, &LoopTop);
|
||||
cbnz(EmitSize, TMP3, &LoopTop);
|
||||
mov(EmitSize, Dst, Expected);
|
||||
(void)b(&LoopExpected);
|
||||
b(&LoopExpected);
|
||||
|
||||
(void)Bind(&LoopNotExpected);
|
||||
Bind(&LoopNotExpected);
|
||||
mov(EmitSize, Dst, TMP2.R());
|
||||
// exclusive monitor needs to be cleared here
|
||||
// Might have hit the case where ldaxr was hit but stlxr wasn't
|
||||
clrex();
|
||||
(void)Bind(&LoopExpected);
|
||||
Bind(&LoopExpected);
|
||||
}
|
||||
}
|
||||
|
||||
DEF_OP(AtomicAdd) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicAdd>();
|
||||
const auto EmitSize = ConvertSize(IROp);
|
||||
const auto SubEmitSize = ConvertSubRegSize8(IROp->Size);
|
||||
|
||||
auto MemSrc = GetReg(Op->Addr);
|
||||
auto Src = GetReg(Op->Value);
|
||||
|
||||
if (CTX->HostFeatures.SupportsAtomics) {
|
||||
staddl(SubEmitSize, Src, MemSrc);
|
||||
} else {
|
||||
ARMEmitter::BackwardLabel LoopTop;
|
||||
Bind(&LoopTop);
|
||||
ldaxr(SubEmitSize, TMP2, MemSrc);
|
||||
add(EmitSize, TMP2, TMP2, Src);
|
||||
stlxr(SubEmitSize, TMP2, TMP2, MemSrc);
|
||||
cbnz(EmitSize, TMP2, &LoopTop);
|
||||
}
|
||||
}
|
||||
|
||||
DEF_OP(AtomicSub) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicSub>();
|
||||
const auto EmitSize = ConvertSize(IROp);
|
||||
const auto SubEmitSize = ConvertSubRegSize8(IROp->Size);
|
||||
|
||||
auto MemSrc = GetReg(Op->Addr);
|
||||
auto Src = GetReg(Op->Value);
|
||||
|
||||
if (CTX->HostFeatures.SupportsAtomics) {
|
||||
neg(EmitSize, TMP2, Src);
|
||||
staddl(SubEmitSize, TMP2, MemSrc);
|
||||
} else {
|
||||
ARMEmitter::BackwardLabel LoopTop;
|
||||
Bind(&LoopTop);
|
||||
ldaxr(SubEmitSize, TMP2, MemSrc);
|
||||
sub(EmitSize, TMP2, TMP2, Src);
|
||||
stlxr(SubEmitSize, TMP2, TMP2, MemSrc);
|
||||
cbnz(EmitSize, TMP2, &LoopTop);
|
||||
}
|
||||
}
|
||||
|
||||
DEF_OP(AtomicAnd) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicAnd>();
|
||||
const auto EmitSize = ConvertSize(IROp);
|
||||
const auto SubEmitSize = ConvertSubRegSize8(IROp->Size);
|
||||
|
||||
auto MemSrc = GetReg(Op->Addr);
|
||||
auto Src = GetReg(Op->Value);
|
||||
|
||||
if (CTX->HostFeatures.SupportsAtomics) {
|
||||
mvn(EmitSize, TMP2, Src);
|
||||
stclrl(SubEmitSize, TMP2, MemSrc);
|
||||
} else {
|
||||
ARMEmitter::BackwardLabel LoopTop;
|
||||
Bind(&LoopTop);
|
||||
ldaxr(SubEmitSize, TMP2, MemSrc);
|
||||
and_(EmitSize, TMP2, TMP2, Src);
|
||||
stlxr(SubEmitSize, TMP2, TMP2, MemSrc);
|
||||
cbnz(EmitSize, TMP2, &LoopTop);
|
||||
}
|
||||
}
|
||||
|
||||
DEF_OP(AtomicCLR) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicCLR>();
|
||||
const auto EmitSize = ConvertSize(IROp);
|
||||
const auto SubEmitSize = ConvertSubRegSize8(IROp->Size);
|
||||
|
||||
auto MemSrc = GetReg(Op->Addr);
|
||||
auto Src = GetReg(Op->Value);
|
||||
|
||||
if (CTX->HostFeatures.SupportsAtomics) {
|
||||
stclrl(SubEmitSize, Src, MemSrc);
|
||||
} else {
|
||||
ARMEmitter::BackwardLabel LoopTop;
|
||||
Bind(&LoopTop);
|
||||
ldaxr(SubEmitSize, TMP2, MemSrc);
|
||||
bic(EmitSize, TMP2, TMP2, Src);
|
||||
stlxr(SubEmitSize, TMP2, TMP2, MemSrc);
|
||||
cbnz(EmitSize, TMP2, &LoopTop);
|
||||
}
|
||||
}
|
||||
|
||||
DEF_OP(AtomicOr) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicOr>();
|
||||
const auto EmitSize = ConvertSize(IROp);
|
||||
const auto SubEmitSize = ConvertSubRegSize8(IROp->Size);
|
||||
|
||||
auto MemSrc = GetReg(Op->Addr);
|
||||
auto Src = GetReg(Op->Value);
|
||||
|
||||
if (CTX->HostFeatures.SupportsAtomics) {
|
||||
stsetl(SubEmitSize, Src, MemSrc);
|
||||
} else {
|
||||
ARMEmitter::BackwardLabel LoopTop;
|
||||
Bind(&LoopTop);
|
||||
ldaxr(SubEmitSize, TMP2, MemSrc);
|
||||
orr(EmitSize, TMP2, TMP2, Src);
|
||||
stlxr(SubEmitSize, TMP2, TMP2, MemSrc);
|
||||
cbnz(EmitSize, TMP2, &LoopTop);
|
||||
}
|
||||
}
|
||||
|
||||
DEF_OP(AtomicXor) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicXor>();
|
||||
const auto EmitSize = ConvertSize(IROp);
|
||||
const auto SubEmitSize = ConvertSubRegSize8(IROp->Size);
|
||||
|
||||
auto MemSrc = GetReg(Op->Addr);
|
||||
auto Src = GetReg(Op->Value);
|
||||
|
||||
if (CTX->HostFeatures.SupportsAtomics) {
|
||||
steorl(SubEmitSize, Src, MemSrc);
|
||||
} else {
|
||||
ARMEmitter::BackwardLabel LoopTop;
|
||||
Bind(&LoopTop);
|
||||
ldaxr(SubEmitSize, TMP2, MemSrc);
|
||||
eor(EmitSize, TMP2, TMP2, Src);
|
||||
stlxr(SubEmitSize, TMP2, TMP2, MemSrc);
|
||||
cbnz(EmitSize, TMP2, &LoopTop);
|
||||
}
|
||||
}
|
||||
|
||||
DEF_OP(AtomicNeg) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicNeg>();
|
||||
const auto EmitSize = ConvertSize(IROp);
|
||||
const auto SubEmitSize = ConvertSubRegSize8(IROp->Size);
|
||||
|
||||
auto MemSrc = GetReg(Op->Addr);
|
||||
|
||||
ARMEmitter::BackwardLabel LoopTop;
|
||||
Bind(&LoopTop);
|
||||
ldaxr(SubEmitSize, TMP2, MemSrc);
|
||||
neg(EmitSize, TMP3, TMP2);
|
||||
stlxr(SubEmitSize, TMP4, TMP3, MemSrc);
|
||||
cbnz(EmitSize, TMP4, &LoopTop);
|
||||
}
|
||||
|
||||
DEF_OP(AtomicSwap) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicSwap>();
|
||||
const auto OpSize = IROp->Size;
|
||||
@@ -159,10 +296,10 @@ DEF_OP(AtomicSwap) {
|
||||
ldswpal(SubEmitSize, Src, GetReg(Node), MemSrc);
|
||||
} else {
|
||||
ARMEmitter::BackwardLabel LoopTop;
|
||||
(void)Bind(&LoopTop);
|
||||
Bind(&LoopTop);
|
||||
ldaxr(SubEmitSize, TMP2, MemSrc);
|
||||
stlxr(SubEmitSize, TMP4, Src, MemSrc);
|
||||
(void)cbnz(EmitSize, TMP4, &LoopTop);
|
||||
cbnz(EmitSize, TMP4, &LoopTop);
|
||||
ubfm(EmitSize, GetReg(Node), TMP2, 0, IR::OpSizeAsBits(OpSize) - 1);
|
||||
}
|
||||
}
|
||||
@@ -179,11 +316,11 @@ DEF_OP(AtomicFetchAdd) {
|
||||
ldaddal(SubEmitSize, Src, GetReg(Node), MemSrc);
|
||||
} else {
|
||||
ARMEmitter::BackwardLabel LoopTop;
|
||||
(void)Bind(&LoopTop);
|
||||
Bind(&LoopTop);
|
||||
ldaxr(SubEmitSize, TMP2, MemSrc);
|
||||
add(EmitSize, TMP3, TMP2, Src);
|
||||
stlxr(SubEmitSize, TMP4, TMP3, MemSrc);
|
||||
(void)cbnz(EmitSize, TMP4, &LoopTop);
|
||||
cbnz(EmitSize, TMP4, &LoopTop);
|
||||
mov(EmitSize, GetReg(Node), TMP2.R());
|
||||
}
|
||||
}
|
||||
@@ -201,11 +338,11 @@ DEF_OP(AtomicFetchSub) {
|
||||
ldaddal(SubEmitSize, TMP2, GetReg(Node), MemSrc);
|
||||
} else {
|
||||
ARMEmitter::BackwardLabel LoopTop;
|
||||
(void)Bind(&LoopTop);
|
||||
Bind(&LoopTop);
|
||||
ldaxr(SubEmitSize, TMP2, MemSrc);
|
||||
sub(EmitSize, TMP3, TMP2, Src);
|
||||
stlxr(SubEmitSize, TMP4, TMP3, MemSrc);
|
||||
(void)cbnz(EmitSize, TMP4, &LoopTop);
|
||||
cbnz(EmitSize, TMP4, &LoopTop);
|
||||
mov(EmitSize, GetReg(Node), TMP2.R());
|
||||
}
|
||||
}
|
||||
@@ -223,11 +360,11 @@ DEF_OP(AtomicFetchAnd) {
|
||||
ldclral(SubEmitSize, TMP2, GetReg(Node), MemSrc);
|
||||
} else {
|
||||
ARMEmitter::BackwardLabel LoopTop;
|
||||
(void)Bind(&LoopTop);
|
||||
Bind(&LoopTop);
|
||||
ldaxr(SubEmitSize, TMP2, MemSrc);
|
||||
and_(EmitSize, TMP3, TMP2, Src);
|
||||
stlxr(SubEmitSize, TMP4, TMP3, MemSrc);
|
||||
(void)cbnz(EmitSize, TMP4, &LoopTop);
|
||||
cbnz(EmitSize, TMP4, &LoopTop);
|
||||
mov(EmitSize, GetReg(Node), TMP2.R());
|
||||
}
|
||||
}
|
||||
@@ -244,11 +381,11 @@ DEF_OP(AtomicFetchCLR) {
|
||||
ldclral(SubEmitSize, Src, GetReg(Node), MemSrc);
|
||||
} else {
|
||||
ARMEmitter::BackwardLabel LoopTop;
|
||||
(void)Bind(&LoopTop);
|
||||
Bind(&LoopTop);
|
||||
ldaxr(SubEmitSize, TMP2, MemSrc);
|
||||
bic(EmitSize, TMP3, TMP2, Src);
|
||||
stlxr(SubEmitSize, TMP4, TMP3, MemSrc);
|
||||
(void)cbnz(EmitSize, TMP4, &LoopTop);
|
||||
cbnz(EmitSize, TMP4, &LoopTop);
|
||||
mov(EmitSize, GetReg(Node), TMP2.R());
|
||||
}
|
||||
}
|
||||
@@ -265,11 +402,11 @@ DEF_OP(AtomicFetchOr) {
|
||||
ldsetal(SubEmitSize, Src, GetReg(Node), MemSrc);
|
||||
} else {
|
||||
ARMEmitter::BackwardLabel LoopTop;
|
||||
(void)Bind(&LoopTop);
|
||||
Bind(&LoopTop);
|
||||
ldaxr(SubEmitSize, TMP2, MemSrc);
|
||||
orr(EmitSize, TMP3, TMP2, Src);
|
||||
stlxr(SubEmitSize, TMP4, TMP3, MemSrc);
|
||||
(void)cbnz(EmitSize, TMP4, &LoopTop);
|
||||
cbnz(EmitSize, TMP4, &LoopTop);
|
||||
mov(EmitSize, GetReg(Node), TMP2.R());
|
||||
}
|
||||
}
|
||||
@@ -286,11 +423,11 @@ DEF_OP(AtomicFetchXor) {
|
||||
ldeoral(SubEmitSize, Src, GetReg(Node), MemSrc);
|
||||
} else {
|
||||
ARMEmitter::BackwardLabel LoopTop;
|
||||
(void)Bind(&LoopTop);
|
||||
Bind(&LoopTop);
|
||||
ldaxr(SubEmitSize, TMP2, MemSrc);
|
||||
eor(EmitSize, TMP3, TMP2, Src);
|
||||
stlxr(SubEmitSize, TMP4, TMP3, MemSrc);
|
||||
(void)cbnz(EmitSize, TMP4, &LoopTop);
|
||||
cbnz(EmitSize, TMP4, &LoopTop);
|
||||
mov(EmitSize, GetReg(Node), TMP2.R());
|
||||
}
|
||||
}
|
||||
@@ -302,26 +439,13 @@ DEF_OP(AtomicFetchNeg) {
|
||||
|
||||
auto MemSrc = GetReg(Op->Addr);
|
||||
|
||||
if (CTX->HostFeatures.SupportsAtomics) {
|
||||
// Use a CAS loop to avoid needing to emulate unaligned LLSC atomics
|
||||
ldr(SubEmitSize, TMP2, MemSrc);
|
||||
ARMEmitter::BackwardLabel LoopTop;
|
||||
(void)Bind(&LoopTop);
|
||||
mov(EmitSize, TMP4, TMP2);
|
||||
neg(EmitSize, TMP3, TMP2);
|
||||
casal(SubEmitSize, TMP2, TMP3, MemSrc);
|
||||
sub(EmitSize, TMP3, TMP2, TMP4);
|
||||
(void)cbnz(EmitSize, TMP3, &LoopTop);
|
||||
mov(EmitSize, GetReg(Node), TMP2.R());
|
||||
} else {
|
||||
ARMEmitter::BackwardLabel LoopTop;
|
||||
(void)Bind(&LoopTop);
|
||||
ldaxr(SubEmitSize, TMP2, MemSrc);
|
||||
neg(EmitSize, TMP3, TMP2);
|
||||
stlxr(SubEmitSize, TMP4, TMP3, MemSrc);
|
||||
(void)cbnz(EmitSize, TMP4, &LoopTop);
|
||||
mov(EmitSize, GetReg(Node), TMP2.R());
|
||||
}
|
||||
ARMEmitter::BackwardLabel LoopTop;
|
||||
Bind(&LoopTop);
|
||||
ldaxr(SubEmitSize, TMP2, MemSrc);
|
||||
neg(EmitSize, TMP3, TMP2);
|
||||
stlxr(SubEmitSize, TMP4, TMP3, MemSrc);
|
||||
cbnz(EmitSize, TMP4, &LoopTop);
|
||||
mov(EmitSize, GetReg(Node), TMP2.R());
|
||||
}
|
||||
|
||||
DEF_OP(TelemetrySetValue) {
|
||||
@@ -329,7 +453,7 @@ DEF_OP(TelemetrySetValue) {
|
||||
auto Op = IROp->C<IR::IROp_TelemetrySetValue>();
|
||||
auto Src = GetReg(Op->Value);
|
||||
|
||||
ldr(TMP2, STATE_PTR(CpuStateFrame, Pointers.TelemetryValueAddresses[Op->TelemetryValueIndex]));
|
||||
ldr(TMP2, STATE_PTR(CpuStateFrame, Pointers.Common.TelemetryValueAddresses[Op->TelemetryValueIndex]));
|
||||
|
||||
// Cortex fuses cmp+cset.
|
||||
cmp(ARMEmitter::Size::i32Bit, Src, 0);
|
||||
@@ -339,11 +463,11 @@ DEF_OP(TelemetrySetValue) {
|
||||
stsetl(ARMEmitter::SubRegSize::i64Bit, TMP1, TMP2);
|
||||
} else {
|
||||
ARMEmitter::BackwardLabel LoopTop;
|
||||
(void)Bind(&LoopTop);
|
||||
Bind(&LoopTop);
|
||||
ldaxr(ARMEmitter::SubRegSize::i64Bit, TMP3, TMP2);
|
||||
orr(ARMEmitter::Size::i32Bit, TMP3, TMP3, Src);
|
||||
stlxr(ARMEmitter::SubRegSize::i64Bit, TMP3, TMP3, TMP2);
|
||||
(void)cbnz(ARMEmitter::Size::i32Bit, TMP3, &LoopTop);
|
||||
cbnz(ARMEmitter::Size::i32Bit, TMP3, &LoopTop);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -56,176 +56,73 @@ DEF_OP(ExitFunction) {
|
||||
uint64_t NewRIP;
|
||||
|
||||
if (IsInlineConstant(Op->NewRIP, &NewRIP) || IsInlineEntrypointOffset(Op->NewRIP, &NewRIP)) {
|
||||
#ifdef ARCHITECTURE_arm64ec
|
||||
#ifdef _M_ARM_64EC
|
||||
if (NewRIP < EC_CODE_BITMAP_MAX_ADDRESS && RtlIsEcCode(NewRIP)) {
|
||||
str(REG_CALLRET_SP, STATE_PTR(CpuStateFrame, State.callret_sp));
|
||||
add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, StaticRegisters[X86State::REG_RSP], 0);
|
||||
InsertGuestRIPMove(EC_CALL_CHECKER_PC_REG, NewRIP);
|
||||
ldr(TMP2, STATE_PTR(CpuStateFrame, Pointers.ExitFunctionEC));
|
||||
LoadConstant(ARMEmitter::Size::i64Bit, EC_CALL_CHECKER_PC_REG, NewRIP);
|
||||
ldr(TMP2, STATE_PTR(CpuStateFrame, Pointers.Common.ExitFunctionEC));
|
||||
br(TMP2);
|
||||
} else {
|
||||
#endif
|
||||
// In order to support direct branches without constantly hitting the L1 cache, we emit a call to a block linker,
|
||||
// this will compile the branch target block when it is hit and replace the branch to the linker at the callsite
|
||||
// with a direct branch to the destination block. Upon invalidation of the target block the backpatch is undone.
|
||||
//
|
||||
// In addition, to avoid needing to lookup in the cache for returns and any indirect branch prediction penalty,
|
||||
// a shadow stack of <GuestReturnRIP, HostReturnPC> pairs is maintained, acting as a first level cache for any
|
||||
// return operations. As the guest may not balance calls and returns exactly, an exception handler is expected to
|
||||
// be installed by the frontend, to reset the shadow stack to the middle of its valid bounds on overflow/underflow.
|
||||
// This shadow stack is also cleared on block invalidation operations or codebuffer switches, to ensure all pointed-to
|
||||
// host code is always valid.
|
||||
|
||||
// This code will be backpatched by Arm64JITCore_ExitFunctionLink, below is an enumeration of all the possible cases.
|
||||
// Jump thunks are emitted in JIT.cpp after compilation of the entire multiblock.
|
||||
//
|
||||
// Call with known return block - unlinked
|
||||
// 00: adr TMP1, 0xC
|
||||
// 04: stp RetReg, TMP1, [SpReg, -0x10]!
|
||||
// 08: bl JmpThunk00
|
||||
// JmpThunk00:
|
||||
// 00: b 0x8
|
||||
// 04: br TMP1
|
||||
// 08: ldr TMP1, <Shared exit linker>
|
||||
// 0c: blr TMP1
|
||||
// 10: HostCode
|
||||
// 18: GuestRIP
|
||||
// 20: CallerOffset
|
||||
//
|
||||
// Call with known return block after backpatching - linked in branch immediate range
|
||||
// 00: adr TMP1, 0xC
|
||||
// 04: stp RetReg, TMP1, [SpReg, -0x10]!
|
||||
// 08: bl HostCode - MODIFIED
|
||||
//
|
||||
// Call with known return block after backpatching - linked out of range
|
||||
// 00: adr TMP1, 0xC
|
||||
// 04: stp RetReg, TMP1, [SpReg, -0x10]!
|
||||
// 08: bl JmpThunk00
|
||||
// JmpThunk00:
|
||||
// 00: ldr TMP1, 0x10 - MODIFIED 2nd
|
||||
// 04: br TMP1
|
||||
// 08: ldr TMP1, <Shared exit linker>
|
||||
// 0c: blr TMP1
|
||||
// 10: HostCode - MODIFIED 1st
|
||||
// 18: GuestRIP
|
||||
// 20: CallerOffset
|
||||
//
|
||||
// Jump - unlinked
|
||||
// 00: b JmpThunk00
|
||||
// JmpThunk00:
|
||||
// 00: b 0x8
|
||||
// 04: br TMP1
|
||||
// 08: ldr TMP1, <Shared exit linker>
|
||||
// 0c: blr TMP1
|
||||
// 10: HostCode
|
||||
// 18: GuestRIP
|
||||
// 20: CallerOffset
|
||||
//
|
||||
// Jump after backpatching - linked in branch immediate range
|
||||
// 00: b HostCode - MODIFIED
|
||||
//
|
||||
// Jump after backpatching - linked out of range
|
||||
// 00: b JmpThunk00
|
||||
// JmpThunk00:
|
||||
// 00: ldr TMP1, 0x10 - MODIFIED 2nd
|
||||
// 04: br TMP1
|
||||
// 08: ldr TMP1, <Shared exit linker>
|
||||
// 0c: blr TMP1
|
||||
// 10: HostCode - MODIFIED 1st
|
||||
// 18: GuestRIP
|
||||
// 20: CallerOffset
|
||||
// Align to 16 byte to allow atomic patching of the following 16 byte
|
||||
// of code (excluding the RIP data) on platforms that support LSE2
|
||||
Align16B();
|
||||
|
||||
ARMEmitter::ForwardLabel l_BranchHost;
|
||||
ARMEmitter::ForwardLabel l_CallReturn;
|
||||
if (Op->Hint == IR::BranchHint::Call) {
|
||||
if (!Op->CallReturnBlock.IsInvalid()) {
|
||||
auto CallReturnAddressReg = GetReg(Op->CallReturnAddress).X();
|
||||
PendingCallReturnTargetLabel = &CallReturnTargets.try_emplace(Op->CallReturnBlock.ID()).first->second;
|
||||
(void)adr(TMP1, &l_CallReturn);
|
||||
stp<ARMEmitter::IndexType::PRE>(CallReturnAddressReg, TMP1, REG_CALLRET_SP, -0x10);
|
||||
} else {
|
||||
stp<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::zr, ARMEmitter::XReg::zr, REG_CALLRET_SP, -0x10);
|
||||
}
|
||||
} else if (Op->Hint == IR::BranchHint::CheckTF) {
|
||||
ARMEmitter::ForwardLabel TFUnset;
|
||||
ldrb(TMP1, STATE_PTR(CpuStateFrame, State.flags[X86State::RFLAG_TF_RAW_LOC]));
|
||||
(void)cbz(ARMEmitter::Size::i32Bit, TMP1, &TFUnset);
|
||||
InsertGuestRIPMove(TMP1, NewRIP);
|
||||
str(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, State.rip));
|
||||
ldr(TMP2, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.DispatcherLoopTop));
|
||||
blr(TMP2);
|
||||
(void)Bind(&TFUnset);
|
||||
}
|
||||
ldr(TMP1, &l_BranchHost);
|
||||
blr(TMP1);
|
||||
|
||||
EmitLinkedBranch(NewRIP, Op->Hint == IR::BranchHint::Call);
|
||||
(void)Bind(&l_CallReturn);
|
||||
#ifdef ARCHITECTURE_arm64ec
|
||||
Bind(&l_BranchHost);
|
||||
dc64(ThreadState->CurrentFrame->Pointers.Common.ExitFunctionLinker);
|
||||
dc64(NewRIP);
|
||||
#ifdef _M_ARM_64EC
|
||||
}
|
||||
#endif
|
||||
} else {
|
||||
ARMEmitter::ForwardLabel SkipFullLookup;
|
||||
|
||||
ARMEmitter::ForwardLabel FullLookup;
|
||||
auto RipReg = GetReg(Op->NewRIP);
|
||||
|
||||
if (Op->Hint == IR::BranchHint::Return) {
|
||||
// First try to pop from the call-ret stack, otherwise follow the normal path (but ending in a ret)
|
||||
ldp<ARMEmitter::IndexType::POST>(TMP1, TMP2, REG_CALLRET_SP, 0x10);
|
||||
sub(TMP1, TMP1, RipReg.X());
|
||||
(void)cbz(ARMEmitter::Size::i64Bit, TMP1, &SkipFullLookup);
|
||||
}
|
||||
|
||||
// L1 Cache
|
||||
ldp<ARMEmitter::IndexType::OFFSET>(TMP1, TMP2, STATE, offsetof(FEXCore::Core::CpuStateFrame, State.L1Pointer));
|
||||
ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.L1Pointer));
|
||||
|
||||
// Calculate (tmp1 + ((ripreg & L1_ENTRIES_MASK) << 4)) for the address
|
||||
// L1Mask is pre-shifted.
|
||||
and_(ARMEmitter::Size::i64Bit, TMP2, TMP2, RipReg, ARMEmitter::ShiftType::LSL, FEXCore::ilog2(sizeof(LookupCache::LookupCacheEntry)));
|
||||
add(TMP1, TMP1, TMP2);
|
||||
|
||||
ldp<ARMEmitter::IndexType::OFFSET>(TMP2, TMP1, TMP1, 0);
|
||||
// arithmetic. ubfiz+add is marginally faster on Firestorm than
|
||||
// and+add(shift). Same performance on Cortex.
|
||||
static_assert(LookupCache::L1_ENTRIES_MASK == ((1u << 20) - 1));
|
||||
ubfiz(ARMEmitter::Size::i64Bit, TMP4, RipReg, 4, 20);
|
||||
add(TMP1, TMP1, TMP4);
|
||||
|
||||
// Note: sub+cbnz used over cmp+br to preserve flags.
|
||||
ldp<ARMEmitter::IndexType::OFFSET>(TMP2, TMP1, TMP1, 0);
|
||||
sub(TMP1, TMP1, RipReg.X());
|
||||
(void)cbz(ARMEmitter::Size::i64Bit, TMP1, &SkipFullLookup);
|
||||
ldr(TMP2, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.DispatcherLoopTop));
|
||||
str(RipReg.X(), STATE, offsetof(FEXCore::Core::CpuStateFrame, State.rip));
|
||||
cbnz(ARMEmitter::Size::i64Bit, TMP1, &FullLookup);
|
||||
br(TMP2);
|
||||
|
||||
(void)Bind(&SkipFullLookup);
|
||||
if (Op->Hint == IR::BranchHint::Call) {
|
||||
ARMEmitter::ForwardLabel l_CallReturn;
|
||||
if (!Op->CallReturnBlock.IsInvalid()) {
|
||||
auto CallReturnAddressReg = GetReg(Op->CallReturnAddress).X();
|
||||
PendingCallReturnTargetLabel = &CallReturnTargets.try_emplace(Op->CallReturnBlock.ID()).first->second;
|
||||
(void)adr(TMP1, &l_CallReturn);
|
||||
stp<ARMEmitter::IndexType::PRE>(CallReturnAddressReg, TMP1, REG_CALLRET_SP, -0x10);
|
||||
} else {
|
||||
stp<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::zr, ARMEmitter::XReg::zr, REG_CALLRET_SP, -0x10);
|
||||
}
|
||||
blr(TMP2);
|
||||
(void)Bind(&l_CallReturn);
|
||||
} else if (Op->Hint == IR::BranchHint::Return) {
|
||||
ret(TMP2);
|
||||
} else {
|
||||
br(TMP2);
|
||||
}
|
||||
Bind(&FullLookup);
|
||||
ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.DispatcherLoopTop));
|
||||
str(RipReg.X(), STATE, offsetof(FEXCore::Core::CpuStateFrame, State.rip));
|
||||
br(TMP1);
|
||||
}
|
||||
}
|
||||
|
||||
DEF_OP(Jump) {
|
||||
const auto Op = IROp->C<IR::IROp_Jump>();
|
||||
const auto Target = Op->TargetBlock;
|
||||
|
||||
PendingTargetLabel = JumpTarget(Op->TargetBlock);
|
||||
PendingTargetLabel = &JumpTargets.try_emplace(Target.ID()).first->second;
|
||||
}
|
||||
|
||||
DEF_OP(CondJump) {
|
||||
auto Op = IROp->C<IR::IROp_CondJump>();
|
||||
|
||||
auto TrueTargetLabel = JumpTarget(Op->TrueBlock);
|
||||
auto TrueTargetLabel = &JumpTargets.try_emplace(Op->TrueBlock.ID()).first->second;
|
||||
|
||||
if (Op->FromNZCV) {
|
||||
b_OrRestart(MapCC(Op->Cond), TrueTargetLabel);
|
||||
b(MapCC(Op->Cond), TrueTargetLabel);
|
||||
} else {
|
||||
uint64_t Const;
|
||||
const bool isConst = IsInlineConstant(Op->Cmp2, &Const);
|
||||
[[maybe_unused]] uint64_t Const;
|
||||
[[maybe_unused]] const bool isConst = IsInlineConstant(Op->Cmp2, &Const);
|
||||
|
||||
auto Reg = GetReg(Op->Cmp1);
|
||||
const auto Size = Op->CompareSize == IR::OpSize::i32Bit ? ARMEmitter::Size::i32Bit : ARMEmitter::Size::i64Bit;
|
||||
@@ -233,24 +130,24 @@ DEF_OP(CondJump) {
|
||||
LOGMAN_THROW_A_FMT(IsGPR(Op->Cmp1), "CondJump: Expected GPR");
|
||||
LOGMAN_THROW_A_FMT(isConst, "CondJump: Expected constant source");
|
||||
|
||||
if (Op->Cond == IR::CondClass::EQ) {
|
||||
if (Op->Cond.Val == FEXCore::IR::COND_EQ) {
|
||||
LOGMAN_THROW_A_FMT(Const == 0, "CondJump: Expected 0 source");
|
||||
cbz_OrRestart(Size, Reg, TrueTargetLabel);
|
||||
} else if (Op->Cond == IR::CondClass::NEQ) {
|
||||
cbz(Size, Reg, TrueTargetLabel);
|
||||
} else if (Op->Cond.Val == FEXCore::IR::COND_NEQ) {
|
||||
LOGMAN_THROW_A_FMT(Const == 0, "CondJump: Expected 0 source");
|
||||
cbnz_OrRestart(Size, Reg, TrueTargetLabel);
|
||||
} else if (Op->Cond == IR::CondClass::TSTZ) {
|
||||
cbnz(Size, Reg, TrueTargetLabel);
|
||||
} else if (Op->Cond.Val == FEXCore::IR::COND_TSTZ) {
|
||||
LOGMAN_THROW_A_FMT(Const < 64, "CondJump: Expected valid bit source");
|
||||
tbz_OrRestart(Reg, Const, TrueTargetLabel);
|
||||
} else if (Op->Cond == IR::CondClass::TSTNZ) {
|
||||
tbz(Reg, Const, TrueTargetLabel);
|
||||
} else if (Op->Cond.Val == FEXCore::IR::COND_TSTNZ) {
|
||||
LOGMAN_THROW_A_FMT(Const < 64, "CondJump: Expected valid bit source");
|
||||
tbnz_OrRestart(Reg, Const, TrueTargetLabel);
|
||||
tbnz(Reg, Const, TrueTargetLabel);
|
||||
} else {
|
||||
LOGMAN_THROW_A_FMT(false, "CondJump expected simple condition");
|
||||
}
|
||||
}
|
||||
|
||||
PendingTargetLabel = JumpTarget(Op->FalseBlock);
|
||||
PendingTargetLabel = &JumpTargets.try_emplace(Op->FalseBlock.ID()).first->second;
|
||||
}
|
||||
|
||||
DEF_OP(Syscall) {
|
||||
@@ -260,10 +157,16 @@ DEF_OP(Syscall) {
|
||||
// X1: ThreadState
|
||||
// X2: Pointer to SyscallArguments
|
||||
|
||||
FEXCore::IR::SyscallFlags Flags = Op->Flags;
|
||||
PushDynamicRegs(TMP1);
|
||||
|
||||
uint32_t GPRSpillMask = ~0U;
|
||||
uint32_t FPRSpillMask = ~0U;
|
||||
if ((Flags & FEXCore::IR::SyscallFlags::NOSYNCSTATEONENTRY) == FEXCore::IR::SyscallFlags::NOSYNCSTATEONENTRY) {
|
||||
// Need to spill all caller saved registers still
|
||||
GPRSpillMask = CALLER_GPR_MASK;
|
||||
FPRSpillMask = CALLER_FPR_MASK;
|
||||
}
|
||||
|
||||
SpillStaticRegs(TMP1, true, GPRSpillMask, FPRSpillMask);
|
||||
|
||||
@@ -283,8 +186,8 @@ DEF_OP(Syscall) {
|
||||
str(GetReg(Op->Header.Args[i]).X(), ARMEmitter::Reg::rsp, i * 8);
|
||||
}
|
||||
|
||||
ldr(ARMEmitter::XReg::x0, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.SyscallHandlerObj));
|
||||
ldr(ARMEmitter::XReg::x3, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.SyscallHandlerFunc));
|
||||
ldr(ARMEmitter::XReg::x0, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.SyscallHandlerObj));
|
||||
ldr(ARMEmitter::XReg::x3, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.SyscallHandlerFunc));
|
||||
mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, STATE.R());
|
||||
|
||||
// SP supporting move
|
||||
@@ -297,22 +200,117 @@ DEF_OP(Syscall) {
|
||||
|
||||
add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, SPOffset);
|
||||
|
||||
// Result is now in x0
|
||||
// Fix the stack and any values that were stepped on
|
||||
FillStaticRegs(true, GPRSpillMask, FPRSpillMask, ARMEmitter::Reg::r1, ARMEmitter::Reg::r2);
|
||||
if ((Flags & FEXCore::IR::SyscallFlags::NORETURN) != FEXCore::IR::SyscallFlags::NORETURN) {
|
||||
// Result is now in x0
|
||||
// Fix the stack and any values that were stepped on
|
||||
FillStaticRegs(true, GPRSpillMask, FPRSpillMask, ARMEmitter::Reg::r1, ARMEmitter::Reg::r2);
|
||||
|
||||
// Now the registers we've spilled are back in their original host registers
|
||||
// We can safely claim we are no longer in a syscall
|
||||
str(ARMEmitter::XReg::zr, STATE, offsetof(FEXCore::Core::CpuStateFrame, InSyscallInfo));
|
||||
// Now the registers we've spilled are back in their original host registers
|
||||
// We can safely claim we are no longer in a syscall
|
||||
str(ARMEmitter::XReg::zr, STATE, offsetof(FEXCore::Core::CpuStateFrame, InSyscallInfo));
|
||||
|
||||
PopDynamicRegs();
|
||||
PopDynamicRegs();
|
||||
|
||||
const auto OSABI = CTX->SyscallHandler->GetOSABI();
|
||||
if ((Flags & FEXCore::IR::SyscallFlags::NORETURNEDRESULT) != FEXCore::IR::SyscallFlags::NORETURNEDRESULT) {
|
||||
// Move result to its destination register.
|
||||
// Only if `NORETURNEDRESULT` wasn't set, otherwise we might overwrite the CPUState refilled with `FillStaticRegs`
|
||||
mov(ARMEmitter::Size::i64Bit, GetReg(Node), ARMEmitter::Reg::r0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (OSABI != FEXCore::HLE::SyscallOSABI::OS_GENERIC) {
|
||||
// Move result to its destination register.
|
||||
// Only if `NORETURNEDRESULT` wasn't set, otherwise we might overwrite the CPUState refilled with `FillStaticRegs`
|
||||
mov(ARMEmitter::Size::i64Bit, GetReg(Node), ARMEmitter::Reg::r0);
|
||||
DEF_OP(InlineSyscall) {
|
||||
auto Op = IROp->C<IR::IROp_InlineSyscall>();
|
||||
// Arguments are passed as follows:
|
||||
// X8: SyscallNumber - RA INTERSECT
|
||||
// X0: Arg0 & Return
|
||||
// X1: Arg1
|
||||
// X2: Arg2
|
||||
// X3: Arg3
|
||||
// X4: Arg4 - RA INTERSECT
|
||||
// X5: Arg5 - RA INTERSECT
|
||||
// X6: Arg6 - Doesn't exist in x86-64 land. RA INTERSECT
|
||||
|
||||
// One argument is removed from the SyscallArguments::MAX_ARGS since the first argument was syscall number
|
||||
const static std::array<ARMEmitter::XRegister, FEXCore::HLE::SyscallArguments::MAX_ARGS - 1> RegArgs = {
|
||||
{ARMEmitter::XReg::x0, ARMEmitter::XReg::x1, ARMEmitter::XReg::x2, ARMEmitter::XReg::x3, ARMEmitter::XReg::x4, ARMEmitter::XReg::x5}};
|
||||
|
||||
bool Intersects {};
|
||||
// We always need to spill x8 since we can't know if it is live at this SSA location
|
||||
uint32_t SpillMask = 1U << 8;
|
||||
for (uint32_t i = 0; i < FEXCore::HLE::SyscallArguments::MAX_ARGS - 1; ++i) {
|
||||
if (Op->Header.Args[i].IsInvalid()) {
|
||||
break;
|
||||
}
|
||||
|
||||
auto Reg = GetReg(Op->Header.Args[i]);
|
||||
if (Reg == ARMEmitter::Reg::r8 || Reg == ARMEmitter::Reg::r4 || Reg == ARMEmitter::Reg::r5) {
|
||||
|
||||
SpillMask |= (1U << Reg.Idx());
|
||||
Intersects = true;
|
||||
}
|
||||
}
|
||||
|
||||
// Ordering is incredibly important here
|
||||
// We must spill any overlapping registers first THEN claim we are in a syscall without invalidating state at all
|
||||
// Only spill the registers that intersect with our usage
|
||||
SpillStaticRegs(TMP1, false, SpillMask);
|
||||
|
||||
// Now that we are spilled, store in the state that we are in a syscall
|
||||
// Still without overwriting registers that matter
|
||||
// 16bit LoadConstant to be a single instruction
|
||||
// We must always spill at least one register (x8) so this value always has a bit set
|
||||
// This gives the signal handler a value to check to see if we are in a syscall at all
|
||||
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, SpillMask & 0xFFFF);
|
||||
str(ARMEmitter::XReg::x0, STATE, offsetof(FEXCore::Core::CpuStateFrame, InSyscallInfo));
|
||||
|
||||
// Now that we have claimed to be a syscall we can set up the arguments
|
||||
const auto EmitSize = CTX->Config.Is64BitMode() ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit;
|
||||
const auto EmitSubSize = CTX->Config.Is64BitMode() ? ARMEmitter::SubRegSize::i64Bit : ARMEmitter::SubRegSize::i32Bit;
|
||||
if (Intersects) {
|
||||
for (uint32_t i = 0; i < FEXCore::HLE::SyscallArguments::MAX_ARGS - 1; ++i) {
|
||||
if (Op->Header.Args[i].IsInvalid()) {
|
||||
break;
|
||||
}
|
||||
|
||||
auto Reg = GetReg(Op->Header.Args[i]);
|
||||
if (SpillMask & (1U << Reg.Idx())) {
|
||||
// In the case of intersection with x4, x5, or x8 then these are currently SRA
|
||||
// for registers RAX, RDX, and RSP. Which have just been spilled
|
||||
// Just load back from the context.
|
||||
auto Correlation = GetX86RegRelationToARMReg(Reg);
|
||||
LOGMAN_THROW_A_FMT(Correlation != X86State::REG_INVALID, "Invalid register mapping");
|
||||
ldr(EmitSubSize, RegArgs[i].R(), STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[Correlation]));
|
||||
} else {
|
||||
mov(EmitSize, RegArgs[i].R(), Reg);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (uint32_t i = 0; i < FEXCore::HLE::SyscallArguments::MAX_ARGS - 1; ++i) {
|
||||
if (Op->Header.Args[i].IsInvalid()) {
|
||||
break;
|
||||
}
|
||||
|
||||
mov(EmitSize, RegArgs[i].R(), GetReg(Op->Header.Args[i]));
|
||||
}
|
||||
}
|
||||
|
||||
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r8, Op->HostSyscallNumber);
|
||||
svc(0);
|
||||
// On updated signal mask we can receive a signal RIGHT HERE
|
||||
|
||||
if ((Op->Flags & FEXCore::IR::SyscallFlags::NORETURN) != FEXCore::IR::SyscallFlags::NORETURN) {
|
||||
// Now that we are done in the syscall we need to carefully peel back the state
|
||||
// First unspill the registers from before
|
||||
FillStaticRegs(false, SpillMask, ~0U, ARMEmitter::Reg::r8, ARMEmitter::Reg::r1);
|
||||
|
||||
// Now the registers we've spilled are back in their original host registers
|
||||
// We can safely claim we are no longer in a syscall
|
||||
str(ARMEmitter::XReg::zr, STATE, offsetof(FEXCore::Core::CpuStateFrame, InSyscallInfo));
|
||||
|
||||
// Result is now in x0
|
||||
// Move result to its destination register
|
||||
mov(EmitSize, GetReg(Node), ARMEmitter::Reg::r0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -328,7 +326,8 @@ DEF_OP(Thunk) {
|
||||
|
||||
mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, GetReg(Op->ArgPtr));
|
||||
|
||||
InsertNamedThunkRelocation(ARMEmitter::Reg::r2, Op->ThunkNameHash);
|
||||
auto thunkFn = static_cast<Context::ContextImpl*>(ThreadState->CTX)->ThunkHandler->LookupThunk(Op->ThunkNameHash);
|
||||
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r2, (uintptr_t)thunkFn);
|
||||
if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {
|
||||
GenerateIndirectRuntimeCall<void, void*, void*>(ARMEmitter::Reg::r2);
|
||||
} else {
|
||||
@@ -342,50 +341,48 @@ DEF_OP(Thunk) {
|
||||
|
||||
DEF_OP(ValidateCode) {
|
||||
auto Op = IROp->C<IR::IROp_ValidateCode>();
|
||||
auto OldCode = Op->CodeOriginal.data();
|
||||
auto Base = GetReg(Op->Header.Args[0]).X();
|
||||
const auto* OldCode = (const uint8_t*)&Op->CodeOriginalLow;
|
||||
int len = Op->CodeLength;
|
||||
int Offset = 0;
|
||||
ARMEmitter::ForwardLabel Fail;
|
||||
int idx = 0;
|
||||
|
||||
LoadConstant(ARMEmitter::Size::i64Bit, GetReg(Node), 0);
|
||||
LoadConstant(ARMEmitter::Size::i64Bit, TMP1, Entry + Op->Offset);
|
||||
LoadConstant(ARMEmitter::Size::i64Bit, TMP2, 1);
|
||||
|
||||
const auto Dst = GetReg(Node);
|
||||
|
||||
auto EmitCheck = [&](size_t Size, auto&& LoadData) {
|
||||
while (len >= Size) {
|
||||
LoadData();
|
||||
sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, TMP2);
|
||||
cbnz_OrRestart(ARMEmitter::Size::i64Bit, TMP1, &Fail);
|
||||
len -= Size;
|
||||
Offset += Size;
|
||||
}
|
||||
};
|
||||
|
||||
EmitCheck(8, [&]() {
|
||||
ldr(TMP1, Base, Offset);
|
||||
LoadConstant(ARMEmitter::Size::i64Bit, TMP2, *(const uint64_t*)(OldCode + Offset));
|
||||
});
|
||||
|
||||
EmitCheck(4, [&]() {
|
||||
ldr(TMP1.W(), Base, Offset);
|
||||
LoadConstant(ARMEmitter::Size::i32Bit, TMP2, *(const uint32_t*)(OldCode + Offset));
|
||||
});
|
||||
|
||||
EmitCheck(2, [&]() {
|
||||
ldrh(TMP1.W(), Base, Offset);
|
||||
LoadConstant(ARMEmitter::Size::i32Bit, TMP2, *(const uint16_t*)(OldCode + Offset));
|
||||
});
|
||||
|
||||
EmitCheck(1, [&]() {
|
||||
ldrb(TMP1.W(), Base, Offset);
|
||||
LoadConstant(ARMEmitter::Size::i32Bit, TMP2, *(const uint8_t*)(OldCode + Offset));
|
||||
});
|
||||
|
||||
ARMEmitter::ForwardLabel End;
|
||||
LoadConstant(ARMEmitter::Size::i32Bit, Dst, 0);
|
||||
b_OrRestart(&End);
|
||||
BindOrRestart(&Fail);
|
||||
LoadConstant(ARMEmitter::Size::i32Bit, Dst, 1);
|
||||
BindOrRestart(&End);
|
||||
while (len >= 8) {
|
||||
ldr(ARMEmitter::XReg::x2, TMP1, idx);
|
||||
LoadConstant(ARMEmitter::Size::i64Bit, TMP4, *(const uint64_t*)(OldCode + idx));
|
||||
cmp(ARMEmitter::Size::i64Bit, TMP3, TMP4);
|
||||
csel(ARMEmitter::Size::i64Bit, Dst, Dst, TMP2, ARMEmitter::Condition::CC_EQ);
|
||||
len -= 8;
|
||||
idx += 8;
|
||||
}
|
||||
while (len >= 4) {
|
||||
ldr(ARMEmitter::WReg::w2, TMP1, idx);
|
||||
LoadConstant(ARMEmitter::Size::i64Bit, TMP4, *(const uint32_t*)(OldCode + idx));
|
||||
cmp(ARMEmitter::Size::i32Bit, TMP3, TMP4);
|
||||
csel(ARMEmitter::Size::i64Bit, Dst, Dst, TMP2, ARMEmitter::Condition::CC_EQ);
|
||||
len -= 4;
|
||||
idx += 4;
|
||||
}
|
||||
while (len >= 2) {
|
||||
ldrh(TMP3, TMP1, idx);
|
||||
LoadConstant(ARMEmitter::Size::i64Bit, TMP4, *(const uint16_t*)(OldCode + idx));
|
||||
cmp(ARMEmitter::Size::i32Bit, TMP3, TMP4);
|
||||
csel(ARMEmitter::Size::i64Bit, Dst, Dst, TMP2, ARMEmitter::Condition::CC_EQ);
|
||||
len -= 2;
|
||||
idx += 2;
|
||||
}
|
||||
while (len >= 1) {
|
||||
ldrb(TMP3, TMP1, idx);
|
||||
LoadConstant(ARMEmitter::Size::i64Bit, TMP4, *(const uint8_t*)(OldCode + idx));
|
||||
cmp(ARMEmitter::Size::i32Bit, TMP3, TMP4);
|
||||
csel(ARMEmitter::Size::i64Bit, Dst, Dst, TMP2, ARMEmitter::Condition::CC_EQ);
|
||||
len -= 1;
|
||||
idx += 1;
|
||||
}
|
||||
}
|
||||
|
||||
DEF_OP(ThreadRemoveCodeEntry) {
|
||||
@@ -397,10 +394,9 @@ DEF_OP(ThreadRemoveCodeEntry) {
|
||||
// X1: RIP
|
||||
mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, STATE.R());
|
||||
|
||||
// TODO: Relocations don't seem to be wired up to this...?
|
||||
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, Entry, CPU::Arm64Emitter::PadType::AUTOPAD);
|
||||
LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, Entry);
|
||||
|
||||
ldr(ARMEmitter::XReg::x2, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.ThreadRemoveCodeEntryFromJIT));
|
||||
ldr(ARMEmitter::XReg::x2, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.ThreadRemoveCodeEntryFromJIT));
|
||||
if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {
|
||||
GenerateIndirectRuntimeCall<void, void*, void*>(ARMEmitter::Reg::r2);
|
||||
} else {
|
||||
@@ -425,8 +421,8 @@ DEF_OP(CPUID) {
|
||||
// x0 = CPUID Handler
|
||||
// x1 = CPUID Function
|
||||
// x2 = CPUID Leaf
|
||||
ldr(ARMEmitter::XReg::x0, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.CPUIDObj));
|
||||
ldr(ARMEmitter::XReg::x3, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.CPUIDFunction));
|
||||
ldr(ARMEmitter::XReg::x0, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.CPUIDObj));
|
||||
ldr(ARMEmitter::XReg::x3, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.CPUIDFunction));
|
||||
|
||||
if (!TMP_ABIARGS) {
|
||||
mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, TMP2);
|
||||
@@ -466,8 +462,8 @@ DEF_OP(XGetBV) {
|
||||
|
||||
// x0 = CPUID Handler
|
||||
// x1 = XCR Function
|
||||
ldr(ARMEmitter::XReg::x0, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.CPUIDObj));
|
||||
ldr(ARMEmitter::XReg::x2, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.XCRFunction));
|
||||
ldr(ARMEmitter::XReg::x0, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.CPUIDObj));
|
||||
ldr(ARMEmitter::XReg::x2, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.XCRFunction));
|
||||
if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {
|
||||
GenerateIndirectRuntimeCall<uint64_t, void*, uint32_t>(ARMEmitter::Reg::r2);
|
||||
} else {
|
||||
|
||||
@@ -423,11 +423,11 @@ DEF_OP(Vector_FToI) {
|
||||
const auto Mask = PRED_TMP_32B.Merging();
|
||||
|
||||
switch (Op->Round) {
|
||||
case IR::RoundMode::Nearest: frintn(SubEmitSize, Dst.Z(), Mask, Vector.Z()); break;
|
||||
case IR::RoundMode::NegInfinity: frintm(SubEmitSize, Dst.Z(), Mask, Vector.Z()); break;
|
||||
case IR::RoundMode::PosInfinity: frintp(SubEmitSize, Dst.Z(), Mask, Vector.Z()); break;
|
||||
case IR::RoundMode::TowardsZero: frintz(SubEmitSize, Dst.Z(), Mask, Vector.Z()); break;
|
||||
case IR::RoundMode::Host: frinti(SubEmitSize, Dst.Z(), Mask, Vector.Z()); break;
|
||||
case FEXCore::IR::Round_Nearest.Val: frintn(SubEmitSize, Dst.Z(), Mask, Vector.Z()); break;
|
||||
case FEXCore::IR::Round_Negative_Infinity.Val: frintm(SubEmitSize, Dst.Z(), Mask, Vector.Z()); break;
|
||||
case FEXCore::IR::Round_Positive_Infinity.Val: frintp(SubEmitSize, Dst.Z(), Mask, Vector.Z()); break;
|
||||
case FEXCore::IR::Round_Towards_Zero.Val: frintz(SubEmitSize, Dst.Z(), Mask, Vector.Z()); break;
|
||||
case FEXCore::IR::Round_Host.Val: frinti(SubEmitSize, Dst.Z(), Mask, Vector.Z()); break;
|
||||
}
|
||||
} else {
|
||||
const auto IsScalar = ElementSize == OpSize;
|
||||
@@ -449,21 +449,21 @@ DEF_OP(Vector_FToI) {
|
||||
}
|
||||
|
||||
switch (Op->Round) {
|
||||
case IR::RoundMode::Nearest: ROUNDING_FN(frintn); break;
|
||||
case IR::RoundMode::NegInfinity: ROUNDING_FN(frintm); break;
|
||||
case IR::RoundMode::PosInfinity: ROUNDING_FN(frintp); break;
|
||||
case IR::RoundMode::TowardsZero: ROUNDING_FN(frintz); break;
|
||||
case IR::RoundMode::Host: ROUNDING_FN(frinti); break;
|
||||
case IR::Round_Nearest.Val: ROUNDING_FN(frintn); break;
|
||||
case IR::Round_Negative_Infinity.Val: ROUNDING_FN(frintm); break;
|
||||
case IR::Round_Positive_Infinity.Val: ROUNDING_FN(frintp); break;
|
||||
case IR::Round_Towards_Zero.Val: ROUNDING_FN(frintz); break;
|
||||
case IR::Round_Host.Val: ROUNDING_FN(frinti); break;
|
||||
}
|
||||
|
||||
#undef ROUNDING_FN
|
||||
} else {
|
||||
switch (Op->Round) {
|
||||
case IR::RoundMode::Nearest: frintn(SubEmitSize, Dst.Q(), Vector.Q()); break;
|
||||
case IR::RoundMode::NegInfinity: frintm(SubEmitSize, Dst.Q(), Vector.Q()); break;
|
||||
case IR::RoundMode::PosInfinity: frintp(SubEmitSize, Dst.Q(), Vector.Q()); break;
|
||||
case IR::RoundMode::TowardsZero: frintz(SubEmitSize, Dst.Q(), Vector.Q()); break;
|
||||
case IR::RoundMode::Host: frinti(SubEmitSize, Dst.Q(), Vector.Q()); break;
|
||||
case FEXCore::IR::Round_Nearest.Val: frintn(SubEmitSize, Dst.Q(), Vector.Q()); break;
|
||||
case FEXCore::IR::Round_Negative_Infinity.Val: frintm(SubEmitSize, Dst.Q(), Vector.Q()); break;
|
||||
case FEXCore::IR::Round_Positive_Infinity.Val: frintp(SubEmitSize, Dst.Q(), Vector.Q()); break;
|
||||
case FEXCore::IR::Round_Towards_Zero.Val: frintz(SubEmitSize, Dst.Q(), Vector.Q()); break;
|
||||
case FEXCore::IR::Round_Host.Val: frinti(SubEmitSize, Dst.Q(), Vector.Q()); break;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -539,11 +539,11 @@ DEF_OP(Vector_F64ToI32) {
|
||||
// Then convert to integers using fcvtzs.
|
||||
auto CVTReg = Dst.Z();
|
||||
switch (Round) {
|
||||
case IR::RoundMode::Nearest: frintn(ARMEmitter::SubRegSize::i64Bit, Dst.Z(), Mask, Vector.Z()); break;
|
||||
case IR::RoundMode::NegInfinity: frintm(ARMEmitter::SubRegSize::i64Bit, Dst.Z(), Mask, Vector.Z()); break;
|
||||
case IR::RoundMode::PosInfinity: frintp(ARMEmitter::SubRegSize::i64Bit, Dst.Z(), Mask, Vector.Z()); break;
|
||||
case IR::RoundMode::TowardsZero: CVTReg = Vector.Z(); break;
|
||||
case IR::RoundMode::Host: frinti(ARMEmitter::SubRegSize::i64Bit, Dst.Z(), Mask, Vector.Z()); break;
|
||||
case IR::Round_Nearest.Val: frintn(ARMEmitter::SubRegSize::i64Bit, Dst.Z(), Mask, Vector.Z()); break;
|
||||
case IR::Round_Negative_Infinity.Val: frintm(ARMEmitter::SubRegSize::i64Bit, Dst.Z(), Mask, Vector.Z()); break;
|
||||
case IR::Round_Positive_Infinity.Val: frintp(ARMEmitter::SubRegSize::i64Bit, Dst.Z(), Mask, Vector.Z()); break;
|
||||
case IR::Round_Towards_Zero.Val: CVTReg = Vector.Z(); break;
|
||||
case IR::Round_Host.Val: frinti(ARMEmitter::SubRegSize::i64Bit, Dst.Z(), Mask, Vector.Z()); break;
|
||||
}
|
||||
|
||||
fcvtzs(Dst.Z(), ARMEmitter::SubRegSize::i32Bit, Mask, CVTReg, ARMEmitter::SubRegSize::i64Bit);
|
||||
@@ -567,11 +567,11 @@ DEF_OP(Vector_F64ToI32) {
|
||||
|
||||
///< Round float to integral depending on rounding mode.
|
||||
switch (Round) {
|
||||
case IR::RoundMode::Nearest: frintn(ARMEmitter::SubRegSize::i64Bit, Dst.Q(), Vector.Q()); break;
|
||||
case IR::RoundMode::NegInfinity: frintm(ARMEmitter::SubRegSize::i64Bit, Dst.Q(), Vector.Q()); break;
|
||||
case IR::RoundMode::PosInfinity: frintp(ARMEmitter::SubRegSize::i64Bit, Dst.Q(), Vector.Q()); break;
|
||||
case IR::RoundMode::TowardsZero: frintz(ARMEmitter::SubRegSize::i64Bit, Dst.Q(), Vector.Q()); break;
|
||||
case IR::RoundMode::Host: frinti(ARMEmitter::SubRegSize::i64Bit, Dst.Q(), Vector.Q()); break;
|
||||
case FEXCore::IR::Round_Nearest.Val: frintn(ARMEmitter::SubRegSize::i64Bit, Dst.Q(), Vector.Q()); break;
|
||||
case FEXCore::IR::Round_Negative_Infinity.Val: frintm(ARMEmitter::SubRegSize::i64Bit, Dst.Q(), Vector.Q()); break;
|
||||
case FEXCore::IR::Round_Positive_Infinity.Val: frintp(ARMEmitter::SubRegSize::i64Bit, Dst.Q(), Vector.Q()); break;
|
||||
case FEXCore::IR::Round_Towards_Zero.Val: frintz(ARMEmitter::SubRegSize::i64Bit, Dst.Q(), Vector.Q()); break;
|
||||
case FEXCore::IR::Round_Host.Val: frinti(ARMEmitter::SubRegSize::i64Bit, Dst.Q(), Vector.Q()); break;
|
||||
}
|
||||
|
||||
// Now narrow from f64 to f32.
|
||||
|
||||
@@ -1,35 +0,0 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
#pragma once
|
||||
|
||||
#include <FEXCore/Utils/AllocatorHooks.h>
|
||||
#include <FEXCore/fextl/vector.h>
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
namespace FEXCore::CPU {
|
||||
union Relocation;
|
||||
} // namespace FEXCore::CPU
|
||||
|
||||
namespace FEXCore::Core {
|
||||
struct DebugDataSubblock {
|
||||
uint32_t HostCodeOffset;
|
||||
uint32_t HostCodeSize;
|
||||
};
|
||||
|
||||
struct DebugDataGuestOpcode {
|
||||
uint64_t GuestEntryOffset;
|
||||
ptrdiff_t HostEntryOffset;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Contains debug data for a block of code for later debugger analysis
|
||||
*
|
||||
* Needs to remain around for as long as the code could be executed at least
|
||||
*/
|
||||
struct DebugData : public FEXCore::Allocator::FEXAllocOperators {
|
||||
uint64_t HostCodeSize; ///< The size of the code generated in the host JIT
|
||||
fextl::vector<DebugDataSubblock> Subblocks;
|
||||
fextl::vector<DebugDataGuestOpcode> GuestOpcodes;
|
||||
fextl::vector<FEXCore::CPU::Relocation>* Relocations;
|
||||
};
|
||||
} // namespace FEXCore::Core
|
||||
@@ -16,7 +16,7 @@ DEF_OP(VAESImc) {
|
||||
|
||||
DEF_OP(VAESEnc) {
|
||||
const auto Op = IROp->C<IR::IROp_VAESEnc>();
|
||||
const auto OpSize = IROp->Size;
|
||||
[[maybe_unused]] const auto OpSize = IROp->Size;
|
||||
|
||||
const auto Dst = GetVReg(Node);
|
||||
const auto Key = GetVReg(Op->Key);
|
||||
@@ -41,7 +41,7 @@ DEF_OP(VAESEnc) {
|
||||
|
||||
DEF_OP(VAESEncLast) {
|
||||
const auto Op = IROp->C<IR::IROp_VAESEncLast>();
|
||||
const auto OpSize = IROp->Size;
|
||||
[[maybe_unused]] const auto OpSize = IROp->Size;
|
||||
|
||||
const auto Dst = GetVReg(Node);
|
||||
const auto Key = GetVReg(Op->Key);
|
||||
@@ -64,7 +64,7 @@ DEF_OP(VAESEncLast) {
|
||||
|
||||
DEF_OP(VAESDec) {
|
||||
const auto Op = IROp->C<IR::IROp_VAESDec>();
|
||||
const auto OpSize = IROp->Size;
|
||||
[[maybe_unused]] const auto OpSize = IROp->Size;
|
||||
|
||||
const auto Dst = GetVReg(Node);
|
||||
const auto Key = GetVReg(Op->Key);
|
||||
@@ -89,7 +89,7 @@ DEF_OP(VAESDec) {
|
||||
|
||||
DEF_OP(VAESDecLast) {
|
||||
const auto Op = IROp->C<IR::IROp_VAESDecLast>();
|
||||
const auto OpSize = IROp->Size;
|
||||
[[maybe_unused]] const auto OpSize = IROp->Size;
|
||||
|
||||
const auto Dst = GetVReg(Node);
|
||||
const auto Key = GetVReg(Op->Key);
|
||||
@@ -322,7 +322,7 @@ DEF_OP(VSha256U1) {
|
||||
|
||||
DEF_OP(PCLMUL) {
|
||||
const auto Op = IROp->C<IR::IROp_PCLMUL>();
|
||||
const auto OpSize = IROp->Size;
|
||||
[[maybe_unused]] const auto OpSize = IROp->Size;
|
||||
|
||||
const auto Dst = GetVReg(Node);
|
||||
const auto Src1 = GetVReg(Op->Src1);
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
/*
|
||||
$info$
|
||||
glossary: Splatter ~ a code generator backend that concatenates configurable macros instead of doing isel
|
||||
glossary: Splatter ~ a code generator backend that concaternates configurable macros instead of doing isel
|
||||
glossary: IR ~ Intermediate Representation, our high-level opcode representation, loosely modeling arm64
|
||||
glossary: SSA ~ Single Static Assignment, a form of representing IR in memory
|
||||
glossary: Basic Block ~ A block of instructions with no control flow, terminated by control flow
|
||||
@@ -11,12 +11,15 @@ desc: Main glue logic of the arm64 splatter backend
|
||||
$end_info$
|
||||
*/
|
||||
|
||||
#include "Common/SoftFloat.h"
|
||||
#include "FEXCore/Utils/Telemetry.h"
|
||||
#include "FEXCore/Utils/TypeDefines.h"
|
||||
#include "Interface/Context/Context.h"
|
||||
#include "Interface/Core/LookupCache.h"
|
||||
|
||||
#include "Interface/Core/Dispatcher/Dispatcher.h"
|
||||
#include "Interface/Core/Interpreter/InterpreterOps.h"
|
||||
#include "Interface/Core/JIT/DebugData.h"
|
||||
#include "Interface/Core/JIT/JITClass.h"
|
||||
|
||||
#include "Interface/IR/Passes/RegisterAllocationPass.h"
|
||||
|
||||
#include "Utils/MemberFunctionToPointer.h"
|
||||
@@ -27,16 +30,15 @@ $end_info$
|
||||
#include <FEXCore/Utils/Allocator.h>
|
||||
#include <FEXCore/Utils/CompilerDefs.h>
|
||||
#include <FEXCore/Utils/EnumUtils.h>
|
||||
#include <FEXCore/Utils/LogManager.h>
|
||||
#include <FEXCore/Utils/LongJump.h>
|
||||
#include <FEXCore/Utils/Profiler.h>
|
||||
#include <FEXCore/Utils/Telemetry.h>
|
||||
#include <FEXCore/Utils/TypeDefines.h>
|
||||
#include <FEXCore/HLE/SyscallHandler.h>
|
||||
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include "Interface/Core/Interpreter/InterpreterOps.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <unistd.h>
|
||||
#include <string.h>
|
||||
#include <limits>
|
||||
|
||||
namespace {
|
||||
struct DivRem {
|
||||
@@ -133,8 +135,8 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::Ref Node) {
|
||||
const auto Src1 = GetVReg(IROp->Args[0]);
|
||||
fmov(VTMP1.S(), Src1.S());
|
||||
|
||||
ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
|
||||
ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.FallbackHandlerPointers[Info.HandlerIndex].Func));
|
||||
ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
|
||||
ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].Func));
|
||||
blr(TMP1);
|
||||
|
||||
ldr<ARMEmitter::IndexType::POST>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);
|
||||
@@ -151,8 +153,8 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::Ref Node) {
|
||||
const auto Src1 = GetVReg(IROp->Args[0]);
|
||||
fmov(VTMP1.D(), Src1.D());
|
||||
|
||||
ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
|
||||
ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.FallbackHandlerPointers[Info.HandlerIndex].Func));
|
||||
ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
|
||||
ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].Func));
|
||||
blr(TMP1);
|
||||
|
||||
ldr<ARMEmitter::IndexType::POST>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);
|
||||
@@ -176,8 +178,8 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::Ref Node) {
|
||||
mov(ARMEmitter::Size::i32Bit, TMP2, Src1);
|
||||
}
|
||||
|
||||
ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
|
||||
ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.FallbackHandlerPointers[Info.HandlerIndex].Func));
|
||||
ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
|
||||
ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].Func));
|
||||
blr(TMP1);
|
||||
|
||||
ldr<ARMEmitter::IndexType::POST>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);
|
||||
@@ -194,8 +196,8 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::Ref Node) {
|
||||
const auto Src1 = GetVReg(IROp->Args[0]);
|
||||
mov(VTMP1.Q(), Src1.Q());
|
||||
|
||||
ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
|
||||
ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.FallbackHandlerPointers[Info.HandlerIndex].Func));
|
||||
ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
|
||||
ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].Func));
|
||||
blr(TMP1);
|
||||
|
||||
ldr<ARMEmitter::IndexType::POST>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);
|
||||
@@ -212,15 +214,15 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::Ref Node) {
|
||||
const auto Src1 = GetVReg(IROp->Args[0]);
|
||||
mov(VTMP1.Q(), Src1.Q());
|
||||
|
||||
ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
|
||||
ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.FallbackHandlerPointers[Info.HandlerIndex].Func));
|
||||
ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
|
||||
ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].Func));
|
||||
blr(TMP1);
|
||||
|
||||
ldr<ARMEmitter::IndexType::POST>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);
|
||||
FillF64Result();
|
||||
} break;
|
||||
|
||||
case FABI_F64_F64_PTR: {
|
||||
case FABI_F64_I16_F64_PTR: {
|
||||
// Linux Reg/Win32 Reg:
|
||||
// tmp4 (x4/x13): FallbackHandler
|
||||
// x30: return
|
||||
@@ -230,14 +232,14 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::Ref Node) {
|
||||
const auto Src1 = GetVReg(IROp->Args[0]);
|
||||
fmov(VTMP1.D(), Src1.D());
|
||||
|
||||
ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
|
||||
ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.FallbackHandlerPointers[Info.HandlerIndex].Func));
|
||||
ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
|
||||
ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].Func));
|
||||
blr(TMP1);
|
||||
|
||||
ldr<ARMEmitter::IndexType::POST>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);
|
||||
FillF64Result();
|
||||
} break;
|
||||
case FABI_F64x2_F64_PTR: {
|
||||
case FABI_F64x2_I16_F64_PTR: {
|
||||
// Linux Reg/Win32 Reg:
|
||||
// tmp4 (x4/x13): FallbackHandler
|
||||
// x30: return
|
||||
@@ -254,15 +256,15 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::Ref Node) {
|
||||
|
||||
fmov(VTMP1.D(), Src1.D());
|
||||
|
||||
ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
|
||||
ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.FallbackHandlerPointers[Info.HandlerIndex].Func));
|
||||
ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
|
||||
ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].Func));
|
||||
blr(TMP1);
|
||||
|
||||
ldr<ARMEmitter::IndexType::POST>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);
|
||||
FillF64x2Result(DstLo, DstHi);
|
||||
} break;
|
||||
|
||||
case FABI_F64_F64_F64_PTR: {
|
||||
case FABI_F64_I16_F64_F64_PTR: {
|
||||
// Linux Reg/Win32 Reg:
|
||||
// tmp4 (x4/x13): FallbackHandler
|
||||
// x30: return
|
||||
@@ -276,8 +278,8 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::Ref Node) {
|
||||
fmov(VTMP1.D(), Src1.D());
|
||||
fmov(VTMP2.D(), Src2.D());
|
||||
|
||||
ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
|
||||
ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.FallbackHandlerPointers[Info.HandlerIndex].Func));
|
||||
ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
|
||||
ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].Func));
|
||||
blr(TMP1);
|
||||
|
||||
ldr<ARMEmitter::IndexType::POST>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);
|
||||
@@ -294,8 +296,8 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::Ref Node) {
|
||||
const auto Src1 = GetVReg(IROp->Args[0]);
|
||||
mov(VTMP1.Q(), Src1.Q());
|
||||
|
||||
ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
|
||||
ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.FallbackHandlerPointers[Info.HandlerIndex].Func));
|
||||
ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
|
||||
ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].Func));
|
||||
blr(TMP1);
|
||||
|
||||
ldr<ARMEmitter::IndexType::POST>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);
|
||||
@@ -312,8 +314,8 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::Ref Node) {
|
||||
const auto Src1 = GetVReg(IROp->Args[0]);
|
||||
mov(VTMP1.Q(), Src1.Q());
|
||||
|
||||
ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
|
||||
ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.FallbackHandlerPointers[Info.HandlerIndex].Func));
|
||||
ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
|
||||
ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].Func));
|
||||
blr(TMP1);
|
||||
|
||||
ldr<ARMEmitter::IndexType::POST>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);
|
||||
@@ -330,8 +332,8 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::Ref Node) {
|
||||
const auto Src1 = GetVReg(IROp->Args[0]);
|
||||
mov(VTMP1.Q(), Src1.Q());
|
||||
|
||||
ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
|
||||
ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.FallbackHandlerPointers[Info.HandlerIndex].Func));
|
||||
ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
|
||||
ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].Func));
|
||||
blr(TMP1);
|
||||
|
||||
ldr<ARMEmitter::IndexType::POST>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);
|
||||
@@ -351,8 +353,8 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::Ref Node) {
|
||||
mov(VTMP1.Q(), Src1.Q());
|
||||
mov(VTMP2.Q(), Src2.Q());
|
||||
|
||||
ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
|
||||
ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.FallbackHandlerPointers[Info.HandlerIndex].Func));
|
||||
ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
|
||||
ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].Func));
|
||||
blr(TMP1);
|
||||
|
||||
ldr<ARMEmitter::IndexType::POST>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);
|
||||
@@ -369,8 +371,8 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::Ref Node) {
|
||||
const auto Src1 = GetVReg(IROp->Args[0]);
|
||||
mov(VTMP1.Q(), Src1.Q());
|
||||
|
||||
ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
|
||||
ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.FallbackHandlerPointers[Info.HandlerIndex].Func));
|
||||
ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
|
||||
ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].Func));
|
||||
blr(TMP1);
|
||||
|
||||
ldr<ARMEmitter::IndexType::POST>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);
|
||||
@@ -394,8 +396,8 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::Ref Node) {
|
||||
|
||||
mov(VTMP1.Q(), Src1.Q());
|
||||
|
||||
ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
|
||||
ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.FallbackHandlerPointers[Info.HandlerIndex].Func));
|
||||
ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
|
||||
ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].Func));
|
||||
blr(TMP1);
|
||||
|
||||
ldr<ARMEmitter::IndexType::POST>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);
|
||||
@@ -416,8 +418,8 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::Ref Node) {
|
||||
mov(VTMP1.Q(), Src1.Q());
|
||||
mov(VTMP2.Q(), Src2.Q());
|
||||
|
||||
ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
|
||||
ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.FallbackHandlerPointers[Info.HandlerIndex].Func));
|
||||
ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
|
||||
ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].Func));
|
||||
blr(TMP1);
|
||||
|
||||
ldr<ARMEmitter::IndexType::POST>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);
|
||||
@@ -434,8 +436,8 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::Ref Node) {
|
||||
// tmp2 (x1/x11): source 2
|
||||
// tmp3 (x2/x12): source 3
|
||||
const auto Op = IROp->C<IR::IROp_VPCMPESTRX>();
|
||||
ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
|
||||
ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.FallbackHandlerPointers[Info.HandlerIndex].Func));
|
||||
ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
|
||||
ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].Func));
|
||||
|
||||
stp<ARMEmitter::IndexType::PRE>(TMP1, ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);
|
||||
|
||||
@@ -476,8 +478,8 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::Ref Node) {
|
||||
mov(VTMP2.Q(), Src2.Q());
|
||||
movz(ARMEmitter::Size::i32Bit, TMP1, Control);
|
||||
|
||||
ldr(TMP2, STATE_PTR(CpuStateFrame, Pointers.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
|
||||
ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.FallbackHandlerPointers[Info.HandlerIndex].Func));
|
||||
ldr(TMP2, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].ABIHandler));
|
||||
ldr(TMP4, STATE_PTR(CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex].Func));
|
||||
blr(TMP2);
|
||||
|
||||
ldr<ARMEmitter::IndexType::POST>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);
|
||||
@@ -493,117 +495,59 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::Ref Node) {
|
||||
}
|
||||
}
|
||||
|
||||
static void DirectBlockDelinker(FEXCore::Context::ExitFunctionLinkData* Record, bool Call) {
|
||||
uintptr_t JumpThunkStartAddress = reinterpret_cast<uintptr_t>(Record) - 0x10;
|
||||
uintptr_t CallerAddress = JumpThunkStartAddress + Record->CallerOffset;
|
||||
auto BranchOffset = JumpThunkStartAddress / 4 - CallerAddress / 4;
|
||||
static void DirectBlockDelinker(FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record) {
|
||||
// Emit new 16 bytes of code to a temporary patch, then atomically apply it
|
||||
__uint128_t Patch;
|
||||
ARMEmitter::Emitter emit((uint8_t*)&Patch, sizeof(Patch));
|
||||
emit.ldr(TMP1, 8); // PC-relative value pointing to constant after blr
|
||||
emit.blr(TMP1);
|
||||
emit.dc64(Frame->Pointers.Common.ExitFunctionLinker);
|
||||
|
||||
// Replace the patched callsite with a branch to the jump thunk.
|
||||
uint32_t BranchInst = 0;
|
||||
ARMEmitter::Emitter BranchEmit(reinterpret_cast<uint8_t*>(&BranchInst), 4);
|
||||
if (Call) {
|
||||
BranchEmit.bl(BranchOffset);
|
||||
} else {
|
||||
BranchEmit.b(BranchOffset);
|
||||
}
|
||||
|
||||
std::atomic_ref<uint32_t>(*reinterpret_cast<uint32_t*>(CallerAddress)).store(BranchInst, std::memory_order::relaxed);
|
||||
ARMEmitter::Emitter::ClearICache(reinterpret_cast<void*>(CallerAddress), 4);
|
||||
auto branch = reinterpret_cast<__uint128_t*>((uintptr_t)Record - 8);
|
||||
std::atomic_ref<__uint128_t>(*branch).store(Patch, std::memory_order::relaxed);
|
||||
ARMEmitter::Emitter::ClearICache((void*)branch, sizeof(*branch));
|
||||
}
|
||||
|
||||
static void IndirectBlockDelinker(FEXCore::Context::ExitFunctionLinkData* Record) {
|
||||
uintptr_t JumpThunkStartAddress = reinterpret_cast<uintptr_t>(Record) - 0x10;
|
||||
uint32_t BranchInst = 0;
|
||||
ARMEmitter::Emitter BranchEmit(reinterpret_cast<uint8_t*>(&BranchInst), 4);
|
||||
// Restore branch +2 instructions to jump to the linker block
|
||||
BranchEmit.b(0x2);
|
||||
|
||||
std::atomic_ref<uint32_t>(*reinterpret_cast<uint32_t*>(JumpThunkStartAddress)).store(BranchInst, std::memory_order::relaxed);
|
||||
ARMEmitter::Emitter::ClearICache(reinterpret_cast<void*>(JumpThunkStartAddress), 4);
|
||||
|
||||
// No need to reset HostCode here as the exit linker pointer is stored separately, and if the block is relinked it will be updated.
|
||||
}
|
||||
|
||||
uint64_t Arm64JITCore::ExitFunctionLink(FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record) {
|
||||
static uint64_t Arm64JITCore_ExitFunctionLink(FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record) {
|
||||
auto Thread = Frame->Thread;
|
||||
auto Lock = Thread->LookupCache->AcquireLock();
|
||||
|
||||
bool TFSet = Thread->CurrentFrame->State.flags[X86State::RFLAG_TF_RAW_LOC];
|
||||
uintptr_t HostCode {};
|
||||
auto GuestRip = Record->GuestRIP;
|
||||
|
||||
if (TFSet) {
|
||||
if (!TFSet) {
|
||||
HostCode = Thread->LookupCache->FindBlock(GuestRip);
|
||||
}
|
||||
|
||||
if (TFSet || !HostCode) {
|
||||
// If TF is set, the cache must be skipped as different code needs to be generated.
|
||||
Frame->State.rip = GuestRip;
|
||||
return Frame->Pointers.DispatcherLoopTop;
|
||||
} else {
|
||||
{
|
||||
// Guard the LookupCache lock with the code invalidation mutex, to avoid issues with forking
|
||||
auto lk_inval =
|
||||
GuardSignalDeferringSection<std::shared_lock>(static_cast<Context::ContextImpl*>(Thread->CTX)->CodeInvalidationMutex, Thread);
|
||||
HostCode = Thread->LookupCache->FindBlock(Thread, GuestRip);
|
||||
}
|
||||
if (!HostCode) {
|
||||
// Hold a reference to the code buffer, to avoid linking unmapped code if compilation triggers a recreation.
|
||||
auto CodeBuffer = static_cast<Arm64JITCore*>(Thread->CPUBackend.get())->CurrentCodeBuffer;
|
||||
HostCode = static_cast<Context::ContextImpl*>(Thread->CTX)->CompileBlock(Frame, GuestRip, 0);
|
||||
if (Thread->LookupCache->Shared != CodeBuffer->LookupCache.get()) {
|
||||
return HostCode;
|
||||
}
|
||||
}
|
||||
return Frame->Pointers.Common.DispatcherLoopTop;
|
||||
}
|
||||
|
||||
// See ExitFunction in BranchOps.cpp for an assembly level view of the handled cases.
|
||||
uintptr_t JumpThunkStartAddress = reinterpret_cast<uintptr_t>(Record) - 0x10;
|
||||
uintptr_t CallerAddress = JumpThunkStartAddress + Record->CallerOffset;
|
||||
auto BranchOffset = HostCode / 4 - CallerAddress / 4;
|
||||
uintptr_t branch = (uintptr_t)(Record)-8;
|
||||
LOGMAN_THROW_A_FMT((branch % 16) == 0, "Incorrect alignment for block linking record");
|
||||
|
||||
uint32_t ExpectedKnownCallMarkerInst = 0;
|
||||
ARMEmitter::Emitter ExpectedKnownCallMarkerEmit(reinterpret_cast<uint8_t*>(&ExpectedKnownCallMarkerInst), 4);
|
||||
ExpectedKnownCallMarkerEmit.adr(TMP1, 0xC);
|
||||
|
||||
// Guard the LookupCache lock with the code invalidation mutex, to avoid issues with forking
|
||||
auto lk_inval = GuardSignalDeferringSection<std::shared_lock>(static_cast<Context::ContextImpl*>(Thread->CTX)->CodeInvalidationMutex, Thread);
|
||||
|
||||
// Lock here is necessary to prevent simultaneous linking and delinking
|
||||
auto lk = Thread->LookupCache->AcquireWriteLock();
|
||||
|
||||
// For non-calls, this would extend into the block's code, however that's fine as an out-of-range adr would never
|
||||
// be generated avoiding any false positives.
|
||||
uintptr_t KnownCallMarkerAddr = CallerAddress - 0x8;
|
||||
uint32_t KnownCallMarkerInst = *reinterpret_cast<uint32_t*>(KnownCallMarkerAddr);
|
||||
if (ARMEmitter::Emitter::IsInt26(BranchOffset)) {
|
||||
// Directly patch the callsite with the appropriate branch instruction.
|
||||
uint32_t BranchInst = 0;
|
||||
ARMEmitter::Emitter BranchEmit(reinterpret_cast<uint8_t*>(&BranchInst), 4);
|
||||
|
||||
if (KnownCallMarkerInst == ExpectedKnownCallMarkerInst) {
|
||||
BranchEmit.bl(BranchOffset);
|
||||
Thread->LookupCache->AddBlockLink(
|
||||
GuestRip, Record, [](FEXCore::Context::ExitFunctionLinkData* Record) { DirectBlockDelinker(Record, true); }, lk);
|
||||
} else {
|
||||
BranchEmit.b(BranchOffset);
|
||||
Thread->LookupCache->AddBlockLink(
|
||||
GuestRip, Record, [](FEXCore::Context::ExitFunctionLinkData* Record) { DirectBlockDelinker(Record, false); }, lk);
|
||||
}
|
||||
|
||||
std::atomic_ref<uint32_t>(*reinterpret_cast<uint32_t*>(CallerAddress)).store(BranchInst, std::memory_order::relaxed);
|
||||
ARMEmitter::Emitter::ClearICache(reinterpret_cast<void*>(CallerAddress), 4);
|
||||
auto offset = HostCode / 4 - branch / 4;
|
||||
if (ARMEmitter::Emitter::IsInt26(offset)) {
|
||||
// This is the optimal case, where the target can be encoded in a single instruction.
|
||||
// Atomically patch the code with a relative branch.
|
||||
const uint32_t Patch = (0b0001'01 << 26) | (offset & ((1u << 26) - 1));
|
||||
std::atomic_ref<uint32_t>(*reinterpret_cast<uint32_t*>(branch)).store(Patch, std::memory_order::relaxed);
|
||||
ARMEmitter::Emitter::ClearICache((void*)branch, 4);
|
||||
} else {
|
||||
// This case is common between calls and jumps as the thunk callsite can be left untouched.
|
||||
std::atomic_ref<uint64_t>(Record->HostCode).store(HostCode, std::memory_order::seq_cst);
|
||||
#ifdef ARCHITECTURE_arm64
|
||||
// fallback case - do a soft-er link by patching the pointer
|
||||
std::atomic_ref<uint64_t>(Record->HostBranch).store(HostCode, std::memory_order::seq_cst);
|
||||
#ifdef _M_ARM_64
|
||||
// Make memory write visible to other threads reading the same location
|
||||
asm volatile("dc cvau, %0; dsb ish" : : "r"(Record->HostCode) :);
|
||||
asm volatile("dc cvau, %0; dsb ish" : : "r"(Record->HostBranch) :);
|
||||
#endif
|
||||
|
||||
uint32_t LdrInst = 0;
|
||||
ARMEmitter::Emitter LdrEmit(reinterpret_cast<uint8_t*>(&LdrInst), 4);
|
||||
LdrEmit.ldr(TMP1, reinterpret_cast<uint64_t>(&Record->HostCode) - JumpThunkStartAddress);
|
||||
std::atomic_ref<uint32_t>(*reinterpret_cast<uint32_t*>(JumpThunkStartAddress)).store(LdrInst, std::memory_order::relaxed);
|
||||
ARMEmitter::Emitter::ClearICache(reinterpret_cast<void*>(JumpThunkStartAddress), 4);
|
||||
|
||||
Thread->LookupCache->AddBlockLink(GuestRip, Record, IndirectBlockDelinker, lk);
|
||||
}
|
||||
|
||||
// Add de-linking handler
|
||||
Thread->LookupCache->AddBlockLink(GuestRip, Record, DirectBlockDelinker);
|
||||
|
||||
return HostCode;
|
||||
}
|
||||
|
||||
@@ -622,46 +566,58 @@ Arm64JITCore::Arm64JITCore(FEXCore::Context::ContextImpl* ctx, FEXCore::Core::In
|
||||
|
||||
RAPass = Thread->PassManager->GetPass<IR::RegisterAllocationPass>("RA");
|
||||
|
||||
RAPass->AddRegisters(IR::RegClass::GPR, GeneralRegisters.size());
|
||||
RAPass->AddRegisters(IR::RegClass::GPRFixed, StaticRegisters.size());
|
||||
RAPass->AddRegisters(IR::RegClass::FPR, GeneralFPRegisters.size());
|
||||
RAPass->AddRegisters(IR::RegClass::FPRFixed, StaticFPRegisters.size());
|
||||
RAPass->AddRegisters(FEXCore::IR::GPRClass, GeneralRegisters.size());
|
||||
RAPass->AddRegisters(FEXCore::IR::GPRFixedClass, StaticRegisters.size());
|
||||
RAPass->AddRegisters(FEXCore::IR::FPRClass, GeneralFPRegisters.size());
|
||||
RAPass->AddRegisters(FEXCore::IR::FPRFixedClass, StaticFPRegisters.size());
|
||||
RAPass->PairRegs = PairRegisters;
|
||||
|
||||
{
|
||||
// Set up pointers that the JIT needs to load
|
||||
|
||||
// Common
|
||||
auto& Ptrs = ThreadState->CurrentFrame->Pointers;
|
||||
auto& Common = ThreadState->CurrentFrame->Pointers.Common;
|
||||
|
||||
Ptrs.PrintValue = reinterpret_cast<uint64_t>(PrintValue);
|
||||
Ptrs.PrintVectorValue = reinterpret_cast<uint64_t>(PrintVectorValue);
|
||||
Ptrs.ThreadRemoveCodeEntryFromJIT = reinterpret_cast<uintptr_t>(&Context::ContextImpl::ThreadRemoveCodeEntryFromJit);
|
||||
Ptrs.MonoBackpatcherWrite = reinterpret_cast<uint64_t>(&Context::ContextImpl::MonoBackpatcherWrite);
|
||||
Ptrs.CPUIDObj = reinterpret_cast<uint64_t>(&CTX->CPUID);
|
||||
Common.PrintValue = reinterpret_cast<uint64_t>(PrintValue);
|
||||
Common.PrintVectorValue = reinterpret_cast<uint64_t>(PrintVectorValue);
|
||||
Common.ThreadRemoveCodeEntryFromJIT = reinterpret_cast<uintptr_t>(&Context::ContextImpl::ThreadRemoveCodeEntryFromJit);
|
||||
Common.CPUIDObj = reinterpret_cast<uint64_t>(&CTX->CPUID);
|
||||
|
||||
{
|
||||
FEXCore::Utils::MemberFunctionToPointerCast PMF(&FEXCore::CPUIDEmu::RunFunction);
|
||||
Ptrs.CPUIDFunction = PMF.GetConvertedPointer();
|
||||
Common.CPUIDFunction = PMF.GetConvertedPointer();
|
||||
}
|
||||
|
||||
{
|
||||
FEXCore::Utils::MemberFunctionToPointerCast PMF(&FEXCore::CPUIDEmu::RunXCRFunction);
|
||||
Ptrs.XCRFunction = PMF.GetConvertedPointer();
|
||||
Common.XCRFunction = PMF.GetConvertedPointer();
|
||||
}
|
||||
|
||||
{
|
||||
FEXCore::Utils::MemberFunctionToPointerCast PMF(&FEXCore::HLE::SyscallHandler::HandleSyscall);
|
||||
Ptrs.SyscallHandlerObj = reinterpret_cast<uint64_t>(CTX->SyscallHandler);
|
||||
Ptrs.SyscallHandlerFunc = PMF.GetVTableEntry(CTX->SyscallHandler);
|
||||
Common.SyscallHandlerObj = reinterpret_cast<uint64_t>(CTX->SyscallHandler);
|
||||
Common.SyscallHandlerFunc = PMF.GetVTableEntry(CTX->SyscallHandler);
|
||||
}
|
||||
Ptrs.ExitFunctionLink = reinterpret_cast<uintptr_t>(&Arm64JITCore::ExitFunctionLink);
|
||||
Ptrs.LUDIV = reinterpret_cast<uint64_t>(LUDIV);
|
||||
Ptrs.LDIV = reinterpret_cast<uint64_t>(LDIV);
|
||||
Common.ExitFunctionLink = reinterpret_cast<uintptr_t>(&Context::ContextImpl::ThreadExitFunctionLink<Arm64JITCore_ExitFunctionLink>);
|
||||
|
||||
// Platform Specific
|
||||
auto& AArch64 = ThreadState->CurrentFrame->Pointers.AArch64;
|
||||
|
||||
AArch64.LUDIV = reinterpret_cast<uint64_t>(LUDIV);
|
||||
AArch64.LDIV = reinterpret_cast<uint64_t>(LDIV);
|
||||
}
|
||||
|
||||
CurrentCodeBuffer = CodeBuffers.GetLatest();
|
||||
ThreadState->LookupCache->Shared = CurrentCodeBuffer->LookupCache.get();
|
||||
|
||||
// Setup dynamic dispatch.
|
||||
if (ParanoidTSO()) {
|
||||
RT_LoadMemTSO = &Arm64JITCore::Op_ParanoidLoadMemTSO;
|
||||
RT_StoreMemTSO = &Arm64JITCore::Op_ParanoidStoreMemTSO;
|
||||
} else {
|
||||
RT_LoadMemTSO = &Arm64JITCore::Op_LoadMemTSO;
|
||||
RT_StoreMemTSO = &Arm64JITCore::Op_StoreMemTSO;
|
||||
}
|
||||
}
|
||||
|
||||
void Arm64JITCore::EmitDetectionString() {
|
||||
@@ -673,13 +629,13 @@ void Arm64JITCore::EmitDetectionString() {
|
||||
void Arm64JITCore::ClearCache() {
|
||||
// NOTE: Holding on to the reference here is required to ensure validity of the WriteLock mutex
|
||||
auto PrevCodeBuffer = CurrentCodeBuffer;
|
||||
auto lk = PrevCodeBuffer->LookupCache->AcquireWriteLock();
|
||||
std::lock_guard lk(PrevCodeBuffer->LookupCache->WriteLock);
|
||||
|
||||
auto CodeBuffer = GetEmptyCodeBuffer();
|
||||
SetBuffer(CodeBuffer->Ptr, CodeBuffer->Size);
|
||||
EmitDetectionString();
|
||||
|
||||
ThreadState->LookupCache->ChangeGuestToHostMapping(*PrevCodeBuffer, *CurrentCodeBuffer->LookupCache, lk);
|
||||
ThreadState->LookupCache->ChangeGuestToHostMapping(*PrevCodeBuffer, *CurrentCodeBuffer->LookupCache);
|
||||
}
|
||||
|
||||
Arm64JITCore::~Arm64JITCore() {}
|
||||
@@ -725,48 +681,48 @@ bool Arm64JITCore::IsInlineEntrypointOffset(const IR::OrderedNodeWrapper& WNode,
|
||||
}
|
||||
}
|
||||
|
||||
void Arm64JITCore::EmitTFCheck() {
|
||||
ARMEmitter::ForwardLabel l_TFUnset;
|
||||
ARMEmitter::ForwardLabel l_TFBlocked;
|
||||
void Arm64JITCore::EmitInterruptChecks(bool CheckTF) {
|
||||
if (CheckTF) {
|
||||
ARMEmitter::ForwardLabel l_TFUnset;
|
||||
ARMEmitter::ForwardLabel l_TFBlocked;
|
||||
|
||||
// Note that this needs to be before the below suspend checks, as X86 checks this flag immediately after executing an instruction.
|
||||
ldrb(TMP1, STATE_PTR(CpuStateFrame, State.flags[X86State::RFLAG_TF_RAW_LOC]));
|
||||
// Note that this needs to be before the below suspend checks, as X86 checks this flag immediately after executing an instruction.
|
||||
ldrb(TMP1, STATE_PTR(CpuStateFrame, State.flags[X86State::RFLAG_TF_RAW_LOC]));
|
||||
|
||||
(void)cbz(ARMEmitter::Size::i32Bit, TMP1, &l_TFUnset);
|
||||
cbz(ARMEmitter::Size::i32Bit, TMP1, &l_TFUnset);
|
||||
|
||||
// X86 semantically checks TF after executing each instruction, so e.g. setting a context with TF set will execute a single instruction
|
||||
// and then raise an exception. However on the FEX side this is simpler to implement by checking at the start of each instruction, handle this by having bit 1 being unset in the flag state indicate that TF is blocked for a single instruction.
|
||||
(void)tbz(TMP1, 1, &l_TFBlocked);
|
||||
// X86 semantically checks TF after executing each instruction, so e.g. setting a context with TF set will execute a single instruction
|
||||
// and then raise an exception. However on the FEX side this is simpler to implement by checking at the start of each instruction, handle this by having bit 1 being unset in the flag state indicate that TF is blocked for a single instruction.
|
||||
tbz(TMP1, 1, &l_TFBlocked);
|
||||
|
||||
// Block TF for a single instruction when the frontend jumps to a new context by unsetting bit 1.
|
||||
ldrb(TMP1, STATE_PTR(CpuStateFrame, State.flags[X86State::RFLAG_TF_RAW_LOC]));
|
||||
and_(ARMEmitter::Size::i32Bit, TMP1, TMP1, ~(1 << 1));
|
||||
strb(TMP1, STATE_PTR(CpuStateFrame, State.flags[X86State::RFLAG_TF_RAW_LOC]));
|
||||
// Block TF for a single instruction when the frontend jumps to a new context by unsetting bit 1.
|
||||
ldrb(TMP1, STATE_PTR(CpuStateFrame, State.flags[X86State::RFLAG_TF_RAW_LOC]));
|
||||
and_(ARMEmitter::Size::i32Bit, TMP1, TMP1, ~(1 << 1));
|
||||
strb(TMP1, STATE_PTR(CpuStateFrame, State.flags[X86State::RFLAG_TF_RAW_LOC]));
|
||||
|
||||
Core::CpuStateFrame::SynchronousFaultDataStruct State = {
|
||||
.FaultToTopAndGeneratedException = 1,
|
||||
.Signal = Core::FAULT_SIGTRAP,
|
||||
.TrapNo = X86State::X86_TRAPNO_DB,
|
||||
.si_code = 2,
|
||||
.err_code = 0,
|
||||
};
|
||||
Core::CpuStateFrame::SynchronousFaultDataStruct State = {
|
||||
.FaultToTopAndGeneratedException = 1,
|
||||
.Signal = Core::FAULT_SIGTRAP,
|
||||
.TrapNo = X86State::X86_TRAPNO_DB,
|
||||
.si_code = 2,
|
||||
.err_code = 0,
|
||||
};
|
||||
|
||||
uint64_t Constant {};
|
||||
memcpy(&Constant, &State, sizeof(State));
|
||||
uint64_t Constant {};
|
||||
memcpy(&Constant, &State, sizeof(State));
|
||||
|
||||
LoadConstant(ARMEmitter::Size::i64Bit, TMP1, Constant);
|
||||
str(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, SynchronousFaultData));
|
||||
ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGTRAP));
|
||||
br(TMP1);
|
||||
LoadConstant(ARMEmitter::Size::i64Bit, TMP1, Constant);
|
||||
str(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, SynchronousFaultData));
|
||||
ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.GuestSignal_SIGTRAP));
|
||||
br(TMP1);
|
||||
|
||||
(void)Bind(&l_TFBlocked);
|
||||
// If TF was blocked for this instruction, unblock it for the next.
|
||||
LoadConstant(ARMEmitter::Size::i32Bit, TMP1, 0b11);
|
||||
strb(TMP1, STATE_PTR(CpuStateFrame, State.flags[X86State::RFLAG_TF_RAW_LOC]));
|
||||
(void)Bind(&l_TFUnset);
|
||||
}
|
||||
Bind(&l_TFBlocked);
|
||||
// If TF was blocked for this instruction, unblock it for the next.
|
||||
LoadConstant(ARMEmitter::Size::i32Bit, TMP1, 0b11);
|
||||
strb(TMP1, STATE_PTR(CpuStateFrame, State.flags[X86State::RFLAG_TF_RAW_LOC]));
|
||||
Bind(&l_TFUnset);
|
||||
}
|
||||
|
||||
void Arm64JITCore::EmitSuspendInterruptCheck() {
|
||||
if (CTX->Config.NeedsPendingInterruptFaultCheck) {
|
||||
// Trigger a fault if there are any pending interrupts
|
||||
// Used only for suspend on WIN32 at the moment
|
||||
@@ -774,100 +730,47 @@ void Arm64JITCore::EmitSuspendInterruptCheck() {
|
||||
offsetof(FEXCore::Core::InternalThreadState, InterruptFaultPage) - offsetof(FEXCore::Core::InternalThreadState, BaseFrameState));
|
||||
}
|
||||
|
||||
#ifdef ARCHITECTURE_arm64ec
|
||||
#ifdef _M_ARM_64EC
|
||||
static constexpr uint16_t SuspendMagic {0xCAFE};
|
||||
|
||||
ldr(TMP2.W(), STATE_PTR(CpuStateFrame, SuspendDoorbell));
|
||||
ARMEmitter::ForwardLabel l_NoSuspend;
|
||||
(void)cbz(ARMEmitter::Size::i32Bit, TMP2, &l_NoSuspend);
|
||||
cbz(ARMEmitter::Size::i32Bit, TMP2, &l_NoSuspend);
|
||||
brk(SuspendMagic);
|
||||
(void)Bind(&l_NoSuspend);
|
||||
Bind(&l_NoSuspend);
|
||||
#endif
|
||||
}
|
||||
|
||||
void Arm64JITCore::EmitEntryPoint(ARMEmitter::BackwardLabel& HeaderLabel, bool CheckTF) {
|
||||
// Get the address of the JITCodeHeader and store in to the core state.
|
||||
// Two instruction cost, each 1 cycle.
|
||||
adr_OrRestart(TMP1, &HeaderLabel);
|
||||
str(TMP1, STATE, offsetof(FEXCore::Core::CPUState, InlineJITBlockHeader));
|
||||
|
||||
if (CheckTF) {
|
||||
EmitTFCheck();
|
||||
}
|
||||
|
||||
if (SpillSlots) {
|
||||
const auto TotalSpillSlotsSize = SpillSlots * MaxSpillSlotSize;
|
||||
|
||||
if (ARMEmitter::IsImmAddSub(TotalSpillSlotsSize)) {
|
||||
sub(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, TotalSpillSlotsSize);
|
||||
} else {
|
||||
LoadConstant(ARMEmitter::Size::i64Bit, TMP1, TotalSpillSlotsSize);
|
||||
sub(ARMEmitter::Size::i64Bit, ARMEmitter::XReg::rsp, ARMEmitter::XReg::rsp, TMP1, ARMEmitter::ExtendedType::LSL_64, 0);
|
||||
}
|
||||
}
|
||||
|
||||
EmitSuspendInterruptCheck();
|
||||
}
|
||||
|
||||
CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size, bool SingleInst, const FEXCore::IR::IRListView* IR,
|
||||
FEXCore::Core::DebugData* DebugData, bool CheckTF) {
|
||||
FEXCORE_PROFILE_SCOPED("Arm64::CompileCode");
|
||||
|
||||
const auto PrevNumAllocations = Relocations.size();
|
||||
JumpTargets.clear();
|
||||
uint32_t SSACount = IR->GetSSACount();
|
||||
|
||||
this->Entry = Entry;
|
||||
this->DebugData = DebugData;
|
||||
this->IR = IR;
|
||||
RequiresFarARM64Jumps = false;
|
||||
SSANodeMultiplier = 24;
|
||||
|
||||
// Prepare restart via long jump in case branch encoding fails.
|
||||
// This uses UncheckedLongJump since we don't implement std::longjmp in WoA setups
|
||||
switch (static_cast<RestartOptions::Control>(FEXCore::UncheckedLongJump::SetJump(ThreadState->RestartJump))) {
|
||||
case RestartOptions::Control::Incoming:
|
||||
// Nothing
|
||||
break;
|
||||
case RestartOptions::Control::EnableFarARM64Jumps: RequiresFarARM64Jumps = true; break;
|
||||
case RestartOptions::Control::NeedsLargerJITSpace:
|
||||
// Get rid of the claimed buffer immediately, we can't fit in it at all.
|
||||
TempAllocator.UnclaimBuffer();
|
||||
SSANodeMultiplier *= 2;
|
||||
break;
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled Arm64 restart condition!");
|
||||
}
|
||||
|
||||
uint32_t SSACount = IR->GetSSACount();
|
||||
JumpTargets.clear();
|
||||
CallReturnTargets.clear();
|
||||
PendingJumpThunks.clear();
|
||||
JumpTargets.resize(IR->GetHeader()->BlockCount, {});
|
||||
|
||||
CodeData.EntryPoints.clear();
|
||||
|
||||
// Fairly excessive buffer range to make sure we don't overflow
|
||||
// One page baseline, plus SSANodeMultipler bytes, plus another page for guard page.
|
||||
const uint32_t DesiredBufferRange = AlignUp(FEXCore::Utils::FEX_PAGE_SIZE * 2 + SSACount * SSANodeMultiplier, FEXCore::Utils::FEX_PAGE_SIZE);
|
||||
uint32_t BufferRange = 0x100 + SSACount * 24;
|
||||
|
||||
// JIT output is first written to a temporary buffer and later relocated to the CodeBuffer.
|
||||
// This minimizes lock contention of CodeBufferWriteMutex.
|
||||
auto TempCodeBufferInfo = TempAllocator.ReownOrClaimBufferWithSize(DesiredBufferRange);
|
||||
auto TempCodeBuffer = TempCodeBufferInfo.Ptr;
|
||||
const uint32_t UsableBufferRange = TempCodeBufferInfo.Size - FEXCore::Utils::FEX_PAGE_SIZE;
|
||||
|
||||
SetBuffer(TempCodeBuffer, UsableBufferRange);
|
||||
|
||||
ThreadState->JITGuardPage = reinterpret_cast<uintptr_t>(TempCodeBuffer) + UsableBufferRange;
|
||||
ThreadState->JITGuardOverflowArgument = FEXCore::ToUnderlying(RestartOptions::Control::NeedsLargerJITSpace);
|
||||
auto TempCodeBuffer = TempAllocator.ReownOrClaimBuffer(BufferRange);
|
||||
SetBuffer(TempCodeBuffer, BufferRange);
|
||||
|
||||
CodeData.BlockBegin = GetCursorAddress<uint8_t*>();
|
||||
|
||||
// Put the code header at the start of the data block.
|
||||
ARMEmitter::BackwardLabel JITCodeHeaderLabel {};
|
||||
(void)Bind(&JITCodeHeaderLabel);
|
||||
Bind(&JITCodeHeaderLabel);
|
||||
JITCodeHeader* CodeHeader = GetCursorAddress<JITCodeHeader*>();
|
||||
CursorIncrement(sizeof(JITCodeHeader));
|
||||
|
||||
auto CodeBegin = GetCursorAddress<uint8_t*>();
|
||||
#ifdef VIXL_DISASSEMBLER
|
||||
const auto DisasmBegin = GetCursorAddress<const vixl::aarch64::Instruction*>();
|
||||
#endif
|
||||
|
||||
// AAPCS64
|
||||
// r30 = LR
|
||||
@@ -889,64 +792,55 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
|
||||
// X1-X3 = Temp
|
||||
// X4-r18 = RA
|
||||
|
||||
CodeData.BlockEntry = GetCursorAddress<uint8_t*>();
|
||||
|
||||
// Get the address of the JITCodeHeader and store in to the core state.
|
||||
// Two instruction cost, each 1 cycle.
|
||||
adr(TMP1, &JITCodeHeaderLabel);
|
||||
str(TMP1, STATE, offsetof(FEXCore::Core::CPUState, InlineJITBlockHeader));
|
||||
|
||||
EmitInterruptChecks(CheckTF);
|
||||
|
||||
SpillSlots = IR->SpillSlots();
|
||||
|
||||
if (SpillSlots) {
|
||||
const auto TotalSpillSlotsSize = SpillSlots * MaxSpillSlotSize;
|
||||
|
||||
if (ARMEmitter::IsImmAddSub(TotalSpillSlotsSize)) {
|
||||
sub(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, TotalSpillSlotsSize);
|
||||
} else {
|
||||
LoadConstant(ARMEmitter::Size::i64Bit, TMP1, TotalSpillSlotsSize);
|
||||
sub(ARMEmitter::Size::i64Bit, ARMEmitter::XReg::rsp, ARMEmitter::XReg::rsp, TMP1, ARMEmitter::ExtendedType::LSL_64, 0);
|
||||
}
|
||||
}
|
||||
|
||||
PendingTargetLabel = nullptr;
|
||||
PendingCallReturnTargetLabel = nullptr;
|
||||
|
||||
for (auto [BlockNode, BlockHeader] : IR->GetBlocks()) {
|
||||
using namespace FEXCore::IR;
|
||||
auto BlockIROp = BlockHeader->CW<FEXCore::IR::IROp_CodeBlock>();
|
||||
#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
|
||||
auto BlockIROp = BlockHeader->CW<FEXCore::IR::IROp_CodeBlock>();
|
||||
LOGMAN_THROW_A_FMT(BlockIROp->Header.Op == IR::OP_CODEBLOCK, "IR type failed to be a code block");
|
||||
#endif
|
||||
|
||||
auto BlockStartHostCode = GetCursorAddress<uint8_t*>();
|
||||
{
|
||||
const auto Node = IR->GetID(BlockNode);
|
||||
const auto Target = &JumpTargets[BlockIROp->ID];
|
||||
const auto IsTarget = JumpTargets.try_emplace(Node).first;
|
||||
|
||||
// if there's a pending branch, and it is not fall-through
|
||||
if (PendingTargetLabel && PendingTargetLabel != Target) {
|
||||
if (PendingTargetLabel->Backward.Location) {
|
||||
EmitSuspendInterruptCheck();
|
||||
}
|
||||
b_OrRestart(PendingTargetLabel);
|
||||
PendingTargetLabel = nullptr;
|
||||
}
|
||||
|
||||
if (BlockIROp->EntryPoint) {
|
||||
uint64_t BlockStartRIP = Entry + BlockIROp->GuestEntryOffset;
|
||||
|
||||
const auto IsReturnTarget = CallReturnTargets.try_emplace(Node).first;
|
||||
if (PendingTargetLabel) {
|
||||
// If there is a fallthrough branch to this block, skip over the entrypoint code.
|
||||
b_OrRestart(Target);
|
||||
} else if (PendingCallReturnTargetLabel && PendingCallReturnTargetLabel != &IsReturnTarget->second) {
|
||||
// If we just emitted a call, but the block we're now emitting is not the return block so don't fallthrough.
|
||||
b_OrRestart(PendingCallReturnTargetLabel);
|
||||
}
|
||||
PendingCallReturnTargetLabel = nullptr;
|
||||
|
||||
BindOrRestart(&IsReturnTarget->second);
|
||||
CodeData.EntryPoints.emplace(BlockStartRIP, GetCursorAddress<uint8_t*>());
|
||||
DebugData->GuestOpcodes.push_back({BlockIROp->GuestEntryOffset, GetCursorAddress<uint8_t*>() - CodeData.BlockBegin});
|
||||
|
||||
EmitEntryPoint(JITCodeHeaderLabel, CheckTF);
|
||||
}
|
||||
|
||||
if (PendingCallReturnTargetLabel) {
|
||||
// If there is still a pending call return target, then the block we're emitting is not the return block so don't fallthrough.
|
||||
b_OrRestart(PendingCallReturnTargetLabel);
|
||||
PendingCallReturnTargetLabel = nullptr;
|
||||
if (PendingTargetLabel && PendingTargetLabel != &IsTarget->second) {
|
||||
b(PendingTargetLabel);
|
||||
}
|
||||
PendingTargetLabel = nullptr;
|
||||
|
||||
BindOrRestart(Target);
|
||||
Bind(&IsTarget->second);
|
||||
}
|
||||
|
||||
for (auto [CodeNode, IROp] : IR->GetCode(BlockNode)) {
|
||||
switch (IROp->Op) {
|
||||
#define REGISTER_OP_RT(op, x) \
|
||||
case FEXCore::IR::IROps::OP_##op: std::invoke(RT_##x, this, IROp, CodeNode); break
|
||||
#define REGISTER_OP(op, x) \
|
||||
case FEXCore::IR::IROps::OP_##op: Op_##x(IROp, CodeNode); break
|
||||
|
||||
@@ -958,57 +852,24 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
|
||||
}
|
||||
}
|
||||
|
||||
DebugData->Subblocks.push_back({static_cast<uint32_t>(BlockStartHostCode - CodeData.BlockBegin),
|
||||
DebugData->Subblocks.push_back({static_cast<uint32_t>(BlockStartHostCode - CodeData.BlockEntry),
|
||||
static_cast<uint32_t>(GetCursorAddress<uint8_t*>() - BlockStartHostCode)});
|
||||
}
|
||||
|
||||
// Make sure last branch is generated. It certainly can't be eliminated here.
|
||||
if (PendingTargetLabel) {
|
||||
if (PendingTargetLabel->Backward.Location) {
|
||||
EmitSuspendInterruptCheck();
|
||||
}
|
||||
b_OrRestart(PendingTargetLabel);
|
||||
b(PendingTargetLabel);
|
||||
}
|
||||
PendingTargetLabel = nullptr;
|
||||
|
||||
ARMEmitter::ForwardLabel l_ExitLink;
|
||||
for (auto& PendingJumpThunk : PendingJumpThunks) {
|
||||
// Align as 64-bit atomics are used on the HostCode field.
|
||||
Align(8);
|
||||
// CodeSize not including the tail data.
|
||||
const uint64_t CodeOnlySize = GetCursorAddress<uint8_t*>() - CodeData.BlockBegin;
|
||||
|
||||
ARMEmitter::ForwardLabel l_DoLink;
|
||||
uint64_t ThunkAddress = GetCursorAddress<uint64_t>();
|
||||
BindOrRestart(&PendingJumpThunk.Label);
|
||||
b_OrRestart(&l_DoLink);
|
||||
br(TMP1);
|
||||
BindOrRestart(&l_DoLink);
|
||||
ldr(TMP1, &l_ExitLink);
|
||||
blr(TMP1);
|
||||
|
||||
// This is a ExitFunctionLinkData struct
|
||||
BindOrRestart(&l_ExitLink);
|
||||
dc64(0); // HostCode
|
||||
PlaceNamedSymbolLiteral(InsertGuestRIPLiteral(PendingJumpThunk.GuestRIP)); // GuestRIP
|
||||
dc64(PendingJumpThunk.CallerAddress - ThunkAddress); // CallerOffset
|
||||
}
|
||||
|
||||
BindOrRestart(&l_ExitLink);
|
||||
PlaceNamedSymbolLiteral(InsertNamedSymbolLiteral(RelocNamedSymbolLiteral::NamedSymbol::SYMBOL_LITERAL_EXITFUNCTION_LINKER));
|
||||
|
||||
// CodeSize not including the header or tail data.
|
||||
const uint64_t CodeOnlySize = GetCursorAddress<uint8_t*>() - CodeBegin;
|
||||
|
||||
// Add the JitCodeTail (written later)
|
||||
// Add the JitCodeTail
|
||||
Align(alignof(JITCodeTail));
|
||||
const auto JITBlockTailLocation = GetCursorAddress<uint8_t*>();
|
||||
CodeHeader->OffsetToBlockTail = JITBlockTailLocation - CodeData.BlockBegin;
|
||||
|
||||
JITCodeTail JITBlockTail {
|
||||
.RIP = Entry,
|
||||
.GuestSize = Size,
|
||||
.SpinLockFutex = 0,
|
||||
.SingleInst = SingleInst,
|
||||
};
|
||||
auto JITBlockTailLocation = GetCursorAddress<uint8_t*>();
|
||||
auto JITBlockTail = GetCursorAddress<JITCodeTail*>();
|
||||
CursorIncrement(sizeof(JITCodeTail));
|
||||
|
||||
// Entries that live after the JITCodeTail.
|
||||
// These entries correlate JIT code regions with guest RIP regions.
|
||||
@@ -1026,13 +887,23 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
|
||||
// FEXCore::Utils::vl64 GuestRIPOffset;
|
||||
// };
|
||||
|
||||
const auto JITRIPEntriesBegin = JITBlockTailLocation + sizeof(JITBlockTail);
|
||||
auto JITRIPEntriesBegin = GetCursorAddress<uint8_t*>();
|
||||
|
||||
// Put the block's RIP entry in the tail.
|
||||
// This will be used for RIP reconstruction in the future.
|
||||
// TODO: This needs to be a data RIP relocation once code caching works.
|
||||
// Current relocation code doesn't support this feature yet.
|
||||
JITBlockTail->RIP = Entry;
|
||||
JITBlockTail->GuestSize = Size;
|
||||
JITBlockTail->SingleInst = SingleInst;
|
||||
JITBlockTail->SpinLockFutex = 0;
|
||||
|
||||
auto JITRIPEntriesLocation = JITRIPEntriesBegin;
|
||||
|
||||
{
|
||||
// Store the RIP entries.
|
||||
JITBlockTail.NumberOfRIPEntries = DebugData->GuestOpcodes.size();
|
||||
JITBlockTail.OffsetToRIPEntries = JITRIPEntriesBegin - JITBlockTailLocation;
|
||||
JITBlockTail->NumberOfRIPEntries = DebugData->GuestOpcodes.size();
|
||||
JITBlockTail->OffsetToRIPEntries = JITRIPEntriesBegin - JITBlockTailLocation;
|
||||
uintptr_t CurrentRIPOffset = 0;
|
||||
uint64_t CurrentPCOffset = 0;
|
||||
|
||||
@@ -1048,20 +919,14 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
|
||||
}
|
||||
}
|
||||
|
||||
SetCursorOffset(JITRIPEntriesLocation - CodeData.BlockBegin);
|
||||
CursorIncrement(JITRIPEntriesLocation - JITRIPEntriesBegin);
|
||||
Align();
|
||||
|
||||
CodeHeader->OffsetToBlockTail = JITBlockTailLocation - CodeData.BlockBegin;
|
||||
|
||||
CodeData.Size = GetCursorAddress<uint8_t*>() - CodeData.BlockBegin;
|
||||
|
||||
// Finalize and write block tail data
|
||||
JITBlockTail.Size = CodeData.Size;
|
||||
{
|
||||
auto PrevCur = GetCursorOffset();
|
||||
memcpy(JITBlockTailLocation, &JITBlockTail, sizeof(JITBlockTail));
|
||||
SetCursorOffset(JITBlockTailLocation - CodeData.BlockBegin + offsetof(JITCodeTail, RIP));
|
||||
PlaceNamedSymbolLiteral(InsertGuestRIPLiteral(JITBlockTail.RIP));
|
||||
SetCursorOffset(PrevCur);
|
||||
}
|
||||
JITBlockTail->Size = CodeData.Size;
|
||||
|
||||
// Migrate the compile output from temporary storage to the actual CodeBuffer.
|
||||
// This can block progress in other compiling threads, so the duration of the lock should be as small as possible.
|
||||
@@ -1070,15 +935,14 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
|
||||
|
||||
// Query size of generated code
|
||||
const auto TempSize = GetCursorOffset();
|
||||
LOGMAN_THROW_A_FMT(TempSize <= BufferRange, "Exceeded bounds of temporary buffer ({:#x} vs {:#x})", TempSize, BufferRange);
|
||||
|
||||
// Bring CodeBuffer up to date
|
||||
{
|
||||
LOGMAN_THROW_A_FMT(CurrentCodeBuffer->LookupCache.get() == ThreadState->LookupCache->Shared, "INVARIANT VIOLATED: SharedLookupCache "
|
||||
"doesn't match up!\n");
|
||||
if (auto Prev = CheckCodeBufferUpdate()) {
|
||||
Allocator::VirtualDontNeed(ThreadState->CallRetStackBase, FEXCore::Core::InternalThreadState::CALLRET_STACK_SIZE);
|
||||
auto lk = ThreadState->LookupCache->AcquireWriteLock();
|
||||
ThreadState->LookupCache->ChangeGuestToHostMapping(*Prev, *CurrentCodeBuffer->LookupCache, lk);
|
||||
ThreadState->LookupCache->ChangeGuestToHostMapping(*Prev, *CurrentCodeBuffer->LookupCache);
|
||||
}
|
||||
|
||||
// NOTE: 16-byte alignment of the new cursor offset must be preserved for block linking records
|
||||
@@ -1096,14 +960,7 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
|
||||
// Adjust host addresses
|
||||
const auto Delta = GetCursorAddress<uint8_t*>() - CodeData.BlockBegin;
|
||||
CodeData.BlockBegin += Delta;
|
||||
for (auto& EntryPoint : CodeData.EntryPoints) {
|
||||
EntryPoint.second += Delta;
|
||||
}
|
||||
CodeBegin += Delta;
|
||||
|
||||
for (std::size_t Idx = PrevNumAllocations; Idx != Relocations.size(); ++Idx) {
|
||||
Relocations[Idx].Header.Offset += CodeBuffers.LatestOffset;
|
||||
}
|
||||
CodeData.BlockEntry += Delta;
|
||||
|
||||
// Copy over CodeBuffer contents
|
||||
memcpy(GetCursorAddress<uint8_t*>(), TempCodeBuffer, TempSize);
|
||||
@@ -1114,7 +971,7 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
|
||||
|
||||
TempAllocator.DelayedDisownBuffer();
|
||||
|
||||
ClearICache(CodeBegin, CodeOnlySize);
|
||||
ClearICache(CodeData.BlockBegin, CodeOnlySize);
|
||||
|
||||
#ifdef VIXL_DISASSEMBLER
|
||||
if (Disassemble() & FEXCore::Config::Disassemble::STATS) {
|
||||
@@ -1128,8 +985,7 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
|
||||
}
|
||||
|
||||
if (Disassemble() & FEXCore::Config::Disassemble::BLOCKS) {
|
||||
const auto DisasmBegin = reinterpret_cast<const vixl::aarch64::Instruction*>(CodeBegin);
|
||||
const auto DisasmEnd = reinterpret_cast<const vixl::aarch64::Instruction*>(CodeBegin + CodeOnlySize);
|
||||
const auto DisasmEnd = reinterpret_cast<const vixl::aarch64::Instruction*>(JITBlockTailLocation);
|
||||
LogMan::Msg::IFmt("Disassemble Begin");
|
||||
for (auto PCToDecode = DisasmBegin; PCToDecode < DisasmEnd; PCToDecode += 4) {
|
||||
DisasmDecoder->Decode(PCToDecode);
|
||||
@@ -1145,7 +1001,7 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size
|
||||
|
||||
this->IR = nullptr;
|
||||
|
||||
return std::move(CodeData);
|
||||
return CodeData;
|
||||
}
|
||||
|
||||
void Arm64JITCore::ResetStack() {
|
||||
|
||||
@@ -10,39 +10,26 @@ $end_info$
|
||||
#include "Interface/Core/ArchHelpers/Arm64Emitter.h"
|
||||
#include "Interface/Core/CPUBackend.h"
|
||||
#include "Interface/Core/Dispatcher/Dispatcher.h"
|
||||
#include "Interface/Core/JIT/Relocations.h"
|
||||
#include "Interface/IR/IR.h"
|
||||
#include "Interface/IR/IntrusiveIRList.h"
|
||||
#include "Interface/IR/RegisterAllocationData.h"
|
||||
|
||||
#include <FEXCore/Config/Config.h>
|
||||
#include <FEXCore/Core/CoreState.h>
|
||||
#include <FEXCore/IR/IR.h>
|
||||
#include <FEXCore/Utils/LogManager.h>
|
||||
#include <FEXCore/fextl/map.h>
|
||||
#include <FEXCore/fextl/memory.h>
|
||||
#include <FEXCore/fextl/string.h>
|
||||
#include <FEXCore/fextl/vector.h>
|
||||
#include <FEXCore/Utils/LongJump.h>
|
||||
|
||||
#include <CodeEmitter/Emitter.h>
|
||||
|
||||
#include <array>
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <optional>
|
||||
#include <utility>
|
||||
#include <variant>
|
||||
|
||||
namespace FEXCore::Core {
|
||||
struct InternalThreadState;
|
||||
}
|
||||
namespace FEXCore::Context {
|
||||
struct ExitFunctionLinkData;
|
||||
}
|
||||
namespace FEXCore::IR {
|
||||
class RegisterAllocationPass;
|
||||
}
|
||||
|
||||
namespace FEXCore::CPU {
|
||||
class Arm64JITCore final : public CPUBackend, public Arm64Emitter {
|
||||
@@ -61,63 +48,31 @@ public:
|
||||
}
|
||||
|
||||
private:
|
||||
FEX_CONFIG_OPT(ParanoidTSO, PARANOIDTSO);
|
||||
|
||||
const bool HostSupportsSVE128 {};
|
||||
const bool HostSupportsSVE256 {};
|
||||
const bool HostSupportsAVX256 {};
|
||||
const bool HostSupportsRPRES {};
|
||||
const bool HostSupportsAFP {};
|
||||
|
||||
struct RestartOptions {
|
||||
enum class Control : uint64_t {
|
||||
Incoming = 0,
|
||||
EnableFarARM64Jumps = 1,
|
||||
NeedsLargerJITSpace = 2,
|
||||
};
|
||||
};
|
||||
|
||||
// FEXCore makes assumptions in the JIT about certain conditions being true.
|
||||
// In the rare case when those assumptions are broken, FEX needs to safely restart the JIT.
|
||||
RestartOptions RestartControl {};
|
||||
bool RequiresFarARM64Jumps {};
|
||||
// Default to 6 instructions per SSA node.
|
||||
uint32_t SSANodeMultiplier {24};
|
||||
|
||||
ARMEmitter::BiDirectionalLabel* PendingTargetLabel {};
|
||||
ARMEmitter::BiDirectionalLabel* PendingCallReturnTargetLabel {};
|
||||
FEXCore::Context::ContextImpl* CTX {};
|
||||
const FEXCore::IR::IRListView* IR {};
|
||||
uint64_t Entry {};
|
||||
CPUBackend::CompiledCode CodeData {};
|
||||
|
||||
fextl::vector<ARMEmitter::BiDirectionalLabel> JumpTargets;
|
||||
|
||||
ARMEmitter::BiDirectionalLabel* JumpTarget(IR::OrderedNodeWrapper Node) {
|
||||
auto Block = IR->GetOp<IR::IROp_CodeBlock>(Node);
|
||||
return &JumpTargets[Block->ID];
|
||||
}
|
||||
|
||||
fextl::map<IR::NodeID, ARMEmitter::BiDirectionalLabel> CallReturnTargets;
|
||||
|
||||
struct PendingJumpThunk {
|
||||
uint64_t CallerAddress;
|
||||
uint64_t GuestRIP;
|
||||
ARMEmitter::ForwardLabel Label;
|
||||
};
|
||||
fextl::vector<PendingJumpThunk> PendingJumpThunks;
|
||||
fextl::map<IR::NodeID, ARMEmitter::BiDirectionalLabel> JumpTargets;
|
||||
|
||||
Utils::PoolBufferWithTimedRetirement<uint8_t*, 5000, 500> TempAllocator;
|
||||
|
||||
static uint64_t ExitFunctionLink(FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record);
|
||||
|
||||
[[nodiscard]]
|
||||
ARMEmitter::Register GetReg(IR::PhysicalRegister Reg) const {
|
||||
const auto RegClass = Reg.AsRegClass();
|
||||
LOGMAN_THROW_A_FMT(Reg.Class == IR::GPRFixedClass.Val || Reg.Class == IR::GPRClass.Val, "Unexpected Class: {}", Reg.Class);
|
||||
|
||||
LOGMAN_THROW_A_FMT(RegClass == IR::RegClass::GPRFixed || RegClass == IR::RegClass::GPR, "Unexpected Class: {}", Reg.Class);
|
||||
|
||||
if (RegClass == IR::RegClass::GPRFixed) {
|
||||
if (Reg.Class == IR::GPRFixedClass.Val) {
|
||||
return StaticRegisters[Reg.Reg];
|
||||
} else if (RegClass == IR::RegClass::GPR) {
|
||||
} else if (Reg.Class == IR::GPRClass.Val) {
|
||||
return GeneralRegisters[Reg.Reg];
|
||||
}
|
||||
|
||||
@@ -136,13 +91,11 @@ private:
|
||||
|
||||
[[nodiscard]]
|
||||
ARMEmitter::VRegister GetVReg(IR::PhysicalRegister Reg) const {
|
||||
const auto RegClass = Reg.AsRegClass();
|
||||
LOGMAN_THROW_A_FMT(Reg.Class == IR::FPRFixedClass.Val || Reg.Class == IR::FPRClass.Val, "Unexpected Class: {}", Reg.Class);
|
||||
|
||||
LOGMAN_THROW_A_FMT(RegClass == IR::RegClass::FPRFixed || RegClass == IR::RegClass::FPR, "Unexpected Class: {}", Reg.Class);
|
||||
|
||||
if (RegClass == IR::RegClass::FPRFixed) {
|
||||
if (Reg.Class == IR::FPRFixedClass.Val) {
|
||||
return StaticFPRegisters[Reg.Reg];
|
||||
} else if (RegClass == IR::RegClass::FPR) {
|
||||
} else if (Reg.Class == IR::FPRClass.Val) {
|
||||
return GeneralFPRegisters[Reg.Reg];
|
||||
}
|
||||
|
||||
@@ -160,8 +113,8 @@ private:
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
static IR::RegClass GetRegClass(IR::Ref Node) {
|
||||
return IR::PhysicalRegister(Node).AsRegClass();
|
||||
FEXCore::IR::RegisterClassType GetRegClass(IR::Ref Node) const {
|
||||
return FEXCore::IR::RegisterClassType {IR::PhysicalRegister(Node).Class};
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
@@ -178,7 +131,7 @@ private:
|
||||
// Converts IR-base shift type to ARMEmitter shift type.
|
||||
// Will be a no-op, only a type conversion since the two definitions match.
|
||||
[[nodiscard]]
|
||||
static ARMEmitter::ShiftType ConvertIRShiftType(IR::ShiftType Shift) {
|
||||
ARMEmitter::ShiftType ConvertIRShiftType(IR::ShiftType Shift) const {
|
||||
return Shift == IR::ShiftType::LSL ? ARMEmitter::ShiftType::LSL :
|
||||
Shift == IR::ShiftType::LSR ? ARMEmitter::ShiftType::LSR :
|
||||
Shift == IR::ShiftType::ASR ? ARMEmitter::ShiftType::ASR :
|
||||
@@ -186,23 +139,18 @@ private:
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
static ARMEmitter::Size ConvertSize(const IR::IROp_Header* Op) {
|
||||
ARMEmitter::Size ConvertSize(const IR::IROp_Header* Op) {
|
||||
return Op->Size == IR::OpSize::i64Bit ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit;
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
static ARMEmitter::Size ConvertSize48(const IR::IROp_Header* Op) {
|
||||
ARMEmitter::Size ConvertSize48(const IR::IROp_Header* Op) {
|
||||
LOGMAN_THROW_A_FMT(Op->Size == IR::OpSize::i32Bit || Op->Size == IR::OpSize::i64Bit, "Invalid size");
|
||||
return ConvertSize(Op);
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
static ARMEmitter::Size ConvertSize(IR::OpSize Size) {
|
||||
return Size == IR::OpSize::i64Bit ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit;
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
static ARMEmitter::SubRegSize ConvertSubRegSize16(IR::OpSize ElementSize) {
|
||||
ARMEmitter::SubRegSize ConvertSubRegSize16(IR::OpSize ElementSize) {
|
||||
LOGMAN_THROW_A_FMT(ElementSize == IR::OpSize::i8Bit || ElementSize == IR::OpSize::i16Bit || ElementSize == IR::OpSize::i32Bit ||
|
||||
ElementSize == IR::OpSize::i64Bit || ElementSize == IR::OpSize::i128Bit,
|
||||
"Invalid size");
|
||||
@@ -214,105 +162,105 @@ private:
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
static ARMEmitter::SubRegSize ConvertSubRegSize16(const IR::IROp_Header* Op) {
|
||||
ARMEmitter::SubRegSize ConvertSubRegSize16(const IR::IROp_Header* Op) {
|
||||
return ConvertSubRegSize16(Op->ElementSize);
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
static ARMEmitter::SubRegSize ConvertSubRegSize8(IR::OpSize ElementSize) {
|
||||
ARMEmitter::SubRegSize ConvertSubRegSize8(IR::OpSize ElementSize) {
|
||||
LOGMAN_THROW_A_FMT(ElementSize != IR::OpSize::i128Bit, "Invalid size");
|
||||
return ConvertSubRegSize16(ElementSize);
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
static ARMEmitter::SubRegSize ConvertSubRegSize8(const IR::IROp_Header* Op) {
|
||||
ARMEmitter::SubRegSize ConvertSubRegSize8(const IR::IROp_Header* Op) {
|
||||
return ConvertSubRegSize8(Op->ElementSize);
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
static ARMEmitter::SubRegSize ConvertSubRegSize4(const IR::IROp_Header* Op) {
|
||||
ARMEmitter::SubRegSize ConvertSubRegSize4(const IR::IROp_Header* Op) {
|
||||
LOGMAN_THROW_A_FMT(Op->ElementSize != IR::OpSize::i64Bit, "Invalid size");
|
||||
return ConvertSubRegSize8(Op);
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
static ARMEmitter::SubRegSize ConvertSubRegSize248(const IR::IROp_Header* Op) {
|
||||
ARMEmitter::SubRegSize ConvertSubRegSize248(const IR::IROp_Header* Op) {
|
||||
LOGMAN_THROW_A_FMT(Op->ElementSize != IR::OpSize::i8Bit, "Invalid size");
|
||||
return ConvertSubRegSize8(Op);
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
static ARMEmitter::VectorRegSizePair ConvertSubRegSizePair16(const IR::IROp_Header* Op) {
|
||||
ARMEmitter::VectorRegSizePair ConvertSubRegSizePair16(const IR::IROp_Header* Op) {
|
||||
return ARMEmitter::ToVectorSizePair(ConvertSubRegSize16(Op));
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
static ARMEmitter::VectorRegSizePair ConvertSubRegSizePair8(const IR::IROp_Header* Op) {
|
||||
ARMEmitter::VectorRegSizePair ConvertSubRegSizePair8(const IR::IROp_Header* Op) {
|
||||
LOGMAN_THROW_A_FMT(Op->ElementSize != IR::OpSize::i128Bit, "Invalid size");
|
||||
return ConvertSubRegSizePair16(Op);
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
static ARMEmitter::VectorRegSizePair ConvertSubRegSizePair248(const IR::IROp_Header* Op) {
|
||||
ARMEmitter::VectorRegSizePair ConvertSubRegSizePair248(const IR::IROp_Header* Op) {
|
||||
LOGMAN_THROW_A_FMT(Op->ElementSize != IR::OpSize::i8Bit, "Invalid size");
|
||||
return ConvertSubRegSizePair8(Op);
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
static ARMEmitter::Condition MapCC(IR::CondClass Cond) {
|
||||
switch (Cond) {
|
||||
case IR::CondClass::EQ: return ARMEmitter::Condition::CC_EQ;
|
||||
case IR::CondClass::NEQ: return ARMEmitter::Condition::CC_NE;
|
||||
case IR::CondClass::SGE: return ARMEmitter::Condition::CC_GE;
|
||||
case IR::CondClass::SLT: return ARMEmitter::Condition::CC_LT;
|
||||
case IR::CondClass::SGT: return ARMEmitter::Condition::CC_GT;
|
||||
case IR::CondClass::SLE: return ARMEmitter::Condition::CC_LE;
|
||||
case IR::CondClass::UGE: return ARMEmitter::Condition::CC_CS;
|
||||
case IR::CondClass::ULT: return ARMEmitter::Condition::CC_CC;
|
||||
case IR::CondClass::UGT: return ARMEmitter::Condition::CC_HI;
|
||||
case IR::CondClass::ULE: return ARMEmitter::Condition::CC_LS;
|
||||
case IR::CondClass::FLU: return ARMEmitter::Condition::CC_LT;
|
||||
case IR::CondClass::FGE: return ARMEmitter::Condition::CC_GE;
|
||||
case IR::CondClass::FLEU: return ARMEmitter::Condition::CC_LE;
|
||||
case IR::CondClass::FGT: return ARMEmitter::Condition::CC_GT;
|
||||
case IR::CondClass::FU:
|
||||
case IR::CondClass::VS: return ARMEmitter::Condition::CC_VS;
|
||||
case IR::CondClass::FNU:
|
||||
case IR::CondClass::VC: return ARMEmitter::Condition::CC_VC;
|
||||
case IR::CondClass::MI: return ARMEmitter::Condition::CC_MI;
|
||||
case IR::CondClass::PL: return ARMEmitter::Condition::CC_PL;
|
||||
ARMEmitter::Condition MapCC(IR::CondClassType Cond) {
|
||||
switch (Cond.Val) {
|
||||
case FEXCore::IR::COND_EQ: return ARMEmitter::Condition::CC_EQ;
|
||||
case FEXCore::IR::COND_NEQ: return ARMEmitter::Condition::CC_NE;
|
||||
case FEXCore::IR::COND_SGE: return ARMEmitter::Condition::CC_GE;
|
||||
case FEXCore::IR::COND_SLT: return ARMEmitter::Condition::CC_LT;
|
||||
case FEXCore::IR::COND_SGT: return ARMEmitter::Condition::CC_GT;
|
||||
case FEXCore::IR::COND_SLE: return ARMEmitter::Condition::CC_LE;
|
||||
case FEXCore::IR::COND_UGE: return ARMEmitter::Condition::CC_CS;
|
||||
case FEXCore::IR::COND_ULT: return ARMEmitter::Condition::CC_CC;
|
||||
case FEXCore::IR::COND_UGT: return ARMEmitter::Condition::CC_HI;
|
||||
case FEXCore::IR::COND_ULE: return ARMEmitter::Condition::CC_LS;
|
||||
case FEXCore::IR::COND_FLU: return ARMEmitter::Condition::CC_LT;
|
||||
case FEXCore::IR::COND_FGE: return ARMEmitter::Condition::CC_GE;
|
||||
case FEXCore::IR::COND_FLEU: return ARMEmitter::Condition::CC_LE;
|
||||
case FEXCore::IR::COND_FGT: return ARMEmitter::Condition::CC_GT;
|
||||
case FEXCore::IR::COND_FU: return ARMEmitter::Condition::CC_VS;
|
||||
case FEXCore::IR::COND_FNU: return ARMEmitter::Condition::CC_VC;
|
||||
case FEXCore::IR::COND_VS:
|
||||
case FEXCore::IR::COND_VC:
|
||||
case FEXCore::IR::COND_MI: return ARMEmitter::Condition::CC_MI;
|
||||
case FEXCore::IR::COND_PL: return ARMEmitter::Condition::CC_PL;
|
||||
default: LOGMAN_MSG_A_FMT("Unsupported compare type"); return ARMEmitter::Condition::CC_NV;
|
||||
}
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
static bool IsFPR(IR::RegClass Class) {
|
||||
return Class == IR::RegClass::FPR || Class == IR::RegClass::FPRFixed;
|
||||
bool IsFPR(IR::RegisterClassType Class) const {
|
||||
return Class == IR::FPRClass || Class == IR::FPRFixedClass;
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
static bool IsGPR(IR::RegClass Class) {
|
||||
return Class == IR::RegClass::GPR || Class == IR::RegClass::GPRFixed;
|
||||
bool IsGPR(IR::RegisterClassType Class) const {
|
||||
return Class == IR::GPRClass || Class == IR::GPRFixedClass;
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
static bool IsGPR(IR::Ref Node) {
|
||||
bool IsGPR(IR::Ref Node) {
|
||||
return IsGPR(GetRegClass(Node));
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
static bool IsFPR(IR::Ref Node) {
|
||||
bool IsFPR(IR::Ref Node) {
|
||||
return IsFPR(GetRegClass(Node));
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
static bool IsGPR(IR::OrderedNodeWrapper Wrap) {
|
||||
return IsGPR(IR::PhysicalRegister(Wrap).AsRegClass());
|
||||
bool IsGPR(IR::OrderedNodeWrapper Wrap) {
|
||||
return IsGPR(IR::RegisterClassType {IR::PhysicalRegister(Wrap).Class});
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
static bool IsFPR(IR::OrderedNodeWrapper Wrap) {
|
||||
return IsFPR(IR::PhysicalRegister(Wrap).AsRegClass());
|
||||
bool IsFPR(IR::OrderedNodeWrapper Wrap) {
|
||||
return IsFPR(IR::RegisterClassType {IR::PhysicalRegister(Wrap).Class});
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
@@ -342,190 +290,6 @@ private:
|
||||
uint32_t End;
|
||||
};
|
||||
|
||||
void EmitLinkedBranch(uint64_t GuestRIP, bool Call) {
|
||||
PendingJumpThunks.push_back({GetCursorAddress<uint64_t>(), GuestRIP, {}});
|
||||
auto& Thunk = PendingJumpThunks.back();
|
||||
BindOrRestart(&Thunk.Label);
|
||||
if (Call) {
|
||||
bl_OrRestart(&Thunk.Label);
|
||||
} else {
|
||||
b_OrRestart(&Thunk.Label);
|
||||
}
|
||||
}
|
||||
|
||||
// Restart helpers
|
||||
template<ARMEmitter::IsLabel T>
|
||||
void bl_OrRestart(T* Label) {
|
||||
if (bl(Label) == ARMEmitter::BranchEncodeSucceeded::Success) {
|
||||
return;
|
||||
}
|
||||
|
||||
// We can support this but currently unnecessary.
|
||||
ERROR_AND_DIE_FMT("Tried to branch larger than 128MB away!");
|
||||
FEXCore::UncheckedLongJump::LongJump(ThreadState->RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));
|
||||
}
|
||||
|
||||
template<ARMEmitter::IsLabel T>
|
||||
void b_OrRestart(T* Label) {
|
||||
if (b(Label) == ARMEmitter::BranchEncodeSucceeded::Success) {
|
||||
return;
|
||||
}
|
||||
|
||||
// We can support this but currently unnecessary.
|
||||
ERROR_AND_DIE_FMT("Tried to branch larger than 128MB away!");
|
||||
FEXCore::UncheckedLongJump::LongJump(ThreadState->RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));
|
||||
}
|
||||
|
||||
template<ARMEmitter::IsLabel T>
|
||||
void b_OrRestart(ARMEmitter::Condition Cond, T* Label) {
|
||||
if (RequiresFarARM64Jumps) {
|
||||
ARMEmitter::ForwardLabel Skip {};
|
||||
// Wrap a manual Cond check around an unconditional branch; this can encode larger offsets
|
||||
(void)b(InvertCondition(Cond), &Skip);
|
||||
if (b(Label) == ARMEmitter::BranchEncodeSucceeded::Failure) {
|
||||
ERROR_AND_DIE_FMT("Tried to branch larger than 128MB away!");
|
||||
}
|
||||
|
||||
(void)Bind(&Skip);
|
||||
return;
|
||||
}
|
||||
|
||||
if (b(Cond, Label) == ARMEmitter::BranchEncodeSucceeded::Success) {
|
||||
return;
|
||||
}
|
||||
|
||||
FEXCore::UncheckedLongJump::LongJump(ThreadState->RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));
|
||||
}
|
||||
|
||||
template<ARMEmitter::IsLabel T>
|
||||
void cbz_OrRestart(ARMEmitter::Size s, ARMEmitter::Register rt, T* Label) {
|
||||
if (RequiresFarARM64Jumps) {
|
||||
ARMEmitter::ForwardLabel Skip {};
|
||||
// Wrap a manual Cond check around an unconditional branch; this can encode larger offsets
|
||||
(void)cbnz(s, rt, &Skip);
|
||||
if (b(Label) == ARMEmitter::BranchEncodeSucceeded::Failure) {
|
||||
ERROR_AND_DIE_FMT("Tried to branch larger than 128MB away!");
|
||||
}
|
||||
|
||||
(void)Bind(&Skip);
|
||||
return;
|
||||
}
|
||||
|
||||
if (cbz(s, rt, Label) == ARMEmitter::BranchEncodeSucceeded::Success) {
|
||||
return;
|
||||
}
|
||||
|
||||
FEXCore::UncheckedLongJump::LongJump(ThreadState->RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));
|
||||
}
|
||||
|
||||
template<ARMEmitter::IsLabel T>
|
||||
void cbnz_OrRestart(ARMEmitter::Size s, ARMEmitter::Register rt, T* Label) {
|
||||
if (RequiresFarARM64Jumps) {
|
||||
ARMEmitter::ForwardLabel Skip {};
|
||||
// Wrap a manual Cond check around an unconditional branch; this can encode larger offsets
|
||||
(void)cbz(s, rt, &Skip);
|
||||
if (b(Label) == ARMEmitter::BranchEncodeSucceeded::Failure) {
|
||||
ERROR_AND_DIE_FMT("Tried to branch larger than 128MB away!");
|
||||
}
|
||||
|
||||
(void)Bind(&Skip);
|
||||
return;
|
||||
}
|
||||
|
||||
if (cbnz(s, rt, Label) == ARMEmitter::BranchEncodeSucceeded::Success) {
|
||||
return;
|
||||
}
|
||||
|
||||
FEXCore::UncheckedLongJump::LongJump(ThreadState->RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));
|
||||
}
|
||||
|
||||
template<ARMEmitter::IsLabel T>
|
||||
void tbz_OrRestart(ARMEmitter::Register rt, uint32_t Bit, T* Label) {
|
||||
if (RequiresFarARM64Jumps) {
|
||||
ARMEmitter::ForwardLabel Skip {};
|
||||
// Wrap a manual Cond check around an unconditional branch; this can encode larger offsets
|
||||
(void)tbnz(rt, Bit, &Skip);
|
||||
if (b(Label) == ARMEmitter::BranchEncodeSucceeded::Failure) {
|
||||
ERROR_AND_DIE_FMT("Tried to branch larger than 128MB away!");
|
||||
}
|
||||
|
||||
(void)Bind(&Skip);
|
||||
return;
|
||||
}
|
||||
|
||||
if (tbz(rt, Bit, Label) == ARMEmitter::BranchEncodeSucceeded::Success) {
|
||||
return;
|
||||
}
|
||||
|
||||
FEXCore::UncheckedLongJump::LongJump(ThreadState->RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));
|
||||
}
|
||||
|
||||
template<ARMEmitter::IsLabel T>
|
||||
void tbnz_OrRestart(ARMEmitter::Register rt, uint32_t Bit, T* Label) {
|
||||
if (RequiresFarARM64Jumps) {
|
||||
ARMEmitter::ForwardLabel Skip {};
|
||||
// Wrap a manual Cond check around an unconditional branch; this can encode larger offsets
|
||||
(void)tbz(rt, Bit, &Skip);
|
||||
if (b(Label) == ARMEmitter::BranchEncodeSucceeded::Failure) {
|
||||
ERROR_AND_DIE_FMT("Tried to branch larger than 128MB away!");
|
||||
}
|
||||
|
||||
(void)Bind(&Skip);
|
||||
return;
|
||||
}
|
||||
|
||||
if (tbnz(rt, Bit, Label) == ARMEmitter::BranchEncodeSucceeded::Success) {
|
||||
return;
|
||||
}
|
||||
|
||||
FEXCore::UncheckedLongJump::LongJump(ThreadState->RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));
|
||||
}
|
||||
|
||||
template<ARMEmitter::IsLabel T>
|
||||
void adr_OrRestart(ARMEmitter::Register rd, T* Label) {
|
||||
if (RequiresFarARM64Jumps) {
|
||||
if (LongAddressGen(rd, Label) == ARMEmitter::BranchEncodeSucceeded::Failure) {
|
||||
ERROR_AND_DIE_FMT("Unable to encode long ADR.");
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (adr(rd, Label) == ARMEmitter::BranchEncodeSucceeded::Success) {
|
||||
return;
|
||||
}
|
||||
|
||||
FEXCore::UncheckedLongJump::LongJump(ThreadState->RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));
|
||||
}
|
||||
|
||||
template<ARMEmitter::IsLabel T>
|
||||
void adrp_OrRestart(ARMEmitter::Register rd, T* Label) {
|
||||
if (RequiresFarARM64Jumps) {
|
||||
if (LongAddressGen(rd, Label) == ARMEmitter::BranchEncodeSucceeded::Failure) {
|
||||
ERROR_AND_DIE_FMT("Unable to encode long ADRP.");
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (adrp(rd, Label) == ARMEmitter::BranchEncodeSucceeded::Success) {
|
||||
return;
|
||||
}
|
||||
|
||||
FEXCore::UncheckedLongJump::LongJump(ThreadState->RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));
|
||||
}
|
||||
|
||||
template<ARMEmitter::IsLabel T>
|
||||
void BindOrRestart(T* Label) {
|
||||
if (Bind(Label)) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (RequiresFarARM64Jumps) {
|
||||
// This should have been caught before this point.
|
||||
ERROR_AND_DIE_FMT("Unhandled long bind");
|
||||
return;
|
||||
}
|
||||
|
||||
FEXCore::UncheckedLongJump::LongJump(ThreadState->RestartJump, FEXCore::ToUnderlying(RestartOptions::Control::EnableFarARM64Jumps));
|
||||
}
|
||||
|
||||
// This is purely a debugging aid for developers to see if they are in JIT code space when inspecting raw memory
|
||||
void EmitDetectionString();
|
||||
IR::RegisterAllocationPass* RAPass {};
|
||||
@@ -536,6 +300,8 @@ private:
|
||||
* @name Relocations
|
||||
* @{ */
|
||||
|
||||
uint64_t GetNamedSymbolLiteral(FEXCore::CPU::RelocNamedSymbolLiteral::NamedSymbol Op);
|
||||
|
||||
/**
|
||||
* @brief A literal pair relocation object for named symbol literals
|
||||
*/
|
||||
@@ -572,30 +338,17 @@ private:
|
||||
*/
|
||||
NamedSymbolLiteralPair InsertNamedSymbolLiteral(FEXCore::CPU::RelocNamedSymbolLiteral::NamedSymbol Op);
|
||||
|
||||
/**
|
||||
* @brief Inserts a relocation for a constant value relative to the guest entrypoint
|
||||
*
|
||||
* @param Reg - The GPR to move the guest RIP in to
|
||||
* @param Constant - The guest RIP that will be relocated
|
||||
*/
|
||||
NamedSymbolLiteralPair InsertGuestRIPLiteral(uint64_t GuestRIP);
|
||||
|
||||
/**
|
||||
* @brief Place the named symbol literal relocation in memory
|
||||
*
|
||||
* @param Lit - Which literal to place
|
||||
*/
|
||||
void PlaceNamedSymbolLiteral(NamedSymbolLiteralPair Lit);
|
||||
void PlaceNamedSymbolLiteral(NamedSymbolLiteralPair& Lit);
|
||||
|
||||
fextl::vector<FEXCore::CPU::Relocation> Relocations;
|
||||
|
||||
/**
|
||||
* Returns any relocations generated since the last call to TakeRelocations.
|
||||
*
|
||||
* GuestBaseAddress must match the base virtual address to which the
|
||||
* input x86 binary is mapped.
|
||||
*/
|
||||
fextl::vector<FEXCore::CPU::Relocation> TakeRelocations(uint64_t GuestBaseAddress) override;
|
||||
///< Relocation code loading
|
||||
bool ApplyRelocations(uint64_t GuestEntry, uint64_t CodeEntry, uint64_t CursorEntry, size_t NumRelocations, const char* EntryRelocations);
|
||||
|
||||
/** @} */
|
||||
|
||||
@@ -618,16 +371,21 @@ private:
|
||||
void Emulate128BitGather(IR::OpSize Size, IR::OpSize ElementSize, ARMEmitter::VRegister Dst, ARMEmitter::VRegister IncomingDst,
|
||||
std::optional<ARMEmitter::Register> BaseAddr, ARMEmitter::VRegister VectorIndexLow,
|
||||
std::optional<ARMEmitter::VRegister> VectorIndexHigh, ARMEmitter::VRegister MaskReg, IR::OpSize VectorIndexSize,
|
||||
size_t DataElementOffsetStart, size_t IndexElementOffsetStart, uint8_t OffsetScale, IR::OpSize AddrSize);
|
||||
size_t DataElementOffsetStart, size_t IndexElementOffsetStart, uint8_t OffsetScale);
|
||||
|
||||
void EmitTFCheck();
|
||||
void EmitInterruptChecks(bool CheckTF);
|
||||
|
||||
void EmitSuspendInterruptCheck();
|
||||
|
||||
void EmitEntryPoint(ARMEmitter::BackwardLabel& HeaderLabel, bool CheckTF);
|
||||
// Runtime selection;
|
||||
// Load and store TSO memory style
|
||||
OpType RT_LoadMemTSO;
|
||||
OpType RT_StoreMemTSO;
|
||||
|
||||
#define DEF_OP(x) void Op_##x(IR::IROp_Header const* IROp, IR::Ref Node)
|
||||
|
||||
// Dynamic Dispatcher supporting operations
|
||||
DEF_OP(ParanoidLoadMemTSO);
|
||||
DEF_OP(ParanoidStoreMemTSO);
|
||||
|
||||
///< Unhandled handler
|
||||
DEF_OP(Unhandled);
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ DEF_OP(LoadContext) {
|
||||
const auto Op = IROp->C<IR::IROp_LoadContext>();
|
||||
const auto OpSize = IROp->Size;
|
||||
|
||||
if (Op->Class == IR::RegClass::GPR) {
|
||||
if (Op->Class == FEXCore::IR::GPRClass) {
|
||||
auto Dst = GetReg(Node);
|
||||
|
||||
switch (OpSize) {
|
||||
@@ -52,7 +52,7 @@ DEF_OP(LoadContext) {
|
||||
DEF_OP(LoadContextPair) {
|
||||
const auto Op = IROp->C<IR::IROp_LoadContextPair>();
|
||||
|
||||
if (Op->Class == IR::RegClass::GPR) {
|
||||
if (Op->Class == FEXCore::IR::GPRClass) {
|
||||
const auto Dst1 = GetReg(Op->OutValue1);
|
||||
const auto Dst2 = GetReg(Op->OutValue2);
|
||||
|
||||
@@ -78,7 +78,7 @@ DEF_OP(StoreContext) {
|
||||
const auto Op = IROp->C<IR::IROp_StoreContext>();
|
||||
const auto OpSize = IROp->Size;
|
||||
|
||||
if (Op->Class == IR::RegClass::GPR) {
|
||||
if (Op->Class == FEXCore::IR::GPRClass) {
|
||||
auto Src = GetZeroableReg(Op->Value);
|
||||
|
||||
switch (OpSize) {
|
||||
@@ -110,7 +110,7 @@ DEF_OP(StoreContextPair) {
|
||||
const auto Op = IROp->C<IR::IROp_StoreContextPair>();
|
||||
const auto OpSize = IROp->Size;
|
||||
|
||||
if (Op->Class == IR::RegClass::GPR) {
|
||||
if (Op->Class == FEXCore::IR::GPRClass) {
|
||||
auto Src1 = GetZeroableReg(Op->Value1);
|
||||
auto Src2 = GetZeroableReg(Op->Value2);
|
||||
|
||||
@@ -135,12 +135,12 @@ DEF_OP(StoreContextPair) {
|
||||
DEF_OP(LoadRegister) {
|
||||
const auto Op = IROp->C<IR::IROp_LoadRegister>();
|
||||
|
||||
if (Op->Class == IR::RegClass::GPR) {
|
||||
if (Op->Class == IR::GPRClass) {
|
||||
LOGMAN_THROW_A_FMT(Op->Reg < StaticRegisters.size(), "out of range reg");
|
||||
|
||||
mov(GetReg(Node).X(), StaticRegisters[Op->Reg].X());
|
||||
} else if (Op->Class == IR::RegClass::FPR) {
|
||||
const auto regSize = HostSupportsAVX256 ? IR::OpSize::i256Bit : IR::OpSize::i128Bit;
|
||||
} else if (Op->Class == IR::FPRClass) {
|
||||
[[maybe_unused]] const auto regSize = HostSupportsAVX256 ? IR::OpSize::i256Bit : IR::OpSize::i128Bit;
|
||||
LOGMAN_THROW_A_FMT(Op->Reg < StaticFPRegisters.size(), "out of range reg");
|
||||
LOGMAN_THROW_A_FMT(IROp->Size == regSize, "expected sized");
|
||||
|
||||
@@ -175,14 +175,13 @@ DEF_OP(LoadAF) {
|
||||
|
||||
DEF_OP(StoreRegister) {
|
||||
const auto Op = IROp->C<IR::IROp_StoreRegister>();
|
||||
const auto Reg = IR::PhysicalRegister(Node);
|
||||
const auto RegClass = Reg.AsRegClass();
|
||||
auto Reg = IR::PhysicalRegister(Node);
|
||||
|
||||
if (RegClass == IR::RegClass::GPRFixed) {
|
||||
if (Reg.Class == IR::GPRFixedClass) {
|
||||
// Always use 64-bit, it's faster. Upper bits ignored for 32-bit mode.
|
||||
mov(ARMEmitter::Size::i64Bit, GetReg(Reg), GetReg(Op->Value));
|
||||
} else if (RegClass == IR::RegClass::FPRFixed) {
|
||||
const auto regSize = HostSupportsAVX256 ? IR::OpSize::i256Bit : IR::OpSize::i128Bit;
|
||||
} else if (Reg.Class == IR::FPRFixedClass) {
|
||||
[[maybe_unused]] const auto regSize = HostSupportsAVX256 ? IR::OpSize::i256Bit : IR::OpSize::i128Bit;
|
||||
LOGMAN_THROW_A_FMT(IROp->Size == regSize, "expected sized");
|
||||
|
||||
const auto guest = GetVReg(Reg);
|
||||
@@ -194,7 +193,7 @@ DEF_OP(StoreRegister) {
|
||||
mov(guest.Q(), host.Q());
|
||||
}
|
||||
} else {
|
||||
LOGMAN_THROW_A_FMT(false, "Unhandled Op->Class {}", RegClass);
|
||||
LOGMAN_THROW_A_FMT(false, "Unhandled Op->Class {}", Reg.Class);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -226,7 +225,7 @@ DEF_OP(LoadContextIndexed) {
|
||||
|
||||
const auto Index = GetReg(Op->Index);
|
||||
|
||||
if (Op->Class == IR::RegClass::GPR) {
|
||||
if (Op->Class == FEXCore::IR::GPRClass) {
|
||||
switch (Op->Stride) {
|
||||
case 1:
|
||||
case 2:
|
||||
@@ -289,7 +288,7 @@ DEF_OP(StoreContextIndexed) {
|
||||
|
||||
const auto Index = GetReg(Op->Index);
|
||||
|
||||
if (Op->Class == IR::RegClass::GPR) {
|
||||
if (Op->Class == FEXCore::IR::GPRClass) {
|
||||
const auto Value = GetReg(Op->Value);
|
||||
|
||||
switch (Op->Stride) {
|
||||
@@ -349,31 +348,12 @@ DEF_OP(StoreContextIndexed) {
|
||||
}
|
||||
}
|
||||
|
||||
DEF_OP(FormContextAddress) {
|
||||
const auto Op = IROp->C<IR::IROp_FormContextAddress>();
|
||||
const auto Index = GetReg(Op->Index);
|
||||
const auto Dst = GetReg(Node);
|
||||
|
||||
switch (Op->Stride) {
|
||||
case 1:
|
||||
case 2:
|
||||
case 4:
|
||||
case 8:
|
||||
case 16:
|
||||
case 32: {
|
||||
add(ARMEmitter::Size::i64Bit, Dst, STATE, Index, ARMEmitter::ShiftType::LSL, FEXCore::ilog2(Op->Stride));
|
||||
break;
|
||||
}
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled FormContextAddress stride: {}", Op->Stride); break;
|
||||
}
|
||||
}
|
||||
|
||||
DEF_OP(SpillRegister) {
|
||||
const auto Op = IROp->C<IR::IROp_SpillRegister>();
|
||||
const auto OpSize = IROp->Size;
|
||||
const uint32_t SlotOffset = Op->Slot * MaxSpillSlotSize;
|
||||
|
||||
if (Op->Class == IR::RegClass::GPR) {
|
||||
if (Op->Class == FEXCore::IR::GPRClass) {
|
||||
const auto Src = GetReg(Op->Value);
|
||||
switch (OpSize) {
|
||||
case IR::OpSize::i8Bit: {
|
||||
@@ -414,7 +394,7 @@ DEF_OP(SpillRegister) {
|
||||
}
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled SpillRegister size: {}", OpSize); break;
|
||||
}
|
||||
} else if (Op->Class == FEXCore::IR::RegClass::FPR) {
|
||||
} else if (Op->Class == FEXCore::IR::FPRClass) {
|
||||
const auto Src = GetVReg(Op->Value);
|
||||
|
||||
switch (OpSize) {
|
||||
@@ -453,7 +433,7 @@ DEF_OP(SpillRegister) {
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled SpillRegister size: {}", OpSize); break;
|
||||
}
|
||||
} else {
|
||||
LOGMAN_MSG_A_FMT("Unhandled SpillRegister class: {}", Op->Class);
|
||||
LOGMAN_MSG_A_FMT("Unhandled SpillRegister class: {}", Op->Class.Val);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -462,7 +442,7 @@ DEF_OP(FillRegister) {
|
||||
const auto OpSize = IROp->Size;
|
||||
const uint32_t SlotOffset = Op->Slot * MaxSpillSlotSize;
|
||||
|
||||
if (Op->Class == IR::RegClass::GPR) {
|
||||
if (Op->Class == FEXCore::IR::GPRClass) {
|
||||
const auto Dst = GetReg(Node);
|
||||
switch (OpSize) {
|
||||
case IR::OpSize::i8Bit: {
|
||||
@@ -503,7 +483,7 @@ DEF_OP(FillRegister) {
|
||||
}
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled FillRegister size: {}", OpSize); break;
|
||||
}
|
||||
} else if (Op->Class == FEXCore::IR::RegClass::FPR) {
|
||||
} else if (Op->Class == FEXCore::IR::FPRClass) {
|
||||
const auto Dst = GetVReg(Node);
|
||||
|
||||
switch (OpSize) {
|
||||
@@ -542,7 +522,7 @@ DEF_OP(FillRegister) {
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled FillRegister size: {}", OpSize); break;
|
||||
}
|
||||
} else {
|
||||
LOGMAN_MSG_A_FMT("Unhandled FillRegister class: {}", Op->Class);
|
||||
LOGMAN_MSG_A_FMT("Unhandled FillRegister class: {}", Op->Class.Val);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -579,14 +559,14 @@ ARMEmitter::ExtendedMemOperand Arm64JITCore::GenerateMemOperand(
|
||||
return ARMEmitter::ExtendedMemOperand(Base.X(), ARMEmitter::IndexType::OFFSET, Const);
|
||||
} else {
|
||||
auto RegOffset = GetReg(Offset);
|
||||
switch (OffsetType) {
|
||||
case IR::MemOffsetType::SXTX:
|
||||
switch (OffsetType.Val) {
|
||||
case IR::MEM_OFFSET_SXTX.Val:
|
||||
return ARMEmitter::ExtendedMemOperand(Base.X(), RegOffset.X(), ARMEmitter::ExtendedType::SXTX, FEXCore::ilog2(OffsetScale));
|
||||
case IR::MemOffsetType::UXTW:
|
||||
case IR::MEM_OFFSET_UXTW.Val:
|
||||
return ARMEmitter::ExtendedMemOperand(Base.X(), RegOffset.X(), ARMEmitter::ExtendedType::UXTW, FEXCore::ilog2(OffsetScale));
|
||||
case IR::MemOffsetType::SXTW:
|
||||
case IR::MEM_OFFSET_SXTW.Val:
|
||||
return ARMEmitter::ExtendedMemOperand(Base.X(), RegOffset.X(), ARMEmitter::ExtendedType::SXTW, FEXCore::ilog2(OffsetScale));
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled GenerateMemOperand OffsetType: {}", OffsetType); break;
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled GenerateMemOperand OffsetType: {}", OffsetType.Val); break;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -613,20 +593,20 @@ ARMEmitter::Register Arm64JITCore::ApplyMemOperand(IR::OpSize AccessSize, ARMEmi
|
||||
add(ARMEmitter::Size::i64Bit, Tmp, Base, Tmp, ARMEmitter::ShiftType::LSL, FEXCore::ilog2(OffsetScale));
|
||||
} else {
|
||||
auto RegOffset = GetReg(Offset);
|
||||
switch (OffsetType) {
|
||||
case IR::MemOffsetType::SXTX:
|
||||
switch (OffsetType.Val) {
|
||||
case IR::MEM_OFFSET_SXTX.Val:
|
||||
add(ARMEmitter::Size::i64Bit, Tmp, Base, RegOffset, ARMEmitter::ExtendedType::SXTX, FEXCore::ilog2(OffsetScale));
|
||||
break;
|
||||
|
||||
case IR::MemOffsetType::UXTW:
|
||||
case IR::MEM_OFFSET_UXTW.Val:
|
||||
add(ARMEmitter::Size::i64Bit, Tmp, Base, RegOffset, ARMEmitter::ExtendedType::UXTW, FEXCore::ilog2(OffsetScale));
|
||||
break;
|
||||
|
||||
case IR::MemOffsetType::SXTW:
|
||||
case IR::MEM_OFFSET_SXTW.Val:
|
||||
add(ARMEmitter::Size::i64Bit, Tmp, Base, RegOffset, ARMEmitter::ExtendedType::SXTW, FEXCore::ilog2(OffsetScale));
|
||||
break;
|
||||
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled OffsetType: {}", OffsetType); break;
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled OffsetType: {}", OffsetType.Val); break;
|
||||
}
|
||||
}
|
||||
return Tmp;
|
||||
@@ -677,7 +657,7 @@ ARMEmitter::SVEMemOperand Arm64JITCore::GenerateSVEMemOperand(IR::OpSize AccessS
|
||||
// Note that we do nothing with the offset type and offset scale,
|
||||
// since SVE loads and stores don't have the ability to perform an
|
||||
// optional extension or shift as part of their behavior.
|
||||
LOGMAN_THROW_A_FMT(OffsetType == IR::MemOffsetType::SXTX, "Currently only the default offset type (SXTX) is supported.");
|
||||
LOGMAN_THROW_A_FMT(OffsetType.Val == IR::MEM_OFFSET_SXTX.Val, "Currently only the default offset type (SXTX) is supported.");
|
||||
|
||||
const auto RegOffset = GetReg(Offset);
|
||||
return ARMEmitter::SVEMemOperand(Base.X(), RegOffset.X());
|
||||
@@ -690,7 +670,7 @@ DEF_OP(LoadMem) {
|
||||
const auto MemReg = GetReg(Op->Addr);
|
||||
const auto MemSrc = GenerateMemOperand(OpSize, MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale);
|
||||
|
||||
if (Op->Class == IR::RegClass::GPR) {
|
||||
if (Op->Class == FEXCore::IR::GPRClass) {
|
||||
const auto Dst = GetReg(Node);
|
||||
|
||||
switch (OpSize) {
|
||||
@@ -724,7 +704,7 @@ DEF_OP(LoadMemPair) {
|
||||
const auto Op = IROp->C<IR::IROp_LoadMemPair>();
|
||||
const auto Addr = GetReg(Op->Addr);
|
||||
|
||||
if (Op->Class == IR::RegClass::GPR) {
|
||||
if (Op->Class == FEXCore::IR::GPRClass) {
|
||||
const auto Dst1 = GetReg(Op->OutValue1);
|
||||
const auto Dst2 = GetReg(Op->OutValue2);
|
||||
|
||||
@@ -752,17 +732,17 @@ DEF_OP(LoadMemTSO) {
|
||||
|
||||
const auto MemReg = GetReg(Op->Addr);
|
||||
|
||||
if (Op->Class == IR::RegClass::GPR) {
|
||||
if (Op->Class == FEXCore::IR::GPRClass) {
|
||||
LOGMAN_THROW_A_FMT(Op->Offset.IsInvalid() || CTX->HostFeatures.SupportsTSOImm9, "unexpected offset");
|
||||
LOGMAN_THROW_A_FMT(Op->OffsetScale == 1, "unexpected offset scale");
|
||||
LOGMAN_THROW_A_FMT(Op->OffsetType == IR::MemOffsetType::SXTX, "unexpected offset type");
|
||||
LOGMAN_THROW_A_FMT(Op->OffsetType == IR::MEM_OFFSET_SXTX, "unexpected offset type");
|
||||
}
|
||||
|
||||
if (CTX->HostFeatures.SupportsTSOImm9 && Op->Class == IR::RegClass::GPR) {
|
||||
if (CTX->HostFeatures.SupportsTSOImm9 && Op->Class == FEXCore::IR::GPRClass) {
|
||||
const auto Dst = GetReg(Node);
|
||||
uint64_t Offset = 0;
|
||||
if (!Op->Offset.IsInvalid()) {
|
||||
bool IsInline = IsInlineConstant(Op->Offset, &Offset);
|
||||
[[maybe_unused]] bool IsInline = IsInlineConstant(Op->Offset, &Offset);
|
||||
LOGMAN_THROW_A_FMT(IsInline, "expected immediate");
|
||||
}
|
||||
|
||||
@@ -780,7 +760,7 @@ DEF_OP(LoadMemTSO) {
|
||||
// Half-barrier once back-patched.
|
||||
nop();
|
||||
}
|
||||
} else if (CTX->HostFeatures.SupportsRCPC && Op->Class == IR::RegClass::GPR) {
|
||||
} else if (CTX->HostFeatures.SupportsRCPC && Op->Class == FEXCore::IR::GPRClass) {
|
||||
const auto Dst = GetReg(Node);
|
||||
if (OpSize == IR::OpSize::i8Bit) {
|
||||
// 8bit load is always aligned to natural alignment
|
||||
@@ -795,7 +775,7 @@ DEF_OP(LoadMemTSO) {
|
||||
// Half-barrier once back-patched.
|
||||
nop();
|
||||
}
|
||||
} else if (Op->Class == IR::RegClass::GPR) {
|
||||
} else if (Op->Class == FEXCore::IR::GPRClass) {
|
||||
const auto Dst = GetReg(Node);
|
||||
if (OpSize == IR::OpSize::i8Bit) {
|
||||
// 8bit load is always aligned to natural alignment
|
||||
@@ -912,7 +892,7 @@ DEF_OP(VLoadVectorMasked) {
|
||||
|
||||
// If the sign bit is zero then skip the load
|
||||
ARMEmitter::ForwardLabel Skip {};
|
||||
(void)tbz(WorkingReg, ElementSizeInBits - 1, &Skip);
|
||||
tbz(WorkingReg, ElementSizeInBits - 1, &Skip);
|
||||
// Do the gather load for this element into the destination
|
||||
switch (IROp->ElementSize) {
|
||||
case IR::OpSize::i8Bit: ld1<ARMEmitter::SubRegSize::i8Bit>(TempDst.Q(), i, TempMemReg); break;
|
||||
@@ -923,7 +903,7 @@ DEF_OP(VLoadVectorMasked) {
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, IROp->ElementSize); return;
|
||||
}
|
||||
|
||||
(void)Bind(&Skip);
|
||||
Bind(&Skip);
|
||||
|
||||
if ((i + 1) != NumElements) {
|
||||
// Handle register rename to save a move.
|
||||
@@ -1013,7 +993,7 @@ DEF_OP(VStoreVectorMasked) {
|
||||
|
||||
// If the sign bit is zero then skip the load
|
||||
ARMEmitter::ForwardLabel Skip {};
|
||||
(void)tbz(WorkingReg, ElementSizeInBits - 1, &Skip);
|
||||
tbz(WorkingReg, ElementSizeInBits - 1, &Skip);
|
||||
// Do the gather load for this element into the destination
|
||||
switch (IROp->ElementSize) {
|
||||
case IR::OpSize::i8Bit: st1<ARMEmitter::SubRegSize::i8Bit>(RegData.Q(), i, TempMemReg); break;
|
||||
@@ -1024,7 +1004,7 @@ DEF_OP(VStoreVectorMasked) {
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, IROp->ElementSize); return;
|
||||
}
|
||||
|
||||
(void)Bind(&Skip);
|
||||
Bind(&Skip);
|
||||
|
||||
if ((i + 1) != NumElements) {
|
||||
// Handle register rename to save a move.
|
||||
@@ -1040,7 +1020,7 @@ void Arm64JITCore::Emulate128BitGather(IR::OpSize Size, IR::OpSize ElementSize,
|
||||
ARMEmitter::VRegister IncomingDst, std::optional<ARMEmitter::Register> BaseAddr,
|
||||
ARMEmitter::VRegister VectorIndexLow, std::optional<ARMEmitter::VRegister> VectorIndexHigh,
|
||||
ARMEmitter::VRegister MaskReg, IR::OpSize VectorIndexSize, size_t DataElementOffsetStart,
|
||||
size_t IndexElementOffsetStart, uint8_t OffsetScale, IR::OpSize AddrSize) {
|
||||
size_t IndexElementOffsetStart, uint8_t OffsetScale) {
|
||||
LOGMAN_THROW_A_FMT(ElementSize >= IR::OpSize::i8Bit && ElementSize <= IR::OpSize::i64Bit, "Invalid element size");
|
||||
|
||||
const auto PerformSMove = [this](IR::OpSize ElementSize, const ARMEmitter::Register Dst, const ARMEmitter::VRegister Vector, int index) {
|
||||
@@ -1102,7 +1082,7 @@ void Arm64JITCore::Emulate128BitGather(IR::OpSize Size, IR::OpSize ElementSize,
|
||||
PerformMove(ElementSize, WorkingReg, MaskReg, i);
|
||||
|
||||
// Skip if the mask's sign bit isn't set
|
||||
(void)tbz(WorkingReg, ElementSizeInBits - 1, &Skip);
|
||||
tbz(WorkingReg, ElementSizeInBits - 1, &Skip);
|
||||
|
||||
// Extract Index Element
|
||||
if ((IndexElement * IR::OpSizeToSize(VectorIndexSize)) >= 16) {
|
||||
@@ -1116,17 +1096,17 @@ void Arm64JITCore::Emulate128BitGather(IR::OpSize Size, IR::OpSize ElementSize,
|
||||
// Calculate memory position for this gather load
|
||||
if (BaseAddr.has_value()) {
|
||||
if (VectorIndexSize == IR::OpSize::i32Bit) {
|
||||
add(ConvertSize(AddrSize), TempMemReg, *BaseAddr, WorkingReg, ARMEmitter::ExtendedType::SXTW, FEXCore::ilog2(OffsetScale));
|
||||
add(ARMEmitter::Size::i64Bit, TempMemReg, *BaseAddr, WorkingReg, ARMEmitter::ExtendedType::SXTW, FEXCore::ilog2(OffsetScale));
|
||||
} else {
|
||||
add(ConvertSize(AddrSize), TempMemReg, *BaseAddr, WorkingReg, ARMEmitter::ShiftType::LSL, FEXCore::ilog2(OffsetScale));
|
||||
add(ARMEmitter::Size::i64Bit, TempMemReg, *BaseAddr, WorkingReg, ARMEmitter::ShiftType::LSL, FEXCore::ilog2(OffsetScale));
|
||||
}
|
||||
} else {
|
||||
///< In this case we have no base address, All addresses come from the vector register itself
|
||||
if (VectorIndexSize == IR::OpSize::i32Bit) {
|
||||
// Sign extend and shift in to the 64-bit register
|
||||
sbfiz(ConvertSize(AddrSize), TempMemReg, WorkingReg, FEXCore::ilog2(OffsetScale), 32);
|
||||
sbfiz(ARMEmitter::Size::i64Bit, TempMemReg, WorkingReg, FEXCore::ilog2(OffsetScale), 32);
|
||||
} else {
|
||||
lsl(ConvertSize(AddrSize), TempMemReg, WorkingReg, FEXCore::ilog2(OffsetScale));
|
||||
lsl(ARMEmitter::Size::i64Bit, TempMemReg, WorkingReg, FEXCore::ilog2(OffsetScale));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1140,7 +1120,7 @@ void Arm64JITCore::Emulate128BitGather(IR::OpSize Size, IR::OpSize ElementSize,
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, ElementSize); FEX_UNREACHABLE;
|
||||
}
|
||||
|
||||
(void)Bind(&Skip);
|
||||
Bind(&Skip);
|
||||
}
|
||||
|
||||
if (NeedsDestTmp) {
|
||||
@@ -1184,8 +1164,7 @@ DEF_OP(VLoadVectorGatherMasked) {
|
||||
|
||||
///< If the host supports SVE and the offset scale matches SVE limitations then it can do an SVE style load.
|
||||
const bool SupportsSVELoad = (HostSupportsSVE128 || HostSupportsSVE256) &&
|
||||
(OffsetScale == 1 || OffsetScale == IR::OpSizeToSize(VectorIndexSize)) &&
|
||||
VectorIndexSize == IROp->ElementSize && Op->AddrSize == IR::OpSize::i64Bit;
|
||||
(OffsetScale == 1 || OffsetScale == IR::OpSizeToSize(VectorIndexSize)) && VectorIndexSize == IROp->ElementSize;
|
||||
|
||||
if (SupportsSVELoad) {
|
||||
uint8_t SVEScale = FEXCore::ilog2(OffsetScale);
|
||||
@@ -1243,7 +1222,7 @@ DEF_OP(VLoadVectorGatherMasked) {
|
||||
} else {
|
||||
LOGMAN_THROW_A_FMT(!Is256Bit, "Can't emulate this gather load in the backend! Programming error!");
|
||||
Emulate128BitGather(IROp->Size, IROp->ElementSize, Dst, IncomingDst, BaseAddr, VectorIndexLow, VectorIndexHigh, MaskReg,
|
||||
VectorIndexSize, DataElementOffsetStart, IndexElementOffsetStart, OffsetScale, Op->AddrSize);
|
||||
VectorIndexSize, DataElementOffsetStart, IndexElementOffsetStart, OffsetScale);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1268,9 +1247,7 @@ DEF_OP(VLoadVectorGatherMaskedQPS) {
|
||||
!Op->VectorIndexHigh.IsInvalid() ? std::make_optional(GetVReg(Op->VectorIndexHigh)) : std::nullopt;
|
||||
|
||||
///< If the host supports SVE and the offset scale matches SVE limitations then it can do an SVE style load.
|
||||
const bool SupportsSVELoad = HostSupportsSVE128 && (OffsetScale == 1 || OffsetScale == 4) && Op->AddrSize == IR::OpSize::i64Bit;
|
||||
|
||||
if (SupportsSVELoad) {
|
||||
if (HostSupportsSVE128 && (OffsetScale == 1 || OffsetScale == 4)) {
|
||||
ARMEmitter::SVEModType ModType = ARMEmitter::SVEModType::MOD_NONE;
|
||||
if (OffsetScale != 1) {
|
||||
ModType = ARMEmitter::SVEModType::MOD_LSL;
|
||||
@@ -1324,7 +1301,7 @@ DEF_OP(VLoadVectorGatherMaskedQPS) {
|
||||
}
|
||||
} else {
|
||||
Emulate128BitGather(IR::OpSize::i128Bit, IR::OpSize::i32Bit, Dst, IncomingDst, BaseAddr, VectorIndexLow, VectorIndexHigh, MaskReg,
|
||||
IR::OpSize::i64Bit, 0, 0, OffsetScale, Op->AddrSize);
|
||||
IR::OpSize::i64Bit, 0, 0, OffsetScale);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1625,7 +1602,7 @@ DEF_OP(StoreMem) {
|
||||
const auto MemReg = GetReg(Op->Addr);
|
||||
const auto MemSrc = GenerateMemOperand(OpSize, MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale);
|
||||
|
||||
if (Op->Class == IR::RegClass::GPR) {
|
||||
if (Op->Class == FEXCore::IR::GPRClass) {
|
||||
const auto Src = GetZeroableReg(Op->Value);
|
||||
switch (OpSize) {
|
||||
case IR::OpSize::i8Bit: strb(Src, MemSrc); break;
|
||||
@@ -1736,7 +1713,7 @@ DEF_OP(StoreMemPair) {
|
||||
const auto OpSize = IROp->Size;
|
||||
const auto Addr = GetReg(Op->Addr);
|
||||
|
||||
if (Op->Class == IR::RegClass::GPR) {
|
||||
if (Op->Class == FEXCore::IR::GPRClass) {
|
||||
const auto Src1 = GetZeroableReg(Op->Value1);
|
||||
const auto Src2 = GetZeroableReg(Op->Value2);
|
||||
switch (OpSize) {
|
||||
@@ -1763,17 +1740,17 @@ DEF_OP(StoreMemTSO) {
|
||||
|
||||
const auto MemReg = GetReg(Op->Addr);
|
||||
|
||||
if (Op->Class == IR::RegClass::GPR) {
|
||||
if (Op->Class == FEXCore::IR::GPRClass) {
|
||||
LOGMAN_THROW_A_FMT(Op->Offset.IsInvalid() || CTX->HostFeatures.SupportsTSOImm9, "unexpected offset");
|
||||
LOGMAN_THROW_A_FMT(Op->OffsetScale == 1, "unexpected offset scale");
|
||||
LOGMAN_THROW_A_FMT(Op->OffsetType == IR::MemOffsetType::SXTX, "unexpected offset type");
|
||||
LOGMAN_THROW_A_FMT(Op->OffsetType == IR::MEM_OFFSET_SXTX, "unexpected offset type");
|
||||
}
|
||||
|
||||
if (CTX->HostFeatures.SupportsTSOImm9 && Op->Class == IR::RegClass::GPR) {
|
||||
if (CTX->HostFeatures.SupportsTSOImm9 && Op->Class == FEXCore::IR::GPRClass) {
|
||||
const auto Src = GetZeroableReg(Op->Value);
|
||||
uint64_t Offset = 0;
|
||||
if (!Op->Offset.IsInvalid()) {
|
||||
bool IsInline = IsInlineConstant(Op->Offset, &Offset);
|
||||
[[maybe_unused]] bool IsInline = IsInlineConstant(Op->Offset, &Offset);
|
||||
LOGMAN_THROW_A_FMT(IsInline, "expected immediate");
|
||||
}
|
||||
|
||||
@@ -1790,7 +1767,7 @@ DEF_OP(StoreMemTSO) {
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled StoreMemTSO size: {}", OpSize); break;
|
||||
}
|
||||
}
|
||||
} else if (Op->Class == IR::RegClass::GPR) {
|
||||
} else if (Op->Class == FEXCore::IR::GPRClass) {
|
||||
const auto Src = GetZeroableReg(Op->Value);
|
||||
|
||||
if (OpSize == IR::OpSize::i8Bit) {
|
||||
@@ -1874,7 +1851,7 @@ DEF_OP(MemSet) {
|
||||
|
||||
if (!DirectionIsInline) {
|
||||
// Backward or forwards implementation depends on flag
|
||||
(void)tbnz(DirectionReg, 1, &BackwardImpl);
|
||||
tbnz(DirectionReg, 1, &BackwardImpl);
|
||||
}
|
||||
|
||||
auto MemStore = [this](auto Value, uint32_t OpSize, int32_t Size) {
|
||||
@@ -1922,7 +1899,7 @@ DEF_OP(MemSet) {
|
||||
ARMEmitter::ForwardLabel DoneInternal {};
|
||||
|
||||
// Early exit if zero count.
|
||||
(void)cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
|
||||
cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
|
||||
|
||||
if (!IsAtomic) {
|
||||
ARMEmitter::ForwardLabel AgainInternal256Exit {};
|
||||
@@ -1939,50 +1916,50 @@ DEF_OP(MemSet) {
|
||||
// Do this in two parts, to fallback to the byte by byte loop if size < 32, and to the
|
||||
// single copy loop if size < 64.
|
||||
sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);
|
||||
(void)tbnz(TMP1, 63, &AgainInternal128Exit);
|
||||
tbnz(TMP1, 63, &AgainInternal128Exit);
|
||||
|
||||
// Fill VTMP2 with the set pattern
|
||||
dup(SubRegSize, VTMP2.Q(), Value);
|
||||
|
||||
sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);
|
||||
(void)tbnz(TMP1, 63, &AgainInternal256Exit);
|
||||
tbnz(TMP1, 63, &AgainInternal256Exit);
|
||||
|
||||
(void)Bind(&AgainInternal256);
|
||||
Bind(&AgainInternal256);
|
||||
stp<ARMEmitter::IndexType::POST>(VTMP2.Q(), VTMP2.Q(), TMP2, 32 * Direction);
|
||||
stp<ARMEmitter::IndexType::POST>(VTMP2.Q(), VTMP2.Q(), TMP2, 32 * Direction);
|
||||
sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 64 / Size);
|
||||
(void)tbz(TMP1, 63, &AgainInternal256);
|
||||
tbz(TMP1, 63, &AgainInternal256);
|
||||
|
||||
(void)Bind(&AgainInternal256Exit);
|
||||
Bind(&AgainInternal256Exit);
|
||||
add(ARMEmitter::Size::i64Bit, TMP1, TMP1, 64 / Size);
|
||||
(void)cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
|
||||
cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
|
||||
|
||||
sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);
|
||||
(void)tbnz(TMP1, 63, &AgainInternal128Exit);
|
||||
(void)Bind(&AgainInternal128);
|
||||
tbnz(TMP1, 63, &AgainInternal128Exit);
|
||||
Bind(&AgainInternal128);
|
||||
stp<ARMEmitter::IndexType::POST>(VTMP2.Q(), VTMP2.Q(), TMP2, 32 * Direction);
|
||||
sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);
|
||||
(void)tbz(TMP1, 63, &AgainInternal128);
|
||||
tbz(TMP1, 63, &AgainInternal128);
|
||||
|
||||
(void)Bind(&AgainInternal128Exit);
|
||||
Bind(&AgainInternal128Exit);
|
||||
add(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);
|
||||
(void)cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
|
||||
cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
|
||||
|
||||
if (Direction == -1) {
|
||||
add(ARMEmitter::Size::i64Bit, TMP2, TMP2, 32 - Size);
|
||||
}
|
||||
}
|
||||
|
||||
(void)Bind(&AgainInternal);
|
||||
Bind(&AgainInternal);
|
||||
if (IsAtomic) {
|
||||
MemStoreTSO(Value, OpSize, SizeDirection);
|
||||
} else {
|
||||
MemStore(Value, OpSize, SizeDirection);
|
||||
}
|
||||
sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 1);
|
||||
(void)cbnz(ARMEmitter::Size::i64Bit, TMP1, &AgainInternal);
|
||||
cbnz(ARMEmitter::Size::i64Bit, TMP1, &AgainInternal);
|
||||
|
||||
(void)Bind(&DoneInternal);
|
||||
Bind(&DoneInternal);
|
||||
|
||||
if (SizeDirection >= 0) {
|
||||
switch (OpSize) {
|
||||
@@ -2012,12 +1989,12 @@ DEF_OP(MemSet) {
|
||||
EmitMemset(Direction);
|
||||
|
||||
if (Direction == 1) {
|
||||
(void)b(&Done);
|
||||
(void)Bind(&BackwardImpl);
|
||||
b(&Done);
|
||||
Bind(&BackwardImpl);
|
||||
}
|
||||
}
|
||||
|
||||
(void)Bind(&Done);
|
||||
Bind(&Done);
|
||||
// Destination already set to the final pointer.
|
||||
}
|
||||
}
|
||||
@@ -2067,7 +2044,7 @@ DEF_OP(MemCpy) {
|
||||
|
||||
if (!DirectionIsInline) {
|
||||
// Backward or forwards implementation depends on flag
|
||||
(void)tbnz(DirectionReg, 1, &BackwardImpl);
|
||||
tbnz(DirectionReg, 1, &BackwardImpl);
|
||||
}
|
||||
|
||||
auto MemCpy = [this](uint32_t OpSize, int32_t Size) {
|
||||
@@ -2164,7 +2141,7 @@ DEF_OP(MemCpy) {
|
||||
ARMEmitter::ForwardLabel DoneInternal {};
|
||||
|
||||
// Early exit if zero count.
|
||||
(void)cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
|
||||
cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
|
||||
|
||||
if (!IsAtomic) {
|
||||
ARMEmitter::ForwardLabel AbsPos {};
|
||||
@@ -2174,11 +2151,11 @@ DEF_OP(MemCpy) {
|
||||
ARMEmitter::BackwardLabel AgainInternal256 {};
|
||||
|
||||
sub(ARMEmitter::Size::i64Bit, TMP4, TMP2, TMP3);
|
||||
(void)tbz(TMP4, 63, &AbsPos);
|
||||
tbz(TMP4, 63, &AbsPos);
|
||||
neg(ARMEmitter::Size::i64Bit, TMP4, TMP4);
|
||||
(void)Bind(&AbsPos);
|
||||
Bind(&AbsPos);
|
||||
sub(ARMEmitter::Size::i64Bit, TMP4, TMP4, 32);
|
||||
(void)tbnz(TMP4, 63, &AgainInternal);
|
||||
tbnz(TMP4, 63, &AgainInternal);
|
||||
|
||||
if (Direction == -1) {
|
||||
sub(ARMEmitter::Size::i64Bit, TMP2, TMP2, 32 - Size);
|
||||
@@ -2190,30 +2167,30 @@ DEF_OP(MemCpy) {
|
||||
// Do this in two parts, to fallback to the byte by byte loop if size < 32, and to the
|
||||
// single copy loop if size < 64.
|
||||
sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);
|
||||
(void)tbnz(TMP1, 63, &AgainInternal128Exit);
|
||||
tbnz(TMP1, 63, &AgainInternal128Exit);
|
||||
sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);
|
||||
(void)tbnz(TMP1, 63, &AgainInternal256Exit);
|
||||
tbnz(TMP1, 63, &AgainInternal256Exit);
|
||||
|
||||
(void)Bind(&AgainInternal256);
|
||||
Bind(&AgainInternal256);
|
||||
MemCpy(32, 32 * Direction);
|
||||
MemCpy(32, 32 * Direction);
|
||||
sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 64 / Size);
|
||||
(void)tbz(TMP1, 63, &AgainInternal256);
|
||||
tbz(TMP1, 63, &AgainInternal256);
|
||||
|
||||
(void)Bind(&AgainInternal256Exit);
|
||||
Bind(&AgainInternal256Exit);
|
||||
add(ARMEmitter::Size::i64Bit, TMP1, TMP1, 64 / Size);
|
||||
(void)cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
|
||||
cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
|
||||
|
||||
sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);
|
||||
(void)tbnz(TMP1, 63, &AgainInternal128Exit);
|
||||
(void)Bind(&AgainInternal128);
|
||||
tbnz(TMP1, 63, &AgainInternal128Exit);
|
||||
Bind(&AgainInternal128);
|
||||
MemCpy(32, 32 * Direction);
|
||||
sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);
|
||||
(void)tbz(TMP1, 63, &AgainInternal128);
|
||||
tbz(TMP1, 63, &AgainInternal128);
|
||||
|
||||
(void)Bind(&AgainInternal128Exit);
|
||||
Bind(&AgainInternal128Exit);
|
||||
add(ARMEmitter::Size::i64Bit, TMP1, TMP1, 32 / Size);
|
||||
(void)cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
|
||||
cbz(ARMEmitter::Size::i64Bit, TMP1, &DoneInternal);
|
||||
|
||||
if (Direction == -1) {
|
||||
add(ARMEmitter::Size::i64Bit, TMP2, TMP2, 32 - Size);
|
||||
@@ -2221,16 +2198,16 @@ DEF_OP(MemCpy) {
|
||||
}
|
||||
}
|
||||
|
||||
(void)Bind(&AgainInternal);
|
||||
Bind(&AgainInternal);
|
||||
if (IsAtomic) {
|
||||
MemCpyTSO(OpSize, SizeDirection);
|
||||
} else {
|
||||
MemCpy(OpSize, SizeDirection);
|
||||
}
|
||||
sub(ARMEmitter::Size::i64Bit, TMP1, TMP1, 1);
|
||||
(void)cbnz(ARMEmitter::Size::i64Bit, TMP1, &AgainInternal);
|
||||
cbnz(ARMEmitter::Size::i64Bit, TMP1, &AgainInternal);
|
||||
|
||||
(void)Bind(&DoneInternal);
|
||||
Bind(&DoneInternal);
|
||||
|
||||
// Needs to use temporaries just in case of overwrite
|
||||
mov(TMP1, MemRegDest.X());
|
||||
@@ -2288,15 +2265,186 @@ DEF_OP(MemCpy) {
|
||||
for (int32_t Direction : {1, -1}) {
|
||||
EmitMemcpy(Direction);
|
||||
if (Direction == 1) {
|
||||
(void)b(&Done);
|
||||
(void)Bind(&BackwardImpl);
|
||||
b(&Done);
|
||||
Bind(&BackwardImpl);
|
||||
}
|
||||
}
|
||||
(void)Bind(&Done);
|
||||
Bind(&Done);
|
||||
// Destination already set to the final pointer.
|
||||
}
|
||||
}
|
||||
|
||||
DEF_OP(ParanoidLoadMemTSO) {
|
||||
const auto Op = IROp->C<IR::IROp_LoadMemTSO>();
|
||||
const auto OpSize = IROp->Size;
|
||||
|
||||
auto MemReg = GetReg(Op->Addr);
|
||||
|
||||
if (CTX->HostFeatures.SupportsTSOImm9 && Op->Class == FEXCore::IR::GPRClass) {
|
||||
const auto Dst = GetReg(Node);
|
||||
uint64_t Offset = 0;
|
||||
if (!Op->Offset.IsInvalid()) {
|
||||
if (!IsInlineConstant(Op->Offset, &Offset)) {
|
||||
MemReg = ApplyMemOperand(OpSize, MemReg, TMP4, Op->Offset, Op->OffsetType, Op->OffsetScale);
|
||||
}
|
||||
}
|
||||
|
||||
if (OpSize == IR::OpSize::i8Bit) {
|
||||
// 8bit load is always aligned to natural alignment
|
||||
const auto Dst = GetReg(Node);
|
||||
ldapurb(Dst, MemReg, Offset);
|
||||
} else {
|
||||
switch (OpSize) {
|
||||
case IR::OpSize::i16Bit: ldapurh(Dst, MemReg, Offset); break;
|
||||
case IR::OpSize::i32Bit: ldapur(Dst.W(), MemReg, Offset); break;
|
||||
case IR::OpSize::i64Bit: ldapur(Dst.X(), MemReg, Offset); break;
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled ParanoidLoadMemTSO size: {}", OpSize); break;
|
||||
}
|
||||
}
|
||||
} else if (CTX->HostFeatures.SupportsRCPC && Op->Class == FEXCore::IR::GPRClass) {
|
||||
const auto Dst = GetReg(Node);
|
||||
MemReg = ApplyMemOperand(OpSize, MemReg, TMP4, Op->Offset, Op->OffsetType, Op->OffsetScale);
|
||||
if (OpSize == IR::OpSize::i8Bit) {
|
||||
// 8bit load is always aligned to natural alignment
|
||||
ldaprb(Dst.W(), MemReg);
|
||||
} else {
|
||||
switch (OpSize) {
|
||||
case IR::OpSize::i16Bit: ldaprh(Dst.W(), MemReg); break;
|
||||
case IR::OpSize::i32Bit: ldapr(Dst.W(), MemReg); break;
|
||||
case IR::OpSize::i64Bit: ldapr(Dst.X(), MemReg); break;
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled ParanoidLoadMemTSO size: {}", OpSize); break;
|
||||
}
|
||||
}
|
||||
} else if (Op->Class == FEXCore::IR::GPRClass) {
|
||||
const auto Dst = GetReg(Node);
|
||||
MemReg = ApplyMemOperand(OpSize, MemReg, TMP4, Op->Offset, Op->OffsetType, Op->OffsetScale);
|
||||
switch (OpSize) {
|
||||
case IR::OpSize::i8Bit: ldarb(Dst, MemReg); break;
|
||||
case IR::OpSize::i16Bit: ldarh(Dst, MemReg); break;
|
||||
case IR::OpSize::i32Bit: ldar(Dst.W(), MemReg); break;
|
||||
case IR::OpSize::i64Bit: ldar(Dst.X(), MemReg); break;
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled ParanoidLoadMemTSO size: {}", OpSize); break;
|
||||
}
|
||||
} else {
|
||||
const auto Dst = GetVReg(Node);
|
||||
MemReg = ApplyMemOperand(OpSize, MemReg, TMP4, Op->Offset, Op->OffsetType, Op->OffsetScale);
|
||||
switch (OpSize) {
|
||||
case IR::OpSize::i8Bit:
|
||||
ldarb(TMP1, MemReg);
|
||||
fmov(ARMEmitter::Size::i32Bit, Dst.S(), TMP1.W());
|
||||
break;
|
||||
case IR::OpSize::i16Bit:
|
||||
ldarh(TMP1, MemReg);
|
||||
fmov(ARMEmitter::Size::i32Bit, Dst.S(), TMP1.W());
|
||||
break;
|
||||
case IR::OpSize::i32Bit:
|
||||
ldar(TMP1.W(), MemReg);
|
||||
fmov(ARMEmitter::Size::i32Bit, Dst.S(), TMP1.W());
|
||||
break;
|
||||
case IR::OpSize::i64Bit:
|
||||
ldar(TMP1, MemReg);
|
||||
fmov(ARMEmitter::Size::i64Bit, Dst.D(), TMP1);
|
||||
break;
|
||||
case IR::OpSize::i128Bit:
|
||||
ldaxp(ARMEmitter::Size::i64Bit, TMP1, TMP2, MemReg);
|
||||
clrex();
|
||||
ins(ARMEmitter::SubRegSize::i64Bit, Dst, 0, TMP1);
|
||||
ins(ARMEmitter::SubRegSize::i64Bit, Dst, 1, TMP2);
|
||||
break;
|
||||
case IR::OpSize::i256Bit:
|
||||
LOGMAN_THROW_A_FMT(HostSupportsSVE256, "Need SVE256 support in order to use {} with 256-bit operation", __func__);
|
||||
dmb(ARMEmitter::BarrierScope::ISH);
|
||||
ld1b<ARMEmitter::SubRegSize::i8Bit>(Dst.Z(), PRED_TMP_32B.Zeroing(), MemReg);
|
||||
dmb(ARMEmitter::BarrierScope::ISH);
|
||||
break;
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled ParanoidLoadMemTSO size: {}", OpSize); break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
DEF_OP(ParanoidStoreMemTSO) {
|
||||
const auto Op = IROp->C<IR::IROp_StoreMemTSO>();
|
||||
const auto OpSize = IROp->Size;
|
||||
|
||||
auto MemReg = GetReg(Op->Addr);
|
||||
|
||||
if (CTX->HostFeatures.SupportsTSOImm9 && Op->Class == FEXCore::IR::GPRClass) {
|
||||
const auto Src = GetZeroableReg(Op->Value);
|
||||
uint64_t Offset = 0;
|
||||
if (!Op->Offset.IsInvalid()) {
|
||||
if (!IsInlineConstant(Op->Offset, &Offset)) {
|
||||
MemReg = ApplyMemOperand(OpSize, MemReg, TMP1, Op->Offset, Op->OffsetType, Op->OffsetScale);
|
||||
}
|
||||
}
|
||||
|
||||
if (OpSize == IR::OpSize::i8Bit) {
|
||||
// 8bit load is always aligned to natural alignment
|
||||
stlurb(Src, MemReg, Offset);
|
||||
} else {
|
||||
switch (OpSize) {
|
||||
case IR::OpSize::i16Bit: stlurh(Src, MemReg, Offset); break;
|
||||
case IR::OpSize::i32Bit: stlur(Src.W(), MemReg, Offset); break;
|
||||
case IR::OpSize::i64Bit: stlur(Src.X(), MemReg, Offset); break;
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled ParanoidStoreMemTSO size: {}", OpSize); break;
|
||||
}
|
||||
}
|
||||
} else if (Op->Class == FEXCore::IR::GPRClass) {
|
||||
const auto Src = GetZeroableReg(Op->Value);
|
||||
MemReg = ApplyMemOperand(OpSize, MemReg, TMP1, Op->Offset, Op->OffsetType, Op->OffsetScale);
|
||||
switch (OpSize) {
|
||||
case IR::OpSize::i8Bit: stlrb(Src, MemReg); break;
|
||||
case IR::OpSize::i16Bit: stlrh(Src, MemReg); break;
|
||||
case IR::OpSize::i32Bit: stlr(Src.W(), MemReg); break;
|
||||
case IR::OpSize::i64Bit: stlr(Src.X(), MemReg); break;
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled ParanoidStoreMemTSO size: {}", OpSize); break;
|
||||
}
|
||||
} else {
|
||||
const auto Src = GetVReg(Op->Value);
|
||||
|
||||
MemReg = ApplyMemOperand(OpSize, MemReg, TMP4, Op->Offset, Op->OffsetType, Op->OffsetScale);
|
||||
|
||||
switch (OpSize) {
|
||||
case IR::OpSize::i8Bit:
|
||||
umov<ARMEmitter::SubRegSize::i8Bit>(TMP1, Src, 0);
|
||||
stlrb(TMP1, MemReg);
|
||||
break;
|
||||
case IR::OpSize::i16Bit:
|
||||
umov<ARMEmitter::SubRegSize::i16Bit>(TMP1, Src, 0);
|
||||
stlrh(TMP1, MemReg);
|
||||
break;
|
||||
case IR::OpSize::i32Bit:
|
||||
umov<ARMEmitter::SubRegSize::i32Bit>(TMP1, Src, 0);
|
||||
stlr(TMP1.W(), MemReg);
|
||||
break;
|
||||
case IR::OpSize::i64Bit:
|
||||
umov<ARMEmitter::SubRegSize::i64Bit>(TMP1, Src, 0);
|
||||
stlr(TMP1, MemReg);
|
||||
break;
|
||||
case IR::OpSize::i128Bit: {
|
||||
// Move vector to GPRs
|
||||
umov<ARMEmitter::SubRegSize::i64Bit>(TMP1, Src, 0);
|
||||
umov<ARMEmitter::SubRegSize::i64Bit>(TMP2, Src, 1);
|
||||
ARMEmitter::BackwardLabel B;
|
||||
Bind(&B);
|
||||
|
||||
// ldaxp must not have both the destination registers be the same
|
||||
ldaxp(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::zr, TMP3, MemReg); // <- Can hit SIGBUS. Overwritten with DMB
|
||||
stlxp(ARMEmitter::Size::i64Bit, TMP3, TMP1, TMP2, MemReg); // <- Can also hit SIGBUS
|
||||
cbnz(ARMEmitter::Size::i64Bit, TMP3, &B); // < Overwritten with DMB
|
||||
break;
|
||||
}
|
||||
case IR::OpSize::i256Bit: {
|
||||
LOGMAN_THROW_A_FMT(HostSupportsSVE256, "Need SVE256 support in order to use {} with 256-bit operation", __func__);
|
||||
dmb(ARMEmitter::BarrierScope::ISH);
|
||||
st1b<ARMEmitter::SubRegSize::i8Bit>(Src.Z(), PRED_TMP_32B, MemReg, 0);
|
||||
dmb(ARMEmitter::BarrierScope::ISH);
|
||||
break;
|
||||
}
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled ParanoidStoreMemTSO size: {}", OpSize); break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
DEF_OP(CacheLineClear) {
|
||||
if (!CTX->HostFeatures.SupportsCacheMaintenanceOps) {
|
||||
dmb(ARMEmitter::BarrierScope::SY);
|
||||
@@ -2438,7 +2586,7 @@ DEF_OP(VStoreNonTemporalPair) {
|
||||
const auto Op = IROp->C<IR::IROp_VStoreNonTemporalPair>();
|
||||
const auto OpSize = IROp->Size;
|
||||
|
||||
const auto Is128Bit = OpSize == IR::OpSize::i128Bit;
|
||||
[[maybe_unused]] const auto Is128Bit = OpSize == IR::OpSize::i128Bit;
|
||||
LOGMAN_THROW_A_FMT(Is128Bit, "This IR operation only operates at 128-bit wide");
|
||||
|
||||
const auto ValueLow = GetVReg(Op->ValueLow);
|
||||
|
||||
@@ -10,35 +10,13 @@ $end_info$
|
||||
#endif
|
||||
|
||||
#include "Interface/Context/Context.h"
|
||||
#include "Interface/Core/JIT/DebugData.h"
|
||||
#include "Interface/Core/JIT/JITClass.h"
|
||||
#include "FEXCore/Debug/InternalThreadState.h"
|
||||
|
||||
#include <FEXCore/Core/SignalDelegator.h>
|
||||
#include <FEXCore/Debug/InternalThreadState.h>
|
||||
#include <FEXCore/Utils/EnumUtils.h>
|
||||
|
||||
namespace FEXCore::CPU {
|
||||
|
||||
DEF_OP(WFET) {
|
||||
auto Op = IROp->C<IR::IROp_WFET>();
|
||||
const auto Lower = GetReg(Op->Lower);
|
||||
const auto Upper = GetReg(Op->Upper);
|
||||
|
||||
// Combine registers.
|
||||
mov(ARMEmitter::Size::i64Bit, TMP1, Lower);
|
||||
bfi(ARMEmitter::Size::i64Bit, TMP1, Upper, 32, 32);
|
||||
if (CTX->Config.TSCScale) {
|
||||
// Scale back to ARM64 TSC scale if necessary
|
||||
lsr(ARMEmitter::Size::i64Bit, TMP1, TMP1, CTX->Config.TSCScale);
|
||||
}
|
||||
|
||||
// Clear the exclusive monitor so it can't spuriously wake up with that event.
|
||||
clrex();
|
||||
|
||||
// Execute wfet to wait until the TSC.
|
||||
wfet(TMP1);
|
||||
}
|
||||
|
||||
DEF_OP(GuestOpcode) {
|
||||
auto Op = IROp->C<IR::IROp_GuestOpcode>();
|
||||
// metadata
|
||||
@@ -48,10 +26,10 @@ DEF_OP(GuestOpcode) {
|
||||
DEF_OP(Fence) {
|
||||
auto Op = IROp->C<IR::IROp_Fence>();
|
||||
switch (Op->Fence) {
|
||||
case IR::FenceType::Load: dmb(ARMEmitter::BarrierScope::LD); break;
|
||||
case IR::FenceType::LoadStore: dmb(ARMEmitter::BarrierScope::SY); break;
|
||||
case IR::FenceType::Store: dmb(ARMEmitter::BarrierScope::ST); break;
|
||||
case IR::FenceType::Inst: isb(); break;
|
||||
case IR::Fence_Load.Val: dmb(ARMEmitter::BarrierScope::LD); break;
|
||||
case IR::Fence_LoadStore.Val: dmb(ARMEmitter::BarrierScope::SY); break;
|
||||
case IR::Fence_Store.Val: dmb(ARMEmitter::BarrierScope::ST); break;
|
||||
case IR::Fence_Inst.Val: isb(); break;
|
||||
default: LOGMAN_MSG_A_FMT("Unknown Fence: {}", Op->Fence); break;
|
||||
}
|
||||
}
|
||||
@@ -78,19 +56,19 @@ DEF_OP(Break) {
|
||||
|
||||
switch (Op->Reason.Signal) {
|
||||
case Core::FAULT_SIGILL:
|
||||
ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGILL));
|
||||
ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.GuestSignal_SIGILL));
|
||||
br(TMP1);
|
||||
break;
|
||||
case Core::FAULT_SIGTRAP:
|
||||
ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGTRAP));
|
||||
ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.GuestSignal_SIGTRAP));
|
||||
br(TMP1);
|
||||
break;
|
||||
case Core::FAULT_SIGSEGV:
|
||||
ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGSEGV));
|
||||
ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.GuestSignal_SIGSEGV));
|
||||
br(TMP1);
|
||||
break;
|
||||
default:
|
||||
ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGTRAP));
|
||||
ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.GuestSignal_SIGTRAP));
|
||||
br(TMP1);
|
||||
break;
|
||||
}
|
||||
@@ -108,10 +86,10 @@ DEF_OP(GetRoundingMode) {
|
||||
// zero. Just swapping 01 and 10. That's a bitfield reverse. Round mode is in
|
||||
// bottom two bits. After reversing as a 32-bit operation, it'll be in [31:30]
|
||||
// and ripe for reinsertion back at 0.
|
||||
static_assert(FEXCore::ToUnderlying(IR::RoundMode::Nearest) == 0);
|
||||
static_assert(FEXCore::ToUnderlying(IR::RoundMode::NegInfinity) == 1);
|
||||
static_assert(FEXCore::ToUnderlying(IR::RoundMode::PosInfinity) == 2);
|
||||
static_assert(FEXCore::ToUnderlying(IR::RoundMode::TowardsZero) == 3);
|
||||
static_assert(IR::ROUND_MODE_NEAREST == 0);
|
||||
static_assert(IR::ROUND_MODE_NEGATIVE_INFINITY == 1);
|
||||
static_assert(IR::ROUND_MODE_POSITIVE_INFINITY == 2);
|
||||
static_assert(IR::ROUND_MODE_TOWARDS_ZERO == 3);
|
||||
|
||||
rbit(ARMEmitter::Size::i32Bit, TMP1, Dst);
|
||||
bfi(ARMEmitter::Size::i64Bit, Dst, TMP1, 30, 2);
|
||||
@@ -189,11 +167,11 @@ DEF_OP(Print) {
|
||||
|
||||
if (IsGPR(Op->Value)) {
|
||||
mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, GetReg(Op->Value));
|
||||
ldr(ARMEmitter::XReg::x3, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.PrintValue));
|
||||
ldr(ARMEmitter::XReg::x3, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.PrintValue));
|
||||
} else {
|
||||
fmov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, GetVReg(Op->Value), false);
|
||||
fmov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, GetVReg(Op->Value), true);
|
||||
ldr(ARMEmitter::XReg::x3, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.PrintVectorValue));
|
||||
ldr(ARMEmitter::XReg::x3, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.PrintVectorValue));
|
||||
}
|
||||
|
||||
if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {
|
||||
@@ -241,7 +219,7 @@ DEF_OP(ProcessorID) {
|
||||
sub(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, ARMEmitter::Reg::rsp, 16);
|
||||
|
||||
// Load the getcpu syscall number
|
||||
#if defined(ARCHITECTURE_x86_64)
|
||||
#if defined(_M_X86_64)
|
||||
// Just to ensure the syscall number doesn't change if compiled for an x86_64 host.
|
||||
constexpr auto GetCPUSyscallNum = 0xa8;
|
||||
#else
|
||||
@@ -288,43 +266,4 @@ DEF_OP(Yield) {
|
||||
yield();
|
||||
}
|
||||
|
||||
DEF_OP(MonoBackpatcherWrite) {
|
||||
auto Op = IROp->C<IR::IROp_MonoBackpatcherWrite>();
|
||||
|
||||
mov(ARMEmitter::Size::i64Bit, TMP3, GetReg(Op->Addr));
|
||||
mov(ARMEmitter::Size::i64Bit, TMP4, GetReg(Op->Value));
|
||||
|
||||
PushDynamicRegs(TMP1);
|
||||
SpillStaticRegs(TMP1);
|
||||
|
||||
mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, STATE.R());
|
||||
mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, IR::OpSizeToSize(Op->Size));
|
||||
|
||||
if (!TMP_ABIARGS) {
|
||||
mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r2, TMP3);
|
||||
mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r3, TMP4);
|
||||
}
|
||||
|
||||
#ifdef ARCHITECTURE_arm64ec
|
||||
ldr(TMP2, ARMEmitter::XReg::x18, TEB_CPU_AREA_OFFSET);
|
||||
LoadConstant(ARMEmitter::Size::i32Bit, TMP1, 1);
|
||||
strb(TMP1.W(), TMP2, CPU_AREA_IN_SYSCALL_CALLBACK_OFFSET);
|
||||
#endif
|
||||
|
||||
ldr(ARMEmitter::XReg::x4, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.MonoBackpatcherWrite));
|
||||
if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {
|
||||
GenerateIndirectRuntimeCall<void, void*, uint8_t, uint64_t, uint64_t>(ARMEmitter::Reg::r4);
|
||||
} else {
|
||||
blr(ARMEmitter::Reg::r4);
|
||||
}
|
||||
|
||||
#ifdef ARCHITECTURE_arm64ec
|
||||
ldr(TMP2, ARMEmitter::XReg::x18, TEB_CPU_AREA_OFFSET);
|
||||
strb(ARMEmitter::WReg::zr, TMP2, CPU_AREA_IN_SYSCALL_CALLBACK_OFFSET);
|
||||
#endif
|
||||
|
||||
FillStaticRegs();
|
||||
PopDynamicRegs();
|
||||
}
|
||||
|
||||
} // namespace FEXCore::CPU
|
||||
@@ -18,4 +18,18 @@ DEF_OP(RMWHandle) {
|
||||
mov(ARMEmitter::Size::i64Bit, GetReg(Node), GetReg(IROp->Args[0]));
|
||||
}
|
||||
|
||||
DEF_OP(Swap1) {
|
||||
auto Op = IROp->C<IR::IROp_Swap1>();
|
||||
auto A = GetReg(Op->A), B = GetReg(Op->B);
|
||||
LOGMAN_THROW_A_FMT(B == GetReg(Node), "Invariant");
|
||||
|
||||
mov(ARMEmitter::Size::i64Bit, TMP1, A);
|
||||
mov(ARMEmitter::Size::i64Bit, A, B);
|
||||
mov(ARMEmitter::Size::i64Bit, B, TMP1);
|
||||
}
|
||||
|
||||
DEF_OP(Swap2) {
|
||||
// Implemented above
|
||||
}
|
||||
|
||||
} // namespace FEXCore::CPU
|
||||
@@ -1,100 +0,0 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
#pragma once
|
||||
#include <FEXCore/IR/IR.h>
|
||||
#include <FEXCore/Utils/CompilerDefs.h>
|
||||
|
||||
namespace FEXCore::Context {
|
||||
class ContextImpl;
|
||||
}
|
||||
|
||||
namespace FEXCore::CPU {
|
||||
enum class RelocationTypes : uint32_t {
|
||||
// 8 byte literal in memory for symbol
|
||||
// Aligned to struct RelocNamedSymbolLiteral
|
||||
RELOC_NAMED_SYMBOL_LITERAL,
|
||||
|
||||
// Fixed size named thunk move
|
||||
// 4 instruction constant generation
|
||||
// Aligned to struct RelocNamedThunkMove
|
||||
RELOC_NAMED_THUNK_MOVE,
|
||||
|
||||
// 8 byte literal (relative to binary base address)
|
||||
RELOC_GUEST_RIP_LITERAL,
|
||||
|
||||
// Fixed size guest RIP move
|
||||
// 4 instruction constant generation
|
||||
// Aligned to struct RelocGuestRIP
|
||||
RELOC_GUEST_RIP_MOVE,
|
||||
};
|
||||
|
||||
struct FEX_PACKED RelocationHeader final {
|
||||
// Offset to the relocated host code data
|
||||
uint64_t Offset {};
|
||||
|
||||
RelocationTypes Type;
|
||||
};
|
||||
|
||||
struct RelocNamedSymbolLiteral final {
|
||||
enum class NamedSymbol : uint32_t {
|
||||
///< Thread specific relocations
|
||||
// JIT Literal pointers
|
||||
SYMBOL_LITERAL_EXITFUNCTION_LINKER,
|
||||
};
|
||||
|
||||
RelocationHeader Header {};
|
||||
|
||||
NamedSymbol Symbol;
|
||||
|
||||
uint32_t Pad[8];
|
||||
};
|
||||
|
||||
struct RelocNamedThunkMove final {
|
||||
RelocationHeader Header {};
|
||||
|
||||
// GPR index the constant is being moved to
|
||||
uint32_t RegisterIndex;
|
||||
|
||||
// The thunk SHA256 hash
|
||||
IR::SHA256Sum Symbol;
|
||||
};
|
||||
|
||||
struct RelocGuestRIP final {
|
||||
RelocationHeader Header {};
|
||||
|
||||
// GPR index the constant is being moved to (for non-literal relocations)
|
||||
uint8_t RegisterIndex;
|
||||
|
||||
char Pad[3];
|
||||
|
||||
// The base RIP (to be moved by the register for non-literal relocations).
|
||||
// In a serialized code cache, this is relative to the binary base address.
|
||||
uint64_t GuestRIP;
|
||||
|
||||
uint32_t pad2[6] {};
|
||||
};
|
||||
|
||||
union Relocation {
|
||||
// Clang 16 Can't default-initialize this union
|
||||
static Relocation Default() {
|
||||
#if __clang_major__ < 17
|
||||
Relocation Ret {.Header {}};
|
||||
memset(&Ret, 0, sizeof(Ret));
|
||||
return Ret;
|
||||
#else
|
||||
return {};
|
||||
#endif
|
||||
}
|
||||
|
||||
RelocationHeader Header {};
|
||||
|
||||
RelocNamedSymbolLiteral NamedSymbolLiteral;
|
||||
// This makes our union of relocations at least 48 bytes
|
||||
// It might be more efficient to not use a union
|
||||
RelocNamedThunkMove NamedThunkMove;
|
||||
|
||||
RelocGuestRIP GuestRIP;
|
||||
};
|
||||
|
||||
uint64_t GetNamedSymbolLiteral(FEXCore::Context::ContextImpl&, RelocNamedSymbolLiteral::NamedSymbol);
|
||||
|
||||
} // namespace FEXCore::CPU
|
||||
@@ -41,7 +41,6 @@ namespace FEXCore::CPU {
|
||||
const auto Op = IROp->C<IR::IROp_##FEXOp>(); \
|
||||
const auto OpSize = IROp->Size; \
|
||||
const auto Is256Bit = OpSize == IR::OpSize::i256Bit; \
|
||||
const auto Is128Bit = OpSize == IR::OpSize::i128Bit; \
|
||||
LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, "Need SVE256 support in order to use {} with 256-bit operation", __func__); \
|
||||
\
|
||||
const auto Dst = GetVReg(Node); \
|
||||
@@ -50,10 +49,8 @@ namespace FEXCore::CPU {
|
||||
\
|
||||
if (HostSupportsSVE256 && Is256Bit) { \
|
||||
ARMOp(Dst.Z(), Vector1.Z(), Vector2.Z()); \
|
||||
} else if (Is128Bit) { \
|
||||
ARMOp(Dst.Q(), Vector1.Q(), Vector2.Q()); \
|
||||
} else { \
|
||||
ARMOp(Dst.D(), Vector1.D(), Vector2.D()); \
|
||||
ARMOp(Dst.Q(), Vector1.Q(), Vector2.Q()); \
|
||||
} \
|
||||
}
|
||||
|
||||
@@ -196,29 +193,29 @@ namespace FEXCore::CPU {
|
||||
VFScalarOperation(IROp->Size, ElementSize, Op->ZeroUpperBits, ScalarEmit, Dst, Vector1, Vector2); \
|
||||
}
|
||||
|
||||
#define DEF_FMAOP_SCALAR_INSERT(FEXOp, ARMOp) \
|
||||
DEF_OP(FEXOp) { \
|
||||
const auto Op = IROp->C<IR::IROp_##FEXOp>(); \
|
||||
const auto ElementSize = Op->Header.ElementSize; \
|
||||
\
|
||||
auto ScalarEmit = [this, ElementSize](ARMEmitter::VRegister Dst, ARMEmitter::VRegister Src1, ARMEmitter::VRegister Src2, \
|
||||
ARMEmitter::VRegister Src3) { \
|
||||
if (ElementSize == IR::OpSize::i16Bit) { \
|
||||
ARMOp(Dst.H(), Src1.H(), Src2.H(), Src3.H()); \
|
||||
} else if (ElementSize == IR::OpSize::i32Bit) { \
|
||||
ARMOp(Dst.S(), Src1.S(), Src2.S(), Src3.S()); \
|
||||
} else if (ElementSize == IR::OpSize::i64Bit) { \
|
||||
ARMOp(Dst.D(), Src1.D(), Src2.D(), Src3.D()); \
|
||||
} \
|
||||
}; \
|
||||
\
|
||||
const auto Dst = GetVReg(Node); \
|
||||
const auto Upper = GetVReg(Op->Upper); \
|
||||
const auto Vector1 = GetVReg(Op->Vector1); \
|
||||
const auto Vector2 = GetVReg(Op->Vector2); \
|
||||
const auto Addend = GetVReg(Op->Addend); \
|
||||
\
|
||||
VFScalarFMAOperation(IROp->Size, ElementSize, ScalarEmit, Dst, Upper, Vector1, Vector2, Addend); \
|
||||
#define DEF_FMAOP_SCALAR_INSERT(FEXOp, ARMOp) \
|
||||
DEF_OP(FEXOp) { \
|
||||
const auto Op = IROp->C<IR::IROp_##FEXOp>(); \
|
||||
const auto ElementSize = Op->Header.ElementSize; \
|
||||
\
|
||||
auto ScalarEmit = \
|
||||
[this, ElementSize](ARMEmitter::VRegister Dst, ARMEmitter::VRegister Src1, ARMEmitter::VRegister Src2, ARMEmitter::VRegister Src3) { \
|
||||
if (ElementSize == IR::OpSize::i16Bit) { \
|
||||
ARMOp(Dst.H(), Src1.H(), Src2.H(), Src3.H()); \
|
||||
} else if (ElementSize == IR::OpSize::i32Bit) { \
|
||||
ARMOp(Dst.S(), Src1.S(), Src2.S(), Src3.S()); \
|
||||
} else if (ElementSize == IR::OpSize::i64Bit) { \
|
||||
ARMOp(Dst.D(), Src1.D(), Src2.D(), Src3.D()); \
|
||||
} \
|
||||
}; \
|
||||
\
|
||||
const auto Dst = GetVReg(Node); \
|
||||
const auto Upper = GetVReg(Op->Upper); \
|
||||
const auto Vector1 = GetVReg(Op->Vector1); \
|
||||
const auto Vector2 = GetVReg(Op->Vector2); \
|
||||
const auto Addend = GetVReg(Op->Addend); \
|
||||
\
|
||||
VFScalarFMAOperation(IROp->Size, ElementSize, ScalarEmit, Dst, Upper, Vector1, Vector2, Addend); \
|
||||
}
|
||||
|
||||
DEF_UNOP(VAbs, abs, true)
|
||||
@@ -747,11 +744,11 @@ DEF_OP(VFToIScalarInsert) {
|
||||
auto Src = *std::get_if<ARMEmitter::VRegister>(&SrcVar);
|
||||
|
||||
switch (RoundMode) {
|
||||
case IR::RoundMode::Nearest: frintn(SubRegSize.Scalar, Dst, Src); break;
|
||||
case IR::RoundMode::NegInfinity: frintm(SubRegSize.Scalar, Dst, Src); break;
|
||||
case IR::RoundMode::PosInfinity: frintp(SubRegSize.Scalar, Dst, Src); break;
|
||||
case IR::RoundMode::TowardsZero: frintz(SubRegSize.Scalar, Dst, Src); break;
|
||||
case IR::RoundMode::Host: frinti(SubRegSize.Scalar, Dst, Src); break;
|
||||
case IR::Round_Nearest: frintn(SubRegSize.Scalar, Dst, Src); break;
|
||||
case IR::Round_Negative_Infinity: frintm(SubRegSize.Scalar, Dst, Src); break;
|
||||
case IR::Round_Positive_Infinity: frintp(SubRegSize.Scalar, Dst, Src); break;
|
||||
case IR::Round_Towards_Zero: frintz(SubRegSize.Scalar, Dst, Src); break;
|
||||
case IR::Round_Host: frinti(SubRegSize.Scalar, Dst, Src); break;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -806,8 +803,8 @@ DEF_OP(VFCMPScalarInsert) {
|
||||
default: break;
|
||||
}
|
||||
};
|
||||
auto ScalarEmitUNO = [this, SubRegSize, ZeroUpperBits, Is256Bit](ARMEmitter::VRegister Dst, ARMEmitter::VRegister Src1,
|
||||
ARMEmitter::VRegister Src2) {
|
||||
auto ScalarEmitUNO =
|
||||
[this, SubRegSize, ZeroUpperBits, Is256Bit](ARMEmitter::VRegister Dst, ARMEmitter::VRegister Src1, ARMEmitter::VRegister Src2) {
|
||||
switch (SubRegSize.Scalar) {
|
||||
case ARMEmitter::ScalarRegSize::i16Bit: {
|
||||
fcmge(VTMP1.H(), Src1.H(), Src2.H());
|
||||
@@ -841,8 +838,8 @@ DEF_OP(VFCMPScalarInsert) {
|
||||
}
|
||||
}
|
||||
};
|
||||
auto ScalarEmitNEQ = [this, SubRegSize, ZeroUpperBits, Is256Bit](ARMEmitter::VRegister Dst, ARMEmitter::VRegister Src1,
|
||||
ARMEmitter::VRegister Src2) {
|
||||
auto ScalarEmitNEQ =
|
||||
[this, SubRegSize, ZeroUpperBits, Is256Bit](ARMEmitter::VRegister Dst, ARMEmitter::VRegister Src1, ARMEmitter::VRegister Src2) {
|
||||
switch (SubRegSize.Scalar) {
|
||||
case ARMEmitter::ScalarRegSize::i16Bit: {
|
||||
fcmeq(VTMP1.H(), Src2.H(), Src1.H());
|
||||
@@ -871,8 +868,8 @@ DEF_OP(VFCMPScalarInsert) {
|
||||
}
|
||||
}
|
||||
};
|
||||
auto ScalarEmitORD = [this, SubRegSize, ZeroUpperBits, Is256Bit](ARMEmitter::VRegister Dst, ARMEmitter::VRegister Src1,
|
||||
ARMEmitter::VRegister Src2) {
|
||||
auto ScalarEmitORD =
|
||||
[this, SubRegSize, ZeroUpperBits, Is256Bit](ARMEmitter::VRegister Dst, ARMEmitter::VRegister Src1, ARMEmitter::VRegister Src2) {
|
||||
switch (SubRegSize.Scalar) {
|
||||
case ARMEmitter::ScalarRegSize::i16Bit: {
|
||||
fcmge(VTMP1.H(), Src1.H(), Src2.H());
|
||||
@@ -977,7 +974,7 @@ DEF_OP(LoadNamedVectorConstant) {
|
||||
}
|
||||
// Load the pointer.
|
||||
auto GenerateMemOperand = [this](IR::OpSize OpSize, uint32_t NamedConstant, ARMEmitter::Register Base) {
|
||||
const auto ConstantOffset = offsetof(FEXCore::Core::CpuStateFrame, Pointers.NamedVectorConstants[NamedConstant]);
|
||||
const auto ConstantOffset = offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.NamedVectorConstants[NamedConstant]);
|
||||
|
||||
if (ConstantOffset <= 255 || // Unscaled 9-bit signed
|
||||
((ConstantOffset & (IR::OpSizeToSize(OpSize) - 1)) == 0 &&
|
||||
@@ -985,13 +982,13 @@ DEF_OP(LoadNamedVectorConstant) {
|
||||
return ARMEmitter::ExtendedMemOperand(Base.X(), ARMEmitter::IndexType::OFFSET, ConstantOffset);
|
||||
}
|
||||
|
||||
ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.NamedVectorConstantPointers[NamedConstant]));
|
||||
ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.NamedVectorConstantPointers[NamedConstant]));
|
||||
return ARMEmitter::ExtendedMemOperand(TMP1, ARMEmitter::IndexType::OFFSET, 0);
|
||||
};
|
||||
|
||||
if (OpSize == IR::OpSize::i256Bit) {
|
||||
// Handle SVE 32-byte variant upfront.
|
||||
ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.NamedVectorConstantPointers[Op->Constant]));
|
||||
ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.NamedVectorConstantPointers[Op->Constant]));
|
||||
ld1b<ARMEmitter::SubRegSize::i8Bit>(Dst.Z(), PRED_TMP_32B.Zeroing(), TMP1, 0);
|
||||
return;
|
||||
}
|
||||
@@ -1013,7 +1010,7 @@ DEF_OP(LoadNamedVectorIndexedConstant) {
|
||||
const auto Dst = GetVReg(Node);
|
||||
|
||||
// Load the pointer.
|
||||
ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.IndexedNamedVectorConstantPointers[Op->Constant]));
|
||||
ldr(TMP1, STATE_PTR(CpuStateFrame, Pointers.Common.IndexedNamedVectorConstantPointers[Op->Constant]));
|
||||
|
||||
switch (OpSize) {
|
||||
case IR::OpSize::i8Bit: ldrb(Dst, TMP1, Op->Index); break;
|
||||
@@ -1118,7 +1115,7 @@ DEF_OP(VAddP) {
|
||||
}
|
||||
|
||||
DEF_OP(VFAddV) {
|
||||
const auto Op = IROp->C<IR::IROp_VFAddV>();
|
||||
const auto Op = IROp->C<IR::IROp_VAddV>();
|
||||
const auto OpSize = IROp->Size;
|
||||
const auto Is256Bit = OpSize == IR::OpSize::i256Bit;
|
||||
LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, "Need SVE256 support in order to use {} with 256-bit operation", __func__);
|
||||
@@ -1355,7 +1352,7 @@ DEF_OP(VFMin) {
|
||||
|
||||
const auto ElementSize = Op->Header.ElementSize;
|
||||
const auto SubRegSize = ConvertSubRegSize248(IROp);
|
||||
const auto IsScalar = ElementSize == OpSize;
|
||||
[[maybe_unused]] const auto IsScalar = ElementSize == OpSize;
|
||||
const auto Is256Bit = OpSize == IR::OpSize::i256Bit;
|
||||
LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, "Need SVE256 support in order to use {} with 256-bit operation", __func__);
|
||||
|
||||
@@ -1428,7 +1425,7 @@ DEF_OP(VFMax) {
|
||||
|
||||
const auto ElementSize = Op->Header.ElementSize;
|
||||
const auto SubRegSize = ConvertSubRegSize248(IROp);
|
||||
const auto IsScalar = ElementSize == OpSize;
|
||||
[[maybe_unused]] const auto IsScalar = ElementSize == OpSize;
|
||||
const auto Is256Bit = OpSize == IR::OpSize::i256Bit;
|
||||
LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, "Need SVE256 support in order to use {} with 256-bit operation", __func__);
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ $end_info$
|
||||
|
||||
namespace FEXCore {
|
||||
GuestToHostMap::GuestToHostMap()
|
||||
: BlockLinks_mbr {"FEXMem_BlockLinks"} {
|
||||
: BlockLinks_mbr {fextl::pmr::get_default_resource()} {
|
||||
BlockLinks_pma = fextl::make_unique<std::pmr::polymorphic_allocator<std::byte>>(&BlockLinks_mbr);
|
||||
// Setup our PMR map.
|
||||
BlockLinks = BlockLinks_pma->new_object<BlockLinksMapType>();
|
||||
@@ -24,7 +24,7 @@ GuestToHostMap::GuestToHostMap()
|
||||
LookupCache::LookupCache(FEXCore::Context::ContextImpl* CTX)
|
||||
: ctx {CTX} {
|
||||
|
||||
TotalCacheSize = ctx->Config.VirtualMemSize / FEXCore::Utils::FEX_PAGE_SIZE * 8 + CODE_SIZE + MAX_L1_SIZE;
|
||||
TotalCacheSize = ctx->Config.VirtualMemSize / 4096 * 8 + CODE_SIZE + L1_SIZE;
|
||||
|
||||
// Block cache ends up looking like this
|
||||
// PageMemoryMap[VirtualMemoryRegion >> 12]
|
||||
@@ -39,10 +39,6 @@ LookupCache::LookupCache(FEXCore::Context::ContextImpl* CTX)
|
||||
// We need one pointer per page of virtual memory
|
||||
// At 64GB of virtual memory this will allocate 128MB of virtual memory space
|
||||
PagePointer = reinterpret_cast<uintptr_t>(FEXCore::Allocator::VirtualAlloc(TotalCacheSize, false, false));
|
||||
LOGMAN_THROW_A_FMT(PagePointer != -1ULL, "Failed to allocate PagePointer");
|
||||
|
||||
FEXCore::Allocator::VirtualName("FEXMem_Lookup", reinterpret_cast<void*>(PagePointer),
|
||||
ctx->Config.VirtualMemSize / FEXCore::Utils::FEX_PAGE_SIZE * 8 + CODE_SIZE);
|
||||
CTX->SyscallHandler->MarkOvercommitRange(PagePointer, TotalCacheSize);
|
||||
|
||||
// Allocate our memory backing our pages
|
||||
@@ -50,21 +46,14 @@ LookupCache::LookupCache(FEXCore::Context::ContextImpl* CTX)
|
||||
// XXX: We can drop down to 16KB if we store 4byte offsets from the code base
|
||||
// We currently limit to 128MB of real memory for caching for the total cache size.
|
||||
// Can end up being inefficient if we compile a small number of blocks per page
|
||||
PageMemory = PagePointer + ctx->Config.VirtualMemSize / FEXCore::Utils::FEX_PAGE_SIZE * 8;
|
||||
PageMemory = PagePointer + ctx->Config.VirtualMemSize / 4096 * 8;
|
||||
LOGMAN_THROW_A_FMT(PageMemory != -1ULL, "Failed to allocate page memory");
|
||||
|
||||
// L1 Cache
|
||||
L1Pointer = PageMemory + CODE_SIZE;
|
||||
FEXCore::Allocator::VirtualName("FEXMem_Lookup_L1", reinterpret_cast<void*>(L1Pointer), MAX_L1_SIZE);
|
||||
LOGMAN_THROW_A_FMT(L1Pointer != -1ULL, "Failed to allocate L1Pointer");
|
||||
|
||||
VirtualMemSize = ctx->Config.VirtualMemSize;
|
||||
|
||||
if (DynamicL1Cache()) {
|
||||
// Start at minimum size when dynamic.
|
||||
L1PointerMask = MIN_L1_ENTRIES - 1;
|
||||
} else {
|
||||
// Start at maximum instead.
|
||||
L1PointerMask = MAX_L1_ENTRIES - 1;
|
||||
}
|
||||
}
|
||||
|
||||
LookupCache::~LookupCache() {
|
||||
@@ -75,27 +64,31 @@ LookupCache::~LookupCache() {
|
||||
// These will get freed when their memory allocators are deallocated.
|
||||
}
|
||||
|
||||
void LookupCache::ClearL2Cache(const FEXCore::LookupCacheBaseLockToken& lk) {
|
||||
void LookupCache::ClearL2Cache() {
|
||||
auto lk = Shared->AcquireLock();
|
||||
// Clear out the page memory
|
||||
// PagePointer and PageMemory are sequential with each other. Clear both at once.
|
||||
FEXCore::Allocator::VirtualDontNeed(reinterpret_cast<void*>(PagePointer),
|
||||
ctx->Config.VirtualMemSize / FEXCore::Utils::FEX_PAGE_SIZE * 8 + CODE_SIZE, false);
|
||||
FEXCore::Allocator::VirtualDontNeed(reinterpret_cast<void*>(PagePointer), ctx->Config.VirtualMemSize / 4096 * 8 + CODE_SIZE, false);
|
||||
AllocateOffset = 0;
|
||||
}
|
||||
|
||||
void LookupCache::ClearThreadLocalCaches(const LookupCacheWriteLockToken&) {
|
||||
void LookupCache::ClearThreadLocalCaches() {
|
||||
auto lk = Shared->AcquireLock();
|
||||
|
||||
// Clear L1 and L2 by clearing the full cache.
|
||||
FEXCore::Allocator::VirtualDontNeed(reinterpret_cast<void*>(PagePointer), TotalCacheSize, false);
|
||||
CachedCodePages.clear();
|
||||
}
|
||||
|
||||
void LookupCache::ClearCache(const LookupCacheWriteLockToken& lk) {
|
||||
void LookupCache::ClearCache() {
|
||||
auto lk = Shared->AcquireLock();
|
||||
|
||||
// Clear L1 and L2 by clearing the full cache.
|
||||
ClearThreadLocalCaches(lk);
|
||||
FEXCore::Allocator::VirtualDontNeed(reinterpret_cast<void*>(PagePointer), TotalCacheSize, false);
|
||||
|
||||
Shared->ClearCache(lk);
|
||||
}
|
||||
|
||||
void GuestToHostMap::ClearCache(const LookupCacheWriteLockToken&) {
|
||||
void GuestToHostMap::ClearCache(const LockToken&) {
|
||||
// Allocate a new pointer from the BlockLinks pma again.
|
||||
BlockLinks = BlockLinks_pma->new_object<BlockLinksMapType>();
|
||||
// All code is gone, clear the block list
|
||||
|
||||
Loaded 100 of 1166 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user