Compare commits

..
1 Commits
Author SHA1 Message Date
Ryan Houdek 33fe6813fc Docs: Update for release FEX-2104 2021-04-02 11:35:29 -07:00
326 changed files with 5983 additions and 18843 deletions

No files matched your search

+1 -14
View File
@@ -13,14 +13,13 @@ env:
BUILD_TYPE: Release
CC: clang
CXX: clang++
FEX_FORCE32BITALLOCATOR: 1
jobs:
build:
runs-on: ${{ matrix.arch }}
strategy:
matrix:
arch: [[self-hosted, x64], [self-hosted, ARMv8.0], [self-hosted, ARMv8.2], [self-hosted, ARMv8.4]]
arch: [[self-hosted, x64], [self-hosted, ARMv8.0], [self-hosted, ARMv8.2]]
fail-fast: false
steps:
@@ -117,18 +116,6 @@ jobs:
working-directory: ${{runner.workspace}}/build
run: mv ${{runner.workspace}}/build/Testing/Temporary/LastTest.log ${{runner.workspace}}/build/Testing/Temporary/LastTest_GCC64.log || true
- name: gcc target tests 32
working-directory: ${{runner.workspace}}/build
shell: bash
# Execute the gvisor tests
run: cmake --build . --config $BUILD_TYPE --target gcc_target_tests_32
- name: GCC32 Test Results move
if: ${{ always() }}
shell: bash
working-directory: ${{runner.workspace}}/build
run: mv ${{runner.workspace}}/build/Testing/Temporary/LastTest.log ${{runner.workspace}}/build/Testing/Temporary/LastTest_GCC32.log || true
- name: Struct verifier tests
working-directory: ${{runner.workspace}}/build
shell: bash
-9
View File
@@ -30,12 +30,3 @@
shallow = true
path = External/fex-gcc-target-tests-bins
url = https://github.com/FEX-Emu/fex-gcc-target-tests-bins.git
[submodule "External/jemalloc"]
path = External/jemalloc
url = https://github.com/FEX-Emu/jemalloc.git
[submodule "External/fmt"]
path = External/fmt
url = https://github.com/fmtlib/fmt.git
[submodule "External/drm-headers"]
path = External/drm-headers
url = https://github.com/FEX-Emu/drm-headers.git
+13 -92
View File
@@ -61,8 +61,8 @@ endif()
if (ENABLE_ASAN)
add_definitions(-DENABLE_ASAN=1)
add_compile_options(-fno-omit-frame-pointer -fsanitize=address -fsanitize-address-use-after-scope)
link_libraries(-fno-omit-frame-pointer -fsanitize=address -fsanitize-address-use-after-scope)
add_compile_options(-fno-omit-frame-pointer -fsanitize=address)
link_libraries(-fno-omit-frame-pointer -fsanitize=address)
endif()
if (ENABLE_TSAN)
@@ -103,21 +103,14 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
message(FATAL_ERROR "FEX doesn't support getting compiled with GCC!")
endif()
find_package(PkgConfig REQUIRED)
find_package(Python 3.0 REQUIRED COMPONENTS Interpreter)
pkg_check_modules(XXHASH libxxhash REQUIRED)
add_definitions(-Wno-trigraphs)
add_definitions(-DGLOBAL_DATA_DIRECTORY="${DATA_DIRECTORY}/")
add_subdirectory(External/jemalloc/)
include_directories(External/jemalloc/pregen/include/)
add_subdirectory(External/cpp-optparse/)
include_directories(External/cpp-optparse/)
add_subdirectory(External/fmt/)
add_subdirectory(External/imgui/)
include_directories(External/imgui/)
@@ -165,27 +158,18 @@ if(ENABLE_WERROR OR ENABLE_STRICT_WERROR)
endif()
if(_M_ARM_64)
if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 999999.0)
# Clang 12.0 fixed the -mcpu=native bug with mixed big.little implementers
# Clang can not currently check for native Apple M1 type in hypervisor. Currently disabled
check_cxx_compiler_flag("-mcpu=native" COMPILER_SUPPORTS_CPU_TYPE)
if(COMPILER_SUPPORTS_CPU_TYPE)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcpu=native")
endif()
else()
# Due to an oversight in llvm, it declares any reasonably new Kryo CPU to only be ARMv8.0
# Manually detect newer CPU revisions until clang and llvm fixes their bug
# This script will either provide a supported CPU or 'native'
# Additionally -march doesn't work under AArch64+Clang, so you have to use -mcpu or -mtune
execute_process(COMMAND python3 "${PROJECT_SOURCE_DIR}/Scripts/aarch64_fit_native.py" "/proc/cpuinfo" "${CMAKE_CXX_COMPILER_VERSION}"
OUTPUT_VARIABLE AARCH64_CPU)
# Due to an oversight in llvm, it declares any reasonably new Kryo CPU to only be ARMv8.0
# Manually detect newer CPU revisions until clang and llvm fixes their bug
# This script will either provide a supported CPU or 'native'
# Additionally -march doesn't work under AArch64+Clang, so you have to use -mcpu or -mtune
execute_process(COMMAND python3 "${PROJECT_SOURCE_DIR}/Scripts/aarch64_fit_native.py" "/proc/cpuinfo"
OUTPUT_VARIABLE AARCH64_CPU)
string(STRIP ${AARCH64_CPU} AARCH64_CPU)
string(STRIP ${AARCH64_CPU} AARCH64_CPU)
check_cxx_compiler_flag("-mcpu=${AARCH64_CPU}" COMPILER_SUPPORTS_CPU_TYPE)
if(COMPILER_SUPPORTS_CPU_TYPE)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcpu=${AARCH64_CPU}")
endif()
check_cxx_compiler_flag("-mcpu=${AARCH64_CPU}" COMPILER_SUPPORTS_CPU_TYPE)
if(COMPILER_SUPPORTS_CPU_TYPE)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcpu=${AARCH64_CPU}")
endif()
endif()
@@ -252,7 +236,7 @@ add_compile_options(-Wall)
configure_file(
${CMAKE_CURRENT_SOURCE_DIR}/include/Config.h.in
${CMAKE_BINARY_DIR}/generated/ConfigDefines.h)
${CMAKE_BINARY_DIR}/generated/Config.h)
if (BUILD_TESTS)
include(CTest)
@@ -261,9 +245,6 @@ if (BUILD_TESTS)
endif()
add_subdirectory(External/FEXCore)
# Binfmt_misc files must be installed prior to Source/ installs
add_subdirectory(Data/binfmts/)
add_subdirectory(Source/)
add_subdirectory(Data/AppConfig/)
@@ -310,63 +291,3 @@ if (BUILD_THUNKS)
DEPENDS guest-libs
)
endif()
set(FEX_VERSION_MAJOR "0")
set(FEX_VERSION_MINOR "0")
set(FEX_VERSION_PATCH "0")
find_package(Git)
if (GIT_FOUND)
execute_process(
COMMAND ${GIT_EXECUTABLE} describe --abbrev=0
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
OUTPUT_VARIABLE GIT_DESCRIBE_STRING
RESULT_VARIABLE GIT_ERROR
ERROR_QUIET
OUTPUT_STRIP_TRAILING_WHITESPACE
)
if (NOT ${GIT_ERROR} EQUAL 0)
# Likely built in a way that doesn't have tags
# Setup a version tag that is unknown
set(GIT_DESCRIBE_STRING "FEX-0000")
endif()
# Change something like `FEX-2106.1-76-<hash>` in to a list
string(REPLACE "-" ";" DESCRIBE_LIST ${GIT_DESCRIBE_STRING})
# Extract the `2106.1` element
list(GET DESCRIBE_LIST 1 DESCRIBE_LIST)
# Change `2106.1` in to a list
string(REPLACE "." ";" DESCRIBE_LIST ${DESCRIBE_LIST})
# Calculate list size
list(LENGTH DESCRIBE_LIST LIST_SIZE)
# Pull out the major version
list(GET DESCRIBE_LIST 0 FEX_VERSION_MAJOR)
# Minor version only exists if there is a .1 at the end
# eg: 2106 versus 2106.1
if (LIST_SIZE GREATER 1)
list(GET DESCRIBE_LIST 1 FEX_VERSION_MINOR)
endif()
endif()
# Package creation
set (CPACK_GENERATOR "DEB")
set (CPACK_PACKAGE_CONTACT "team@fex-emu.org")
set (CPACK_PACKAGE_VERSION_MAJOR "${FEX_VERSION_MAJOR}")
set (CPACK_PACKAGE_VERSION_MINOR "${FEX_VERSION_MINOR}")
set (CPACK_PACKAGE_VERSION_PATCH "${FEX_VERSION_PATCH}")
# Debian defines
set (CPACK_DEBIAN_PACKAGE_DEPENDS "libstdc++6")
set (CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${CMAKE_CURRENT_SOURCE_DIR}/CPack/postinst;${CMAKE_CURRENT_SOURCE_DIR}/CPack/prerm")
if (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64")
# binfmt_misc conflicts with qemu-user-static
# We also only install binfmt_misc on aarch64 hosts
set (CPACK_DEBIAN_PACKAGE_CONFLICTS "qemu-user-static")
endif()
include (CPack)
-18
View File
@@ -1,18 +0,0 @@
#!/bin/sh
set -e
update_binfmt() {
# Check for update-binfmts
command -v update-binfmts >/dev/null || return 0
# Setup binfmt_misc
update-binfmts --import FEX-x86
update-binfmts --import FEX-x86_64
}
# Install FEXInterpreter hardlink
# Needs to be done before setting up binfmt_misc
ln -f /usr/bin/FEXLoader /usr/bin/FEXInterpreter
if [ $(uname -m) = 'aarch64' ]; then
update_binfmt
fi
-17
View File
@@ -1,17 +0,0 @@
#!/bin/sh
set -e
update_binfmt() {
# Check for update-binfmts
command -v update-binfmts >/dev/null || return 0
# Uninstall
update-binfmts --unimport FEX-x86
update-binfmts --unimport FEX-x86_64
}
if [ $(uname -m) = 'aarch64' ]; then
update_binfmt
fi
# Remove FEXInterpreter hardlink
unlink /usr/bin/FEXInterpreter
-4
View File
@@ -1,4 +0,0 @@
install(FILES FEX-x86
DESTINATION ${CMAKE_INSTALL_PREFIX}/share/binfmts/)
install(FILES FEX-x86_64
DESTINATION ${CMAKE_INSTALL_PREFIX}/share/binfmts/)
-9
View File
@@ -1,9 +0,0 @@
package fex
interpreter /usr/bin/FEXInterpreter
magic \x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x03\x00
offset 0
mask \xff\xff\xff\xff\xff\xfe\xfe\x00\x00\x00\x00\xff\xff\xff\xff\xff\xfe\xff\xff\xff
credentials yes
fix_binary yes
preserve no
-8
View File
@@ -1,8 +0,0 @@
package fex
interpreter /usr/bin/FEXInterpreter
magic \x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x3e\x00
offset 0
mask \xff\xff\xff\xff\xff\xfe\xfe\x00\x00\x00\x00\xff\xff\xff\xff\xff\xfe\xff\xff\xff
credentials yes
fix_binary yes
preserve no
+2 -2
View File
@@ -3,7 +3,7 @@ FROM ubuntu:20.04 as builder
RUN DEBIAN_FRONTEND="noninteractive" apt-get update
RUN DEBIAN_FRONTEND="noninteractive" apt install -y cmake \
clang-10 llvm-10 nasm ninja-build \
clang-10 llvm-10 nasm ninja-build libnuma-dev \
libcap-dev libglfw3-dev libepoxy-dev python3-dev \
python3 linux-headers-generic
@@ -23,7 +23,7 @@ FROM ubuntu:20.04
RUN DEBIAN_FRONTEND="noninteractive" apt-get update
RUN DEBIAN_FRONTEND="noninteractive" apt install -y \
libcap-dev libglfw3-dev libepoxy-dev
libnuma-dev libcap-dev libglfw3-dev libepoxy-dev
COPY --from=builder /opt/FEX/build/Bin/* /usr/bin/
-35
View File
@@ -159,42 +159,7 @@ def print_man_environment(options):
default
)
print_man_environment_tail()
output_man.write(".El\n")
def print_man_environment_tail():
# Additional environment variables that live outside of the normal loop
print_man_env_option(
"FEX_APP_CONFIG_LOCATION",
[
"Allows the user to override where FEX looks for configuration files",
"By default FEX will look in {$HOME, $XDG_CONFIG_HOME}/.fex-emu/",
"This will override the full path",
],
"''")
print_man_env_option(
"FEX_APP_CONFIG",
[
"Allows the user to override where FEX looks for only the application config file",
"By default FEX will look in {$HOME, $XDG_CONFIG_HOME}/.fex-emu/Config.json",
"This will override this file location",
"One must be careful with this option as it will override any applications that load with execve as well"
"If you need to support applications that execve then use FEX_APP_CONFIG_LOCATION instead"
],
"''")
print_man_env_option(
"FEX_APP_DATA_LOCATION",
[
"Allows the user to override where FEX looks for data files",
"By default FEX will look in {$HOME, $XDG_DATA_HOME}/.fex-emu/",
"This will override the full path",
"This is the folder where FEX stores generated files like IR cache"
],
"''")
def print_man_header():
header ='''.Dd {0}
.Dt FEX
+19 -25
View File
@@ -108,10 +108,10 @@ def print_ir_sizes(ops, defines):
output_file.write("[[maybe_unused]] static size_t GetSize(IROps Op) { return IRSizes[Op]; }\n\n")
output_file.write("__attribute__((const)) __attribute__((visibility(\"default\"))) std::string_view const& GetName(IROps Op);\n")
output_file.write("__attribute__((const)) __attribute__((visibility(\"default\"))) uint8_t GetArgs(IROps Op);\n")
output_file.write("__attribute__((const)) __attribute__((visibility(\"default\"))) FEXCore::IR::RegisterClassType GetRegClass(IROps Op);\n\n")
output_file.write("__attribute__((const)) __attribute__((visibility(\"default\"))) bool HasSideEffects(IROps Op);\n")
output_file.write("std::string_view const& GetName(IROps Op);\n")
output_file.write("uint8_t GetArgs(IROps Op);\n")
output_file.write("FEXCore::IR::RegisterClassType GetRegClass(IROps Op);\n\n")
output_file.write("bool HasSideEffects(IROps Op);\n")
output_file.write("#undef IROP_SIZES\n")
output_file.write("#endif\n\n")
@@ -277,7 +277,7 @@ def print_ir_allocator_helpers(ops, defines):
output_file.write("\tusing IRPair = Wrapper<T>;\n\n")
output_file.write("\tIRPair<IROp_Header> AllocateRawOp(size_t HeaderSize) {\n")
output_file.write("\t\tauto Op = reinterpret_cast<IROp_Header*>(DualListData.DataAllocate(HeaderSize));\n")
output_file.write("\t\tauto Op = reinterpret_cast<IROp_Header*>(Data.Allocate(HeaderSize));\n")
output_file.write("\t\tmemset(Op, 0, HeaderSize);\n")
output_file.write("\t\tOp->Op = IROps::OP_DUMMY;\n")
output_file.write("\t\treturn IRPair<IROp_Header>{Op, CreateNode(Op)};\n")
@@ -286,7 +286,7 @@ def print_ir_allocator_helpers(ops, defines):
output_file.write("\ttemplate<class T, IROps T2>\n")
output_file.write("\tT *AllocateOrphanOp() {\n")
output_file.write("\t\tsize_t Size = FEXCore::IR::GetSize(T2);\n")
output_file.write("\t\tauto Op = reinterpret_cast<T*>(DualListData.DataAllocate(Size));\n")
output_file.write("\t\tauto Op = reinterpret_cast<T*>(Data.Allocate(Size));\n")
output_file.write("\t\tmemset(Op, 0, Size);\n")
output_file.write("\t\tOp->Header.Op = T2;\n")
output_file.write("\t\treturn Op;\n")
@@ -295,25 +295,25 @@ def print_ir_allocator_helpers(ops, defines):
output_file.write("\ttemplate<class T, IROps T2>\n")
output_file.write("\tIRPair<T> AllocateOp() {\n")
output_file.write("\t\tsize_t Size = FEXCore::IR::GetSize(T2);\n")
output_file.write("\t\tauto Op = reinterpret_cast<T*>(DualListData.DataAllocate(Size));\n")
output_file.write("\t\tauto Op = reinterpret_cast<T*>(Data.Allocate(Size));\n")
output_file.write("\t\tmemset(Op, 0, Size);\n")
output_file.write("\t\tOp->Header.Op = T2;\n")
output_file.write("\t\treturn IRPair<T>{Op, CreateNode(&Op->Header)};\n")
output_file.write("\t}\n\n")
output_file.write("\tuint8_t GetOpSize(OrderedNode *Op) const {\n")
output_file.write("\t\tauto HeaderOp = Op->Header.Value.GetNode(DualListData.DataBegin());\n")
output_file.write("\t\tauto HeaderOp = Op->Header.Value.GetNode(Data.Begin());\n")
output_file.write("\t\treturn HeaderOp->Size;\n")
output_file.write("\t}\n\n")
output_file.write("\tuint8_t GetOpElements(OrderedNode *Op) const {\n")
output_file.write("\t\tauto HeaderOp = Op->Header.Value.GetNode(DualListData.DataBegin());\n")
output_file.write("\t\tLOGMAN_THROW_A(HeaderOp->HasDest, \"Op %s has no dest\\n\", GetName(HeaderOp->Op));\n")
output_file.write("\t\tauto HeaderOp = Op->Header.Value.GetNode(Data.Begin());\n")
output_file.write("\t\tLogMan::Throw::A(HeaderOp->HasDest, \"Op %s has no dest\\n\", GetName(HeaderOp->Op));\n")
output_file.write("\t\treturn HeaderOp->Size / HeaderOp->ElementSize;\n")
output_file.write("\t}\n\n")
output_file.write("\tbool OpHasDest(OrderedNode *Op) const {\n")
output_file.write("\t\tauto HeaderOp = Op->Header.Value.GetNode(DualListData.DataBegin());\n")
output_file.write("\t\tauto HeaderOp = Op->Header.Value.GetNode(Data.Begin());\n")
output_file.write("\t\treturn HeaderOp->HasDest;\n")
output_file.write("\t}\n\n")
@@ -387,14 +387,11 @@ def print_ir_allocator_helpers(ops, defines):
output_file.write(") {\n")
output_file.write("\t\tauto Op = AllocateOp<IROp_%s, IROps::OP_%s>();\n" % (op_key, op_key.upper()))
if (SSAArgs != 0):
output_file.write("\t\tauto ListDataBegin = DualListData.ListBegin();\n")
for i in range(0, SSAArgs):
output_file.write("\t\tOp.first->Header.Args[%d] = ssa%d->Wrapped(ListDataBegin);\n" % (i, i))
output_file.write("\t\tOp.first->Header.NumArgs = %d;\n" % (SSAArgs))
if (SSAArgs != 0):
for i in range(0, SSAArgs):
output_file.write("\t\tOp.first->Header.Args[%d] = ssa%d->Wrapped(ListData.Begin());\n" % (i, i))
output_file.write("\t\tssa%d->AddUse();\n" % (i))
if (HasArgs):
@@ -402,6 +399,11 @@ def print_ir_allocator_helpers(ops, defines):
data_name = op_vals["Args"][i]
output_file.write("\t\tOp.first->%s = %s;\n" % (data_name, data_name))
if (HasFixedDestSize):
output_file.write("\t\tOp.first->Header.Size = %d;\n" % FixedDestSize)
if (HasDestSize):
output_file.write("\t\tOp.first->Header.Size = %s;\n" % DestSize)
if (HasDest):
# We can only infer a size if we have arguments
if not (HasFixedDestSize or HasDestSize):
@@ -410,18 +412,10 @@ def print_ir_allocator_helpers(ops, defines):
if (SSAArgs != 0):
for i in range(0, SSAArgs):
output_file.write("\t\tuint8_t Size%d = GetOpSize(ssa%s);\n" % (i, i))
for i in range(0, SSAArgs):
output_file.write("\t\tInferSize = std::max(InferSize, Size%d);\n" % (i))
output_file.write("\t\tOp.first->Header.Size = InferSize;\n")
output_file.write("\t\tOp.first->Header.NumArgs = %d;\n" % (SSAArgs))
if (HasFixedDestSize):
output_file.write("\t\tOp.first->Header.Size = %d;\n" % FixedDestSize)
if (HasDestSize):
output_file.write("\t\tOp.first->Header.Size = %s;\n" % DestSize)
output_file.write("\t\tOp.first->Header.ElementSize = Op.first->Header.Size / (%s);\n" % NumElements)
if (HasDest):
@@ -505,7 +499,7 @@ def print_ir_parser_allocator_helpers(ops, defines):
if (SSAArgs != 0):
for i in range(0, SSAArgs):
output_file.write("\t\tOp.first->Header.Args[%d] = ssa%d->Wrapped(DualListData.ListBegin());\n" % (i, i))
output_file.write("\t\tOp.first->Header.Args[%d] = ssa%d->Wrapped(ListData.Begin());\n" % (i, i))
output_file.write("\t\tssa%d->AddUse();\n" % (i))
if (HasArgs):
+4 -23
View File
@@ -114,7 +114,6 @@ set (SRCS
Interface/IR/Passes/DeadContextStoreElimination.cpp
Interface/IR/Passes/IRCompaction.cpp
Interface/IR/Passes/IRValidation.cpp
Interface/IR/Passes/LongDivideRemovalPass.cpp
Interface/IR/Passes/ValueDominanceValidation.cpp
Interface/IR/Passes/PhiValidation.cpp
Interface/IR/Passes/RedundantFlagCalculationElimination.cpp
@@ -122,8 +121,8 @@ set (SRCS
Interface/IR/Passes/StaticRegisterAllocationPass.cpp
Interface/IR/Passes/RegisterAllocationPass.cpp
Interface/IR/Passes/SyscallOptimization.cpp
Utils/Allocator.cpp
Utils/Allocator/64BitAllocator.cpp
Utils/ELFLoader.cpp
Utils/ELFSymbolDatabase.cpp
Utils/LogManager.cpp
Utils/Threads.cpp
)
@@ -265,11 +264,8 @@ function(AddObject Name Type)
add_dependencies(${Name} IR_INC)
add_dependencies(${Name} CONFIG_INC)
target_link_libraries(${Name} pthread vixl dl fmt::fmt xxhash FEX_jemalloc)
target_link_libraries(${Name} pthread rt vixl ${LINUX_LIBS} dl)
set_target_properties(${Name} PROPERTIES OUTPUT_NAME FEXCore)
set_target_properties(${Name} PROPERTIES C_VISIBILITY_PRESET hidden)
set_target_properties(${Name} PROPERTIES CXX_VISIBILITY_PRESET hidden)
set_target_properties(${Name} PROPERTIES VISIBILITY_INLINES_HIDDEN TRUE)
target_include_directories(${Name} PUBLIC "${CMAKE_CURRENT_BINARY_DIR}")
@@ -284,12 +280,9 @@ function(AddObject Name Type)
target_compile_options(${Name}
PRIVATE
-Wall
-Werror=cast-qual
-Werror=ignored-qualifiers
-Werror=implicit-fallthrough
-Wno-trigraphs
-ffunction-sections
)
if (GCC_COLOR)
@@ -306,24 +299,12 @@ endfunction()
function(AddLibrary Name Type)
add_library(${Name} ${Type} $<TARGET_OBJECTS:${PROJECT_NAME}_object>)
target_link_libraries(${Name} pthread vixl dl fmt::fmt xxhash FEX_jemalloc)
target_link_libraries(${Name} pthread rt vixl ${LINUX_LIBS} dl)
set_target_properties(${Name} PROPERTIES OUTPUT_NAME FEXCore)
set_target_properties(${Name} PROPERTIES C_VISIBILITY_PRESET hidden)
set_target_properties(${Name} PROPERTIES CXX_VISIBILITY_PRESET hidden)
set_target_properties(${Name} PROPERTIES VISIBILITY_INLINES_HIDDEN TRUE)
target_include_directories(${Name} PUBLIC "${CMAKE_CURRENT_BINARY_DIR}")
target_include_directories(${Name} PUBLIC "${PROJECT_SOURCE_DIR}/include/")
target_include_directories(${Name} PUBLIC "${CMAKE_BINARY_DIR}/include/")
if (CMAKE_BUILD_TYPE MATCHES "RELEASE")
target_link_options(${Name}
PRIVATE
"LINKER:--gc-sections"
"LINKER:--strip-all"
"LINKER:--as-needed"
)
endif()
endfunction()
AddObject(${PROJECT_NAME}_object OBJECT)
+6 -7
View File
@@ -1,6 +1,5 @@
#pragma once
#include "Common/MathUtils.h"
#include <FEXCore/Utils/Allocator.h>
#include <FEXCore/Utils/LogManager.h>
#include <cstring>
@@ -17,16 +16,16 @@ struct BitSet final {
ElementType *Memory;
void Allocate(size_t Elements) {
size_t AllocateSize = AlignUp(Elements, MinimumSizeBits) / MinimumSize;
LOGMAN_THROW_A((AllocateSize * MinimumSize) >= Elements, "Fail");
Memory = static_cast<ElementType*>(FEXCore::Allocator::malloc(AllocateSize));
LogMan::Throw::A((AllocateSize * MinimumSize) >= Elements, "Fail");
Memory = static_cast<ElementType*>(malloc(AllocateSize));
}
void Realloc(size_t Elements) {
size_t AllocateSize = AlignUp(Elements, MinimumSizeBits) / MinimumSize;
LOGMAN_THROW_A((AllocateSize * MinimumSize) >= Elements, "Fail");
Memory = static_cast<ElementType*>(FEXCore::Allocator::realloc(Memory, AllocateSize));
LogMan::Throw::A((AllocateSize * MinimumSize) >= Elements, "Fail");
Memory = static_cast<ElementType*>(realloc(Memory, AllocateSize));
}
void Free() {
FEXCore::Allocator::free(Memory);
free(Memory);
Memory = nullptr;
}
bool Get(T Element) {
@@ -61,7 +60,7 @@ struct BitSetView final {
ElementType *Memory;
void GetView(BitSet<T> &Set, uint64_t ElementOffset) {
LOGMAN_THROW_A((ElementOffset % MinimumSize) == 0,
LogMan::Throw::A((ElementOffset % MinimumSize) == 0,
"Bitset view offset needs to be aligned to size of backing element");
Memory = &Set.Memory[ElementOffset / MinimumSizeBits];
}
+11 -19
View File
@@ -6,13 +6,10 @@
#include <sys/stat.h>
namespace FEXCore::Paths {
std::unique_ptr<std::string> CachePath;
std::unique_ptr<std::string> EntryCache;
std::string CachePath;
std::string EntryCache;
void InitializePaths() {
CachePath = std::make_unique<std::string>();
EntryCache = std::make_unique<std::string>();
char const *HomeDir = getenv("HOME");
if (!HomeDir) {
@@ -25,34 +22,29 @@ namespace FEXCore::Paths {
char *XDGDataDir = getenv("XDG_DATA_DIR");
if (XDGDataDir) {
*CachePath = XDGDataDir;
CachePath = XDGDataDir;
}
else {
if (HomeDir) {
*CachePath = HomeDir;
CachePath = HomeDir;
}
}
*CachePath += "/.fex-emu/";
*EntryCache = *CachePath + "/EntryCache/";
CachePath += "/.fex-emu/";
EntryCache = CachePath + "/EntryCache/";
// Ensure the folder structure is created for our Data
if (!std::filesystem::exists(*EntryCache) &&
!std::filesystem::create_directories(*EntryCache)) {
LogMan::Msg::D("Couldn't create EntryCache directory: '%s'", EntryCache->c_str());
if (!std::filesystem::exists(EntryCache) &&
!std::filesystem::create_directories(EntryCache)) {
LogMan::Msg::D("Couldn't create EntryCache directory: '%s'", EntryCache.c_str());
}
}
void ShutdownPaths() {
CachePath.reset();
EntryCache.reset();
}
std::string GetCachePath() {
return *CachePath;
return CachePath;
}
std::string GetEntryCachePath() {
return *EntryCache;
return EntryCache;
}
}
-1
View File
@@ -3,7 +3,6 @@
namespace FEXCore::Paths {
void InitializePaths();
void ShutdownPaths();
std::string GetCachePath();
std::string GetEntryCachePath();
}
+11 -13
View File
@@ -1,6 +1,4 @@
#pragma once
#include <FEXCore/Utils/BitUtils.h>
#include <FEXCore/Utils/LogManager.h>
#include <cmath>
@@ -160,18 +158,18 @@ struct X80SoftFloat {
}
operator float() const {
const float32_t Result = extF80_to_f32(*this);
return FEXCore::BitCast<float>(Result);
float32_t Result = extF80_to_f32(*this);
return *(float*)&Result;
}
operator double() const {
const float64_t Result = extF80_to_f64(*this);
return FEXCore::BitCast<double>(Result);
float64_t Result = extF80_to_f64(*this);
return *(double*)&Result;
}
operator BIGFLOAT() const {
const float128_t Result = extF80_to_f128(*this);
return FEXCore::BitCast<BIGFLOAT>(Result);
float128_t Result = extF80_to_f128(*this);
return *(BIGFLOAT*)&Result;
}
operator int16_t() const {
@@ -198,11 +196,11 @@ struct X80SoftFloat {
}
void operator=(const float rhs) {
*this = f32_to_extF80(FEXCore::BitCast<float32_t>(rhs));
*this = f32_to_extF80(*(float32_t*)&rhs);
}
void operator=(const double rhs) {
*this = f64_to_extF80(FEXCore::BitCast<float64_t>(rhs));
*this = f64_to_extF80(*(float64_t*)&rhs);
}
void operator=(const int16_t rhs) {
@@ -228,15 +226,15 @@ struct X80SoftFloat {
}
X80SoftFloat(const float rhs) {
*this = f32_to_extF80(FEXCore::BitCast<float32_t>(rhs));
*this = f32_to_extF80(*(float32_t*)&rhs);
}
X80SoftFloat(const double rhs) {
*this = f64_to_extF80(FEXCore::BitCast<float64_t>(rhs));
*this = f64_to_extF80(*(float64_t*)&rhs);
}
X80SoftFloat(BIGFLOAT rhs) {
*this = f128_to_extF80(FEXCore::BitCast<float128_t>(rhs));
*this = f128_to_extF80(*(float128_t*)&rhs);
}
X80SoftFloat(const int16_t rhs) {
+8 -46
View File
@@ -48,15 +48,8 @@ namespace FEXCore::Config {
else {
char const *HomeDir = GetHomeDirectory();
char const *ConfigXDG = getenv("XDG_CONFIG_HOME");
char const *ConfigOverride = getenv("FEX_APP_CONFIG_LOCATION");
if (ConfigOverride) {
// Config override completely overrides the config directory
ConfigDir = ConfigOverride;
}
else {
ConfigDir = ConfigXDG ? ConfigXDG : HomeDir;
ConfigDir += "/.fex-emu/";
}
ConfigDir = ConfigXDG ? ConfigXDG : HomeDir;
ConfigDir += "/.fex-emu/";
// Ensure the folder structure is created for our configuration
if (!std::filesystem::exists(ConfigDir) &&
@@ -71,40 +64,21 @@ namespace FEXCore::Config {
}
std::string GetConfigFileLocation() {
std::string ConfigFile{};
const char *AppConfig = getenv("FEX_APP_CONFIG");
if (AppConfig) {
// App config environment variable overwrites only the config file
ConfigFile = AppConfig;
}
else {
ConfigFile = GetConfigDirectory(false) + "Config.json";
}
std::string ConfigFile = GetConfigDirectory(false) + "Config.json";
return ConfigFile;
}
std::string GetApplicationConfig(const std::string &Filename, bool Global) {
std::string GetApplicationConfig(std::string &Filename, bool Global) {
std::string ConfigFile = GetConfigDirectory(Global);
if (!Global &&
!std::filesystem::exists(ConfigFile) &&
!std::filesystem::create_directories(ConfigFile)) {
LogMan::Msg::D("Couldn't create config directory: '%s'", ConfigFile.c_str());
// Let's go local in this case
return "./" + Filename + ".json";
return "./";
}
ConfigFile += "AppConfig/";
// Attempt to create the local folder if it doesn't exist
if (!Global &&
!std::filesystem::exists(ConfigFile) &&
!std::filesystem::create_directories(ConfigFile)) {
LogMan::Msg::D("Couldn't create AppConfig directory: '%s'", ConfigFile.c_str());
// Let's go local in this case
return "./" + Filename + ".json";
}
ConfigFile += Filename + ".json";
ConfigFile += "AppConfig/" + Filename + ".json";
return ConfigFile;
}
@@ -113,15 +87,8 @@ namespace FEXCore::Config {
char const *HomeDir = GetHomeDirectory();
char const *DataXDG = getenv("XDG_DATA_HOME");
char const *DataOverride = getenv("FEX_APP_DATA_LOCATION");
if (DataOverride) {
// Data override will override the complete directory
DataDir = DataOverride;
}
else {
DataDir = DataXDG ?: HomeDir;
DataDir += "/.fex-emu/";
}
DataDir = DataXDG ?: HomeDir;
DataDir += "/.fex-emu/";
return DataDir;
}
@@ -359,10 +326,6 @@ namespace FEXCore::Config {
Meta->Set(Option, Data);
}
void Erase(ConfigOption Option) {
Meta->Erase(Option);
}
void EraseSet(ConfigOption Option, std::string Data) {
Meta->EraseSet(Option, Data);
}
@@ -426,6 +389,5 @@ namespace FEXCore::Config {
*List = **Value;
}
}
template void Value<std::string>::GetListIfExists(FEXCore::Config::ConfigOption Option, std::list<std::string> *List);
}
+1 -28
View File
@@ -129,18 +129,7 @@
"Desc": [
"Disables optimizations passes for debugging."
]
},
"Force32BitAllocator": {
"Type": "bool",
"Default": "false",
"Desc": [
"Forces use of the 32-bit allocator on 32-bit applications",
"Used to work around ulimit problems of CI runner",
"Potentially useful for debugging memory problems",
"32-bit allocator is always used if your host kernel is older than 4.17"
]
}
},
"Logging": {
"SilentLog": {
@@ -153,7 +142,7 @@
},
"OutputLog": {
"Type": "str",
"Default": "stderr",
"Default": "stdout",
"ShortArg": "o",
"Desc": [
"File to write FEX output to.",
@@ -199,14 +188,6 @@
"Removes the calculation of the parity flag from GPR instructions.",
"Assuming no uses rely on it"
]
},
"ParanoidTSO": {
"Type": "bool",
"Default": "false",
"Desc": [
"Makes TSO operations even more strict.",
"Forces vector loadstores to also become atomic."
]
}
},
"Misc": {
@@ -218,14 +199,6 @@
"Captures both the loaded executable and libraries it loads."
]
},
"AOTIRGenerate": {
"Type": "bool",
"Default": "false",
"Desc": [
"Scans file for executable code and generates an AOT IR cache.",
"Does not run the executable."
]
},
"AOTIRLoad": {
"Type": "bool",
"Default": "false",
+11 -25
View File
@@ -14,10 +14,6 @@ namespace FEXCore::Context {
IR::InstallOpcodeHandlers(Mode);
}
void ShutdownStaticTables() {
FEXCore::Paths::ShutdownPaths();
}
FEXCore::Context::Context *CreateNewContext() {
return new FEXCore::Context::Context{};
}
@@ -37,11 +33,12 @@ namespace FEXCore::Context {
return CTX->InitCore(Loader);
}
void SetExitHandler(FEXCore::Context::Context *CTX, ExitHandler handler) {
CTX->CustomExitHandler = std::move(handler);
void SetExitHandler(FEXCore::Context::Context *CTX,
std::function<void(uint64_t ThreadId, FEXCore::Context::ExitReason)> handler) {
CTX->CustomExitHandler = handler;
}
ExitHandler GetExitHandler(FEXCore::Context::Context *CTX) {
std::function<void(uint64_t ThreadId, FEXCore::Context::ExitReason)> GetExitHandler(FEXCore::Context::Context *CTX) {
return CTX->CustomExitHandler;
}
@@ -53,9 +50,6 @@ namespace FEXCore::Context {
CTX->Step();
}
void CompileRIP(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP) {
Thread->CTX->CompileBlock(Thread->CurrentFrame, GuestRIP);
}
FEXCore::Context::ExitReason RunUntilExit(FEXCore::Context::Context *CTX) {
return CTX->RunUntilExit();
@@ -104,12 +98,12 @@ namespace FEXCore::Context {
CTX->HandleCallback(RIP);
}
void RegisterHostSignalHandler(FEXCore::Context::Context *CTX, int Signal, HostSignalDelegatorFunction Func, bool Required) {
CTX->RegisterHostSignalHandler(Signal, Func, Required);
void RegisterHostSignalHandler(FEXCore::Context::Context *CTX, int Signal, HostSignalDelegatorFunction Func) {
CTX->RegisterHostSignalHandler(Signal, Func);
}
void RegisterFrontendHostSignalHandler(FEXCore::Context::Context *CTX, int Signal, HostSignalDelegatorFunction Func, bool Required) {
CTX->RegisterFrontendHostSignalHandler(Signal, Func, Required);
void RegisterFrontendHostSignalHandler(FEXCore::Context::Context *CTX, int Signal, HostSignalDelegatorFunction Func) {
CTX->RegisterFrontendHostSignalHandler(Signal, Func);
}
FEXCore::Core::InternalThreadState* CreateThread(FEXCore::Context::Context *CTX, FEXCore::Core::CPUState *NewThreadState, uint64_t ParentTID) {
@@ -148,20 +142,12 @@ namespace FEXCore::Context {
return CTX->CPUID.RunFunction(Function, Leaf);
}
void SetAOTIRLoader(FEXCore::Context::Context *CTX, std::function<int(const std::string&)> CacheReader) {
void SetAOTIRLoader(FEXCore::Context::Context *CTX, std::function<std::unique_ptr<std::istream>(const std::string&)> CacheReader) {
CTX->AOTIRLoader = CacheReader;
}
void SetAOTIRWriter(FEXCore::Context::Context *CTX, std::function<std::unique_ptr<std::ostream>(const std::string&)> CacheWriter) {
CTX->AOTIRWriter = CacheWriter;
}
void FinalizeAOTIRCache(FEXCore::Context::Context *CTX) {
CTX->FinalizeAOTIRCache();
}
void WriteFilesWithCode(FEXCore::Context::Context *CTX, std::function<void(const std::string& fileid, const std::string& filename)> Writer) {
CTX->WriteFilesWithCode(Writer);
bool WriteAOTIR(FEXCore::Context::Context *CTX, std::function<std::unique_ptr<std::ostream>(const std::string&)> CacheWriter) {
return CTX->WriteAOTIRCache(CacheWriter);
}
void AddNamedRegion(FEXCore::Context::Context *CTX, uintptr_t Base, uintptr_t Length, uintptr_t Offset, const std::string& Name) {
+30 -92
View File
@@ -1,5 +1,4 @@
#pragma once
#include "Common/JitSymbols.h"
#include "Interface/Core/CPUID.h"
#include "Interface/Core/Frontend.h"
@@ -10,7 +9,6 @@
#include "Interface/IR/Passes/RegisterAllocationPass.h"
#include <FEXCore/Config/Config.h>
#include <FEXCore/Core/CPUBackend.h>
#include <FEXCore/Utils/CompilerDefs.h>
#include <FEXCore/Utils/Event.h>
#include <stdint.h>
@@ -22,9 +20,7 @@
#include <optional>
#include <ostream>
#include <set>
#include <shared_mutex>
#include <unordered_map>
#include <queue>
namespace FEXCore {
class ThunkHandler;
@@ -56,38 +52,6 @@ namespace FEXCore::Context {
MODE_SINGLESTEP = 1,
};
struct AOTIRInlineEntry {
uint64_t GuestHash;
uint64_t GuestLength;
/* RAData followed by IRData */
uint8_t InlineData[0];
IR::RegisterAllocationData *GetRAData();
IR::IRListView *GetIRData();
};
struct AOTIRInlineIndexEntry {
uint64_t GuestStart;
uint64_t DataOffset;
};
struct AOTIRInlineIndex {
uint64_t Count;
uint64_t DataBase;
AOTIRInlineIndexEntry Entries[0];
AOTIRInlineEntry *Find(uint64_t GuestStart);
AOTIRInlineEntry *GetInlineEntry(uint64_t DataOffset);
};
struct AOTIRCaptureCacheEntry {
std::unique_ptr<std::ostream> Stream;
std::map<uint64_t, uint64_t> Index;
void AppendAOTIRCaptureCache(uint64_t GuestRIP, uint64_t Start, uint64_t Length, uint64_t Hash, FEXCore::IR::IRListView *IRList, FEXCore::IR::RegisterAllocationData *RAData);
};
struct Context {
friend class FEXCore::HLE::SyscallHandler;
#ifdef JIT_ARM64
@@ -114,7 +78,6 @@ namespace FEXCore::Context {
FEX_CONFIG_OPT(ABILocalFlags, ABILOCALFLAGS);
FEX_CONFIG_OPT(ABINoPF, ABINOPF);
FEX_CONFIG_OPT(AOTIRCapture, AOTIRCAPTURE);
FEX_CONFIG_OPT(AOTIRGenerate, AOTIRGENERATE);
FEX_CONFIG_OPT(AOTIRLoad, AOTIRLOAD);
FEX_CONFIG_OPT(SMCChecks, SMCCHECKS);
FEX_CONFIG_OPT(Core, CORE);
@@ -124,7 +87,7 @@ namespace FEXCore::Context {
FEX_CONFIG_OPT(DumpIR, DUMPIR);
} Config;
using IntCallbackReturn = FEX_NAKED void(*)(FEXCore::Core::InternalThreadState *Thread, volatile void *Host_RSP);
using IntCallbackReturn = __attribute__((naked)) void(*)(FEXCore::Core::InternalThreadState *Thread, volatile void *Host_RSP);
IntCallbackReturn InterpreterCallbackReturn;
FEXCore::HostFeatures HostFeatures;
@@ -147,31 +110,29 @@ namespace FEXCore::Context {
std::unique_ptr<FEXCore::ThunkHandler> ThunkHandler;
CustomCPUFactoryType CustomCPUFactory;
FEXCore::Context::ExitHandler CustomExitHandler;
std::function<void(uint64_t ThreadId, FEXCore::Context::ExitReason)> CustomExitHandler;
struct AOTIRCacheEntry {
AOTIRInlineIndex *Array;
void *mapping;
size_t size;
uint64_t start;
uint64_t len;
uint64_t crc;
IR::IRListView *IR;
IR::RegisterAllocationData *RAData;
};
std::unordered_map<std::string, AOTIRCacheEntry> AOTIRCache;
std::function<int(const std::string&)> AOTIRLoader;
std::function<std::unique_ptr<std::ostream>(const std::string&)> AOTIRWriter;
std::unordered_map<std::string, AOTIRCaptureCacheEntry> AOTIRCaptureCache;
std::function<std::unique_ptr<std::istream>(const std::string&)> AOTIRLoader;
std::unordered_map<std::string, std::map<uint64_t, AOTIRCacheEntry>> AOTIRCache;
struct AddrToFileEntry {
uint64_t Start;
uint64_t Len;
uint64_t Offset;
std::string fileid;
std::string filename;
void *CachedFileEntry;
bool ContainsCode;
};
std::map<uint64_t, AddrToFileEntry> AddrToFile;
std::map<std::string, std::string> FilesWithCode;
#ifdef BLOCKSTATS
std::unique_ptr<FEXCore::BlockSamplingData> BlockData;
@@ -185,7 +146,7 @@ namespace FEXCore::Context {
bool InitCore(FEXCore::CodeLoader *Loader);
FEXCore::Context::ExitReason RunUntilExit();
int GetProgramStatus() const;
int GetProgramStatus();
bool IsPaused() const { return !Running; }
void Pause();
void Run();
@@ -196,12 +157,12 @@ namespace FEXCore::Context {
void StopThread(FEXCore::Core::InternalThreadState *Thread);
void SignalThread(FEXCore::Core::InternalThreadState *Thread, FEXCore::Core::SignalEvent Event);
bool GetGdbServerStatus() const { return DebugServer != nullptr; }
bool GetGdbServerStatus() { return (bool)DebugServer; }
void StartGdbServer();
void StopGdbServer();
void HandleCallback(uint64_t RIP);
void RegisterHostSignalHandler(int Signal, HostSignalDelegatorFunction Func, bool Required);
void RegisterFrontendHostSignalHandler(int Signal, HostSignalDelegatorFunction Func, bool Required);
void RegisterHostSignalHandler(int Signal, HostSignalDelegatorFunction Func);
void RegisterFrontendHostSignalHandler(int Signal, HostSignalDelegatorFunction Func);
static void RemoveCodeEntry(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP);
@@ -217,37 +178,18 @@ namespace FEXCore::Context {
bool GetDebugDataForRIP(uint64_t RIP, FEXCore::Core::DebugData *Data);
bool FindHostCodeForRIP(uint64_t RIP, uint8_t **Code);
struct GenerateIRResult {
FEXCore::IR::IRListView* IRList;
// User's responsibility to deallocate this.
FEXCore::IR::RegisterAllocationData* RAData;
uint64_t TotalInstructions;
uint64_t TotalInstructionsLength;
uint64_t StartAddr;
uint64_t Length;
};
[[nodiscard]] GenerateIRResult GenerateIR(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP);
// XXX:
// bool FindIRForRIP(uint64_t RIP, FEXCore::IR::IntrusiveIRList **ir);
// void SetIRForRIP(uint64_t RIP, FEXCore::IR::IntrusiveIRList *const ir);
void LoadEntryList();
struct CompileCodeResult {
void* CompiledCode;
FEXCore::IR::IRListView* IRData;
FEXCore::Core::DebugData* DebugData;
// User's responsibility to deallocate this.
FEXCore::IR::RegisterAllocationData* RAData;
bool GeneratedIR;
uint64_t StartAddr;
uint64_t Length;
};
[[nodiscard]] CompileCodeResult CompileCode(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP);
std::tuple<FEXCore::IR::IRListView *, FEXCore::IR::RegisterAllocationData *, uint64_t, uint64_t, uint64_t, uint64_t> GenerateIR(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP);
std::tuple<void *, FEXCore::IR::IRListView *, FEXCore::Core::DebugData *, FEXCore::IR::RegisterAllocationData *, bool, uint64_t, uint64_t> CompileCode(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP);
uintptr_t CompileBlock(FEXCore::Core::CpuStateFrame *Frame, uint64_t GuestRIP);
// same as CompileBlock, but aborts on failure
void CompileBlockJit(FEXCore::Core::CpuStateFrame *Frame, uint64_t GuestRIP);
bool LoadAOTIRCache(int streamfd);
void FinalizeAOTIRCache();
void WriteFilesWithCode(std::function<void(const std::string& fileid, const std::string& filename)> Writer);
bool LoadAOTIRCache(std::istream &stream);
bool WriteAOTIRCache(std::function<std::unique_ptr<std::ostream>(const std::string&)> CacheWriter);
// Used for thread creation from syscalls
void InitializeCompiler(FEXCore::Core::InternalThreadState* State, bool CompileThread);
FEXCore::Core::InternalThreadState* CreateThread(FEXCore::Core::CPUState *NewThreadState, uint64_t ParentTID);
@@ -259,9 +201,7 @@ namespace FEXCore::Context {
void DestroyThread(FEXCore::Core::InternalThreadState *Thread);
void CleanupAfterFork(FEXCore::Core::InternalThreadState *ExceptForThread);
std::vector<FEXCore::Core::InternalThreadState*>* GetThreads() { return &Threads; }
uint8_t GetGPRSize() const { return Config.Is64BitMode ? 8 : 4; }
std::vector<FEXCore::Core::InternalThreadState*> *const GetThreads() { return &Threads; }
void AddNamedRegion(uintptr_t Base, uintptr_t Size, uintptr_t Offset, const std::string &filename);
void RemoveNamedRegion(uintptr_t Base, uintptr_t Size);
@@ -282,21 +222,19 @@ namespace FEXCore::Context {
void NotifyPause();
void AddBlockMapping(FEXCore::Core::InternalThreadState *Thread, uint64_t Address, void *Ptr, uint64_t Start, uint64_t Length);
FEXCore::CodeLoader *LocalLoader{};
// Entry Cache
std::optional<std::string> GetFilenameHash(std::string const &Filename) const;
void AddThreadRIPsToEntryList(FEXCore::Core::InternalThreadState *Thread);
void SaveEntryList();
std::set<uint64_t> EntryList;
std::vector<uint64_t> InitLocations;
uint64_t StartingRIP;
std::mutex ExitMutex;
std::unique_ptr<GdbServer> DebugServer;
std::shared_mutex AOTIRCacheLock;
std::shared_mutex AOTIRCaptureCacheWriteoutLock;
std::atomic<bool> AOTIRCaptureCacheWriteoutFlusing;
std::queue<std::function<void()>> AOTIRCaptureCacheWriteoutQueue;
void AOTIRCaptureCacheWriteoutQueue_Flush();
void AOTIRCaptureCacheWriteoutQueue_Append(const std::function<void()> &fn);
bool StartPaused = false;
FEX_CONFIG_OPT(AppFilename, APP_FILENAME);
};
+179 -237
View File
@@ -157,13 +157,6 @@ bool HandleCASPAL(void *_ucontext, void *_info, uint32_t Instr) {
mcontext->regs[ExpectedReg2] = FailedResult >> 32;
return true;
}
// This happens in the case that between Load and CAS that something has store our desired in to the memory location
// This means our CAS fails because what we wanted to store was already stored
uint64_t FailedResult = FailedResultOurBits >> (Alignment * 8);
mcontext->regs[ExpectedReg1] = FailedResult & ~0U;
mcontext->regs[ExpectedReg2] = FailedResult >> 32;
return true;
}
}
else {
@@ -207,18 +200,20 @@ bool HandleCASPAL(void *_ucontext, void *_info, uint32_t Instr) {
__uint128_t FailedResultOurBits = TmpExpected & Mask;
__uint128_t FailedResultNotOurBits = TmpExpected & NegMask;
__uint128_t FailedDesiredOurBits = TmpDesired & Mask;
__uint128_t FailedDesiredNotOurBits = TmpDesired & NegMask;
if ((FailedResultNotOurBits ^ FailedDesiredNotOurBits) != 0) {
// If the bits changed that weren't part of our regular CAS then we need to try again
continue;
}
// This happens in the case that between Load and CAS that something has store our desired in to the memory location
// This means our CAS fails because what we wanted to store was already stored
uint64_t FailedResult = FailedResultOurBits >> (Alignment * 8);
mcontext->regs[ExpectedReg1] = FailedResult & ~0U;
mcontext->regs[ExpectedReg2] = FailedResult >> 32;
return true;
if ((FailedResultOurBits ^ FailedDesiredOurBits) != 0) {
// If the bits changed that we were wanting to change then we have failed and can return
// We need to extract the bits and return them in EXPECTED
uint64_t FailedResult = FailedResultOurBits >> (Alignment * 8);
mcontext->regs[ExpectedReg1] = FailedResult & ~0U;
mcontext->regs[ExpectedReg2] = FailedResult >> 32;
return true;
}
}
}
}
@@ -226,19 +221,13 @@ bool HandleCASPAL(void *_ucontext, void *_info, uint32_t Instr) {
return false;
}
template <typename T>
using CASExpectedFn = T (*)(T Src, T Expected);
template <typename T>
using CASDesiredFn = T (*)(T Src, T Desired);
template<bool Retry>
static
uint16_t DoCAS16(
std::tuple<uint16_t, bool> DoCAS16(
uint16_t DesiredSrc,
uint16_t ExpectedSrc,
uint64_t Addr,
CASExpectedFn<uint16_t> ExpectedFunction,
CASDesiredFn<uint16_t> DesiredFunction) {
std::function<uint16_t(uint16_t SrcVal, uint16_t Expected)> ExpectedFunction,
std::function<uint16_t(uint16_t SrcVal, uint16_t Desired)> DesiredFunction) {
// 16 bit
uint64_t AlignmentMask = 0b1111;
if ((Addr & AlignmentMask) == 15) {
@@ -246,66 +235,49 @@ uint16_t DoCAS16(
// Need a dual 8bit CAS loop
uint64_t AddrUpper = Addr + 1;
while (1) {
uint8_t ActualUpper{};
uint8_t ActualLower{};
// Careful ordering here
ActualUpper = LoadAcquire8(AddrUpper);
ActualLower = LoadAcquire8(Addr);
uint8_t ActualUpper{};
uint8_t ActualLower{};
// Careful ordering here
ActualUpper = LoadAcquire8(AddrUpper);
ActualLower = LoadAcquire8(Addr);
uint16_t Actual = ActualUpper;
Actual <<= 8;
Actual |= ActualLower;
uint16_t Actual = ActualUpper;
Actual <<= 8;
Actual |= ActualLower;
uint16_t Desired = DesiredFunction(Actual, DesiredSrc);
uint8_t DesiredLower = Desired;
uint8_t DesiredUpper = Desired >> 8;
uint16_t Desired = DesiredFunction(Actual, DesiredSrc);
uint8_t DesiredLower = Desired;
uint8_t DesiredUpper = Desired >> 8;
uint16_t Expected = ExpectedFunction(Actual, ExpectedSrc);
uint8_t ExpectedLower = Expected;
uint8_t ExpectedUpper = Expected >> 8;
uint16_t Expected = ExpectedFunction(Actual, ExpectedSrc);
uint8_t ExpectedLower = Expected;
uint8_t ExpectedUpper = Expected >> 8;
bool Tear = false;
if (ActualUpper == ExpectedUpper &&
ActualLower == ExpectedLower) {
if (StoreCAS8(ExpectedUpper, DesiredUpper, AddrUpper)) {
if (StoreCAS8(ExpectedLower, DesiredLower, Addr)) {
// Stored successfully
return Expected;
}
else {
// CAS managed to tear, we can't really solve this
// Continue down the path to let the guest know values weren't expected
Tear = true;
}
}
ActualLower = ExpectedLower;
ActualUpper = ExpectedUpper;
}
// If the bits changed that we were wanting to change then we have failed and can return
// We need to extract the bits and return them in EXPECTED
uint16_t FailedResult = ActualUpper;
FailedResult <<= 8;
FailedResult |= ActualLower;
if constexpr (Retry) {
if (Tear) {
// If we are retrying and tearing then we can't do anything here
// XXX: Resolve with TME
return FailedResult;
if (ActualUpper == ExpectedUpper &&
ActualLower == ExpectedLower) {
if (StoreCAS8(ExpectedUpper, DesiredUpper, AddrUpper)) {
if (StoreCAS8(ExpectedLower, DesiredLower, Addr)) {
// Stored successfully
return std::make_tuple(Expected, true);
}
else {
// We can retry safely
// CAS managed to tear, we can't really solve this
// Continue down the path to let the guest know values weren't expected
}
}
else {
// Without Retry (CAS) then we have failed regardless of tear
// CAS failed but handled successfully
return FailedResult;
}
ActualLower = ExpectedLower;
ActualUpper = ExpectedUpper;
}
// If the bits changed that we were wanting to change then we have failed and can return
// We need to extract the bits and return them in EXPECTED
uint16_t FailedResult = ActualUpper;
FailedResult <<= 8;
FailedResult |= ActualLower;
// CAS failed but handled successfully
return std::make_tuple(FailedResult, false);
}
else {
AlignmentMask = 0b111;
@@ -344,30 +316,28 @@ uint16_t DoCAS16(
bool CASResult = Atomic128->compare_exchange_strong(TmpExpected, TmpDesired);
if (CASResult) {
// Successful, so we are done
return Expected >> (Alignment * 8);
return std::make_tuple(Expected >> (Alignment * 8), true);
}
else {
if constexpr (Retry) {
// If we failed but we have enabled retry then just retry without checking results
// CAS can't retry but atomic memory ops need to retry until passing
continue;
}
// Not successful
// Now we need to check the results to see if we need to try again
__uint128_t FailedResultOurBits = TmpExpected & Mask;
__uint128_t FailedResultNotOurBits = TmpExpected & NegMask;
__uint128_t FailedDesiredOurBits = TmpDesired & Mask;
__uint128_t FailedDesiredNotOurBits = TmpDesired & NegMask;
if ((FailedResultNotOurBits ^ FailedDesiredNotOurBits) != 0) {
// If the bits changed that weren't part of our regular CAS then we need to try again
continue;
}
// This happens in the case that between Load and CAS that something has store our desired in to the memory location
// This means our CAS fails because what we wanted to store was already stored
uint16_t FailedResult = FailedResultOurBits >> (Alignment * 8);
// CAS failed but handled successfully
return FailedResult;
if ((FailedResultOurBits ^ FailedDesiredOurBits) != 0) {
// If the bits changed that we were wanting to change then we have failed and can return
// We need to extract the bits and return them in EXPECTED
uint16_t FailedResult = FailedResultOurBits >> (Alignment * 8);
LogMan::Msg::D("Expected 0x%04x, Desired 0x%04x, Result 0x%04x", (uint16_t)(Expected >> (Alignment * 8)), DesiredSrc, FailedResult);
// CAS failed but handled successfully
return std::make_tuple(FailedResult, false);
}
}
}
}
@@ -409,31 +379,28 @@ uint16_t DoCAS16(
bool CASResult = Atomic->compare_exchange_strong(TmpExpected, TmpDesired);
if (CASResult) {
// Successful, so we are done
return Expected >> (Alignment * 8);
return std::make_tuple(Expected >> (Alignment * 8), true);
}
else {
if constexpr (Retry) {
// If we failed but we have enabled retry then just retry without checking results
// CAS can't retry but atomic memory ops need to retry until passing
continue;
}
// Not successful
// Now we need to check the results to see if we can try again
uint64_t FailedResultOurBits = TmpExpected & Mask;
uint64_t FailedResultNotOurBits = TmpExpected & NegMask;
uint64_t FailedDesiredOurBits = TmpDesired & Mask;
uint64_t FailedDesiredNotOurBits = TmpDesired & NegMask;
if ((FailedResultNotOurBits ^ FailedDesiredNotOurBits) != 0) {
// If the bits changed that weren't part of our regular CAS then we need to try again
continue;
}
// This happens in the case that between Load and CAS that something has store our desired in to the memory location
// This means our CAS fails because what we wanted to store was already stored
uint16_t FailedResult = FailedResultOurBits >> (Alignment * 8);
// CAS failed but handled successfully
return FailedResult;
if ((FailedResultOurBits ^ FailedDesiredOurBits) != 0) {
// If the bits changed that we were wanting to change then we have failed and can return
// We need to extract the bits and return them in EXPECTED
uint16_t FailedResult = FailedResultOurBits >> (Alignment * 8);
// CAS failed but handled successfully
return std::make_tuple(FailedResult, false);
}
}
}
}
@@ -475,31 +442,28 @@ uint16_t DoCAS16(
bool CASResult = Atomic->compare_exchange_strong(TmpExpected, TmpDesired);
if (CASResult) {
// Successful, so we are done
return Expected >> (Alignment * 8);
return std::make_tuple(Expected >> (Alignment * 8), true);
}
else {
if constexpr (Retry) {
// If we failed but we have enabled retry then just retry without checking results
// CAS can't retry but atomic memory ops need to retry until passing
continue;
}
// Not successful
// Now we need to check the results to see if we can try again
uint32_t FailedResultOurBits = TmpExpected & Mask;
uint32_t FailedResultNotOurBits = TmpExpected & NegMask;
uint32_t FailedDesiredOurBits = TmpDesired & Mask;
uint32_t FailedDesiredNotOurBits = TmpDesired & NegMask;
if ((FailedResultNotOurBits ^ FailedDesiredNotOurBits) != 0) {
// If the bits changed that weren't part of our regular CAS then we need to try again
continue;
}
// This happens in the case that between Load and CAS that something has store our desired in to the memory location
// This means our CAS fails because what we wanted to store was already stored
uint16_t FailedResult = FailedResultOurBits >> (Alignment * 8);
// CAS failed but handled successfully
return FailedResult;
if ((FailedResultOurBits ^ FailedDesiredOurBits) != 0) {
// If the bits changed that we were wanting to change then we have failed and can return
// We need to extract the bits and return them in EXPECTED
uint16_t FailedResult = FailedResultOurBits >> (Alignment * 8);
// CAS failed but handled successfully
return std::make_tuple(FailedResult, false);
}
}
}
}
@@ -507,14 +471,13 @@ uint16_t DoCAS16(
}
}
template<bool Retry>
static
uint32_t DoCAS32(
std::tuple<uint32_t, bool> DoCAS32(
uint32_t DesiredSrc,
uint32_t ExpectedSrc,
uint64_t Addr,
CASExpectedFn<uint32_t> ExpectedFunction,
CASDesiredFn<uint32_t> DesiredFunction) {
std::function<uint32_t(uint32_t SrcVal, uint32_t Expected)> ExpectedFunction,
std::function<uint32_t(uint32_t SrcVal, uint32_t Desired)> DesiredFunction) {
// 32 bit
uint64_t AlignmentMask = 0b1111;
if ((Addr & AlignmentMask) > 12) {
@@ -546,7 +509,6 @@ uint32_t DoCAS32(
TmpDesired &= NegMask;
TmpDesired |= Desired << (Alignment * 8);
bool Tear = false;
if (TmpExpected == TmpActual) {
uint32_t TmpExpectedLower = TmpExpected;
uint32_t TmpExpectedUpper = TmpExpected >> 32;
@@ -557,12 +519,11 @@ uint32_t DoCAS32(
if (StoreCAS32(TmpExpectedUpper, TmpDesiredUpper, AddrUpper)) {
if (StoreCAS32(TmpExpectedLower, TmpDesiredLower, Addr)) {
// Stored successfully
return Expected;
return std::make_tuple(Expected, true);
}
else {
// CAS managed to tear, we can't really solve this
// Continue down the path to let the guest know values weren't expected
Tear = true;
}
}
@@ -580,30 +541,18 @@ uint32_t DoCAS32(
uint64_t FailedResultOurBits = TmpExpected & Mask;
uint64_t FailedResultNotOurBits = TmpExpected & NegMask;
uint64_t FailedDesiredOurBits = TmpDesired & Mask;
uint64_t FailedDesiredNotOurBits = TmpDesired & NegMask;
if ((FailedResultNotOurBits ^ FailedDesiredNotOurBits) != 0) {
// If the bits changed that weren't part of our regular CAS then we need to try again
continue;
}
// This happens in the case that between Load and CAS that something has store our desired in to the memory location
// This means our CAS fails because what we wanted to store was already stored
uint32_t FailedResult = FailedResultOurBits >> (Alignment * 8);
if constexpr (Retry) {
if (Tear) {
// If we are retrying and tearing then we can't do anything here
// XXX: Resolve with TME
return FailedResult;
}
else {
// We can retry safely
}
}
else {
// Without Retry (CAS) then we have failed regardless of tear
if ((FailedResultOurBits ^ FailedDesiredOurBits) != 0) {
// If the bits changed that we were wanting to change then we have failed and can return
// We need to extract the bits and return them in EXPECTED
uint32_t FailedResult = FailedResultOurBits >> (Alignment * 8);
// CAS failed but handled successfully
return FailedResult;
return std::make_tuple(FailedResult, false);
}
}
}
@@ -642,31 +591,27 @@ uint32_t DoCAS32(
bool CASResult = Atomic128->compare_exchange_strong(TmpExpected, TmpDesired);
if (CASResult) {
// Stored successfully
return Expected;
return std::make_tuple(Expected, true);
}
else {
if constexpr (Retry) {
// If we failed but we have enabled retry then just retry without checking results
// CAS can't retry but atomic memory ops need to retry until passing
continue;
}
// Not successful
// Now we need to check the results to see if we need to try again
__uint128_t FailedResultOurBits = TmpExpected & Mask;
__uint128_t FailedResultNotOurBits = TmpExpected & NegMask;
__uint128_t FailedDesiredOurBits = TmpDesired & Mask;
__uint128_t FailedDesiredNotOurBits = TmpDesired & NegMask;
if ((FailedResultNotOurBits ^ FailedDesiredNotOurBits) != 0) {
// If the bits changed that weren't part of our regular CAS then we need to try again
continue;
}
// This happens in the case that between Load and CAS that something has store our desired in to the memory location
// This means our CAS fails because what we wanted to store was already stored
uint32_t FailedResult = FailedResultOurBits >> (Alignment * 8);
// CAS failed but handled successfully
return FailedResult;
if ((FailedResultOurBits ^ FailedDesiredOurBits) != 0) {
// If the bits changed that we were wanting to change then we have failed and can return
// We need to extract the bits and return them in EXPECTED
uint32_t FailedResult = FailedResultOurBits >> (Alignment * 8);
// CAS failed but handled successfully
return std::make_tuple(FailedResult, false);
}
}
}
}
@@ -705,46 +650,41 @@ uint32_t DoCAS32(
bool CASResult = Atomic->compare_exchange_strong(TmpExpected, TmpDesired);
if (CASResult) {
// Stored successfully
return Expected;
return std::make_tuple(Expected, true);
}
else {
if constexpr (Retry) {
// If we failed but we have enabled retry then just retry without checking results
// CAS can't retry but atomic memory ops need to retry until passing
continue;
}
// Not successful
// Now we need to check the results to see if we can try again
uint64_t FailedResultOurBits = TmpExpected & Mask;
uint64_t FailedResultNotOurBits = TmpExpected & NegMask;
uint64_t FailedDesiredOurBits = TmpDesired & Mask;
uint64_t FailedDesiredNotOurBits = TmpDesired & NegMask;
if ((FailedResultNotOurBits ^ FailedDesiredNotOurBits) != 0) {
// If the bits changed that weren't part of our regular CAS then we need to try again
continue;
}
// This happens in the case that between Load and CAS that something has store our desired in to the memory location
// This means our CAS fails because what we wanted to store was already stored
uint32_t FailedResult = FailedResultOurBits >> (Alignment * 8);
// CAS failed but handled successfully
return FailedResult;
if ((FailedResultOurBits ^ FailedDesiredOurBits) != 0) {
// If the bits changed that we were wanting to change then we have failed and can return
// We need to extract the bits and return them in EXPECTED
uint32_t FailedResult = FailedResultOurBits >> (Alignment * 8);
// CAS failed but handled successfully
return std::make_tuple(FailedResult, false);
}
}
}
}
}
}
template<bool Retry>
static
uint64_t DoCAS64(
std::tuple<uint64_t, bool> DoCAS64(
uint64_t DesiredSrc,
uint64_t ExpectedSrc,
uint64_t Addr,
CASExpectedFn<uint64_t> ExpectedFunction,
CASDesiredFn<uint64_t> DesiredFunction) {
std::function<uint64_t(uint64_t SrcVal, uint64_t Expected)> ExpectedFunction,
std::function<uint64_t(uint64_t SrcVal, uint64_t Desired)> DesiredFunction) {
// 64bit
uint64_t AlignmentMask = 0b1111;
if ((Addr & AlignmentMask) > 8) {
@@ -784,17 +724,15 @@ uint64_t DoCAS64(
uint64_t TmpDesiredLower = TmpDesired;
uint64_t TmpDesiredUpper = TmpDesired >> 64;
bool Tear = false;
if (TmpExpected == TmpActual) {
if (StoreCAS64(TmpExpectedUpper, TmpDesiredUpper, AddrUpper)) {
if (StoreCAS64(TmpExpectedLower, TmpDesiredLower, Addr)) {
// Stored successfully
return Expected;
return std::make_tuple(Expected, true);
}
else {
// CAS managed to tear, we can't really solve this
// Continue down the path to let the guest know values weren't expected
Tear = true;
}
}
@@ -812,30 +750,18 @@ uint64_t DoCAS64(
__uint128_t FailedResultOurBits = TmpExpected & Mask;
__uint128_t FailedResultNotOurBits = TmpExpected & NegMask;
__uint128_t FailedDesiredOurBits = TmpDesired & Mask;
__uint128_t FailedDesiredNotOurBits = TmpDesired & NegMask;
if ((FailedResultNotOurBits ^ FailedDesiredNotOurBits) != 0) {
// If the bits changed that weren't part of our regular CAS then we need to try again
continue;
}
// This happens in the case that between Load and CAS that something has store our desired in to the memory location
// This means our CAS fails because what we wanted to store was already stored
uint64_t FailedResult = FailedResultOurBits >> (Alignment * 8);
if constexpr (Retry) {
if (Tear) {
// If we are retrying and tearing then we can't do anything here
// XXX: Resolve with TME
return FailedResult;
}
else {
// We can retry safely
}
}
else {
// Without Retry (CAS) then we have failed regardless of tear
if ((FailedResultOurBits ^ FailedDesiredOurBits) != 0) {
// If the bits changed that we were wanting to change then we have failed and can return
// We need to extract the bits and return them in EXPECTED
uint64_t FailedResult = FailedResultOurBits >> (Alignment * 8);
// CAS failed but handled successfully
return FailedResult;
return std::make_tuple(FailedResult, false);
}
}
}
@@ -870,34 +796,35 @@ uint64_t DoCAS64(
bool CASResult = Atomic128->compare_exchange_strong(TmpExpected, TmpDesired);
if (CASResult) {
// Stored successfully
return Expected;
return std::make_tuple(Expected, true);
}
else {
if constexpr (Retry) {
// If we failed but we have enabled retry then just retry without checking results
// CAS can't retry but atomic memory ops need to retry until passing
continue;
}
// Not successful
// Now we need to check the results to see if we need to try again
__uint128_t FailedResultOurBits = TmpExpected & Mask;
__uint128_t FailedResultNotOurBits = TmpExpected & NegMask;
__uint128_t FailedDesiredOurBits = TmpDesired & Mask;
__uint128_t FailedDesiredNotOurBits = TmpDesired & NegMask;
if ((FailedResultNotOurBits ^ FailedDesiredNotOurBits) != 0) {
// If the bits changed that weren't part of our regular CAS then we need to try again
continue;
}
if ((FailedResultOurBits ^ FailedDesiredOurBits) != 0) {
// If the bits changed that we were wanting to change then we have failed and can return
// We need to extract the bits and return them in EXPECTED
uint64_t FailedResult = FailedResultOurBits >> (Alignment * 8);
// CAS failed but handled successfully
return std::make_tuple(FailedResult, false);
}
// This happens in the case that between Load and CAS that something has store our desired in to the memory location
// This means our CAS fails because what we wanted to store was already stored
uint64_t FailedResult = FailedResultOurBits >> (Alignment * 8);
// CAS failed but handled successfully
return FailedResult;
// If we got here, that means the CAS failed
// NotOurBits didn't change and bits we cared about didn't change
ERROR_AND_DIE("Impossible");
}
}
}
}
bool HandleCASAL(void *_ucontext, void *_info, uint32_t Instr) {
@@ -928,7 +855,7 @@ bool HandleCASAL(void *_ucontext, void *_info, uint32_t Instr) {
// 8bit can't be unaligned
// Only need to handle 16, 32, 64
if (Size == 2) {
auto Res = DoCAS16<false>(
auto Res = DoCAS16(
mcontext->regs[DesiredReg],
mcontext->regs[ExpectedReg],
Addr,
@@ -944,12 +871,12 @@ bool HandleCASAL(void *_ucontext, void *_info, uint32_t Instr) {
// Regardless of pass or fail
// We set the result register if it isn't a zero register
if (ExpectedReg != 31) {
mcontext->regs[ExpectedReg] = Res;
mcontext->regs[ExpectedReg] = std::get<0>(Res);
}
return true;
}
else if (Size == 4) {
auto Res = DoCAS32<false>(
auto Res = DoCAS32(
mcontext->regs[DesiredReg],
mcontext->regs[ExpectedReg],
Addr,
@@ -965,12 +892,12 @@ bool HandleCASAL(void *_ucontext, void *_info, uint32_t Instr) {
// Regardless of pass or fail
// We set the result register if it isn't a zero register
if (ExpectedReg != 31) {
mcontext->regs[ExpectedReg] = Res;
mcontext->regs[ExpectedReg] = std::get<0>(Res);
}
return true;
}
else if (Size == 8) {
auto Res = DoCAS64<false>(
auto Res = DoCAS64(
mcontext->regs[DesiredReg],
mcontext->regs[ExpectedReg],
Addr,
@@ -986,7 +913,7 @@ bool HandleCASAL(void *_ucontext, void *_info, uint32_t Instr) {
// Regardless of pass or fail
// We set the result register if it isn't a zero register
if (ExpectedReg != 31) {
mcontext->regs[ExpectedReg] = Res;
mcontext->regs[ExpectedReg] = std::get<0>(Res);
}
return true;
}
@@ -1037,7 +964,7 @@ bool HandleAtomicMemOp(void *_ucontext, void *_info, uint32_t Instr) {
return Desired;
};
CASDesiredFn<uint16_t> DesiredFunction{};
std::function<uint16_t(uint16_t SrcVal, uint16_t Desired)> DesiredFunction;
switch (Op) {
case ATOMIC_ADD_OP:
@@ -1061,16 +988,21 @@ bool HandleAtomicMemOp(void *_ucontext, void *_info, uint32_t Instr) {
break;
}
auto Res = DoCAS16<true>(
mcontext->regs[SourceReg],
0, // Unused
Addr,
NOPExpected,
DesiredFunction);
// If we passed and our destination register is not zero
// Then we need to update the result register with what was in memory
if (ResultReg != 31) {
mcontext->regs[ResultReg] = Res;
bool Passed = false;
while (!Passed) {
auto Res = DoCAS16(
mcontext->regs[SourceReg],
0, // Unused
Addr,
NOPExpected,
DesiredFunction);
Passed = std::get<1>(Res);
// If we passed and our destination register is not zero
// Then we need to update the result register with what was in memory
if (Passed &&
ResultReg != 31) {
mcontext->regs[ResultReg] = std::get<0>(Res);
}
}
return true;
}
@@ -1099,7 +1031,7 @@ bool HandleAtomicMemOp(void *_ucontext, void *_info, uint32_t Instr) {
return Desired;
};
CASDesiredFn<uint32_t> DesiredFunction{};
std::function<uint32_t(uint32_t SrcVal, uint32_t Desired)> DesiredFunction;
switch (Op) {
case ATOMIC_ADD_OP:
@@ -1123,16 +1055,21 @@ bool HandleAtomicMemOp(void *_ucontext, void *_info, uint32_t Instr) {
break;
}
auto Res = DoCAS32<true>(
mcontext->regs[SourceReg],
0, // Unused
Addr,
NOPExpected,
DesiredFunction);
// If we passed and our destination register is not zero
// Then we need to update the result register with what was in memory
if (ResultReg != 31) {
mcontext->regs[ResultReg] = Res;
bool Passed = false;
while (!Passed) {
auto Res = DoCAS32(
mcontext->regs[SourceReg],
0, // Unused
Addr,
NOPExpected,
DesiredFunction);
Passed = std::get<1>(Res);
// If we passed and our destination register is not zero
// Then we need to update the result register with what was in memory
if (Passed &&
ResultReg != 31) {
mcontext->regs[ResultReg] = std::get<0>(Res);
}
}
return true;
}
@@ -1161,7 +1098,7 @@ bool HandleAtomicMemOp(void *_ucontext, void *_info, uint32_t Instr) {
return Desired;
};
CASDesiredFn<uint64_t> DesiredFunction{};
std::function<uint64_t(uint64_t SrcVal, uint64_t Desired)> DesiredFunction;
switch (Op) {
case ATOMIC_ADD_OP:
@@ -1185,16 +1122,21 @@ bool HandleAtomicMemOp(void *_ucontext, void *_info, uint32_t Instr) {
break;
}
auto Res = DoCAS64<true>(
mcontext->regs[SourceReg],
0, // Unused
Addr,
NOPExpected,
DesiredFunction);
// If we passed and our destination register is not zero
// Then we need to update the result register with what was in memory
if (ResultReg != 31) {
mcontext->regs[ResultReg] = Res;
bool Passed = false;
while (!Passed) {
auto Res = DoCAS64(
mcontext->regs[SourceReg],
0, // Unused
Addr,
NOPExpected,
DesiredFunction);
Passed = std::get<1>(Res);
// If we passed and our destination register is not zero
// Then we need to update the result register with what was in memory
if (Passed &&
ResultReg != 31) {
mcontext->regs[ResultReg] = std::get<0>(Res);
}
}
return true;
}
@@ -12,12 +12,6 @@ namespace FEXCore::ArchHelpers::Arm64 {
constexpr uint32_t ATOMIC_MEM_MASK = 0x3B200C00;
constexpr uint32_t ATOMIC_MEM_INST = 0x38200000;
constexpr uint32_t LDAXP_MASK = 0xBF'FF'80'00;
constexpr uint32_t LDAXP_INST = 0x88'7F'80'00;
constexpr uint32_t STLXP_MASK = 0xBF'E0'80'00;
constexpr uint32_t STLXP_INST = 0x88'20'80'00;
// Load ops are 4 bits
// Acquire and release bits are independent on the instruction
constexpr uint32_t ATOMIC_ADD_OP = 0b0000;
@@ -29,17 +29,6 @@ Arm64Emitter::Arm64Emitter(size_t size) : vixl::aarch64::Assembler(size, vixl::a
if (!SupportsAtomics) {
WARN_ONCE("Host CPU doesn't support atomics. Expect bad performance");
}
#ifdef _M_ARM_64
// We need to get the CPU's cache line size
// We expect sane targets that have correct cacheline sizes across clusters
uint64_t CTR;
__asm volatile ("mrs %[ctr], ctr_el0"
: [ctr] "=r"(CTR));
DCacheLineSize = 4 << ((CTR >> 16) & 0xF);
ICacheLineSize = 4 << (CTR & 0xF);
#endif
}
void Arm64Emitter::LoadConstant(vixl::aarch64::Register Reg, uint64_t Constant) {
@@ -69,9 +69,6 @@ protected:
void Align16B();
uint32_t SpillSlots{};
uint32_t DCacheLineSize{};
uint32_t ICacheLineSize{};
};
}
@@ -111,7 +111,7 @@ static inline void BackupContext(void* ucontext, T *Backup) {
// Host FPR state starts at _mcontext->reserved[0];
HostFPRState *HostState = reinterpret_cast<HostFPRState*>(&_mcontext->__reserved[0]);
LOGMAN_THROW_A(HostState->Head.Magic == FPR_MAGIC, "Wrong FPR Magic: 0x%08x", HostState->Head.Magic);
LogMan::Throw::A(HostState->Head.Magic == FPR_MAGIC, "Wrong FPR Magic: 0x%08x", HostState->Head.Magic);
Backup->FPSR = HostState->FPSR;
Backup->FPCR = HostState->FPCR;
memcpy(&Backup->FPRs[0], &HostState->FPRs[0], 32 * sizeof(__uint128_t));
@@ -126,7 +126,7 @@ static inline void RestoreContext(void* ucontext, T *Backup) {
auto _mcontext = GetMContext(ucontext);
HostFPRState *HostState = reinterpret_cast<HostFPRState*>(&_mcontext->__reserved[0]);
LOGMAN_THROW_A(HostState->Head.Magic == FPR_MAGIC, "Wrong FPR Magic: 0x%08x", HostState->Head.Magic);
LogMan::Throw::A(HostState->Head.Magic == FPR_MAGIC, "Wrong FPR Magic: 0x%08x", HostState->Head.Magic);
memcpy(&HostState->FPRs[0], &Backup->FPRs[0], 32 * sizeof(__uint128_t));
HostState->FPCR = Backup->FPCR;
HostState->FPSR = Backup->FPSR;
+162 -486
View File
@@ -15,26 +15,7 @@ $end_info$
#endif
namespace FEXCore {
constexpr uint32_t SUPPORTS_AVX = 0;
// #define CPUID_AMD
#ifdef CPUID_AMD
constexpr uint32_t FAMILY_IDENTIFIER =
0 | // Stepping
(0xA << 4) | // Model
(0xF << 8) | // Family ID
(0 << 12) | // Processor type
(0 << 16) | // Extended model ID
(1 << 20); // Extended family ID
#else
constexpr uint32_t FAMILY_IDENTIFIER =
0 | // Stepping
(0x7 << 4) | // Model
(0x6 << 8) | // Family ID
(0 << 12) | // Processor type
(1 << 16) | // Extended model ID
(0x0 << 20); // Extended family ID
#endif
//#define CPUID_AMD
#ifdef _M_ARM_64
static uint32_t GetCycleCounterFrequency() {
uint64_t Result{};
@@ -57,7 +38,7 @@ static uint32_t GetCycleCounterFrequency() {
}
#endif
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_0h(uint32_t Leaf) {
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_0h() {
FEXCore::CPUID::FunctionResults Res{};
// EBX, EDX, ECX become the manufacturer id string
@@ -76,23 +57,25 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_0h(uint32_t Leaf) {
}
// Processor Info and Features bits
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_01h(uint32_t Leaf) {
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_01h() {
FEXCore::CPUID::FunctionResults Res{};
uint32_t CoreCount = Cores();
Res.eax = FAMILY_IDENTIFIER;
Res.eax = 0 | // Stepping
(0 << 4) | // Model
(0xF << 8) | // Family ID
(0 << 12) | // Processor type
(0 << 16) | // Extended model ID
(0 << 20); // Extended family ID
Res.ebx = 0 | // Brand index
(8 << 8) | // Cache line size in bytes
(CoreCount << 16) | // Number of addressable IDs for the logical cores in the physical CPU
(8 << 16) | // Number of addressable IDs for the logical cores in the physical CPU
(0 << 24); // Local APIC ID
Res.ecx =
(1 << 0) | // SSE3
(0 << 1) | // PCLMULQDQ
(1 << 2) | // DS area supports 64bit layout
(1 << 3) | // MWait
(0 << 4) | // DS-CPL
(1 << 4) | // DS-CPL
(0 << 5) | // VMX
(0 << 6) | // SMX
(0 << 7) | // Intel SpeedStep
@@ -106,8 +89,8 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_01h(uint32_t Leaf) {
(0 << 15) | // Perfmon and debug capability
(0 << 16) | // Reserved
(0 << 17) | // Process-context identifiers
(0 << 18) | // Prefetching from memory mapped device
(1 << 19) | // SSE4.1
(1 << 18) | // Prefetching from memory mapped device
(0 << 19) | // SSE4.1
(0 << 20) | // SSE4.2
(0 << 21) | // X2APIC
(1 << 22) | // MOVBE
@@ -116,7 +99,7 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_01h(uint32_t Leaf) {
(CTX->HostFeatures.SupportsAES << 25) | // AES
(0 << 26) | // XSAVE
(0 << 27) | // OSXSAVE
(SUPPORTS_AVX << 28) | // AVX
(0 << 28) | // AVX
(0 << 29) | // F16C
(0 << 30) | // RDRAND
(0 << 31); // Hypervisor always returns zero
@@ -141,7 +124,7 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_01h(uint32_t Leaf) {
(1 << 16) | // Page Attribute Table
(1 << 17) | // 36bit page size extension
(0 << 18) | // Processor serial number
(1 << 19) | // CLFLUSH
(0 << 19) | // CLFLUSH
(0 << 20) | // Reserved
(0 << 21) | // Debug store
(0 << 22) | // Thermal monitor and software controled clock
@@ -149,16 +132,16 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_01h(uint32_t Leaf) {
(1 << 24) | // FXSAVE/FXRSTOR
(1 << 25) | // SSE
(1 << 26) | // SSE2
(0 << 27) | // Self Snoop
(1 << 27) | // Self Snoop
(1 << 28) | // Max APIC IDs reserved field is valid
(0 << 29) | // Thermal monitor
(1 << 29) | // Thermal monitor
(0 << 30) | // Reserved
(0 << 31); // Pending break enable
(1 << 31); // Pending break enable
return Res;
}
// 2: Cache and TLB information
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_02h(uint32_t Leaf) {
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_02h() {
FEXCore::CPUID::FunctionResults Res{};
// returns default values from i7 model 1Ah
@@ -182,286 +165,124 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_02h(uint32_t Leaf) {
return Res;
}
// 4: Deterministic cache parameters for each level
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_04h(uint32_t Leaf) {
FEXCore::CPUID::FunctionResults Res{};
constexpr uint32_t CacheType_Data = 1;
constexpr uint32_t CacheType_Instruction = 2;
constexpr uint32_t CacheType_Unified = 3;
if (Leaf == 0) {
// Report L1D
uint32_t CoreCount = Cores() - 1;
Res.eax = CacheType_Data | // Cache type
(0b001 << 5) | // Cache level
(1 << 8) | // Self initializing cache level
(0 << 9) | // Fully associative
(0 << 14) | // Maximum number of addressable IDs for logical processors sharing this cache (With SMT this would be 1)
(CoreCount << 26); // Maximum number of addressable IDs for processor cores in the physical package
Res.ebx =
(63 << 0) | // Line Size - 1 : Claiming 64 byte
(0 << 12) | // Physical Line partitions
(7 << 22); // Associativity - 1 : Claiming 8 way
// 32KB
Res.ecx = 63; // Number of sets - 1 : Claiming 64 sets
Res.edx =
(0 << 0) | // Write-back invalidate
(0 << 1) | // Cache inclusiveness - Includes lower caches
(0 << 2); // Complex cache indexing - 0: Direct, 1: Complex
}
else if (Leaf == 1) {
// Report L1I
uint32_t CoreCount = Cores() - 1;
Res.eax = CacheType_Instruction | // Cache type
(0b001 << 5) | // Cache level
(1 << 8) | // Self initializing cache level
(0 << 9) | // Fully associative
(0 << 14) | // Maximum number of addressable IDs for logical processors sharing this cache (With SMT this would be 1)
(CoreCount << 26); // Maximum number of addressable IDs for processor cores in the physical package
Res.ebx =
(63 << 0) | // Line Size - 1 : Claiming 64 byte
(0 << 12) | // Physical Line partitions
(7 << 22); // Associativity - 1 : Claiming 8 way
// 32KB
Res.ecx = 63; // Number of sets - 1 : Claiming 64 sets
Res.edx =
(0 << 0) | // Write-back invalidate
(0 << 1) | // Cache inclusiveness - Includes lower caches
(0 << 2); // Complex cache indexing - 0: Direct, 1: Complex
}
else if (Leaf == 2) {
// Report L2
uint32_t CoreCount = Cores() - 1;
Res.eax = CacheType_Unified | // Cache type
(0b010 << 5) | // Cache level
(1 << 8) | // Self initializing cache level
(0 << 9) | // Fully associative
(0 << 14) | // Maximum number of addressable IDs for logical processors sharing this cache
(CoreCount << 26); // Maximum number of addressable IDs for processor cores in the physical package
Res.ebx =
(63 << 0) | // Line Size - 1 : Claiming 64 byte
(0 << 12) | // Physical Line partitions
(7 << 22); // Associativity - 1 : Claiming 8 way
// 512KB
Res.ecx = 0x3FF; // Number of sets - 1 : Claiming 1024 sets
Res.edx =
(0 << 0) | // Write-back invalidate
(0 << 1) | // Cache inclusiveness - Includes lower caches
(0 << 2); // Complex cache indexing - 0: Direct, 1: Complex
}
else if (Leaf == 3) {
// Report L3
uint32_t CoreCount = Cores() - 1;
Res.eax = CacheType_Unified | // Cache type
(0b011 << 5) | // Cache level
(1 << 8) | // Self initializing cache level
(0 << 9) | // Fully associative
(CoreCount << 14) | // Maximum number of addressable IDs for logical processors sharing this cache
(CoreCount << 26); // Maximum number of addressable IDs for processor cores in the physical package
Res.ebx =
(63 << 0) | // Line Size - 1 : Claiming 64 byte
(0 << 12) | // Physical Line partitions
(7 << 22); // Associativity - 1 : Claiming 8 way
// 8MB
Res.ecx = 0x4000; // Number of sets - 1 : Claiming 16384 sets
Res.edx =
(0 << 0) | // Write-back invalidate
(0 << 1) | // Cache inclusiveness - Includes lower caches
(1 << 2); // Complex cache indexing - 0: Direct, 1: Complex
}
return Res;
}
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_06h(uint32_t Leaf) {
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_06h() {
FEXCore::CPUID::FunctionResults Res{};
Res.eax = (1 << 2); // Always running APIC
Res.ecx = (0 << 3); // Intel performance energy bias preference (EPB)
return Res;
}
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_07h(uint32_t Leaf) {
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_07h() {
FEXCore::CPUID::FunctionResults Res{};
if (Leaf == 0) {
// Number of subfunctions
Res.eax = 0x0;
Res.ebx =
(1 << 0) | // FS/GS support
(0 << 1) | // TSC adjust MSR
(0 << 2) | // SGX
(0 << 3) | // BMI1
(0 << 4) | // Intel Hardware Lock Elison
(0 << 5) | // AVX2 support
(1 << 6) | // FPU data pointer updated only on exception
(1 << 7) | // SMEP support
(0 << 8) | // BMI2
(0 << 9) | // Enhanced REP MOVSB/STOSB
(1 << 10) | // INVPCID for system software control of process-context
(0 << 11) | // Restricted transactional memory
(0 << 12) | // Intel resource directory technology Monitoring
(1 << 13) | // Deprecates FPU CS and DS
(0 << 14) | // Intel MPX
(0 << 15) | // Intel Resource Directory Technology Allocation
(0 << 16) | // Reserved
(0 << 17) | // Reserved
(0 << 18) | // RDSEED
(0 << 19) | // ADCX and ADOX instructions
(0 << 20) | // SMAP Supervisor mode access prevention and CLAC/STAC instructions
(0 << 21) | // Reserved
(0 << 22) | // Reserved
(0 << 23) | // CLFLUSHOPT instruction
(0 << 24) | // CLWB instruction
(0 << 25) | // Intel processor trace
(0 << 26) | // Reserved
(0 << 27) | // Reserved
(0 << 28) | // Reserved
(0 << 29) | // SHA instructions
(0 << 30) | // Reserved
(0 << 31); // Reserved
Res.ecx =
(1 << 0) | // PREFETCHWT1
(0 << 1) | // AVX512VBMI
(0 << 2) | // Usermode instruction prevention
(0 << 3) | // Protection keys for user mode pages
(0 << 4) | // OS protection keys
(0 << 5) | // waitpkg
(0 << 6) | // AVX512_VBMI2
(0 << 7) | // CET shadow stack
(0 << 8) | // GFNI
(0 << 9) | // VAES
(0 << 10) | // VPCLMULQDQ
(0 << 11) | // AVX512_VNNI
(0 << 12) | // AVX512_BITALG
(0 << 13) | // Intel Total Memory Encryption
(0 << 14) | // AVX512_VPOPCNTDQ
(0 << 15) | // Reserved
(0 << 16) | // 5 Level page tables
(0 << 17) | // MPX MAWAU
(0 << 18) | // MPX MAWAU
(0 << 19) | // MPX MAWAU
(0 << 20) | // MPX MAWAU
(0 << 21) | // MPX MAWAU
(0 << 22) | // RDPID Read Processor ID
(0 << 23) | // Reserved
(0 << 24) | // Reserved
(0 << 25) | // CLDEMOTE
(0 << 26) | // Reserved
(0 << 27) | // MOVDIRI
(0 << 28) | // MOVDIR64B
(0 << 29) | // Reserved
(0 << 30) | // SGX Launch configuration
(0 << 31); // Reserved
// Number of subfunctions
Res.eax = 0x0;
Res.ebx =
(1 << 0) | // FS/GS support
(0 << 1) | // TSC adjust MSR
(0 << 2) | // SGX
(0 << 3) | // BMI1
(0 << 4) | // Intel Hardware Lock Elison
(0 << 5) | // AVX2 support
(1 << 6) | // FPU data pointer updated only on exception
(1 << 7) | // SMEP support
(0 << 8) | // BMI2
(0 << 9) | // Enhanced REP MOVSB/STOSB
(1 << 10) | // INVPCID for system software control of process-context
(0 << 11) | // Restricted transactional memory
(0 << 12) | // Intel resource directory technology Monitoring
(1 << 13) | // Deprecates FPU CS and DS
(0 << 14) | // Intel MPX
(0 << 15) | // Intel Resource Directory Technology Allocation
(0 << 16) | // Reserved
(0 << 17) | // Reserved
(0 << 18) | // RDSEED
(0 << 19) | // ADCX and ADOX instructions
(0 << 20) | // SMAP Supervisor mode access prevention and CLAC/STAC instructions
(0 << 21) | // Reserved
(0 << 22) | // Reserved
(0 << 23) | // CLFLUSHOPT instruction
(0 << 24) | // CLWB instruction
(0 << 25) | // Intel processor trace
(0 << 26) | // Reserved
(0 << 27) | // Reserved
(0 << 28) | // Reserved
(0 << 29) | // SHA instructions
(0 << 30) | // Reserved
(0 << 31); // Reserved
Res.edx =
(0 << 0) | // Reserved
(0 << 1) | // Reserved
(0 << 2) | // AVX512_4VNNIW
(0 << 3) | // AVX512_4FMAPS
(0 << 4) | // Fast Short Rep Mov
(0 << 5) | // Reserved
(0 << 6) | // Reserved
(0 << 7) | // Reserved
(0 << 8) | // AVX512_VP2INTERSECT
(0 << 9) | // Reserved
(0 << 10) | // VERW clears CPU buffers
(0 << 11) | // Reserved
(0 << 12) | // Reserved
(0 << 13) | // Reserved
(0 << 14) | // SERIALIZE instruction
(0 << 15) | // Reserved
(0 << 16) | // Reserved
(0 << 17) | // Reserved
(0 << 18) | // Intel PCONFIG
(0 << 19) | // Intel Architectural LBR
(0 << 20) | // Intel CET
(0 << 21) | // Reserved
(0 << 22) | // Reserved
(0 << 23) | // Reserved
(0 << 24) | // Reserved
(0 << 25) | // Reserved
(0 << 26) | // Reserved
(0 << 27) | // Reserved
(0 << 28) | // L1D Flush
(0 << 29) | // Arch capabilities
(0 << 30) | // Reserved
(0 << 31); // Reserved
}
Res.ecx =
(1 << 0) | // PREFETCHWT1
(0 << 1) | // AVX512VBMI
(0 << 2) | // Usermode instruction prevention
(0 << 3) | // Protection keys for user mode pages
(1 << 4) | // OS protection keys
(0 << 5) | // waitpkg
(0 << 6) | // AVX512_VBMI2
(0 << 7) | // CET shadow stack
(0 << 8) | // GFNI
(0 << 9) | // VAES
(0 << 10) | // VPCLMULQDQ
(0 << 11) | // AVX512_VNNI
(0 << 12) | // AVX512_BITALG
(0 << 13) | // Intel Total Memory Encryption
(0 << 14) | // AVX512_VPOPCNTDQ
(0 << 15) | // Reserved
(0 << 16) | // 5 Level page tables
(0 << 17) | // MPX MAWAU
(0 << 18) | // MPX MAWAU
(0 << 19) | // MPX MAWAU
(0 << 20) | // MPX MAWAU
(0 << 21) | // MPX MAWAU
(0 << 22) | // RDPID Read Processor ID
(0 << 23) | // Reserved
(0 << 24) | // Reserved
(0 << 25) | // CLDEMOTE
(0 << 26) | // Reserved
(0 << 27) | // MOVDIRI
(0 << 28) | // MOVDIR64B
(0 << 29) | // Reserved
(0 << 30) | // SGX Launch configuration
(0 << 31); // Reserved
Res.edx =
(0 << 0) | // Reserved
(0 << 1) | // Reserved
(0 << 2) | // AVX512_4VNNIW
(0 << 3) | // AVX512_4FMAPS
(0 << 4) | // Fast Short Rep Mov
(0 << 5) | // Reserved
(0 << 6) | // Reserved
(0 << 7) | // Reserved
(0 << 8) | // AVX512_VP2INTERSECT
(0 << 9) | // Reserved
(0 << 10) | // VERW clears CPU buffers
(0 << 11) | // Reserved
(0 << 12) | // Reserved
(0 << 13) | // Reserved
(0 << 14) | // SERIALIZE instruction
(0 << 15) | // Reserved
(0 << 16) | // Reserved
(0 << 17) | // Reserved
(0 << 18) | // Intel PCONFIG
(0 << 19) | // Intel Architectural LBR
(0 << 20) | // Intel CET
(0 << 21) | // Reserved
(0 << 22) | // Reserved
(0 << 23) | // Reserved
(0 << 24) | // Reserved
(0 << 25) | // Reserved
(0 << 26) | // Reserved
(0 << 27) | // Reserved
(0 << 28) | // L1D Flush
(0 << 29) | // Arch capabilities
(0 << 30) | // Reserved
(0 << 31); // Reserved
return Res;
}
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_0Dh(uint32_t Leaf) {
// Leaf 0
FEXCore::CPUID::FunctionResults Res{};
uint32_t XFeatureSupportedSizeMax = SUPPORTS_AVX ? 0x0000'0340 : 0x0000'0240; // XFeatureEnabledSizeMax: Legacy Header + FPU/SSE + AVX
if (Leaf == 0) {
// XFeatureSupportedMask[31:0]
Res.eax =
(1 << 0) | // X87 support
(1 << 1) | // 128-bit SSE support
(SUPPORTS_AVX << 2) | // 256-bit AVX support
(0b00 << 3) | // MPX State
(0b000 << 5) | // AVX-512 state
(0 << 8) | // "Used for IA32_XSS" ... Used for what?
(0 << 9); // PKRU state
// EBX and ECX doesn't need to match if a feature is supported but not enabled
Res.ebx = XFeatureSupportedSizeMax;
Res.ecx = XFeatureSupportedSizeMax; // XFeatureSupportedSizeMax: Size in bytes of XSAVE/XRSTOR area
// XFeatureSupportedMask[63:32]
Res.edx = 0; // Upper 32-bits of XFeatureSupportedMask
}
else if (Leaf == 1) {
Res.eax =
(0 << 0) | // XSAVEOPT
(0 << 1) | // XSAVEC (and XRSTOR)
(0 << 2) | // XGETBV - XGETBV with ECX=1 supported
(0 << 3); // XSAVES - XSAVES, XRSTORS, and IA32_XSS supported
// Same information as Leaf 0 for ebx
Res.ebx = XFeatureSupportedSizeMax;
// Lower supported 32bits of IA32_XSS MSR. IA32_XSS[n] can only be set to 1 if ECX[n] is 1
Res.ecx =
(0b0000'0000 << 0) | // Used for XCR0
(0 << 8) | // PT state
(0 << 9); // Used for XCR0
// Upper supported 32bits of IA32_XSS MSR. IA32_XSS[n+32] can only be set to 1 if EDX[n] is 1
// Entirely reserved atm
Res.edx = 0;
}
else if (Leaf == 2) {
Res.eax = SUPPORTS_AVX ? 0x0000'0100 : 0; // YmmSaveStateSize
Res.ebx = SUPPORTS_AVX ? 0x0000'0240 : 0; // YmmSaveStateOffset
// Reserved
Res.ecx = 0;
Res.edx = 0;
}
return Res;
}
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_15h(uint32_t Leaf) {
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_15h() {
FEXCore::CPUID::FunctionResults Res{};
// TSC frequency = ECX * EBX / EAX
uint32_t FrequencyHz = GetCycleCounterFrequency();
@@ -474,7 +295,7 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_15h(uint32_t Leaf) {
}
// Highest extended function implemented
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0000h(uint32_t Leaf) {
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0000h() {
FEXCore::CPUID::FunctionResults Res{};
Res.eax = 0x8000001F;
@@ -493,10 +314,15 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0000h(uint32_t Leaf) {
}
// Extended processor and feature bits
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0001h(uint32_t Leaf) {
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0001h() {
FEXCore::CPUID::FunctionResults Res{};
Res.eax = FAMILY_IDENTIFIER;
Res.eax = 0 | // Stepping
(0 << 4) | // Model
(0 << 8) | // Family ID
(0 << 12) | // Processor type
(0 << 16) | // Extended model ID
(0 << 20); // Extended family ID
Res.ecx =
(1 << 0) | // LAHF/SAHF
@@ -521,13 +347,13 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0001h(uint32_t Leaf) {
(0 << 19) | // Reserved
(0 << 20) | // Reserved
(0 << 21) | // Reserved
(0 << 22) | // Topology extensions support
(0 << 23) | // Core performance counter extensions
(0 << 24) | // NB performance counter extensions
(1 << 22) | // Topology extensions support
(1 << 23) | // Core performance counter extensions
(1 << 24) | // NB performance counter extensions
(0 << 25) | // Reserved
(0 << 26) | // Data breakpoints extensions
(0 << 27) | // Performance TSC
(0 << 28) | // L2 perf counter extensions
(1 << 27) | // Performance TSC
(1 << 28) | // L2 perf counter extensions
(0 << 29) | // Reserved
(0 << 30) | // Reserved
(0 << 31); // Reserved
@@ -559,7 +385,7 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0001h(uint32_t Leaf) {
(1 << 23) | // MMX
(1 << 24) | // FXSAVE/FXRSTOR
(1 << 25) | // FXSAVE/FXRSTOR Optimizations
(0 << 26) | // 1 gigabit pages
(1 << 26) | // 1 gigabit pages
(0 << 27) | // RDTSCP
(0 << 28) | // Reserved
(1 << 29) | // Long Mode
@@ -574,26 +400,26 @@ constexpr char ProcessorBrand[48] = {
};
//Processor brand string
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0002h(uint32_t Leaf) {
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0002h() {
FEXCore::CPUID::FunctionResults Res{};
memcpy(&Res, &ProcessorBrand[0], sizeof(FEXCore::CPUID::FunctionResults));
return Res;
}
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0003h(uint32_t Leaf) {
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0003h() {
FEXCore::CPUID::FunctionResults Res{};
memcpy(&Res, &ProcessorBrand[16], sizeof(FEXCore::CPUID::FunctionResults));
return Res;
}
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0004h(uint32_t Leaf) {
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0004h() {
FEXCore::CPUID::FunctionResults Res{};
memcpy(&Res, &ProcessorBrand[32], sizeof(FEXCore::CPUID::FunctionResults));
return Res;
}
// L1 Cache and TLB identifiers
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0005h(uint32_t Leaf) {
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0005h() {
FEXCore::CPUID::FunctionResults Res{};
// L1 TLB Information for 2MB and 4MB pages
@@ -628,7 +454,7 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0005h(uint32_t Leaf) {
}
// L2 Cache identifiers
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0006h(uint32_t Leaf) {
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0006h() {
FEXCore::CPUID::FunctionResults Res{};
// L2 TLB Information for 2MB and 4MB pages
@@ -662,7 +488,7 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0006h(uint32_t Leaf) {
}
// Advanced power management
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0007h(uint32_t Leaf) {
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0007h() {
FEXCore::CPUID::FunctionResults Res{};
Res.eax = (1 << 2); // APIC timer not affected by p-state
Res.edx =
@@ -670,167 +496,27 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0007h(uint32_t Leaf) {
return Res;
}
// Virtual and physical address sizes
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0008h(uint32_t Leaf) {
FEXCore::CPUID::FunctionResults Res{};
Res.eax =
(48 << 0) | // PhysAddrSize = 48-bit
(48 << 8) | // LinAddrSize = 48-bit
(0 << 16); // GuestPhysAddrSize == PhysAddrSize
Res.ebx =
(0 << 2) | // XSaveErPtr: Saving and restoring error pointers
(0 << 1) | // IRPerf: Instructions retired count support
(0 << 0); // CLZERO support
uint32_t CoreCount = Cores() - 1;
Res.ecx =
(0 << 16) | // PerfTscSize: Performance timestamp count size
((uint32_t)std::log2(CoreCount + 1) << 12) | // ApicIdSize: Number of bits in ApicID
(CoreCount << 0); // Count count subtract one
return Res;
}
// TLB 1GB page identifiers
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0019h(uint32_t Leaf) {
FEXCore::CPUID::FunctionResults Res{};
Res.eax =
(0xF << 28) | // L1 DTLB associativity for 1GB pages
(64 << 16) | // L1 DTLB entry count for 1GB pages
(0xF << 12) | // L1 ITLB associativity for 1GB pages
(64 << 0); // L1 ITLB entry count for 1GB pages
Res.ebx =
(0 << 28) | // L2 DTLB associativity for 1GB pages
(0 << 16) | // L2 DTLB entry count for 1GB pages
(0 << 12) | // L2 ITLB associativity for 1GB pages
(0 << 0); // L2 ITLB entry count for 1GB pages
return Res;
}
// Deterministic cache parameters for each level
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_001Dh(uint32_t Leaf) {
// This is nearly a copy of CPUID function 4h
// There are some minor changes though
FEXCore::CPUID::FunctionResults Res{};
constexpr uint32_t CacheType_Data = 1;
constexpr uint32_t CacheType_Instruction = 2;
constexpr uint32_t CacheType_Unified = 3;
if (Leaf == 0) {
// Report L1D
Res.eax = CacheType_Data | // Cache type
(0b001 << 5) | // Cache level
(1 << 8) | // Self initializing cache level
(0 << 9) | // Fully associative
(0 << 14); // Maximum number of addressable IDs for logical processors sharing this cache (With SMT this would be 1)
Res.ebx =
(63 << 0) | // Line Size - 1 : Claiming 64 byte
(0 << 12) | // Physical Line partitions
(7 << 22); // Associativity - 1 : Claiming 8 way
// 32KB
Res.ecx = 63; // Number of sets - 1 : Claiming 64 sets
Res.edx =
(0 << 0) | // Write-back invalidate
(0 << 1); // Cache inclusiveness - Includes lower caches
}
else if (Leaf == 1) {
// Report L1I
Res.eax = CacheType_Instruction | // Cache type
(0b001 << 5) | // Cache level
(1 << 8) | // Self initializing cache level
(0 << 9) | // Fully associative
(0 << 14); // Maximum number of addressable IDs for logical processors sharing this cache (With SMT this would be 1)
Res.ebx =
(63 << 0) | // Line Size - 1 : Claiming 64 byte
(0 << 12) | // Physical Line partitions
(7 << 22); // Associativity - 1 : Claiming 8 way
// 32KB
Res.ecx = 63; // Number of sets - 1 : Claiming 64 sets
Res.edx =
(0 << 0) | // Write-back invalidate
(0 << 1); // Cache inclusiveness - Includes lower caches
}
else if (Leaf == 2) {
// Report L2
Res.eax = CacheType_Unified | // Cache type
(0b010 << 5) | // Cache level
(1 << 8) | // Self initializing cache level
(0 << 9) | // Fully associative
(0 << 14); // Maximum number of addressable IDs for logical processors sharing this cache
Res.ebx =
(63 << 0) | // Line Size - 1 : Claiming 64 byte
(0 << 12) | // Physical Line partitions
(7 << 22); // Associativity - 1 : Claiming 8 way
// 512KB
Res.ecx = 0x3FF; // Number of sets - 1 : Claiming 1024 sets
Res.edx =
(0 << 0) | // Write-back invalidate
(0 << 1); // Cache inclusiveness - Includes lower caches
}
else if (Leaf == 3) {
// Report L3
uint32_t CoreCount = Cores() - 1;
Res.eax = CacheType_Unified | // Cache type
(0b011 << 5) | // Cache level
(1 << 8) | // Self initializing cache level
(0 << 9) | // Fully associative
(CoreCount << 14); // Maximum number of addressable IDs for logical processors sharing this cache
Res.ebx =
(63 << 0) | // Line Size - 1 : Claiming 64 byte
(0 << 12) | // Physical Line partitions
(7 << 22); // Associativity - 1 : Claiming 8 way
// 8MB
Res.ecx = 0x4000; // Number of sets - 1 : Claiming 16384 sets
Res.edx =
(0 << 0) | // Write-back invalidate
(0 << 1); // Cache inclusiveness - Includes lower caches
}
return Res;
}
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_Reserved(uint32_t Leaf) {
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_Reserved() {
FEXCore::CPUID::FunctionResults Res{};
return Res;
}
void CPUIDEmu::Init(FEXCore::Context::Context *ctx) {
CTX = ctx;
using namespace std::placeholders;
RegisterFunction(0, std::bind(&CPUIDEmu::Function_0h, this, _1));
RegisterFunction(1, std::bind(&CPUIDEmu::Function_01h, this, _1));
RegisterFunction(2, std::bind(&CPUIDEmu::Function_02h, this, _1));
RegisterFunction(0, std::bind(&CPUIDEmu::Function_0h, this));
RegisterFunction(1, std::bind(&CPUIDEmu::Function_01h, this));
RegisterFunction(2, std::bind(&CPUIDEmu::Function_02h, this));
// 3: Serial Number(previously), now reserved
#ifndef CPUID_AMD
// Deterministic cache parameters for each level
RegisterFunction(0x4, std::bind(&CPUIDEmu::Function_04h, this, _1));
#endif
// 4: Deterministic cache parameters for each level
// 5: Monitor/mwait
// Thermal and power management
RegisterFunction(6, std::bind(&CPUIDEmu::Function_06h, this, _1));
RegisterFunction(6, std::bind(&CPUIDEmu::Function_06h, this));
// Extended feature flags
RegisterFunction(7, std::bind(&CPUIDEmu::Function_07h, this, _1));
RegisterFunction(7, std::bind(&CPUIDEmu::Function_07h, this));
// 9: Direct Cache Access information
// 0x0A: Architectural performance monitoring
// 0x0B: Extended topology enumeration
// 0x0D: Processor extended state enumeration
RegisterFunction(0x0D, std::bind(&CPUIDEmu::Function_0Dh, this, _1));
// 0x0F: Intel RDT monitoring
// 0x10: Intel RDT allocation enumeration
// 0x12: Intel SGX capability enumeration
@@ -839,47 +525,37 @@ void CPUIDEmu::Init(FEXCore::Context::Context *ctx) {
#ifndef CPUID_AMD
// Timestamp counter information
// Doesn't exist on AMD hardware
RegisterFunction(0x15, std::bind(&CPUIDEmu::Function_15h, this, _1));
RegisterFunction(0x15, std::bind(&CPUIDEmu::Function_15h, this));
#endif
// 0x16: Processor frequency information
// 0x17: SoC vendor attribute enumeration
// Largest extended function number
RegisterFunction(0x8000'0000, std::bind(&CPUIDEmu::Function_8000_0000h, this, _1));
RegisterFunction(0x8000'0000, std::bind(&CPUIDEmu::Function_8000_0000h, this));
// Processor vendor
RegisterFunction(0x8000'0001, std::bind(&CPUIDEmu::Function_8000_0001h, this, _1));
RegisterFunction(0x8000'0001, std::bind(&CPUIDEmu::Function_8000_0001h, this));
// Processor brand string
RegisterFunction(0x8000'0002, std::bind(&CPUIDEmu::Function_8000_0002h, this, _1));
RegisterFunction(0x8000'0002, std::bind(&CPUIDEmu::Function_8000_0002h, this));
// Processor brand string continued
RegisterFunction(0x8000'0003, std::bind(&CPUIDEmu::Function_8000_0003h, this, _1));
RegisterFunction(0x8000'0003, std::bind(&CPUIDEmu::Function_8000_0003h, this));
// Processor brand string continued
RegisterFunction(0x8000'0004, std::bind(&CPUIDEmu::Function_8000_0004h, this, _1));
RegisterFunction(0x8000'0004, std::bind(&CPUIDEmu::Function_8000_0004h, this));
// 0x8000'0005: L1 Cache and TLB identifiers
#ifdef CPUID_AMD
RegisterFunction(0x8000'0005, std::bind(&CPUIDEmu::Function_8000_0005h, this, _1));
#else
// This is full reserved on Intel platforms
RegisterFunction(0x8000'0005, std::bind(&CPUIDEmu::Function_Reserved, this, _1));
RegisterFunction(0x8000'0005, std::bind(&CPUIDEmu::Function_8000_0005h, this));
#endif
// 0x8000'0006: L2 Cache identifiers
RegisterFunction(0x8000'0006, std::bind(&CPUIDEmu::Function_8000_0006h, this, _1));
RegisterFunction(0x8000'0006, std::bind(&CPUIDEmu::Function_8000_0006h, this));
// Advanced power management information
RegisterFunction(0x8000'0007, std::bind(&CPUIDEmu::Function_8000_0007h, this, _1));
// Virtual and physical address sizes
RegisterFunction(0x8000'0008, std::bind(&CPUIDEmu::Function_8000_0008h, this, _1));
RegisterFunction(0x8000'0007, std::bind(&CPUIDEmu::Function_8000_0007h, this));
// 0x8000'0008: Virtual and physical address sizes
// 0x8000'000A: SVM Revision
// TLB 1GB page identifiers
RegisterFunction(0x8000'0019, std::bind(&CPUIDEmu::Function_8000_0019h, this, _1));
// 0x8000'0019: TLB 1GB page identifiers
// 0x8000'001A: Performance optimization identifiers
// 0x8000'001B: Instruction based sampling identifiers
// 0x8000'001C: Lightweight profiling capabilities
// 0x8000'001D: Cache properties
#ifdef CPUID_AMD
// Deterministic cache parameters for each level
RegisterFunction(0x8000'001D, std::bind(&CPUIDEmu::Function_8000_001Dh, this, _1));
#endif
// 0x8000'001E: Extended APIC ID
// 0x8000'001F: AMD Secure Encryption
}
+21 -28
View File
@@ -3,7 +3,6 @@
#include <unordered_map>
#include <FEXCore/Core/CPUID.h>
#include <FEXCore/Config/Config.h>
#include <FEXCore/Utils/LogManager.h>
namespace FEXCore {
@@ -24,23 +23,22 @@ private:
public:
void Init(FEXCore::Context::Context *ctx);
FEXCore::CPUID::FunctionResults RunFunction(uint32_t Function, uint32_t Leaf) {
FEXCore::CPUID::FunctionResults RunFunction(uint32_t Function, [[maybe_unused]] uint32_t Leaf) {
auto Handler = FunctionHandlers.find(Function);
if (Handler == FunctionHandlers.end()) {
#ifndef NDEBUG
LogMan::Msg::E("Unhandled CPU ID function, 0x%x-0x%x", Function, Leaf);
LogMan::Msg::E("Unhandled CPU ID function, 0x%x", Function);
#endif
return Function_Reserved(Leaf);
return Function_Reserved();
}
return Handler->second(Leaf);
return Handler->second();
}
private:
FEXCore::Context::Context *CTX;
FEX_CONFIG_OPT(Cores, THREADS);
using FunctionHandler = std::function<FEXCore::CPUID::FunctionResults(uint32_t Leaf)>;
using FunctionHandler = std::function<FEXCore::CPUID::FunctionResults()>;
void RegisterFunction(uint32_t Function, FunctionHandler Handler) {
FunctionHandlers[Function] = Handler;
}
@@ -48,26 +46,21 @@ private:
std::unordered_map<uint32_t, FunctionHandler> FunctionHandlers;
// Functions
FEXCore::CPUID::FunctionResults Function_0h(uint32_t Leaf);
FEXCore::CPUID::FunctionResults Function_01h(uint32_t Leaf);
FEXCore::CPUID::FunctionResults Function_02h(uint32_t Leaf);
FEXCore::CPUID::FunctionResults Function_04h(uint32_t Leaf);
FEXCore::CPUID::FunctionResults Function_06h(uint32_t Leaf);
FEXCore::CPUID::FunctionResults Function_07h(uint32_t Leaf);
FEXCore::CPUID::FunctionResults Function_0Dh(uint32_t Leaf);
FEXCore::CPUID::FunctionResults Function_15h(uint32_t Leaf);
FEXCore::CPUID::FunctionResults Function_8000_0000h(uint32_t Leaf);
FEXCore::CPUID::FunctionResults Function_8000_0001h(uint32_t Leaf);
FEXCore::CPUID::FunctionResults Function_8000_0002h(uint32_t Leaf);
FEXCore::CPUID::FunctionResults Function_8000_0003h(uint32_t Leaf);
FEXCore::CPUID::FunctionResults Function_8000_0004h(uint32_t Leaf);
FEXCore::CPUID::FunctionResults Function_8000_0005h(uint32_t Leaf);
FEXCore::CPUID::FunctionResults Function_8000_0006h(uint32_t Leaf);
FEXCore::CPUID::FunctionResults Function_8000_0007h(uint32_t Leaf);
FEXCore::CPUID::FunctionResults Function_8000_0008h(uint32_t Leaf);
FEXCore::CPUID::FunctionResults Function_8000_0009h(uint32_t Leaf);
FEXCore::CPUID::FunctionResults Function_8000_0019h(uint32_t Leaf);
FEXCore::CPUID::FunctionResults Function_8000_001Dh(uint32_t Leaf);
FEXCore::CPUID::FunctionResults Function_Reserved(uint32_t Leaf);
FEXCore::CPUID::FunctionResults Function_0h();
FEXCore::CPUID::FunctionResults Function_01h();
FEXCore::CPUID::FunctionResults Function_02h();
FEXCore::CPUID::FunctionResults Function_06h();
FEXCore::CPUID::FunctionResults Function_07h();
FEXCore::CPUID::FunctionResults Function_15h();
FEXCore::CPUID::FunctionResults Function_8000_0000h();
FEXCore::CPUID::FunctionResults Function_8000_0001h();
FEXCore::CPUID::FunctionResults Function_8000_0002h();
FEXCore::CPUID::FunctionResults Function_8000_0003h();
FEXCore::CPUID::FunctionResults Function_8000_0004h();
FEXCore::CPUID::FunctionResults Function_8000_0005h();
FEXCore::CPUID::FunctionResults Function_8000_0006h();
FEXCore::CPUID::FunctionResults Function_8000_0007h();
FEXCore::CPUID::FunctionResults Function_Reserved();
};
}
+3 -3
View File
@@ -61,7 +61,7 @@ namespace FEXCore {
}
}
LOGMAN_THROW_A(CompileThreadData->LocalIRCache.size() == 0, "Compile service must never have LocalIRCache");
LogMan::Throw::A(CompileThreadData->LocalIRCache.size() == 0, "Compile service must never have LocalIRCache");
CompileMutex.unlock();
}
@@ -124,7 +124,7 @@ namespace FEXCore {
// If we had a work item then work on it
if (Item) {
// Make sure it's not in lookup cache by accident
LOGMAN_THROW_A(CompileThreadData->LookupCache->FindBlock(Item->RIP) == 0, "Compile Service must never have entries in the LookupCache");
LogMan::Throw::A(CompileThreadData->LookupCache->FindBlock(Item->RIP) == 0, "Compile Service must never have entries in the LookupCache");
// Code isn't in cache, compile now
// Set our thread state's RIP
@@ -132,7 +132,7 @@ namespace FEXCore {
auto [CodePtr, IRList, DebugData, RAData, Generated, StartAddr, Length] = CTX->CompileCode(CompileThreadData.get(), Item->RIP);
LOGMAN_THROW_A(Generated == true, "Compile Service doesn't have IR Cache");
LogMan::Throw::A(Generated == true, "Compile Service doesn't have IR Cache");
if (!CodePtr) {
// XXX: We currently have the expectation that compile service code will be significantly smaller than regular thread's code
File diff suppressed because it is too large. Load diff
@@ -5,7 +5,6 @@
#include "Interface/Context/Context.h"
#include <FEXCore/Core/X86Enums.h>
#include <bit>
#include <cmath>
#include "aarch64/assembler-aarch64.h"
@@ -68,7 +67,6 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::
// We want to ensure that we are 16 byte aligned at the top of this loop
Align16B();
aarch64::Label FullLookup{};
aarch64::Label CallBlock{};
aarch64::Label LoopTop{};
aarch64::Label ExitSpillSRA{};
aarch64::Label ThreadPauseHandler{};
@@ -81,19 +79,16 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::
ldr(x2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.rip)));
auto RipReg = x2;
// L1 Cache
ldr(x0, &l_L1Ptr);
and_(x3, RipReg, LookupCache::L1_ENTRIES_MASK);
add(x0, x0, Operand(x3, Shift::LSL, 4));
ldp(x3, x0, MemOperand(x0));
cmp(x0, RipReg);
b(&FullLookup, Condition::ne);
if (!config.ExecuteBlocksWithCall) {
br(x3);
} else {
b(&CallBlock);
// L1 Cache
ldr(x0, &l_L1Ptr);
and_(x3, RipReg, LookupCache::L1_ENTRIES_MASK);
add(x0, x0, Operand(x3, Shift::LSL, 4));
ldp(x1, x0, MemOperand(x0));
cmp(x0, RipReg);
b(&FullLookup, Condition::ne);
br(x1);
}
// L1C check failed, do a full lookup
@@ -104,7 +99,7 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::
ldr(x0, &l_PagePtr);
// Mask the address by the virtual address size so we can check for aliases
if (std::popcount(VirtualMemorySize) == 1) {
if (__builtin_popcountl(VirtualMemorySize) == 1) {
and_(x3, RipReg, Thread->LookupCache->GetVirtualMemorySize() - 1);
}
else {
@@ -141,37 +136,40 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::
// If we've made it here then we have a real compiled block
{
// update L1 cache
ldr(x0, &l_L1Ptr);
and_(x1, RipReg, LookupCache::L1_ENTRIES_MASK);
add(x0, x0, Operand(x1, Shift::LSL, 4));
stp(x3, x2, MemOperand(x0));
// Jump to the block
if (!config.ExecuteBlocksWithCall) {
// update L1 cache
ldr(x0, &l_L1Ptr);
and_(x1, RipReg, LookupCache::L1_ENTRIES_MASK);
add(x0, x0, Operand(x1, Shift::LSL, 4));
stp(x3, x2, MemOperand(x0));
br(x3);
} else {
bind(&CallBlock);
mov(x0, STATE);
blr(x3);
}
}
if (CTX->GetGdbServerStatus()) {
// If we have a gdb server running then run in a less efficient mode that checks if we need to exit
// This happens when single stepping
if (config.ExecuteBlocksWithCall) {
// Interpreter continues execution here
if (CTX->GetGdbServerStatus()) {
// If we have a gdb server running then run in a less efficient mode that checks if we need to exit
// This happens when single stepping
static_assert(sizeof(CTX->Config.RunningMode) == 4, "This is expected to be size of 4");
ldr(x0, &l_CTX);
ldr(w0, MemOperand(x0, offsetof(FEXCore::Context::Context, Config.RunningMode)));
// If the value == 0 then branch to the top
cbz(x0, &LoopTop);
// Else we need to pause now
b(&ThreadPauseHandler);
} else {
// Unconditionally loop to the top
// We will only stop on error when compiling a block or signal
b(&LoopTop);
}
static_assert(sizeof(CTX->Config.RunningMode) == 4, "This is expected to be size of 4");
ldr(x0, &l_CTX);
ldr(w0, MemOperand(x0, offsetof(FEXCore::Context::Context, Config.RunningMode)));
// If the value == 0 then branch to the top
cbz(x0, &LoopTop);
// Else we need to pause now
b(&ThreadPauseHandler);
}
else {
// Unconditionally loop to the top
// We will only stop on error when compiling a block or signal
b(&LoopTop);
}
}
}
@@ -352,7 +350,7 @@ void InterpreterCore::CreateAsmDispatch(FEXCore::Context::Context *ctx, FEXCore:
DispatcherConfig config;
config.ExecuteBlocksWithCall = true;
Dispatcher = std::make_unique<Arm64Dispatcher>(ctx, Thread, config);
Dispatcher = new Arm64Dispatcher(ctx, Thread, config);
DispatchPtr = Dispatcher->DispatchPtr;
CallbackPtr = Dispatcher->CallbackPtr;
@@ -108,7 +108,7 @@ bool Dispatcher::HandleGuestSignal(int Signal, void *info, void *ucontext, Guest
if (GuestAction->sa_flags & SA_SIGINFO) {
if (SRAEnabled) {
if (!IsAddressInJITCode(ArchHelpers::Context::GetPc(ucontext), false)) {
LOGMAN_THROW_A(!IsAddressInJITCode(ArchHelpers::Context::GetPc(ucontext), true), "Signals in dispatcher have unsynchronized context");
LogMan::Throw::A(!IsAddressInJITCode(ArchHelpers::Context::GetPc(ucontext), true), "Signals in dispatcher have unsynchronized context");
} else {
// We are in jit, SRA must be spilled
SpillSRA(ucontext);
@@ -121,11 +121,7 @@ bool Dispatcher::HandleGuestSignal(int Signal, void *info, void *ucontext, Guest
NewGuestSP -= sizeof(FEXCore::x86_64::ucontext_t);
uint64_t UContextLocation = NewGuestSP;
NewGuestSP -= sizeof(siginfo_t);
uint64_t SigInfoLocation = NewGuestSP;
FEXCore::x86_64::ucontext_t *guest_uctx = reinterpret_cast<FEXCore::x86_64::ucontext_t*>(UContextLocation);
siginfo_t *guest_siginfo = reinterpret_cast<siginfo_t*>(SigInfoLocation);
// We have extended float information
guest_uctx->uc_flags |= FEXCore::x86_64::UC_FP_XSTATE;
@@ -173,14 +169,8 @@ bool Dispatcher::HandleGuestSignal(int Signal, void *info, void *ucontext, Guest
guest_uctx->uc_stack.ss_sp = GuestStack->ss_sp;
guest_uctx->uc_stack.ss_size = GuestStack->ss_size;
// siginfo_t
siginfo_t *HostSigInfo = reinterpret_cast<siginfo_t*>(info);
// aarch64 and x86_64 siginfo_t matches. We can just copy this over
// SI_USER could also potentially have random data in it, needs to be bit perfect
// For guest faults we don't have a real way to reconstruct state to a real guest RIP
*guest_siginfo = *HostSigInfo;
Frame->State.gregs[X86State::REG_RSI] = SigInfoLocation;
// XXX: siginfo_t(RSI)
Frame->State.gregs[X86State::REG_RSI] = 0x4142434445460000;
Frame->State.gregs[X86State::REG_RDX] = UContextLocation;
}
else {
@@ -188,36 +178,7 @@ bool Dispatcher::HandleGuestSignal(int Signal, void *info, void *ucontext, Guest
NewGuestSP -= sizeof(FEXCore::x86::ucontext_t);
uint64_t UContextLocation = 0; // NewGuestSP;
NewGuestSP -= sizeof(FEXCore::x86::siginfo_t);
uint64_t SigInfoLocation = NewGuestSP;
FEXCore::x86::siginfo_t *guest_siginfo = reinterpret_cast<FEXCore::x86::siginfo_t*>(SigInfoLocation);
siginfo_t *HostSigInfo = reinterpret_cast<siginfo_t*>(info);
// These three elements are in every siginfo
guest_siginfo->si_signo = HostSigInfo->si_signo;
guest_siginfo->si_errno = HostSigInfo->si_errno;
guest_siginfo->si_code = HostSigInfo->si_code;
switch (Signal) {
case SIGSEGV:
case SIGBUS:
// Macro expansion to get the si_addr
// Can't really give a real result here. Pull from the context for now
guest_siginfo->_sifields._sigfault.addr = Frame->State.rip;
break;
case SIGCHLD:
guest_siginfo->_sifields._sigchld.pid = HostSigInfo->si_pid;
guest_siginfo->_sifields._sigchld.uid = HostSigInfo->si_uid;
guest_siginfo->_sifields._sigchld.status = HostSigInfo->si_status;
guest_siginfo->_sifields._sigchld.utime = HostSigInfo->si_utime;
guest_siginfo->_sifields._sigchld.stime = HostSigInfo->si_stime;
break;
default:
LogMan::Msg::D("Unhandled siginfo_t signal: %d", Signal);
// Hope for the best, most things just copy over
memcpy(guest_siginfo, info, sizeof(siginfo_t));
break;
}
uint64_t SigInfoLocation = 0; // NewGuestSP;
NewGuestSP -= 4;
*(uint32_t*)NewGuestSP = UContextLocation;
@@ -242,7 +203,7 @@ bool Dispatcher::HandleGuestSignal(int Signal, void *info, void *ucontext, Guest
else {
NewGuestSP -= 4;
*(uint32_t*)NewGuestSP = CTX->X86CodeGen.SignalReturn;
LOGMAN_THROW_A(CTX->X86CodeGen.SignalReturn < 0x1'0000'0000ULL, "This needs to be below 4GB");
LogMan::Throw::A(CTX->X86CodeGen.SignalReturn < 0x1'0000'0000ULL, "This needs to be below 4GB");
Frame->State.gregs[X86State::REG_RSP] = NewGuestSP;
}
@@ -276,7 +237,7 @@ bool Dispatcher::HandleSignalPause(int Signal, void *info, void *ucontext) {
FEXCore::Core::SignalEvent SignalReason = ThreadState->SignalReason.load();
auto Frame = ThreadState->CurrentFrame;
if (SignalReason == FEXCore::Core::SignalEvent::Pause) {
if (SignalReason == FEXCore::Core::SignalEvent::SIGNALEVENT_PAUSE) {
// Store our thread state so we can come back to this
StoreThreadState(Signal, ucontext);
@@ -286,7 +247,7 @@ bool Dispatcher::HandleSignalPause(int Signal, void *info, void *ucontext) {
} else {
if (SRAEnabled) {
// We are in non-jit, SRA is already spilled
LOGMAN_THROW_A(!IsAddressInJITCode(ArchHelpers::Context::GetPc(ucontext), true), "Signals in dispatcher have unsynchronized context");
LogMan::Throw::A(!IsAddressInJITCode(ArchHelpers::Context::GetPc(ucontext), true), "Signals in dispatcher have unsynchronized context");
}
ArchHelpers::Context::SetPc(ucontext, ThreadPauseHandlerAddress);
}
@@ -301,11 +262,11 @@ bool Dispatcher::HandleSignalPause(int Signal, void *info, void *ucontext) {
// We use this to track if it is safe to clear cache
++SignalHandlerRefCounter;
ThreadState->SignalReason.store(FEXCore::Core::SignalEvent::Nothing);
ThreadState->SignalReason.store(FEXCore::Core::SIGNALEVENT_NONE);
return true;
}
if (SignalReason == FEXCore::Core::SignalEvent::Stop) {
if (SignalReason == FEXCore::Core::SignalEvent::SIGNALEVENT_STOP) {
// Our thread is stopping
// We don't care about anything at this point
// Set the stack to our starting location when we entered the core and get out safely
@@ -321,23 +282,23 @@ bool Dispatcher::HandleSignalPause(int Signal, void *info, void *ucontext) {
} else {
if (SRAEnabled) {
// We are in non-jit, SRA is already spilled
LOGMAN_THROW_A(!IsAddressInJITCode(ArchHelpers::Context::GetPc(ucontext), true), "Signals in dispatcher have unsynchronized context");
LogMan::Throw::A(!IsAddressInJITCode(ArchHelpers::Context::GetPc(ucontext), true), "Signals in dispatcher have unsynchronized context");
}
ArchHelpers::Context::SetPc(ucontext, ThreadStopHandlerAddress);
}
ThreadState->SignalReason.store(FEXCore::Core::SignalEvent::Nothing);
ThreadState->SignalReason.store(FEXCore::Core::SIGNALEVENT_NONE);
return true;
}
if (SignalReason == FEXCore::Core::SignalEvent::Return) {
if (SignalReason == FEXCore::Core::SignalEvent::SIGNALEVENT_RETURN) {
RestoreThreadState(ucontext);
// Ref count our faults
// We use this to track if it is safe to clear cache
--SignalHandlerRefCounter;
ThreadState->SignalReason.store(FEXCore::Core::SignalEvent::Nothing);
ThreadState->SignalReason.store(FEXCore::Core::SIGNALEVENT_NONE);
return true;
}
@@ -345,14 +306,14 @@ bool Dispatcher::HandleSignalPause(int Signal, void *info, void *ucontext) {
}
uint64_t Dispatcher::GetCompileBlockPtr() {
using ClassPtrType = void (FEXCore::Context::Context::*)(FEXCore::Core::CpuStateFrame *, uint64_t);
using ClassPtrType = uintptr_t (FEXCore::Context::Context::*)(FEXCore::Core::CpuStateFrame *, uint64_t);
union PtrCast {
ClassPtrType ClassPtr;
uintptr_t Data;
};
PtrCast CompileBlockPtr;
CompileBlockPtr.ClassPtr = &FEXCore::Context::Context::CompileBlockJit;
CompileBlockPtr.ClassPtr = &FEXCore::Context::Context::CompileBlock;
return CompileBlockPtr.Data;
}
@@ -366,7 +327,7 @@ void Dispatcher::RemoveCodeBuffer(uint8_t* start_to_remove) {
}
}
bool Dispatcher::IsAddressInJITCode(uint64_t Address, bool IncludeDispatcher) const {
bool Dispatcher::IsAddressInJITCode(uint64_t Address, bool IncludeDispatcher) {
for (auto [start, end] : CodeBuffers) {
if (Address >= start && Address < end) {
return true;
@@ -18,7 +18,6 @@ struct DispatcherConfig {
class Dispatcher {
public:
virtual ~Dispatcher() = default;
CPUBackend::AsmDispatch DispatchPtr;
CPUBackend::JITCallback CallbackPtr;
FEXCore::Context::Context::IntCallbackReturn ReturnPtr;
@@ -54,8 +53,8 @@ public:
void RemoveCodeBuffer(uint8_t* start);
bool IsAddressInJITCode(uint64_t Address, bool IncludeDispatcher = true) const;
bool IsAddressInDispatcher(uint64_t Address) const {
bool IsAddressInJITCode(uint64_t Address, bool IncludeDispatcher = true);
bool IsAddressInDispatcher(uint64_t Address) {
return Address >= Start && Address < End;
}
@@ -82,4 +81,4 @@ private:
std::vector<std::tuple<uint64_t, uint64_t>> CodeBuffers; // Start, End
};
}
}
@@ -12,7 +12,7 @@ static constexpr size_t MAX_DISPATCHER_CODE_SIZE = 4096;
X86Dispatcher::X86Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, DispatcherConfig &config)
: Dispatcher(ctx, Thread)
, Xbyak::CodeGenerator(MAX_DISPATCHER_CODE_SIZE, nullptr, this) {
, Xbyak::CodeGenerator(MAX_DISPATCHER_CODE_SIZE) {
using namespace Xbyak;
using namespace Xbyak::util;
@@ -66,7 +66,6 @@ X86Dispatcher::X86Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::Inte
Label LoopTop;
Label FullLookup;
Label CallBlock;
Label NoBlock;
Label ExitBlock;
Label ThreadPauseHandler;
@@ -78,20 +77,17 @@ X86Dispatcher::X86Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::Inte
// Load our RIP
mov(rdx, qword [STATE + offsetof(FEXCore::Core::CPUState, rip)]);
// L1 Cache
mov(r13, Thread->LookupCache->GetL1Pointer());
mov(rax, rdx);
if (!config.ExecuteBlocksWithCall)
{
// L1 Cache
mov(r13, Thread->LookupCache->GetL1Pointer());
mov(rax, rdx);
and_(rax, LookupCache::L1_ENTRIES_MASK);
shl(rax, 4);
cmp(qword[r13 + rax + 8], rdx);
jne(FullLookup);
if (!config.ExecuteBlocksWithCall) {
and_(rax, LookupCache::L1_ENTRIES_MASK);
shl(rax, 4);
cmp(qword[r13 + rax + 8], rdx);
jne(FullLookup);
jmp(qword[r13 + rax + 0]);
} else {
mov(rax, qword[r13 + rax + 0]);
jmp(CallBlock);
}
L(FullLookup);
@@ -126,19 +122,19 @@ X86Dispatcher::X86Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::Inte
je(NoBlock);
// Update L1
mov(r13, Thread->LookupCache->GetL1Pointer());
mov(rcx, rdx);
and_(rcx, LookupCache::L1_ENTRIES_MASK);
shl(rcx, 1);
mov(qword[r13 + rcx*8 + 8], rdx);
mov(qword[r13 + rcx*8 + 0], rax);
if (config.ExecuteBlocksWithCall) {
mov(r13, Thread->LookupCache->GetL1Pointer());
mov(rcx, rdx);
and_(rcx, LookupCache::L1_ENTRIES_MASK);
shl(rcx, 1);
mov(qword[r13 + rcx*8 + 8], rdx);
mov(qword[r13 + rcx*8 + 0], rax);
}
// Real block if we made it here
if (!config.ExecuteBlocksWithCall) {
jmp(rax);
} else {
L(CallBlock);
mov(rdi, STATE);
call(rax);
@@ -181,10 +177,19 @@ X86Dispatcher::X86Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::Inte
{
L(NoBlock);
using ClassPtrType = uintptr_t (FEXCore::Context::Context::*)(FEXCore::Core::CpuStateFrame *, uint64_t);
union PtrCast {
ClassPtrType ClassPtr;
uintptr_t Data;
};
PtrCast Ptr;
Ptr.ClassPtr = &FEXCore::Context::Context::CompileBlock;
// {rdi, rsi, rdx}
mov(rdi, reinterpret_cast<uint64_t>(CTX));
mov(rsi, STATE);
mov(rax, GetCompileBlockPtr());
mov(rax, Ptr.Data);
call(rax);
@@ -266,7 +271,7 @@ X86Dispatcher::X86Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::Inte
{
ReturnPtr = getCurr<FEXCore::Context::Context::IntCallbackReturn>();
// using CallbackReturn = FEX_NAKED void(*)(FEXCore::Core::InternalThreadState *Thread, volatile void *Host_RSP);
// using CallbackReturn = __attribute__((naked)) void(*)(FEXCore::Core::InternalThreadState *Thread, volatile void *Host_RSP);
// rdi = thread
// rsi = rsp
@@ -293,7 +298,7 @@ X86Dispatcher::X86Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::Inte
#if ENABLE_JITSYMBOLS
std::string Name = "Dispatch_" + std::to_string(::gettid());
CTX->Symbols.Register(reinterpret_cast<void*>(Start), End-Start, Name);
CTX->Symbols.Register(Start, End-Start, Name);
#endif
}
@@ -307,7 +312,7 @@ void InterpreterCore::CreateAsmDispatch(FEXCore::Context::Context *ctx, FEXCore:
DispatcherConfig config;
config.ExecuteBlocksWithCall = true;
Dispatcher = std::make_unique<X86Dispatcher>(ctx, Thread, config);
Dispatcher = new X86Dispatcher(ctx, Thread, config);
DispatchPtr = Dispatcher->DispatchPtr;
CallbackPtr = Dispatcher->CallbackPtr;
@@ -2,26 +2,16 @@
#include "Interface/Core/Dispatcher/Dispatcher.h"
#include <FEXCore/Utils/Allocator.h>
#define XBYAK64
#include <xbyak/xbyak.h>
namespace FEXCore::CPU {
class X86Dispatcher final : public Dispatcher, public Xbyak::CodeGenerator, public Xbyak::Allocator {
class X86Dispatcher final : public Dispatcher, public Xbyak::CodeGenerator {
public:
X86Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, DispatcherConfig &config);
virtual ~X86Dispatcher() override;
// Xbyak::Allocator
Xbyak::uint8 *alloc(size_t size) override { Size = size; return reinterpret_cast<uint8_t*>(FEXCore::Allocator::mmap(nullptr, size, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)); }
void free(Xbyak::uint8 *p) override { FEXCore::Allocator::munmap(p, Size); }
bool useProtect() const override { return false; }
private:
size_t Size{};
};
}
}
+68 -102
View File
@@ -15,15 +15,12 @@ $end_info$
#include <FEXCore/Core/X86Enums.h>
#include <FEXCore/Debug/X86Tables.h>
#include <FEXCore/Utils/LogManager.h>
#include <set>
namespace FEXCore::Frontend {
using namespace FEXCore::X86Tables;
static uint32_t MapModRMToReg(uint8_t REX, uint8_t bits, bool HighBits, bool HasREX, bool HasXMM, bool HasMM, uint8_t InvalidOffset = 16) {
using GPRArray = std::array<uint32_t, 16>;
static constexpr GPRArray GPRIndexes = {
constexpr std::array<uint64_t, 16> GPRIndexes = {
// Classical ordering?
FEXCore::X86State::REG_RAX,
FEXCore::X86State::REG_RCX,
@@ -43,7 +40,7 @@ static uint32_t MapModRMToReg(uint8_t REX, uint8_t bits, bool HighBits, bool Has
FEXCore::X86State::REG_R15,
};
static constexpr GPRArray GPR8BitHighIndexes = {
constexpr std::array<uint64_t, 16> GPR8BitHighIndexes = {
// Classical ordering?
FEXCore::X86State::REG_RAX,
FEXCore::X86State::REG_RCX,
@@ -63,7 +60,7 @@ static uint32_t MapModRMToReg(uint8_t REX, uint8_t bits, bool HighBits, bool Has
FEXCore::X86State::REG_R15,
};
static constexpr GPRArray XMMIndexes = {
constexpr std::array<uint64_t, 16> XMMIndexes = {
FEXCore::X86State::REG_XMM_0,
FEXCore::X86State::REG_XMM_1,
FEXCore::X86State::REG_XMM_2,
@@ -82,7 +79,7 @@ static uint32_t MapModRMToReg(uint8_t REX, uint8_t bits, bool HighBits, bool Has
FEXCore::X86State::REG_XMM_15,
};
static constexpr GPRArray MMIndexes = {
constexpr std::array<uint64_t, 16> MMIndexes = {
FEXCore::X86State::REG_MM_0,
FEXCore::X86State::REG_MM_1,
FEXCore::X86State::REG_MM_2,
@@ -101,7 +98,7 @@ static uint32_t MapModRMToReg(uint8_t REX, uint8_t bits, bool HighBits, bool Has
FEXCore::X86State::REG_INVALID
};
const GPRArray *GPRs = &GPRIndexes;
const std::array<uint64_t, 16> *GPRs = &GPRIndexes;
if (HasXMM) {
GPRs = &XMMIndexes;
}
@@ -127,13 +124,13 @@ Decoder::Decoder(FEXCore::Context::Context *ctx)
uint8_t Decoder::ReadByte() {
uint8_t Byte = InstStream[InstructionSize];
LOGMAN_THROW_A(InstructionSize < MAX_INST_SIZE, "Max instruction size exceeded!");
LogMan::Throw::A(InstructionSize < MAX_INST_SIZE, "Max instruction size exceeded!");
Instruction[InstructionSize] = Byte;
InstructionSize++;
return Byte;
}
uint8_t Decoder::PeekByte(uint8_t Offset) const {
uint8_t Decoder::PeekByte(uint8_t Offset) {
uint8_t Byte = InstStream[InstructionSize + Offset];
return Byte;
}
@@ -144,7 +141,7 @@ uint64_t Decoder::ReadData(uint8_t Size) {
}
if (Size > sizeof(uint64_t)) {
LOGMAN_MSG_A("Unknown data size to read");
LogMan::Msg::A("Unknown data size to read");
return 0;
}
@@ -199,9 +196,9 @@ void Decoder::DecodeModRM_16(X86Tables::DecodedOperand *Operand, X86Tables::ModR
}
}
Operand->Type = DecodedOperand::OpType::SIB;
Operand->Data.SIB.Scale = 1;
Operand->Data.SIB.Offset = Literal;
Operand->TypeSIB.Type = DecodedOperand::TYPE_SIB;
Operand->TypeSIB.Scale = 1;
Operand->TypeSIB.Offset = Literal;
// Only called when ModRM.mod != 0b11
struct Encodings {
@@ -240,8 +237,8 @@ void Decoder::DecodeModRM_16(X86Tables::DecodedOperand *Operand, X86Tables::ModR
uint8_t LookupIndex = ModRM.mod << 3 | ModRM.rm;
auto it = Lookup[LookupIndex];
Operand->Data.SIB.Base = it.Base;
Operand->Data.SIB.Index = it.Index;
Operand->TypeSIB.Base = it.Base;
Operand->TypeSIB.Index = it.Index;
}
void Decoder::DecodeModRM_64(X86Tables::DecodedOperand *Operand, X86Tables::ModRMDecoded ModRM) {
@@ -279,21 +276,21 @@ void Decoder::DecodeModRM_64(X86Tables::DecodedOperand *Operand, X86Tables::ModR
}
// SIB
Operand->Type = DecodedOperand::OpType::SIB;
Operand->Data.SIB.Scale = 1 << SIB.scale;
Operand->TypeSIB.Type = DecodedOperand::TYPE_SIB;
Operand->TypeSIB.Scale = 1 << SIB.scale;
// The invalid encoding types are described at Table 1-12. "promoted nsigned is always non-zero"
Operand->Data.SIB.Index = MapModRMToReg(DecodeInst->Flags & DecodeFlags::FLAG_REX_XGPR_X ? 1 : 0, SIB.index, false, false, false, false, 0b100);
Operand->Data.SIB.Base = MapModRMToReg(DecodeInst->Flags & DecodeFlags::FLAG_REX_XGPR_B ? 1 : 0, SIB.base, false, false, false, false, ModRM.mod == 0 ? 0b101 : 16);
Operand->TypeSIB.Index = MapModRMToReg(DecodeInst->Flags & DecodeFlags::FLAG_REX_XGPR_X ? 1 : 0, SIB.index, false, false, false, false, 0b100);
Operand->TypeSIB.Base = MapModRMToReg(DecodeInst->Flags & DecodeFlags::FLAG_REX_XGPR_B ? 1 : 0, SIB.base, false, false, false, false, ModRM.mod == 0 ? 0b101 : 16);
uint64_t Literal {0};
LOGMAN_THROW_A(Displacement <= 4, "Number of bytes should be <= 4 for literal src");
LogMan::Throw::A(Displacement <= 4, "Number of bytes should be <= 4 for literal src");
Literal = ReadData(Displacement);
if (Displacement == 1) {
Literal = static_cast<int8_t>(Literal);
}
Operand->Data.SIB.Offset = Literal;
Operand->TypeSIB.Offset = Literal;
}
else if (ModRM.mod == 0) {
// Explained in Table 1-14. "Operand Addressing Using ModRM and SIB Bytes"
@@ -302,13 +299,13 @@ void Decoder::DecodeModRM_64(X86Tables::DecodedOperand *Operand, X86Tables::ModR
uint32_t Literal;
Literal = ReadData(4);
Operand->Type = DecodedOperand::OpType::RIPRelative;
Operand->Data.RIPLiteral.Value.u = Literal;
Operand->TypeRIPLiteral.Type = DecodedOperand::TYPE_RIP_RELATIVE;
Operand->TypeRIPLiteral.Literal.u = Literal;
}
else {
// Register-direct addressing
Operand->Type = DecodedOperand::OpType::GPRDirect;
Operand->Data.GPR.GPR = MapModRMToReg(DecodeInst->Flags & DecodeFlags::FLAG_REX_XGPR_B ? 1 : 0, ModRM.rm, false, false, false, false);
Operand->TypeGPR.Type = DecodedOperand::TYPE_GPR_DIRECT;
Operand->TypeGPR.GPR = MapModRMToReg(DecodeInst->Flags & DecodeFlags::FLAG_REX_XGPR_B ? 1 : 0, ModRM.rm, false, false, false, false);
}
}
else {
@@ -320,9 +317,9 @@ void Decoder::DecodeModRM_64(X86Tables::DecodedOperand *Operand, X86Tables::ModR
}
Displacement = DisplacementSize;
Operand->Type = DecodedOperand::OpType::GPRIndirect;
Operand->Data.GPRIndirect.GPR = MapModRMToReg(DecodeInst->Flags & DecodeFlags::FLAG_REX_XGPR_B ? 1 : 0, ModRM.rm, false, false, false, false);
Operand->Data.GPRIndirect.Displacement = Literal;
Operand->TypeGPRIndirect.Type = DecodedOperand::TYPE_GPR_INDIRECT;
Operand->TypeGPRIndirect.GPR = MapModRMToReg(DecodeInst->Flags & DecodeFlags::FLAG_REX_XGPR_B ? 1 : 0, ModRM.rm, false, false, false, false);
Operand->TypeGPRIndirect.Displacement = Literal;
}
}
@@ -346,7 +343,7 @@ bool Decoder::NormalOp(FEXCore::X86Tables::X86InstInfo const *Info, uint16_t Op)
return false;
}
LOGMAN_THROW_A(!(Info->Type >= FEXCore::X86Tables::TYPE_GROUP_1 && Info->Type <= FEXCore::X86Tables::TYPE_GROUP_P),
LogMan::Throw::A(!(Info->Type >= FEXCore::X86Tables::TYPE_GROUP_1 && Info->Type <= FEXCore::X86Tables::TYPE_GROUP_P),
"Group Ops should have been decoded before this!");
uint8_t DestSize{};
@@ -462,25 +459,22 @@ bool Decoder::NormalOp(FEXCore::X86Tables::X86InstInfo const *Info, uint16_t Op)
if (HAS_NON_XMM_SUBFLAG(Info->Flags, FEXCore::X86Tables::InstFlags::FLAGS_SF_DST_RAX) ||
HAS_NON_XMM_SUBFLAG(Info->Flags, FEXCore::X86Tables::InstFlags::FLAGS_SF_DST_RDX)) {
// Some instructions hardcode their destination as RAX
CurrentDest->Type = DecodedOperand::OpType::GPR;
CurrentDest->Data.GPR.HighBits = false;
CurrentDest->Data.GPR.GPR = HAS_NON_XMM_SUBFLAG(Info->Flags, FEXCore::X86Tables::InstFlags::FLAGS_SF_DST_RAX) ? FEXCore::X86State::REG_RAX : FEXCore::X86State::REG_RDX;
CurrentDest->TypeGPR.Type = DecodedOperand::TYPE_GPR;
CurrentDest->TypeGPR.HighBits = false;
CurrentDest->TypeGPR.GPR = HAS_NON_XMM_SUBFLAG(Info->Flags, FEXCore::X86Tables::InstFlags::FLAGS_SF_DST_RAX) ? FEXCore::X86State::REG_RAX : FEXCore::X86State::REG_RDX;
CurrentDest = &DecodeInst->Src[0];
}
if (HAS_NON_XMM_SUBFLAG(Info->Flags, FEXCore::X86Tables::InstFlags::FLAGS_SF_REX_IN_BYTE)) {
LOGMAN_THROW_A(!HasMODRM, "This instruction shouldn't have ModRM!");
LogMan::Throw::A(!HasMODRM, "This instruction shouldn't have ModRM!");
// If the REX is in the byte that means the lower nibble of the OP contains the destination GPR
// This also means that the destination is always a GPR on these ones
// ADDITIONALLY:
// If there is a REX prefix then that allows extended GPR usage
CurrentDest->Type = DecodedOperand::OpType::GPR;
DecodeInst->Dest.Data.GPR.HighBits = (Is8BitDest && !HasREX && (Op & 0b111) >= 0b100) || HasHighXMM;
CurrentDest->Data.GPR.GPR = MapModRMToReg(DecodeInst->Flags & DecodeFlags::FLAG_REX_XGPR_B ? 1 : 0, Op & 0b111, Is8BitDest, HasREX, false, false);
if (CurrentDest->Data.GPR.GPR == FEXCore::X86State::REG_INVALID)
return false;
CurrentDest->TypeGPR.Type = DecodedOperand::TYPE_GPR;
DecodeInst->Dest.TypeGPR.HighBits = (Is8BitDest && !HasREX && (Op & 0b111) >= 0b100) || HasHighXMM;
CurrentDest->TypeGPR.GPR = MapModRMToReg(DecodeInst->Flags & DecodeFlags::FLAG_REX_XGPR_B ? 1 : 0, Op & 0b111, Is8BitDest, HasREX, false, false);
}
uint8_t Bytes = Info->MoreBytes;
@@ -503,63 +497,55 @@ bool Decoder::NormalOp(FEXCore::X86Tables::X86InstInfo const *Info, uint16_t Op)
ModRM.Hex = DecodeInst->ModRM;
// Decode the GPR source first
GPR.Type = DecodedOperand::OpType::GPR;
GPR.Data.GPR.HighBits = (GPR8Bit && ModRM.reg >= 0b100 && !HasREX) || HasHighXMM;
GPR.Data.GPR.GPR = MapModRMToReg(DecodeInst->Flags & DecodeFlags::FLAG_REX_XGPR_R ? 1 : 0, ModRM.reg, GPR8Bit, HasREX, HasXMMGPR, HasMMGPR);
if (GPR.Data.GPR.GPR == FEXCore::X86State::REG_INVALID)
return false;
GPR.TypeGPR.Type = DecodedOperand::TYPE_GPR;
GPR.TypeGPR.HighBits = (GPR8Bit && ModRM.reg >= 0b100 && !HasREX) || HasHighXMM;
GPR.TypeGPR.GPR = MapModRMToReg(DecodeInst->Flags & DecodeFlags::FLAG_REX_XGPR_R ? 1 : 0, ModRM.reg, GPR8Bit, HasREX, HasXMMGPR, HasMMGPR);
// ModRM.mod == 0b11 == Register
// ModRM.Mod != 0b11 == Register-direct addressing
if (ModRM.mod == 0b11) {
NonGPR.Type = DecodedOperand::OpType::GPR;
NonGPR.Data.GPR.HighBits = (NonGPR8Bit && ModRM.rm >= 0b100 && !HasREX) || HasHighXMM;
NonGPR.Data.GPR.GPR = MapModRMToReg(DecodeInst->Flags & DecodeFlags::FLAG_REX_XGPR_B ? 1 : 0, ModRM.rm, NonGPR8Bit, HasREX, HasXMMNonGPR, HasMMNonGPR);
if (NonGPR.Data.GPR.GPR == FEXCore::X86State::REG_INVALID)
return false;
NonGPR.TypeGPR.Type = DecodedOperand::TYPE_GPR;
NonGPR.TypeGPR.HighBits = (NonGPR8Bit && ModRM.rm >= 0b100 && !HasREX) || HasHighXMM;
NonGPR.TypeGPR.GPR = MapModRMToReg(DecodeInst->Flags & DecodeFlags::FLAG_REX_XGPR_B ? 1 : 0, ModRM.rm, NonGPR8Bit, HasREX, HasXMMNonGPR, HasMMNonGPR);
}
else {
auto Disp = DecodeModRMs_Disp[Has16BitAddressing];
(this->*Disp)(&NonGPR, ModRM);
}
return true;
};
size_t CurrentSrc = 0;
if (Info->Flags & FEXCore::X86Tables::InstFlags::FLAGS_MODRM) {
if (Info->Flags & FEXCore::X86Tables::InstFlags::FLAGS_SF_MOD_DST) {
if (!ModRMOperand(DecodeInst->Src[CurrentSrc], DecodeInst->Dest, HasXMMSrc, HasXMMDst, HasMMSrc, HasMMDst, Is8BitSrc, Is8BitDest))
return false;
ModRMOperand(DecodeInst->Src[CurrentSrc], DecodeInst->Dest, HasXMMSrc, HasXMMDst, HasMMSrc, HasMMDst, Is8BitSrc, Is8BitDest);
}
else {
if (!ModRMOperand(DecodeInst->Dest, DecodeInst->Src[CurrentSrc], HasXMMDst, HasXMMSrc, HasMMDst, HasMMSrc, Is8BitDest, Is8BitSrc))
return false;
ModRMOperand(DecodeInst->Dest, DecodeInst->Src[CurrentSrc], HasXMMDst, HasXMMSrc, HasMMDst, HasMMSrc, Is8BitDest, Is8BitSrc);
}
++CurrentSrc;
}
if (HAS_NON_XMM_SUBFLAG(Info->Flags, FEXCore::X86Tables::InstFlags::FLAGS_SF_SRC_RAX)) {
DecodeInst->Src[CurrentSrc].Type = DecodedOperand::OpType::GPR;
DecodeInst->Src[CurrentSrc].Data.GPR.HighBits = false;
DecodeInst->Src[CurrentSrc].Data.GPR.GPR = FEXCore::X86State::REG_RAX;
DecodeInst->Src[CurrentSrc].TypeGPR.Type = DecodedOperand::TYPE_GPR;
DecodeInst->Src[CurrentSrc].TypeGPR.HighBits = false;
DecodeInst->Src[CurrentSrc].TypeGPR.GPR = FEXCore::X86State::REG_RAX;
++CurrentSrc;
}
else if (HAS_NON_XMM_SUBFLAG(Info->Flags, FEXCore::X86Tables::InstFlags::FLAGS_SF_SRC_RCX)) {
DecodeInst->Src[CurrentSrc].Type = DecodedOperand::OpType::GPR;
DecodeInst->Src[CurrentSrc].Data.GPR.HighBits = false;
DecodeInst->Src[CurrentSrc].Data.GPR.GPR = FEXCore::X86State::REG_RCX;
DecodeInst->Src[CurrentSrc].TypeGPR.Type = DecodedOperand::TYPE_GPR;
DecodeInst->Src[CurrentSrc].TypeGPR.HighBits = false;
DecodeInst->Src[CurrentSrc].TypeGPR.GPR = FEXCore::X86State::REG_RCX;
++CurrentSrc;
}
if (Bytes != 0) {
LOGMAN_THROW_A(Bytes <= 8, "Number of bytes should be <= 8 for literal src");
LogMan::Throw::A(Bytes <= 8, "Number of bytes should be <= 8 for literal src");
DecodeInst->Src[CurrentSrc].Data.Literal.Size = Bytes;
DecodeInst->Src[CurrentSrc].TypeLiteral.Size = Bytes;
uint64_t Literal = ReadData(Bytes);
uint64_t Literal {0};
Literal = ReadData(Bytes);
if ((Info->Flags & FEXCore::X86Tables::InstFlags::FLAGS_SRC_SEXT) ||
(DecodeFlags::GetSizeDstFlags(DecodeInst->Flags) == DecodeFlags::SIZE_64BIT && Info->Flags & FEXCore::X86Tables::InstFlags::FLAGS_SRC_SEXT64BIT)) {
@@ -572,15 +558,15 @@ bool Decoder::NormalOp(FEXCore::X86Tables::X86InstInfo const *Info, uint16_t Op)
else {
Literal = static_cast<int32_t>(Literal);
}
DecodeInst->Src[CurrentSrc].Data.Literal.Size = DestSize;
DecodeInst->Src[CurrentSrc].TypeLiteral.Size = DestSize;
}
Bytes = 0;
DecodeInst->Src[CurrentSrc].Type = DecodedOperand::OpType::Literal;
DecodeInst->Src[CurrentSrc].Data.Literal.Value = Literal;
DecodeInst->Src[CurrentSrc].TypeLiteral.Type = DecodedOperand::TYPE_LITERAL;
DecodeInst->Src[CurrentSrc].TypeLiteral.Literal = Literal;
}
LOGMAN_THROW_A(Bytes == 0, "Inst at 0x%lx: 0x%04x '%s' Had an instruction of size %d with %d remaining", DecodeInst->PC, DecodeInst->OP, DecodeInst->TableInfo->Name, InstructionSize, Bytes);
LogMan::Throw::A(Bytes == 0, "Inst at 0x%lx: 0x%04x '%s' Had an instruction of size %d with %d remaining", DecodeInst->PC, DecodeInst->OP, DecodeInst->TableInfo->Name, InstructionSize, Bytes);
DecodeInst->InstSize = InstructionSize;
return true;
}
@@ -605,7 +591,7 @@ bool Decoder::NormalOpHeader(FEXCore::X86Tables::X86InstInfo const *Info, uint16
return false;
}
LOGMAN_THROW_A(Info->Type != FEXCore::X86Tables::TYPE_REX_PREFIX, "REX PREFIX should have been decoded before this!");
LogMan::Throw::A(Info->Type != FEXCore::X86Tables::TYPE_REX_PREFIX, "REX PREFIX should have been decoded before this!");
if (Info->Type >= FEXCore::X86Tables::TYPE_GROUP_1 &&
Info->Type <= FEXCore::X86Tables::TYPE_GROUP_11) {
@@ -661,7 +647,7 @@ bool Decoder::NormalOpHeader(FEXCore::X86Tables::X86InstInfo const *Info, uint16
3,
};
uint8_t Field = RegToField[ModRM.reg];
LOGMAN_THROW_A(Field != 255, "Invalid field selected!");
LogMan::Throw::A(Field != 255, "Invalid field selected!");
LocalOp = (Field << 3) | ModRM.rm;
return NormalOp(&SecondModRMTableOps[LocalOp], LocalOp);
@@ -695,10 +681,7 @@ bool Decoder::NormalOpHeader(FEXCore::X86Tables::X86InstInfo const *Info, uint16
uint8_t Byte2 = ReadByte();
pp = Byte2 & 0b11;
map_select = Byte1 & 0b11111;
if (!(map_select >= 1 && map_select <= 3)) {
LogMan::Msg::E("We don't understand a map_select of: %d", map_select);
return false;
}
LogMan::Throw::A(map_select >= 1 && map_select <= 3, "We don't understand a map_select of: %d", map_select);
}
uint16_t VEXOp = ReadByte();
@@ -746,8 +729,6 @@ bool Decoder::DecodeInstruction(uint64_t PC) {
DecodeInst->PC = PC;
for(;;) {
if (InstructionSize >= MAX_INST_SIZE)
return false;
uint8_t Op = ReadByte();
switch (Op) {
case 0x0F: {// Escape Op
@@ -898,7 +879,7 @@ bool Decoder::DecodeInstruction(uint64_t PC) {
auto Info = &FEXCore::X86Tables::BaseOps[Op];
if (Info->Type == FEXCore::X86Tables::TYPE_REX_PREFIX) {
LOGMAN_THROW_A(CTX->Config.Is64BitMode, "Got REX prefix in 32bit mode");
LogMan::Throw::A(CTX->Config.Is64BitMode, "Got REX prefix in 32bit mode");
DecodeInst->Flags |= DecodeFlags::FLAG_REX_PREFIX;
// Widening displacement
@@ -928,10 +909,6 @@ bool Decoder::DecodeInstruction(uint64_t PC) {
}
if (DecodeInst->Dest.IsGPR()) {
assert(DecodeInst->Dest.Data.GPR.GPR != 255);
}
return true;
}
@@ -941,7 +918,7 @@ void Decoder::BranchTargetInMultiblockRange() {
// If the RIP setting is conditional AND within our symbol range then it can be considered for multiblock
uint64_t TargetRIP = 0;
const uint8_t GPRSize = CTX->GetGPRSize();
uint8_t GPRSize = CTX->Config.Is64BitMode ? 8 : 4;
bool Conditional = true;
switch (DecodeInst->OP) {
@@ -951,23 +928,19 @@ void Decoder::BranchTargetInMultiblockRange() {
// auto RIPOffset = LoadSource(Op, Op->Src[0], Op->Flags);
// auto RIPTargetConst = _Constant(Op->PC + Op->InstSize);
// Target offset is PC + InstSize + Literal
LOGMAN_THROW_A(DecodeInst->Src[0].IsLiteral(), "Had wrong operand type");
TargetRIP = DecodeInst->PC + DecodeInst->InstSize + DecodeInst->Src[0].Data.Literal.Value;
LogMan::Throw::A(DecodeInst->Src[0].TypeNone.Type == DecodedOperand::TYPE_LITERAL, "Had wrong operand type");
TargetRIP = DecodeInst->PC + DecodeInst->InstSize + DecodeInst->Src[0].TypeLiteral.Literal;
break;
}
case 0xE9:
case 0xEB: // Both are unconditional JMP instructions
LOGMAN_THROW_A(DecodeInst->Src[0].IsLiteral(), "Had wrong operand type");
TargetRIP = DecodeInst->PC + DecodeInst->InstSize + DecodeInst->Src[0].Data.Literal.Value;
LogMan::Throw::A(DecodeInst->Src[0].TypeNone.Type == DecodedOperand::TYPE_LITERAL, "Had wrong operand type");
TargetRIP = DecodeInst->PC + DecodeInst->InstSize + DecodeInst->Src[0].TypeLiteral.Literal;
Conditional = false;
break;
case 0xE8: // Call - Immediate target, We don't want to inline calls
if (ExternalBranches) {
ExternalBranches->insert(DecodeInst->PC + DecodeInst->InstSize);
}
[[fallthrough]];
case 0xC2: // RET imm
case 0xC3: // RET
case 0xE8: // Call - Immediate target, We don't want to inline calls
default:
return;
break;
@@ -997,10 +970,6 @@ void Decoder::BranchTargetInMultiblockRange() {
BlocksToDecode.find(TargetRIP) == BlocksToDecode.end()) {
BlocksToDecode.emplace(TargetRIP);
}
} else {
if (ExternalBranches) {
ExternalBranches->insert(TargetRIP);
}
}
}
@@ -1024,7 +993,7 @@ bool Decoder::DecodeInstructionsAtEntry(uint8_t const* _InstStream, uint64_t PC)
// If we don't have symbols available then we become a bit optimistic about multiblock ranges
if (!SymbolAvailable) {
// If we don't have a symbol available then assume all branches are valid for multiblock
SymbolMaxAddress = SectionMaxAddress;
SymbolMaxAddress = ~0ULL;
SymbolMinAddress = EntryPoint;
}
@@ -1054,10 +1023,7 @@ bool Decoder::DecodeInstructionsAtEntry(uint8_t const* _InstStream, uint64_t PC)
if (ErrorDuringDecoding) {
LogMan::Msg::D("Couldn't Decode something at 0x%lx, Started at 0x%lx", PC + PCOffset, PC);
if (Blocks.size() == 1) {
return false;
}
LOGMAN_THROW_A(Blocks.size() != 1, "Decode Error in entry block");
LogMan::Throw::A(Blocks.size() != 1, "Decode Error in entry block");
CurrentBlockDecoding.HasInvalidInstruction = true;
if (ErrorDuringDecoding && Blocks.size() != 1) {
+2 -6
View File
@@ -26,15 +26,13 @@ public:
Decoder(FEXCore::Context::Context *ctx);
bool DecodeInstructionsAtEntry(uint8_t const* InstStream, uint64_t PC);
std::vector<DecodedBlocks> const *GetDecodedBlocks() const {
std::vector<DecodedBlocks> const *GetDecodedBlocks() {
return &Blocks;
}
uint64_t DecodedMinAddress {};
uint64_t DecodedMaxAddress {~0ULL};
void SetSectionMaxAddress(uint64_t v) { SectionMaxAddress = v; }
void SetExternalBranches(std::set<uint64_t> *v) { ExternalBranches = v; }
private:
FEXCore::Context::Context *CTX;
@@ -43,7 +41,7 @@ private:
void BranchTargetInMultiblockRange();
uint8_t ReadByte();
uint8_t PeekByte(uint8_t Offset) const;
uint8_t PeekByte(uint8_t Offset);
uint64_t ReadData(uint8_t Size);
void SkipBytes(uint8_t Size) { InstructionSize += Size; }
bool NormalOp(FEXCore::X86Tables::X86InstInfo const *Info, uint16_t Op);
@@ -67,12 +65,10 @@ private:
uint64_t MaxCondBranchBackwards {~0ULL};
uint64_t SymbolMaxAddress {};
uint64_t SymbolMinAddress {~0ULL};
uint64_t SectionMaxAddress {~0ULL};
std::vector<DecodedBlocks> Blocks;
std::set<uint64_t> BlocksToDecode;
std::set<uint64_t> HasBlocks;
std::set<uint64_t> *ExternalBranches {nullptr};
// ModRM rm decoding
using DecodeModRMPtr = void (FEXCore::Frontend::Decoder::*)(X86Tables::DecodedOperand *Operand, X86Tables::ModRMDecoded ModRM);
+93 -77
View File
@@ -15,19 +15,15 @@ $end_info$
#include <optional>
#include "Common/NetStream.h"
#include "Common/SoftFloat.h"
#include <FEXCore/Utils/CompilerDefs.h>
#include <FEXCore/Utils/LogManager.h>
#include <cstring>
#include <fcntl.h>
#include <fmt/format.h>
#include <fstream>
#include <netdb.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netdb.h>
#include <string.h>
#include <fcntl.h>
#include <unistd.h>
#include <fstream>
#include "GdbServer.h"
#include <FEXCore/Core/CodeLoader.h>
@@ -38,20 +34,20 @@ namespace FEXCore
void GdbServer::Break(int signal) {
std::lock_guard lk(sendMutex);
if (!CommsStream) {
return;
}
const auto str = fmt::format("S{:02x}", signal);
SendPacket(*CommsStream, str);
std::ostringstream ss;
ss << "S" << std::setfill('0') << std::setw(2) << std::hex << signal;
if (CommsStream)
SendPacket(*CommsStream, ss.str());
}
GdbServer::GdbServer(FEXCore::Context::Context *ctx) : CTX(ctx) {
Context::SetExitHandler(ctx, [this](uint64_t ThreadId, FEXCore::Context::ExitReason ExitReason) {
ctx->CustomExitHandler = [this](uint64_t ThreadId, FEXCore::Context::ExitReason ExitReason) {
if (ExitReason == FEXCore::Context::ExitReason::EXIT_DEBUG) {
this->Break(SIGTRAP);
}
});
};
// This is a total hack as there is currently no way to resume once hitting a segfault
// But it's semi-useful for debugging.
@@ -64,12 +60,12 @@ GdbServer::GdbServer(FEXCore::Context::Context *ctx) : CTX(ctx) {
usleep(100000);
return true;
}, true);
});
StartThread();
}
static int calculateChecksum(const std::string &packet) {
static int calculateChecksum(std::string &packet) {
unsigned char checksum = 0;
for (const char &c : packet) {
checksum += c;
@@ -103,9 +99,11 @@ static std::string encodeHex(unsigned char *data, size_t length) {
}
static std::string getThreadName(uint32_t ThreadID) {
const auto ThreadFile = fmt::format("/proc/{}/task/{}/comm", getpid(), ThreadID);
std::fstream fs(ThreadFile, std::fstream::in | std::fstream::binary);
std::fstream fs;
std::ostringstream ThreadFile;
ThreadFile << "/proc/" << getpid() << "/task/" << ThreadID << "/comm";
fs.open(ThreadFile.str(), std::fstream::in | std::fstream::binary);
if (fs.is_open()) {
std::string ThreadName;
fs >> ThreadName;
@@ -137,7 +135,7 @@ std::string GdbServer::ReadPacket(std::iostream &stream) {
switch(c) {
case '$': // start of packet
if (packet.size() != 0)
LogMan::Msg::EFmt("Dropping unexpected data: \"{}\"", packet);
LogMan::Msg::E("Dropping unexpected data: \"%s\"", packet.c_str());
// clear any existing data, must have been a mistake.
packet = std::string();
@@ -158,7 +156,7 @@ std::string GdbServer::ReadPacket(std::iostream &stream) {
if (calculateChecksum(packet) == expected_checksum) {
return packet;
} else {
LogMan::Msg::EFmt("Received Invalid Packet: ${}#{:02x}", packet, expected_checksum);
LogMan::Msg::E("Received Invalid Packet: $%s#%02x %c%c", packet.c_str(), expected_checksum);
}
break;
}
@@ -171,10 +169,10 @@ std::string GdbServer::ReadPacket(std::iostream &stream) {
return "";
}
static std::string escapePacket(const std::string& packet) {
static std::string escapePacket(std::string packet) {
std::ostringstream ss;
for(const auto &c : packet) {
for(auto &c : packet) {
switch (c) {
case '$':
case '#':
@@ -193,11 +191,13 @@ static std::string escapePacket(const std::string& packet) {
return ss.str();
}
void GdbServer::SendPacket(std::ostream &stream, const std::string& packet) {
const auto escaped = escapePacket(packet);
const auto str = fmt::format("${}#{:02x}", escaped, calculateChecksum(escaped));
void GdbServer::SendPacket(std::ostream &stream, std::string packet) {
auto escaped = escapePacket(packet);
std::ostringstream ss;
stream << str << std::flush;
ss << '$' << escaped << '#';
ss << std::setfill('0') << std::setw(2) << std::hex << (int)calculateChecksum(escaped);
stream << ss.str() << std::flush;
}
void GdbServer::SendACK(std::ostream &stream, bool NACK) {
@@ -218,7 +218,7 @@ void GdbServer::SendACK(std::ostream &stream, bool NACK) {
}
}
struct FEX_PACKED GDBContextDefinition {
struct __attribute__((packed)) GDBContextDefinition {
uint64_t gregs[16];
uint64_t rip;
uint32_t eflags;
@@ -279,7 +279,7 @@ std::string GdbServer::readRegs() {
return encodeHex((unsigned char *)&GDB, sizeof(GDBContextDefinition));
}
GdbServer::HandledPacketType GdbServer::readReg(const std::string& packet) {
GdbServer::HandledPacketType GdbServer::readReg(std::string& packet) {
size_t addr;
auto ss = std::istringstream(packet);
ss.get(); // Drop first letter
@@ -357,7 +357,7 @@ GdbServer::HandledPacketType GdbServer::readReg(const std::string& packet) {
return {encodeHex((unsigned char *)(&Empty), sizeof(uint32_t)), HandledPacketType::TYPE_ACK};
}
LogMan::Msg::EFmt("Unknown GDB register 0x{:x}", addr);
LogMan::Msg::E("Unknown GDB register 0x%lx", addr);
return {"E00", HandledPacketType::TYPE_ACK};
}
@@ -462,7 +462,7 @@ std::string buildTargetXML() {
return xml.str();
}
GdbServer::HandledPacketType GdbServer::handleXfer(const std::string &packet) {
GdbServer::HandledPacketType GdbServer::handleXfer(std::string &packet) {
std::string object;
std::string rw;
std::string annex;
@@ -548,9 +548,10 @@ GdbServer::HandledPacketType GdbServer::handleXfer(const std::string &packet) {
static size_t CheckMemMapping(uint64_t Address, size_t Size) {
uint64_t AddressEnd = Address + Size;
std::fstream fs("/proc/self/maps", std::fstream::in | std::fstream::binary);
std::string Line;
std::fstream fs;
fs.open("/proc/self/maps", std::fstream::in | std::fstream::binary);
std::string Line;
while (std::getline(fs, Line)) {
if (fs.eof()) break;
uint64_t Begin, End;
@@ -567,29 +568,32 @@ static size_t CheckMemMapping(uint64_t Address, size_t Size) {
}
}
fs.close();
return 0;
}
GdbServer::HandledPacketType GdbServer::handleProgramOffsets() {
std::fstream fs("/proc/self/maps", std::fstream::in | std::fstream::binary);
std::fstream fs;
fs.open("/proc/self/maps", std::fstream::in | std::fstream::binary);
std::string Line;
std::string const &RuntimeExecutable = Filename();
while (std::getline(fs, Line)) {
uint64_t Begin, End;
char Filename[255];
if (sscanf(Line.c_str(), "%lx-%lx %*c%*c%*c%*c %*x %*x:%*x %*d%s", &Begin, &End, Filename) == 3) {
if (RuntimeExecutable == Filename) {
auto str = fmt::format("Text={:x};Data={:x};Bss={:x}", Begin, Begin, Begin);
return {std::move(str), HandledPacketType::TYPE_ACK};
std::ostringstream ss;
ss << "Text=" << std::hex << Begin << ";Data=" << std::hex << Begin << ";Bss=" << std::hex << Begin;
ss << std::flush;
return {ss.str(), HandledPacketType::TYPE_ACK};
}
}
}
fs.close();
return {"Text=0;Data=0;Bss=0", HandledPacketType::TYPE_ACK};
}
GdbServer::HandledPacketType GdbServer::handleMemory(const std::string &packet) {
GdbServer::HandledPacketType GdbServer::handleMemory(std::string &packet) {
bool write;
size_t addr;
size_t length;
@@ -630,8 +634,8 @@ GdbServer::HandledPacketType GdbServer::handleMemory(const std::string &packet)
}
GdbServer::HandledPacketType GdbServer::handleQuery(const std::string &packet) {
const auto match = [&](const char *str) -> bool { return packet.rfind(str, 0) == 0; };
GdbServer::HandledPacketType GdbServer::handleQuery(std::string &packet) {
auto match = [&](const char *str) -> bool { return packet.rfind(str, 0) == 0; };
if (match("qSupported")) {
return {"PacketSize=5000;xmlRegisters=i386;qXfer:exec-file:read+;qXfer:features:read+;", HandledPacketType::TYPE_ACK};
@@ -689,8 +693,8 @@ GdbServer::HandledPacketType GdbServer::handleQuery(const std::string &packet) {
return {"", HandledPacketType::TYPE_UNKNOWN};
}
GdbServer::HandledPacketType GdbServer::handleV(const std::string& packet) {
const auto match = [&](const std::string& str) -> std::optional<std::istringstream> {
GdbServer::HandledPacketType GdbServer::handleV(std::string& packet) {
auto match = [&](std::string str) -> std::optional<std::istringstream> {
if (packet.rfind(str, 0) == 0) {
auto ss = std::istringstream(packet);
ss.seekg(str.size());
@@ -699,11 +703,18 @@ GdbServer::HandledPacketType GdbServer::handleV(const std::string& packet) {
return std::nullopt;
};
const auto F = [](int result) { return fmt::format("F{:x}", result); };
const auto F_error = [] { return fmt::format("F-1,{:x}", errno); };
const auto F_data = [](int result, const std::string& data) {
return fmt::format("F{:x};{}", result, data);
};
auto F = [](int result) {
std::ostringstream ss;
ss << "F" << std::hex << result;
return ss.str(); };
auto F_error = [&]() {
std::ostringstream ss;
ss << "F-1," << std::hex << errno;
return ss.str(); };
auto F_data = [&](int result, std::string data) {
std::ostringstream ss;
ss << "F" << std::hex << result << ";" << data;
return ss.str(); };
std::optional<std::istringstream> ss;
if((ss = match("vFile:open:"))) {
@@ -725,11 +736,11 @@ GdbServer::HandledPacketType GdbServer::handleV(const std::string& packet) {
return {F(pid == 0 ? 0 : -1), HandledPacketType::TYPE_ACK}; // Only support the common filesystem
}
if((ss = match("vFile:close:"))) {
int fd;
*ss >> std::hex >> fd;
close(fd);
return {F(0), HandledPacketType::TYPE_ACK};
}
int fd;
*ss >> std::hex >> fd;
close(fd);
return {F(0), HandledPacketType::TYPE_ACK};
}
if((ss = match("vFile:pread:"))) {
int fd, count, offset;
@@ -766,7 +777,7 @@ GdbServer::HandledPacketType GdbServer::handleV(const std::string& packet) {
}
if (ss->fail()) {
return {"E00", HandledPacketType::TYPE_ACK};
return {"E00", HandledPacketType::TYPE_ACK};
}
switch (action) {
@@ -776,25 +787,27 @@ GdbServer::HandledPacketType GdbServer::handleV(const std::string& packet) {
}
case 's': {
CTX->Step();
SendPacketPair({"OK", HandledPacketType::TYPE_ACK});
auto str = fmt::format("T05thread:{:02x};core:2c;", getpid());
SendPacketPair({std::move(str), HandledPacketType::TYPE_ACK});
SendPacketPair({"OK", HandledPacketType::TYPE_ACK});
std::ostringstream ss;
ss << "T05thread:" << std::setfill('0') << std::setw(2) << std::hex << getpid() << ";core:2c;";
SendPacketPair({ss.str(), HandledPacketType::TYPE_ACK});
return {"OK", HandledPacketType::TYPE_ACK};
}
case 't':
// This thread isn't part of the thread pool
CTX->Stop(false /* Ignore current thread */);
return {"OK", HandledPacketType::TYPE_ACK};
return {"OK", HandledPacketType::TYPE_ACK};
default:
return {"E00", HandledPacketType::TYPE_ACK};
return {"E00", HandledPacketType::TYPE_ACK};
}
}
return {"", HandledPacketType::TYPE_ACK};
return {"", HandledPacketType::TYPE_ACK};
}
GdbServer::HandledPacketType GdbServer::handleThreadOp(const std::string &packet) {
const auto match = [&](const char *str) -> bool { return packet.rfind(str, 0) == 0; };
GdbServer::HandledPacketType GdbServer::handleThreadOp(std::string &packet) {
auto match = [&](const char *str) -> bool { return packet.rfind(str, 0) == 0; };
if (match("Hc")) {
// Sets thread to this ID for stepping
@@ -810,7 +823,7 @@ GdbServer::HandledPacketType GdbServer::handleThreadOp(const std::string &packet
if (match("Hg")) {
// Sets thread for "other" operations
auto ss = std::istringstream(packet);
ss.seekg(std::string_view("Hg").size());
ss.seekg(std::string("Hg").size());
ss >> std::hex >> CurrentDebuggingThread;
// This must return quick otherwise IDA complains
@@ -821,7 +834,7 @@ GdbServer::HandledPacketType GdbServer::handleThreadOp(const std::string &packet
return {"", HandledPacketType::TYPE_UNKNOWN};
}
GdbServer::HandledPacketType GdbServer::handleBreakpoint(const std::string &packet) {
GdbServer::HandledPacketType GdbServer::handleBreakpoint(std::string &packet) {
auto ss = std::istringstream(packet);
bool Set{};
@@ -837,15 +850,17 @@ GdbServer::HandledPacketType GdbServer::handleBreakpoint(const std::string &pack
return {"OK", HandledPacketType::TYPE_ACK};
}
GdbServer::HandledPacketType GdbServer::ProcessPacket(const std::string &packet) {
GdbServer::HandledPacketType GdbServer::ProcessPacket(std::string &packet) {
switch (packet[0]) {
case '?': {
// Indicates the reason that the thread has stopped
// Behaviour changes if the target is in non-stop mode
// Binja doesn't support S response here
//return {"S00", HandledPacketType::TYPE_ACK};
auto str = fmt::format("T00thread:{:02x};core:2c;", getpid());
return {std::move(str), HandledPacketType::TYPE_ACK};
std::ostringstream ss;
ss << "T00thread:" << std::setfill('0') << std::setw(2) << std::hex << getpid() << ";core:2c;";
return {ss.str(), HandledPacketType::TYPE_ACK};
}
case 'g':
return {readRegs(), HandledPacketType::TYPE_ACK};
@@ -875,14 +890,14 @@ GdbServer::HandledPacketType GdbServer::ProcessPacket(const std::string &packet)
}
}
void GdbServer::SendPacketPair(const HandledPacketType& response) {
void GdbServer::SendPacketPair(HandledPacketType response) {
std::lock_guard lk(sendMutex);
if (response.TypeResponse == HandledPacketType::TYPE_ACK ||
response.TypeResponse == HandledPacketType::TYPE_ONLYACK) {
SendACK(*CommsStream, false);
}
else if (response.TypeResponse == HandledPacketType::TYPE_NACK ||
response.TypeResponse == HandledPacketType::TYPE_ONLYNACK) {
response.TypeResponse == HandledPacketType::TYPE_ONLYNACK) {
SendACK(*CommsStream, true);
}
@@ -890,8 +905,8 @@ void GdbServer::SendPacketPair(const HandledPacketType& response) {
SendPacket(*CommsStream, "");
}
else if (response.TypeResponse != HandledPacketType::TYPE_ONLYNACK &&
response.TypeResponse != HandledPacketType::TYPE_ONLYACK &&
response.TypeResponse != HandledPacketType::TYPE_NONE) {
response.TypeResponse != HandledPacketType::TYPE_ONLYACK &&
response.TypeResponse != HandledPacketType::TYPE_NONE) {
SendPacket(*CommsStream, response.Response);
}
}
@@ -912,7 +927,7 @@ void GdbServer::GdbServerLoop() {
response = ProcessPacket(packet);
SendPacketPair(response);
if (response.TypeResponse == HandledPacketType::TYPE_UNKNOWN) {
LogMan::Msg::DFmt("Unknown packet {}", packet);
LogMan::Msg::D("Unknown packet %s", packet.c_str());
}
break;
}
@@ -928,12 +943,13 @@ void GdbServer::GdbServerLoop() {
break;
case '\x03': { // ASCII EOT
CTX->Pause();
auto str = fmt::format("T02thread:{:02x};core:2c;", getpid());
SendPacketPair({std::move(str), HandledPacketType::TYPE_ACK});
std::ostringstream ss;
ss << "T02thread:" << std::setfill('0') << std::setw(2) << std::hex << getpid() << ";core:2c;";
SendPacketPair({ss.str(), HandledPacketType::TYPE_ACK});
break;
}
default:
LogMan::Msg::DFmt("GdbServer: Unexpected byte {} ({:02x})", static_cast<char>(c), c);
LogMan::Msg::D("GdbServer: Unexpected byte %c (%02x)", c, c);
}
}
@@ -987,7 +1003,7 @@ std::unique_ptr<std::iostream> GdbServer::OpenSocket() {
// Block until a connection arrives
LogMan::Msg::IFmt("GdbServer, waiting for connection on localhost:8086");
LogMan::Msg::I("GdbServer, waiting for connection on localhost:8086");
listen(sockfd, 1);
new_fd = accept(sockfd, (struct sockaddr *)&their_addr, &addr_size);
+10 -10
View File
@@ -30,7 +30,7 @@ private:
std::unique_ptr<std::iostream> OpenSocket();
void StartThread();
std::string ReadPacket(std::iostream &stream);
void SendPacket(std::ostream &stream, const std::string& packet);
void SendPacket(std::ostream &stream, std::string packet);
void SendACK(std::ostream &stream, bool NACK);
@@ -47,18 +47,18 @@ private:
ResponseType TypeResponse{};
};
void SendPacketPair(const HandledPacketType& packetPair);
HandledPacketType ProcessPacket(const std::string &packet);
HandledPacketType handleQuery(const std::string &packet);
HandledPacketType handleXfer(const std::string &packet);
HandledPacketType handleMemory(const std::string &packet);
HandledPacketType handleV(const std::string& packet);
HandledPacketType handleThreadOp(const std::string &packet);
HandledPacketType handleBreakpoint(const std::string &packet);
void SendPacketPair(HandledPacketType packetPair);
HandledPacketType ProcessPacket(std::string &packet);
HandledPacketType handleQuery(std::string &packet);
HandledPacketType handleXfer(std::string &packet);
HandledPacketType handleMemory(std::string &packet);
HandledPacketType handleV(std::string& packet);
HandledPacketType handleThreadOp(std::string &packet);
HandledPacketType handleBreakpoint(std::string &packet);
HandledPacketType handleProgramOffsets();
std::string readRegs();
HandledPacketType readReg(const std::string& packet);
HandledPacketType readReg(std::string& packet);
FEXCore::Context::Context *CTX;
std::unique_ptr<FEXCore::Threads::Thread> gdbServerThread;
@@ -22,8 +22,9 @@ using DestMapType = std::vector<uint32_t>;
class InterpreterCore final : public CPUBackend {
public:
explicit InterpreterCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, bool CompileThread);
~InterpreterCore() override;
std::string GetName() override { return "Interpreter"; }
void *CompileCode(uint64_t Entry, FEXCore::IR::IRListView const *IR, FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) override;
void *CompileCode(FEXCore::IR::IRListView const *IR, FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) override;
void *MapRegion(void* HostPtr, uint64_t, uint64_t) override { return HostPtr; }
@@ -45,7 +46,7 @@ private:
template<typename Res>
Res GetSrc(void* SSAData, IR::OrderedNodeWrapper Src);
std::unique_ptr<Dispatcher> Dispatcher{};
Dispatcher *Dispatcher{};
};
}
@@ -31,7 +31,7 @@ static void InterpreterExecution(FEXCore::Core::CpuStateFrame *Frame) {
auto LocalEntry = Thread->LocalIRCache.find(Thread->CurrentFrame->State.rip);
InterpreterOps::InterpretIR(Thread, Thread->CurrentFrame->State.rip, LocalEntry->second.IR.get(), LocalEntry->second.DebugData.get());
InterpreterOps::InterpretIR(Thread, LocalEntry->second.IR.get(), LocalEntry->second.DebugData.get());
}
bool InterpreterCore::HandleSIGBUS(int Signal, void *info, void *ucontext) {
@@ -93,12 +93,12 @@ InterpreterCore::InterpreterCore(FEXCore::Context::Context *ctx, FEXCore::Core::
CTX->SignalDelegation->RegisterHostSignalHandler(SignalDelegator::SIGNAL_FOR_PAUSE, [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext) -> bool {
InterpreterCore *Core = reinterpret_cast<InterpreterCore*>(Thread->CPUBackend.get());
return Core->Dispatcher->HandleSignalPause(Signal, info, ucontext);
}, true);
});
CTX->SignalDelegation->RegisterHostSignalHandler(SIGBUS, [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext) -> bool {
InterpreterCore *Core = reinterpret_cast<InterpreterCore*>(Thread->CPUBackend.get());
return Core->HandleSIGBUS(Signal, info, ucontext);
}, true);
});
auto GuestSignalHandler = [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext, GuestSigAction *GuestAction, stack_t *GuestStack) -> bool {
InterpreterCore *Core = reinterpret_cast<InterpreterCore*>(Thread->CPUBackend.get());
@@ -111,12 +111,18 @@ InterpreterCore::InterpreterCore(FEXCore::Context::Context *ctx, FEXCore::Core::
}
}
void *InterpreterCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IRListView const *IR, [[maybe_unused]] FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) {
InterpreterCore::~InterpreterCore() {
delete Dispatcher;
}
void *InterpreterCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const *IR, [[maybe_unused]] FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) {
return reinterpret_cast<void*>(InterpreterExecution);
}
std::unique_ptr<CPUBackend> CreateInterpreterCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, bool CompileThread) {
return std::make_unique<InterpreterCore>(ctx, Thread, CompileThread);
FEXCore::CPU::CPUBackend *CreateInterpreterCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, bool CompileThread) {
return new InterpreterCore(ctx, Thread, CompileThread);
}
}
@@ -1,7 +1,5 @@
#pragma once
#include <memory>
namespace FEXCore::Context {
struct Context;
}
@@ -13,6 +11,6 @@ namespace FEXCore::Core {
namespace FEXCore::CPU {
class CPUBackend;
std::unique_ptr<CPUBackend> CreateInterpreterCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, bool CompileThread);
FEXCore::CPU::CPUBackend *CreateInterpreterCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, bool CompileThread);
}
File diff suppressed because it is too large. Load diff
@@ -36,7 +36,7 @@ namespace FEXCore::CPU {
class InterpreterOps {
public:
static void InterpretIR(FEXCore::Core::InternalThreadState *Thread, uint64_t Entry, FEXCore::IR::IRListView *CurrentIR, FEXCore::Core::DebugData *DebugData);
static void InterpretIR(FEXCore::Core::InternalThreadState *Thread, FEXCore::IR::IRListView *CurrentIR, FEXCore::Core::DebugData *DebugData);
static bool GetFallbackHandler(IR::IROp_Header *IROp, FallbackInfo *Info);
};
};
+54 -65
View File
@@ -46,7 +46,7 @@ DEF_OP(TruncElementPair) {
mov(Dst.second, Src.second);
break;
}
default: LOGMAN_MSG_A("Unhandled Truncation size: %d", Op->Size); break;
default: LogMan::Msg::A("Unhandled Truncation size: %d", Op->Size); break;
}
}
@@ -59,7 +59,7 @@ DEF_OP(Constant) {
DEF_OP(EntrypointOffset) {
auto Op = IROp->C<IR::IROp_EntrypointOffset>();
auto Constant = Entry + Op->Offset;
auto Constant = IR->GetHeader()->Entry + Op->Offset;
auto Dst = GetReg<RA_64>(Node);
LoadConstant(Dst, Constant);
}
@@ -95,7 +95,7 @@ DEF_OP(Add) {
case 8:
add(GetReg<RA_64>(Node), GetReg<RA_64>(Op->Header.Args[0].ID()), Const);
break;
default: LOGMAN_MSG_A("Unsupported Add size: %d", OpSize);
default: LogMan::Msg::A("Unsupported Add size: %d", OpSize);
}
} else {
switch (OpSize) {
@@ -105,7 +105,7 @@ DEF_OP(Add) {
case 8:
add(GetReg<RA_64>(Node), GetReg<RA_64>(Op->Header.Args[0].ID()), GetReg<RA_64>(Op->Header.Args[1].ID()));
break;
default: LOGMAN_MSG_A("Unsupported Add size: %d", OpSize);
default: LogMan::Msg::A("Unsupported Add size: %d", OpSize);
}
}
}
@@ -121,7 +121,7 @@ DEF_OP(Sub) {
case 8:
sub(GRS(Node), GRS(Op->Header.Args[0].ID()), Const);
break;
default: LOGMAN_MSG_A("Unsupported Sub size: %d", OpSize);
default: LogMan::Msg::A("Unsupported Sub size: %d", OpSize);
}
} else {
switch (OpSize) {
@@ -131,7 +131,7 @@ DEF_OP(Sub) {
case 8:
sub(GetReg<RA_64>(Node), GetReg<RA_64>(Op->Header.Args[0].ID()), GetReg<RA_64>(Op->Header.Args[1].ID()));
break;
default: LOGMAN_MSG_A("Unsupported Sub size: %d", OpSize);
default: LogMan::Msg::A("Unsupported Sub size: %d", OpSize);
}
}
@@ -147,7 +147,7 @@ DEF_OP(Neg) {
case 8:
neg(GetReg<RA_64>(Node), GetReg<RA_64>(Op->Header.Args[0].ID()));
break;
default: LOGMAN_MSG_A("Unsupported Not size: %d", OpSize);
default: LogMan::Msg::A("Unsupported Not size: %d", OpSize);
}
}
@@ -159,11 +159,12 @@ DEF_OP(Mul) {
switch (OpSize) {
case 4:
mul(Dst.W(), GetReg<RA_32>(Op->Header.Args[0].ID()), GetReg<RA_32>(Op->Header.Args[1].ID()));
sxtw(Dst, Dst);
break;
case 8:
mul(Dst, GetReg<RA_64>(Op->Header.Args[0].ID()), GetReg<RA_64>(Op->Header.Args[1].ID()));
break;
default: LOGMAN_MSG_A("Unknown Sext size: %d", OpSize);
default: LogMan::Msg::A("Unknown Sext size: %d", OpSize);
}
}
@@ -179,7 +180,7 @@ DEF_OP(UMul) {
case 8:
mul(Dst, GetReg<RA_64>(Op->Header.Args[0].ID()), GetReg<RA_64>(Op->Header.Args[1].ID()));
break;
default: LOGMAN_MSG_A("Unknown Sext size: %d", OpSize);
default: LogMan::Msg::A("Unknown Sext size: %d", OpSize);
}
}
@@ -216,7 +217,7 @@ DEF_OP(Div) {
sdiv(GetReg<RA_64>(Node), GetReg<RA_64>(Op->Header.Args[0].ID()), GetReg<RA_64>(Op->Header.Args[1].ID()));
break;
}
default: LOGMAN_MSG_A("Unknown DIV Size: %d", Size); break;
default: LogMan::Msg::A("Unknown DIV Size: %d", Size); break;
}
}
@@ -243,7 +244,7 @@ DEF_OP(UDiv) {
udiv(GetReg<RA_64>(Node), GetReg<RA_64>(Op->Header.Args[0].ID()), GetReg<RA_64>(Op->Header.Args[1].ID()));
break;
}
default: LOGMAN_MSG_A("Unknown UDIV Size: %d", Size); break;
default: LogMan::Msg::A("Unknown UDIV Size: %d", Size); break;
}
}
@@ -290,7 +291,7 @@ DEF_OP(Rem) {
msub(GetReg<RA_64>(Node), TMP1, Divisor, Dividend);
break;
}
default: LOGMAN_MSG_A("Unknown REM Size: %d", OpSize); break;
default: LogMan::Msg::A("Unknown REM Size: %d", OpSize); break;
}
}
@@ -332,7 +333,7 @@ DEF_OP(URem) {
msub(GetReg<RA_64>(Node), TMP1, Divisor, Dividend);
break;
}
default: LOGMAN_MSG_A("Unknown UREM Size: %d", OpSize); break;
default: LogMan::Msg::A("Unknown UREM Size: %d", OpSize); break;
}
}
@@ -344,12 +345,12 @@ DEF_OP(MulH) {
sxtw(TMP1, GetReg<RA_64>(Op->Header.Args[0].ID()));
sxtw(TMP2, GetReg<RA_64>(Op->Header.Args[1].ID()));
mul(TMP1, TMP1, TMP2);
ubfx(GetReg<RA_64>(Node), TMP1, 32, 32);
sbfx(GetReg<RA_64>(Node), TMP1, 32, 32);
break;
case 8:
smulh(GetReg<RA_64>(Node), GetReg<RA_64>(Op->Header.Args[0].ID()), GetReg<RA_64>(Op->Header.Args[1].ID()));
break;
default: LOGMAN_MSG_A("Unknown Sext size: %d", OpSize);
default: LogMan::Msg::A("Unknown Sext size: %d", OpSize);
}
}
@@ -366,7 +367,7 @@ DEF_OP(UMulH) {
case 8:
umulh(GetReg<RA_64>(Node), GetReg<RA_64>(Op->Header.Args[0].ID()), GetReg<RA_64>(Op->Header.Args[1].ID()));
break;
default: LOGMAN_MSG_A("Unknown Sext size: %d", OpSize);
default: LogMan::Msg::A("Unknown Sext size: %d", OpSize);
}
}
@@ -462,7 +463,7 @@ DEF_OP(Ror) {
break;
}
default: LOGMAN_MSG_A("Unhandled ROR size: %d", OpSize);
default: LogMan::Msg::A("Unhandled ROR size: %d", OpSize);
}
} else {
switch (OpSize) {
@@ -475,7 +476,7 @@ DEF_OP(Ror) {
break;
}
default: LOGMAN_MSG_A("Unhandled ROR size: %d", OpSize);
default: LogMan::Msg::A("Unhandled ROR size: %d", OpSize);
}
}
}
@@ -494,7 +495,7 @@ DEF_OP(Extr) {
break;
}
default: LOGMAN_MSG_A("Unhandled EXTR size: %d", OpSize);
default: LogMan::Msg::A("Unhandled EXTR size: %d", OpSize);
}
}
@@ -539,7 +540,7 @@ DEF_OP(LDiv) {
mov(GetReg<RA_64>(Node), x0);
break;
}
default: LOGMAN_MSG_A("Unknown LDIV Size: %d", Size); break;
default: LogMan::Msg::A("Unknown LDIV Size: %d", Size); break;
}
}
@@ -582,7 +583,7 @@ DEF_OP(LUDiv) {
mov(GetReg<RA_64>(Node), x0);
break;
}
default: LOGMAN_MSG_A("Unknown LUDIV Size: %d", Size); break;
default: LogMan::Msg::A("Unknown LUDIV Size: %d", Size); break;
}
}
@@ -635,7 +636,7 @@ DEF_OP(LRem) {
mov(GetReg<RA_64>(Node), x0);
break;
}
default: LOGMAN_MSG_A("Unknown LREM Size: %d", Size); break;
default: LogMan::Msg::A("Unknown LREM Size: %d", Size); break;
}
}
@@ -685,7 +686,7 @@ DEF_OP(LURem) {
mov(GetReg<RA_64>(Node), x0);
break;
}
default: LOGMAN_MSG_A("Unknown LUREM Size: %d", OpSize); break;
default: LogMan::Msg::A("Unknown LUREM Size: %d", OpSize); break;
}
}
@@ -699,7 +700,7 @@ DEF_OP(Not) {
case 8:
mvn(GetReg<RA_64>(Node), GetReg<RA_64>(Op->Header.Args[0].ID()));
break;
default: LOGMAN_MSG_A("Unsupported Not size: %d", OpSize);
default: LogMan::Msg::A("Unsupported Not size: %d", OpSize);
}
}
@@ -730,7 +731,7 @@ DEF_OP(Popcount) {
// fmov has zero extended, unused bytes are zero
addv(VTMP1.B(), VTMP1.V8B());
break;
default: LOGMAN_MSG_A("Unsupported Popcount size: %d", OpSize);
default: LogMan::Msg::A("Unsupported Popcount size: %d", OpSize);
}
auto Dst = GetReg<RA_32>(Node);
@@ -779,7 +780,7 @@ DEF_OP(FindMSB) {
clz(Dst, GetReg<RA_64>(Op->Header.Args[0].ID()));
sub(Dst, TMP1, Dst);
break;
default: LOGMAN_MSG_A("Unknown REV size: %d", OpSize); break;
default: LogMan::Msg::A("Unknown REV size: %d", OpSize); break;
}
}
@@ -800,7 +801,7 @@ DEF_OP(FindTrailingZeros) {
rbit(GetReg<RA_64>(Node), GetReg<RA_64>(Op->Header.Args[0].ID()));
clz(GetReg<RA_64>(Node), GetReg<RA_64>(Node));
break;
default: LOGMAN_MSG_A("Unknown size: %d", OpSize); break;
default: LogMan::Msg::A("Unknown size: %d", OpSize); break;
}
}
@@ -819,7 +820,7 @@ DEF_OP(CountLeadingZeroes) {
case 8:
clz(GetReg<RA_64>(Node), GetReg<RA_64>(Op->Header.Args[0].ID()));
break;
default: LOGMAN_MSG_A("Unknown size: %d", OpSize); break;
default: LogMan::Msg::A("Unknown size: %d", OpSize); break;
}
}
@@ -837,7 +838,7 @@ DEF_OP(Rev) {
case 8:
rev(GetReg<RA_64>(Node), GetReg<RA_64>(Op->Header.Args[0].ID()));
break;
default: LOGMAN_MSG_A("Unknown REV size: %d", OpSize); break;
default: LogMan::Msg::A("Unknown REV size: %d", OpSize); break;
}
}
@@ -859,14 +860,15 @@ DEF_OP(Bfi) {
bfi(TMP1, GetReg<RA_64>(Op->Header.Args[1].ID()), Op->lsb, Op->Width);
mov(GetReg<RA_64>(Node), TMP1);
break;
default: LOGMAN_MSG_A("Unknown BFI size: %d", OpSize); break;
default: LogMan::Msg::A("Unknown BFI size: %d", OpSize); break;
}
}
DEF_OP(Bfe) {
auto Op = IROp->C<IR::IROp_Bfe>();
LOGMAN_THROW_A(IROp->Size <= 8, "OpSize is too large for BFE: %d", IROp->Size);
LOGMAN_THROW_A(Op->Width != 0, "Invalid BFE width of 0");
uint8_t OpSize = IROp->Size;
LogMan::Throw::A(OpSize <= 8, "OpSize is too large for BFE: %d", OpSize);
LogMan::Throw::A(Op->Width != 0, "Invalid BFE width of 0");
auto Dst = GetReg<RA_64>(Node);
ubfx(Dst, GetReg<RA_64>(Op->Header.Args[0].ID()), Op->lsb, Op->Width);
@@ -911,7 +913,7 @@ Condition MapSelectCC(IR::CondClassType Cond) {
case FEXCore::IR::COND_MI:
case FEXCore::IR::COND_PL:
default:
LOGMAN_MSG_A("Unsupported compare type");
LogMan::Msg::A("Unsupported compare type");
return Condition::nv;
}
}
@@ -929,7 +931,7 @@ DEF_OP(Select) {
} else if (IsFPR(Op->Cmp1.ID())) {
fcmp(GRFCMP(Op->Cmp1.ID()), GRFCMP(Op->Cmp2.ID()));
} else {
LOGMAN_MSG_A("Select: Expected GPR or FPR");
LogMan::Msg::A("Select: Expected GPR or FPR");
}
auto cc = MapSelectCC(Op->Cond);
@@ -940,7 +942,7 @@ DEF_OP(Select) {
if (is_const_true || is_const_false) {
if (is_const_false != true || is_const_true != true || const_true != 1 || const_false != 0) {
LOGMAN_MSG_A("Select: Unsupported compare inline parameters");
LogMan::Msg::A("Select: Unsupported compare inline parameters");
}
cset(GRS(Node), cc);
} else {
@@ -964,53 +966,38 @@ DEF_OP(VExtractToGPR) {
case 8:
umov(GetReg<RA_64>(Node), GetSrc(Op->Header.Args[0].ID()).V2D(), Op->Idx);
break;
default: LOGMAN_MSG_A("Unhandled ExtractElementSize: %d", OpSize);
default: LogMan::Msg::A("Unhandled ExtractElementSize: %d", OpSize);
}
}
DEF_OP(Float_ToGPR_ZU) {
LogMan::Msg::D("Unimplemented");
}
DEF_OP(Float_ToGPR_ZS) {
auto Op = IROp->C<IR::IROp_Float_ToGPR_ZS>();
aarch64::Register Dst{};
aarch64::VRegister Src{};
if (Op->SrcElementSize == 8) {
Src = GetSrc(Op->Header.Args[0].ID()).D();
if (Op->Header.ElementSize == 8) {
fcvtzs(GetReg<RA_64>(Node), GetSrc(Op->Header.Args[0].ID()).D());
}
else {
Src = GetSrc(Op->Header.Args[0].ID()).S();
fcvtzs(GetReg<RA_32>(Node), GetSrc(Op->Header.Args[0].ID()).S());
}
}
if (IROp->Size == 8) {
Dst = GetReg<RA_64>(Node);
}
else {
Dst = GetReg<RA_32>(Node);
}
fcvtzs(Dst, Src);
DEF_OP(Float_ToGPR_U) {
LogMan::Msg::D("Unimplemented");
}
DEF_OP(Float_ToGPR_S) {
auto Op = IROp->C<IR::IROp_Float_ToGPR_S>();
aarch64::Register Dst{};
aarch64::VRegister Src{};
if (Op->SrcElementSize == 8) {
if (Op->Header.ElementSize == 8) {
frinti(VTMP1.D(), GetSrc(Op->Header.Args[0].ID()).D());
Src = VTMP1.D();
fcvtzs(GetReg<RA_64>(Node), VTMP1.D());
}
else {
frinti(VTMP1.S(), GetSrc(Op->Header.Args[0].ID()).S());
Src = VTMP1.S();
fcvtzs(GetReg<RA_32>(Node), VTMP1.S());
}
if (IROp->Size == 8) {
Dst = GetReg<RA_64>(Node);
}
else {
Dst = GetReg<RA_32>(Node);
}
fcvtzs(Dst, Src);
}
DEF_OP(FCmp) {
@@ -1027,7 +1014,7 @@ DEF_OP(FCmp) {
bool set = false;
if (Op->Flags & (1 << IR::FCMP_FLAG_EQ)) {
LOGMAN_THROW_A(IR::FCMP_FLAG_EQ == 0, "IR::FCMP_FLAG_EQ must equal 0");
LogMan::Throw::A(IR::FCMP_FLAG_EQ == 0, "IR::FCMP_FLAG_EQ must equal 0");
// EQ or unordered
cset(Dst, Condition::eq); // Z = 1
csinc(Dst, Dst, xzr, Condition::vc); // IF !V ? Z : 1
@@ -1101,7 +1088,9 @@ void Arm64JITCore::RegisterALUHandlers() {
REGISTER_OP(SBFE, Sbfe);
REGISTER_OP(SELECT, Select);
REGISTER_OP(VEXTRACTTOGPR, VExtractToGPR);
REGISTER_OP(FLOAT_TOGPR_ZU, Float_ToGPR_ZU);
REGISTER_OP(FLOAT_TOGPR_ZS, Float_ToGPR_ZS);
REGISTER_OP(FLOAT_TOGPR_U, Float_ToGPR_U);
REGISTER_OP(FLOAT_TOGPR_S, Float_ToGPR_S);
REGISTER_OP(FCMP, FCmp);
@@ -34,7 +34,7 @@ DEF_OP(CASPair) {
mov(Dst.first, TMP3);
mov(Dst.second, TMP4);
break;
default: LOGMAN_MSG_A("Unsupported: %d", OpSize);
default: LogMan::Msg::A("Unsupported: %d", OpSize);
}
}
else {
@@ -89,7 +89,7 @@ DEF_OP(CASPair) {
bind(&LoopExpected);
break;
}
default: LOGMAN_MSG_A("Unsupported: %d", OpSize);
default: LogMan::Msg::A("Unsupported: %d", OpSize);
}
}
}
@@ -115,7 +115,7 @@ DEF_OP(CAS) {
case 2: casalh(TMP2.W(), Desired.W(), MemOperand(MemSrc)); break;
case 4: casal(TMP2.W(), Desired.W(), MemOperand(MemSrc)); break;
case 8: casal(TMP2.X(), Desired.X(), MemOperand(MemSrc)); break;
default: LOGMAN_MSG_A("Unsupported: %d", OpSize);
default: LogMan::Msg::A("Unsupported: %d", OpSize);
}
mov(GetReg<RA_64>(Node), TMP2);
}
@@ -206,7 +206,7 @@ DEF_OP(CAS) {
break;
}
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", OpSize);
default: LogMan::Msg::A("Unhandled Atomic size: %d", OpSize);
}
}
}
@@ -222,7 +222,7 @@ DEF_OP(AtomicAdd) {
case 2: staddlh(GetReg<RA_32>(Op->Header.Args[1].ID()), MemOperand(MemSrc)); break;
case 4: staddl(GetReg<RA_32>(Op->Header.Args[1].ID()), MemOperand(MemSrc)); break;
case 8: staddl(GetReg<RA_64>(Op->Header.Args[1].ID()), MemOperand(MemSrc)); break;
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
else {
@@ -264,7 +264,7 @@ DEF_OP(AtomicAdd) {
cbnz(TMP2, &LoopTop);
break;
}
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
}
@@ -281,7 +281,7 @@ DEF_OP(AtomicSub) {
case 2: staddlh(TMP2.W(), MemOperand(MemSrc)); break;
case 4: staddl(TMP2.W(), MemOperand(MemSrc)); break;
case 8: staddl(TMP2.X(), MemOperand(MemSrc)); break;
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
else {
@@ -323,7 +323,7 @@ DEF_OP(AtomicSub) {
cbnz(TMP2, &LoopTop);
break;
}
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
}
@@ -340,7 +340,7 @@ DEF_OP(AtomicAnd) {
case 2: stclrlh(TMP2.W(), MemOperand(MemSrc)); break;
case 4: stclrl(TMP2.W(), MemOperand(MemSrc)); break;
case 8: stclrl(TMP2.X(), MemOperand(MemSrc)); break;
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
else {
@@ -382,7 +382,7 @@ DEF_OP(AtomicAnd) {
cbnz(TMP2, &LoopTop);
break;
}
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
}
@@ -398,7 +398,7 @@ DEF_OP(AtomicOr) {
case 2: stsetlh(GetReg<RA_32>(Op->Header.Args[1].ID()), MemOperand(MemSrc)); break;
case 4: stsetl(GetReg<RA_32>(Op->Header.Args[1].ID()), MemOperand(MemSrc)); break;
case 8: stsetl(GetReg<RA_64>(Op->Header.Args[1].ID()), MemOperand(MemSrc)); break;
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
else {
@@ -440,7 +440,7 @@ DEF_OP(AtomicOr) {
cbnz(TMP2, &LoopTop);
break;
}
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
}
@@ -456,7 +456,7 @@ DEF_OP(AtomicXor) {
case 2: steorlh(GetReg<RA_32>(Op->Header.Args[1].ID()), MemOperand(MemSrc)); break;
case 4: steorl(GetReg<RA_32>(Op->Header.Args[1].ID()), MemOperand(MemSrc)); break;
case 8: steorl(GetReg<RA_64>(Op->Header.Args[1].ID()), MemOperand(MemSrc)); break;
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
else {
@@ -498,7 +498,7 @@ DEF_OP(AtomicXor) {
cbnz(TMP2, &LoopTop);
break;
}
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
}
@@ -515,7 +515,7 @@ DEF_OP(AtomicSwap) {
case 2: swplh(TMP2.W(), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
case 4: swpl(TMP2.W(), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
case 8: swpl(TMP2.X(), GetReg<RA_64>(Node), MemOperand(MemSrc)); break;
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
else {
@@ -558,7 +558,7 @@ DEF_OP(AtomicSwap) {
mov(GetReg<RA_64>(Node), TMP2.X());
break;
}
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
}
@@ -573,7 +573,7 @@ DEF_OP(AtomicFetchAdd) {
case 2: ldaddalh(GetReg<RA_32>(Op->Header.Args[1].ID()), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
case 4: ldaddal(GetReg<RA_32>(Op->Header.Args[1].ID()), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
case 8: ldaddal(GetReg<RA_64>(Op->Header.Args[1].ID()), GetReg<RA_64>(Node), MemOperand(MemSrc)); break;
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
else {
@@ -619,7 +619,7 @@ DEF_OP(AtomicFetchAdd) {
mov(GetReg<RA_64>(Node), TMP2);
break;
}
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
}
@@ -635,7 +635,7 @@ DEF_OP(AtomicFetchSub) {
case 2: ldaddalh(TMP2.W(), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
case 4: ldaddal(TMP2.W(), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
case 8: ldaddal(TMP2.X(), GetReg<RA_64>(Node), MemOperand(MemSrc)); break;
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
else {
@@ -681,7 +681,7 @@ DEF_OP(AtomicFetchSub) {
mov(GetReg<RA_64>(Node), TMP2);
break;
}
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
}
@@ -697,7 +697,7 @@ DEF_OP(AtomicFetchAnd) {
case 2: ldclralh(TMP2.W(), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
case 4: ldclral(TMP2.W(), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
case 8: ldclral(TMP2.X(), GetReg<RA_64>(Node), MemOperand(MemSrc)); break;
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
else {
@@ -743,7 +743,7 @@ DEF_OP(AtomicFetchAnd) {
mov(GetReg<RA_64>(Node), TMP2);
break;
}
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
}
@@ -758,7 +758,7 @@ DEF_OP(AtomicFetchOr) {
case 2: ldsetalh(GetReg<RA_32>(Op->Header.Args[1].ID()), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
case 4: ldsetal(GetReg<RA_32>(Op->Header.Args[1].ID()), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
case 8: ldsetal(GetReg<RA_64>(Op->Header.Args[1].ID()), GetReg<RA_64>(Node), MemOperand(MemSrc)); break;
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
else {
@@ -804,7 +804,7 @@ DEF_OP(AtomicFetchOr) {
mov(GetReg<RA_64>(Node), TMP2);
break;
}
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
}
@@ -819,7 +819,7 @@ DEF_OP(AtomicFetchXor) {
case 2: ldeoralh(GetReg<RA_32>(Op->Header.Args[1].ID()), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
case 4: ldeoral(GetReg<RA_32>(Op->Header.Args[1].ID()), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
case 8: ldeoral(GetReg<RA_64>(Op->Header.Args[1].ID()), GetReg<RA_64>(Node), MemOperand(MemSrc)); break;
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
else {
@@ -865,7 +865,7 @@ DEF_OP(AtomicFetchXor) {
mov(GetReg<RA_64>(Node), TMP2);
break;
}
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
}
@@ -73,7 +73,7 @@ DEF_OP(ExitFunction) {
uint64_t NewRIP;
if (IsInlineConstant(Op->NewRIP, &NewRIP) || IsInlineEntrypointOffset(Op->NewRIP, &NewRIP)) {
Literal l_BranchHost{ThreadSharedData.Dispatcher->ExitFunctionLinkerAddress};
Literal l_BranchHost{Dispatcher->ExitFunctionLinkerAddress};
Literal l_BranchGuest{NewRIP};
ldr(x0, &l_BranchHost);
@@ -96,7 +96,7 @@ DEF_OP(ExitFunction) {
br(x1);
bind(&FullLookup);
LoadConstant(TMP1, ThreadSharedData.Dispatcher->AbsoluteLoopTopAddress);
LoadConstant(TMP1, Dispatcher->AbsoluteLoopTopAddress);
str(RipReg, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.rip)));
br(TMP1);
}
@@ -142,7 +142,7 @@ Condition MapBranchCC(IR::CondClassType Cond) {
case FEXCore::IR::COND_MI:
case FEXCore::IR::COND_PL:
default:
LOGMAN_MSG_A("Unsupported compare type");
LogMan::Msg::A("Unsupported compare type");
return Condition::nv;
}
}
@@ -169,10 +169,10 @@ DEF_OP(CondJump) {
bool isConst = IsInlineConstant(Op->Cmp2, &Const);
if (isConst && Const == 0 && Op->Cond.Val == FEXCore::IR::COND_EQ) {
LOGMAN_THROW_A(IsGPR(Op->Cmp1.ID()), "CondJump: Expected GPR");
LogMan::Throw::A(IsGPR(Op->Cmp1.ID()), "CondJump: Expected GPR");
cbz(GRCMP(Op->Cmp1.ID()), TrueTargetLabel);
} else if (isConst && Const == 0 && Op->Cond.Val == FEXCore::IR::COND_NEQ) {
LOGMAN_THROW_A(IsGPR(Op->Cmp1.ID()), "CondJump: Expected GPR");
LogMan::Throw::A(IsGPR(Op->Cmp1.ID()), "CondJump: Expected GPR");
cbnz(GRCMP(Op->Cmp1.ID()), TrueTargetLabel);
} else {
if (IsGPR(Op->Cmp1.ID())) {
@@ -183,7 +183,7 @@ DEF_OP(CondJump) {
} else if (IsFPR(Op->Cmp1.ID())) {
fcmp(GRFCMP(Op->Cmp1.ID()), GRFCMP(Op->Cmp2.ID()));
} else {
LOGMAN_MSG_A("CondJump: Expected GPR or FPR");
LogMan::Msg::A("CondJump: Expected GPR or FPR");
}
b(TrueTargetLabel, MapBranchCC(Op->Cond));
@@ -257,18 +257,18 @@ DEF_OP(Thunk) {
DEF_OP(ValidateCode) {
auto Op = IROp->C<IR::IROp_ValidateCode>();
const auto *OldCode = (const uint8_t *)&Op->CodeOriginalLow;
uint8_t *OldCode = (uint8_t *)&Op->CodeOriginalLow;
int len = Op->CodeLength;
int idx = 0;
LoadConstant(GetReg<RA_64>(Node), 0);
LoadConstant(x0, Entry + Op->Offset);
LoadConstant(x0, IR->GetHeader()->Entry + Op->Offset);
LoadConstant(x1, 1);
while (len >= 8)
{
ldr(x2, MemOperand(x0, idx));
LoadConstant(x3, *(const uint32_t *)(OldCode + idx));
LoadConstant(x3, *(uint32_t *)(OldCode + idx));
cmp(x2, x3);
csel(GetReg<RA_64>(Node), GetReg<RA_64>(Node), x1, Condition::eq);
len -= 8;
@@ -277,7 +277,7 @@ DEF_OP(ValidateCode) {
while (len >= 4)
{
ldr(w2, MemOperand(x0, idx));
LoadConstant(w3, *(const uint32_t *)(OldCode + idx));
LoadConstant(w3, *(uint32_t *)(OldCode + idx));
cmp(w2, w3);
csel(GetReg<RA_64>(Node), GetReg<RA_64>(Node), x1, Condition::eq);
len -= 4;
@@ -286,7 +286,7 @@ DEF_OP(ValidateCode) {
while (len >= 2)
{
ldrh(w2, MemOperand(x0, idx));
LoadConstant(w3, *(const uint16_t *)(OldCode + idx));
LoadConstant(w3, *(uint16_t *)(OldCode + idx));
cmp(w2, w3);
csel(GetReg<RA_64>(Node), GetReg<RA_64>(Node), x1, Condition::eq);
len -= 2;
@@ -295,7 +295,7 @@ DEF_OP(ValidateCode) {
while (len >= 1)
{
ldrb(w2, MemOperand(x0, idx));
LoadConstant(w3, *(const uint8_t *)(OldCode + idx));
LoadConstant(w3, *(uint8_t *)(OldCode + idx));
cmp(w2, w3);
csel(GetReg<RA_64>(Node), GetReg<RA_64>(Node), x1, Condition::eq);
len -= 1;
@@ -311,7 +311,7 @@ DEF_OP(RemoveCodeEntry) {
PushDynamicRegsAndLR();
mov(x0, STATE);
LoadConstant(x1, Entry);
LoadConstant(x1, IR->GetHeader()->Entry);
LoadConstant(x2, reinterpret_cast<uintptr_t>(&Context::Context::RemoveCodeEntryFromJit));
SpillStaticRegs();
@@ -31,7 +31,7 @@ DEF_OP(VInsGPR) {
ins(GetDst(Node).V2D(), Op->Index, GetReg<RA_64>(Op->Header.Args[1].ID()));
break;
}
default: LOGMAN_MSG_A("Unknown Element Size: %d", Op->Header.ElementSize); break;
default: LogMan::Msg::A("Unknown Element Size: %d", Op->Header.ElementSize); break;
}
}
@@ -52,10 +52,14 @@ DEF_OP(VCastFromGPR) {
case 8:
fmov(GetDst(Node).D(), GetReg<RA_64>(Op->Header.Args[0].ID()).X());
break;
default: LOGMAN_MSG_A("Unknown castGPR element size: %d", Op->Header.ElementSize);
default: LogMan::Msg::A("Unknown castGPR element size: %d", Op->Header.ElementSize);
}
}
DEF_OP(Float_FromGPR_U) {
LogMan::Msg::A("Unimplemented");
}
DEF_OP(Float_FromGPR_S) {
auto Op = IROp->C<IR::IROp_Float_FromGPR_S>();
uint16_t Conv = (Op->Header.ElementSize << 8) | Op->SrcElementSize;
@@ -91,7 +95,20 @@ DEF_OP(Float_FToF) {
fcvt(GetDst(Node).S(), GetSrc(Op->Header.Args[0].ID()).D());
break;
}
default: LOGMAN_MSG_A("Unknown FCVT sizes: 0x%x", Conv);
default: LogMan::Msg::A("Unknown FCVT sizes: 0x%x", Conv);
}
}
DEF_OP(Vector_UToF) {
auto Op = IROp->C<IR::IROp_Vector_UToF>();
switch (Op->Header.ElementSize) {
case 4:
ucvtf(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S());
break;
case 8:
ucvtf(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D());
break;
default: LogMan::Msg::A("Unknown castGPR element size: %d", Op->Header.ElementSize);
}
}
@@ -104,7 +121,20 @@ DEF_OP(Vector_SToF) {
case 8:
scvtf(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D());
break;
default: LOGMAN_MSG_A("Unknown castGPR element size: %d", Op->Header.ElementSize);
default: LogMan::Msg::A("Unknown castGPR element size: %d", Op->Header.ElementSize);
}
}
DEF_OP(Vector_FToZU) {
auto Op = IROp->C<IR::IROp_Vector_FToZU>();
switch (Op->Header.ElementSize) {
case 4:
fcvtzu(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S());
break;
case 8:
fcvtzu(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D());
break;
default: LogMan::Msg::A("Unknown castGPR element size: %d", Op->Header.ElementSize);
}
}
@@ -117,7 +147,22 @@ DEF_OP(Vector_FToZS) {
case 8:
fcvtzs(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D());
break;
default: LOGMAN_MSG_A("Unknown castGPR element size: %d", Op->Header.ElementSize);
default: LogMan::Msg::A("Unknown castGPR element size: %d", Op->Header.ElementSize);
}
}
DEF_OP(Vector_FToU) {
auto Op = IROp->C<IR::IROp_Vector_FToU>();
switch (Op->Header.ElementSize) {
case 4:
frinti(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S());
fcvtzu(GetDst(Node).V4S(), GetDst(Node).V4S());
break;
case 8:
frinti(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D());
fcvtzu(GetDst(Node).V2D(), GetDst(Node).V2D());
break;
default: LogMan::Msg::A("Unknown castGPR element size: %d", Op->Header.ElementSize);
}
}
@@ -132,7 +177,7 @@ DEF_OP(Vector_FToS) {
frinti(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D());
fcvtzs(GetDst(Node).V2D(), GetDst(Node).V2D());
break;
default: LOGMAN_MSG_A("Unknown castGPR element size: %d", Op->Header.ElementSize);
default: LogMan::Msg::A("Unknown castGPR element size: %d", Op->Header.ElementSize);
}
}
@@ -149,63 +194,7 @@ DEF_OP(Vector_FToF) {
fcvtn(GetDst(Node).V2S(), GetSrc(Op->Header.Args[0].ID()).V2D());
break;
}
default: LOGMAN_MSG_A("Unknown Conversion Type : 0%04x", Conv); break;
}
}
DEF_OP(Vector_FToI) {
auto Op = IROp->C<IR::IROp_Vector_FToI>();
switch (Op->Round) {
case FEXCore::IR::Round_Nearest.Val:
switch (Op->Header.ElementSize) {
case 4:
frintn(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S());
break;
case 8:
frintn(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D());
break;
}
break;
case FEXCore::IR::Round_Negative_Infinity.Val:
switch (Op->Header.ElementSize) {
case 4:
frintm(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S());
break;
case 8:
frintm(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D());
break;
}
break;
case FEXCore::IR::Round_Positive_Infinity.Val:
switch (Op->Header.ElementSize) {
case 4:
frintp(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S());
break;
case 8:
frintp(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D());
break;
}
break;
case FEXCore::IR::Round_Towards_Zero.Val:
switch (Op->Header.ElementSize) {
case 4:
frintz(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S());
break;
case 8:
frintz(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D());
break;
}
break;
case FEXCore::IR::Round_Host.Val:
switch (Op->Header.ElementSize) {
case 4:
frinti(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S());
break;
case 8:
frinti(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D());
break;
}
break;
default: LogMan::Msg::A("Unknown Conversion Type : 0%04x", Conv); break;
}
}
@@ -214,13 +203,16 @@ void Arm64JITCore::RegisterConversionHandlers() {
#define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &Arm64JITCore::Op_##x
REGISTER_OP(VINSGPR, VInsGPR);
REGISTER_OP(VCASTFROMGPR, VCastFromGPR);
REGISTER_OP(FLOAT_FROMGPR_U, Float_FromGPR_U);
REGISTER_OP(FLOAT_FROMGPR_S, Float_FromGPR_S);
REGISTER_OP(FLOAT_FTOF, Float_FToF);
REGISTER_OP(VECTOR_UTOF, Vector_UToF);
REGISTER_OP(VECTOR_STOF, Vector_SToF);
REGISTER_OP(VECTOR_FTOZU, Vector_FToZU);
REGISTER_OP(VECTOR_FTOZS, Vector_FToZS);
REGISTER_OP(VECTOR_FTOU, Vector_FToU);
REGISTER_OP(VECTOR_FTOS, Vector_FToS);
REGISTER_OP(VECTOR_FTOF, Vector_FToF);
REGISTER_OP(VECTOR_FTOI, Vector_FToI);
#undef REGISTER_OP
}
}
+55 -100
View File
@@ -22,8 +22,6 @@ $end_info$
#include <FEXCore/Core/X86Enums.h>
#include <FEXCore/Core/UContext.h>
#include <FEXCore/Utils/Allocator.h>
#include <FEXCore/Utils/CompilerDefs.h>
#include "Interface/Core/Interpreter/InterpreterOps.h"
#include <sys/mman.h>
@@ -44,10 +42,8 @@ using namespace vixl::aarch64;
void Arm64JITCore::Op_Unhandled(FEXCore::IR::IROp_Header *IROp, uint32_t Node) {
FallbackInfo Info;
if (!InterpreterOps::GetFallbackHandler(IROp, &Info)) {
#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
auto Name = FEXCore::IR::GetName(IROp->Op);
LOGMAN_MSG_A("Unhandled IR Op: %s", std::string(Name).c_str());
#endif
LogMan::Msg::A("Unhandled IR Op: %s", std::string(Name).c_str());
} else {
switch(Info.ABI) {
case FABI_VOID_U16:{
@@ -294,11 +290,8 @@ void Arm64JITCore::Op_Unhandled(FEXCore::IR::IROp_Header *IROp, uint32_t Node) {
case FABI_UNKNOWN:
default:
#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
auto Name = FEXCore::IR::GetName(IROp->Op);
LOGMAN_MSG_A("Unhandled IR Fallback abi: %s %d", std::string(Name).c_str(), Info.ABI);
#endif
break;
auto Name = FEXCore::IR::GetName(IROp->Op);
LogMan::Msg::A("Unhandled IR Fallback abi: %s %d", std::string(Name).c_str(), Info.ABI);
}
}
}
@@ -310,18 +303,18 @@ Arm64JITCore::CodeBuffer Arm64JITCore::AllocateNewCodeBuffer(size_t Size) {
CodeBuffer Buffer;
Buffer.Size = Size;
Buffer.Ptr = static_cast<uint8_t*>(
FEXCore::Allocator::mmap(nullptr,
mmap(nullptr,
Buffer.Size,
PROT_READ | PROT_WRITE | PROT_EXEC,
MAP_PRIVATE | MAP_ANONYMOUS,
-1, 0));
LOGMAN_THROW_A(!!Buffer.Ptr, "Couldn't allocate code buffer");
LogMan::Throw::A(!!Buffer.Ptr, "Couldn't allocate code buffer");
Dispatcher->RegisterCodeBuffer(Buffer.Ptr, Buffer.Size);
return Buffer;
}
void Arm64JITCore::FreeCodeBuffer(CodeBuffer Buffer) {
FEXCore::Allocator::munmap(Buffer.Ptr, Buffer.Size);
munmap(Buffer.Ptr, Buffer.Size);
Dispatcher->RemoveCodeBuffer(Buffer.Ptr);
}
@@ -366,34 +359,6 @@ bool Arm64JITCore::HandleSIGBUS(int Signal, void *info, void *ucontext) {
// Back up one instruction and have another go
ArchHelpers::Context::SetPc(ucontext, ArchHelpers::Context::GetPc(ucontext) - 4);
}
else if ((Instr & FEXCore::ArchHelpers::Arm64::LDAXP_MASK) == FEXCore::ArchHelpers::Arm64::LDAXP_INST) { // LDAXP
uint32_t DataReg2 = (Instr >> 10) & 0x1F;
// Convert to LDP
uint32_t LDP = 0b0010'1001'0100'0000'0000'0000'0000'0000;
LDP |= Size << 31;
LDP |= DataReg2 << 10;
LDP |= AddrReg << 5;
LDP |= DataReg;
PC[-1] = DMB;
PC[0] = LDP;
PC[1] = DMB;
// Back up one instruction and have another go
ArchHelpers::Context::SetPc(ucontext, ArchHelpers::Context::GetPc(ucontext) - 4);
}
else if ((Instr & FEXCore::ArchHelpers::Arm64::STLXP_MASK) == FEXCore::ArchHelpers::Arm64::STLXP_INST) { // STLXP
uint32_t DataReg2 = (Instr >> 10) & 0x1F;
// Convert to STP
uint32_t STP = 0b0010'1001'0000'0000'0000'0000'0000'0000;
STP |= Size << 31;
STP |= DataReg2 << 10;
STP |= AddrReg << 5;
STP |= DataReg;
PC[-1] = DMB;
PC[0] = STP;
PC[1] = DMB;
// Back up one instruction and have another go
ArchHelpers::Context::SetPc(ucontext, ArchHelpers::Context::GetPc(ucontext) - 4);
}
else if ((Instr & FEXCore::ArchHelpers::Arm64::CASPAL_MASK) == FEXCore::ArchHelpers::Arm64::CASPAL_INST) { // CASPAL
if (FEXCore::ArchHelpers::Arm64::HandleCASPAL(ucontext, info, Instr)) {
// Skip this instruction now
@@ -447,7 +412,7 @@ Arm64JITCore::Arm64JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::Intern
config.ExitFunctionLinkThis = reinterpret_cast<uintptr_t>(this);
config.StaticRegisterAssignment = true;
Dispatcher = std::make_unique<Arm64Dispatcher>(CTX, ThreadState, config);
Dispatcher = new Arm64Dispatcher(CTX, ThreadState, config);
DispatchPtr = Dispatcher->DispatchPtr;
CallbackPtr = Dispatcher->CallbackPtr;
}
@@ -501,23 +466,22 @@ Arm64JITCore::Arm64JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::Intern
if (!CompileThread) {
ThreadSharedData.SignalHandlerRefCounterPtr = &Dispatcher->SignalHandlerRefCounter;
ThreadSharedData.SignalReturnInstruction = Dispatcher->SignalHandlerReturnAddress;
ThreadSharedData.Dispatcher = Dispatcher.get();
// This will register the host signal handler per thread, which is fine
CTX->SignalDelegation->RegisterHostSignalHandler(SIGILL, [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext) -> bool {
Arm64JITCore *Core = reinterpret_cast<Arm64JITCore*>(Thread->CPUBackend.get());
return Core->Dispatcher->HandleSIGILL(Signal, info, ucontext);
}, true);
});
CTX->SignalDelegation->RegisterHostSignalHandler(SIGBUS, [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext) -> bool {
Arm64JITCore *Core = reinterpret_cast<Arm64JITCore*>(Thread->CPUBackend.get());
return Core->HandleSIGBUS(Signal, info, ucontext);
}, true);
});
CTX->SignalDelegation->RegisterHostSignalHandler(SignalDelegator::SIGNAL_FOR_PAUSE, [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext) -> bool {
Arm64JITCore *Core = reinterpret_cast<Arm64JITCore*>(Thread->CPUBackend.get());
return Core->Dispatcher->HandleSignalPause(Signal, info, ucontext);
}, true);
});
auto GuestSignalHandler = [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext, GuestSigAction *GuestAction, stack_t *GuestStack) -> bool {
Arm64JITCore *Core = reinterpret_cast<Arm64JITCore*>(Thread->CPUBackend.get());
@@ -581,85 +545,77 @@ Arm64JITCore::~Arm64JITCore() {
FreeCodeBuffer(InitialCodeBuffer);
}
IR::PhysicalRegister Arm64JITCore::GetPhys(uint32_t Node) const {
static IR::PhysicalRegister GetPhys(IR::RegisterAllocationData *RAData, uint32_t Node) {
auto PhyReg = RAData->GetNodeRegister(Node);
LOGMAN_THROW_A(!PhyReg.IsInvalid(), "Couldn't Allocate register for node: ssa%d. Class: %d", Node, PhyReg.Class);
LogMan::Throw::A(!PhyReg.IsInvalid(), "Couldn't Allocate register for node: ssa%d. Class: %d", Node, PhyReg.Class);
return PhyReg;
}
template<>
aarch64::Register Arm64JITCore::GetReg<Arm64JITCore::RA_32>(uint32_t Node) const {
auto Reg = GetPhys(Node);
aarch64::Register Arm64JITCore::GetReg<Arm64JITCore::RA_32>(uint32_t Node) {
auto Reg = GetPhys(RAData, Node);
if (Reg.Class == IR::GPRFixedClass.Val) {
return SRA64[Reg.Reg].W();
} else if (Reg.Class == IR::GPRClass.Val) {
return RA64[Reg.Reg].W();
} else {
LOGMAN_THROW_A(false, "Unexpected Class: %d", Reg.Class);
LogMan::Throw::A(false, "Unexpected Class: %d", Reg.Class);
}
FEX_UNREACHABLE;
__builtin_unreachable();
}
template<>
aarch64::Register Arm64JITCore::GetReg<Arm64JITCore::RA_64>(uint32_t Node) const {
auto Reg = GetPhys(Node);
aarch64::Register Arm64JITCore::GetReg<Arm64JITCore::RA_64>(uint32_t Node) {
auto Reg = GetPhys(RAData, Node);
if (Reg.Class == IR::GPRFixedClass.Val) {
return SRA64[Reg.Reg];
} else if (Reg.Class == IR::GPRClass.Val) {
return RA64[Reg.Reg];
} else {
LOGMAN_THROW_A(false, "Unexpected Class: %d", Reg.Class);
LogMan::Throw::A(false, "Unexpected Class: %d", Reg.Class);
}
FEX_UNREACHABLE;
__builtin_unreachable();
}
template<>
std::pair<aarch64::Register, aarch64::Register> Arm64JITCore::GetSrcPair<Arm64JITCore::RA_32>(uint32_t Node) const {
uint32_t Reg = GetPhys(Node).Reg;
std::pair<aarch64::Register, aarch64::Register> Arm64JITCore::GetSrcPair<Arm64JITCore::RA_32>(uint32_t Node) {
uint32_t Reg = GetPhys(RAData, Node).Reg;
return RA32Pair[Reg];
}
template<>
std::pair<aarch64::Register, aarch64::Register> Arm64JITCore::GetSrcPair<Arm64JITCore::RA_64>(uint32_t Node) const {
uint32_t Reg = GetPhys(Node).Reg;
std::pair<aarch64::Register, aarch64::Register> Arm64JITCore::GetSrcPair<Arm64JITCore::RA_64>(uint32_t Node) {
uint32_t Reg = GetPhys(RAData, Node).Reg;
return RA64Pair[Reg];
}
aarch64::VRegister Arm64JITCore::GetSrc(uint32_t Node) const {
auto Reg = GetPhys(Node);
aarch64::VRegister Arm64JITCore::GetSrc(uint32_t Node) {
auto Reg = GetPhys(RAData, Node);
if (Reg.Class == IR::FPRFixedClass.Val) {
return SRAFPR[Reg.Reg];
} else if (Reg.Class == IR::FPRClass.Val) {
return RAFPR[Reg.Reg];
} else {
LOGMAN_THROW_A(false, "Unexpected Class: %d", Reg.Class);
LogMan::Throw::A(false, "Unexpected Class: %d", Reg.Class);
}
FEX_UNREACHABLE;
__builtin_unreachable();
}
aarch64::VRegister Arm64JITCore::GetDst(uint32_t Node) const {
auto Reg = GetPhys(Node);
aarch64::VRegister Arm64JITCore::GetDst(uint32_t Node) {
auto Reg = GetPhys(RAData, Node);
if (Reg.Class == IR::FPRFixedClass.Val) {
return SRAFPR[Reg.Reg];
} else if (Reg.Class == IR::FPRClass.Val) {
return RAFPR[Reg.Reg];
} else {
LOGMAN_THROW_A(false, "Unexpected Class: %d", Reg.Class);
LogMan::Throw::A(false, "Unexpected Class: %d", Reg.Class);
}
FEX_UNREACHABLE;
__builtin_unreachable();
}
bool Arm64JITCore::IsInlineConstant(const IR::OrderedNodeWrapper& WNode, uint64_t* Value) const {
bool Arm64JITCore::IsInlineConstant(const IR::OrderedNodeWrapper& WNode, uint64_t* Value) {
auto OpHeader = IR->GetOp<IR::IROp_Header>(WNode);
if (OpHeader->Op == IR::IROps::OP_INLINECONSTANT) {
@@ -673,13 +629,13 @@ bool Arm64JITCore::IsInlineConstant(const IR::OrderedNodeWrapper& WNode, uint64_
}
}
bool Arm64JITCore::IsInlineEntrypointOffset(const IR::OrderedNodeWrapper& WNode, uint64_t* Value) const {
bool Arm64JITCore::IsInlineEntrypointOffset(const IR::OrderedNodeWrapper& WNode, uint64_t* Value) {
auto OpHeader = IR->GetOp<IR::IROp_Header>(WNode);
if (OpHeader->Op == IR::IROps::OP_INLINEENTRYPOINTOFFSET) {
auto Op = OpHeader->C<IR::IROp_InlineEntrypointOffset>();
if (Value) {
*Value = Entry + Op->Offset;
*Value = IR->GetHeader()->Entry + Op->Offset;
}
return true;
} else {
@@ -687,33 +643,34 @@ bool Arm64JITCore::IsInlineEntrypointOffset(const IR::OrderedNodeWrapper& WNode,
}
}
FEXCore::IR::RegisterClassType Arm64JITCore::GetRegClass(uint32_t Node) const {
return FEXCore::IR::RegisterClassType {GetPhys(Node).Class};
FEXCore::IR::RegisterClassType Arm64JITCore::GetRegClass(uint32_t Node) {
return FEXCore::IR::RegisterClassType {GetPhys(RAData, Node).Class};
}
bool Arm64JITCore::IsFPR(uint32_t Node) const {
bool Arm64JITCore::IsFPR(uint32_t Node) {
auto Class = GetRegClass(Node);
return Class == IR::FPRClass || Class == IR::FPRFixedClass;
}
bool Arm64JITCore::IsGPR(uint32_t Node) const {
bool Arm64JITCore::IsGPR(uint32_t Node) {
auto Class = GetRegClass(Node);
return Class == IR::GPRClass || Class == IR::GPRFixedClass;
}
void *Arm64JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IRListView const *IR, [[maybe_unused]] FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) {
void *Arm64JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const *IR, [[maybe_unused]] FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) {
using namespace aarch64;
JumpTargets.clear();
uint32_t SSACount = IR->GetSSACount();
this->Entry = Entry;
this->RAData = RAData;
auto HeaderOp = IR->GetHeader();
#ifndef NDEBUG
LoadConstant(x0, Entry);
LoadConstant(x0, HeaderOp->Entry);
#endif
this->IR = IR;
@@ -745,7 +702,7 @@ void *Arm64JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IR
// X4-r18 = RA
auto Buffer = GetBuffer();
auto GuestEntry = Buffer->GetOffsetAddress<uint64_t>(GetCursorOffset());
auto Entry = Buffer->GetOffsetAddress<uint64_t>(GetCursorOffset());
if (CTX->GetGdbServerStatus()) {
aarch64::Label RunBlock;
@@ -762,17 +719,17 @@ void *Arm64JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IR
cbz(w0, &RunBlock);
{
// Make sure RIP is syncronized to the context
LoadConstant(x0, Entry);
LoadConstant(x0, HeaderOp->Entry);
str(x0, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.rip)));
// Stop the thread
LoadConstant(x0, ThreadSharedData.Dispatcher->ThreadPauseHandlerAddressSpillSRA);
LoadConstant(x0, Dispatcher->ThreadPauseHandlerAddressSpillSRA);
br(x0);
}
bind(&RunBlock);
}
//LOGMAN_THROW_A(RAData->HasFullRA(), "Arm64 JIT only works with RA");
//LogMan::Throw::A(RAData->HasFullRA(), "Arm64 JIT only works with RA");
SpillSlots = RAData->SpillSlots();
@@ -789,10 +746,8 @@ void *Arm64JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IR
for (auto [BlockNode, BlockHeader] : IR->GetBlocks()) {
using namespace FEXCore::IR;
#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
auto BlockIROp = BlockHeader->CW<FEXCore::IR::IROp_CodeBlock>();
LOGMAN_THROW_A(BlockIROp->Header.Op == IR::OP_CODEBLOCK, "IR type failed to be a code block");
#endif
LogMan::Throw::A(BlockIROp->Header.Op == IR::OP_CODEBLOCK, "IR type failed to be a code block");
{
uint32_t Node = IR->GetID(BlockNode);
@@ -838,15 +793,15 @@ void *Arm64JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IR
FinalizeCode();
auto CodeEnd = Buffer->GetOffsetAddress<uint64_t>(GetCursorOffset());
CPU.EnsureIAndDCacheCoherency(reinterpret_cast<void*>(GuestEntry), CodeEnd - reinterpret_cast<uint64_t>(GuestEntry));
CPU.EnsureIAndDCacheCoherency(reinterpret_cast<void*>(Entry), CodeEnd - reinterpret_cast<uint64_t>(Entry));
if (DebugData) {
DebugData->HostCodeSize = reinterpret_cast<uintptr_t>(CodeEnd) - reinterpret_cast<uintptr_t>(GuestEntry);
DebugData->HostCodeSize = reinterpret_cast<uintptr_t>(CodeEnd) - reinterpret_cast<uintptr_t>(Entry);
}
this->IR = nullptr;
return reinterpret_cast<void*>(GuestEntry);
return reinterpret_cast<void*>(Entry);
}
uint64_t Arm64JITCore::ExitFunctionLink(Arm64JITCore *core, FEXCore::Core::CpuStateFrame *Frame, uint64_t *record) {
@@ -858,11 +813,11 @@ uint64_t Arm64JITCore::ExitFunctionLink(Arm64JITCore *core, FEXCore::Core::CpuSt
if (!HostCode) {
//printf("ExitFunctionLink: Aborting, %lX not in cache\n", GuestRip);
Frame->State.rip = GuestRip;
return core->ThreadSharedData.Dispatcher->AbsoluteLoopTopAddress;
return core->Dispatcher->AbsoluteLoopTopAddress;
}
uintptr_t branch = (uintptr_t)(record) - 8;
auto LinkerAddress = core->ThreadSharedData.Dispatcher->ExitFunctionLinkerAddress;
auto LinkerAddress = core->Dispatcher->ExitFunctionLinkerAddress;
auto offset = HostCode/4 - branch/4;
if (IsInt26(offset)) {
@@ -898,7 +853,7 @@ uint64_t Arm64JITCore::ExitFunctionLink(Arm64JITCore *core, FEXCore::Core::CpuSt
return HostCode;
}
std::unique_ptr<CPUBackend> CreateArm64JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, bool CompileThread) {
return std::make_unique<Arm64JITCore>(ctx, Thread, CompileThread);
FEXCore::CPU::CPUBackend *CreateArm64JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, bool CompileThread) {
return new Arm64JITCore(ctx, Thread, CompileThread);
}
}
+20 -32
View File
@@ -47,7 +47,7 @@ public:
~Arm64JITCore() override;
std::string GetName() override { return "JIT"; }
void *CompileCode(uint64_t Entry, FEXCore::IR::IRListView const *IR, FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) override;
void *CompileCode(FEXCore::IR::IRListView const *IR, FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) override;
void *MapRegion(void* HostPtr, uint64_t, uint64_t) override { return HostPtr; }
@@ -63,14 +63,11 @@ public:
void CopyNecessaryDataForCompileThread(CPUBackend *Original) override;
private:
FEX_CONFIG_OPT(ParanoidTSO, PARANOIDTSO);
std::unique_ptr<FEXCore::CPU::Dispatcher> Dispatcher;
Dispatcher *Dispatcher;
Label *PendingTargetLabel;
FEXCore::Context::Context *CTX;
FEXCore::Core::InternalThreadState *ThreadState;
FEXCore::IR::IRListView const *IR;
uint64_t Entry;
std::map<IR::OrderedNodeWrapper::NodeOffsetType, aarch64::Label> JumpTargets;
@@ -96,35 +93,33 @@ private:
constexpr static uint8_t RA_FPR = 2;
template<uint8_t RAType>
aarch64::Register GetReg(uint32_t Node) const;
aarch64::Register GetReg(uint32_t Node);
template<>
aarch64::Register GetReg<RA_32>(uint32_t Node) const;
aarch64::Register GetReg<RA_32>(uint32_t Node);
template<>
aarch64::Register GetReg<RA_64>(uint32_t Node) const;
aarch64::Register GetReg<RA_64>(uint32_t Node);
template<uint8_t RAType>
std::pair<aarch64::Register, aarch64::Register> GetSrcPair(uint32_t Node) const;
std::pair<aarch64::Register, aarch64::Register> GetSrcPair(uint32_t Node);
template<>
std::pair<aarch64::Register, aarch64::Register> GetSrcPair<RA_32>(uint32_t Node) const;
std::pair<aarch64::Register, aarch64::Register> GetSrcPair<RA_32>(uint32_t Node);
template<>
std::pair<aarch64::Register, aarch64::Register> GetSrcPair<RA_64>(uint32_t Node) const;
std::pair<aarch64::Register, aarch64::Register> GetSrcPair<RA_64>(uint32_t Node);
aarch64::VRegister GetSrc(uint32_t Node) const;
aarch64::VRegister GetDst(uint32_t Node) const;
aarch64::VRegister GetSrc(uint32_t Node);
aarch64::VRegister GetDst(uint32_t Node);
FEXCore::IR::RegisterClassType GetRegClass(uint32_t Node) const;
FEXCore::IR::RegisterClassType GetRegClass(uint32_t Node);
IR::PhysicalRegister GetPhys(uint32_t Node) const;
bool IsFPR(uint32_t Node) const;
bool IsGPR(uint32_t Node) const;
bool IsFPR(uint32_t Node);
bool IsGPR(uint32_t Node);
MemOperand GenerateMemOperand(uint8_t AccessSize, aarch64::Register Base, IR::OrderedNodeWrapper Offset, IR::MemOffsetType OffsetType, uint8_t OffsetScale);
bool IsInlineConstant(const IR::OrderedNodeWrapper& Node, uint64_t* Value = nullptr) const;
bool IsInlineEntrypointOffset(const IR::OrderedNodeWrapper& WNode, uint64_t* Value) const;
bool IsInlineConstant(const IR::OrderedNodeWrapper& Node, uint64_t* Value = nullptr);
bool IsInlineEntrypointOffset(const IR::OrderedNodeWrapper& WNode, uint64_t* Value);
struct LiveRange {
uint32_t Begin;
@@ -167,7 +162,6 @@ private:
uint64_t SignalReturnInstruction{};
uint32_t *SignalHandlerRefCounterPtr{};
FEXCore::CPU::Dispatcher *Dispatcher{};
};
CompilerSharedData ThreadSharedData;
@@ -240,6 +234,7 @@ private:
DEF_OP(VExtractToGPR);
DEF_OP(Float_ToGPR_ZU);
DEF_OP(Float_ToGPR_ZS);
DEF_OP(Float_ToGPR_U);
DEF_OP(Float_ToGPR_S);
DEF_OP(FCmp);
@@ -276,13 +271,16 @@ private:
///< Conversion ops
DEF_OP(VInsGPR);
DEF_OP(VCastFromGPR);
DEF_OP(Float_FromGPR_U);
DEF_OP(Float_FromGPR_S);
DEF_OP(Float_FToF);
DEF_OP(Vector_UToF);
DEF_OP(Vector_SToF);
DEF_OP(Vector_FToZU);
DEF_OP(Vector_FToZS);
DEF_OP(Vector_FToU);
DEF_OP(Vector_FToS);
DEF_OP(Vector_FToF);
DEF_OP(Vector_FToI);
///< Flag ops
DEF_OP(GetHostFlag);
@@ -302,11 +300,8 @@ private:
DEF_OP(StoreMem);
DEF_OP(LoadMemTSO);
DEF_OP(StoreMemTSO);
DEF_OP(ParanoidLoadMemTSO);
DEF_OP(ParanoidStoreMemTSO);
DEF_OP(VLoadMemElement);
DEF_OP(VStoreMemElement);
DEF_OP(CacheLineClear);
///< Misc ops
DEF_OP(EndBlock);
@@ -332,7 +327,6 @@ private:
DEF_OP(SplatVector4);
DEF_OP(VMov);
DEF_OP(VAnd);
DEF_OP(VBic);
DEF_OP(VOr);
DEF_OP(VXor);
DEF_OP(VAdd);
@@ -343,10 +337,8 @@ private:
DEF_OP(VSQSub);
DEF_OP(VAddP);
DEF_OP(VAddV);
DEF_OP(VUMinV);
DEF_OP(VURAvg);
DEF_OP(VAbs);
DEF_OP(VPopcount);
DEF_OP(VFAdd);
DEF_OP(VFAddP);
DEF_OP(VFSub);
@@ -366,8 +358,6 @@ private:
DEF_OP(VSMax);
DEF_OP(VZip);
DEF_OP(VZip2);
DEF_OP(VUnZip);
DEF_OP(VUnZip2);
DEF_OP(VBSL);
DEF_OP(VCMPEQ);
DEF_OP(VCMPEQZ);
@@ -390,7 +380,6 @@ private:
DEF_OP(VInsElement);
DEF_OP(VInsScalarElement);
DEF_OP(VExtractElement);
DEF_OP(VDupElement);
DEF_OP(VExtr);
DEF_OP(VSLI);
DEF_OP(VSRI);
@@ -413,7 +402,6 @@ private:
DEF_OP(VSMull);
DEF_OP(VUMull2);
DEF_OP(VSMull2);
DEF_OP(VUABDL);
DEF_OP(VTBL1);
///< Encryption ops
+61 -229
View File
@@ -5,7 +5,6 @@ $end_info$
*/
#include "Interface/Core/JIT/Arm64/JITClass.h"
#include <FEXCore/Utils/CompilerDefs.h>
namespace FEXCore::CPU {
@@ -30,7 +29,7 @@ DEF_OP(LoadContext) {
case 8:
ldr(GetReg<RA_64>(Node), MemOperand(STATE, Op->Offset));
break;
default: LOGMAN_MSG_A("Unhandled LoadContext size: %d", OpSize);
default: LogMan::Msg::A("Unhandled LoadContext size: %d", OpSize);
}
}
else {
@@ -51,7 +50,7 @@ DEF_OP(LoadContext) {
case 16:
ldr(Dst, MemOperand(STATE, Op->Offset));
break;
default: LOGMAN_MSG_A("Unhandled LoadContext size: %d", OpSize);
default: LogMan::Msg::A("Unhandled LoadContext size: %d", OpSize);
}
}
}
@@ -73,7 +72,7 @@ DEF_OP(StoreContext) {
case 8:
str(GetReg<RA_64>(Op->Header.Args[0].ID()), MemOperand(STATE, Op->Offset));
break;
default: LOGMAN_MSG_A("Unhandled StoreContext size: %d", OpSize);
default: LogMan::Msg::A("Unhandled StoreContext size: %d", OpSize);
}
}
else {
@@ -94,7 +93,7 @@ DEF_OP(StoreContext) {
case 16:
str(Src, MemOperand(STATE, Op->Offset));
break;
default: LOGMAN_MSG_A("Unhandled LoadContext size: %d", OpSize);
default: LogMan::Msg::A("Unhandled LoadContext size: %d", OpSize);
}
}
}
@@ -107,29 +106,29 @@ DEF_OP(LoadRegister) {
auto regId = (Op->Offset - offsetof(FEXCore::Core::CpuStateFrame, State.gregs[0])) / 8;
auto regOffs = Op->Offset & 7;
LOGMAN_THROW_A(regId < SRA64.size(), "out of range regId");
LogMan::Throw::A(regId < SRA64.size(), "out of range regId");
auto reg = SRA64[regId];
switch(Op->Header.Size) {
case 1:
LOGMAN_THROW_A(regOffs == 0 || regOffs == 1, "unexpected regOffs");
LogMan::Throw::A(regOffs == 0 || regOffs == 1, "unexpected regOffs");
ubfx(GetReg<RA_64>(Node), reg, regOffs * 8, 8);
break;
case 2:
LOGMAN_THROW_A(regOffs == 0, "unexpected regOffs");
LogMan::Throw::A(regOffs == 0, "unexpected regOffs");
ubfx(GetReg<RA_64>(Node), reg, 0, 16);
break;
case 4:
LOGMAN_THROW_A(regOffs == 0, "unexpected regOffs");
LogMan::Throw::A(regOffs == 0, "unexpected regOffs");
if (GetReg<RA_64>(Node).GetCode() != reg.GetCode())
mov(GetReg<RA_32>(Node), reg.W());
break;
case 8:
LOGMAN_THROW_A(regOffs == 0, "unexpected regOffs");
LogMan::Throw::A(regOffs == 0, "unexpected regOffs");
if (GetReg<RA_64>(Node).GetCode() != reg.GetCode())
mov(GetReg<RA_64>(Node), reg);
break;
@@ -138,24 +137,24 @@ DEF_OP(LoadRegister) {
auto regId = (Op->Offset - offsetof(FEXCore::Core::CpuStateFrame, State.xmm[0][0])) / 16;
auto regOffs = Op->Offset & 15;
LOGMAN_THROW_A(regId < SRAFPR.size(), "out of range regId");
LogMan::Throw::A(regId < SRAFPR.size(), "out of range regId");
auto guest = SRAFPR[regId];
auto host = GetSrc(Node);
switch(Op->Header.Size) {
case 1:
LOGMAN_THROW_A(regOffs == 0, "unexpected regOffs");
LogMan::Throw::A(regOffs == 0, "unexpected regOffs");
mov(host.B(), guest.B());
break;
case 2:
LOGMAN_THROW_A(regOffs == 0, "unexpected regOffs");
LogMan::Throw::A(regOffs == 0, "unexpected regOffs");
fmov(host.H(), guest.H());
break;
case 4:
LOGMAN_THROW_A((regOffs & 3) == 0, "unexpected regOffs");
LogMan::Throw::A((regOffs & 3) == 0, "unexpected regOffs");
if (regOffs == 0) {
if (host.GetCode() != guest.GetCode())
fmov(host.S(), guest.S());
@@ -165,7 +164,7 @@ DEF_OP(LoadRegister) {
break;
case 8:
LOGMAN_THROW_A((regOffs & 7) == 0, "unexpected regOffs");
LogMan::Throw::A((regOffs & 7) == 0, "unexpected regOffs");
if (regOffs == 0) {
if (host.GetCode() != guest.GetCode())
mov(host.D(), guest.D());
@@ -175,13 +174,13 @@ DEF_OP(LoadRegister) {
break;
case 16:
LOGMAN_THROW_A(regOffs == 0, "unexpected regOffs");
LogMan::Throw::A(regOffs == 0, "unexpected regOffs");
if (host.GetCode() != guest.GetCode())
mov(host.Q(), guest.Q());
break;
}
} else {
LOGMAN_THROW_A(false, "Unhandled Op->Class %d", Op->Class);
LogMan::Throw::A(false, "Unhandled Op->Class %d", Op->Class);
}
}
@@ -192,28 +191,28 @@ DEF_OP(StoreRegister) {
auto regId = Op->Offset / 8 - 1;
auto regOffs = Op->Offset & 7;
LOGMAN_THROW_A(regId < SRA64.size(), "out of range regId");
LogMan::Throw::A(regId < SRA64.size(), "out of range regId");
auto reg = SRA64[regId];
switch(Op->Header.Size) {
case 1:
LOGMAN_THROW_A(regOffs == 0 || regOffs == 1, "unexpected regOffs");
LogMan::Throw::A(regOffs == 0 || regOffs == 1, "unexpected regOffs");
bfi(reg, GetReg<RA_64>(Op->Value.ID()), regOffs * 8, 8);
break;
case 2:
LOGMAN_THROW_A(regOffs == 0, "unexpected regOffs");
LogMan::Throw::A(regOffs == 0, "unexpected regOffs");
bfi(reg, GetReg<RA_64>(Op->Value.ID()), 0, 16);
break;
case 4:
LOGMAN_THROW_A(regOffs == 0, "unexpected regOffs");
LogMan::Throw::A(regOffs == 0, "unexpected regOffs");
bfi(reg, GetReg<RA_64>(Op->Value.ID()), 0, 32);
break;
case 8:
LOGMAN_THROW_A(regOffs == 0, "unexpected regOffs");
LogMan::Throw::A(regOffs == 0, "unexpected regOffs");
if (GetReg<RA_64>(Op->Value.ID()).GetCode() != reg.GetCode())
mov(reg, GetReg<RA_64>(Op->Value.ID()));
break;
@@ -222,7 +221,7 @@ DEF_OP(StoreRegister) {
auto regId = (Op->Offset - offsetof(FEXCore::Core::CpuStateFrame, State.xmm[0][0])) / 16;
auto regOffs = Op->Offset & 15;
LOGMAN_THROW_A(regId < SRAFPR.size(), "regId out of range");
LogMan::Throw::A(regId < SRAFPR.size(), "regId out of range");
auto guest = SRAFPR[regId];
auto host = GetSrc(Op->Value.ID());
@@ -233,28 +232,28 @@ DEF_OP(StoreRegister) {
break;
case 2:
LOGMAN_THROW_A((regOffs & 1) == 0, "unexpected regOffs");
LogMan::Throw::A((regOffs & 1) == 0, "unexpected regOffs");
ins(guest.V8H(), regOffs/2, host.V8H(), 0);
break;
case 4:
LOGMAN_THROW_A((regOffs & 3) == 0, "unexpected regOffs");
LogMan::Throw::A((regOffs & 3) == 0, "unexpected regOffs");
ins(guest.V4S(), regOffs/4, host.V4S(), 0);
break;
case 8:
LOGMAN_THROW_A((regOffs & 7) == 0, "unexpected regOffs");
LogMan::Throw::A((regOffs & 7) == 0, "unexpected regOffs");
ins(guest.V2D(), regOffs / 8, host.V2D(), 0);
break;
case 16:
LOGMAN_THROW_A(regOffs == 0, "unexpected regOffs");
LogMan::Throw::A(regOffs == 0, "unexpected regOffs");
if (guest.GetCode() != host.GetCode())
mov(guest.Q(), host.Q());
break;
}
} else {
LOGMAN_THROW_A(false, "Unhandled Op->Class %d", Op->Class);
LogMan::Throw::A(false, "Unhandled Op->Class %d", Op->Class);
}
}
@@ -288,15 +287,15 @@ DEF_OP(LoadContextIndexed) {
ldr(GetReg<RA_64>(Node), MemOperand(TMP1, Op->BaseOffset));
break;
default:
LOGMAN_MSG_A("Unhandled LoadContextIndexed size: %d", Op->Size);
LogMan::Msg::A("Unhandled LoadContextIndexed size: %d", Op->Size);
}
break;
}
case 16:
LOGMAN_MSG_A("Invalid Class load of size 16");
LogMan::Msg::A("Invalid Class load of size 16");
break;
default:
LOGMAN_MSG_A("Unhandled LoadContextIndexed stride: %d", Op->Stride);
LogMan::Msg::A("Unhandled LoadContextIndexed stride: %d", Op->Stride);
}
}
else {
@@ -333,12 +332,12 @@ DEF_OP(LoadContextIndexed) {
}
break;
default:
LOGMAN_MSG_A("Unhandled LoadContextIndexed size: %d", Op->Size);
LogMan::Msg::A("Unhandled LoadContextIndexed size: %d", Op->Size);
}
break;
}
default:
LOGMAN_MSG_A("Unhandled LoadContextIndexed stride: %d", Op->Stride);
LogMan::Msg::A("Unhandled LoadContextIndexed stride: %d", Op->Stride);
}
}
}
@@ -374,15 +373,15 @@ DEF_OP(StoreContextIndexed) {
str(value, MemOperand(TMP1, Op->BaseOffset));
break;
default:
LOGMAN_MSG_A("Unhandled LoadContextIndexed size: %d", Op->Size);
LogMan::Msg::A("Unhandled LoadContextIndexed size: %d", Op->Size);
}
break;
}
case 16:
LOGMAN_MSG_A("Invalid Class load of size 16");
LogMan::Msg::A("Invalid Class load of size 16");
break;
default:
LOGMAN_MSG_A("Unhandled LoadContextIndexed stride: %d", Op->Stride);
LogMan::Msg::A("Unhandled LoadContextIndexed stride: %d", Op->Stride);
}
}
else {
@@ -421,12 +420,12 @@ DEF_OP(StoreContextIndexed) {
}
break;
default:
LOGMAN_MSG_A("Unhandled LoadContextIndexed size: %d", Op->Size);
LogMan::Msg::A("Unhandled LoadContextIndexed size: %d", Op->Size);
}
break;
}
default:
LOGMAN_MSG_A("Unhandled LoadContextIndexed stride: %d", Op->Stride);
LogMan::Msg::A("Unhandled LoadContextIndexed stride: %d", Op->Stride);
}
}
}
@@ -454,7 +453,7 @@ DEF_OP(SpillRegister) {
str(GetReg<RA_64>(Op->Header.Args[0].ID()), MemOperand(sp, SlotOffset));
break;
}
default: LOGMAN_MSG_A("Unhandled SpillRegister size: %d", OpSize);
default: LogMan::Msg::A("Unhandled SpillRegister size: %d", OpSize);
}
} else if (Op->Class == FEXCore::IR::FPRClass) {
switch (OpSize) {
@@ -470,10 +469,10 @@ DEF_OP(SpillRegister) {
str(GetSrc(Op->Header.Args[0].ID()), MemOperand(sp, SlotOffset));
break;
}
default: LOGMAN_MSG_A("Unhandled SpillRegister size: %d", OpSize);
default: LogMan::Msg::A("Unhandled SpillRegister size: %d", OpSize);
}
} else {
LOGMAN_MSG_A("Unhandled SpillRegister class: %d", Op->Class.Val);
LogMan::Msg::A("Unhandled SpillRegister class: %d", Op->Class.Val);
}
}
@@ -500,7 +499,7 @@ DEF_OP(FillRegister) {
ldr(GetReg<RA_64>(Node), MemOperand(sp, SlotOffset));
break;
}
default: LOGMAN_MSG_A("Unhandled SpillRegister size: %d", OpSize);
default: LogMan::Msg::A("Unhandled SpillRegister size: %d", OpSize);
}
} else if (Op->Class == FEXCore::IR::FPRClass) {
switch (OpSize) {
@@ -516,10 +515,10 @@ DEF_OP(FillRegister) {
ldr(GetDst(Node), MemOperand(sp, SlotOffset));
break;
}
default: LOGMAN_MSG_A("Unhandled SpillRegister size: %d", OpSize);
default: LogMan::Msg::A("Unhandled SpillRegister size: %d", OpSize);
}
} else {
LOGMAN_MSG_A("Unhandled FillRegister class: %d", Op->Class.Val);
LogMan::Msg::A("Unhandled FillRegister class: %d", Op->Class.Val);
}
}
@@ -539,7 +538,7 @@ MemOperand Arm64JITCore::GenerateMemOperand(uint8_t AccessSize, aarch64::Registe
return MemOperand(Base);
} else {
if (OffsetScale != 1 && OffsetScale != AccessSize) {
LOGMAN_MSG_A("Unhandled GenerateMemOperand OffsetScale: %d", OffsetScale);
LogMan::Msg::A("Unhandled GenerateMemOperand OffsetScale: %d", OffsetScale);
}
uint64_t Const;
if (IsInlineConstant(Offset, &Const)) {
@@ -551,12 +550,11 @@ MemOperand Arm64JITCore::GenerateMemOperand(uint8_t AccessSize, aarch64::Registe
case IR::MEM_OFFSET_UXTW.Val: return MemOperand(Base, RegOffset.W(), Extend::UXTW, (int)std::log2(OffsetScale) );
case IR::MEM_OFFSET_SXTW.Val: return MemOperand(Base, RegOffset.W(), Extend::SXTW, (int)std::log2(OffsetScale) );
default: LOGMAN_MSG_A("Unhandled GenerateMemOperand OffsetType: %d", OffsetType.Val); break;
default: LogMan::Msg::A("Unhandled GenerateMemOperand OffsetType: %d", OffsetType.Val); break;
}
}
}
FEX_UNREACHABLE;
__builtin_unreachable();
}
DEF_OP(LoadMem) {
@@ -580,7 +578,7 @@ DEF_OP(LoadMem) {
case 8:
ldr(Dst, MemSrc);
break;
default: LOGMAN_MSG_A("Unhandled LoadMem size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled LoadMem size: %d", Op->Size);
}
}
else {
@@ -601,7 +599,7 @@ DEF_OP(LoadMem) {
case 16:
ldr(Dst, MemSrc);
break;
default: LOGMAN_MSG_A("Unhandled LoadMem size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled LoadMem size: %d", Op->Size);
}
}
}
@@ -612,7 +610,7 @@ DEF_OP(LoadMemTSO) {
auto MemSrc = MemOperand(GetReg<RA_64>(Op->Header.Args[0].ID()));
if (!Op->Offset.IsInvalid()) {
LOGMAN_MSG_A("LoadMemTSO: No offset allowed");
LogMan::Msg::A("LoadMemTSO: No offset allowed");
}
if (SupportsRCPC && Op->Class == FEXCore::IR::GPRClass) {
@@ -635,7 +633,7 @@ DEF_OP(LoadMemTSO) {
case 8:
ldapr(Dst, MemSrc);
break;
default: LOGMAN_MSG_A("Unhandled LoadMem size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled LoadMem size: %d", Op->Size);
}
nop();
}
@@ -660,7 +658,7 @@ DEF_OP(LoadMemTSO) {
case 8:
ldar(Dst, MemSrc);
break;
default: LOGMAN_MSG_A("Unhandled LoadMem size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled LoadMem size: %d", Op->Size);
}
nop();
}
@@ -681,7 +679,7 @@ DEF_OP(LoadMemTSO) {
case 16:
ldr(Dst, MemSrc);
break;
default: LOGMAN_MSG_A("Unhandled LoadMem size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled LoadMem size: %d", Op->Size);
}
dmb(InnerShareable, BarrierAll);
}
@@ -708,7 +706,7 @@ DEF_OP(StoreMem) {
case 8:
str(GetReg<RA_64>(Op->Header.Args[1].ID()), MemSrc);
break;
default: LOGMAN_MSG_A("Unhandled StoreMem size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled StoreMem size: %d", Op->Size);
}
}
else {
@@ -729,7 +727,7 @@ DEF_OP(StoreMem) {
case 16:
str(Src, MemSrc);
break;
default: LOGMAN_MSG_A("Unhandled StoreMem size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled StoreMem size: %d", Op->Size);
}
}
}
@@ -739,7 +737,7 @@ DEF_OP(StoreMemTSO) {
auto MemSrc = MemOperand(GetReg<RA_64>(Op->Header.Args[0].ID()));
if (!Op->Offset.IsInvalid()) {
LOGMAN_MSG_A("StoreMemTSO: No offset allowed");
LogMan::Msg::A("StoreMemTSO: No offset allowed");
}
if (Op->Class == FEXCore::IR::GPRClass) {
@@ -759,7 +757,7 @@ DEF_OP(StoreMemTSO) {
case 8:
stlr(GetReg<RA_64>(Op->Header.Args[1].ID()), MemSrc);
break;
default: LOGMAN_MSG_A("Unhandled StoreMem size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled StoreMem size: %d", Op->Size);
}
nop();
}
@@ -783,177 +781,18 @@ DEF_OP(StoreMemTSO) {
case 16:
str(Src, MemSrc);
break;
default: LOGMAN_MSG_A("Unhandled StoreMem size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled StoreMem size: %d", Op->Size);
}
dmb(InnerShareable, BarrierAll);
}
}
DEF_OP(ParanoidLoadMemTSO) {
auto Op = IROp->C<IR::IROp_LoadMemTSO>();
auto MemSrc = MemOperand(GetReg<RA_64>(Op->Header.Args[0].ID()));
if (!Op->Offset.IsInvalid()) {
LOGMAN_MSG_A("LoadMemTSO: No offset allowed");
}
if (Op->Class == FEXCore::IR::GPRClass) {
if (Op->Size == 1) {
// 8bit load is always aligned to natural alignment
auto Dst = GetReg<RA_64>(Node);
ldarb(Dst, MemSrc);
}
else {
auto Dst = GetReg<RA_64>(Node);
nop();
switch (Op->Size) {
case 2:
ldarh(Dst, MemSrc);
break;
case 4:
ldar(Dst.W(), MemSrc);
break;
case 8:
ldar(Dst, MemSrc);
break;
default: LOGMAN_MSG_A("Unhandled LoadMem size: %d", Op->Size);
}
nop();
}
}
else {
auto Dst = GetDst(Node);
switch (Op->Size) {
case 2:
nop();
ldarh(TMP1, MemSrc);
nop();
fmov(Dst, TMP1);
break;
case 4:
nop();
ldar(TMP1.W(), MemSrc);
nop();
fmov(Dst, TMP1);
break;
case 8:
nop();
ldar(TMP1, MemSrc);
nop();
fmov(Dst, TMP1);
break;
case 16:
nop();
ldaxp(TMP1, TMP2, MemSrc);
clrex();
mov(Dst.V2D(), 0, TMP1);
mov(Dst.V2D(), 1, TMP2);
break;
default: LOGMAN_MSG_A("Unhandled LoadMem size: %d", Op->Size);
}
}
}
DEF_OP(ParanoidStoreMemTSO) {
auto Op = IROp->C<IR::IROp_StoreMemTSO>();
auto MemSrc = MemOperand(GetReg<RA_64>(Op->Header.Args[0].ID()));
if (!Op->Offset.IsInvalid()) {
LOGMAN_MSG_A("StoreMemTSO: No offset allowed");
}
if (Op->Class == FEXCore::IR::GPRClass) {
if (Op->Size == 1) {
// 8bit load is always aligned to natural alignment
stlrb(GetReg<RA_64>(Op->Header.Args[1].ID()), MemSrc);
}
else {
nop();
switch (Op->Size) {
case 2:
stlrh(GetReg<RA_64>(Op->Header.Args[1].ID()), MemSrc);
break;
case 4:
stlr(GetReg<RA_32>(Op->Header.Args[1].ID()), MemSrc);
break;
case 8:
stlr(GetReg<RA_64>(Op->Header.Args[1].ID()), MemSrc);
break;
default: LOGMAN_MSG_A("Unhandled StoreMem size: %d", Op->Size);
}
nop();
}
}
else {
auto Src = GetSrc(Op->Header.Args[1].ID());
if (Op->Size == 1) {
// 8bit load is always aligned to natural alignment
mov(TMP1, Src.V4S(), 0);
stlrb(TMP1, MemSrc);
}
else {
switch (Op->Size) {
case 2:
mov(TMP1, Src.V4S(), 0);
nop();
stlrh(TMP1, MemSrc);
nop();
break;
case 4:
mov(TMP1, Src.V4S(), 0);
nop();
stlr(TMP1.W(), MemSrc);
nop();
break;
case 8:
mov(TMP1, Src.V2D(), 0);
nop();
stlr(TMP1, MemSrc);
nop();
break;
case 16: {
// Move vector to GPRs
mov(TMP1, Src.V2D(), 0);
mov(TMP2, Src.V2D(), 1);
Label B;
bind(&B);
nop(); // < Overwritten with DMB
// ldaxp must not have both the destination registers be the same
ldaxp(xzr, TMP3, MemSrc); // <- Can hit SIGBUS
nop(); // < Overwritten with DMB
stlxp(TMP3, TMP1, TMP2, MemSrc); // <- Can also hit SIGBUS
cbnz(TMP3, &B); // < Overwritten with DMB
break;
}
default: LOGMAN_MSG_A("Unhandled StoreMem size: %d", Op->Size);
}
}
}
}
DEF_OP(VLoadMemElement) {
LOGMAN_MSG_A("Unimplemented");
LogMan::Msg::A("Unimplemented");
}
DEF_OP(VStoreMemElement) {
LOGMAN_MSG_A("Unimplemented");
}
DEF_OP(CacheLineClear) {
auto Op = IROp->C<IR::IROp_CacheLineClear>();
auto MemReg = GetReg<RA_64>(Op->Header.Args[0].ID());
// Clear dcache only
// icache doesn't matter here since the guest application shouldn't be calling clflush on JIT code.
mov(TMP1, MemReg);
for (size_t i = 0; i < std::max(1U, DCacheLineSize / 64U); ++i) {
dc(DataCacheOp::CVAU, TMP1);
add(TMP1, TMP1, DCacheLineSize);
}
dsb(InnerShareable, BarrierAll);
LogMan::Msg::A("Unimplemented");
}
#undef DEF_OP
@@ -971,17 +810,10 @@ void Arm64JITCore::RegisterMemoryHandlers() {
REGISTER_OP(STOREFLAG, StoreFlag);
REGISTER_OP(LOADMEM, LoadMem);
REGISTER_OP(STOREMEM, StoreMem);
if (ParanoidTSO()) {
REGISTER_OP(LOADMEMTSO, ParanoidLoadMemTSO);
REGISTER_OP(STOREMEMTSO, ParanoidStoreMemTSO);
}
else {
REGISTER_OP(LOADMEMTSO, LoadMemTSO);
REGISTER_OP(STOREMEMTSO, StoreMemTSO);
}
REGISTER_OP(LOADMEMTSO, LoadMemTSO);
REGISTER_OP(STOREMEMTSO, StoreMemTSO);
REGISTER_OP(VLOADMEMELEMENT, VLoadMemElement);
REGISTER_OP(VSTOREMEMELEMENT, VStoreMemElement);
REGISTER_OP(CACHELINECLEAR, CacheLineClear);
#undef REGISTER_OP
}
}
@@ -7,13 +7,6 @@ $end_info$
#include "Interface/Core/JIT/Arm64/JITClass.h"
namespace FEXCore::CPU {
static void PrintValue(uint64_t Value) {
LogMan::Msg::D("Value: 0x%lx", Value);
}
static void PrintVectorValue(uint64_t Value, uint64_t ValueUpper) {
LogMan::Msg::D("Value: 0x%016lx'%016lx", ValueUpper, Value);
}
using namespace vixl;
using namespace vixl::aarch64;
@@ -31,7 +24,7 @@ DEF_OP(Fence) {
case IR::Fence_Store.Val:
dmb(FullSystem, BarrierWrites);
break;
default: LOGMAN_MSG_A("Unknown Fence: %d", Op->Fence); break;
default: LogMan::Msg::A("Unknown Fence: %d", Op->Fence); break;
}
}
@@ -42,15 +35,6 @@ DEF_OP(Break) {
case 5: // Guest ud2
hlt(4);
break;
case 1: // Int <imm8>
hlt(4);
break;
case 2: // overflow
hlt(4);
break;
case 3: // int 1
hlt(4);
break;
case 4: { // HLT
// Time to quit
// Set our stack to the starting stack location
@@ -58,18 +42,18 @@ DEF_OP(Break) {
add(sp, TMP1, 0);
// Now we need to jump to the thread stop handler
LoadConstant(TMP1, ThreadSharedData.Dispatcher->ThreadStopHandlerAddressSpillSRA);
LoadConstant(TMP1, Dispatcher->ThreadStopHandlerAddressSpillSRA);
br(TMP1);
break;
}
case 6: { // INT3
ResetStack();
LoadConstant(TMP1, ThreadSharedData.Dispatcher->ThreadPauseHandlerAddressSpillSRA);
LoadConstant(TMP1, Dispatcher->ThreadPauseHandlerAddressSpillSRA);
br(TMP1);
break;
}
default: LOGMAN_MSG_A("Unknown Break reason: %d", Op->Reason);
default: LogMan::Msg::A("Unknown Break reason: %d", Op->Reason);
}
}
@@ -131,28 +115,6 @@ DEF_OP(SetRoundingMode) {
msr(FPCR, TMP1);
}
DEF_OP(Print) {
auto Op = IROp->C<IR::IROp_Print>();
PushDynamicRegsAndLR();
if (IsGPR(Op->Header.Args[0].ID())) {
mov(x0, GetReg<RA_64>(Op->Header.Args[0].ID()));
LoadConstant(x3, reinterpret_cast<uint64_t>(PrintValue));
}
else {
fmov(x0, GetSrc(Op->Header.Args[0].ID()).V1D());
// Bug in vixl that source vector needs to b V1D rather than V2D?
fmov(x1, GetSrc(Op->Header.Args[0].ID()).V1D(), 1);
LoadConstant(x3, reinterpret_cast<uint64_t>(PrintVectorValue));
}
SpillStaticRegs();
blr(x3);
FillStaticRegs();
PopDynamicRegsAndLR();
}
#undef DEF_OP
void Arm64JITCore::RegisterMiscHandlers() {
#define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &Arm64JITCore::Op_##x
@@ -165,7 +127,7 @@ void Arm64JITCore::RegisterMiscHandlers() {
REGISTER_OP(BREAK, Break);
REGISTER_OP(PHI, NoOp);
REGISTER_OP(PHIVALUE, NoOp);
REGISTER_OP(PRINT, Print);
REGISTER_OP(PRINT, Unhandled);
REGISTER_OP(GETROUNDINGMODE, GetRoundingMode);
REGISTER_OP(SETROUNDINGMODE, SetRoundingMode);
REGISTER_OP(INVALIDATEFLAGS, NoOp);
@@ -26,7 +26,7 @@ DEF_OP(ExtractElementPair) {
mov (GetReg<RA_64>(Node), Regs[Op->Element]);
break;
}
default: LOGMAN_MSG_A("Unknown Size"); break;
default: LogMan::Msg::A("Unknown Size"); break;
}
}
@@ -52,7 +52,7 @@ DEF_OP(CreateElementPair) {
RegTmp = x0;
break;
}
default: LOGMAN_MSG_A("Unknown Size"); break;
default: LogMan::Msg::A("Unknown Size"); break;
}
if (Dst.first.GetCode() != RegSecond.GetCode()) {
File diff suppressed because it is too large. Load diff
+2 -4
View File
@@ -1,7 +1,5 @@
#pragma once
#include <memory>
namespace FEXCore::Context {
struct Context;
}
@@ -13,6 +11,6 @@ struct InternalThreadState;
namespace FEXCore::CPU {
class CPUBackend;
std::unique_ptr<CPUBackend> CreateX86JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, bool CompileThread);
std::unique_ptr<CPUBackend> CreateArm64JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, bool CompileThread);
FEXCore::CPU::CPUBackend *CreateX86JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, bool CompileThread);
FEXCore::CPU::CPUBackend *CreateArm64JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, bool CompileThread);
}
+60 -77
View File
@@ -20,7 +20,7 @@ DEF_OP(TruncElementPair) {
mov(Dst.second, Src.second);
break;
}
default: LOGMAN_MSG_A("Unhandled Truncation size: %d", Op->Size); break;
default: LogMan::Msg::A("Unhandled Truncation size: %d", Op->Size); break;
}
}
@@ -32,7 +32,7 @@ DEF_OP(Constant) {
DEF_OP(EntrypointOffset) {
auto Op = IROp->C<IR::IROp_EntrypointOffset>();
auto Constant = Entry + Op->Offset;
auto Constant = IR->GetHeader()->Entry + Op->Offset;
mov(GetDst<RA_64>(Node), Constant);
}
@@ -70,7 +70,7 @@ DEF_OP(Add) {
case 8:
add(rax, Const);
break;
default: LOGMAN_MSG_A("Unhandled Add size: %d", OpSize);
default: LogMan::Msg::A("Unhandled Add size: %d", OpSize);
break;
}
} else {
@@ -81,7 +81,7 @@ DEF_OP(Add) {
case 8:
add(rax, GetSrc<RA_64>(Op->Header.Args[1].ID()));
break;
default: LOGMAN_MSG_A("Unhandled Add size: %d", OpSize);
default: LogMan::Msg::A("Unhandled Add size: %d", OpSize);
break;
}
}
@@ -103,7 +103,7 @@ DEF_OP(Sub) {
case 8:
sub(rax, Const);
break;
default: LOGMAN_MSG_A("Unhandled Sub size: %d", OpSize);
default: LogMan::Msg::A("Unhandled Sub size: %d", OpSize);
break;
}
} else {
@@ -114,7 +114,7 @@ DEF_OP(Sub) {
case 8:
sub(rax, GetSrc<RA_64>(Op->Header.Args[1].ID()));
break;
default: LOGMAN_MSG_A("Unhandled Sub size: %d", OpSize);
default: LogMan::Msg::A("Unhandled Sub size: %d", OpSize);
break;
}
}
@@ -136,7 +136,7 @@ DEF_OP(Neg) {
Src = GetSrc<RA_64>(Op->Header.Args[0].ID());
Dst = GetDst<RA_64>(Node);
break;
default: LOGMAN_MSG_A("Unhandled Neg size: %d", OpSize);
default: LogMan::Msg::A("Unhandled Neg size: %d", OpSize);
break;
}
mov(Dst, Src);
@@ -160,7 +160,7 @@ DEF_OP(Mul) {
imul(rax, GetSrc<RA_64>(Op->Header.Args[1].ID()));
mov(Dst, rax);
break;
default: LOGMAN_MSG_A("Unknown Sext size: %d", OpSize);
default: LogMan::Msg::A("Unknown Sext size: %d", OpSize);
}
}
@@ -179,7 +179,7 @@ DEF_OP(UMul) {
mul(GetSrc<RA_64>(Op->Header.Args[1].ID()));
mov(GetDst<RA_64>(Node), rax);
break;
default: LOGMAN_MSG_A("Unknown Sext size: %d", OpSize);
default: LogMan::Msg::A("Unknown Sext size: %d", OpSize);
}
}
@@ -218,7 +218,7 @@ DEF_OP(Div) {
mov(GetDst<RA_64>(Node), rax);
break;
}
default: LOGMAN_MSG_A("Unknown UDIV Size: %d", Size); break;
default: LogMan::Msg::A("Unknown UDIV Size: %d", Size); break;
}
}
@@ -261,7 +261,7 @@ DEF_OP(UDiv) {
mov(GetDst<RA_64>(Node), rax);
break;
}
default: LOGMAN_MSG_A("Unknown UDIV OpSize: %d", OpSize); break;
default: LogMan::Msg::A("Unknown UDIV OpSize: %d", OpSize); break;
}
}
@@ -298,7 +298,7 @@ DEF_OP(Rem) {
mov(GetDst<RA_64>(Node), rdx);
break;
}
default: LOGMAN_MSG_A("Unknown UDIV Size: %d", OpSize); break;
default: LogMan::Msg::A("Unknown UDIV Size: %d", OpSize); break;
}
}
@@ -341,7 +341,7 @@ DEF_OP(URem) {
mov(GetDst<RA_64>(Node), rdx);
break;
}
default: LOGMAN_MSG_A("Unknown UDIV OpSize: %d", OpSize); break;
default: LogMan::Msg::A("Unknown UDIV OpSize: %d", OpSize); break;
}
}
@@ -360,7 +360,7 @@ DEF_OP(MulH) {
imul(GetSrc<RA_64>(Op->Header.Args[1].ID()));
mov(GetDst<RA_64>(Node), rdx);
break;
default: LOGMAN_MSG_A("Unknown Sext size: %d", OpSize);
default: LogMan::Msg::A("Unknown Sext size: %d", OpSize);
}
}
@@ -379,7 +379,7 @@ DEF_OP(UMulH) {
mul(GetSrc<RA_64>(Op->Header.Args[1].ID()));
mov(GetDst<RA_64>(Node), rdx);
break;
default: LOGMAN_MSG_A("Unknown Sext size: %d", OpSize);
default: LogMan::Msg::A("Unknown Sext size: %d", OpSize);
}
}
@@ -441,7 +441,7 @@ DEF_OP(Lshl) {
mov(GetDst<RA_64>(Node), GetSrc<RA_64>(Op->Header.Args[0].ID()));
shl(GetDst<RA_64>(Node), Const);
break;
default: LOGMAN_MSG_A("Unknown LSHL Size: %d\n", OpSize); break;
default: LogMan::Msg::A("Unknown LSHL Size: %d\n", OpSize); break;
};
} else {
mov(rcx, GetSrc<RA_64>(Op->Header.Args[1].ID()));
@@ -456,7 +456,7 @@ DEF_OP(Lshl) {
mov(GetDst<RA_64>(Node), GetSrc<RA_64>(Op->Header.Args[0].ID()));
shl(GetDst<RA_64>(Node), cl);
break;
default: LOGMAN_MSG_A("Unknown LSHL Size: %d\n", OpSize); break;
default: LogMan::Msg::A("Unknown LSHL Size: %d\n", OpSize); break;
};
}
}
@@ -488,7 +488,7 @@ DEF_OP(Lshr) {
mov(GetDst<RA_64>(Node), GetSrc<RA_64>(Op->Header.Args[0].ID()));
shr(GetDst<RA_64>(Node), Const);
break;
default: LOGMAN_MSG_A("Unknown Size: %d\n", OpSize); break;
default: LogMan::Msg::A("Unknown Size: %d\n", OpSize); break;
};
} else {
@@ -512,7 +512,7 @@ DEF_OP(Lshr) {
mov(GetDst<RA_64>(Node), GetSrc<RA_64>(Op->Header.Args[0].ID()));
shr(GetDst<RA_64>(Node), cl);
break;
default: LOGMAN_MSG_A("Unknown Size: %d\n", OpSize); break;
default: LogMan::Msg::A("Unknown Size: %d\n", OpSize); break;
};
}
}
@@ -546,7 +546,7 @@ DEF_OP(Ashr) {
mov(GetDst<RA_64>(Node), GetSrc<RA_64>(Op->Header.Args[0].ID()));
sar(GetDst<RA_64>(Node), Const);
break;
default: LOGMAN_MSG_A("Unknown ASHR Size: %d\n", OpSize); break;
default: LogMan::Msg::A("Unknown ASHR Size: %d\n", OpSize); break;
};
} else {
@@ -571,7 +571,7 @@ DEF_OP(Ashr) {
mov(GetDst<RA_64>(Node), GetSrc<RA_64>(Op->Header.Args[0].ID()));
sar(GetDst<RA_64>(Node), cl);
break;
default: LOGMAN_MSG_A("Unknown ASHR Size: %d\n", OpSize); break;
default: LogMan::Msg::A("Unknown ASHR Size: %d\n", OpSize); break;
};
}
}
@@ -596,7 +596,7 @@ DEF_OP(Ror) {
ror(rax, Const);
break;
}
default: LOGMAN_MSG_A("Unknown ROR Size: %d\n", OpSize); break;
default: LogMan::Msg::A("Unknown ROR Size: %d\n", OpSize); break;
}
} else {
mov (rcx, GetSrc<RA_64>(Op->Header.Args[1].ID()));
@@ -612,7 +612,7 @@ DEF_OP(Ror) {
ror(rax, cl);
break;
}
default: LOGMAN_MSG_A("Unknown ROR Size: %d\n", OpSize); break;
default: LogMan::Msg::A("Unknown ROR Size: %d\n", OpSize); break;
}
}
mov(GetDst<RA_64>(Node), rax);
@@ -668,7 +668,7 @@ DEF_OP(LDiv) {
mov(GetDst<RA_64>(Node), rax);
break;
}
default: LOGMAN_MSG_A("Unknown LDIV OpSize: %d", OpSize); break;
default: LogMan::Msg::A("Unknown LDIV OpSize: %d", OpSize); break;
}
}
@@ -700,7 +700,7 @@ DEF_OP(LUDiv) {
mov(GetDst<RA_64>(Node), rax);
break;
}
default: LOGMAN_MSG_A("Unknown LUDIV OpSize: %d", OpSize); break;
default: LogMan::Msg::A("Unknown LUDIV OpSize: %d", OpSize); break;
}
}
@@ -732,7 +732,7 @@ DEF_OP(LRem) {
mov(GetDst<RA_64>(Node), rdx);
break;
}
default: LOGMAN_MSG_A("Unknown LREM OpSize: %d", OpSize); break;
default: LogMan::Msg::A("Unknown LREM OpSize: %d", OpSize); break;
}
}
@@ -764,7 +764,7 @@ DEF_OP(LURem) {
mov(GetDst<RA_64>(Node), rdx);
break;
}
default: LOGMAN_MSG_A("Unknown LUDIV OpSize: %d", OpSize); break;
default: LogMan::Msg::A("Unknown LUDIV OpSize: %d", OpSize); break;
}
}
@@ -829,7 +829,7 @@ DEF_OP(FindMSB) {
case 8:
bsr(GetDst<RA_64>(Node), GetSrc<RA_64>(Op->Header.Args[0].ID()));
break;
default: LOGMAN_MSG_A("Unknown OpSize: %d", OpSize);
default: LogMan::Msg::A("Unknown OpSize: %d", OpSize);
}
}
@@ -853,7 +853,7 @@ DEF_OP(FindTrailingZeros) {
mov(rax, 0x40);
cmovz(GetDst<RA_64>(Node), rax);
break;
default: LOGMAN_MSG_A("Unknown size: %d", OpSize); break;
default: LogMan::Msg::A("Unknown size: %d", OpSize); break;
}
}
@@ -876,7 +876,7 @@ DEF_OP(CountLeadingZeroes) {
lzcnt(GetDst<RA_64>(Node), GetSrc<RA_64>(Op->Header.Args[0].ID()));
break;
}
default: LOGMAN_MSG_A("Unknown size: %d", OpSize); break;
default: LogMan::Msg::A("Unknown size: %d", OpSize); break;
}
}
else {
@@ -915,7 +915,7 @@ DEF_OP(CountLeadingZeroes) {
mov(GetDst<RA_64>(Node), rax);
break;
}
default: LOGMAN_MSG_A("Unknown size: %d", OpSize); break;
default: LogMan::Msg::A("Unknown size: %d", OpSize); break;
}
}
}
@@ -937,7 +937,7 @@ DEF_OP(Rev) {
mov (GetDst<RA_64>(Node), GetSrc<RA_64>(Op->Header.Args[0].ID()));
bswap(GetDst<RA_64>(Node).cvt64());
break;
default: LOGMAN_MSG_A("Unknown REV size: %d", OpSize); break;
default: LogMan::Msg::A("Unknown REV size: %d", OpSize); break;
}
}
@@ -970,7 +970,9 @@ DEF_OP(Bfi) {
DEF_OP(Bfe) {
auto Op = IROp->C<IR::IROp_Bfe>();
LOGMAN_THROW_A(IROp->Size <= 8, "OpSize is too large for BFE: %d", IROp->Size);
uint8_t OpSize = IROp->Size;
LogMan::Throw::A(OpSize <= 8, "OpSize is too large for BFE: %d", OpSize);
auto Dst = GetDst<RA_64>(Node);
@@ -1071,7 +1073,7 @@ DEF_OP(Select) {
if (is_const_true || is_const_false) {
if (is_const_false != true || is_const_true != true || const_true != 1 || const_false != 0) {
LOGMAN_MSG_A("Select: Unsupported compare inline parameters");
LogMan::Msg::A("Select: Unsupported compare inline parameters");
}
(this->*SetCC)(al);
movzx(Dst, al);
@@ -1102,67 +1104,46 @@ DEF_OP(VExtractToGPR) {
pextrq(GetDst<RA_64>(Node), GetSrc(Op->Header.Args[0].ID()), Op->Idx);
break;
}
default: LOGMAN_MSG_A("Unknown Element Size: %d", Op->Header.ElementSize); break;
default: LogMan::Msg::A("Unknown Element Size: %d", Op->Header.ElementSize); break;
}
}
DEF_OP(Float_ToGPR_ZU) {
LogMan::Msg::D("Unimplemented");
}
DEF_OP(Float_ToGPR_ZS) {
auto Op = IROp->C<IR::IROp_Float_ToGPR_ZS>();
uint16_t Conv = (IROp->Size << 8) | Op->SrcElementSize;
switch (Conv) {
case 0x0804: // int64_t <- float
cvttss2si(GetDst<RA_64>(Node), GetSrc(Op->Header.Args[0].ID()));
break;
case 0x0808: // int64_t <- double
cvttsd2si(GetDst<RA_64>(Node), GetSrc(Op->Header.Args[0].ID()));
break;
case 0x0404: // int32_t <- float
cvttss2si(GetDst<RA_32>(Node), GetSrc(Op->Header.Args[0].ID()));
break;
case 0x0408: // int32_t <- double
cvttsd2si(GetDst<RA_32>(Node), GetSrc(Op->Header.Args[0].ID()));
break;
if (Op->Header.ElementSize == 8) {
cvttsd2si(GetDst<RA_64>(Node), GetSrc(Op->Header.Args[0].ID()));
}
else {
cvttss2si(GetDst<RA_32>(Node), GetSrc(Op->Header.Args[0].ID()));
}
}
DEF_OP(Float_ToGPR_U) {
LogMan::Msg::D("Unimplemented");
}
DEF_OP(Float_ToGPR_S) {
auto Op = IROp->C<IR::IROp_Float_ToGPR_S>();
uint16_t Conv = (IROp->Size << 8) | Op->SrcElementSize;
switch (Conv) {
case 0x0804: // int64_t <- float
cvtss2si(GetDst<RA_64>(Node), GetSrc(Op->Header.Args[0].ID()));
break;
case 0x0808: // int64_t <- double
cvtsd2si(GetDst<RA_64>(Node), GetSrc(Op->Header.Args[0].ID()));
break;
case 0x0404: // int32_t <- float
cvtss2si(GetDst<RA_32>(Node), GetSrc(Op->Header.Args[0].ID()));
break;
case 0x0408: // int32_t <- double
cvtsd2si(GetDst<RA_32>(Node), GetSrc(Op->Header.Args[0].ID()));
break;
if (Op->Header.ElementSize == 8) {
cvtsd2si(GetDst<RA_64>(Node), GetSrc(Op->Header.Args[0].ID()));
}
else {
cvtss2si(GetDst<RA_32>(Node), GetSrc(Op->Header.Args[0].ID()));
}
}
DEF_OP(FCmp) {
auto Op = IROp->C<IR::IROp_FCmp>();
if (Op->Flags & (1 << IR::FCMP_FLAG_UNORDERED)) {
if (Op->ElementSize == 4) {
ucomiss(GetSrc(Op->Header.Args[0].ID()), GetSrc(Op->Header.Args[1].ID()));
}
else {
ucomisd(GetSrc(Op->Header.Args[0].ID()), GetSrc(Op->Header.Args[1].ID()));
}
if (Op->ElementSize == 4) {
ucomiss(GetSrc(Op->Header.Args[0].ID()), GetSrc(Op->Header.Args[1].ID()));
}
else {
if (Op->ElementSize == 4) {
comiss(GetSrc(Op->Header.Args[0].ID()), GetSrc(Op->Header.Args[1].ID()));
}
else {
comisd(GetSrc(Op->Header.Args[0].ID()), GetSrc(Op->Header.Args[1].ID()));
}
ucomisd(GetSrc(Op->Header.Args[0].ID()), GetSrc(Op->Header.Args[1].ID()));
}
mov (rdx, 0);
@@ -1236,7 +1217,9 @@ void X86JITCore::RegisterALUHandlers() {
REGISTER_OP(SBFE, Sbfe);
REGISTER_OP(SELECT, Select);
REGISTER_OP(VEXTRACTTOGPR, VExtractToGPR);
REGISTER_OP(FLOAT_TOGPR_ZU, Float_ToGPR_ZU);
REGISTER_OP(FLOAT_TOGPR_ZS, Float_ToGPR_ZS);
REGISTER_OP(FLOAT_TOGPR_U, Float_ToGPR_U);
REGISTER_OP(FLOAT_TOGPR_S, Float_ToGPR_S);
REGISTER_OP(FCMP, FCmp);
#undef REGISTER_OP
@@ -55,7 +55,7 @@ DEF_OP(CASPair) {
mov(Dst.second, rdx);
break;
}
default: LOGMAN_MSG_A("Unsupported: %d", OpSize);
default: LogMan::Msg::A("Unsupported: %d", OpSize);
}
}
@@ -74,6 +74,7 @@ DEF_OP(CAS) {
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Header.Args[2].ID());
mov(rdx, GetSrc<RA_64>(Op->Header.Args[1].ID()));
mov(rax, GetSrc<RA_64>(Op->Header.Args[0].ID()));
// RCX now contains pointer
@@ -81,31 +82,31 @@ DEF_OP(CAS) {
// RDX contains our desired
lock();
switch (OpSize) {
case 1: {
cmpxchg(byte [MemReg], GetSrc<RA_8>(Op->Header.Args[1].ID()));
movzx(GetDst<RA_64>(Node), al);
cmpxchg(byte [MemReg], dl);
movzx(rax, al);
break;
}
case 2: {
cmpxchg(word [MemReg], GetSrc<RA_16>(Op->Header.Args[1].ID()));
movzx(GetDst<RA_64>(Node), ax);
cmpxchg(word [MemReg], dx);
movzx(rax, ax);
break;
}
case 4: {
cmpxchg(dword [MemReg], GetSrc<RA_32>(Op->Header.Args[1].ID()));
// RAX now contains the result
mov (GetDst<RA_64>(Node), eax);
cmpxchg(dword [MemReg], edx);
break;
}
case 8: {
cmpxchg(qword [MemReg], GetSrc<RA_64>(Op->Header.Args[1].ID()));
// RAX now contains the result
mov (GetDst<RA_64>(Node), rax);
cmpxchg(qword [MemReg], rdx);
break;
}
default: LOGMAN_MSG_A("Unsupported: %d", OpSize);
default: LogMan::Msg::A("Unsupported: %d", OpSize);
}
// RAX now contains the result
mov (GetDst<RA_64>(Node), rax);
}
DEF_OP(AtomicAdd) {
@@ -127,7 +128,7 @@ DEF_OP(AtomicAdd) {
case 8:
add(qword [MemReg], GetSrc<RA_64>(Op->Header.Args[1].ID()));
break;
default: LOGMAN_MSG_A("Unhandled AtomicAdd size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled AtomicAdd size: %d", Op->Size);
}
}
@@ -149,7 +150,7 @@ DEF_OP(AtomicSub) {
case 8:
sub(qword [MemReg], GetSrc<RA_64>(Op->Header.Args[1].ID()));
break;
default: LOGMAN_MSG_A("Unhandled AtomicAdd size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled AtomicAdd size: %d", Op->Size);
}
}
@@ -171,7 +172,7 @@ DEF_OP(AtomicAnd) {
case 8:
and_(qword [MemReg], GetSrc<RA_64>(Op->Header.Args[1].ID()));
break;
default: LOGMAN_MSG_A("Unhandled AtomicAdd size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled AtomicAdd size: %d", Op->Size);
}
}
@@ -193,7 +194,7 @@ DEF_OP(AtomicOr) {
case 8:
or_(qword [MemReg], GetSrc<RA_64>(Op->Header.Args[1].ID()));
break;
default: LOGMAN_MSG_A("Unhandled AtomicAdd size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled AtomicAdd size: %d", Op->Size);
}
}
@@ -215,7 +216,7 @@ DEF_OP(AtomicXor) {
case 8:
xor_(qword [MemReg], GetSrc<RA_64>(Op->Header.Args[1].ID()));
break;
default: LOGMAN_MSG_A("Unhandled AtomicAdd size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled AtomicAdd size: %d", Op->Size);
}
}
@@ -227,17 +228,17 @@ DEF_OP(AtomicSwap) {
switch (Op->Size) {
case 1:
movzx(GetDst<RA_64>(Node), GetSrc<RA_8>(Op->Header.Args[1].ID()));
mov(GetDst<RA_8>(Node), GetSrc<RA_8>(Op->Header.Args[1].ID()));
lock();
xchg(byte [MemReg], GetDst<RA_8>(Node));
break;
case 2:
movzx(GetDst<RA_64>(Node), GetSrc<RA_16>(Op->Header.Args[1].ID()));
mov(GetDst<RA_16>(Node), GetSrc<RA_16>(Op->Header.Args[1].ID()));
lock();
xchg(word [MemReg], GetDst<RA_16>(Node));
break;
case 4:
mov(GetDst<RA_64>(Node), GetSrc<RA_32>(Op->Header.Args[1].ID()));
mov(GetDst<RA_32>(Node), GetSrc<RA_32>(Op->Header.Args[1].ID()));
lock();
xchg(dword [MemReg], GetDst<RA_32>(Node));
break;
@@ -246,7 +247,7 @@ DEF_OP(AtomicSwap) {
lock();
xchg(qword [MemReg], GetDst<RA_64>(Node));
break;
default: LOGMAN_MSG_A("Unhandled AtomicAdd size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled AtomicAdd size: %d", Op->Size);
}
}
@@ -256,13 +257,13 @@ DEF_OP(AtomicFetchAdd) {
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Header.Args[0].ID());
switch (Op->Size) {
case 1:
movzx(rcx, GetSrc<RA_8>(Op->Header.Args[1].ID()));
mov(cl, GetSrc<RA_8>(Op->Header.Args[1].ID()));
lock();
xadd(byte [MemReg], cl);
movzx(GetDst<RA_32>(Node), cl);
break;
case 2:
movzx(rcx, GetSrc<RA_16>(Op->Header.Args[1].ID()));
mov(cx, GetSrc<RA_16>(Op->Header.Args[1].ID()));
lock();
xadd(word [MemReg], cx);
movzx(GetDst<RA_32>(Node), cx);
@@ -271,7 +272,7 @@ DEF_OP(AtomicFetchAdd) {
mov(ecx, GetSrc<RA_32>(Op->Header.Args[1].ID()));
lock();
xadd(dword [MemReg], ecx);
mov(GetDst<RA_64>(Node), ecx);
mov(GetDst<RA_32>(Node), ecx);
break;
case 8:
mov(rcx, GetSrc<RA_64>(Op->Header.Args[1].ID()));
@@ -279,7 +280,7 @@ DEF_OP(AtomicFetchAdd) {
xadd(qword [MemReg], rcx);
mov(GetDst<RA_64>(Node), rcx);
break;
default: LOGMAN_MSG_A("Unhandled AtomicFetchAdd size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled AtomicFetchAdd size: %d", Op->Size);
}
}
@@ -316,7 +317,7 @@ DEF_OP(AtomicFetchSub) {
xadd(qword [MemReg], rcx);
mov(GetDst<RA_64>(Node), rcx);
break;
default: LOGMAN_MSG_A("Unhandled AtomicFetchAdd size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled AtomicFetchAdd size: %d", Op->Size);
}
}
@@ -394,7 +395,7 @@ DEF_OP(AtomicFetchAnd) {
mov(GetDst<RA_64>(Node), TMP3.cvt64());
break;
}
default: LOGMAN_MSG_A("Unhandled AtomicFetchAdd size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled AtomicFetchAdd size: %d", Op->Size);
}
}
@@ -471,7 +472,7 @@ DEF_OP(AtomicFetchOr) {
mov(GetDst<RA_64>(Node), TMP3.cvt64());
break;
}
default: LOGMAN_MSG_A("Unhandled AtomicFetchAdd size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled AtomicFetchAdd size: %d", Op->Size);
}
}
@@ -548,7 +549,7 @@ DEF_OP(AtomicFetchXor) {
mov(GetDst<RA_64>(Node), TMP3.cvt64());
break;
}
default: LOGMAN_MSG_A("Unhandled AtomicFetchAdd size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled AtomicFetchAdd size: %d", Op->Size);
}
}
@@ -81,7 +81,7 @@ DEF_OP(ExitFunction) {
jmp(qword[rax]);
L(l_BranchHost);
dq(ThreadSharedData.Dispatcher->ExitFunctionLinkerAddress);
dq(Dispatcher->ExitFunctionLinkerAddress);
L(l_BranchGuest);
dq(NewRIP);
} else {
@@ -101,7 +101,7 @@ DEF_OP(ExitFunction) {
jmp(qword[LookupBase + 0]);
L(FullLookup);
mov(rax, ThreadSharedData.Dispatcher->AbsoluteLoopTopAddress);
mov(rax, Dispatcher->AbsoluteLoopTopAddress);
mov(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, State.rip)], RipReg);
jmp(rax);
}
@@ -248,28 +248,28 @@ DEF_OP(Thunk) {
DEF_OP(ValidateCode) {
auto Op = IROp->C<IR::IROp_ValidateCode>();
const auto* OldCode = (const uint8_t*)&Op->CodeOriginalLow;
uint8_t* OldCode = (uint8_t*)&Op->CodeOriginalLow;
int len = Op->CodeLength;
int idx = 0;
xor_(GetDst<RA_64>(Node), GetDst<RA_64>(Node));
mov(rax, Entry + Op->Offset);
mov(rax, IR->GetHeader()->Entry + Op->Offset);
mov(rbx, 1);
while (len >= 4) {
cmp(dword[rax + idx], *(const uint32_t*)(OldCode + idx));
cmp(dword[rax + idx], *(uint32_t*)(OldCode + idx));
cmovne(GetDst<RA_64>(Node), rbx);
len-=4;
idx+=4;
}
while (len >= 2) {
mov(rcx, *(const uint16_t*)(OldCode + idx));
mov(rcx, *(uint16_t*)(OldCode + idx));
cmp(word[rax + idx], cx);
cmovne(GetDst<RA_64>(Node), rbx);
len-=2;
idx+=2;
}
while (len >= 1) {
cmp(byte[rax + idx], *(const uint8_t*)(OldCode + idx));
cmp(byte[rax + idx], *(uint8_t*)(OldCode + idx));
cmovne(GetDst<RA_64>(Node), rbx);
len-=1;
idx+=1;
@@ -286,7 +286,7 @@ DEF_OP(RemoveCodeEntry) {
sub(rsp, 8); // Align
mov(rdi, STATE);
mov(rax, Entry); // imm64 move
mov(rax, IR->GetHeader()->Entry); // imm64 move
mov(rsi, rax);
@@ -319,9 +319,8 @@ DEF_OP(CPUID) {
//
// Result: RAX, RDX. 4xi32
// rsi can be in the source registers, so copy argument to edx first
mov (edx, GetSrc<RA_32>(Op->Header.Args[1].ID()));
mov (esi, GetSrc<RA_32>(Op->Header.Args[0].ID()));
mov (rsi, GetSrc<RA_64>(Op->Header.Args[0].ID()));
mov (rdx, GetSrc<RA_64>(Op->Header.Args[1].ID()));
mov (rdi, reinterpret_cast<uint64_t>(&CTX->CPUID));
auto NumPush = RA64.size();
@@ -31,7 +31,7 @@ DEF_OP(VInsGPR) {
pinsrq(GetDst(Node), GetSrc<RA_64>(Op->Header.Args[1].ID()), Op->Index);
break;
}
default: LOGMAN_MSG_A("Unknown Element Size: %d", Op->Header.ElementSize); break;
default: LogMan::Msg::A("Unknown Element Size: %d", Op->Header.ElementSize); break;
}
}
@@ -52,10 +52,14 @@ DEF_OP(VCastFromGPR) {
case 8:
vmovq(GetDst(Node), GetSrc<RA_64>(Op->Header.Args[0].ID()).cvt64());
break;
default: LOGMAN_MSG_A("Unknown castGPR element size: %d", Op->Header.ElementSize);
default: LogMan::Msg::A("Unknown castGPR element size: %d", Op->Header.ElementSize);
}
}
DEF_OP(Float_FromGPR_U) {
LogMan::Msg::A("Unimplemented");
}
DEF_OP(Float_FromGPR_S) {
auto Op = IROp->C<IR::IROp_Float_FromGPR_S>();
uint16_t Conv = (Op->Header.ElementSize << 8) | Op->SrcElementSize;
@@ -91,10 +95,14 @@ DEF_OP(Float_FToF) {
cvtsd2ss(GetDst(Node), GetSrc(Op->Header.Args[0].ID()));
break;
}
default: LOGMAN_MSG_A("Unknown FCVT sizes: 0x%x", Conv);
default: LogMan::Msg::A("Unknown FCVT sizes: 0x%x", Conv);
}
}
DEF_OP(Vector_UToF) {
LogMan::Msg::A("Unimplemented");
}
DEF_OP(Vector_SToF) {
auto Op = IROp->C<IR::IROp_Vector_SToF>();
switch (Op->Header.ElementSize) {
@@ -113,10 +121,14 @@ DEF_OP(Vector_SToF) {
cvtsi2sd(xmm15, rax);
movlhps(GetDst(Node), xmm15);
break;
default: LOGMAN_MSG_A("Unknown castGPR element size: %d", Op->Header.ElementSize);
default: LogMan::Msg::A("Unknown castGPR element size: %d", Op->Header.ElementSize);
}
}
DEF_OP(Vector_FToZU) {
LogMan::Msg::A("Unimplemented");
}
DEF_OP(Vector_FToZS) {
auto Op = IROp->C<IR::IROp_Vector_FToZS>();
switch (Op->Header.ElementSize) {
@@ -126,10 +138,14 @@ DEF_OP(Vector_FToZS) {
case 8:
cvttpd2dq(GetDst(Node), GetSrc(Op->Header.Args[0].ID()));
break;
default: LOGMAN_MSG_A("Unknown castGPR element size: %d", Op->Header.ElementSize);
default: LogMan::Msg::A("Unknown castGPR element size: %d", Op->Header.ElementSize);
}
}
DEF_OP(Vector_FToU) {
LogMan::Msg::A("Unimplemented");
}
DEF_OP(Vector_FToS) {
auto Op = IROp->C<IR::IROp_Vector_FToS>();
switch (Op->Header.ElementSize) {
@@ -139,7 +155,7 @@ DEF_OP(Vector_FToS) {
case 8:
cvtpd2dq(GetDst(Node), GetSrc(Op->Header.Args[0].ID()));
break;
default: LOGMAN_MSG_A("Unknown castGPR element size: %d", Op->Header.ElementSize);
default: LogMan::Msg::A("Unknown castGPR element size: %d", Op->Header.ElementSize);
}
}
@@ -156,39 +172,7 @@ DEF_OP(Vector_FToF) {
cvtpd2ps(GetDst(Node), GetSrc(Op->Header.Args[0].ID()));
break;
}
default: LOGMAN_MSG_A("Unknown Conversion Type : 0%04x", Conv); break;
}
}
DEF_OP(Vector_FToI) {
auto Op = IROp->C<IR::IROp_Vector_FToI>();
uint8_t RoundMode{};
switch (Op->Round) {
case FEXCore::IR::Round_Nearest.Val:
RoundMode = 0b0000'0'0'00;
break;
case FEXCore::IR::Round_Negative_Infinity.Val:
RoundMode = 0b0000'0'0'01;
break;
case FEXCore::IR::Round_Positive_Infinity.Val:
RoundMode = 0b0000'0'0'10;
break;
case FEXCore::IR::Round_Towards_Zero.Val:
RoundMode = 0b0000'0'0'11;
break;
case FEXCore::IR::Round_Host.Val:
RoundMode = 0b0000'0'1'00;
break;
}
switch (Op->Header.ElementSize) {
case 4:
roundps(GetDst(Node), GetSrc(Op->Header.Args[0].ID()), RoundMode);
break;
case 8:
roundpd(GetDst(Node), GetSrc(Op->Header.Args[0].ID()), RoundMode);
break;
default: LogMan::Msg::A("Unknown Conversion Type : 0%04x", Conv); break;
}
}
@@ -197,13 +181,16 @@ void X86JITCore::RegisterConversionHandlers() {
#define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &X86JITCore::Op_##x
REGISTER_OP(VINSGPR, VInsGPR);
REGISTER_OP(VCASTFROMGPR, VCastFromGPR);
REGISTER_OP(FLOAT_FROMGPR_U, Float_FromGPR_U);
REGISTER_OP(FLOAT_FROMGPR_S, Float_FromGPR_S);
REGISTER_OP(FLOAT_FTOF, Float_FToF);
REGISTER_OP(VECTOR_UTOF, Vector_UToF);
REGISTER_OP(VECTOR_STOF, Vector_SToF);
REGISTER_OP(VECTOR_FTOZU, Vector_FToZU);
REGISTER_OP(VECTOR_FTOZS, Vector_FToZS);
REGISTER_OP(VECTOR_FTOU, Vector_FToU);
REGISTER_OP(VECTOR_FTOS, Vector_FToS);
REGISTER_OP(VECTOR_FTOF, Vector_FToF);
REGISTER_OP(VECTOR_FTOI, Vector_FToI);
#undef REGISTER_OP
}
}
+57 -66
View File
@@ -15,7 +15,6 @@ $end_info$
#include <FEXCore/Core/X86Enums.h>
#include <FEXCore/Core/UContext.h>
#include <FEXCore/Utils/Allocator.h>
#include <cmath>
#include <signal.h>
@@ -31,17 +30,17 @@ CodeBuffer AllocateNewCodeBuffer(size_t Size) {
CodeBuffer Buffer;
Buffer.Size = Size;
Buffer.Ptr = static_cast<uint8_t*>(
FEXCore::Allocator::mmap(nullptr,
mmap(nullptr,
Buffer.Size,
PROT_READ | PROT_WRITE | PROT_EXEC,
MAP_PRIVATE | MAP_ANONYMOUS,
-1, 0));
LOGMAN_THROW_A(Buffer.Ptr != reinterpret_cast<uint8_t*>(~0ULL), "Couldn't allocate code buffer");
LogMan::Throw::A(Buffer.Ptr != reinterpret_cast<uint8_t*>(~0ULL), "Couldn't allocate code buffer");
return Buffer;
}
void FreeCodeBuffer(CodeBuffer Buffer) {
FEXCore::Allocator::munmap(Buffer.Ptr, Buffer.Size);
munmap(Buffer.Ptr, Buffer.Size);
}
}
@@ -84,10 +83,8 @@ void X86JITCore::PopRegs() {
void X86JITCore::Op_Unhandled(FEXCore::IR::IROp_Header *IROp, uint32_t Node) {
FallbackInfo Info;
if (!InterpreterOps::GetFallbackHandler(IROp, &Info)) {
#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
auto Name = FEXCore::IR::GetName(IROp->Op);
LOGMAN_MSG_A("Unhandled IR Op: %s", std::string(Name).c_str());
#endif
LogMan::Msg::A("Unhandled IR Op: %s", std::string(Name).c_str());
} else {
switch(Info.ABI) {
case FABI_VOID_U16: {
@@ -284,11 +281,8 @@ void X86JITCore::Op_Unhandled(FEXCore::IR::IROp_Header *IROp, uint32_t Node) {
case FABI_UNKNOWN:
default:
#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
auto Name = FEXCore::IR::GetName(IROp->Op);
LOGMAN_MSG_A("Unhandled IR Fallback abi: %s %d", std::string(Name).c_str(), Info.ABI);
#endif
break;
LogMan::Msg::A("Unhandled IR Fallback abi: %s %d", std::string(Name).c_str(), Info.ABI);
}
}
}
@@ -336,24 +330,24 @@ X86JITCore::X86JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalTh
config.ExitFunctionLink = reinterpret_cast<uintptr_t>(&ExitFunctionLink);
config.ExitFunctionLinkThis = reinterpret_cast<uintptr_t>(this);
Dispatcher = std::make_unique<X86Dispatcher>(CTX, ThreadState, config);
Dispatcher = new X86Dispatcher(CTX, ThreadState, config);
DispatchPtr = Dispatcher->DispatchPtr;
CallbackPtr = Dispatcher->CallbackPtr;
ThreadSharedData.SignalHandlerRefCounterPtr = &Dispatcher->SignalHandlerRefCounter;
ThreadSharedData.SignalHandlerReturnAddress = Dispatcher->SignalHandlerReturnAddress;
ThreadSharedData.Dispatcher = Dispatcher.get();
// This will register the host signal handler per thread, which is fine
CTX->SignalDelegation->RegisterHostSignalHandler(SIGILL, [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext) -> bool {
X86JITCore *Core = reinterpret_cast<X86JITCore*>(Thread->CPUBackend.get());
return Core->Dispatcher->HandleSIGILL(Signal, info, ucontext);
}, true);
});
CTX->SignalDelegation->RegisterHostSignalHandler(SignalDelegator::SIGNAL_FOR_PAUSE, [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext) -> bool {
X86JITCore *Core = reinterpret_cast<X86JITCore*>(Thread->CPUBackend.get());
return Core->Dispatcher->HandleSignalPause(Signal, info, ucontext);
}, true);
});
auto GuestSignalHandler = [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext, GuestSigAction *GuestAction, stack_t *GuestStack) -> bool {
X86JITCore *Core = reinterpret_cast<X86JITCore*>(Thread->CPUBackend.get());
@@ -416,106 +410,106 @@ void X86JITCore::ClearCache() {
}
}
IR::PhysicalRegister X86JITCore::GetPhys(uint32_t Node) const {
IR::PhysicalRegister X86JITCore::GetPhys(uint32_t Node) {
auto PhyReg = RAData->GetNodeRegister(Node);
LOGMAN_THROW_A(PhyReg.Raw != 255, "Couldn't Allocate register for node: ssa%d. Class: %d", Node, PhyReg.Class);
LogMan::Throw::A(PhyReg.Raw != 255, "Couldn't Allocate register for node: ssa%d. Class: %d", Node, PhyReg.Class);
return PhyReg;
}
bool X86JITCore::IsFPR(uint32_t Node) const {
bool X86JITCore::IsFPR(uint32_t Node) {
return RAData->GetNodeRegister(Node).Class == IR::FPRClass.Val;
}
bool X86JITCore::IsGPR(uint32_t Node) const {
bool X86JITCore::IsGPR(uint32_t Node) {
return RAData->GetNodeRegister(Node).Class == IR::GPRClass.Val;
}
template<uint8_t RAType>
Xbyak::Reg X86JITCore::GetSrc(uint32_t Node) const {
Xbyak::Reg X86JITCore::GetSrc(uint32_t Node) {
// rax, rcx, rdx, rsi, r8, r9,
// r10
// Callee Saved
// rbx, rbp, r12, r13, r14, r15
auto PhyReg = GetPhys(Node);
if constexpr (RAType == RA_64)
if (RAType == RA_64)
return RA64[PhyReg.Reg].cvt64();
else if constexpr (RAType == RA_XMM)
else if (RAType == RA_XMM)
return RAXMM[PhyReg.Reg];
else if constexpr (RAType == RA_32)
else if (RAType == RA_32)
return RA64[PhyReg.Reg].cvt32();
else if constexpr (RAType == RA_16)
else if (RAType == RA_16)
return RA64[PhyReg.Reg].cvt16();
else if constexpr (RAType == RA_8)
else if (RAType == RA_8)
return RA64[PhyReg.Reg].cvt8();
}
template
Xbyak::Reg X86JITCore::GetSrc<X86JITCore::RA_64>(uint32_t Node) const;
Xbyak::Reg X86JITCore::GetSrc<X86JITCore::RA_64>(uint32_t Node);
template
Xbyak::Reg X86JITCore::GetSrc<X86JITCore::RA_32>(uint32_t Node) const;
Xbyak::Reg X86JITCore::GetSrc<X86JITCore::RA_32>(uint32_t Node);
template
Xbyak::Reg X86JITCore::GetSrc<X86JITCore::RA_16>(uint32_t Node) const;
Xbyak::Reg X86JITCore::GetSrc<X86JITCore::RA_16>(uint32_t Node);
template
Xbyak::Reg X86JITCore::GetSrc<X86JITCore::RA_8>(uint32_t Node) const;
Xbyak::Reg X86JITCore::GetSrc<X86JITCore::RA_8>(uint32_t Node);
Xbyak::Xmm X86JITCore::GetSrc(uint32_t Node) const {
Xbyak::Xmm X86JITCore::GetSrc(uint32_t Node) {
auto PhyReg = GetPhys(Node);
return RAXMM_x[PhyReg.Reg];
}
template<uint8_t RAType>
Xbyak::Reg X86JITCore::GetDst(uint32_t Node) const {
Xbyak::Reg X86JITCore::GetDst(uint32_t Node) {
auto PhyReg = GetPhys(Node);
if constexpr (RAType == RA_64)
if (RAType == RA_64)
return RA64[PhyReg.Reg].cvt64();
else if constexpr (RAType == RA_XMM)
else if (RAType == RA_XMM)
return RAXMM[PhyReg.Reg];
else if constexpr (RAType == RA_32)
else if (RAType == RA_32)
return RA64[PhyReg.Reg].cvt32();
else if constexpr (RAType == RA_16)
else if (RAType == RA_16)
return RA64[PhyReg.Reg].cvt16();
else if constexpr (RAType == RA_8)
else if (RAType == RA_8)
return RA64[PhyReg.Reg].cvt8();
}
template
Xbyak::Reg X86JITCore::GetDst<X86JITCore::RA_64>(uint32_t Node) const;
Xbyak::Reg X86JITCore::GetDst<X86JITCore::RA_64>(uint32_t Node);
template
Xbyak::Reg X86JITCore::GetDst<X86JITCore::RA_32>(uint32_t Node) const;
Xbyak::Reg X86JITCore::GetDst<X86JITCore::RA_32>(uint32_t Node);
template
Xbyak::Reg X86JITCore::GetDst<X86JITCore::RA_16>(uint32_t Node) const;
Xbyak::Reg X86JITCore::GetDst<X86JITCore::RA_16>(uint32_t Node);
template
Xbyak::Reg X86JITCore::GetDst<X86JITCore::RA_8>(uint32_t Node) const;
Xbyak::Reg X86JITCore::GetDst<X86JITCore::RA_8>(uint32_t Node);
template<uint8_t RAType>
std::pair<Xbyak::Reg, Xbyak::Reg> X86JITCore::GetSrcPair(uint32_t Node) const {
std::pair<Xbyak::Reg, Xbyak::Reg> X86JITCore::GetSrcPair(uint32_t Node) {
auto PhyReg = GetPhys(Node);
if constexpr (RAType == RA_64)
if (RAType == RA_64)
return RA64Pair[PhyReg.Reg];
else if constexpr (RAType == RA_32)
else if (RAType == RA_32)
return {RA64Pair[PhyReg.Reg].first.cvt32(), RA64Pair[PhyReg.Reg].second.cvt32()};
}
template
std::pair<Xbyak::Reg, Xbyak::Reg> X86JITCore::GetSrcPair<X86JITCore::RA_64>(uint32_t Node) const;
std::pair<Xbyak::Reg, Xbyak::Reg> X86JITCore::GetSrcPair<X86JITCore::RA_64>(uint32_t Node);
template
std::pair<Xbyak::Reg, Xbyak::Reg> X86JITCore::GetSrcPair<X86JITCore::RA_32>(uint32_t Node) const;
std::pair<Xbyak::Reg, Xbyak::Reg> X86JITCore::GetSrcPair<X86JITCore::RA_32>(uint32_t Node);
Xbyak::Xmm X86JITCore::GetDst(uint32_t Node) const {
Xbyak::Xmm X86JITCore::GetDst(uint32_t Node) {
auto PhyReg = GetPhys(Node);
return RAXMM_x[PhyReg.Reg];
}
bool X86JITCore::IsInlineConstant(const IR::OrderedNodeWrapper& WNode, uint64_t* Value) const {
bool X86JITCore::IsInlineConstant(const IR::OrderedNodeWrapper& WNode, uint64_t* Value) {
auto OpHeader = IR->GetOp<IR::IROp_Header>(WNode);
if (OpHeader->Op == IR::IROps::OP_INLINECONSTANT) {
@@ -529,13 +523,13 @@ bool X86JITCore::IsInlineConstant(const IR::OrderedNodeWrapper& WNode, uint64_t*
}
}
bool X86JITCore::IsInlineEntrypointOffset(const IR::OrderedNodeWrapper& WNode, uint64_t* Value) const {
bool X86JITCore::IsInlineEntrypointOffset(const IR::OrderedNodeWrapper& WNode, uint64_t* Value) {
auto OpHeader = IR->GetOp<IR::IROp_Header>(WNode);
if (OpHeader->Op == IR::IROps::OP_INLINEENTRYPOINTOFFSET) {
auto Op = OpHeader->C<IR::IROp_InlineEntrypointOffset>();
if (Value) {
*Value = Entry + Op->Offset;
*Value = IR->GetHeader()->Entry + Op->Offset;
}
return true;
} else {
@@ -568,7 +562,7 @@ std::tuple<X86JITCore::SetCC, X86JITCore::CMovCC, X86JITCore::JCC> X86JITCore::G
case FEXCore::IR::COND_VS:
case FEXCore::IR::COND_VC:
default:
LOGMAN_MSG_A("Unsupported compare type");
LogMan::Msg::A("Unsupported compare type");
break;
}
@@ -576,11 +570,10 @@ std::tuple<X86JITCore::SetCC, X86JITCore::CMovCC, X86JITCore::JCC> X86JITCore::G
return { &CodeGenerator::sete , &CodeGenerator::cmove , &CodeGenerator::je };
}
void *X86JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IRListView const *IR, [[maybe_unused]] FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) {
void *X86JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const *IR, [[maybe_unused]] FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) {
JumpTargets.clear();
uint32_t SSACount = IR->GetSSACount();
this->Entry = Entry;
this->RAData = RAData;
// Fairly excessive buffer range to make sure we don't overflow
@@ -589,7 +582,7 @@ void *X86JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IRLi
ThreadState->CTX->ClearCodeCache(ThreadState, false);
}
void *GuestEntry = getCurr<void*>();
void *Entry = getCurr<void*>();
this->IR = IR;
if (CTX->GetGdbServerStatus()) {
@@ -604,14 +597,14 @@ void *X86JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IRLi
cmp(dword [rax + (offsetof(FEXCore::Context::Context, Config.RunningMode))], 0);
je(RunBlock);
// Else we need to pause now
mov(rax, ThreadSharedData.Dispatcher->ThreadPauseHandlerAddress);
mov(rax, Dispatcher->ThreadPauseHandlerAddress);
jmp(rax);
ud2();
L(RunBlock);
}
LOGMAN_THROW_A(RAData != nullptr, "Needs RA");
LogMan::Throw::A(RAData != nullptr, "Needs RA");
SpillSlots = RAData->SpillSlots();
@@ -620,7 +613,7 @@ void *X86JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IRLi
}
#ifdef BLOCKSTATS
BlockSamplingData::BlockData *SamplingData = CTX->BlockData->GetBlockData(Entry);
BlockSamplingData::BlockData *SamplingData = CTX->BlockData->GetBlockData(HeaderOp->Entry);
if (GetSamplingData) {
mov(rcx, reinterpret_cast<uintptr_t>(SamplingData));
rdtsc();
@@ -667,10 +660,8 @@ void *X86JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IRLi
for (auto [BlockNode, BlockHeader] : IR->GetBlocks()) {
using namespace FEXCore::IR;
{
#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
auto BlockIROp = BlockHeader->CW<IROp_CodeBlock>();
LOGMAN_THROW_A(BlockIROp->Header.Op == IR::OP_CODEBLOCK, "IR type failed to be a code block");
#endif
LogMan::Throw::A(BlockIROp->Header.Op == IR::OP_CODEBLOCK, "IR type failed to be a code block");
uint32_t Node = IR->GetID(BlockNode);
auto IsTarget = JumpTargets.find(Node);
@@ -739,15 +730,15 @@ void *X86JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IRLi
}
PendingTargetLabel = nullptr;
void *GuestExit = getCurr<void*>();
void *Exit = getCurr<void*>();
this->IR = nullptr;
ready();
if (DebugData) {
DebugData->HostCodeSize = reinterpret_cast<uintptr_t>(GuestExit) - reinterpret_cast<uintptr_t>(GuestEntry);
DebugData->HostCodeSize = reinterpret_cast<uintptr_t>(Exit) - reinterpret_cast<uintptr_t>(Entry);
}
return GuestEntry;
return Entry;
}
uint64_t X86JITCore::ExitFunctionLink(X86JITCore *core, FEXCore::Core::CpuStateFrame *Frame, uint64_t *record) {
@@ -758,10 +749,10 @@ uint64_t X86JITCore::ExitFunctionLink(X86JITCore *core, FEXCore::Core::CpuStateF
if (!HostCode) {
Thread->CurrentFrame->State.rip = GuestRip;
return core->ThreadSharedData.Dispatcher->AbsoluteLoopTopAddress;
return core->Dispatcher->AbsoluteLoopTopAddress;
}
auto LinkerAddress = core->ThreadSharedData.Dispatcher->ExitFunctionLinkerAddress;
auto LinkerAddress = core->Dispatcher->ExitFunctionLinkerAddress;
Thread->LookupCache->AddBlockLink(GuestRip, (uintptr_t)record, [record, LinkerAddress]{
// undo the link
record[0] = LinkerAddress;
@@ -771,7 +762,7 @@ uint64_t X86JITCore::ExitFunctionLink(X86JITCore *core, FEXCore::Core::CpuStateF
return HostCode;
}
std::unique_ptr<CPUBackend> CreateX86JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, bool CompileThread) {
return std::make_unique<X86JITCore>(ctx, Thread, AllocateNewCodeBuffer(CompileThread ? X86JITCore::MAX_CODE_SIZE : X86JITCore::INITIAL_CODE_SIZE), CompileThread);
FEXCore::CPU::CPUBackend *CreateX86JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, bool CompileThread) {
return new X86JITCore(ctx, Thread, AllocateNewCodeBuffer(CompileThread ? X86JITCore::MAX_CODE_SIZE : X86JITCore::INITIAL_CODE_SIZE), CompileThread);
}
}
+18 -24
View File
@@ -66,7 +66,7 @@ public:
explicit X86JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, CodeBuffer Buffer, bool CompileThread);
~X86JITCore() override;
std::string GetName() override { return "JIT"; }
void *CompileCode(uint64_t Entry, FEXCore::IR::IRListView const *IR, FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) override;
void *CompileCode(FEXCore::IR::IRListView const *IR, FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) override;
void *MapRegion(void* HostPtr, uint64_t, uint64_t) override { return HostPtr; }
@@ -83,8 +83,7 @@ private:
FEXCore::Context::Context *CTX;
FEXCore::Core::InternalThreadState *ThreadState;
FEXCore::IR::IRListView const *IR;
std::unique_ptr<FEXCore::CPU::Dispatcher> Dispatcher;
uint64_t Entry;
FEXCore::CPU::Dispatcher *Dispatcher;
std::unordered_map<IR::OrderedNodeWrapper::NodeOffsetType, Label> JumpTargets;
Xbyak::util::Cpu Features{};
@@ -112,26 +111,26 @@ private:
constexpr static uint8_t RA_64 = 3;
constexpr static uint8_t RA_XMM = 4;
IR::PhysicalRegister GetPhys(uint32_t Node) const;
IR::PhysicalRegister GetPhys(uint32_t Node);
bool IsFPR(uint32_t Node) const;
bool IsGPR(uint32_t Node) const;
bool IsFPR(uint32_t Node);
bool IsGPR(uint32_t Node);
template<uint8_t RAType>
Xbyak::Reg GetSrc(uint32_t Node) const;
Xbyak::Reg GetSrc(uint32_t Node);
template<uint8_t RAType>
std::pair<Xbyak::Reg, Xbyak::Reg> GetSrcPair(uint32_t Node) const;
std::pair<Xbyak::Reg, Xbyak::Reg> GetSrcPair(uint32_t Node);
template<uint8_t RAType>
Xbyak::Reg GetDst(uint32_t Node) const;
Xbyak::Reg GetDst(uint32_t Node);
Xbyak::Xmm GetSrc(uint32_t Node) const;
Xbyak::Xmm GetDst(uint32_t Node) const;
Xbyak::Xmm GetSrc(uint32_t Node);
Xbyak::Xmm GetDst(uint32_t Node);
Xbyak::RegExp GenerateModRM(Xbyak::Reg Base, IR::OrderedNodeWrapper Offset, IR::MemOffsetType OffsetType, uint8_t OffsetScale) const;
Xbyak::RegExp GenerateModRM(Xbyak::Reg Base, IR::OrderedNodeWrapper Offset, IR::MemOffsetType OffsetType, uint8_t OffsetScale);
bool IsInlineConstant(const IR::OrderedNodeWrapper& Node, uint64_t* Value = nullptr) const;
bool IsInlineEntrypointOffset(const IR::OrderedNodeWrapper& WNode, uint64_t* Value) const;
bool IsInlineConstant(const IR::OrderedNodeWrapper& Node, uint64_t* Value = nullptr);
bool IsInlineEntrypointOffset(const IR::OrderedNodeWrapper& WNode, uint64_t* Value);
IR::RegisterAllocationPass *RAPass;
FEXCore::IR::RegisterAllocationData *RAData;
@@ -162,7 +161,6 @@ private:
uint64_t SignalHandlerReturnAddress{};
uint32_t *SignalHandlerRefCounterPtr{};
FEXCore::CPU::Dispatcher *Dispatcher{};
};
CompilerSharedData ThreadSharedData;
@@ -241,7 +239,9 @@ private:
DEF_OP(Sbfe);
DEF_OP(Select);
DEF_OP(VExtractToGPR);
DEF_OP(Float_ToGPR_ZU);
DEF_OP(Float_ToGPR_ZS);
DEF_OP(Float_ToGPR_U);
DEF_OP(Float_ToGPR_S);
DEF_OP(FCmp);
DEF_OP(F80Cmp);
@@ -279,14 +279,16 @@ private:
///< Conversion ops
DEF_OP(VInsGPR);
DEF_OP(VCastFromGPR);
DEF_OP(Float_FromGPR_U);
DEF_OP(Float_FromGPR_S);
DEF_OP(Float_FToF);
DEF_OP(Vector_UToF);
DEF_OP(Vector_SToF);
DEF_OP(Vector_FToZU);
DEF_OP(Vector_FToZS);
DEF_OP(Vector_FToU);
DEF_OP(Vector_FToS);
DEF_OP(Vector_FToF);
DEF_OP(Vector_FToI);
///< Flag ops
DEF_OP(GetHostFlag);
@@ -304,7 +306,6 @@ private:
DEF_OP(StoreMem);
DEF_OP(VLoadMemElement);
DEF_OP(VStoreMemElement);
DEF_OP(CacheLineClear);
///< Misc ops
DEF_OP(EndBlock);
@@ -329,7 +330,6 @@ private:
DEF_OP(SplatVector);
DEF_OP(VMov);
DEF_OP(VAnd);
DEF_OP(VBic);
DEF_OP(VOr);
DEF_OP(VXor);
DEF_OP(VAdd);
@@ -340,10 +340,8 @@ private:
DEF_OP(VSQSub);
DEF_OP(VAddP);
DEF_OP(VAddV);
DEF_OP(VUMinV);
DEF_OP(VURAvg);
DEF_OP(VAbs);
DEF_OP(VPopcount);
DEF_OP(VFAdd);
DEF_OP(VFAddP);
DEF_OP(VFSub);
@@ -363,8 +361,6 @@ private:
DEF_OP(VSMax);
DEF_OP(VZip);
DEF_OP(VZip2);
DEF_OP(VUnZip);
DEF_OP(VUnZip2);
DEF_OP(VBSL);
DEF_OP(VCMPEQ);
DEF_OP(VCMPEQZ);
@@ -387,7 +383,6 @@ private:
DEF_OP(VInsElement);
DEF_OP(VInsScalarElement);
DEF_OP(VExtractElement);
DEF_OP(VDupElement);
DEF_OP(VExtr);
DEF_OP(VSLI);
DEF_OP(VSRI);
@@ -410,7 +405,6 @@ private:
DEF_OP(VSMull);
DEF_OP(VUMull2);
DEF_OP(VSMull2);
DEF_OP(VUABDL);
DEF_OP(VTBL1);
///< Encryption ops
@@ -36,10 +36,10 @@ DEF_OP(LoadContext) {
}
break;
case 16: {
LOGMAN_MSG_A("Invalid GPR load of size 16");
LogMan::Msg::A("Invalid GPR load of size 16");
}
break;
default: LOGMAN_MSG_A("Unhandled LoadContext size: %d", OpSize);
default: LogMan::Msg::A("Unhandled LoadContext size: %d", OpSize);
}
}
else {
@@ -69,7 +69,7 @@ DEF_OP(LoadContext) {
movups(GetDst(Node), xword [STATE + Op->Offset]);
}
break;
default: LOGMAN_MSG_A("Unhandled LoadContext size: %d", OpSize);
default: LogMan::Msg::A("Unhandled LoadContext size: %d", OpSize);
}
}
}
@@ -100,7 +100,7 @@ DEF_OP(StoreContext) {
case 16:
LogMan::Msg::D("Invalid store size of 16");
break;
default: LOGMAN_MSG_A("Unhandled StoreContext size: %d", OpSize);
default: LogMan::Msg::A("Unhandled StoreContext size: %d", OpSize);
}
}
else {
@@ -129,7 +129,7 @@ DEF_OP(StoreContext) {
movups(xword [STATE + Op->Offset], GetSrc(Op->Header.Args[0].ID()));
}
break;
default: LOGMAN_MSG_A("Unhandled StoreContext size: %d", OpSize);
default: LogMan::Msg::A("Unhandled StoreContext size: %d", OpSize);
}
}
}
@@ -160,15 +160,15 @@ DEF_OP(LoadContextIndexed) {
mov(GetDst<RA_64>(Node), qword [rax + index * Op->Stride]);
break;
default:
LOGMAN_MSG_A("Unhandled LoadContextIndexed size: %d", Op->Size);
LogMan::Msg::A("Unhandled LoadContextIndexed size: %d", Op->Size);
}
break;
}
case 16:
LOGMAN_MSG_A("Invalid Class load of size 16");
LogMan::Msg::A("Invalid Class load of size 16");
break;
default:
LOGMAN_MSG_A("Unhandled LoadContextIndexed stride: %d", Op->Stride);
LogMan::Msg::A("Unhandled LoadContextIndexed stride: %d", Op->Stride);
}
}
@@ -195,7 +195,7 @@ DEF_OP(LoadContextIndexed) {
vmovq(GetDst(Node), qword [rax + index * Op->Stride]);
break;
default:
LOGMAN_MSG_A("Unhandled LoadContextIndexed size: %d", Op->Size);
LogMan::Msg::A("Unhandled LoadContextIndexed size: %d", Op->Size);
}
break;
}
@@ -223,12 +223,12 @@ DEF_OP(LoadContextIndexed) {
movups(GetDst(Node), xword [STATE + rax]);
break;
default:
LOGMAN_MSG_A("Unhandled LoadContextIndexed size: %d", Op->Size);
LogMan::Msg::A("Unhandled LoadContextIndexed size: %d", Op->Size);
}
break;
}
default:
LOGMAN_MSG_A("Unhandled LoadContextIndexed stride: %d", Op->Stride);
LogMan::Msg::A("Unhandled LoadContextIndexed stride: %d", Op->Stride);
}
}
}
@@ -248,13 +248,13 @@ DEF_OP(StoreContextIndexed) {
case 4:
case 8: {
if (!(size == 1 || size == 2 || size == 4 || size == 8)) {
LOGMAN_MSG_A("Unhandled StoreContextIndexed size: %d", Op->Size);
LogMan::Msg::A("Unhandled StoreContextIndexed size: %d", Op->Size);
}
mov(AddressFrame(Op->Size * 8) [rax + index * Op->Stride], value);
break;
}
default:
LOGMAN_MSG_A("Unhandled StoreContextIndexed stride: %d", Op->Stride);
LogMan::Msg::A("Unhandled StoreContextIndexed stride: %d", Op->Stride);
}
}
else {
@@ -279,7 +279,7 @@ DEF_OP(StoreContextIndexed) {
vmovq(AddressFrame(Op->Size * 8) [rax + index * Op->Stride], value);
break;
default:
LOGMAN_MSG_A("Unhandled StoreContextIndexed size: %d", size);
LogMan::Msg::A("Unhandled StoreContextIndexed size: %d", size);
}
break;
}
@@ -307,12 +307,12 @@ DEF_OP(StoreContextIndexed) {
movups(xword [STATE + rax], value);
break;
default:
LOGMAN_MSG_A("Unhandled StoreContextIndexed size: %d", size);
LogMan::Msg::A("Unhandled StoreContextIndexed size: %d", size);
}
break;
}
default:
LOGMAN_MSG_A("Unhandled StoreContextIndexed stride: %d", Op->Stride);
LogMan::Msg::A("Unhandled StoreContextIndexed stride: %d", Op->Stride);
}
}
}
@@ -340,7 +340,7 @@ DEF_OP(SpillRegister) {
mov(qword [rsp + SlotOffset], GetSrc<RA_64>(Op->Header.Args[0].ID()));
break;
}
default: LOGMAN_MSG_A("Unhandled SpillRegister size: %d", OpSize);
default: LogMan::Msg::A("Unhandled SpillRegister size: %d", OpSize);
}
} else if (Op->Class == FEXCore::IR::FPRClass) {
switch (OpSize) {
@@ -356,10 +356,10 @@ DEF_OP(SpillRegister) {
movaps(xword [rsp + SlotOffset], GetSrc(Op->Header.Args[0].ID()));
break;
}
default: LOGMAN_MSG_A("Unhandled SpillRegister size: %d", OpSize);
default: LogMan::Msg::A("Unhandled SpillRegister size: %d", OpSize);
}
} else {
LOGMAN_MSG_A("Unhandled SpillRegister class: %d", Op->Class.Val);
LogMan::Msg::A("Unhandled SpillRegister class: %d", Op->Class.Val);
}
@@ -388,7 +388,7 @@ DEF_OP(FillRegister) {
mov(GetDst<RA_64>(Node), qword [rsp + SlotOffset]);
break;
}
default: LOGMAN_MSG_A("Unhandled FillRegister size: %d", OpSize);
default: LogMan::Msg::A("Unhandled FillRegister size: %d", OpSize);
}
} else if (Op->Class == FEXCore::IR::FPRClass) {
switch (OpSize) {
@@ -404,10 +404,10 @@ DEF_OP(FillRegister) {
movaps(GetDst(Node), xword [rsp + SlotOffset]);
break;
}
default: LOGMAN_MSG_A("Unhandled FillRegister size: %d", OpSize);
default: LogMan::Msg::A("Unhandled FillRegister size: %d", OpSize);
}
} else {
LOGMAN_MSG_A("Unhandled FillRegister class: %d", Op->Class.Val);
LogMan::Msg::A("Unhandled FillRegister class: %d", Op->Class.Val);
}
}
@@ -425,16 +425,16 @@ DEF_OP(StoreFlag) {
mov(byte [STATE + (offsetof(FEXCore::Core::CPUState, flags[0]) + Op->Flag)], al);
}
Xbyak::RegExp X86JITCore::GenerateModRM(Xbyak::Reg Base, IR::OrderedNodeWrapper Offset, IR::MemOffsetType OffsetType, uint8_t OffsetScale) const {
Xbyak::RegExp X86JITCore::GenerateModRM(Xbyak::Reg Base, IR::OrderedNodeWrapper Offset, IR::MemOffsetType OffsetType, uint8_t OffsetScale) {
if (Offset.IsInvalid()) {
return Base;
} else {
if (OffsetScale != 1 && OffsetScale != 2 && OffsetScale != 4 && OffsetScale != 8) {
LOGMAN_MSG_A("Unhandled GenerateModRM OffsetScale: %d", OffsetScale);
LogMan::Msg::A("Unhandled GenerateModRM OffsetScale: %d", OffsetScale);
}
if (OffsetType != IR::MEM_OFFSET_SXTX) {
LOGMAN_MSG_A("Unhandled GenerateModRM OffsetType: %d", OffsetType.Val);
LogMan::Msg::A("Unhandled GenerateModRM OffsetType: %d", OffsetType.Val);
}
uint64_t Const;
@@ -475,7 +475,7 @@ DEF_OP(LoadMem) {
mov(Dst, qword [MemPtr]);
}
break;
default: LOGMAN_MSG_A("Unhandled LoadMem size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled LoadMem size: %d", Op->Size);
}
}
else
@@ -511,7 +511,7 @@ DEF_OP(LoadMem) {
}
}
break;
default: LOGMAN_MSG_A("Unhandled LoadMem size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled LoadMem size: %d", Op->Size);
}
}
}
@@ -537,7 +537,7 @@ DEF_OP(StoreMem) {
case 8:
mov(qword [MemPtr], GetSrc<RA_64>(Op->Header.Args[1].ID()));
break;
default: LOGMAN_MSG_A("Unhandled StoreMem size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled StoreMem size: %d", Op->Size);
}
}
else {
@@ -560,25 +560,17 @@ DEF_OP(StoreMem) {
else
movups(xword [MemPtr], GetSrc(Op->Header.Args[1].ID()));
break;
default: LOGMAN_MSG_A("Unhandled StoreMem size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled StoreMem size: %d", Op->Size);
}
}
}
DEF_OP(VLoadMemElement) {
LOGMAN_MSG_A("Unimplemented");
LogMan::Msg::A("Unimplemented");
}
DEF_OP(VStoreMemElement) {
LOGMAN_MSG_A("Unimplemented");
}
DEF_OP(CacheLineClear) {
auto Op = IROp->C<IR::IROp_CacheLineClear>();
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Addr.ID());
clflush(ptr [MemReg]);
LogMan::Msg::A("Unimplemented");
}
#undef DEF_OP
@@ -600,7 +592,6 @@ void X86JITCore::RegisterMemoryHandlers() {
REGISTER_OP(STOREMEMTSO, StoreMem);
REGISTER_OP(VLOADMEMELEMENT, VLoadMemElement);
REGISTER_OP(VSTOREMEMELEMENT, VStoreMemElement);
REGISTER_OP(CACHELINECLEAR, CacheLineClear);
#undef REGISTER_OP
}
}
+18 -29
View File
@@ -12,10 +12,6 @@ static void PrintValue(uint64_t Value) {
LogMan::Msg::D("Value: 0x%lx", Value);
}
static void PrintVectorValue(uint64_t Value, uint64_t ValueUpper) {
LogMan::Msg::D("Value: 0x%016lx'%016lx", ValueUpper, Value);
}
#define DEF_OP(x) void X86JITCore::Op_##x(FEXCore::IR::IROp_Header *IROp, uint32_t Node)
DEF_OP(Fence) {
@@ -30,7 +26,7 @@ DEF_OP(Fence) {
case IR::Fence_Store.Val:
sfence();
break;
default: LOGMAN_MSG_A("Unknown Fence: %d", Op->Fence); break;
default: LogMan::Msg::A("Unknown Fence: %d", Op->Fence); break;
}
}
@@ -41,22 +37,13 @@ DEF_OP(Break) {
case 5: // Guest ud2
ud2();
break;
case 1: // Int <imm8>
ud2();
break;
case 2: // overflow
ud2();
break;
case 3: // int 1
ud2();
break;
case 4: { // HLT
// Time to quit
// Set our stack to the starting stack location
mov(rsp, qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, ReturningStackLocation)]);
// Now we need to jump to the thread stop handler
mov(TMP1, ThreadSharedData.Dispatcher->ThreadStopHandlerAddress);
mov(TMP1, Dispatcher->ThreadStopHandlerAddress);
jmp(TMP1);
break;
}
@@ -69,7 +56,7 @@ DEF_OP(Break) {
}
// This jump target needs to be a constant offset here
mov(TMP1, ThreadSharedData.Dispatcher->ThreadPauseHandlerAddress);
mov(TMP1, Dispatcher->ThreadPauseHandlerAddress);
jmp(TMP1);
}
else {
@@ -78,12 +65,12 @@ DEF_OP(Break) {
mov(rsp, qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, ReturningStackLocation)]);
// Now we need to jump to the thread stop handler
mov(TMP1, ThreadSharedData.Dispatcher->ThreadStopHandlerAddress);
mov(TMP1, Dispatcher->ThreadStopHandlerAddress);
jmp(TMP1);
}
break;
}
default: LOGMAN_MSG_A("Unknown Break reason: %d", Op->Reason);
default: LogMan::Msg::A("Unknown Break reason: %d", Op->Reason);
}
}
@@ -123,22 +110,24 @@ DEF_OP(SetRoundingMode) {
DEF_OP(Print) {
auto Op = IROp->C<IR::IROp_Print>();
PushRegs();
if (IsGPR(Op->Header.Args[0].ID())) {
mov (rdi, GetSrc<RA_64>(Op->Header.Args[0].ID()));
for (auto &Reg : RA64)
push(Reg);
mov(rax, reinterpret_cast<uintptr_t>(PrintValue));
}
else {
pextrq(rdi, GetSrc(Op->Header.Args[0].ID()), 0);
pextrq(rsi, GetSrc(Op->Header.Args[0].ID()), 1);
auto NumPush = RA64.size();
if (NumPush & 1)
sub(rsp, 8); // Align
mov(rax, reinterpret_cast<uintptr_t>(PrintVectorValue));
}
mov (rdi, GetSrc<RA_64>(Op->Header.Args[0].ID()));
mov(rax, reinterpret_cast<uintptr_t>(PrintValue));
call(rax);
PopRegs();
if (NumPush & 1)
add(rsp, 8); // Align
for (uint32_t i = RA64.size(); i > 0; --i)
pop(RA64[i - 1]);
}
#undef DEF_OP
@@ -25,7 +25,7 @@ DEF_OP(ExtractElementPair) {
mov (GetDst<RA_64>(Node), Regs[Op->Element]);
break;
}
default: LOGMAN_MSG_A("Unknown Size"); break;
default: LogMan::Msg::A("Unknown Size"); break;
}
}
@@ -51,7 +51,7 @@ DEF_OP(CreateElementPair) {
RegTmp = rax;
break;
}
default: LOGMAN_MSG_A("Unknown Size"); break;
default: LogMan::Msg::A("Unknown Size"); break;
}
if (Dst.first != RegSecond) {
File diff suppressed because it is too large. Load diff
+8 -10
View File
@@ -8,8 +8,6 @@ $end_info$
#include "Interface/Context/Context.h"
#include "Interface/Core/Core.h"
#include "Interface/Core/LookupCache.h"
#include <FEXCore/Utils/Allocator.h>
#include <sys/mman.h>
namespace FEXCore {
@@ -28,27 +26,27 @@ LookupCache::LookupCache(FEXCore::Context::Context *CTX)
// Allocate a region of memory that we can use to back our block pointers
// We need one pointer per page of virtual memory
// At 64GB of virtual memory this will allocate 128MB of virtual memory space
PagePointer = reinterpret_cast<uintptr_t>(FEXCore::Allocator::mmap(nullptr, ctx->Config.VirtualMemSize / 4096 * 8, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0));
PagePointer = reinterpret_cast<uintptr_t>(mmap(nullptr, ctx->Config.VirtualMemSize / 4096 * 8, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0));
// Allocate our memory backing our pages
// We need 32KB per guest page (One pointer per byte)
// XXX: We can drop down to 16KB if we store 4byte offsets from the code base
// We currently limit to 128MB of real memory for caching for the total cache size.
// Can end up being inefficient if we compile a small number of blocks per page
PageMemory = reinterpret_cast<uintptr_t>(FEXCore::Allocator::mmap(nullptr, CODE_SIZE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0));
LOGMAN_THROW_A(PageMemory != -1ULL, "Failed to allocate page memory");
PageMemory = reinterpret_cast<uintptr_t>(mmap(nullptr, CODE_SIZE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0));
LogMan::Throw::A(PageMemory != -1ULL, "Failed to allocate page memory");
// L1 Cache
L1Pointer = reinterpret_cast<uintptr_t>(FEXCore::Allocator::mmap(nullptr, L1_SIZE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0));
LOGMAN_THROW_A(L1Pointer != -1ULL, "Failed to allocate L1Pointer");
L1Pointer = reinterpret_cast<uintptr_t>(mmap(nullptr, L1_SIZE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0));
LogMan::Throw::A(L1Pointer != -1ULL, "Failed to allocate L1Pointer");
VirtualMemSize = ctx->Config.VirtualMemSize;
}
LookupCache::~LookupCache() {
FEXCore::Allocator::munmap(reinterpret_cast<void*>(PagePointer), ctx->Config.VirtualMemSize / 4096 * 8);
FEXCore::Allocator::munmap(reinterpret_cast<void*>(PageMemory), CODE_SIZE);
FEXCore::Allocator::munmap(reinterpret_cast<void*>(L1Pointer), L1_SIZE);
munmap(reinterpret_cast<void*>(PagePointer), ctx->Config.VirtualMemSize / 4096 * 8);
munmap(reinterpret_cast<void*>(PageMemory), CODE_SIZE);
munmap(reinterpret_cast<void*>(L1Pointer), L1_SIZE);
}
void LookupCache::HintUsedRange(uint64_t Address, uint64_t Size) {
+8 -14
View File
@@ -38,21 +38,14 @@ public:
std::map<uint64_t, std::vector<uint64_t>> CodePages;
void AddBlockMapping(uint64_t Address, void *HostCode, uint64_t Start, uint64_t Length) {
#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
auto InsertPoint =
#endif
BlockList.emplace(Address, (uintptr_t)HostCode);
LOGMAN_THROW_A(InsertPoint.second == true, "Dupplicate block mapping added");
auto InsertPoint = BlockList.emplace(Address, (uintptr_t)HostCode);
LogMan::Throw::A(InsertPoint.second == true, "Dupplicate block mapping added");
for (auto CurrentPage = Start >> 12, EndPage = (Start + Length) >> 12; CurrentPage <= EndPage; CurrentPage++) {
CodePages[CurrentPage].push_back(Address);
}
// There is no need to update L1 or L2, they will get updated on first lookup
// However, adding to L1 here increases performance
auto &L1Entry = reinterpret_cast<LookupCacheEntry*>(L1Pointer)[Address & L1_ENTRIES_MASK];
L1Entry.GuestCode = Address;
L1Entry.HostCode = (uintptr_t)HostCode;
// no need to update L1 or L2, they will get updated on first lookup
}
void Erase(uint64_t Address) {
@@ -101,8 +94,8 @@ public:
void HintUsedRange(uint64_t Address, uint64_t Size);
uintptr_t GetL1Pointer() const { return L1Pointer; }
uintptr_t GetPagePointer() const { return PagePointer; }
uintptr_t GetL1Pointer() { return L1Pointer; }
uintptr_t GetPagePointer() { return PagePointer; }
uintptr_t GetVirtualMemorySize() const { return VirtualMemSize; }
constexpr static size_t L1_ENTRIES = 1 * 1024 * 1024; // Must be a power of 2
@@ -112,8 +105,9 @@ private:
void CacheBlockMapping(uint64_t Address, uintptr_t HostCode) {
// Do L1
auto &L1Entry = reinterpret_cast<LookupCacheEntry*>(L1Pointer)[Address & L1_ENTRIES_MASK];
L1Entry.GuestCode = Address;
L1Entry.HostCode = HostCode;
if (L1Entry.GuestCode == Address) {
L1Entry.GuestCode = L1Entry.HostCode = 0;
}
// Do ful map
auto FullAddress = Address;
File diff suppressed because it is too large. Load diff
+33 -39
View File
@@ -49,7 +49,7 @@ public:
OrderedNode* GetNewJumpBlock(uint64_t RIP) {
auto it = JumpTargets.find(RIP);
LOGMAN_THROW_A(it != JumpTargets.end(), "Couldn't find block generated for 0x%lx", RIP);
LogMan::Throw::A(it != JumpTargets.end(), "Couldn't find block generated for 0x%lx", RIP);
return it->second.BlockEntry;
}
@@ -59,7 +59,7 @@ public:
it->second.HaveEmitted = true;
if (CurrentCodeBlock->Wrapped(DualListData.ListBegin()).ID() == it->second.BlockEntry->Wrapped(DualListData.ListBegin()).ID()) return;
if (CurrentCodeBlock->Wrapped(ListData.Begin()).ID() == it->second.BlockEntry->Wrapped(ListData.Begin()).ID()) return;
// We have hit a RIP that is a jump target
// Thus we need to end up in a new block
@@ -81,14 +81,14 @@ public:
// rdi, 0x8
// cmp qword [rdi-8], 0
// jne .label
if (LastOp && !BlockSetRIP) {
if (!BlockSetRIP) {
auto it = JumpTargets.find(NextRIP);
if (it == JumpTargets.end()) {
if (it == JumpTargets.end() && LastOp) {
const uint8_t GPRSize = CTX->GetGPRSize();
uint8_t GPRSize = CTX->Config.Is64BitMode ? 8 : 4;
// If we don't have a jump target to a new block then we have to leave
// Set the RIP to the next instruction and leave
auto RelocatedNextRIP = _EntrypointOffset(NextRIP - Entry, GPRSize);
auto RelocatedNextRIP = _EntrypointOffset(NextRIP - Current_Header->Entry, GPRSize);
_ExitFunction(RelocatedNextRIP);
}
else if (it != JumpTargets.end()) {
@@ -104,8 +104,7 @@ public:
OpDispatchBuilder(FEXCore::Context::Context *ctx);
void ResetWorkingList();
void ResetDecodeFailure() { DecodeFailure = false; }
bool HadDecodeFailure() const { return DecodeFailure; }
bool HadDecodeFailure() { return DecodeFailure; }
void BeginFunction(uint64_t RIP, std::vector<FEXCore::Frontend::Decoder::DecodedBlocks> const *Blocks);
void Finalize();
@@ -260,6 +259,12 @@ public:
template<size_t ElementSize>
void PSUBQOp(OpcodeArgs);
template<size_t ElementSize>
void PMINUOp(OpcodeArgs);
template<size_t ElementSize>
void PMAXUOp(OpcodeArgs);
void PMINSWOp(OpcodeArgs);
void PMAXSWOp(OpcodeArgs);
template<size_t ElementSize>
void MOVMSKOp(OpcodeArgs);
void MOVMSKOpOne(OpcodeArgs);
template<size_t ElementSize>
@@ -269,6 +274,10 @@ public:
void PSHUFBOp(OpcodeArgs);
template<size_t ElementSize, bool HalfSize, bool Low>
void PSHUFDOp(OpcodeArgs);
template<size_t ElementSize>
void PCMPEQOp(OpcodeArgs);
template<size_t ElementSize>
void PCMPGTOp(OpcodeArgs);
void MOVDOp(OpcodeArgs);
template<size_t ElementSize, bool Scalar, uint32_t SrcIndex>
void PSRLDOp(OpcodeArgs);
@@ -287,21 +296,21 @@ public:
template<size_t ElementSize>
void PAVGOp(OpcodeArgs);
void MOVDDUPOp(OpcodeArgs);
template<size_t DstElementSize>
template<size_t DstElementSize, bool Signed>
void CVTGPR_To_FPR(OpcodeArgs);
template<size_t SrcElementSize, bool HostRoundingMode>
template<size_t SrcElementSize, bool Signed, bool HostRoundingMode>
void CVTFPR_To_GPR(OpcodeArgs);
template<size_t SrcElementSize, bool Widen>
template<size_t SrcElementSize, bool Signed, bool Widen>
void Vector_CVT_Int_To_Float(OpcodeArgs);
template<size_t DstElementSize, size_t SrcElementSize>
void Scalar_CVT_Float_To_Float(OpcodeArgs);
template<size_t DstElementSize, size_t SrcElementSize>
void Vector_CVT_Float_To_Float(OpcodeArgs);
template<size_t SrcElementSize, bool Narrow, bool HostRoundingMode>
template<size_t SrcElementSize, bool Signed, bool Narrow, bool HostRoundingMode>
void Vector_CVT_Float_To_Int(OpcodeArgs);
template<size_t SrcElementSize, bool Signed, bool Widen>
void MMX_To_XMM_Vector_CVT_Int_To_Float(OpcodeArgs);
template<size_t SrcElementSize, bool Narrow, bool HostRoundingMode>
template<size_t SrcElementSize, bool Signed, bool Narrow, bool HostRoundingMode>
void XMM_To_MMX_Vector_CVT_Float_To_Int(OpcodeArgs);
void MASKMOVOp(OpcodeArgs);
void MOVBetweenGPR_FPR(OpcodeArgs);
@@ -315,13 +324,17 @@ public:
void ANDNOp(OpcodeArgs);
template<size_t ElementSize>
void PINSROp(OpcodeArgs);
void InsertPSOp(OpcodeArgs);
template<size_t ElementSize>
void PExtrOp(OpcodeArgs);
template<size_t ElementSize, bool Signed>
void PMULOp(OpcodeArgs);
template<size_t ElementSize>
void PSIGN(OpcodeArgs);
template<size_t ElementSize>
void PABS(OpcodeArgs);
// X87 Ops
template<size_t width>
void FLD(OpcodeArgs);
@@ -445,8 +458,6 @@ public:
template<uint8_t FenceType>
void FenceOp(OpcodeArgs);
void StoreFenceOrCLFlush(OpcodeArgs);
void PSADBW(OpcodeArgs);
void AESImcOp(OpcodeArgs);
@@ -456,23 +467,6 @@ public:
void AESDecLastOp(OpcodeArgs);
void AESKeyGenAssist(OpcodeArgs);
template<size_t ElementSize, size_t DstElementSize, bool Signed>
void ExtendVectorElements(OpcodeArgs);
template<size_t ElementSize, bool Scalar>
void VectorRound(OpcodeArgs);
template<size_t ElementSize>
void VectorBlend(OpcodeArgs);
template<size_t ElementSize>
void VectorVariableBlend(OpcodeArgs);
void PTestOp(OpcodeArgs);
void PHMINPOSUWOp(OpcodeArgs);
template<size_t ElementSize>
void DPPOp(OpcodeArgs);
void MPSADBWOp(OpcodeArgs);
void UnimplementedOp(OpcodeArgs);
#undef OpcodeArgs
@@ -497,8 +491,8 @@ private:
void StoreResult(FEXCore::IR::RegisterClassType Class, FEXCore::X86Tables::DecodedOp Op, FEXCore::X86Tables::DecodedOperand const& Operand, OrderedNode *const Src, int8_t Align);
void StoreResult(FEXCore::IR::RegisterClassType Class, FEXCore::X86Tables::DecodedOp Op, OrderedNode *const Src, int8_t Align);
uint8_t GetDstSize(FEXCore::X86Tables::DecodedOp Op) const;
uint8_t GetSrcSize(FEXCore::X86Tables::DecodedOp Op) const;
uint8_t GetDstSize(FEXCore::X86Tables::DecodedOp Op);
uint8_t GetSrcSize(FEXCore::X86Tables::DecodedOp Op);
template<unsigned BitOffset>
void SetRFLAG(OrderedNode *Value);
@@ -528,12 +522,12 @@ private:
OrderedNode * GetX87Top();
void SetX87Top(OrderedNode *Value);
bool DestIsLockedMem(FEXCore::X86Tables::DecodedOp Op) const {
return DestIsMem(Op) && (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_LOCK) != 0;
bool DestIsLockedMem(FEXCore::X86Tables::DecodedOp Op) {
return Op->Dest.TypeNone.Type !=FEXCore::X86Tables::DecodedOperand::TYPE_GPR && (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_LOCK);
}
bool DestIsMem(FEXCore::X86Tables::DecodedOp Op) const {
return !Op->Dest.IsGPR();
bool DestIsMem(FEXCore::X86Tables::DecodedOp Op) {
return Op->Dest.TypeNone.Type !=FEXCore::X86Tables::DecodedOperand::TYPE_GPR;
}
void CreateJumpBlocks(std::vector<FEXCore::Frontend::Decoder::DecodedBlocks> const *Blocks);
+4 -5
View File
@@ -6,7 +6,6 @@ $end_info$
*/
#include "Interface/Core/X86HelperGen.h"
#include <FEXCore/Utils/Allocator.h>
#include <cstring>
#include <stdlib.h>
@@ -32,7 +31,7 @@ X86GeneratedCode::X86GeneratedCode() {
}
X86GeneratedCode::~X86GeneratedCode() {
FEXCore::Allocator::munmap(CodePtr, CODE_SIZE);
munmap(CodePtr, CODE_SIZE);
}
void* X86GeneratedCode::AllocateGuestCodeSpace(size_t Size) {
@@ -40,7 +39,7 @@ void* X86GeneratedCode::AllocateGuestCodeSpace(size_t Size) {
if (Is64BitMode()) {
// 64bit mode can have its sigret handler anywhere
return FEXCore::Allocator::mmap(nullptr, Size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
return mmap(nullptr, Size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
}
// First 64bit page
@@ -50,14 +49,14 @@ void* X86GeneratedCode::AllocateGuestCodeSpace(size_t Size) {
// We need to have the sigret handler in the lower 32bits of memory space
// Scan top down and try to allocate a location
for (size_t Location = 0xFFFF'E000; Location != 0x0; Location -= 0x1000) {
void *Ptr = FEXCore::Allocator::mmap(reinterpret_cast<void*>(Location), Size, PROT_READ | PROT_WRITE, MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
void *Ptr = mmap(reinterpret_cast<void*>(Location), Size, PROT_READ | PROT_WRITE, MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (Ptr != MAP_FAILED &&
reinterpret_cast<uintptr_t>(Ptr) >= LOCATION_MAX) {
// Failed to map in the lower 32bits
// Try again
// Can happen in the case that host kernel ignores MAP_FIXED_NOREPLACE
FEXCore::Allocator::munmap(Ptr, Size);
munmap(Ptr, Size);
continue;
}
@@ -41,10 +41,10 @@ void InitializeH0F38Tables() {
{OPD(PF_38_NONE, 0x0B), 1, X86InstInfo{"PMULHRSW", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0, nullptr}},
{OPD(PF_38_66, 0x0B), 1, X86InstInfo{"PMULHRSW", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x10), 1, X86InstInfo{"PBLENDVB", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x14), 1, X86InstInfo{"BLENDVPS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x15), 1, X86InstInfo{"BLENDVPD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x17), 1, X86InstInfo{"PTEST", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x10), 1, X86InstInfo{"PBLENDVB", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x14), 1, X86InstInfo{"BLENDVPS", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x15), 1, X86InstInfo{"BLENDVPD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x17), 1, X86InstInfo{"PTEST", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_NONE, 0x1C), 1, X86InstInfo{"PABSB", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0, nullptr}},
{OPD(PF_38_66, 0x1C), 1, X86InstInfo{"PABSB", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_NONE, 0x1D), 1, X86InstInfo{"PABSW", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0, nullptr}},
@@ -52,34 +52,34 @@ void InitializeH0F38Tables() {
{OPD(PF_38_NONE, 0x1E), 1, X86InstInfo{"PABSD", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0, nullptr}},
{OPD(PF_38_66, 0x1E), 1, X86InstInfo{"PABSD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x20), 1, X86InstInfo{"PMOVSXBW", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x21), 1, X86InstInfo{"PMOVSXBD", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x22), 1, X86InstInfo{"PMOVSXBQ", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_16BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x23), 1, X86InstInfo{"PMOVSXWD", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x24), 1, X86InstInfo{"PMOVSXWQ", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x25), 1, X86InstInfo{"PMOVSXDQ", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x28), 1, X86InstInfo{"PMULDQ", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x29), 1, X86InstInfo{"PCMPEQQ", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x20), 1, X86InstInfo{"PMOVSXBW", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x21), 1, X86InstInfo{"PMOVSXBD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x22), 1, X86InstInfo{"PMOVSXBQ", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x23), 1, X86InstInfo{"PMOVSXWD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x24), 1, X86InstInfo{"PMOVSXWQ", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x25), 1, X86InstInfo{"PMOVSXDQ", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x28), 1, X86InstInfo{"PMULDQ", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x29), 1, X86InstInfo{"PCMPEQQ", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x2A), 1, X86InstInfo{"MOVNTDQA", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x2B), 1, X86InstInfo{"PACKUSDW", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x2B), 1, X86InstInfo{"PACKUSDW", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x30), 1, X86InstInfo{"PMOVZXBW", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x31), 1, X86InstInfo{"PMOVZXBD", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x32), 1, X86InstInfo{"PMOVZXBQ", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_16BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x33), 1, X86InstInfo{"PMOVZXWD", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x34), 1, X86InstInfo{"PMOVZXWQ", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x35), 1, X86InstInfo{"PMOVZXDQ", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x38), 1, X86InstInfo{"PMINSB", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x39), 1, X86InstInfo{"PMINSD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x3A), 1, X86InstInfo{"PMINUW", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x3B), 1, X86InstInfo{"PMINUD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x3C), 1, X86InstInfo{"PMAXSB", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x3D), 1, X86InstInfo{"PMAXSD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x3E), 1, X86InstInfo{"PMAXUW", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x3F), 1, X86InstInfo{"PMAXUD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x30), 1, X86InstInfo{"PMOVZXBW", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x31), 1, X86InstInfo{"PMOVZXBD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x32), 1, X86InstInfo{"PMOVZXBQ", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x33), 1, X86InstInfo{"PMOVZXWD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x34), 1, X86InstInfo{"PMOVZXWQ", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x35), 1, X86InstInfo{"PMOVZXDQ", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x38), 1, X86InstInfo{"PMINSB", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x39), 1, X86InstInfo{"PMINSD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x3A), 1, X86InstInfo{"PMINUW", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x3B), 1, X86InstInfo{"PMINUD", TYPE_INST, FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x3C), 1, X86InstInfo{"PMAXSB", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x3D), 1, X86InstInfo{"PMAXSD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x3E), 1, X86InstInfo{"PMAXUW", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x3F), 1, X86InstInfo{"PMAXUD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x40), 1, X86InstInfo{"PMULLD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x41), 1, X86InstInfo{"PHMINPOSUW", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x40), 1, X86InstInfo{"PMULLD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x41), 1, X86InstInfo{"PHMINPOSUW", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0xDB), 1, X86InstInfo{"AESIMC", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0xDC), 1, X86InstInfo{"AESENC", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
@@ -16,26 +16,26 @@ void InitializeH0F3ATables(Context::OperatingMode Mode) {
const U16U8InfoStruct H0F3ATable[] = {
{OPD(0, PF_3A_NONE, 0x0F), 1, X86InstInfo{"PALIGNR", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 1, nullptr}},
{OPD(0, PF_3A_66, 0x08), 1, X86InstInfo{"ROUNDPS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(0, PF_3A_66, 0x09), 1, X86InstInfo{"ROUNDPD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(0, PF_3A_66, 0x0A), 1, X86InstInfo{"ROUNDSS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(0, PF_3A_66, 0x0B), 1, X86InstInfo{"ROUNDSD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(0, PF_3A_66, 0x0C), 1, X86InstInfo{"BLENDPS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(0, PF_3A_66, 0x0D), 1, X86InstInfo{"BLENDPD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(0, PF_3A_66, 0x0E), 1, X86InstInfo{"PBLENDW", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(0, PF_3A_66, 0x08), 1, X86InstInfo{"ROUNDPS", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(0, PF_3A_66, 0x09), 1, X86InstInfo{"ROUNDPD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(0, PF_3A_66, 0x0A), 1, X86InstInfo{"ROUNDSS", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(0, PF_3A_66, 0x0B), 1, X86InstInfo{"ROUNDSD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(0, PF_3A_66, 0x0C), 1, X86InstInfo{"BLENDPS", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(0, PF_3A_66, 0x0D), 1, X86InstInfo{"BLENDPD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(0, PF_3A_66, 0x0E), 1, X86InstInfo{"PBLENDW", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(0, PF_3A_66, 0x0F), 1, X86InstInfo{"PALIGNR", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(0, PF_3A_66, 0x14), 1, X86InstInfo{"PEXTRB", TYPE_INST, GenFlagsSizes(SIZE_8BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_DST_GPR | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(0, PF_3A_66, 0x15), 1, X86InstInfo{"PEXTRW", TYPE_INST, GenFlagsSizes(SIZE_16BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_DST_GPR | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(0, PF_3A_66, 0x16), 1, X86InstInfo{"PEXTRD", TYPE_INST, GenFlagsSizes(SIZE_32BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_DST_GPR | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(0, PF_3A_66, 0x17), 1, X86InstInfo{"EXTRACTPS", TYPE_INST, GenFlagsSizes(SIZE_32BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_DST_GPR | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(0, PF_3A_66, 0x17), 1, X86InstInfo{"EXTRACTPS", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(0, PF_3A_66, 0x20), 1, X86InstInfo{"PINSRB", TYPE_INST, GenFlagsDstSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_SRC_GPR, 1, nullptr}},
{OPD(0, PF_3A_66, 0x21), 1, X86InstInfo{"INSERTPS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(0, PF_3A_66, 0x20), 1, X86InstInfo{"PINSRB", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_8BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_SRC_GPR, 1, nullptr}},
{OPD(0, PF_3A_66, 0x21), 1, X86InstInfo{"INSERTPS", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(0, PF_3A_66, 0x22), 1, X86InstInfo{"PINSRD", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_SRC_GPR, 1, nullptr}},
{OPD(0, PF_3A_66, 0x40), 1, X86InstInfo{"DPPS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(0, PF_3A_66, 0x41), 1, X86InstInfo{"DPPD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(0, PF_3A_66, 0x42), 1, X86InstInfo{"MPSADBW", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(0, PF_3A_66, 0x40), 1, X86InstInfo{"DPPS", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(0, PF_3A_66, 0x41), 1, X86InstInfo{"DPPD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(0, PF_3A_66, 0x42), 1, X86InstInfo{"MPSADBW", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(0, PF_3A_66, 0x44), 1, X86InstInfo{"PCLMULQDQ", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(0, PF_3A_66, 0x60), 1, X86InstInfo{"PCMPESTRM", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
@@ -33,7 +33,7 @@ static inline void GenerateTable(X86InstInfo *FinalTable, U8U8InfoStruct const *
auto OpNum = Op.first;
X86InstInfo const &Info = Op.Info;
for (uint32_t i = 0; i < Op.second; ++i) {
LOGMAN_THROW_A(FinalTable[OpNum + i].Type == TYPE_UNKNOWN, "Duplicate Entry %s->%s", FinalTable[OpNum + i].Name, Info.Name);
LogMan::Throw::A(FinalTable[OpNum + i].Type == TYPE_UNKNOWN, "Duplicate Entry %s->%s", FinalTable[OpNum + i].Name, Info.Name);
FinalTable[OpNum + i] = Info;
#ifndef NDEBUG
++Total;
@@ -50,7 +50,7 @@ static inline void GenerateTable(X86InstInfo *FinalTable, U16U8InfoStruct const
auto OpNum = Op.first;
X86InstInfo const &Info = Op.Info;
for (uint32_t i = 0; i < Op.second; ++i) {
LOGMAN_THROW_A(FinalTable[OpNum + i].Type == TYPE_UNKNOWN, "Duplicate Entry %s->%s", FinalTable[OpNum + i].Name, Info.Name);
LogMan::Throw::A(FinalTable[OpNum + i].Type == TYPE_UNKNOWN, "Duplicate Entry %s->%s", FinalTable[OpNum + i].Name, Info.Name);
FinalTable[OpNum + i] = Info;
#ifndef NDEBUG
++Total;
@@ -67,7 +67,7 @@ static inline void GenerateTableWithCopy(X86InstInfo *FinalTable, U8U8InfoStruct
auto OpNum = Op.first;
X86InstInfo const &Info = Op.Info;
for (uint32_t i = 0; i < Op.second; ++i) {
LOGMAN_THROW_A(FinalTable[OpNum + i].Type == TYPE_UNKNOWN, "Duplicate Entry %s->%s", FinalTable[OpNum + i].Name, Info.Name);
LogMan::Throw::A(FinalTable[OpNum + i].Type == TYPE_UNKNOWN, "Duplicate Entry %s->%s", FinalTable[OpNum + i].Name, Info.Name);
if (Info.Type == TYPE_COPY_OTHER) {
FinalTable[OpNum + i] = OtherLocal[OpNum + i];
}
@@ -89,7 +89,7 @@ static inline void GenerateX87Table(X86InstInfo *FinalTable, U16U8InfoStruct con
auto OpNum = Op.first;
X86InstInfo const &Info = Op.Info;
for (uint32_t i = 0; i < Op.second; ++i) {
LOGMAN_THROW_A(FinalTable[OpNum + i].Type == TYPE_UNKNOWN, "Duplicate Entry %s->%s", FinalTable[OpNum + i].Name, Info.Name);
LogMan::Throw::A(FinalTable[OpNum + i].Type == TYPE_UNKNOWN, "Duplicate Entry %s->%s", FinalTable[OpNum + i].Name, Info.Name);
if ((OpNum & 0b11'000'000) == 0b11'000'000) {
// If the mod field is 0b11 then it is a regular op
FinalTable[OpNum + i] = Info;
@@ -97,7 +97,7 @@ static inline void GenerateX87Table(X86InstInfo *FinalTable, U16U8InfoStruct con
else {
// If the mod field is !0b11 then this instruction is duplicated through the whole mod [0b00, 0b10] range
// and the modrm.rm space because that is used part of the instruction encoding
LOGMAN_THROW_A((OpNum & 0b11'000'000) == 0, "Only support mod field of zero in this path");
LogMan::Throw::A((OpNum & 0b11'000'000) == 0, "Only support mod field of zero in this path");
for (uint16_t mod = 0b00'000'000; mod < 0b11'000'000; mod += 0b01'000'000) {
for (uint16_t rm = 0b000; rm < 0b1'000; ++rm) {
FinalTable[(OpNum | mod | rm) + i] = Info;
+101 -179
View File
@@ -60,12 +60,6 @@
"constexpr static uint8_t ROUND_MODE_TOWARDS_ZERO = 3",
"constexpr static uint8_t ROUND_MODE_FLUSH_TO_ZERO = 1 << 2",
"static constexpr FEXCore::IR::RoundType Round_Nearest {ROUND_MODE_NEAREST}",
"static constexpr FEXCore::IR::RoundType Round_Negative_Infinity {ROUND_MODE_NEGATIVE_INFINITY}",
"static constexpr FEXCore::IR::RoundType Round_Positive_Infinity {ROUND_MODE_POSITIVE_INFINITY}",
"static constexpr FEXCore::IR::RoundType Round_Towards_Zero {ROUND_MODE_TOWARDS_ZERO} /* Truncate */",
"static constexpr FEXCore::IR::RoundType Round_Host {ROUND_MODE_TOWARDS_ZERO + 1}",
"constexpr static FEXCore::IR::MemOffsetType MEM_OFFSET_SXTX {0};",
"constexpr static FEXCore::IR::MemOffsetType MEM_OFFSET_UXTW {1};",
"constexpr static FEXCore::IR::MemOffsetType MEM_OFFSET_SXTW {2};"
@@ -85,6 +79,7 @@
"Blocks"
],
"Args": [
"uint64_t", "Entry",
"uint32_t", "BlockCount"
]
},
@@ -798,17 +793,6 @@
]
},
"CacheLineClear": {
"Desc": ["Does a 64 byte cacheline clear at the address specified"
],
"HasSideEffects": true,
"OpClass": "Memory",
"SSAArgs": "1",
"SSANames": [
"Addr"
]
},
"Add": {
"Desc": [ "Integer Add",
"Will truncate to 64 or 32bits"
@@ -1483,6 +1467,24 @@
]
},
"Float_ToGPR_U": {
"Desc": ["Moves the scalar element to a GPR with conversion",
"Converts the 32bit or 64bit float to an unsigned integer",
"Rounding mode determined by host flag's rounding mode"
],
"OpClass": "ALU",
"HasDest": true,
"DestClass": "GPR",
"DestSize": "ElementSize",
"SSAArgs": "1",
"SSANames": [
"Scalar"
],
"Args": [
"uint8_t", "ElementSize"
]
},
"Float_ToGPR_S": {
"Desc": ["Moves the scalar element to a GPR with conversion",
"Converts the 32bit or 64bit float to an signed integer",
@@ -1491,16 +1493,30 @@
"OpClass": "ALU",
"HasDest": true,
"DestClass": "GPR",
"DestSize": "DestElementSize",
"DestSize": "ElementSize",
"SSAArgs": "1",
"SSANames": [
"Scalar"
],
"HelperArgs": [
"uint8_t", "DestElementSize"
"Args": [
"uint8_t", "ElementSize"
]
},
"Float_ToGPR_ZU": {
"Desc": ["Moves the scalar element to a GPR with conversion",
"Converts the 32bit or 64bit float to an unsigned integer rounding towards zero (Truncating)"
],
"OpClass": "ALU",
"HasDest": true,
"DestClass": "GPR",
"DestSize": "ElementSize",
"SSAArgs": "1",
"SSANames": [
"Scalar"
],
"Args": [
"uint8_t", "SrcElementSize"
"uint8_t", "ElementSize"
]
},
@@ -1511,16 +1527,13 @@
"OpClass": "ALU",
"HasDest": true,
"DestClass": "GPR",
"DestSize": "DestElementSize",
"DestSize": "ElementSize",
"SSAArgs": "1",
"SSANames": [
"Scalar"
],
"HelperArgs": [
"uint8_t", "DestElementSize"
],
"Args": [
"uint8_t", "SrcElementSize"
"uint8_t", "ElementSize"
]
},
@@ -1550,6 +1563,7 @@
"Depending on backend, may only support GPR printing"
],
"OpClass": "Misc",
"DestSize": "GetOpSize(ssa0)",
"SSAArgs": "1",
"SSANames": [
"Value"
@@ -1641,23 +1655,6 @@
]
},
"VBic": {
"OpClass": "Vector",
"HasDest": true,
"DestClass": "FPR",
"DestSize": "RegisterSize",
"NumElements": "RegisterSize / ElementSize",
"SSAArgs": "2",
"SSANames": [
"Vector1",
"Vector2"
],
"HelperArgs": [
"uint8_t", "RegisterSize",
"uint8_t", "ElementSize"
]
},
"VOr": {
"OpClass": "Vector",
"HasDest": true,
@@ -1803,8 +1800,8 @@
"NumElements": "RegisterSize / ElementSize",
"SSAArgs": "2",
"SSANames": [
"VectorLower",
"VectorUpper"
"Vector1",
"Vector2"
],
"HelperArgs": [
"uint8_t", "RegisterSize",
@@ -1831,25 +1828,6 @@
]
},
"VUMinV": {
"OpClass": "Vector",
"Desc": ["Does a horizontal vector unsigned minimum of elements across the source vector",
"Result is a zero extended scalar"
],
"HasDest": true,
"DestClass": "FPR",
"DestSize": "RegisterSize",
"NumElements": "RegisterSize / ElementSize",
"SSAArgs": "1",
"SSANames": [
"Vector"
],
"HelperArgs": [
"uint8_t", "RegisterSize",
"uint8_t", "ElementSize"
]
},
"VURAvg": {
"OpClass": "Vector",
"Desc": ["Does an unsigned rounded average", "dst_elem = (src1_elem + src2_elem + 1) >> 1"],
@@ -1886,24 +1864,6 @@
]
},
"VPopcount": {
"OpClass": "Vector",
"Desc": ["Does a popcount for each element of the register"
],
"HasDest": true,
"DestClass": "FPR",
"DestSize": "RegisterSize",
"NumElements": "RegisterSize / ElementSize",
"SSAArgs": "1",
"SSANames": [
"Vector"
],
"HelperArgs": [
"uint8_t", "RegisterSize",
"uint8_t", "ElementSize"
]
},
"VFAdd": {
"OpClass": "Vector",
"HasDest": true,
@@ -1930,8 +1890,8 @@
"NumElements": "RegisterSize / ElementSize",
"SSAArgs": "2",
"SSANames": [
"VectorLow",
"VectorHigh"
"Vector1",
"Vector2"
],
"HelperArgs": [
"uint8_t", "RegisterSize",
@@ -2223,40 +2183,6 @@
]
},
"VUnZip": {
"OpClass": "Vector",
"HasDest": true,
"DestClass": "FPR",
"DestSize": "RegisterSize",
"NumElements": "RegisterSize / ElementSize",
"SSAArgs": "2",
"SSANames": [
"Lower",
"Upper"
],
"HelperArgs": [
"uint8_t", "RegisterSize",
"uint8_t", "ElementSize"
]
},
"VUnZip2": {
"OpClass": "Vector",
"HasDest": true,
"DestClass": "FPR",
"DestSize": "RegisterSize",
"NumElements": "RegisterSize / ElementSize",
"SSAArgs": "2",
"SSANames": [
"Lower",
"Upper"
],
"HelperArgs": [
"uint8_t", "RegisterSize",
"uint8_t", "ElementSize"
]
},
"VBSL": {
"Desc": ["Does a vector bitwise select.",
"If the bit in the field is 1 then the corresponding bit is pulled from VectorTrue",
@@ -2648,26 +2574,6 @@
]
},
"VDupElement": {
"Desc": ["Duplicates one element from the source register across the whole register"],
"OpClass": "Vector",
"HasDest": true,
"DestClass": "FPR",
"DestSize": "RegisterSize",
"NumElements": "RegisterSize / ElementSize",
"SSAArgs": "1",
"SSANames": [
"Vector"
],
"HelperArgs": [
"uint8_t", "RegisterSize",
"uint8_t", "ElementSize"
],
"Args": [
"uint8_t", "Index"
]
},
"VExtr": {
"Desc": ["Concats two vector registers together and extracts a full width register from the element index",
"Index is an element index. So it is offset by ElementSize argument",
@@ -3020,6 +2926,27 @@
]
},
"Float_FromGPR_U": {
"OpClass": "Conv",
"Desc": ["Scalar op: Converts unsigned GPR to Scalar float",
"Zeroes the upper bits of the vector register"
],
"HasDest": true,
"DestClass": "FPR",
"DestSize": "DstElementSize",
"NumElements": "1",
"SSAArgs": "1",
"SSANames": [
"GPR"
],
"HelperArgs": [
"uint8_t", "DstElementSize"
],
"Args": [
"uint8_t", "SrcElementSize"
]
},
"Float_FromGPR_S": {
"OpClass": "Conv",
"Desc": ["Scalar op: Converts signed GPR to Scalar float",
@@ -3096,6 +3023,25 @@
]
},
"Vector_FToU": {
"OpClass": "Conv",
"Desc": ["Vector op: Converts float to unsigned integer",
"Rounding mode determined by host rounding mode"
],
"HasDest": true,
"DestClass": "FPR",
"DestSize": "RegisterSize",
"NumElements": "RegisterSize / ElementSize",
"SSAArgs": "1",
"SSANames": [
"Vector"
],
"HelperArgs": [
"uint8_t", "RegisterSize",
"uint8_t", "ElementSize"
]
},
"Vector_FToS": {
"OpClass": "Conv",
"Desc": ["Vector op: Converts float to signed integer, rounding towards zero",
@@ -3115,6 +3061,23 @@
]
},
"Vector_FToZU": {
"OpClass": "Conv",
"Desc": "Vector op: Converts float to unsigned integer, rounding towards zero",
"HasDest": true,
"DestClass": "FPR",
"DestSize": "RegisterSize",
"NumElements": "RegisterSize / ElementSize",
"SSAArgs": "1",
"SSANames": [
"Vector"
],
"HelperArgs": [
"uint8_t", "RegisterSize",
"uint8_t", "ElementSize"
]
},
"Vector_FToZS": {
"OpClass": "Conv",
"Desc": "Vector op: Converts float to signed integer, rounding towards zero",
@@ -3152,28 +3115,6 @@
]
},
"Vector_FToI": {
"OpClass": "Conv",
"Desc": ["Vector op: Rounds float to integral",
"Rounding mode determined by argument"
],
"HasDest": true,
"DestClass": "FPR",
"DestSize": "RegisterSize",
"NumElements": "RegisterSize / ElementSize",
"SSAArgs": "1",
"SSANames": [
"Vector"
],
"HelperArgs": [
"uint8_t", "RegisterSize",
"uint8_t", "ElementSize"
],
"Args":[
"FEXCore::IR::RoundType", "Round"
]
},
"VUMul": {
"OpClass": "Vector",
"HasDest": true,
@@ -3281,25 +3222,6 @@
]
},
"VUABDL": {
"OpClass": "Vector",
"Desc": ["Unsigned Absolute Difference Long"
],
"HasDest": true,
"DestClass": "FPR",
"DestSize": "RegisterSize",
"NumElements": "RegisterSize / (ElementSize << 1)",
"SSAArgs": "2",
"SSANames": [
"Vector1",
"Vector2"
],
"HelperArgs": [
"uint8_t", "RegisterSize",
"uint8_t", "ElementSize"
]
},
"VTBL1": {
"Desc": ["Does a vector table lookup from one register in to the destination",
"Lookup is byte sized per byte element.",
+3 -13
View File
@@ -37,7 +37,7 @@ static void PrintArg(std::stringstream *out, [[maybe_unused]] IRListView const*
}
static void PrintArg(std::stringstream *out, [[maybe_unused]] IRListView const* IR, CondClassType Arg) {
static constexpr std::array<std::string_view, 22> CondNames = {
std::array<std::string, 22> CondNames = {
"EQ",
"NEQ",
"UGE",
@@ -66,7 +66,7 @@ static void PrintArg(std::stringstream *out, [[maybe_unused]] IRListView const*
}
static void PrintArg(std::stringstream *out, [[maybe_unused]] IRListView const* IR, MemOffsetType Arg) {
static constexpr std::array<std::string_view, 3> Names = {
std::array<std::string, 3> Names = {
"SXTX",
"UXTW",
"SXTW",
@@ -154,17 +154,6 @@ static void PrintArg(std::stringstream *out, [[maybe_unused]] IRListView const*
}
}
static void PrintArg(std::stringstream *out, [[maybe_unused]] IRListView const* IR, FEXCore::IR::RoundType Arg) {
switch (Arg) {
case FEXCore::IR::Round_Nearest: *out << "Nearest"; break;
case FEXCore::IR::Round_Negative_Infinity: *out << "-Inf"; break;
case FEXCore::IR::Round_Positive_Infinity: *out << "+Inf"; break;
case FEXCore::IR::Round_Towards_Zero: *out << "Towards Zero"; break;
case FEXCore::IR::Round_Host: *out << "Host"; break;
default: *out << "<Unknown Round Type>"; break;
}
}
void Dump(std::stringstream *out, IRListView const* IR, IR::RegisterAllocationData *RAData) {
auto HeaderOp = IR->GetHeader();
@@ -178,6 +167,7 @@ void Dump(std::stringstream *out, IRListView const* IR, IR::RegisterAllocationDa
++CurrentIndent;
AddIndent();
*out << "(%ssa0) " << "IRHeader ";
*out << "#0x" << std::hex << HeaderOp->Entry << ", ";
*out << "%ssa" << HeaderOp->Blocks.ID() << ", ";
*out << "#" << std::dec << HeaderOp->BlockCount << std::endl;
+17 -16
View File
@@ -9,17 +9,18 @@ $end_info$
namespace FEXCore::IR {
void IREmitter::ResetWorkingList() {
DualListData.Reset();
Data.Reset();
ListData.Reset();
CodeBlocks.clear();
CurrentWriteCursor = nullptr;
// This is necessary since we do "null" pointer checks
InvalidNode = reinterpret_cast<OrderedNode*>(DualListData.ListAllocate(sizeof(OrderedNode)));
InvalidNode = reinterpret_cast<OrderedNode*>(ListData.Allocate(sizeof(OrderedNode)));
memset(InvalidNode, 0, sizeof(OrderedNode));
CurrentCodeBlock = nullptr;
}
void IREmitter::ReplaceAllUsesWithRange(OrderedNode *Node, OrderedNode *NewNode, AllNodesIterator After, AllNodesIterator End) {
uintptr_t ListBegin = DualListData.ListBegin();
uintptr_t ListBegin = ListData.Begin();
auto NodeId = Node->Wrapped(ListBegin).ID();
while (After != End) {
@@ -44,8 +45,8 @@ void IREmitter::ReplaceAllUsesWithRange(OrderedNode *Node, OrderedNode *NewNode,
}
void IREmitter::ReplaceNodeArgument(OrderedNode *Node, uint8_t Arg, OrderedNode *NewArg) {
uintptr_t ListBegin = DualListData.ListBegin();
uintptr_t DataBegin = DualListData.DataBegin();
uintptr_t ListBegin = ListData.Begin();
uintptr_t DataBegin = Data.Begin();
FEXCore::IR::IROp_Header *IROp = Node->Op(DataBegin);
OrderedNodeWrapper OldArgWrapper = IROp->Args[Arg];
@@ -56,8 +57,8 @@ void IREmitter::ReplaceNodeArgument(OrderedNode *Node, uint8_t Arg, OrderedNode
}
void IREmitter::RemoveArgUses(OrderedNode *Node) {
uintptr_t ListBegin = DualListData.ListBegin();
uintptr_t DataBegin = DualListData.DataBegin();
uintptr_t ListBegin = ListData.Begin();
uintptr_t DataBegin = Data.Begin();
FEXCore::IR::IROp_Header *IROp = Node->Op(DataBegin);
@@ -71,7 +72,7 @@ void IREmitter::RemoveArgUses(OrderedNode *Node) {
void IREmitter::Remove(OrderedNode *Node) {
RemoveArgUses(Node);
Node->Unlink(DualListData.ListBegin());
Node->Unlink(ListData.Begin());
}
IREmitter::IRPair<IROp_CodeBlock> IREmitter::CreateNewCodeBlockAfter(OrderedNode* insertAfter) {
@@ -82,14 +83,14 @@ IREmitter::IRPair<IROp_CodeBlock> IREmitter::CreateNewCodeBlockAfter(OrderedNode
if (insertAfter) {
LinkCodeBlocks(insertAfter, CodeNode);
} else {
LOGMAN_THROW_A(CurrentCodeBlock != nullptr, "CurrentCodeBlock must not be null here");
LogMan::Throw::A(CurrentCodeBlock != nullptr, "CurrentCodeBlock must not be null here");
// Find last block
auto LastBlock = CurrentCodeBlock;
while (LastBlock->Header.Next.GetNode(DualListData.ListBegin()) != InvalidNode)
LastBlock = LastBlock->Header.Next.GetNode(DualListData.ListBegin());
while (LastBlock->Header.Next.GetNode(ListData.Begin()) != InvalidNode)
LastBlock = LastBlock->Header.Next.GetNode(ListData.Begin());
// Append it after the last block
LinkCodeBlocks(LastBlock, CodeNode);
}
@@ -101,12 +102,12 @@ IREmitter::IRPair<IROp_CodeBlock> IREmitter::CreateNewCodeBlockAfter(OrderedNode
void IREmitter::SetCurrentCodeBlock(OrderedNode *Node) {
CurrentCodeBlock = Node;
LOGMAN_THROW_A(Node->Op(DualListData.DataBegin())->Op == OP_CODEBLOCK, "Node wasn't codeblock. It was '%s'", std::string(IR::GetName(Node->Op(DualListData.DataBegin())->Op)).c_str());
SetWriteCursor(Node->Op(DualListData.DataBegin())->CW<IROp_CodeBlock>()->Begin.GetNode(DualListData.ListBegin()));
LogMan::Throw::A(Node->Op(Data.Begin())->Op == OP_CODEBLOCK, "Node wasn't codeblock. It was '%s'", std::string(IR::GetName(Node->Op(Data.Begin())->Op)).c_str());
SetWriteCursor(Node->Op(Data.Begin())->CW<IROp_CodeBlock>()->Begin.GetNode(ListData.Begin()));
}
void IREmitter::ReplaceWithConstant(OrderedNode *Node, uint64_t Value) {
auto Header = Node->Op(DualListData.DataBegin());
auto Header = Node->Op(Data.Begin());
if (IRSizes[Header->Op] >= sizeof(IROp_Constant)) {
// Unlink any arguments the node currently has
+117 -113
View File
@@ -66,8 +66,7 @@ std::string DecodeErrorToString(DecodeFailure Failure) {
case DecodeFailure::DECODE_INVALID_CONDFLAG: return "Invalid Conditional name";
case DecodeFailure::DECODE_INVALID_MEMOFFSETTYPE: return "Invalid Memory Offset Type";
case DecodeFailure::DECODE_INVALID_FENCETYPE: return "Invalid Fence Type";
}
return "Unknown Error";
};
}
std::unordered_map<std::string_view, FEXCore::IR::IROps> NameToOpMap;
@@ -75,22 +74,22 @@ std::unordered_map<std::string_view, FEXCore::IR::IROps> NameToOpMap;
class IRParser: public FEXCore::IR::IREmitter {
public:
template<typename Type>
std::pair<DecodeFailure, Type> DecodeValue(const std::string &Arg) {
std::pair<DecodeFailure, Type> DecodeValue(std::string &Arg) {
return {DecodeFailure::DECODE_UNKNOWN_TYPE, {}};
}
template<>
std::pair<DecodeFailure, uint8_t> DecodeValue(const std::string &Arg) {
std::pair<DecodeFailure, uint8_t> DecodeValue(std::string &Arg) {
if (Arg.at(0) != '#') return {DecodeFailure::DECODE_INVALIDCHAR, 0};
uint8_t Result = strtoul(&Arg.at(1), nullptr, 0);
if (errno == ERANGE) return {DecodeFailure::DECODE_INVALIDRANGE, 0};
return {DecodeFailure::DECODE_OKAY, Result};
}
}
template<>
std::pair<DecodeFailure, bool> DecodeValue(const std::string &Arg) {
std::pair<DecodeFailure, bool> DecodeValue(std::string &Arg) {
if (Arg.at(0) != '#') return {DecodeFailure::DECODE_INVALIDCHAR, 0};
uint8_t Result = strtoul(&Arg.at(1), nullptr, 0);
@@ -99,7 +98,7 @@ class IRParser: public FEXCore::IR::IREmitter {
}
template<>
std::pair<DecodeFailure, uint16_t> DecodeValue(const std::string &Arg) {
std::pair<DecodeFailure, uint16_t> DecodeValue(std::string &Arg) {
if (Arg.at(0) != '#') return {DecodeFailure::DECODE_INVALIDCHAR, 0};
uint16_t Result = strtoul(&Arg.at(1), nullptr, 0);
@@ -108,7 +107,7 @@ class IRParser: public FEXCore::IR::IREmitter {
}
template<>
std::pair<DecodeFailure, uint32_t> DecodeValue(const std::string &Arg) {
std::pair<DecodeFailure, uint32_t> DecodeValue(std::string &Arg) {
if (Arg.at(0) != '#') return {DecodeFailure::DECODE_INVALIDCHAR, 0};
uint32_t Result = strtoul(&Arg.at(1), nullptr, 0);
@@ -117,7 +116,7 @@ class IRParser: public FEXCore::IR::IREmitter {
}
template<>
std::pair<DecodeFailure, uint64_t> DecodeValue(const std::string &Arg) {
std::pair<DecodeFailure, uint64_t> DecodeValue(std::string &Arg) {
if (Arg.at(0) != '#') return {DecodeFailure::DECODE_INVALIDCHAR, 0};
uint64_t Result = strtoull(&Arg.at(1), nullptr, 0);
@@ -126,7 +125,7 @@ class IRParser: public FEXCore::IR::IREmitter {
}
template<>
std::pair<DecodeFailure, int64_t> DecodeValue(const std::string &Arg) {
std::pair<DecodeFailure, int64_t> DecodeValue(std::string &Arg) {
if (Arg.at(0) != '#') return {DecodeFailure::DECODE_INVALIDCHAR, 0};
int64_t Result = (int64_t)strtoull(&Arg.at(1), nullptr, 0);
@@ -135,7 +134,7 @@ class IRParser: public FEXCore::IR::IREmitter {
}
template<>
std::pair<DecodeFailure, IR::SHA256Sum> DecodeValue(const std::string &Arg) {
std::pair<DecodeFailure, IR::SHA256Sum> DecodeValue(std::string &Arg) {
IR::SHA256Sum Result;
if (Arg.at(0) != 's' || Arg.at(1) != 'h' || Arg.at(2) != 'a' || Arg.at(3) != '2' || Arg.at(4) != '5' || Arg.at(5) != '6' || Arg.at(6) != ':')
@@ -166,7 +165,7 @@ class IRParser: public FEXCore::IR::IREmitter {
}
template<>
std::pair<DecodeFailure, FEXCore::IR::RegisterClassType> DecodeValue(const std::string &Arg) {
std::pair<DecodeFailure, FEXCore::IR::RegisterClassType> DecodeValue(std::string &Arg) {
if (Arg == "GPR") {
return {DecodeFailure::DECODE_OKAY, FEXCore::IR::GPRClass};
}
@@ -184,7 +183,7 @@ class IRParser: public FEXCore::IR::IREmitter {
}
template<>
std::pair<DecodeFailure, FEXCore::IR::TypeDefinition> DecodeValue(const std::string &Arg) {
std::pair<DecodeFailure, FEXCore::IR::TypeDefinition> DecodeValue(std::string &Arg) {
uint8_t Size{}, Elements{1};
int NumArgs = sscanf(Arg.c_str(), "i%hhdv%hhd", &Size, &Elements);
@@ -196,8 +195,8 @@ class IRParser: public FEXCore::IR::IREmitter {
}
template<>
std::pair<DecodeFailure, FEXCore::IR::CondClassType> DecodeValue(const std::string &Arg) {
static constexpr std::array<std::string_view, 22> CondNames = {
std::pair<DecodeFailure, FEXCore::IR::CondClassType> DecodeValue(std::string &Arg) {
std::array<std::string, 22> CondNames = {
"EQ",
"NEQ",
"UGE",
@@ -231,8 +230,8 @@ class IRParser: public FEXCore::IR::IREmitter {
}
template<>
std::pair<DecodeFailure, FEXCore::IR::MemOffsetType> DecodeValue(const std::string &Arg) {
static constexpr std::array<std::string_view, 3> Names = {
std::pair<DecodeFailure, FEXCore::IR::MemOffsetType> DecodeValue(std::string &Arg) {
std::array<std::string, 3> Names = {
"SXTX",
"UXTW",
"SXTW",
@@ -247,8 +246,8 @@ class IRParser: public FEXCore::IR::IREmitter {
}
template<>
std::pair<DecodeFailure, FEXCore::IR::FenceType> DecodeValue(const std::string &Arg) {
static constexpr std::array<std::string_view, 3> Names = {
std::pair<DecodeFailure, FEXCore::IR::FenceType> DecodeValue(std::string &Arg) {
std::array<std::string, 3> Names = {
"Loads",
"Stores",
"LoadStores",
@@ -263,22 +262,23 @@ class IRParser: public FEXCore::IR::IREmitter {
}
template<>
std::pair<DecodeFailure, OrderedNode*> DecodeValue(const std::string &Arg) {
std::pair<DecodeFailure, OrderedNode*> DecodeValue(std::string &Arg) {
if (Arg.at(0) != '%') return {DecodeFailure::DECODE_INVALIDCHAR, 0};
// Strip off the type qualifier from the ssa value
size_t ArgEnd = std::string::npos;
std::string SSAName = trim(Arg);
const size_t ArgEnd = SSAName.find_first_of(' ');
ArgEnd = SSAName.find_first_of(" ");
if (ArgEnd != std::string::npos) {
SSAName = SSAName.substr(0, ArgEnd);
}
SSAName = SSAName.substr(0, ArgEnd);
}
// Forward declarations may make this not succed
// Forward declarations may make this not succed
auto Op = SSANameMapper.find(SSAName);
if (Op == SSANameMapper.end()) {
if (Op == SSANameMapper.end()) {
return {DecodeFailure::DECODE_UNKNOWN_SSA, nullptr};
}
}
return {DecodeFailure::DECODE_OKAY, Op->second};
}
@@ -302,21 +302,21 @@ class IRParser: public FEXCore::IR::IREmitter {
IRParser(std::istream *text) {
InitializeStaticTables();
std::string TmpLine;
while (!text->eof()) {
std::getline(*text, TmpLine);
if (text->eof()) {
break;
}
if (text->eof()) {
break;
}
if (text->fail()) {
LogMan::Msg::EFmt("Failed to getline on line: {}", Lines.size());
LogMan::Msg::E("Failed to getline on line: %ld", Lines.size());
return;
}
Lines.emplace_back(TmpLine);
}
ResetWorkingList();
ResetWorkingList();
Loaded = Parse();
}
@@ -327,11 +327,11 @@ class IRParser: public FEXCore::IR::IREmitter {
bool Parse() {
const auto CheckPrintError = [&](const LineDefinition &Def, DecodeFailure Failure) -> bool {
auto CheckPrintError = [&](LineDefinition &Def, DecodeFailure Failure) -> bool {
if (Failure != DecodeFailure::DECODE_OKAY) {
LogMan::Msg::EFmt("Error on Line: {}", Def.LineNumber);
LogMan::Msg::EFmt("{}", Lines[Def.LineNumber]);
LogMan::Msg::EFmt("Value Couldn't be decoded due to {}", DecodeErrorToString(Failure));
LogMan::Msg::E("Error on Line: %d", Def.LineNumber);
LogMan::Msg::E("%s", Lines[Def.LineNumber].c_str());
LogMan::Msg::E("Value Couldn't be decoded due to %s", DecodeErrorToString(Failure).c_str());
return false;
}
@@ -339,13 +339,13 @@ class IRParser: public FEXCore::IR::IREmitter {
};
// String parse every line for our definitions
for (size_t i = 0; i < Lines.size(); ++i) {
std::string Line = Lines[i];
for (size_t i = 0; i < Lines.size(); ++i) {
std::string Line = Lines[i];
LineDefinition Def{};
CurrentDef = &Def;
CurrentDef = &Def;
Def.LineNumber = i;
Line = trim(Line);
Line = trim(Line);
// Skip empty lines
if (Line.empty()) {
@@ -359,37 +359,35 @@ class IRParser: public FEXCore::IR::IREmitter {
}
size_t CurrentPos{};
// Let's see if this node is assigning something first
if (Line[0] == '%') {
// Let's see if this node is assigning something first
if (Line[0] == '%') {
size_t DefinitionEnd = std::string::npos;
if ((DefinitionEnd = Line.find_first_of('=', CurrentPos)) != std::string::npos) {
if ((DefinitionEnd = Line.find_first_of("=", CurrentPos)) != std::string::npos) {
Def.Definition = Line.substr(0, DefinitionEnd);
Def.Definition = trim(Def.Definition);
Def.HasDefinition = true;
CurrentPos = DefinitionEnd + 1; // +1 to ensure we go past then assignment
}
else {
LogMan::Msg::EFmt("Error on Line: {}", i);
LogMan::Msg::EFmt("{}", Lines[i]);
LogMan::Msg::EFmt("SSA declaration without assignment");
LogMan::Msg::E("Error on Line: %d", i);
LogMan::Msg::E("%s", Lines[i].c_str());
LogMan::Msg::E("SSA declaration without assignment");
return false;
}
}
}
// Check if we are pulling in some IR from the IR Printer
// Prints (%ssa%d) at the start of lines without a definition
if (Line[0] == '(') {
size_t DefinitionEnd = std::string::npos;
if ((DefinitionEnd = Line.find_first_of(')', CurrentPos)) != std::string::npos) {
if ((DefinitionEnd = Line.find_first_of(")", CurrentPos)) != std::string::npos) {
size_t SSAEnd = std::string::npos;
if ((SSAEnd = Line.find_last_of(' ', DefinitionEnd)) != std::string::npos) {
if ((SSAEnd = Line.find_last_of(" ", DefinitionEnd)) != std::string::npos) {
std::string Type = Line.substr(SSAEnd + 1, DefinitionEnd - SSAEnd - 1);
Type = trim(Type);
auto DefinitionSize = DecodeValue<FEXCore::IR::TypeDefinition>(Type);
if (!CheckPrintError(Def, DefinitionSize.first)) {
return false;
}
if (!CheckPrintError(Def, DefinitionSize.first)) return false;
Def.Size = DefinitionSize.second;
}
@@ -398,9 +396,9 @@ class IRParser: public FEXCore::IR::IREmitter {
CurrentPos = DefinitionEnd + 1;
}
else {
LogMan::Msg::EFmt("Error on Line: {}", i);
LogMan::Msg::EFmt("{}", Lines[i]);
LogMan::Msg::EFmt("SSA value with numbered SSA provided but no closing parentheses");
LogMan::Msg::E("Error on Line: %d", i);
LogMan::Msg::E("%s", Lines[i].c_str());
LogMan::Msg::E("SSA value with numbered SSA provided but no closing parentheses");
return false;
}
}
@@ -408,7 +406,7 @@ class IRParser: public FEXCore::IR::IREmitter {
if (Def.HasDefinition) {
// Let's check if we have a size declared with this variable
size_t NameEnd = std::string::npos;
if ((NameEnd = Def.Definition.find_first_of(' ')) != std::string::npos) {
if ((NameEnd = Def.Definition.find_first_of(" ")) != std::string::npos) {
std::string Type = Def.Definition.substr(NameEnd + 1);
Type = trim(Type);
Def.Definition = trim(Def.Definition.substr(0, NameEnd));
@@ -419,9 +417,9 @@ class IRParser: public FEXCore::IR::IREmitter {
}
if (Def.Definition == "%Invalid") {
LogMan::Msg::EFmt("Error on Line: {}", i);
LogMan::Msg::EFmt("{}", Lines[i]);
LogMan::Msg::EFmt("Definition tried to define reserved %Invalid ssa node");
LogMan::Msg::E("Error on Line: %d", i);
LogMan::Msg::E("%s", Lines[i].c_str());
LogMan::Msg::E("Definition tried to define reserved %Invalid ssa node");
return false;
}
}
@@ -438,9 +436,9 @@ class IRParser: public FEXCore::IR::IREmitter {
}
else {
if (RemainingLine.empty()) {
LogMan::Msg::EFmt("Error on Line: {}", i);
LogMan::Msg::EFmt("{}", Lines[i]);
LogMan::Msg::EFmt("Line without an IROp?");
LogMan::Msg::E("Error on Line: %d", i);
LogMan::Msg::E("%s", Lines[i].c_str());
LogMan::Msg::E("Line without an IROp?");
return false;
}
@@ -457,10 +455,12 @@ class IRParser: public FEXCore::IR::IREmitter {
}
else {
while (!RemainingLine.empty()) {
const size_t ArgEnd = RemainingLine.find(',');
std::string Arg = trim(RemainingLine.substr(0, ArgEnd));
size_t ArgEnd = std::string::npos;
ArgEnd = RemainingLine.find_first_of(",");
Def.Args.emplace_back(std::move(Arg));
std::string Arg = RemainingLine.substr(0, ArgEnd);
Arg = trim(Arg);
Def.Args.emplace_back(Arg);
RemainingLine.erase(0, ArgEnd+1); // +1 to ensure we go past the ','
if (ArgEnd == std::string::npos)
@@ -469,17 +469,17 @@ class IRParser: public FEXCore::IR::IREmitter {
}
}
CurrentDef = &Defs.emplace_back(std::move(Def));
}
Defs.emplace_back(Def);
}
// Ensure all of the ops are real ops
for(size_t i = 0; i < Defs.size(); ++i) {
auto &Def = Defs[i];
auto Op = NameToOpMap.find(Def.IROp);
if (Op == NameToOpMap.end()) {
LogMan::Msg::EFmt("Error on Line: {}", Def.LineNumber);
LogMan::Msg::EFmt("{}", Lines[Def.LineNumber]);
LogMan::Msg::EFmt("IROp '{}' doesn't exist", Def.IROp);
LogMan::Msg::E("Error on Line: %d", Def.LineNumber);
LogMan::Msg::E("%s", Lines[Def.LineNumber].c_str());
LogMan::Msg::E("IROp '%s' doesn't exist", Def.IROp.c_str());
return false;
}
Def.OpEnum = Op->second;
@@ -489,38 +489,40 @@ class IRParser: public FEXCore::IR::IREmitter {
IRPair<IROp_IRHeader> IRHeader;
{
auto &Def = Defs[0];
CurrentDef = &Def;
CurrentDef = &Def;
if (Def.OpEnum != FEXCore::IR::IROps::OP_IRHEADER) {
LogMan::Msg::EFmt("Error on Line: {}", Def.LineNumber);
LogMan::Msg::EFmt("{}", Lines[Def.LineNumber]);
LogMan::Msg::EFmt("First op needs to be IRHeader. Was '{}'", Def.IROp);
LogMan::Msg::E("Error on Line: %d", Def.LineNumber);
LogMan::Msg::E("%s", Lines[Def.LineNumber].c_str());
LogMan::Msg::E("First op needs to be IRHeader. Was '%s'", Def.IROp.c_str());
return false;
}
auto CodeBlockCount = DecodeValue<uint64_t>(Def.Args[1]);
auto Entry = DecodeValue<uint64_t>(Def.Args[0]);
auto CodeBlockCount = DecodeValue<uint64_t>(Def.Args[2]);
if (!CheckPrintError(Def, Entry.first)) return false;
if (!CheckPrintError(Def, CodeBlockCount.first)) return false;
IRHeader = _IRHeader(InvalidNode, CodeBlockCount.second);
IRHeader = _IRHeader(InvalidNode, Entry.second, CodeBlockCount.second);
}
SetWriteCursor(nullptr); // isolate the header from everything following
// Initialize SSANameMapper with Invalid value
SSANameMapper.insert_or_assign("%Invalid", Invalid());
SSANameMapper["%Invalid"] = Invalid();
// Spin through the blocks and generate basic block ops
for(size_t i = 0; i < Defs.size(); ++i) {
auto &Def = Defs[i];
if (Def.OpEnum == FEXCore::IR::IROps::OP_CODEBLOCK) {
auto CodeBlock = _CodeBlock(InvalidNode, InvalidNode);
SSANameMapper.insert_or_assign(Def.Definition, CodeBlock.Node);
SSANameMapper[Def.Definition] = CodeBlock.Node;
Def.Node = CodeBlock.Node;
if (i == 1) {
// First code block is the entry block
// Link the header to the first block
IRHeader.first->Blocks = CodeBlock.Node->Wrapped(DualListData.ListBegin());
IRHeader.first->Blocks = CodeBlock.Node->Wrapped(ListData.Begin());
}
CodeBlocks.emplace_back(CodeBlock.Node);
}
@@ -532,64 +534,63 @@ class IRParser: public FEXCore::IR::IREmitter {
FEXCore::IR::IROp_CodeBlock *CurrentBlockOp{};
for(size_t i = 1; i < Defs.size(); ++i) {
auto &Def = Defs[i];
CurrentDef = &Def;
CurrentDef = &Def;
switch (Def.OpEnum) {
// Special handled
case FEXCore::IR::IROps::OP_IRHEADER:
LogMan::Msg::EFmt("Error on Line: {}", Def.LineNumber);
LogMan::Msg::EFmt("{}", Lines[Def.LineNumber]);
LogMan::Msg::EFmt("IRHEADER used in the middle of the block!");
LogMan::Msg::E("Error on Line: %d", Def.LineNumber);
LogMan::Msg::E("%s", Lines[Def.LineNumber].c_str());
LogMan::Msg::E("IRHEADER used in the middle of the block!");
return false; // only one OP_IRHEADER allowed per block
case FEXCore::IR::IROps::OP_CODEBLOCK: {
SetWriteCursor(nullptr); // isolate from previous block
if (CurrentBlock != nullptr) {
LogMan::Msg::EFmt("Error on Line: {}", Def.LineNumber);
LogMan::Msg::EFmt("{}", Lines[Def.LineNumber]);
LogMan::Msg::EFmt("CodeBlock being used inside of already existing codeblock!");
LogMan::Msg::E("Error on Line: %d", Def.LineNumber);
LogMan::Msg::E("%s", Lines[Def.LineNumber].c_str());
LogMan::Msg::E("CodeBlock being used inside of already existing codeblock!");
return false;
}
CurrentBlock = Def.Node;
CurrentBlockOp = CurrentBlock->Op(DualListData.DataBegin())->CW<FEXCore::IR::IROp_CodeBlock>();
CurrentBlockOp = CurrentBlock->Op(Data.Begin())->CW<FEXCore::IR::IROp_CodeBlock>();
break;
}
case FEXCore::IR::IROps::OP_BEGINBLOCK: {
if (CurrentBlock == nullptr) {
LogMan::Msg::EFmt("Error on Line: {}", Def.LineNumber);
LogMan::Msg::EFmt("{}", Lines[Def.LineNumber]);
LogMan::Msg::EFmt("EndBlock being used outside of a block!");
LogMan::Msg::E("Error on Line: %d", Def.LineNumber);
LogMan::Msg::E("%s", Lines[Def.LineNumber].c_str());
LogMan::Msg::E("EndBlock being used outside of a block!");
return false;
}
auto Adjust = DecodeValue<OrderedNode*>(Def.Args[0]);
if (!CheckPrintError(Def, Adjust.first)) {
return false;
}
if (!CheckPrintError(Def, Adjust.first)) return false;
Def.Node = _BeginBlock(Adjust.second);
CurrentBlockOp->Begin = Def.Node->Wrapped(DualListData.ListBegin());
CurrentBlockOp->Begin = Def.Node->Wrapped(ListData.Begin());
break;
}
case FEXCore::IR::IROps::OP_ENDBLOCK: {
if (CurrentBlock == nullptr) {
LogMan::Msg::EFmt("Error on Line: {}", Def.LineNumber);
LogMan::Msg::EFmt("{}", Lines[Def.LineNumber]);
LogMan::Msg::EFmt("EndBlock being used outside of a block!");
LogMan::Msg::E("Error on Line: %d", Def.LineNumber);
LogMan::Msg::E("%s", Lines[Def.LineNumber].c_str());
LogMan::Msg::E("EndBlock being used outside of a block!");
return false;
}
auto Adjust = DecodeValue<OrderedNode*>(Def.Args[0]);
if (!CheckPrintError(Def, Adjust.first)) {
return false;
}
if (!CheckPrintError(Def, Adjust.first)) return false;
Def.Node = _EndBlock(Adjust.second);
CurrentBlockOp->Last = Def.Node->Wrapped(DualListData.ListBegin());
CurrentBlockOp->Last = Def.Node->Wrapped(ListData.Begin());
CurrentBlock = nullptr;
CurrentBlockOp = nullptr;
@@ -598,23 +599,25 @@ class IRParser: public FEXCore::IR::IREmitter {
}
case FEXCore::IR::IROps::OP_DUMMY: {
LogMan::Msg::EFmt("Error on Line: {}", Def.LineNumber);
LogMan::Msg::EFmt("{}", Lines[Def.LineNumber]);
LogMan::Msg::EFmt("Dummy op must not be used");
LogMan::Msg::E("Error on Line: %d", Def.LineNumber);
LogMan::Msg::E("%s", Lines[Def.LineNumber].c_str());
LogMan::Msg::E("Dummy op must not be used");
break;
}
#define IROP_PARSER_SWITCH_HELPERS
#include <FEXCore/IR/IRDefines.inc>
default: {
LogMan::Msg::EFmt("Error on Line: {}", Def.LineNumber);
LogMan::Msg::EFmt("{}", Lines[Def.LineNumber]);
LogMan::Msg::EFmt("Unhandled Op enum '{}' in parser", Def.IROp);
LogMan::Msg::E("Error on Line: %d", Def.LineNumber);
LogMan::Msg::E("%s", Lines[Def.LineNumber].c_str());
LogMan::Msg::E("Unhandled Op enum '%s' in parser", Def.IROp.c_str());
return false;
break;
}
}
if (Def.HasDefinition) {
auto IROp = Def.Node->Op(DualListData.DataBegin());
auto IROp = Def.Node->Op(Data.Begin());
if (Def.Size.Elements()) {
IROp->Size = Def.Size.Bytes() * Def.Size.Elements();
IROp->ElementSize = Def.Size.Bytes();
@@ -623,7 +626,7 @@ class IRParser: public FEXCore::IR::IREmitter {
IROp->Size = Def.Size.Bytes();
IROp->ElementSize = 0;
}
SSANameMapper.insert_or_assign(Def.Definition, Def.Node);
SSANameMapper[Def.Definition] = Def.Node;
}
}
@@ -631,11 +634,11 @@ class IRParser: public FEXCore::IR::IREmitter {
}
void InitializeStaticTables() {
if (NameToOpMap.empty()) {
if (NameToOpMap.size() == 0) {
for (FEXCore::IR::IROps Op = FEXCore::IR::IROps::OP_DUMMY;
Op <= FEXCore::IR::IROps::OP_LAST;
Op = static_cast<FEXCore::IR::IROps>(static_cast<uint32_t>(Op) + 1)) {
NameToOpMap.insert_or_assign(FEXCore::IR::GetName(Op), Op);
NameToOpMap[FEXCore::IR::GetName(Op)] = Op;
}
}
}
@@ -643,12 +646,13 @@ class IRParser: public FEXCore::IR::IREmitter {
} // anon namespace
std::unique_ptr<IREmitter> Parse(std::istream *in) {
auto parser = std::make_unique<IRParser>(in);
IREmitter* Parse(std::istream *in) {
auto parser = new IRParser(in);
if (parser->Loaded) {
return parser;
} else {
delete parser;
return nullptr;
}
}
+4 -9
View File
@@ -19,13 +19,6 @@ void PassManager::AddDefaultPasses(bool InlineConstants, bool StaticRegisterAllo
if (!DisablePasses()) {
InsertPass(CreateContextLoadStoreElimination());
if (Is64BitMode()) {
// This needs to run after RCLSE
// This only matters for 64-bit code since these instructions don't exist in 32-bit
InsertPass(CreateLongDivideEliminationPass());
}
InsertPass(CreateDeadStoreElimination());
InsertPass(CreatePassDeadCodeElimination());
InsertPass(CreateConstProp(InlineConstants));
@@ -45,9 +38,10 @@ void PassManager::AddDefaultPasses(bool InlineConstants, bool StaticRegisterAllo
InsertPass(CreateStaticRegisterAllocationPass());
}
CompactionPass = CreateIRCompaction();
// If the IR is compacted post-RA then the node indexing gets messed up and the backend isn't able to find the register assigned to a node
// Compact before IR, don't worry about RA generating spills/fills
CompactionPass = InsertPass(CreateIRCompaction());
InsertPass(CompactionPass);
}
void PassManager::AddDefaultValidationPasses() {
@@ -59,7 +53,8 @@ void PassManager::AddDefaultValidationPasses() {
}
void PassManager::InsertRegisterAllocationPass(bool OptimizeSRA) {
RAPass = InsertPass(IR::CreateRegisterAllocationPass(CompactionPass, OptimizeSRA));
RAPass = IR::CreateRegisterAllocationPass(CompactionPass, OptimizeSRA);
InsertPass(RAPass);
}
bool PassManager::Run(IREmitter *IREmit) {
+6 -9
View File
@@ -6,7 +6,6 @@ $end_info$
#pragma once
#include <FEXCore/Config/Config.h>
#include <FEXCore/IR/IntrusiveIRList.h>
#include <FEXCore/IR/IREmitter.h>
@@ -42,9 +41,9 @@ class PassManager final {
public:
void AddDefaultPasses(bool InlineConstants, bool StaticRegisterAllocation);
void AddDefaultValidationPasses();
Pass* InsertPass(std::unique_ptr<Pass> Pass) {
void InsertPass(Pass *Pass) {
Pass->RegisterPassManager(this);
return Passes.emplace_back(std::move(Pass)).get();
Passes.emplace_back(Pass);
}
void InsertRegisterAllocationPass(bool OptimizeSRA);
@@ -52,7 +51,7 @@ public:
bool Run(IREmitter *IREmit);
void RegisterExitHandler(ShouldExitHandler Handler) {
ExitHandler = std::move(Handler);
ExitHandler = Handler;
}
bool HasRAPass() const {
@@ -73,19 +72,17 @@ protected:
private:
Pass *RAPass{};
Pass *CompactionPass{};
FEXCore::IR::Pass *CompactionPass{};
std::vector<std::unique_ptr<Pass>> Passes;
#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
std::vector<std::unique_ptr<Pass>> ValidationPasses;
void InsertValidationPass(std::unique_ptr<Pass> Pass) {
void InsertValidationPass(Pass *Pass) {
Pass->RegisterPassManager(this);
ValidationPasses.emplace_back(std::move(Pass));
ValidationPasses.emplace_back(Pass);
}
#endif
FEX_CONFIG_OPT(Is64BitMode, IS64BIT_MODE);
};
}
+12 -15
View File
@@ -1,27 +1,24 @@
#pragma once
#include <memory>
namespace FEXCore::IR {
class Pass;
class RegisterAllocationPass;
class RegisterAllocationData;
std::unique_ptr<FEXCore::IR::Pass> CreateConstProp(bool InlineConstants);
std::unique_ptr<FEXCore::IR::Pass> CreateContextLoadStoreElimination();
std::unique_ptr<FEXCore::IR::Pass> CreateSyscallOptimization();
std::unique_ptr<FEXCore::IR::Pass> CreateDeadFlagCalculationEliminination();
std::unique_ptr<FEXCore::IR::Pass> CreateDeadStoreElimination();
std::unique_ptr<FEXCore::IR::Pass> CreatePassDeadCodeElimination();
std::unique_ptr<FEXCore::IR::Pass> CreateIRCompaction();
std::unique_ptr<FEXCore::IR::RegisterAllocationPass> CreateRegisterAllocationPass(FEXCore::IR::Pass* CompactionPass, bool OptimizeSRA);
std::unique_ptr<FEXCore::IR::Pass> CreateStaticRegisterAllocationPass();
std::unique_ptr<FEXCore::IR::Pass> CreateLongDivideEliminationPass();
FEXCore::IR::Pass* CreateConstProp(bool InlineConstants);
FEXCore::IR::Pass* CreateContextLoadStoreElimination();
FEXCore::IR::Pass* CreateSyscallOptimization();
FEXCore::IR::Pass* CreateDeadFlagCalculationEliminination();
FEXCore::IR::Pass* CreateDeadStoreElimination();
FEXCore::IR::Pass* CreatePassDeadCodeElimination();
FEXCore::IR::Pass* CreateIRCompaction();
FEXCore::IR::RegisterAllocationPass* CreateRegisterAllocationPass(FEXCore::IR::Pass* CompactionPass, bool OptimizeSRA);
FEXCore::IR::Pass* CreateStaticRegisterAllocationPass();
namespace Validation {
std::unique_ptr<FEXCore::IR::Pass> CreateIRValidation();
std::unique_ptr<FEXCore::IR::Pass> CreatePhiValidation();
std::unique_ptr<FEXCore::IR::Pass> CreateValueDominanceValidation();
FEXCore::IR::Pass* CreateIRValidation();
FEXCore::IR::Pass* CreatePhiValidation();
FEXCore::IR::Pass* CreateValueDominanceValidation();
}
}
+322 -377
View File
@@ -19,6 +19,15 @@ $end_info$
namespace FEXCore::IR {
class ConstProp final : public FEXCore::IR::Pass {
std::unordered_map<uint64_t, OrderedNode*> ConstPool;
std::map<OrderedNode*, uint64_t> AddressgenConsts;
public:
bool Run(IREmitter *IREmit) override;
bool InlineConstants;
ConstProp(bool DoInlineConstants) : InlineConstants(DoInlineConstants) { }
};
template<typename T>
uint64_t getMask(T Op) {
uint64_t NumBits = Op->Header.Size * 8;
@@ -60,7 +69,8 @@ static bool IsImmMemory(uint64_t imm, uint8_t AccessSize) {
}
}
static std::tuple<MemOffsetType, uint8_t, OrderedNode*, OrderedNode*> MemExtendedAddressing(IREmitter *IREmit, uint8_t AccessSize, IROp_Header* AddressHeader) {
std::tuple<MemOffsetType, uint8_t, OrderedNode*, OrderedNode*> MemExtendedAddressing(IREmitter *IREmit, uint8_t AccessSize, IROp_Header* AddressHeader) {
auto Src0Header = IREmit->GetOpHeader(AddressHeader->Args[0]);
if (Src0Header->Size == 8) {
//Try to optimize: Base + MUL(Offset, Scale)
@@ -114,7 +124,7 @@ static std::tuple<MemOffsetType, uint8_t, OrderedNode*, OrderedNode*> MemExtende
return { MEM_OFFSET_SXTX, 1, IREmit->UnwrapNode(AddressHeader->Args[0]), IREmit->UnwrapNode(AddressHeader->Args[1]) };
}
static OrderedNodeWrapper RemoveUselessMasking(IREmitter *IREmit, OrderedNodeWrapper src, uint64_t mask) {
OrderedNodeWrapper RemoveUselessMasking(IREmitter *IREmit, OrderedNodeWrapper src, uint64_t mask) {
#if 1 // HOTFIX: We need to clear up the meaning of opsize and dest size. See #594
return src;
#else
@@ -141,7 +151,7 @@ static OrderedNodeWrapper RemoveUselessMasking(IREmitter *IREmit, OrderedNodeWra
#endif
}
static bool IsBfeAlreadyDone(IREmitter *IREmit, OrderedNodeWrapper src, uint64_t Width) {
bool IsBfeAlreadyDone(IREmitter *IREmit, OrderedNodeWrapper src, uint64_t Width) {
auto IROp = IREmit->GetOpHeader(src);
if (IROp->Op == OP_BFE) {
auto Op = IROp->C<IR::IROp_Bfe>();
@@ -152,55 +162,31 @@ static bool IsBfeAlreadyDone(IREmitter *IREmit, OrderedNodeWrapper src, uint64_t
return false;
}
class ConstProp final : public FEXCore::IR::Pass {
public:
explicit ConstProp(bool DoInlineConstants) : InlineConstants(DoInlineConstants) { }
bool ConstProp::Run(IREmitter *IREmit) {
bool Run(IREmitter *IREmit) override;
bool InlineConstants;
private:
bool HandleConstantPools(IREmitter *IREmit, const IRListView& CurrentIR);
void CodeMotionAroundSelects(IREmitter *IREmit, const IRListView& CurrentIR);
void FCMPOptimization(IREmitter *IREmit, const IRListView& CurrentIR);
void LoadMemStoreMemImmediatePooling(IREmitter *IREmit, const IRListView& CurrentIR);
bool ZextAndMaskingElimination(IREmitter *IREmit, const IRListView& CurrentIR,
OrderedNode* CodeNode, IROp_Header* IROp);
bool ConstantPropagation(IREmitter *IREmit, const IRListView& CurrentIR,
OrderedNode* CodeNode, IROp_Header* IROp);
bool ConstantInlining(IREmitter *IREmit, const IRListView& CurrentIR);
std::unordered_map<uint64_t, OrderedNode*> ConstPool;
std::map<OrderedNode*, uint64_t> AddressgenConsts;
};
bool ConstProp::HandleConstantPools(IREmitter *IREmit, const IRListView& CurrentIR) {
bool Changed = false;
auto CurrentIR = IREmit->ViewIR();
auto OriginalWriteCursor = IREmit->GetWriteCursor();
// constants are pooled per block
for (auto [BlockNode, BlockHeader] : CurrentIR.GetBlocks()) {
for (auto [CodeNode, IROp] : CurrentIR.GetCode(BlockNode)) {
if (IROp->Op == OP_CONSTANT) {
auto Op = IROp->C<IR::IROp_Constant>();
if (ConstPool.count(Op->Constant)) {
IREmit->ReplaceAllUsesWith(CodeNode, ConstPool[Op->Constant]);
Changed = true;
} else {
ConstPool[Op->Constant] = CodeNode;
{
// constants are pooled per block
for (auto [BlockNode, BlockHeader] : CurrentIR.GetBlocks()) {
for (auto [CodeNode, IROp] : CurrentIR.GetCode(BlockNode)) {
if (IROp->Op == OP_CONSTANT) {
auto Op = IROp->C<IR::IROp_Constant>();
if (ConstPool.count(Op->Constant)) {
IREmit->ReplaceAllUsesWith(CodeNode, ConstPool[Op->Constant]);
Changed = true;
} else {
ConstPool[Op->Constant] = CodeNode;
}
}
}
ConstPool.clear();
}
ConstPool.clear();
}
return Changed;
}
// Code motion around selects
// Moves unary ops that depend on a select before the select, if both inputs are constants
// assumes that unary ops without side effects on constants will be constprop'd
void ConstProp::CodeMotionAroundSelects(IREmitter *IREmit, const IRListView& CurrentIR) {
// Code motion around selects
// Moves unary ops that depend on a select before the select, if both inputs are constants
// assumes that unary ops without side effects on constants will be constprop'd
@@ -257,9 +243,9 @@ void ConstProp::CodeMotionAroundSelects(IREmitter *IREmit, const IRListView& Cur
}
}
}
}
void ConstProp::FCMPOptimization(IREmitter *IREmit, const IRListView& CurrentIR) {
// FCMP optimization
// Make all FCMPs set no flags
for (auto [CodeNode, IROp] : CurrentIR.GetAllCode()) {
if (IROp->Op == OP_FCMP) {
@@ -274,17 +260,16 @@ void ConstProp::FCMPOptimization(IREmitter *IREmit, const IRListView& CurrentIR)
auto ghf = IROp->CW<IR::IROp_GetHostFlag>();
auto fcmp = IREmit->GetOpHeader(ghf->GPR)->CW<IR::IROp_FCmp>();
LOGMAN_THROW_A(fcmp->Header.Op == OP_FCMP || fcmp->Header.Op == OP_F80CMP, "Unexpected OP_GETHOSTFLAG source");
LogMan::Throw::A(fcmp->Header.Op == OP_FCMP || fcmp->Header.Op == OP_F80CMP, "Unexpected OP_GETHOSTFLAG source");
if(fcmp->Header.Op == OP_FCMP) {
fcmp->Flags |= 1 << ghf->Flag;
}
}
}
}
// LoadMem / StoreMem imm pooling
// If imms are close by, use address gen to generate the values instead of using a new imm
void ConstProp::LoadMemStoreMemImmediatePooling(IREmitter *IREmit, const IRListView& CurrentIR) {
// LoadMem / StoreMem imm pooling
// If imms are close by, use address gen to generate the values instead of using a new imm
for (auto [BlockNode, BlockIROp] : CurrentIR.GetBlocks()) {
for (auto [CodeNode, IROp] : CurrentIR.GetCode(BlockNode)) {
if (IROp->Op == OP_LOADMEM || IROp->Op == OP_STOREMEM) {
@@ -308,163 +293,152 @@ void ConstProp::LoadMemStoreMemImmediatePooling(IREmitter *IREmit, const IRListV
}
AddressgenConsts.clear();
}
}
bool ConstProp::ZextAndMaskingElimination(IREmitter *IREmit, const IRListView& CurrentIR,
OrderedNode* CodeNode, IROp_Header* IROp) {
bool Changed = false;
switch (IROp->Op) {
// Generic handling
case OP_OR:
case OP_XOR:
case OP_NOT:
case OP_ADD:
case OP_SUB:
case OP_MUL:
case OP_UMUL:
case OP_DIV:
case OP_UDIV:
case OP_LSHR:
case OP_ASHR:
case OP_LSHL:
case OP_ROR: {
for (int i = 0; i < IROp->NumArgs; i++) {
auto newArg = RemoveUselessMasking(IREmit, IROp->Args[i], getMask(IROp));
if (newArg.ID() != IROp->Args[i].ID()) {
IREmit->ReplaceNodeArgument(CodeNode, i, IREmit->UnwrapNode(newArg));
Changed = true;
for (auto [CodeNode, IROp] : CurrentIR.GetAllCode()) {
// zext / masking elimination
switch (IROp->Op) {
// Generic handling
case OP_OR:
case OP_XOR:
case OP_NOT:
case OP_ADD:
case OP_SUB:
case OP_MUL:
case OP_UMUL:
case OP_DIV:
case OP_UDIV:
case OP_LSHR:
case OP_ASHR:
case OP_LSHL:
case OP_ROR: {
for (int i = 0; i < IROp->NumArgs; i++) {
auto newArg = RemoveUselessMasking(IREmit, IROp->Args[i], getMask(IROp));
if (newArg.ID() != IROp->Args[i].ID()) {
IREmit->ReplaceNodeArgument(CodeNode, i, IREmit->UnwrapNode(newArg));
Changed = true;
}
}
}
break;
}
case OP_AND: {
// if AND's arguments are imms, they are masking
for (int i = 0; i < IROp->NumArgs; i++) {
auto mask = getMask(IROp);
uint64_t imm = 0;
if (IREmit->IsValueConstant(IROp->Args[i^1], &imm))
mask = imm;
auto newArg = RemoveUselessMasking(IREmit, IROp->Args[i], imm);
if (newArg.ID() != IROp->Args[i].ID()) {
IREmit->ReplaceNodeArgument(CodeNode, i, IREmit->UnwrapNode(newArg));
Changed = true;
}
}
break;
}
case OP_BFE: {
auto Op = IROp->C<IR::IROp_Bfe>();
// Is this value already BFE'd?
if (IsBfeAlreadyDone(IREmit, IROp->Args[0], Op->Width)) {
IREmit->ReplaceAllUsesWith(CodeNode, CurrentIR.GetNode(IROp->Args[0]));
//printf("Removed BFE once \n");
break;
}
// Is this value already ZEXT'd?
if (Op->lsb == 0) {
//LoadMem, LoadMemTSO & LoadContext ZExt
case OP_AND: {
// if AND's arguments are imms, they are masking
for (int i = 0; i < IROp->NumArgs; i++) {
auto mask = getMask(IROp);
uint64_t imm = 0;
if (IREmit->IsValueConstant(IROp->Args[i^1], &imm))
mask = imm;
auto newArg = RemoveUselessMasking(IREmit, IROp->Args[i], imm);
if (newArg.ID() != IROp->Args[i].ID()) {
IREmit->ReplaceNodeArgument(CodeNode, i, IREmit->UnwrapNode(newArg));
Changed = true;
}
}
break;
}
case OP_BFE: {
auto Op = IROp->C<IR::IROp_Bfe>();
// Is this value already BFE'd?
if (IsBfeAlreadyDone(IREmit, IROp->Args[0], Op->Width)) {
IREmit->ReplaceAllUsesWith(CodeNode, CurrentIR.GetNode(IROp->Args[0]));
//printf("Removed BFE once \n");
break;
}
// Is this value already ZEXT'd?
if (Op->lsb == 0) {
//LoadMem, LoadMemTSO & LoadContext ZExt
auto source = IROp->Args[0];
auto sourceHeader = IREmit->GetOpHeader(source);
if (Op->Width >= (sourceHeader->Size*8) &&
(sourceHeader->Op == OP_LOADMEM || sourceHeader->Op == OP_LOADMEMTSO || sourceHeader->Op == OP_LOADCONTEXT)
) {
//printf("Eliminated needless zext bfe\n");
// Load mem / load ctx zexts, no need to vmem
IREmit->ReplaceAllUsesWith(CodeNode, CurrentIR.GetNode(source));
break;
}
}
// BFE does implicit masking, remove any masks leading to this, if possible
uint64_t imm = 1ULL << (Op->Width-1);
imm = (imm-1) *2 + 1;
imm <<= Op->lsb;
auto newArg = RemoveUselessMasking(IREmit, IROp->Args[0], imm);
if (newArg.ID() != IROp->Args[0].ID()) {
IREmit->ReplaceNodeArgument(CodeNode, 0, IREmit->UnwrapNode(newArg));
Changed = true;
}
break;
}
case OP_SBFE: {
auto Op = IROp->C<IR::IROp_Sbfe>();
// BFE does implicit masking
uint64_t imm = 1ULL << (Op->Width-1);
imm = (imm-1) *2 + 1;
imm <<= Op->lsb;
auto newArg = RemoveUselessMasking(IREmit, IROp->Args[0], imm);
if (newArg.ID() != IROp->Args[0].ID()) {
IREmit->ReplaceNodeArgument(CodeNode, 0, IREmit->UnwrapNode(newArg));
Changed = true;
}
break;
}
case OP_VFADD:
case OP_VFSUB:
case OP_VFMUL:
case OP_VFDIV:
case OP_FCMP: {
auto flopSize = IROp->Size;
for (int i = 0; i < IROp->NumArgs; i++) {
auto argHeader = IREmit->GetOpHeader(IROp->Args[i]);
if (argHeader->Op == OP_VMOV) {
auto source = argHeader->Args[0];
auto sourceHeader = IREmit->GetOpHeader(source);
if (sourceHeader->Size >= flopSize) {
IREmit->ReplaceNodeArgument(CodeNode, i, IREmit->UnwrapNode(source));
//printf("VMOV bypassed\n");
}
}
}
break;
}
case OP_VMOV: {
// elim from load mem
auto source = IROp->Args[0];
auto sourceHeader = IREmit->GetOpHeader(source);
if (Op->Width >= (sourceHeader->Size*8) &&
if (IROp->Size >= sourceHeader->Size &&
(sourceHeader->Op == OP_LOADMEM || sourceHeader->Op == OP_LOADMEMTSO || sourceHeader->Op == OP_LOADCONTEXT)
) {
//printf("Eliminated needless zext bfe\n");
) {
//printf("Eliminated needless zext VMOV\n");
// Load mem / load ctx zexts, no need to vmem
IREmit->ReplaceAllUsesWith(CodeNode, CurrentIR.GetNode(source));
break;
} else if (IROp->Size == sourceHeader->Size) {
// VMOV of same size
//printf("printf vmov of same size?!\n");
IREmit->ReplaceAllUsesWith(CodeNode, CurrentIR.GetNode(source));
}
break;
}
// BFE does implicit masking, remove any masks leading to this, if possible
uint64_t imm = 1ULL << (Op->Width-1);
imm = (imm-1) *2 + 1;
imm <<= Op->lsb;
auto newArg = RemoveUselessMasking(IREmit, IROp->Args[0], imm);
if (newArg.ID() != IROp->Args[0].ID()) {
IREmit->ReplaceNodeArgument(CodeNode, 0, IREmit->UnwrapNode(newArg));
Changed = true;
}
break;
default: break;
}
case OP_SBFE: {
auto Op = IROp->C<IR::IROp_Sbfe>();
// BFE does implicit masking
uint64_t imm = 1ULL << (Op->Width-1);
imm = (imm-1) *2 + 1;
imm <<= Op->lsb;
auto newArg = RemoveUselessMasking(IREmit, IROp->Args[0], imm);
if (newArg.ID() != IROp->Args[0].ID()) {
IREmit->ReplaceNodeArgument(CodeNode, 0, IREmit->UnwrapNode(newArg));
Changed = true;
}
break;
}
case OP_VFADD:
case OP_VFSUB:
case OP_VFMUL:
case OP_VFDIV:
case OP_FCMP: {
auto flopSize = IROp->Size;
for (int i = 0; i < IROp->NumArgs; i++) {
auto argHeader = IREmit->GetOpHeader(IROp->Args[i]);
if (argHeader->Op == OP_VMOV) {
auto source = argHeader->Args[0];
auto sourceHeader = IREmit->GetOpHeader(source);
if (sourceHeader->Size >= flopSize) {
IREmit->ReplaceNodeArgument(CodeNode, i, IREmit->UnwrapNode(source));
//printf("VMOV bypassed\n");
}
}
}
break;
}
case OP_VMOV: {
// elim from load mem
auto source = IROp->Args[0];
auto sourceHeader = IREmit->GetOpHeader(source);
if (IROp->Size >= sourceHeader->Size &&
(sourceHeader->Op == OP_LOADMEM || sourceHeader->Op == OP_LOADMEMTSO || sourceHeader->Op == OP_LOADCONTEXT)
) {
//printf("Eliminated needless zext VMOV\n");
// Load mem / load ctx zexts, no need to vmem
IREmit->ReplaceAllUsesWith(CodeNode, CurrentIR.GetNode(source));
} else if (IROp->Size == sourceHeader->Size) {
// VMOV of same size
//printf("printf vmov of same size?!\n");
IREmit->ReplaceAllUsesWith(CodeNode, CurrentIR.GetNode(source));
}
break;
}
default:
break;
}
return Changed;
}
// constprop + some more per instruction logic
bool ConstProp::ConstantPropagation(IREmitter *IREmit, const IRListView& CurrentIR,
OrderedNode* CodeNode, IROp_Header* IROp) {
bool Changed = false;
// constprop + some more per instruction logic
switch (IROp->Op) {
/*
case OP_UMUL:
@@ -489,7 +463,7 @@ bool ConstProp::ConstantPropagation(IREmitter *IREmit, const IRListView& Current
if (IREmit->IsValueConstant(IROp->Args[0], &Constant1) &&
IREmit->IsValueConstant(IROp->Args[1], &Constant2)) {
LOGMAN_MSG_A("Could const prop op: %s", std::string(IR::GetName(IROp->Op)).c_str());
LogMan::Msg::A("Could const prop op: %s", std::string(IR::GetName(IROp->Op)).c_str());
}
break;
}
@@ -505,7 +479,7 @@ bool ConstProp::ConstantPropagation(IREmitter *IREmit, const IRListView& Current
uint64_t Constant1;
if (IREmit->IsValueConstant(IROp->Args[0], &Constant1)) {
LOGMAN_MSG_A("Could const prop op: %s", std::string(IR::GetName(IROp->Op)).c_str());
LogMan::Msg::A("Could const prop op: %s", std::string(IR::GetName(IROp->Op)).c_str());
}
break;
}
@@ -516,6 +490,7 @@ bool ConstProp::ConstantPropagation(IREmitter *IREmit, const IRListView& Current
auto AddressHeader = IREmit->GetOpHeader(Op->Header.Args[0]);
if (AddressHeader->Op == OP_ADD && AddressHeader->Size == 8) {
auto [OffsetType, OffsetScale, Arg0, Arg1] = MemExtendedAddressing(IREmit, Op->Size, AddressHeader);
Op->OffsetType = OffsetType;
@@ -555,6 +530,7 @@ bool ConstProp::ConstantPropagation(IREmitter *IREmit, const IRListView& Current
uint64_t NewConstant = (Constant1 + Constant2) & getMask(Op) ;
IREmit->ReplaceWithConstant(CodeNode, NewConstant);
Changed = true;
continue;
}
break;
}
@@ -735,9 +711,9 @@ bool ConstProp::ConstantPropagation(IREmitter *IREmit, const IRListView& Current
uint64_t NewConstant = (Constant1 * Constant2) & getMask(Op);
IREmit->ReplaceWithConstant(CodeNode, NewConstant);
Changed = true;
} else if (IREmit->IsValueConstant(Op->Header.Args[1], &Constant2) && std::popcount(Constant2) == 1) {
} else if (IREmit->IsValueConstant(Op->Header.Args[1], &Constant2) && __builtin_popcountl(Constant2) == 1) {
if (IROp->Size == 4 || IROp->Size == 8) {
uint64_t amt = std::countr_zero(Constant2);
uint64_t amt = __builtin_ctzl(Constant2);
IREmit->SetWriteCursor(CodeNode);
auto shift = IREmit->_Lshl(CurrentIR.GetNode(Op->Header.Args[0]), IREmit->_Constant(amt));
shift.first->Header.Size = IROp->Size; // force Lshl to be the same size as the original Mul
@@ -777,223 +753,192 @@ bool ConstProp::ConstantPropagation(IREmitter *IREmit, const IRListView& Current
default:
break;
}
}
return Changed;
}
bool ConstProp::ConstantInlining(IREmitter *IREmit, const IRListView& CurrentIR) {
bool Changed = false;
for (auto [CodeNode, IROp] : CurrentIR.GetAllCode()) {
switch(IROp->Op) {
case OP_LSHR:
case OP_ASHR:
case OP_ROR:
case OP_LSHL:
{
auto Op = IROp->C<IR::IROp_Lshr>();
uint64_t Constant2{};
if (IREmit->IsValueConstant(Op->Header.Args[1], &Constant2)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Header.Args[1]));
// this shouldn't be here, but rather on the emitter themselves or the constprop transformation?
if (IROp->Size <=4)
Constant2 &= 31;
else
Constant2 &= 63;
IREmit->ReplaceNodeArgument(CodeNode, 1, IREmit->_InlineConstant(Constant2));
Changed = true;
}
break;
}
case OP_ADD:
case OP_SUB:
{
auto Op = IROp->C<IR::IROp_Add>();
uint64_t Constant2{};
if (IREmit->IsValueConstant(Op->Header.Args[1], &Constant2)) {
if (IsImmAddSub(Constant2)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Header.Args[1]));
IREmit->ReplaceNodeArgument(CodeNode, 1, IREmit->_InlineConstant(Constant2));
Changed = true;
}
}
break;
}
case OP_SELECT:
{
auto Op = IROp->C<IR::IROp_Select>();
uint64_t Constant1{};
if (IREmit->IsValueConstant(Op->Header.Args[1], &Constant1)) {
if (IsImmAddSub(Constant1)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Header.Args[1]));
IREmit->ReplaceNodeArgument(CodeNode, 1, IREmit->_InlineConstant(Constant1));
Changed = true;
}
}
uint64_t Constant2{};
uint64_t Constant3{};
if (IREmit->IsValueConstant(Op->Header.Args[2], &Constant2) &&
IREmit->IsValueConstant(Op->Header.Args[3], &Constant3) &&
Constant2 == 1 &&
Constant3 == 0)
// constant inlining
if (InlineConstants) {
for (auto [CodeNode, IROp] : CurrentIR.GetAllCode()) {
switch(IROp->Op) {
case OP_LSHR:
case OP_ASHR:
case OP_ROR:
case OP_LSHL:
{
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Header.Args[2]));
auto Op = IROp->C<IR::IROp_Lshr>();
IREmit->ReplaceNodeArgument(CodeNode, 2, IREmit->_InlineConstant(Constant2));
IREmit->ReplaceNodeArgument(CodeNode, 3, IREmit->_InlineConstant(Constant3));
}
break;
}
case OP_CONDJUMP:
{
auto Op = IROp->C<IR::IROp_CondJump>();
uint64_t Constant2{};
if (IREmit->IsValueConstant(Op->Header.Args[1], &Constant2)) {
if (IsImmAddSub(Constant2)) {
uint64_t Constant2{};
if (IREmit->IsValueConstant(Op->Header.Args[1], &Constant2)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Header.Args[1]));
// this shouldn't be here, but rather on the emitter themselves or the constprop transformation?
if (IROp->Size <=4)
Constant2 &= 31;
else
Constant2 &= 63;
IREmit->ReplaceNodeArgument(CodeNode, 1, IREmit->_InlineConstant(Constant2));
Changed = true;
}
break;
}
break;
}
case OP_EXITFUNCTION:
{
auto Op = IROp->C<IR::IROp_ExitFunction>();
case OP_ADD:
case OP_SUB:
{
auto Op = IROp->C<IR::IROp_Add>();
uint64_t Constant{};
if (IREmit->IsValueConstant(Op->NewRIP, &Constant)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->NewRIP));
uint64_t Constant2{};
if (IREmit->IsValueConstant(Op->Header.Args[1], &Constant2)) {
if (IsImmAddSub(Constant2)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Header.Args[1]));
IREmit->ReplaceNodeArgument(CodeNode, 0, IREmit->_InlineConstant(Constant));
IREmit->ReplaceNodeArgument(CodeNode, 1, IREmit->_InlineConstant(Constant2));
Changed = true;
} else {
auto NewRIP = IREmit->GetOpHeader(Op->NewRIP);
if (NewRIP->Op == OP_ENTRYPOINTOFFSET) {
auto EO = NewRIP->C<IR::IROp_EntrypointOffset>();
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->NewRIP));
IREmit->ReplaceNodeArgument(CodeNode, 0, IREmit->_InlineEntrypointOffset(EO->Offset, EO->Header.Size));
Changed = true;
Changed = true;
}
}
break;
}
break;
}
case OP_OR:
case OP_XOR:
case OP_AND:
{
auto Op = IROp->CW<IR::IROp_Or>();
case OP_SELECT:
{
auto Op = IROp->C<IR::IROp_Select>();
uint64_t Constant2{};
if (IREmit->IsValueConstant(Op->Header.Args[1], &Constant2)) {
if (IsImmLogical(Constant2, IROp->Size * 8)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Header.Args[1]));
uint64_t Constant1{};
if (IREmit->IsValueConstant(Op->Header.Args[1], &Constant1)) {
if (IsImmAddSub(Constant1)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Header.Args[1]));
IREmit->ReplaceNodeArgument(CodeNode, 1, IREmit->_InlineConstant(Constant2));
IREmit->ReplaceNodeArgument(CodeNode, 1, IREmit->_InlineConstant(Constant1));
Changed = true;
Changed = true;
}
}
}
break;
}
case OP_LOADMEM:
{
auto Op = IROp->CW<IR::IROp_LoadMem>();
uint64_t Constant2{};
if (Op->OffsetType == MEM_OFFSET_SXTX && IREmit->IsValueConstant(Op->Header.Args[1], &Constant2)) {
if (IsImmMemory(Constant2, Op->Size)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Header.Args[1]));
IREmit->ReplaceNodeArgument(CodeNode, 1, IREmit->_InlineConstant(Constant2));
Changed = true;
}
}
break;
}
case OP_STOREMEM:
{
auto Op = IROp->CW<IR::IROp_StoreMem>();
uint64_t Constant2{};
if (Op->OffsetType == MEM_OFFSET_SXTX && IREmit->IsValueConstant(Op->Header.Args[2], &Constant2)) {
if (IsImmMemory(Constant2, Op->Size)) {
uint64_t Constant2{};
uint64_t Constant3{};
if (IREmit->IsValueConstant(Op->Header.Args[2], &Constant2) &&
IREmit->IsValueConstant(Op->Header.Args[3], &Constant3) &&
Constant2 == 1 &&
Constant3 == 0)
{
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Header.Args[2]));
IREmit->ReplaceNodeArgument(CodeNode, 2, IREmit->_InlineConstant(Constant2));
IREmit->ReplaceNodeArgument(CodeNode, 3, IREmit->_InlineConstant(Constant3));
}
break;
}
case OP_CONDJUMP:
{
auto Op = IROp->C<IR::IROp_CondJump>();
uint64_t Constant2{};
if (IREmit->IsValueConstant(Op->Header.Args[1], &Constant2)) {
if (IsImmAddSub(Constant2)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Header.Args[1]));
IREmit->ReplaceNodeArgument(CodeNode, 1, IREmit->_InlineConstant(Constant2));
Changed = true;
}
}
break;
}
case OP_EXITFUNCTION:
{
auto Op = IROp->C<IR::IROp_ExitFunction>();
uint64_t Constant{};
if (IREmit->IsValueConstant(Op->NewRIP, &Constant)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->NewRIP));
IREmit->ReplaceNodeArgument(CodeNode, 0, IREmit->_InlineConstant(Constant));
Changed = true;
} else {
auto NewRIP = IREmit->GetOpHeader(Op->NewRIP);
if (NewRIP->Op == OP_ENTRYPOINTOFFSET) {
auto EO = NewRIP->C<IR::IROp_EntrypointOffset>();
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->NewRIP));
IREmit->ReplaceNodeArgument(CodeNode, 0, IREmit->_InlineEntrypointOffset(EO->Offset, EO->Header.Size));
Changed = true;
}
}
break;
}
break;
case OP_OR:
case OP_XOR:
case OP_AND:
{
auto Op = IROp->CW<IR::IROp_Or>();
uint64_t Constant2{};
if (IREmit->IsValueConstant(Op->Header.Args[1], &Constant2)) {
if (IsImmLogical(Constant2, IROp->Size * 8)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Header.Args[1]));
IREmit->ReplaceNodeArgument(CodeNode, 1, IREmit->_InlineConstant(Constant2));
Changed = true;
}
}
break;
}
case OP_LOADMEM:
{
auto Op = IROp->CW<IR::IROp_LoadMem>();
uint64_t Constant2{};
if (Op->OffsetType == MEM_OFFSET_SXTX && IREmit->IsValueConstant(Op->Header.Args[1], &Constant2)) {
if (IsImmMemory(Constant2, Op->Size)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Header.Args[1]));
IREmit->ReplaceNodeArgument(CodeNode, 1, IREmit->_InlineConstant(Constant2));
Changed = true;
}
}
break;
}
case OP_STOREMEM:
{
auto Op = IROp->CW<IR::IROp_StoreMem>();
uint64_t Constant2{};
if (Op->OffsetType == MEM_OFFSET_SXTX && IREmit->IsValueConstant(Op->Header.Args[2], &Constant2)) {
if (IsImmMemory(Constant2, Op->Size)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Header.Args[2]));
IREmit->ReplaceNodeArgument(CodeNode, 2, IREmit->_InlineConstant(Constant2));
Changed = true;
}
}
break;
}
default: break;
}
default:
break;
}
}
return Changed;
}
bool ConstProp::Run(IREmitter *IREmit) {
bool Changed = false;
auto CurrentIR = IREmit->ViewIR();
auto OriginalWriteCursor = IREmit->GetWriteCursor();
if (HandleConstantPools(IREmit, CurrentIR)) {
Changed = true;
}
CodeMotionAroundSelects(IREmit, CurrentIR);
FCMPOptimization(IREmit, CurrentIR);
LoadMemStoreMemImmediatePooling(IREmit, CurrentIR);
for (auto [CodeNode, IROp] : CurrentIR.GetAllCode()) {
if (ZextAndMaskingElimination(IREmit, CurrentIR, CodeNode, IROp)) {
Changed = true;
}
if (ConstantPropagation(IREmit, CurrentIR, CodeNode, IROp)) {
Changed = true;
}
}
if (InlineConstants && ConstantInlining(IREmit, CurrentIR)) {
Changed = true;
}
IREmit->SetWriteCursor(OriginalWriteCursor);
return Changed;
}
std::unique_ptr<FEXCore::IR::Pass> CreateConstProp(bool InlineConstants) {
return std::make_unique<ConstProp>(InlineConstants);
FEXCore::IR::Pass* CreateConstProp(bool InlineConstants) {
return new ConstProp(InlineConstants);
}
}
@@ -59,8 +59,10 @@ void DeadCodeElimination::markUsed(OrderedNodeWrapper *CodeOp, IROp_Header *IROp
}
std::unique_ptr<FEXCore::IR::Pass> CreatePassDeadCodeElimination() {
return std::make_unique<DeadCodeElimination>();
FEXCore::IR::Pass* CreatePassDeadCodeElimination() {
return new DeadCodeElimination{};
}
}
@@ -211,18 +211,18 @@ namespace {
size_t ClassifiedStructSize{};
ContextClassificationInfo->Lookup.reserve(sizeof(FEXCore::Core::CPUState));
for (auto &it : *ContextClassification) {
LOGMAN_THROW_A(it.Class.Offset == ContextClassificationInfo->Lookup.size(), "Offset missmatch %d %d", it.Class.Offset == ContextClassificationInfo->Lookup.size());
LogMan::Throw::A(it.Class.Offset == ContextClassificationInfo->Lookup.size(), "Offset missmatch %d %d", it.Class.Offset == ContextClassificationInfo->Lookup.size());
for (int i = 0; i < it.Class.Size; i++) {
ContextClassificationInfo->Lookup.push_back(&it);
}
ClassifiedStructSize += it.Class.Size;
}
LOGMAN_THROW_A(ClassifiedStructSize == sizeof(FEXCore::Core::CPUState),
LogMan::Throw::A(ClassifiedStructSize == sizeof(FEXCore::Core::CPUState),
"Classified CPUStruct size doesn't match real CPUState struct size! %ld != %ld",
ClassifiedStructSize, sizeof(FEXCore::Core::CPUState));
LOGMAN_THROW_A(ContextClassificationInfo->Lookup.size() == sizeof(FEXCore::Core::CPUState),
LogMan::Throw::A(ContextClassificationInfo->Lookup.size() == sizeof(FEXCore::Core::CPUState),
"Classified CPUStruct size doesn't match real CPUState struct size! %ld != %ld",
ContextClassificationInfo->Lookup.size(), sizeof(FEXCore::Core::CPUState));
}
@@ -283,7 +283,7 @@ class RCLSE final : public FEXCore::IR::Pass {
public:
RCLSE() {
ClassifyContextStruct(&ClassifiedStruct);
DCE = FEXCore::IR::CreatePassDeadCodeElimination();
DCE.reset(FEXCore::IR::CreatePassDeadCodeElimination());
}
bool Run(FEXCore::IR::IREmitter *IREmit) override;
private:
@@ -306,15 +306,15 @@ ContextMemberInfo *RCLSE::FindMemberInfo(ContextInfo *ContextClassificationInfo,
}
ContextMemberInfo *RCLSE::RecordAccess(ContextMemberInfo *Info, FEXCore::IR::RegisterClassType RegClass, uint32_t Offset, uint8_t Size, LastAccessType AccessType, FEXCore::IR::OrderedNode *Node, FEXCore::IR::OrderedNode *StoreNode) {
LOGMAN_THROW_A((Offset + Size) <= (Info->Class.Offset + Info->Class.Size), "Access to context item went over member size");
LOGMAN_THROW_A(Info->Accessed != ACCESS_INVALID, "Tried to access invalid member");
LogMan::Throw::A((Offset + Size) <= (Info->Class.Offset + Info->Class.Size), "Access to context item went over member size");
LogMan::Throw::A(Info->Accessed != ACCESS_INVALID, "Tried to access invalid member");
// If we aren't fully overwriting the member then it is a partial write that we need to track
if (Size < Info->Class.Size) {
AccessType = AccessType == ACCESS_WRITE ? ACCESS_PARTIAL_WRITE : ACCESS_PARTIAL_READ;
}
if (Size > Info->Class.Size) {
LOGMAN_MSG_A("Can't handle this");
LogMan::Msg::A("Can't handle this");
}
Info->Accessed = AccessType;
@@ -500,7 +500,7 @@ bool RCLSE::RedundantStoreLoadElimination(FEXCore::IR::IREmitter *IREmit) {
else if (LastClass == GPRClass) {
LastNode = IREmit->_Bfe(Info->AccessSize, TruncateSize * 8, 0, LastNode);
} else {
LOGMAN_MSG_A("Unhandled Register class");
LogMan::Msg::A("Unhandled Register class");
}
}
@@ -578,7 +578,7 @@ bool RCLSE::RedundantStoreLoadElimination(FEXCore::IR::IREmitter *IREmit) {
if (LastStoreNode != nullptr)
{
IREmit->SetWriteCursor(CodeNode);
RecordAccess(&LocalInfo, FEXCore::IR::GPRClass, offsetof(FEXCore::Core::CPUState, flags[0]) + F, 1, ACCESS_WRITE, IREmit->_Constant(0), CodeNode);
RecordAccess(&LocalInfo, FEXCore::IR::GPRClass, offsetof(FEXCore::Core::CPUState, flags[0]) + F, 1, ACCESS_WRITE, IREmit->_InlineConstant(0), CodeNode);
IREmit->Remove(LastStoreNode);
Changed = true;
@@ -636,8 +636,8 @@ bool RCLSE::Run(FEXCore::IR::IREmitter *IREmit) {
namespace FEXCore::IR {
std::unique_ptr<FEXCore::IR::Pass> CreateContextLoadStoreElimination() {
return std::make_unique<RCLSE>();
FEXCore::IR::Pass* CreateContextLoadStoreElimination() {
return new RCLSE{};
}
}
@@ -106,7 +106,7 @@ uint64_t FPRBit(uint32_t Offset, uint32_t Size) {
else if (Size == 4)
return 1UL << (bitn);
else
LOGMAN_MSG_A("Unexpected FPR size %d", Size);
LogMan::Msg::A("Unexpected FPR size %d", Size);
return 7UL << (bitn); // Return maximum on failure case
}
@@ -331,8 +331,8 @@ bool DeadStoreElimination::Run(IREmitter *IREmit) {
return Changed;
}
std::unique_ptr<FEXCore::IR::Pass> CreateDeadStoreElimination() {
return std::make_unique<DeadStoreElimination>();
FEXCore::IR::Pass* CreateDeadStoreElimination() {
return new DeadStoreElimination{};
}
}
@@ -66,7 +66,7 @@ bool IRCompaction::Run(IREmitter *IREmit) {
auto HeaderNode = CurrentIR.GetHeaderNode();
auto HeaderOp = CurrentIR.GetHeader();
LOGMAN_THROW_A(HeaderOp->Header.Op == OP_IRHEADER, "First op wasn't IRHeader");
LogMan::Throw::A(HeaderOp->Header.Op == OP_IRHEADER, "First op wasn't IRHeader");
// This compaction pass is something that we need to ensure correct ordering and distances between IROps
// Later on we assume that an IROp's SSA value live range is its Node locations
@@ -84,13 +84,13 @@ bool IRCompaction::Run(IREmitter *IREmit) {
// Zero is always zero(invalid)
OldToNewRemap[0].NodeID = 0;
auto LocalHeaderOp = LocalBuilder._IRHeader(OrderedNodeWrapper::WrapOffset(0).GetNode(ListBegin), HeaderOp->BlockCount);
auto LocalHeaderOp = LocalBuilder._IRHeader(OrderedNodeWrapper::WrapOffset(0).GetNode(ListBegin), HeaderOp->Entry, HeaderOp->BlockCount);
OldToNewRemap[CurrentIR.GetID(HeaderNode)].NodeID = LocalIR.GetID(LocalHeaderOp.Node);
{
// Generate our codeblocks and link them together
for (auto [BlockNode, BlockHeader] : CurrentIR.GetBlocks()) {
LOGMAN_THROW_A(BlockHeader->Op == OP_CODEBLOCK, "IR type failed to be a code block");
LogMan::Throw::A(BlockHeader->Op == OP_CODEBLOCK, "IR type failed to be a code block");
auto LocalBlockIRNode = LocalBuilder._CodeBlock(LocalHeaderOp, LocalHeaderOp); // Use LocalHeaderOp as a dummy arg for now
OldToNewRemap[CurrentIR.GetID(BlockNode)].NodeID = LocalIR.GetID(LocalBlockIRNode.Node);
@@ -153,10 +153,8 @@ bool IRCompaction::Run(IREmitter *IREmit) {
{
// Fixup the arguments of all the IROps
for (auto &Block : GeneratedCodeBlocks) {
#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
auto BlockIROp = LocalIR.GetOp<FEXCore::IR::IROp_CodeBlock>(Block.NewNode);
LOGMAN_THROW_A(BlockIROp->Header.Op == OP_CODEBLOCK, "IR type failed to be a code block");
#endif
LogMan::Throw::A(BlockIROp->Header.Op == OP_CODEBLOCK, "IR type failed to be a code block");
for (auto [LocalNode, LocalIROp] : LocalIR.GetCode(Block.NewNode)) {
@@ -167,7 +165,7 @@ bool IRCompaction::Run(IREmitter *IREmit) {
for (uint8_t i = 0; i < NumArgs; ++i) {
uint32_t OldArg = LocalIROp->Args[i].ID();
#ifndef NDEBUG
LOGMAN_THROW_A(OldToNewRemap[OldArg].NodeID != ~0U, "Tried remapping unfound node %%ssa%d", OldArg);
LogMan::Throw::A(OldToNewRemap[OldArg].NodeID != ~0U, "Tried remapping unfound node %%ssa%d", OldArg);
#endif
LocalIROp->Args[i].NodeOffset = OldToNewRemap[OldArg].NodeID * sizeof(OrderedNode);
}
@@ -193,7 +191,7 @@ bool IRCompaction::Run(IREmitter *IREmit) {
// if (NewListSize > OldListSize ||
// NewDataSize > OldDataSize) {
// LOGMAN_MSG_A("Whoa. Compaction made the IR a different size when it shouldn't have. 0x%lx > 0x%lx or 0x%lx > 0x%lx",NewListSize, OldListSize, NewDataSize, OldDataSize);
// LogMan::Msg::A("Whoa. Compaction made the IR a different size when it shouldn't have. 0x%lx > 0x%lx or 0x%lx > 0x%lx",NewListSize, OldListSize, NewDataSize, OldDataSize);
// }
IREmit->CopyData(LocalBuilder);
@@ -201,8 +199,8 @@ bool IRCompaction::Run(IREmitter *IREmit) {
return true;
}
std::unique_ptr<FEXCore::IR::Pass> CreateIRCompaction() {
return std::make_unique<IRCompaction>();
FEXCore::IR::Pass* CreateIRCompaction() {
return new IRCompaction{};
}
}
@@ -11,7 +11,7 @@ $end_info$
#include "Interface/Core/OpcodeDispatcher.h"
#include "Common/BitSet.h"
#include <sstream>
#include <iostream>
namespace {
struct BlockInfo {
@@ -54,10 +54,8 @@ bool IRValidation::Run(IREmitter *IREmit) {
std::vector<uint32_t> Uses(CurrentIR.GetSSACount(), 0);
#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
auto HeaderOp = CurrentIR.GetHeader();
LOGMAN_THROW_A(HeaderOp->Header.Op == OP_IRHEADER, "First op wasn't IRHeader");
#endif
LogMan::Throw::A(HeaderOp->Header.Op == OP_IRHEADER, "First op wasn't IRHeader");
IR::RegisterAllocationData * RAData{};
if (Manager->HasRAPass()) {
@@ -68,7 +66,7 @@ bool IRValidation::Run(IREmitter *IREmit) {
for (auto [BlockNode, BlockHeader] : CurrentIR.GetBlocks()) {
auto BlockIROp = BlockHeader->CW<FEXCore::IR::IROp_CodeBlock>();
LOGMAN_THROW_A(BlockIROp->Header.Op == OP_CODEBLOCK, "IR type failed to be a code block");
LogMan::Throw::A(BlockIROp->Header.Op == OP_CODEBLOCK, "IR type failed to be a code block");
uint32_t BlockID = CurrentIR.GetID(BlockNode);
@@ -211,7 +209,7 @@ bool IRValidation::Run(IREmitter *IREmit) {
break;
}
default:
// LOGMAN_MSG_A("Unknown IR Op: %d(%s)", IROp->Op, FEXCore::IR::GetName(IROp->Op).data());
// LogMan::Msg::A("Unknown IR Op: %d(%s)", IROp->Op, FEXCore::IR::GetName(IROp->Op).data());
break;
}
}
@@ -281,13 +279,13 @@ bool IRValidation::Run(IREmitter *IREmit) {
Out << "Warnings:" << std::endl << Warnings.str() << std::endl;
}
LogMan::Msg::EFmt("{}", Out.str());
LogMan::Msg::E("%s", Out.str().c_str());
}
return false;
}
std::unique_ptr<FEXCore::IR::Pass> CreateIRValidation() {
return std::make_unique<IRValidation>();
FEXCore::IR::Pass* CreateIRValidation() {
return new IRValidation{};
}
}
@@ -1,112 +0,0 @@
/*
$info$
tags: ir|opts
desc: Long divide elimination pass
$end_info$
*/
#include "Interface/IR/PassManager.h"
#include <FEXCore/Utils/LogManager.h>
namespace FEXCore::IR {
class LongDivideEliminationPass final : public FEXCore::IR::Pass {
public:
bool Run(IREmitter *IREmit) override;
private:
bool IsZeroOp(IREmitter *IREmit, OrderedNodeWrapper Arg);
bool IsSextOp(IREmitter *IREmit, OrderedNodeWrapper Lower, OrderedNodeWrapper Upper);
};
bool LongDivideEliminationPass::IsZeroOp(IREmitter *IREmit, OrderedNodeWrapper Arg) {
auto IROp = IREmit->GetOpHeader(Arg);
uint64_t Value;
// XOR based zero
if (IROp->Op == OP_XOR) {
return IROp->Args[0] == IROp->Args[1];
}
else if (IREmit->IsValueConstant(Arg, &Value)) {
// Zero constant based zero op
return Value == 0;
}
return false;
}
bool LongDivideEliminationPass::IsSextOp(IREmitter *IREmit, OrderedNodeWrapper Lower, OrderedNodeWrapper Upper) {
// We need to check if the upper source is a sext of the lower source
auto UpperIROp = IREmit->GetOpHeader(Upper);
if (UpperIROp->Op == OP_SBFE) {
auto Op = UpperIROp->C<IR::IROp_Sbfe>();
if (Op->Width == 1 && Op->lsb == 63) {
// CQO: OrderedNode *Upper = _Sbfe(1, Size * 8 - 1, Src);
// If the lower is the upper in this case then it can be optimized
return Op->Header.Args[0] == Lower;
}
}
return false;
}
bool LongDivideEliminationPass::Run(IREmitter *IREmit) {
bool Changed = false;
auto CurrentIR = IREmit->ViewIR();
auto OriginalWriteCursor = IREmit->GetWriteCursor();
for (auto [BlockNode, BlockHeader] : CurrentIR.GetBlocks()) {
for (auto [CodeNode, IROp] : CurrentIR.GetCode(BlockNode)) {
if (IROp->Size == 8) {
if (IROp->Op == OP_LDIV ||
IROp->Op == OP_LREM) {
auto Op = IROp->C<IR::IROp_LDiv>();
// Check upper Op to see if it came from a CQO
// CQO: OrderedNode *Upper = _Sbfe(1, Size * 8 - 1, Src);
// If it does then it we only need a 64bit SDIV
if (IsSextOp(IREmit, Op->Lower, Op->Upper)) {
IREmit->SetWriteCursor(CodeNode);
OrderedNode *Lower = CurrentIR.GetNode(Op->Lower);
OrderedNode *Divisor = CurrentIR.GetNode(Op->Divisor);
OrderedNode *SDivOp{};
if (IROp->Op == OP_LDIV) {
SDivOp = IREmit->_Div(Lower, Divisor);
}
else {
SDivOp = IREmit->_Rem(Lower, Divisor);
}
IREmit->ReplaceAllUsesWith(CodeNode, SDivOp);
Changed = true;
}
}
else if (IROp->Op == OP_LUDIV ||
IROp->Op == OP_LUREM) {
auto Op = IROp->C<IR::IROp_LUDiv>();
// Check upper Op to see if it came from a xor zeroing op
// XOR: Result = _Xor(Dest, Src);
// If it does then it we only need a 64bit UDIV
if (IsZeroOp(IREmit, Op->Upper)) {
IREmit->SetWriteCursor(CodeNode);
OrderedNode *Lower = CurrentIR.GetNode(Op->Lower);
OrderedNode *Divisor = CurrentIR.GetNode(Op->Divisor);
OrderedNode *UDivOp{};
if (IROp->Op == OP_LUDIV) {
UDivOp = IREmit->_UDiv(Lower, Divisor);
}
else {
UDivOp = IREmit->_URem(Lower, Divisor);
}
IREmit->ReplaceAllUsesWith(CodeNode, UDivOp);
Changed = true;
}
}
}
}
}
IREmit->SetWriteCursor(OriginalWriteCursor);
return Changed;
}
std::unique_ptr<FEXCore::IR::Pass> CreateLongDivideEliminationPass() {
return std::make_unique<LongDivideEliminationPass>();
}
}
@@ -8,7 +8,7 @@ $end_info$
#include "Interface/IR/PassManager.h"
#include "Interface/Core/OpcodeDispatcher.h"
#include <sstream>
#include <iostream>
namespace FEXCore::IR::Validation {
@@ -59,15 +59,15 @@ bool PhiValidation::Run(IREmitter *IREmit) {
Out << "Errors:" << std::endl << Errors.str() << std::endl;
LogMan::Msg::EFmt("{}", Out.str());
LogMan::Msg::E(Out.str().c_str());
}
return false;
}
std::unique_ptr<FEXCore::IR::Pass> CreatePhiValidation() {
return std::make_unique<PhiValidation>();
FEXCore::IR::Pass* CreatePhiValidation() {
return new PhiValidation{};
}
}
@@ -59,8 +59,8 @@ bool DeadFlagCalculationEliminination::Run(IREmitter *IREmit) {
return Changed;
}
std::unique_ptr<FEXCore::IR::Pass> CreateDeadFlagCalculationEliminination() {
return std::make_unique<DeadFlagCalculationEliminination>();
FEXCore::IR::Pass* CreateDeadFlagCalculationEliminination() {
return new DeadFlagCalculationEliminination{};
}
}
@@ -8,7 +8,6 @@ $end_info$
#include "Interface/IR/Passes/RegisterAllocationPass.h"
#include "Interface/IR/Passes.h"
#include "Interface/Core/OpcodeDispatcher.h"
#include <FEXCore/Utils/Allocator.h>
#include <iterator>
#include <unordered_set>
@@ -46,7 +45,7 @@ namespace {
for (int i = 1; i < Size; i++)
Items[i] = 0xDEADBEEF;
#endif
Next.reset();
Next.release();
}
BucketList() {
@@ -66,7 +65,7 @@ namespace {
Enumerator(Item);
if (++i == Bucket->Size) {
LOGMAN_THROW_A(Bucket->Next != nullptr, "Interference bug");
LogMan::Throw::A(Bucket->Next != nullptr, "Interference bug");
Bucket = Bucket->Next.get();
i = 0;
}
@@ -87,7 +86,7 @@ namespace {
return true;
if (++i == Bucket->Size) {
LOGMAN_THROW_A(Bucket->Next != nullptr, "Bucket in bad state");
LogMan::Throw::A(Bucket->Next != nullptr, "Bucket in bad state");
Bucket = Bucket->Next.get();
i = 0;
}
@@ -131,7 +130,7 @@ namespace {
}
else if (++i == Size) {
i = 0;
LOGMAN_THROW_A(that->Next != nullptr, "Bucket::Erase but element not contained");
LogMan::Throw::A(that->Next != nullptr, "Bucket::Erase but element not contained");
that = that->Next.get();
}
}
@@ -144,7 +143,7 @@ namespace {
}
else if (++i == Size) {
if (that->Next->Items[0] == 0) {
that->Next.reset();
that->Next.release();
foundThat->Items[foundI] = that->Items[Size-1];
that->Items[Size-1] = 0;
break;
@@ -263,7 +262,7 @@ namespace {
Graph->Nodes.resize(NodeCount);
Graph->VisitedNodePredecessors.clear();
Graph->AllocData.reset();
Graph->AllocData.reset((FEXCore::IR::RegisterAllocationData*)FEXCore::Allocator::malloc(FEXCore::IR::RegisterAllocationData::Size(NodeCount)));
Graph->AllocData.reset((FEXCore::IR::RegisterAllocationData*)malloc(FEXCore::IR::RegisterAllocationData::Size(NodeCount)));
memset(&Graph->AllocData->Map[0], INVALID_REGCLASS.Raw, NodeCount);
Graph->AllocData->MapCount = NodeCount;
Graph->AllocData->IsShared = false; // not shared by default
@@ -446,8 +445,8 @@ namespace FEXCore::IR {
}
void ConstrainedRAPass::AllocateRegisterSet(uint32_t RegisterCount, uint32_t ClassCount) {
LOGMAN_THROW_A(RegisterCount <= INVALID_REG, "Up to %d regs supported", INVALID_REG);
LOGMAN_THROW_A(ClassCount <= INVALID_CLASS, "Up to %d classes supported", INVALID_CLASS);
LogMan::Throw::A(RegisterCount <= INVALID_REG, "Up to %d regs supported", INVALID_REG);
LogMan::Throw::A(ClassCount <= INVALID_CLASS, "Up to %d classes supported", INVALID_CLASS);
Graph = AllocateRegisterGraph(ClassCount);
@@ -460,7 +459,7 @@ namespace FEXCore::IR {
}
void ConstrainedRAPass::AddRegisters(FEXCore::IR::RegisterClassType Class, uint32_t RegisterCount) {
LOGMAN_THROW_A(RegisterCount <= INVALID_REG, "Up to %d regs supported", INVALID_REG);
LogMan::Throw::A(RegisterCount <= INVALID_REG, "Up to %d regs supported", INVALID_REG);
AllocatePhysicalRegisters(Graph, Class, RegisterCount);
}
@@ -487,7 +486,7 @@ namespace FEXCore::IR {
auto Op = IROp->C<IROp_CodeBlock>();
LOGMAN_THROW_A(Op->Header.Op == OP_CODEBLOCK, "Block not defined by codeblock?");
LogMan::Throw::A(Op->Header.Op == OP_CODEBLOCK, "Block not defined by codeblock?");
LiveRange->Begin = std::min(LiveRange->Begin, Op->Begin.ID());
LiveRange->End = std::max(LiveRange->End, Op->Begin.ID());
@@ -515,7 +514,7 @@ namespace FEXCore::IR {
// If the destination hasn't yet been set then set it now
if (IROp->HasDest) {
LOGMAN_THROW_A(LiveRanges[Node].Begin == ~0U, "Node begin already defined?");
LogMan::Throw::A(LiveRanges[Node].Begin == ~0U, "Node begin already defined?");
LiveRanges[Node].Begin = Node;
// Default to ending right where after it starts
LiveRanges[Node].End = Node + 1;
@@ -547,7 +546,7 @@ namespace FEXCore::IR {
if (IR->GetOp<IROp_Header>(IROp->Args[i])->Op == OP_INLINEENTRYPOINTOFFSET) continue;
if (IR->GetOp<IROp_Header>(IROp->Args[i])->Op == OP_IRHEADER) continue;
uint32_t ArgNode = IROp->Args[i].ID();
LOGMAN_THROW_A(LiveRanges[ArgNode].Begin != ~0U, "%%ssa%d used by %%ssa%d before defined?", ArgNode, Node);
LogMan::Throw::A(LiveRanges[ArgNode].Begin != ~0U, "%%ssa%d used by %%ssa%d before defined?", ArgNode, Node);
auto ArgNodeBlockID = Graph->Nodes[ArgNode].Head.BlockID;
if (ArgNodeBlockID == BlockNodeID) {
@@ -601,7 +600,7 @@ namespace FEXCore::IR {
} else if (StaticClass == FPRFixedClass) {
return Size == 16;
} else {
LOGMAN_THROW_A(false, "Unexpected static class %d", StaticClass);
LogMan::Throw::A(false, "Unexpected static class %d", StaticClass);
}
return false; // Unknown
};
@@ -613,7 +612,7 @@ namespace FEXCore::IR {
} else if (StaticClass == FPRFixedClass) {
return (Size == 16 /*|| Size == 8 || Size == 4*/) && ((Offset & 15) == 0); // We need more meta info to support not-size-of-reg
} else {
LOGMAN_THROW_A(false, "Unexpected static class %d", StaticClass);
LogMan::Throw::A(false, "Unexpected static class %d", StaticClass);
}
return false; // Unknown
};
@@ -633,7 +632,7 @@ namespace FEXCore::IR {
auto reg = (Offset - beginFpr) / 16;
return PhysicalRegister(FPRFixedClass, reg);
} else {
LOGMAN_THROW_A(false, "Unexpected Offset %d", Offset);
LogMan::Throw::A(false, "Unexpected Offset %d", Offset);
return INVALID_REGCLASS;
}
};
@@ -657,7 +656,7 @@ namespace FEXCore::IR {
auto reg = (Offset - beginFpr) / 16;
return &StaticMaps[GprSize + reg];
} else {
LOGMAN_THROW_A(false, "Unexpected offset %d", Offset);
LogMan::Throw::A(false, "Unexpected offset %d", Offset);
return (LiveRange**)nullptr;
}
};
@@ -669,7 +668,7 @@ namespace FEXCore::IR {
} else if (PhyReg.Class == FPRFixedClass.Val) {
return &StaticMaps[GprSize + PhyReg.Reg];
} else {
LOGMAN_THROW_A(false, "Unexpected Class %d", PhyReg.Class);
LogMan::Throw::A(false, "Unexpected Class %d", PhyReg.Class);
return (LiveRange**)nullptr;
}
};
@@ -808,7 +807,7 @@ namespace FEXCore::IR {
for (auto [BlockNode, BlockHeader] : IR->GetBlocks()) {
auto BlockIROp = BlockHeader->CW<FEXCore::IR::IROp_CodeBlock>();
LOGMAN_THROW_A(BlockIROp->Header.Op == IR::OP_CODEBLOCK, "IR type failed to be a code block");
LogMan::Throw::A(BlockIROp->Header.Op == IR::OP_CODEBLOCK, "IR type failed to be a code block");
BlockInterferences *BlockInterferenceVector = &LocalBlockInterferences.try_emplace(IR->GetID(BlockNode)).first->second;
BlockInterferenceVector->reserve(BlockIROp->Last.ID() - BlockIROp->Begin.ID());
@@ -911,7 +910,7 @@ namespace FEXCore::IR {
SpanEnd.resize(NodeCount);
for (uint32_t i = 0; i < NodeCount; ++i) {
if (LiveRanges[i].Begin != ~0U) {
LOGMAN_THROW_A(LiveRanges[i].Begin < LiveRanges[i].End , "Span must Begin before Ending");
LogMan::Throw::A(LiveRanges[i].Begin < LiveRanges[i].End , "Span must Begin before Ending");
auto Class = GetClass(Graph->AllocData->Map[i]);
SpanStart[LiveRanges[i].Begin].Append(INFO_MAKE(i, Class));
@@ -939,7 +938,7 @@ namespace FEXCore::IR {
});
}
LOGMAN_THROW_A(Active.Items[0] == 0, "Interference bug");
LogMan::Throw::A(Active.Items[0] == 0, "Interference bug");
SpanStart.clear();
SpanEnd.clear();
}
@@ -958,7 +957,7 @@ namespace FEXCore::IR {
RegisterClass *RAClass = &Graph->Set.Classes[RegClass];
if (CurrentNode->Head.PhiPartner) {
LOGMAN_MSG_A("Phi nodes not supported");
LogMan::Msg::A("Phi nodes not supported");
#if 0
// In the case that we have a list of nodes that need the same register allocated we need to do something special
// We need to gather the data from the forward linked list and make sure they all match the virtual register
@@ -1157,7 +1156,7 @@ namespace FEXCore::IR {
// This would ensure something will spill earlier if its previous use and next use are farther away
auto InterferenceNodeNextUse = FindFirstUse(IREmit, InterferenceOrderedNode, NodeOpBeginIter, InterferenceNodeOpEndIter);
auto InterferenceNodePrevUse = FindLastUseBefore(IREmit, InterferenceOrderedNode, InterferenceNodeOpBeginIter, NodeOpBeginIter);
LOGMAN_THROW_A(InterferenceNodeNextUse != IR::NodeIterator::Invalid(), "Couldn't find next usage of op");
LogMan::Throw::A(InterferenceNodeNextUse != IR::NodeIterator::Invalid(), "Couldn't find next usage of op");
// If there is no use of the interference op prior to our op then it only has initial definition
if (InterferenceNodePrevUse == IR::NodeIterator::Invalid()) InterferenceNodePrevUse = InterferenceNodeOpBeginIter;
@@ -1323,7 +1322,7 @@ namespace FEXCore::IR {
LogMan::Msg::D("\tInt%d: %%ssa%d Remat: %d [%d, %d)", j++, InterferenceNode, InterferenceLiveRange->RematCost, InterferenceLiveRange->Begin, InterferenceLiveRange->End);
});
}
LOGMAN_THROW_A(InterferenceIdToSpill != 0, "Couldn't find Node to spill");
LogMan::Throw::A(InterferenceIdToSpill != 0, "Couldn't find Node to spill");
return InterferenceIdToSpill;
}
@@ -1358,7 +1357,7 @@ namespace FEXCore::IR {
auto LastCursor = IREmit->GetWriteCursor();
auto [CodeNode, IROp] = IR.at(SpillPointId)();
LOGMAN_THROW_A(IROp->HasDest, "Can't spill with no dest");
LogMan::Throw::A(IROp->HasDest, "Can't spill with no dest");
uint32_t Node = IR.GetID(CodeNode);
RegisterNode *CurrentNode = &Graph->Nodes[Node];
@@ -1382,7 +1381,7 @@ namespace FEXCore::IR {
// First op post Spill
auto NextIter = IR.at(CodeNode);
auto FirstUseLocation = FindFirstUse(IREmit, ConstantNode, NextIter, NodeIterator::Invalid());
LOGMAN_THROW_A(FirstUseLocation != IR::NodeIterator::Invalid(), "At %%ssa%d Spilling Op %%ssa%d but Failure to find op use", Node, InterferenceNode);
LogMan::Throw::A(FirstUseLocation != IR::NodeIterator::Invalid(), "At %%ssa%d Spilling Op %%ssa%d but Failure to find op use", Node, InterferenceNode);
if (FirstUseLocation != IR::NodeIterator::Invalid()) {
--FirstUseLocation;
auto [FirstUseOrderedNode, _] = FirstUseLocation();
@@ -1399,13 +1398,11 @@ namespace FEXCore::IR {
if (InterferenceNode != ~0U) {
FEXCore::IR::RegisterClassType InterferenceRegClass = FEXCore::IR::RegisterClassType{Graph->AllocData->Map[InterferenceNode].Class};
uint32_t SpillSlot = FindSpillSlot(InterferenceNode, InterferenceRegClass);
#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
RegisterNode *InterferenceRegisterNode = &Graph->Nodes[InterferenceNode];
LOGMAN_THROW_A(SpillSlot != ~0U, "Interference Node doesn't have a spill slot!");
//LOGMAN_THROW_A(InterferenceRegisterNode->Head.RegAndClass.Reg != INVALID_REG, "Interference node never assigned a register?");
LOGMAN_THROW_A(InterferenceRegClass != ~0U, "Interference node never assigned a register class?");
LOGMAN_THROW_A(InterferenceRegisterNode->Head.PhiPartner == nullptr, "We don't support spilling PHI nodes currently");
#endif
LogMan::Throw::A(SpillSlot != ~0U, "Interference Node doesn't have a spill slot!");
//LogMan::Throw::A(InterferenceRegisterNode->Head.RegAndClass.Reg != INVALID_REG, "Interference node never assigned a register?");
LogMan::Throw::A(InterferenceRegClass != ~0U, "Interference node never assigned a register class?");
LogMan::Throw::A(InterferenceRegisterNode->Head.PhiPartner == nullptr, "We don't support spilling PHI nodes currently");
// This is the op that we need to dump
auto [InterferenceOrderedNode, InterferenceIROp] = IR.at(InterferenceNode)();
@@ -1438,7 +1435,7 @@ namespace FEXCore::IR {
++FirstIter;
auto FirstUseLocation = FindFirstUse(IREmit, InterferenceOrderedNode, FirstIter, NodeIterator::Invalid());
LOGMAN_THROW_A(FirstUseLocation != NodeIterator::Invalid(), "At %%ssa%d Spilling Op %%ssa%d but Failure to find op use", Node, InterferenceNode);
LogMan::Throw::A(FirstUseLocation != NodeIterator::Invalid(), "At %%ssa%d Spilling Op %%ssa%d but Failure to find op use", Node, InterferenceNode);
if (FirstUseLocation != IR::NodeIterator::Invalid()) {
// We want to fill just before the first use
--FirstUseLocation;
@@ -1540,7 +1537,7 @@ namespace FEXCore::IR {
return Changed;
}
std::unique_ptr<FEXCore::IR::RegisterAllocationPass> CreateRegisterAllocationPass(FEXCore::IR::Pass* CompactionPass, bool OptimizeSRA) {
return std::make_unique<ConstrainedRAPass>(CompactionPass, OptimizeSRA);
FEXCore::IR::RegisterAllocationPass* CreateRegisterAllocationPass(FEXCore::IR::Pass* CompactionPass, bool OptimizeSRA) {
return new ConstrainedRAPass{CompactionPass, OptimizeSRA};
}
}
@@ -22,7 +22,7 @@ bool IsStaticAllocGpr(uint32_t Offset, RegisterClassType Class) {
if (Offset >= begin && Offset < end) {
auto reg = (Offset - begin) / 8;
LOGMAN_THROW_A(Class == IR::GPRClass, "unexpected Class %d", Class);
LogMan::Throw::A(Class == IR::GPRClass, "unexpected Class %d", Class);
rv = reg < 16; // 0..15 -> 16 in total
}
@@ -37,7 +37,7 @@ bool IsStaticAllocFpr(uint32_t Offset, RegisterClassType Class, bool AllowGpr) {
if (Offset >= begin && Offset < end) {
auto reg = (Offset - begin)/16;
LOGMAN_THROW_A(Class == IR::FPRClass || (AllowGpr && Class == IR::GPRClass), "unexpected Class %d, AllowGpr %d", Class, AllowGpr);
LogMan::Throw::A(Class == IR::FPRClass || (AllowGpr && Class == IR::GPRClass), "unexpected Class %d, AllowGpr %d", Class, AllowGpr);
rv = reg < 16; // 0..15 -> 16 in total
}
@@ -94,8 +94,8 @@ bool StaticRegisterAllocationPass::Run(IREmitter *IREmit) {
return true;
}
std::unique_ptr<FEXCore::IR::Pass> CreateStaticRegisterAllocationPass() {
return std::make_unique<StaticRegisterAllocationPass>();
FEXCore::IR::Pass* CreateStaticRegisterAllocationPass() {
return new StaticRegisterAllocationPass{};
}
}
@@ -44,11 +44,12 @@ bool SyscallOptimization::Run(IREmitter *IREmit) {
}
}
return Changed;
}
std::unique_ptr<FEXCore::IR::Pass> CreateSyscallOptimization() {
return std::make_unique<SyscallOptimization>();
FEXCore::IR::Pass* CreateSyscallOptimization() {
return new SyscallOptimization{};
}
}
@@ -8,9 +8,9 @@ $end_info$
#include "Interface/IR/PassManager.h"
#include "Interface/Core/OpcodeDispatcher.h"
#include <iostream>
#include <map>
#include <list>
#include <sstream>
#include <unordered_map>
namespace {
@@ -206,14 +206,14 @@ bool ValueDominanceValidation::Run(IREmitter *IREmit) {
Out << "Warnings:" << std::endl << Warnings.str() << std::endl;
}
LogMan::Msg::EFmt("{}", Out.str());
LogMan::Msg::E(Out.str().c_str());
}
return false;
}
std::unique_ptr<FEXCore::IR::Pass> CreateValueDominanceValidation() {
return std::make_unique<ValueDominanceValidation>();
FEXCore::IR::Pass* CreateValueDominanceValidation() {
return new ValueDominanceValidation{};
}
}
-118
View File
@@ -1,118 +0,0 @@
#include "Utils/Allocator/HostAllocator.h"
#include <FEXCore/Utils/Allocator.h>
#include <sys/mman.h>
#include <jemalloc/jemalloc.h>
#include <memory>
#include <malloc.h>
extern "C" {
extern void *__libc_malloc(size_t size);
extern void *__libc_realloc(void *ptr, size_t size);
extern void __libc_free(void *ptr);
typedef void* (*mmap_hook_type)(
void *addr, size_t length, int prot, int flags,
int fd, off_t offset);
typedef int (*munmap_hook_type)(void *addr, size_t length);
extern mmap_hook_type __mmap_hook;
extern munmap_hook_type __munmap_hook;
static FEXCore::Allocator::MALLOC_Hook global_malloc {::__libc_malloc};
static FEXCore::Allocator::REALLOC_Hook global_realloc {::__libc_realloc};
static FEXCore::Allocator::FREE_Hook global_free {::__libc_free};
// Override the global functions
FEX_DEFAULT_VISIBILITY void *malloc(size_t size) { return global_malloc(size); }
FEX_DEFAULT_VISIBILITY void *realloc(void *ptr, size_t size) { return global_realloc(ptr, size); }
FEX_DEFAULT_VISIBILITY void free(void *ptr) { return global_free(ptr); }
}
namespace FEXCore::Allocator {
MMAP_Hook mmap {::mmap};
MUNMAP_Hook munmap {::munmap};
MALLOC_Hook malloc {::__libc_malloc};
REALLOC_Hook realloc {::__libc_realloc};
FREE_Hook free {::__libc_free};
using GLIBC_MALLOC_Hook = void*(*)(size_t, const void *caller);
using GLIBC_REALLOC_Hook = void*(*)(void*, size_t, const void *caller);
using GLIBC_FREE_Hook = void(*)(void*, const void *caller);
std::unique_ptr<Alloc::HostAllocator> Alloc64{};
void *FEX_mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset) {
void *Result = Alloc64->Mmap(addr, length, prot, flags, fd, offset);
if (Result >= (void*)-4096) {
errno = -(uint64_t)Result;
return (void*)-1;
}
return Result;
}
int FEX_munmap(void *addr, size_t length) {
int Result = Alloc64->Munmap(addr, length);
if (Result != 0) {
errno = -Result;
return -1;
}
return Result;
}
void *FEX_malloc_hook(size_t size, const void *caller) {
return ::je_malloc(size);
}
void *FEX_realloc_hook(void *ptr, size_t size, const void *caller) {
return ::je_realloc(ptr, size);
}
void FEX_free_hook(void *ptr, const void *caller) {
return ::je_free(ptr);
}
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
void SetupHooks() {
Alloc64 = Alloc::OSAllocator::Create64BitAllocator();
__mmap_hook = FEX_mmap;
__munmap_hook = FEX_munmap;
FEXCore::Allocator::mmap = FEX_mmap;
FEXCore::Allocator::munmap = FEX_munmap;
FEXCore::Allocator::malloc = ::je_malloc;
FEXCore::Allocator::realloc = ::je_realloc;
FEXCore::Allocator::free = ::je_free;
global_malloc = ::je_malloc;
global_realloc = ::je_realloc;
global_free = ::je_free;
__malloc_hook = FEXCore::Allocator::FEX_malloc_hook;
__realloc_hook = FEXCore::Allocator::FEX_realloc_hook;
__free_hook = FEXCore::Allocator::FEX_free_hook;
}
void ClearHooks() {
__mmap_hook = ::mmap;
__munmap_hook = ::munmap;
FEXCore::Allocator::mmap = ::mmap;
FEXCore::Allocator::munmap = ::munmap;
FEXCore::Allocator::malloc = ::__libc_malloc;
FEXCore::Allocator::realloc = ::__libc_realloc;
FEXCore::Allocator::free = ::__libc_free;
global_malloc = ::__libc_malloc;
global_realloc = ::__libc_realloc;
global_free = ::__libc_free;
// Reset's glibc hooks
__malloc_hook = 0;
__realloc_hook = 0;
__free_hook = 0;
}
#pragma GCC diagnostic pop
}
extern "C" {
}
@@ -1,724 +0,0 @@
#include "Utils/Allocator/FlexBitSet.h"
#include "Utils/Allocator/HostAllocator.h"
#include "Utils/Allocator/IntrusiveArenaAllocator.h"
#include <FEXCore/Utils/LogManager.h>
#include <array>
#include <bit>
#include <bitset>
#include <cassert>
#include <cmath>
#include <cstdint>
#include <cstring>
#include <list>
#include <malloc.h>
#include <mutex>
#include <stdio.h>
#include <set>
#include <sys/mman.h>
#include <sys/utsname.h>
#include <sys/resource.h>
#include <syscall.h>
#include <vector>
static constexpr uint64_t PAGE_SHIFT = 12;
static constexpr uint64_t PAGE_MASK = (1 << PAGE_SHIFT) - 1;
namespace Alloc::OSAllocator {
class OSAllocator_64Bit final : public Alloc::HostAllocator {
public:
OSAllocator_64Bit();
virtual ~OSAllocator_64Bit();
void *AllocateSlab(size_t Size) override { return nullptr; }
void DeallocateSlab(void *Ptr, size_t Size) override {}
void *Mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset) override;
int Munmap(void *addr, size_t length) override;
private:
constexpr static uint64_t PAGE_SIZE = 4096;
// Upper bound is the maximum virtual address space of the host processor
uintptr_t UPPER_BOUND = (1ULL << 57);
// Lower bound is the starting of the range just past the lower 32bits
constexpr static uintptr_t LOWER_BOUND = 0x1'0000'0000ULL;
uintptr_t UPPER_BOUND_PAGE = UPPER_BOUND / PAGE_SIZE;
constexpr static uintptr_t LOWER_BOUND_PAGE = LOWER_BOUND / PAGE_SIZE;
struct ReservedVMARegion {
uintptr_t Base;
// Could be number of pages if we want to pack this in to 12 bytes
uint64_t RegionSize;
};
bool MergeReservedRegionIfPossible(ReservedVMARegion *Region, uintptr_t NextPtr, uint64_t NextSize) {
constexpr uint64_t MaxReservedRegionSize = 64ULL * 1024 * 1024 * 1024; // 64GB
uintptr_t RegionEnd = Region->Base + Region->RegionSize;
uint64_t NewRegionSize = Region->RegionSize + NextSize;
if (RegionEnd == NextPtr &&
NewRegionSize <= MaxReservedRegionSize) {
// Append the contiguous region
Region->RegionSize = NewRegionSize;
return true;
}
return false;
}
struct LiveVMARegion {
ReservedVMARegion *SlabInfo;
uint64_t FreeSpace{};
uint32_t LastPageAllocation{};
FlexBitSet<uint64_t> UsedPages;
// This returns the size of the LiveVMARegion in addition to the flex set that tracks the used data
// The LiveVMARegion lives at the start of the VMA region which means on initialization we need to set that
// tracked ranged as used immediately
static size_t GetSizeWithFlexSet(size_t Size) {
// One element per page
// 0x10'0000'0000 bytes
// 0x100'0000 Pages
// 1 bit per page for tracking means 0x20'0000 (Pages / 8) bytes of flex space
// Which is 2MB of tracking
uint64_t NumElements = (Size >> PAGE_SHIFT) * sizeof(uint64_t);
return sizeof(LiveVMARegion) + FlexBitSet<uint64_t>::Size(NumElements);
}
static void InitializeVMARegionUsed(LiveVMARegion *Region, size_t AdditionalSize) {
size_t SizeOfLiveRegion = AlignUp(LiveVMARegion::GetSizeWithFlexSet(Region->SlabInfo->RegionSize), PAGE_SIZE);
size_t SizePlusManagedData = SizeOfLiveRegion + AdditionalSize;
Region->FreeSpace = Region->SlabInfo->RegionSize - SizePlusManagedData;
size_t NumPages = SizePlusManagedData >> PAGE_SHIFT;
// Memset the full tracking to zero to state nothing used
Region->UsedPages.MemSet(Region->SlabInfo->RegionSize >> PAGE_SHIFT);
// Set our reserved pages
for (size_t i = 0; i < NumPages; ++i) {
// Set our used pages
Region->UsedPages.Set(i);
}
Region->LastPageAllocation = NumPages;
}
};
static_assert(std::is_trivially_copyable<LiveVMARegion>::value, "Needs to be trivially copyable");
static_assert(offsetof(LiveVMARegion, UsedPages) == sizeof(LiveVMARegion), "FlexBitSet needs to be at the end");
using ReservedRegionListType = std::pmr::list<ReservedVMARegion*>;
using LiveRegionListType = std::pmr::list<LiveVMARegion*>;
ReservedRegionListType *ReservedRegions{};
LiveRegionListType *LiveRegions{};
Alloc::ForwardOnlyIntrusiveArenaAllocator *ObjectAlloc{};
std::mutex AllocationMutex{};
void DetermineVASize();
LiveVMARegion *MakeRegionActive(ReservedRegionListType::iterator ReservedIterator, uint64_t UsedSize) {
ReservedVMARegion *ReservedRegion = *ReservedIterator;
ReservedRegions->erase(ReservedIterator);
// mprotect the new region we've allocated
size_t SizeOfLiveRegion = AlignUp(LiveVMARegion::GetSizeWithFlexSet(ReservedRegion->RegionSize), PAGE_SIZE);
size_t SizePlusManagedData = UsedSize + SizeOfLiveRegion;
mprotect(reinterpret_cast<void*>(ReservedRegion->Base), SizePlusManagedData, PROT_READ | PROT_WRITE);
LiveVMARegion *LiveRange = new (reinterpret_cast<void*>(ReservedRegion->Base)) LiveVMARegion();
// Copy over the reserved data
LiveRange->SlabInfo = ReservedRegion;
// Initialize VMA
LiveVMARegion::InitializeVMARegionUsed(LiveRange, UsedSize);
// Add to our active tracked ranges
auto LiveIter = LiveRegions->emplace_back(LiveRange);
return LiveIter;
}
// 32-bit old kernel workarounds
struct PtrCache {
uint32_t Ptr;
uint32_t Size;
};
PtrCache *Steal32BitIfOldKernel();
void Clear32BitOnOldKernel(PtrCache *Base);
};
void OSAllocator_64Bit::DetermineVASize() {
const std::vector<uintptr_t> TLBSizes = {{
1ULL << 57,
1ULL << 52,
1ULL << 48,
1ULL << 47,
1ULL << 42,
1ULL << 39,
1ULL << 36,
}};
for (auto Size : TLBSizes) {
// Just try allocating
// We can't actually determine VA size on ARM safely
auto Find = [](uintptr_t Size) -> bool {
for (int i = 0; i < 64; ++i) {
// Try grabbing a some of the top pages of the range
// x86 allocates some high pages in the top end
void *Ptr = ::mmap(reinterpret_cast<void*>(Size - PAGE_SIZE * i), PAGE_SIZE, PROT_NONE, MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (Ptr != (void*)~0ULL) {
::munmap(Ptr, PAGE_SIZE);
if (Ptr == (void*)(Size - PAGE_SIZE * i)) {
return true;
}
}
}
return false;
};
if (Find(Size)) {
UPPER_BOUND = Size;
UPPER_BOUND_PAGE = UPPER_BOUND / PAGE_SIZE;
break;
}
}
}
void *OSAllocator_64Bit::Mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset) {
if (addr != 0 &&
addr < reinterpret_cast<void*>(LOWER_BOUND)) {
// If we are asked to allocate something outside of the 64-bit space
// Then we need to just hand this to the OS
return ::mmap(addr, length, prot, flags, fd, offset);
}
uint64_t Addr = reinterpret_cast<uint64_t>(addr);
// Addr must be page aligned
if (Addr & PAGE_MASK) {
return reinterpret_cast<void*>(-EINVAL);
}
// If FD is provided then offset must also be page aligned
if (fd != -1 &&
offset & PAGE_MASK) {
return reinterpret_cast<void*>(-EINVAL);
}
// 64bit address overflow
if (Addr + length < Addr) {
return reinterpret_cast<void*>(-EOVERFLOW);
}
bool Fixed = (flags & MAP_FIXED) || (flags & MAP_FIXED_NOREPLACE);
length = AlignUp(length, PAGE_SIZE);
uint64_t AddrEnd = Addr + length;
size_t NumberOfPages = length / PAGE_SIZE;
// This needs a mutex to be thread safe
std::scoped_lock<std::mutex> lk{AllocationMutex};
uint64_t AllocatedOffset{};
LiveVMARegion *LiveRegion{};
if (Fixed || Addr != 0) {
// Check active slabs to see if we can fit this
for (auto it = LiveRegions->begin(); it != LiveRegions->end(); ++it) {
uintptr_t RegionBegin = (*it)->SlabInfo->Base;
uintptr_t RegionEnd = RegionBegin + (*it)->SlabInfo->RegionSize;
if (Addr >= RegionBegin &&
Addr < RegionEnd) {
LiveRegion = *it;
// Leave our loop
break;
}
}
// Couldn't find an active region that fit
// Check reserved regions
if (!LiveRegion) {
// Didn't have a slab that fit this range
// Check our reserved regions to see if we have one that fits
for (auto it = ReservedRegions->begin(); it != ReservedRegions->end(); ++it) {
ReservedVMARegion *ReservedRegion = *it;
uintptr_t RegionEnd = ReservedRegion->Base + ReservedRegion->RegionSize;
if (Addr >= ReservedRegion->Base &&
AddrEnd < RegionEnd) {
// Found one, let's make it active
LiveRegion = MakeRegionActive(it, 0);
break;
}
}
}
}
again:
auto CheckIfRangeFits = [&AllocatedOffset](LiveVMARegion *Region, uint64_t length, int prot, int flags, int fd, off_t offset, uint64_t StartingPosition = 0) -> std::pair<LiveVMARegion*, void*> {
uint64_t AllocatedPage{};
uint64_t NumberOfPages = length >> PAGE_SHIFT;
if (Region->FreeSpace >= length) {
uint64_t LastAllocation =
StartingPosition ?
(StartingPosition - Region->SlabInfo->Base) >> PAGE_SHIFT
: Region->LastPageAllocation;
size_t RegionNumberOfPages = Region->SlabInfo->RegionSize >> PAGE_SHIFT;
try_again:
for (size_t CurrentPage = LastAllocation;
CurrentPage < (RegionNumberOfPages - NumberOfPages);) {
// If we have enough free space, check if we have enough free pages that are contiguous
size_t Remaining = NumberOfPages;
assert((CurrentPage + Remaining - 1) < RegionNumberOfPages);
while (Remaining) {
if (Region->UsedPages[CurrentPage + Remaining - 1]) {
// Has an intersecting range
break;
}
--Remaining;
}
if (Remaining) {
// Didn't find a slab range
CurrentPage += Remaining;
}
else {
// We have a slab range
AllocatedPage = CurrentPage;
break;
}
}
if (!AllocatedPage && LastAllocation != 0) {
// Try again but starting from the beginning
LastAllocation = 0;
// Using goto so we don't have recursive mutex shenanigans
goto try_again;
}
if (AllocatedPage) {
AllocatedOffset = Region->SlabInfo->Base + AllocatedPage * PAGE_SIZE;
// We need to setup protections for this
void *MMapResult = ::mmap(reinterpret_cast<void*>(AllocatedOffset),
length,
prot,
(flags & ~MAP_FIXED_NOREPLACE) | MAP_FIXED,
fd, offset);
if (MMapResult == MAP_FAILED) {
return std::make_pair(Region, reinterpret_cast<void*>(-errno));
}
return std::make_pair(Region, MMapResult);
}
}
return std::make_pair(nullptr, nullptr);
};
if (Fixed) {
// Found a region let's allocate to it
if (LiveRegion) {
// Found a slab that fits this
if (flags & MAP_FIXED_NOREPLACE) {
auto Fits = CheckIfRangeFits(LiveRegion, length, prot, flags, fd, offset, Addr);
if (Fits.first && Fits.second == reinterpret_cast<void*>(Addr)) {
// We fit correctly
AllocatedOffset = Addr;
}
else {
// Intersected with something that already existed
return reinterpret_cast<void*>(-EEXIST);
}
}
else {
// We need to mmap the file to this location
void *MMapResult = ::mmap(reinterpret_cast<void*>(Addr),
length,
prot,
(flags & ~MAP_FIXED_NOREPLACE) | MAP_FIXED,
fd, offset);
if (MMapResult == MAP_FAILED) {
return reinterpret_cast<void*>(-errno);
}
AllocatedOffset = Addr;
}
// Fall through to live region tracking
}
}
else {
// Check our active slabs to see if we can fit the allocation
// Slightly different than fixed since it doesn't need exact placement
if (LiveRegion && Addr != 0) {
// We found a LiveRegion that could hold this address. Let's try to place it
// Check if this area is free
auto Fits = CheckIfRangeFits(LiveRegion, length, prot, flags, fd, offset, Addr);
if (Fits.first && Fits.second == reinterpret_cast<void*>(Addr)) {
// We fit correctly
AllocatedOffset = Addr;
}
else {
// Couldn't fit
// We can continue past this point still
LiveRegion = nullptr;
}
}
if (!LiveRegion) {
for (auto it = LiveRegions->begin(); it != LiveRegions->end(); ++it) {
auto Fits = CheckIfRangeFits(*it, length, prot, flags, fd, offset);
if (Fits.first && Fits.second == reinterpret_cast<void*>(AllocatedOffset)) {
// We fit correctly
LiveRegion = Fits.first;
break;
}
// Couldn't fit but mmap gave us an error
if (!Fits.first && Fits.second) {
return Fits.second;
}
// nullptr on both means no error and couldn't fit
}
}
if (!LiveRegion) {
// Couldn't find a fit in the live regions
// Allocate a new reserved region
size_t lengthOfLiveRegion = AlignUp(LiveVMARegion::GetSizeWithFlexSet(length), PAGE_SIZE);
size_t lengthPlusManagedData = length + lengthOfLiveRegion;
for (auto it = ReservedRegions->begin(); it != ReservedRegions->end(); ++it) {
if ((*it)->RegionSize >= lengthPlusManagedData) {
MakeRegionActive(it, 0);
goto again;
}
}
}
}
if (LiveRegion) {
// Mark the pages as used
uintptr_t RegionBegin = LiveRegion->SlabInfo->Base;
uintptr_t MappedBegin = (AllocatedOffset - RegionBegin) >> PAGE_SHIFT;
for (size_t i = 0; i < NumberOfPages; ++i) {
LiveRegion->UsedPages.Set(MappedBegin + i);
}
// Change our last allocation region
LiveRegion->LastPageAllocation = MappedBegin + NumberOfPages;
LiveRegion->FreeSpace -= length;
}
if (!AllocatedOffset) {
AllocatedOffset = -ENOMEM;
}
return reinterpret_cast<void*>(AllocatedOffset);
}
int OSAllocator_64Bit::Munmap(void *addr, size_t length) {
if (addr < reinterpret_cast<void*>(LOWER_BOUND)) {
// If we are asked to allocate something outside of the 64-bit space
// Then we need to just hand this to the OS
return ::munmap(addr, length);
}
uint64_t Addr = reinterpret_cast<uint64_t>(addr);
if (Addr & PAGE_MASK) {
return -EINVAL;
}
if (length & PAGE_MASK) {
return -EINVAL;
}
if (Addr + length < Addr) {
return -EOVERFLOW;
}
// This needs a mutex to be thread safe
std::scoped_lock<std::mutex> lk{AllocationMutex};
length = AlignUp(length, PAGE_SIZE);
uintptr_t PtrBegin = reinterpret_cast<uintptr_t>(addr);
uintptr_t PtrEnd = PtrBegin + length;
// Walk all of the live ranges and find this slab then delete it
for (auto it = LiveRegions->begin(); it != LiveRegions->end(); ++it) {
uintptr_t RegionBegin = (*it)->SlabInfo->Base;
uintptr_t RegionEnd = RegionBegin + (*it)->SlabInfo->RegionSize;
if (RegionBegin <= PtrBegin &&
RegionEnd > PtrEnd) {
// Live region fully encompasses slab range
uint64_t FreedPages{};
uint64_t SlabPageBegin = (PtrBegin - RegionBegin) >> PAGE_SHIFT;
uint64_t PagesToFree = length >> PAGE_SHIFT;
for (size_t i = 0; i < PagesToFree; ++i) {
FreedPages += (*it)->UsedPages.TestAndClear(SlabPageBegin + i) ? 1 : 0;
}
if (FreedPages != 0)
{
// If we were contiuous freeing then make sure to give back the physical address space
// If the region was locked then madvise won't remove the physical backing
// This woul be a bug in the frontend application
// So be careful with mlock/munlock
::madvise(addr, length, MADV_DONTNEED);
::mmap(addr, length, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0);
}
(*it)->FreeSpace += FreedPages * 4096;
// XXX: Move region back to reserved list
return 0;
}
}
// If it didn't match at all then no error
return 0;
}
OSAllocator_64Bit::PtrCache *OSAllocator_64Bit::Steal32BitIfOldKernel() {
// First calculate kernel version
struct utsname buf{};
if (uname(&buf) == -1) {
return nullptr;
}
int32_t Major{};
int32_t Minor{};
int32_t Patch{};
char Tmp{};
std::istringstream ss{buf.release};
ss >> Major;
ss.read(&Tmp, 1);
ss >> Minor;
ss.read(&Tmp, 1);
ss >> Patch;
ss.read(&Tmp, 1);
uint32_t Version = (Major << 24) | (Minor << 16) | Patch;
if (Version >= ((4 << 24) | (17 << 16) | 0)) {
// If the kernel is >= 4.17 then it supports MAP_FIXED_NOREPLACE
return nullptr;
}
OSAllocator_64Bit::PtrCache *Cache{};
uint32_t CacheSize{};
uint32_t CurrentCacheOffset = 0;
constexpr std::array<size_t, 6> ReservedVMARegionSizes = {{
1ULL * 1024 * 1024 * 1024, // 1GB
512ULL * 1024 * 1024, // 512MB
128ULL * 1024 * 1024, // 128MB
32ULL * 1024 * 1024, // 32MB
1ULL * 1024 * 1024, // 1MB
4096ULL // One page
}};
constexpr size_t AllocationSizeMaxIndex = ReservedVMARegionSizes.size() - 1;
uint64_t CurrentSizeIndex = 0;
constexpr size_t LOWER_BOUND_32 = 0x1'0000;
constexpr size_t UPPER_BOUND_32 = LOWER_BOUND;
for (size_t MemoryOffset = LOWER_BOUND_32; MemoryOffset < UPPER_BOUND_32;) {
size_t AllocationSize = ReservedVMARegionSizes[CurrentSizeIndex];
size_t MemoryOffsetUpper = MemoryOffset + AllocationSize;
// If we would go above the upper bound on size then try the next size
if (MemoryOffsetUpper > UPPER_BOUND_32) {
++CurrentSizeIndex;
continue;
}
void *Ptr = ::mmap(reinterpret_cast<void*>(MemoryOffset), AllocationSize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0);
// If we managed to allocate and not get the address we want then unmap it
// This happens with kernels older than 4.17
if (reinterpret_cast<uintptr_t>(Ptr) + AllocationSize > UPPER_BOUND_32) {
munmap(Ptr, AllocationSize);
Ptr = reinterpret_cast<void*>(~0ULL);
}
// If we failed to allocate and we are on the smallest allocation size then just continue onward
// This page was unmappable
if (reinterpret_cast<uintptr_t>(Ptr) == ~0ULL && CurrentSizeIndex == AllocationSizeMaxIndex) {
CurrentSizeIndex = 0;
MemoryOffset += AllocationSize;
continue;
}
// Congratulations we were able to map this bit
// Reset and claim it was available
if (reinterpret_cast<uintptr_t>(Ptr) != ~0ULL) {
if (!Cache) {
Cache = reinterpret_cast<OSAllocator_64Bit::PtrCache *>(Ptr);
CacheSize = AllocationSize;
}
else {
Cache[CurrentCacheOffset] = {
.Ptr = static_cast<uint32_t>(reinterpret_cast<uint64_t>(Ptr)),
.Size = static_cast<uint32_t>(AllocationSize)
};
++CurrentCacheOffset;
}
CurrentSizeIndex = 0;
MemoryOffset += AllocationSize;
continue;
}
// Couldn't allocate at this size
// Increase and continue
++CurrentSizeIndex;
}
Cache[CurrentCacheOffset] = {
.Ptr = static_cast<uint32_t>(reinterpret_cast<uint64_t>(Cache)),
.Size = CacheSize,
};
return Cache;
}
void OSAllocator_64Bit::Clear32BitOnOldKernel(OSAllocator_64Bit::PtrCache *Base) {
if (Base == nullptr) {
return;
}
for (size_t i = 0;; ++i) {
void *Ptr = reinterpret_cast<void*>(Base[i].Ptr);
size_t Size = Base[i].Size;
munmap(Ptr, Size);
if (Ptr == Base) {
break;
}
}
}
OSAllocator_64Bit::OSAllocator_64Bit() {
malloc_trim(0);
DetermineVASize();
auto ArrayPtr = Steal32BitIfOldKernel();
// On allocation try and steal the entire upper 64bits of address space for mapping
constexpr std::array<size_t, 8> ReservedVMARegionSizes = {{
// Anything larger than 64GB fails out
64ULL * 1024 * 1024 * 1024, // 64GB
32ULL * 1024 * 1024 * 1024, // 32GB
16ULL * 1024 * 1024 * 1024, // 16GB
4ULL * 1024 * 1024 * 1024, // 4GB
1ULL * 1024 * 1024 * 1024, // 1GB
512ULL * 1024 * 1024, // 512MB
128ULL * 1024 * 1024, // 128MB
4096ULL // One page
}};
constexpr size_t AllocationSizeMaxIndex = ReservedVMARegionSizes.size() - 1;
// Have the first region only be 4GB VMA
// Avoids conflicts with some tests
uint64_t CurrentSizeIndex = 3;
ReservedVMARegion *PrevReserved{};
for (size_t MemoryOffset = LOWER_BOUND; MemoryOffset < UPPER_BOUND;) {
size_t AllocationSize = ReservedVMARegionSizes[CurrentSizeIndex];
size_t MemoryOffsetUpper = MemoryOffset + AllocationSize;
// If we would go above the upper bound on size then try the next size
if (MemoryOffsetUpper > UPPER_BOUND) {
++CurrentSizeIndex;
continue;
}
void *Ptr = ::mmap(reinterpret_cast<void*>(MemoryOffset), AllocationSize, PROT_NONE, MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0);
// If we managed to allocate and not get the address we want then unmap it
// This happens with kernels older than 4.17
if (reinterpret_cast<uintptr_t>(Ptr) != MemoryOffset &&
reinterpret_cast<uintptr_t>(Ptr) < LOWER_BOUND) {
munmap(Ptr, AllocationSize);
Ptr = reinterpret_cast<void*>(~0ULL);
}
// If we failed to allocate and we are on the smallest allocation size then just continue onward
// This page was unmappable
if (reinterpret_cast<uintptr_t>(Ptr) == ~0ULL && CurrentSizeIndex == AllocationSizeMaxIndex) {
CurrentSizeIndex = 0;
MemoryOffset += AllocationSize;
continue;
}
// Congratulations we were able to map this bit
// Reset and claim it was available
if (reinterpret_cast<uintptr_t>(Ptr) != ~0ULL) {
if (!ObjectAlloc) {
// Steal the first allocation for an intrusive allocator
// Will be mprotected correctly already
int Result = mprotect(Ptr, AllocationSize, PROT_READ | PROT_WRITE);
LogMan::Throw::A(Result == 0, "mprotect(%p, 0x%lx) -> %d (%s)", Ptr, AllocationSize, Result, strerror(errno));
ObjectAlloc = new (Ptr) Alloc::ForwardOnlyIntrusiveArenaAllocator(Ptr, AllocationSize);
ReservedRegions = ObjectAlloc->new_construct(ReservedRegions, ObjectAlloc);
LiveRegions = ObjectAlloc->new_construct(LiveRegions, ObjectAlloc);
}
else {
// If the allocation size is large than a page, then try allowing it to be a huge page
// This enables the kernel to use transparent large pages in the allocator which can reduce memory pressure
// Considering we are allocating the entire VA space, this is a good thing
// If MADV_HUGEPAGE isn't support then this will fail harmlessly
if (AllocationSize > 4096) {
::madvise(Ptr, AllocationSize, MADV_HUGEPAGE);
}
bool Merged = false;
if (PrevReserved) {
Merged = MergeReservedRegionIfPossible(PrevReserved, reinterpret_cast<uint64_t>(Ptr), AllocationSize);
}
if (!Merged) {
ReservedVMARegion *Region = ObjectAlloc->new_construct<ReservedVMARegion>();
Region->Base = reinterpret_cast<uint64_t>(Ptr);
Region->RegionSize = AllocationSize;
ReservedRegions->emplace_back(Region);
PrevReserved = Region;
}
}
CurrentSizeIndex = 0;
MemoryOffset += AllocationSize;
continue;
}
// Couldn't allocate at this size
// Increase and continue
++CurrentSizeIndex;
}
Clear32BitOnOldKernel(ArrayPtr);
}
OSAllocator_64Bit::~OSAllocator_64Bit() {
// For consistency, pull the mutex
std::scoped_lock<std::mutex> lk{AllocationMutex};
// Walk the pages and deallocate
// First walk the live regions
for (auto it = LiveRegions->begin(); it != LiveRegions->end(); ++it) {
::munmap(reinterpret_cast<void*>((*it)->SlabInfo->Base), (*it)->SlabInfo->RegionSize);
}
// Now walk the reserved regions
for (auto it = ReservedRegions->begin(); it != ReservedRegions->end(); ++it) {
::munmap(reinterpret_cast<void*>((*it)->Base), (*it)->RegionSize);
}
}
std::unique_ptr<Alloc::HostAllocator> Create64BitAllocator() {
return std::make_unique<OSAllocator_64Bit>();
}
}
-50
View File
@@ -1,50 +0,0 @@
#pragma once
#include "HostAllocator.h"
#include <cstddef>
#include <cstdint>
#include <cstring>
#include <type_traits>
template<typename T>
struct FlexBitSet final {
using ElementType = T;
constexpr static size_t MinimumSize = sizeof(ElementType);
constexpr static size_t MinimumSizeBits = sizeof(ElementType) * 8;
T Memory[];
bool Get(T Element) {
return (Memory[Element / MinimumSizeBits] & (1ULL << (Element % MinimumSizeBits))) != 0;
}
bool TestAndClear(T Element) {
bool Value = Get(Element);
Memory[Element / MinimumSizeBits] &= ~(1ULL << (Element % MinimumSizeBits));
return Value;
}
void Set(T Element) {
Memory[Element / MinimumSizeBits] |= (1ULL << (Element % MinimumSizeBits));
}
void Clear(T Element) {
Memory[Element / MinimumSizeBits] &= ~(1ULL << (Element % MinimumSizeBits));
}
void MemClear(size_t Elements) {
memset(Memory, 0, Alloc::AlignUp(Elements / MinimumSizeBits, MinimumSizeBits));
}
void MemSet(size_t Elements) {
memset(Memory, 0xFF, Alloc::AlignUp(Elements / MinimumSizeBits, MinimumSizeBits));
}
// This very explicitly doesn't let you take an address
// Is only a getter
bool operator[](T Element) {
return Get(Element);
}
static size_t Size(T Elements) {
return Alloc::AlignUp(Elements / MinimumSizeBits, MinimumSizeBits);
}
};
static_assert(sizeof(FlexBitSet<uint64_t>) == 0, "This needs to be a flex member");
static_assert(std::is_trivially_copyable<FlexBitSet<uint64_t>>::value, "Needsto be trivially copyable");
-45
View File
@@ -1,45 +0,0 @@
#pragma once
#include <cstddef>
#include <cstdint>
#include <memory>
#include <sys/types.h>
constexpr static uint64_t PAGE_SIZE = 4096;
namespace Alloc {
static inline uint64_t AlignUp(uint64_t value, uint64_t size) {
return value + (size - value % size) % size;
};
// HostAllocator is just a page pased slab allocator
// Similar to mmap and munmap only mapping at the page level
class HostAllocator {
public:
HostAllocator() = default;
virtual ~HostAllocator() = default;
virtual void *AllocateSlab(size_t Size) = 0;
virtual void DeallocateSlab(void *Ptr, size_t Size) = 0;
virtual void *Mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset) { return nullptr; }
virtual int Munmap(void *addr, size_t length) { return -1; }
};
class GlobalAllocator {
public:
HostAllocator *Alloc{};
GlobalAllocator(HostAllocator *_Alloc)
: Alloc {_Alloc} {}
virtual ~GlobalAllocator() = default;
virtual void *malloc(size_t Size) = 0;
virtual void *calloc(size_t num, size_t size) = 0;
virtual void *realloc(void *ptr, size_t size) = 0;
virtual void *memalign(size_t alignment, size_t size) = 0;
virtual void free(void *ptr) = 0;
};
}
namespace Alloc::OSAllocator {
std::unique_ptr<Alloc::HostAllocator> Create64BitAllocator();
}
@@ -1,186 +0,0 @@
#pragma once
#include "FlexBitSet.h"
#include "HostAllocator.h"
#include <bitset>
#include <cstddef>
#include <memory_resource>
#include <mutex>
#include <vector>
namespace Alloc {
class ForwardOnlyIntrusiveArenaAllocator final : public std::pmr::memory_resource {
public:
ForwardOnlyIntrusiveArenaAllocator(void* Ptr, size_t _Size)
: Begin {reinterpret_cast<uintptr_t>(Ptr)}
, Size {_Size} {
LastAllocation = sizeof(ForwardOnlyIntrusiveArenaAllocator);
}
~ForwardOnlyIntrusiveArenaAllocator() = default;
template<class U, class... Args>
U *new_construct(Args&&... args) {
void *Ptr = do_allocate(sizeof(U), std::alignment_of<U>::value);
return new (Ptr) U(args...);
}
template<class U, class... Args>
U *new_construct(U *Class, Args&&... args) {
void *Ptr = do_allocate(sizeof(U), std::alignment_of<U>::value);
return new (Ptr) U(args...);
}
size_t AmountAllocated() const { return LastAllocation; }
private:
void *do_allocate(std::size_t bytes, std::size_t alignment) override {
size_t PreviousAligned = Alloc::AlignUp(LastAllocation, alignment);
size_t NewOffset = PreviousAligned + bytes;
if (NewOffset > Size) {
return nullptr;
}
LastAllocation = NewOffset;
return reinterpret_cast<void*>(Begin + PreviousAligned);
}
void do_deallocate(void*, std::size_t, std::size_t) override {
// Do nothing
}
bool do_is_equal(const std::pmr::memory_resource& other) const noexcept override {
// Only if the allocator pointers are the same are they equal
if (this == &other) {
return true;
}
// We don't share state with another allocator so we can't share anything
return false;
}
uintptr_t Begin;
size_t Size;
size_t LastAllocation{};
};
class IntrusiveArenaAllocator final : public std::pmr::memory_resource {
public:
IntrusiveArenaAllocator(void* Ptr, size_t _Size)
: Begin {reinterpret_cast<uintptr_t>(Ptr)}
, Size {_Size} {
uint64_t NumberOfPages = _Size / PAGE_SIZE;
uint64_t UsedBits = Alloc::AlignUp(sizeof(IntrusiveArenaAllocator) +
Size / PAGE_SIZE / 8, PAGE_SIZE);
for (size_t i = 0; i < UsedBits; ++i) {
UsedPages.Set(i);
}
FreePages = NumberOfPages - UsedBits;
}
template<class U, class... Args>
U *new_construct(Args&&... args) {
void *Ptr = do_allocate(sizeof(U), std::alignment_of<U>::value);
return new (Ptr) U(args...);
}
template<class U, class... Args>
U *new_construct(U *Class, Args&&... args) {
void *Ptr = do_allocate(sizeof(U), std::alignment_of<U>::value);
return new (Ptr) U(args...);
}
uintptr_t GetSlabBase() const { return Begin; }
uint64_t GetSlabSize() const { return Size; }
uint64_t GetFreePages() const { return FreePages; }
private:
void *do_allocate(std::size_t bytes, std::size_t alignment) override {
std::scoped_lock<std::mutex> lk{AllocationMutex};
size_t NumberPages = Alloc::AlignUp(bytes, PAGE_SIZE) / PAGE_SIZE;
uintptr_t AllocatedOffset{};
try_again:
for (uintptr_t CurrentPage = LastAllocatedPageOffset; CurrentPage <= (Size - NumberPages);) {
size_t Remaining = NumberPages;
while (Remaining) {
if (UsedPages[CurrentPage + Remaining - 1]) {
// Has an intersecting range
break;
}
--Remaining;
}
if (Remaining) {
// Didn't find an allocation range
CurrentPage += Remaining;
}
else {
// We have a range to allocate
AllocatedOffset = CurrentPage;
break;
}
}
if (!AllocatedOffset && LastAllocatedPageOffset != 0) {
// Try again but starting from the beginning
LastAllocatedPageOffset = 0;
// Using goto so we don't have recursive mutex shenanigans
goto try_again;
}
// Allocated offset must be valid or zero at this point
if (AllocatedOffset) {
// Map the range as no longer available
for (size_t i = 0; i < NumberPages; ++i) {
UsedPages.Set(AllocatedOffset + i);
}
LastAllocatedPageOffset = AllocatedOffset + NumberPages;
// Now convert this base page to a pointer and return it
return reinterpret_cast<void*>(Begin + AllocatedOffset * PAGE_SIZE);
}
return nullptr;
}
void do_deallocate(void* p, std::size_t bytes, std::size_t alignment) override {
std::scoped_lock<std::mutex> lk{AllocationMutex};
uintptr_t PageOffset = (reinterpret_cast<uintptr_t>(p) - Begin) / PAGE_SIZE;
size_t NumPages = AlignUp(bytes, PAGE_SIZE) / PAGE_SIZE;
// Walk the allocation list and deallocate
uint64_t FreedPages{};
for (size_t i = 0; i < NumPages; ++i) {
FreedPages += UsedPages.TestAndClear(PageOffset + i) ? 1 : 0;
}
FreePages += FreedPages;
}
bool do_is_equal(const std::pmr::memory_resource& other) const noexcept override {
// Only if the allocator pointers are the same are they equal
if (this == &other) {
return true;
}
// We don't share state with another allocator so we can't share anything
return false;
}
uintptr_t Begin;
size_t Size;
uint64_t FreePages{};
size_t LastAllocatedPageOffset{};
std::mutex AllocationMutex{};
// For up to 64GB regions this will require up to 2MB tracking
// Needs to be the last element
FlexBitSet<uint64_t> UsedPages;
};
}
@@ -5,8 +5,8 @@ desc: Loads and parses an elf to memory. Also handles some loading & logic.
$end_info$
*/
#include "Common/MathUtils.h"
#include "Linux/Utils/ELFContainer.h"
#include <FEXCore/Utils/Common/MathUtils.h>
#include <FEXCore/Utils/ELFLoader.h>
#include <FEXCore/Utils/LogManager.h>
#include <cstring>
#include <elf.h>
@@ -134,7 +134,7 @@ ELFContainer::ELFContainer(std::string const &Filename, std::string const &RootF
//PrintInitArray();
//PrintDynamicTable();
//LOGMAN_THROW_A(InterpreterHeader == nullptr, "Can only handle static programs");
//LogMan::Throw::A(InterpreterHeader == nullptr, "Can only handle static programs");
}
ELFContainer::~ELFContainer() {
@@ -196,8 +196,8 @@ bool ELFContainer::LoadELF_32() {
memcpy(&Header, reinterpret_cast<Elf32_Ehdr *>(&RawFile.at(0)),
sizeof(Elf32_Ehdr));
LOGMAN_THROW_A(Header._32.e_phentsize == sizeof(Elf32_Phdr), "PH Entry size wasn't correct size");
LOGMAN_THROW_A(Header._32.e_shentsize == sizeof(Elf32_Shdr), "PH Entry size wasn't correct size");
LogMan::Throw::A(Header._32.e_phentsize == sizeof(Elf32_Phdr), "PH Entry size wasn't correct size");
LogMan::Throw::A(Header._32.e_shentsize == sizeof(Elf32_Shdr), "PH Entry size wasn't correct size");
if (Header._32.e_machine != EM_386) {
LogMan::Msg::D("32bit ELF wasn't x86 based");
@@ -237,8 +237,8 @@ bool ELFContainer::LoadELF_64() {
memcpy(&Header, reinterpret_cast<Elf64_Ehdr *>(&RawFile.at(0)),
sizeof(Elf64_Ehdr));
LOGMAN_THROW_A(Header._64.e_phentsize == 56, "PH Entry size wasn't 56");
LOGMAN_THROW_A(Header._64.e_shentsize == 64, "PH Entry size wasn't 64");
LogMan::Throw::A(Header._64.e_phentsize == 56, "PH Entry size wasn't 56");
LogMan::Throw::A(Header._64.e_shentsize == 64, "PH Entry size wasn't 64");
if (Header._64.e_machine != EM_X86_64) {
LogMan::Msg::D("64bit ELF wasn't x86-64 based");
@@ -408,9 +408,9 @@ void ELFContainer::CalculateSymbols() {
uint64_t NumSymTabSymbols = 0;
uint64_t NumDynSymSymbols = 0;
if (SymTabHeader) {
LOGMAN_THROW_A(SymTabHeader->sh_link < SectionHeaders.size(),
LogMan::Throw::A(SymTabHeader->sh_link < SectionHeaders.size(),
"Symbol table string table section is wrong");
LOGMAN_THROW_A(SymTabHeader->sh_entsize == sizeof(Elf32_Sym),
LogMan::Throw::A(SymTabHeader->sh_entsize == sizeof(Elf32_Sym),
"Entry size doesn't match symbol entry");
StringTableHeader = SectionHeaders.at(SymTabHeader->sh_link)._32;
@@ -419,9 +419,9 @@ void ELFContainer::CalculateSymbols() {
}
if (DynSymTabHeader) {
LOGMAN_THROW_A(DynSymTabHeader->sh_link < SectionHeaders.size(),
LogMan::Throw::A(DynSymTabHeader->sh_link < SectionHeaders.size(),
"Symbol table string table section is wrong");
LOGMAN_THROW_A(DynSymTabHeader->sh_entsize == sizeof(Elf32_Sym),
LogMan::Throw::A(DynSymTabHeader->sh_entsize == sizeof(Elf32_Sym),
"Entry size doesn't match symbol entry");
DynStringTableHeader = SectionHeaders.at(DynSymTabHeader->sh_link)._32;
@@ -477,36 +477,6 @@ void ELFContainer::CalculateSymbols() {
}
}
}
Elf32_Shdr const *StrHeader = SectionHeaders.at(Header._32.e_shstrndx)._32;
char const *SHStrings = &RawFile.at(StrHeader->sh_offset);
for (uint32_t i = 0; i < SectionHeaders.size(); ++i) {
Elf32_Shdr const *hdr = SectionHeaders.at(i)._32;
if (strcmp(&SHStrings[hdr->sh_name], ".eh_frame_hdr") == 0) {
auto eh_frame_hdr = &RawFile.at(hdr->sh_offset);
// we only handle this specific unwind table encoding
if (eh_frame_hdr[0] == 1 && eh_frame_hdr[1] == 0x1B && eh_frame_hdr[2] == 0x3 && eh_frame_hdr[3] == 0x3b) {
// ptr enc : 4 bytes, signed, pcrel
// fde count : 4 bytes udata
// table enc : 4 bytes, signed, datarel
int fde_count = *(int*)(eh_frame_hdr + 8);
UnwindEntries.clear();
UnwindEntries.reserve(fde_count);
struct entry {
int32_t pc;
int32_t fde;
};
entry *Table = (entry*)(eh_frame_hdr+12);
for (int f = 0; f < fde_count; f++) {
uintptr_t Entry = (uintptr_t)(Table[f].pc + hdr->sh_offset);
UnwindEntries.push_back(Entry);
}
}
break;
}
}
}
else {
Elf64_Shdr const *SymTabHeader{nullptr};
@@ -541,9 +511,9 @@ void ELFContainer::CalculateSymbols() {
uint64_t NumSymTabSymbols = 0;
uint64_t NumDynSymSymbols = 0;
if (SymTabHeader) {
LOGMAN_THROW_A(SymTabHeader->sh_link < SectionHeaders.size(),
LogMan::Throw::A(SymTabHeader->sh_link < SectionHeaders.size(),
"Symbol table string table section is wrong");
LOGMAN_THROW_A(SymTabHeader->sh_entsize == sizeof(Elf64_Sym),
LogMan::Throw::A(SymTabHeader->sh_entsize == sizeof(Elf64_Sym),
"Entry size doesn't match symbol entry");
StringTableHeader = SectionHeaders.at(SymTabHeader->sh_link)._64;
@@ -552,9 +522,9 @@ void ELFContainer::CalculateSymbols() {
}
if (DynSymTabHeader) {
LOGMAN_THROW_A(DynSymTabHeader->sh_link < SectionHeaders.size(),
LogMan::Throw::A(DynSymTabHeader->sh_link < SectionHeaders.size(),
"Symbol table string table section is wrong");
LOGMAN_THROW_A(DynSymTabHeader->sh_entsize == sizeof(Elf64_Sym),
LogMan::Throw::A(DynSymTabHeader->sh_entsize == sizeof(Elf64_Sym),
"Entry size doesn't match symbol entry");
DynStringTableHeader = SectionHeaders.at(DynSymTabHeader->sh_link)._64;
@@ -610,36 +580,6 @@ void ELFContainer::CalculateSymbols() {
}
}
}
Elf64_Shdr const *StrHeader = SectionHeaders.at(Header._64.e_shstrndx)._64;
char const *SHStrings = &RawFile.at(StrHeader->sh_offset);
for (uint32_t i = 0; i < SectionHeaders.size(); ++i) {
Elf64_Shdr const *hdr = SectionHeaders.at(i)._64;
if (strcmp(&SHStrings[hdr->sh_name], ".eh_frame_hdr") == 0) {
auto eh_frame_hdr = &RawFile.at(hdr->sh_offset);
// we only handle this specific unwind table encoding
if (eh_frame_hdr[0] == 1 && eh_frame_hdr[1] == 0x1B && eh_frame_hdr[2] == 0x3 && eh_frame_hdr[3] == 0x3b) {
// ptr enc : 4 bytes, signed, pcrel
// fde count : 4 bytes udata
// table enc : 4 bytes, signed, datarel
int fde_count = *(int*)(eh_frame_hdr + 8);
UnwindEntries.clear();
UnwindEntries.reserve(fde_count);
struct entry {
int32_t pc;
int32_t fde;
};
entry *Table = (entry*)(eh_frame_hdr+12);
for (int f = 0; f < fde_count; f++) {
uintptr_t Entry = (uintptr_t)(Table[f].pc + hdr->sh_offset);
UnwindEntries.push_back(Entry);
}
}
break;
}
}
}
}
@@ -689,11 +629,6 @@ void ELFContainer::AddSymbols(SymbolAdder Adder) {
}
}
}
void ELFContainer::AddUnwindEntries(UnwindAdder Adder) {
for (auto Entry : UnwindEntries) {
Adder(Entry);
}
}
void ELFContainer::PrintHeader() const {
if (Mode == MODE_32BIT) {
@@ -730,7 +665,7 @@ void ELFContainer::PrintHeader() const {
void ELFContainer::PrintSectionHeaders() const {
if (Mode == MODE_32BIT) {
LOGMAN_THROW_A(Header._32.e_shstrndx < SectionHeaders.size(),
LogMan::Throw::A(Header._32.e_shstrndx < SectionHeaders.size(),
"String index section is wrong index!");
Elf32_Shdr const *StrHeader = SectionHeaders.at(Header._32.e_shstrndx)._32;
char const *SHStrings = &RawFile.at(StrHeader->sh_offset);
@@ -750,7 +685,7 @@ void ELFContainer::PrintSectionHeaders() const {
}
}
else {
LOGMAN_THROW_A(Header._64.e_shstrndx < SectionHeaders.size(),
LogMan::Throw::A(Header._64.e_shstrndx < SectionHeaders.size(),
"String index section is wrong index!");
Elf64_Shdr const *StrHeader = SectionHeaders.at(Header._64.e_shstrndx)._64;
char const *SHStrings = &RawFile.at(StrHeader->sh_offset);
@@ -773,7 +708,7 @@ void ELFContainer::PrintSectionHeaders() const {
void ELFContainer::PrintProgramHeaders() const {
if (Mode == MODE_32BIT) {
LOGMAN_THROW_A(Header._32.e_shstrndx < SectionHeaders.size(),
LogMan::Throw::A(Header._32.e_shstrndx < SectionHeaders.size(),
"String index section is wrong index!");
for (uint32_t i = 0; i < ProgramHeaders.size(); ++i) {
Elf32_Phdr const *hdr = ProgramHeaders.at(i)._32;
@@ -788,7 +723,7 @@ void ELFContainer::PrintProgramHeaders() const {
}
}
else {
LOGMAN_THROW_A(Header._64.e_shstrndx < SectionHeaders.size(),
LogMan::Throw::A(Header._64.e_shstrndx < SectionHeaders.size(),
"String index section is wrong index!");
for (uint32_t i = 0; i < ProgramHeaders.size(); ++i) {
Elf64_Phdr const *hdr = ProgramHeaders.at(i)._64;
@@ -822,9 +757,9 @@ void ELFContainer::PrintSymbolTable() const {
return;
}
LOGMAN_THROW_A(SymTabHeader->sh_link < SectionHeaders.size(),
LogMan::Throw::A(SymTabHeader->sh_link < SectionHeaders.size(),
"Symbol table string table section is wrong");
LOGMAN_THROW_A(SymTabHeader->sh_entsize == sizeof(Elf32_Sym),
LogMan::Throw::A(SymTabHeader->sh_entsize == sizeof(Elf32_Sym),
"Entry size doesn't match symbol entry");
StringTableHeader = SectionHeaders.at(SymTabHeader->sh_link)._32;
@@ -860,9 +795,9 @@ void ELFContainer::PrintSymbolTable() const {
return;
}
LOGMAN_THROW_A(SymTabHeader->sh_link < SectionHeaders.size(),
LogMan::Throw::A(SymTabHeader->sh_link < SectionHeaders.size(),
"Symbol table string table section is wrong");
LOGMAN_THROW_A(SymTabHeader->sh_entsize == sizeof(Elf64_Sym),
LogMan::Throw::A(SymTabHeader->sh_entsize == sizeof(Elf64_Sym),
"Entry size doesn't match symbol entry");
StringTableHeader = SectionHeaders.at(SymTabHeader->sh_link)._64;
@@ -907,12 +842,12 @@ void ELFContainer::PrintRelocationTable() const {
LogMan::Msg::D("Relocation Section: '%s'", &SHStrings[RelaHeader->sh_name]);
if (RelaHeader->sh_info != 0) {
LOGMAN_THROW_A(RelaHeader->sh_info < SectionHeaders.size(), "Rela header pointers to invalid GOT header");
LogMan::Throw::A(RelaHeader->sh_info < SectionHeaders.size(), "Rela header pointers to invalid GOT header");
GOTHeader = SectionHeaders.at(RelaHeader->sh_info)._64;
}
if (RelaHeader->sh_link != 0) {
LOGMAN_THROW_A(RelaHeader->sh_link < SectionHeaders.size(), "Rela header pointers to invalid dyndym header");
LogMan::Throw::A(RelaHeader->sh_link < SectionHeaders.size(), "Rela header pointers to invalid dyndym header");
DynSymHeader = SectionHeaders.at(RelaHeader->sh_link)._64;
StringTableHeader = SectionHeaders.at(DynSymHeader->sh_link)._64;
@@ -930,7 +865,7 @@ void ELFContainer::PrintRelocationTable() const {
LogMan::Msg::D("\toffset: 0x%lx", Entry->r_offset);
LogMan::Msg::D("\tSym: 0x%lx", Sym);
if (DynSymHeader && Sym != 0) {
LOGMAN_THROW_A(DynSymHeader->sh_entsize == sizeof(Elf64_Sym), "Oops, entry size doesn't match");
LogMan::Throw::A(DynSymHeader->sh_entsize == sizeof(Elf64_Sym), "Oops, entry size doesn't match");
uint64_t offset = DynSymHeader->sh_offset + Sym * DynSymHeader->sh_entsize;
Elf64_Sym const *Symbol =
@@ -993,12 +928,12 @@ void ELFContainer::FixupRelocations(void *ELFBase, uint64_t GuestELFBase, Symbol
RelaHeader = hdr;
if (RelaHeader->sh_info != 0) {
LOGMAN_THROW_A(RelaHeader->sh_info < SectionHeaders.size(), "Rela header pointers to invalid GOT header");
LogMan::Throw::A(RelaHeader->sh_info < SectionHeaders.size(), "Rela header pointers to invalid GOT header");
GOTHeader = SectionHeaders.at(RelaHeader->sh_info)._64;
}
if (RelaHeader->sh_link != 0) {
LOGMAN_THROW_A(RelaHeader->sh_link < SectionHeaders.size(), "Rela header pointers to invalid dyndym header");
LogMan::Throw::A(RelaHeader->sh_link < SectionHeaders.size(), "Rela header pointers to invalid dyndym header");
DynSymHeader = SectionHeaders.at(RelaHeader->sh_link)._64;
StringTableHeader = SectionHeaders.at(DynSymHeader->sh_link)._64;
@@ -1015,7 +950,7 @@ void ELFContainer::FixupRelocations(void *ELFBase, uint64_t GuestELFBase, Symbol
Elf64_Sym const *EntrySymbol {nullptr};
char const *EntrySymbolName {nullptr};
if (DynSymHeader && Sym != 0) {
LOGMAN_THROW_A(DynSymHeader->sh_entsize == sizeof(Elf64_Sym), "Oops, entry size doesn't match");
LogMan::Throw::A(DynSymHeader->sh_entsize == sizeof(Elf64_Sym), "Oops, entry size doesn't match");
uint64_t offset = DynSymHeader->sh_offset + Sym * DynSymHeader->sh_entsize;
EntrySymbol =
@@ -1351,4 +1286,5 @@ void ELFContainer::GetInitLocations(uint64_t GuestELFBase, std::vector<uint64_t>
}
}
} // namespace ELFLoader
@@ -5,11 +5,9 @@ desc: Part of our now defunct ld-linux replacement, keeps tracks of all symbols,
$end_info$
*/
#include "Common/MathUtils.h"
#include "Linux/Utils/ELFSymbolDatabase.h"
#include <FEXCore/Utils/Allocator.h>
#include <FEXCore/Utils/ELFSymbolDatabase.h>
#include <FEXCore/Utils/LogManager.h>
#include <FEXCore/Utils/Common/MathUtils.h>
#include <cstring>
#include <elf.h>
@@ -83,11 +81,8 @@ ELFSymbolDatabase::ELFSymbolDatabase(::ELFLoader::ELFContainer *file)
for (auto &Lib : UnfilledDependencies) {
if (NameToELF.find(Lib) == NameToELF.end()) {
std::string LibraryPath;
#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
bool Found =
#endif
FindLibraryFile(&LibraryPath, Lib.c_str());
LOGMAN_THROW_A(Found, "Couldn't find library '%s'", Lib.c_str());
bool Found = FindLibraryFile(&LibraryPath, Lib.c_str());
LogMan::Throw::A(Found, "Couldn't find library '%s'", Lib.c_str());
auto Info = DynamicELFInfo.emplace_back(new ELFInfo{});
Info->Name = Lib;
Info->Container = new ::ELFLoader::ELFContainer(LibraryPath, {}, true);
@@ -115,7 +110,7 @@ ELFSymbolDatabase::ELFSymbolDatabase(::ELFLoader::ELFContainer *file)
FillSymbols();
if (LocalInfo.Container->WasDynamic() && File->GetMode() == ELFContainer::MODE_64BIT) {
ELFBase = FEXCore::Allocator::mmap(nullptr, ELFMemorySize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
ELFBase = mmap(nullptr, ELFMemorySize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
FillMemoryLayouts(reinterpret_cast<uintptr_t>(ELFBase));
FillInitializationOrder();
FillSymbols();
@@ -126,7 +121,7 @@ ELFSymbolDatabase::ELFSymbolDatabase(::ELFLoader::ELFContainer *file)
ELFSymbolDatabase::~ELFSymbolDatabase() {
if (ELFBase) {
FEXCore::Allocator::munmap(ELFBase, ELFMemorySize);
munmap(ELFBase, ELFMemorySize);
ELFBase = nullptr;
}
}
@@ -169,7 +164,7 @@ void ELFSymbolDatabase::FillMemoryLayouts(uint64_t DefinedBase) {
uint64_t CurrentELFAlignedSize = AlignUp(std::get<2>(LocalInfo.CustomLayout), 4096);
if (CurrentELFBase < 0x10000) {
// We can't allocate memory in the first 16KB, Hopefully no elfs require this.
LOGMAN_MSG_A("Elf requires memory mapped in the first 16kb");
LogMan::Msg::A("Elf requires memory mapped in the first 16kb");
}
std::get<2>(LocalInfo.CustomLayout) = CurrentELFAlignedSize;
+1 -19
View File
@@ -37,17 +37,7 @@ void UnInstallHandlers() { Handlers.clear(); }
Handler(Buffer);
}
FEX_TRAP_EXECUTION;
}
void MFmt(const char *fmt, const fmt::format_args& args) {
auto msg = fmt::vformat(fmt, args);
for (auto& Handler : Handlers) {
Handler(msg.c_str());
}
FEX_TRAP_EXECUTION;
__builtin_trap();
}
} // namespace Throw
@@ -77,13 +67,5 @@ void M(DebugLevels Level, const char *fmt, va_list args) {
}
}
void MFmtImpl(DebugLevels level, const char* fmt, const fmt::format_args& args) {
const auto msg = fmt::vformat(fmt, args);
for (auto& Handler : Handlers) {
Handler(level, msg.c_str());
}
}
} // namespace Msg
} // namespace LogMan
Loaded 100 of 326 files, more files were not shown because too many files have changed in this diff. Show more