Compare commits

..
1 Commits
Author SHA1 Message Date
Ryan Houdek 33fe6813fc Docs: Update for release FEX-2104 2021-04-02 11:35:29 -07:00
289 changed files with 4955 additions and 16262 deletions

No files matched your search

+1 -14
View File
@@ -13,14 +13,13 @@ env:
BUILD_TYPE: Release
CC: clang
CXX: clang++
FEX_FORCE32BITALLOCATOR: 1
jobs:
build:
runs-on: ${{ matrix.arch }}
strategy:
matrix:
arch: [[self-hosted, x64], [self-hosted, ARMv8.0], [self-hosted, ARMv8.2], [self-hosted, ARMv8.4]]
arch: [[self-hosted, x64], [self-hosted, ARMv8.0], [self-hosted, ARMv8.2]]
fail-fast: false
steps:
@@ -117,18 +116,6 @@ jobs:
working-directory: ${{runner.workspace}}/build
run: mv ${{runner.workspace}}/build/Testing/Temporary/LastTest.log ${{runner.workspace}}/build/Testing/Temporary/LastTest_GCC64.log || true
- name: gcc target tests 32
working-directory: ${{runner.workspace}}/build
shell: bash
# Execute the gvisor tests
run: cmake --build . --config $BUILD_TYPE --target gcc_target_tests_32
- name: GCC32 Test Results move
if: ${{ always() }}
shell: bash
working-directory: ${{runner.workspace}}/build
run: mv ${{runner.workspace}}/build/Testing/Temporary/LastTest.log ${{runner.workspace}}/build/Testing/Temporary/LastTest_GCC32.log || true
- name: Struct verifier tests
working-directory: ${{runner.workspace}}/build
shell: bash
-9
View File
@@ -30,12 +30,3 @@
shallow = true
path = External/fex-gcc-target-tests-bins
url = https://github.com/FEX-Emu/fex-gcc-target-tests-bins.git
[submodule "External/jemalloc"]
path = External/jemalloc
url = https://github.com/FEX-Emu/jemalloc.git
[submodule "External/fmt"]
path = External/fmt
url = https://github.com/fmtlib/fmt.git
[submodule "External/drm-headers"]
path = External/drm-headers
url = https://github.com/FEX-Emu/drm-headers.git
+12 -28
View File
@@ -61,8 +61,8 @@ endif()
if (ENABLE_ASAN)
add_definitions(-DENABLE_ASAN=1)
add_compile_options(-fno-omit-frame-pointer -fsanitize=address -fsanitize-address-use-after-scope)
link_libraries(-fno-omit-frame-pointer -fsanitize=address -fsanitize-address-use-after-scope)
add_compile_options(-fno-omit-frame-pointer -fsanitize=address)
link_libraries(-fno-omit-frame-pointer -fsanitize=address)
endif()
if (ENABLE_TSAN)
@@ -103,21 +103,14 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
message(FATAL_ERROR "FEX doesn't support getting compiled with GCC!")
endif()
find_package(PkgConfig REQUIRED)
find_package(Python 3.0 REQUIRED COMPONENTS Interpreter)
pkg_check_modules(XXHASH libxxhash REQUIRED)
add_definitions(-Wno-trigraphs)
add_definitions(-DGLOBAL_DATA_DIRECTORY="${DATA_DIRECTORY}/")
add_subdirectory(External/jemalloc/)
include_directories(External/jemalloc/pregen/include/)
add_subdirectory(External/cpp-optparse/)
include_directories(External/cpp-optparse/)
add_subdirectory(External/fmt/)
add_subdirectory(External/imgui/)
include_directories(External/imgui/)
@@ -165,27 +158,18 @@ if(ENABLE_WERROR OR ENABLE_STRICT_WERROR)
endif()
if(_M_ARM_64)
if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 999999.0)
# Clang 12.0 fixed the -mcpu=native bug with mixed big.little implementers
# Clang can not currently check for native Apple M1 type in hypervisor. Currently disabled
check_cxx_compiler_flag("-mcpu=native" COMPILER_SUPPORTS_CPU_TYPE)
if(COMPILER_SUPPORTS_CPU_TYPE)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcpu=native")
endif()
else()
# Due to an oversight in llvm, it declares any reasonably new Kryo CPU to only be ARMv8.0
# Manually detect newer CPU revisions until clang and llvm fixes their bug
# This script will either provide a supported CPU or 'native'
# Additionally -march doesn't work under AArch64+Clang, so you have to use -mcpu or -mtune
execute_process(COMMAND python3 "${PROJECT_SOURCE_DIR}/Scripts/aarch64_fit_native.py" "/proc/cpuinfo" "${CMAKE_CXX_COMPILER_VERSION}"
OUTPUT_VARIABLE AARCH64_CPU)
# Due to an oversight in llvm, it declares any reasonably new Kryo CPU to only be ARMv8.0
# Manually detect newer CPU revisions until clang and llvm fixes their bug
# This script will either provide a supported CPU or 'native'
# Additionally -march doesn't work under AArch64+Clang, so you have to use -mcpu or -mtune
execute_process(COMMAND python3 "${PROJECT_SOURCE_DIR}/Scripts/aarch64_fit_native.py" "/proc/cpuinfo"
OUTPUT_VARIABLE AARCH64_CPU)
string(STRIP ${AARCH64_CPU} AARCH64_CPU)
string(STRIP ${AARCH64_CPU} AARCH64_CPU)
check_cxx_compiler_flag("-mcpu=${AARCH64_CPU}" COMPILER_SUPPORTS_CPU_TYPE)
if(COMPILER_SUPPORTS_CPU_TYPE)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcpu=${AARCH64_CPU}")
endif()
check_cxx_compiler_flag("-mcpu=${AARCH64_CPU}" COMPILER_SUPPORTS_CPU_TYPE)
if(COMPILER_SUPPORTS_CPU_TYPE)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcpu=${AARCH64_CPU}")
endif()
endif()
-35
View File
@@ -159,42 +159,7 @@ def print_man_environment(options):
default
)
print_man_environment_tail()
output_man.write(".El\n")
def print_man_environment_tail():
# Additional environment variables that live outside of the normal loop
print_man_env_option(
"FEX_APP_CONFIG_LOCATION",
[
"Allows the user to override where FEX looks for configuration files",
"By default FEX will look in {$HOME, $XDG_CONFIG_HOME}/.fex-emu/",
"This will override the full path",
],
"''")
print_man_env_option(
"FEX_APP_CONFIG",
[
"Allows the user to override where FEX looks for only the application config file",
"By default FEX will look in {$HOME, $XDG_CONFIG_HOME}/.fex-emu/Config.json",
"This will override this file location",
"One must be careful with this option as it will override any applications that load with execve as well"
"If you need to support applications that execve then use FEX_APP_CONFIG_LOCATION instead"
],
"''")
print_man_env_option(
"FEX_APP_DATA_LOCATION",
[
"Allows the user to override where FEX looks for data files",
"By default FEX will look in {$HOME, $XDG_DATA_HOME}/.fex-emu/",
"This will override the full path",
"This is the folder where FEX stores generated files like IR cache"
],
"''")
def print_man_header():
header ='''.Dd {0}
.Dt FEX
+19 -25
View File
@@ -108,10 +108,10 @@ def print_ir_sizes(ops, defines):
output_file.write("[[maybe_unused]] static size_t GetSize(IROps Op) { return IRSizes[Op]; }\n\n")
output_file.write("__attribute__((const)) __attribute__((visibility(\"default\"))) std::string_view const& GetName(IROps Op);\n")
output_file.write("__attribute__((const)) __attribute__((visibility(\"default\"))) uint8_t GetArgs(IROps Op);\n")
output_file.write("__attribute__((const)) __attribute__((visibility(\"default\"))) FEXCore::IR::RegisterClassType GetRegClass(IROps Op);\n\n")
output_file.write("__attribute__((const)) __attribute__((visibility(\"default\"))) bool HasSideEffects(IROps Op);\n")
output_file.write("std::string_view const& GetName(IROps Op);\n")
output_file.write("uint8_t GetArgs(IROps Op);\n")
output_file.write("FEXCore::IR::RegisterClassType GetRegClass(IROps Op);\n\n")
output_file.write("bool HasSideEffects(IROps Op);\n")
output_file.write("#undef IROP_SIZES\n")
output_file.write("#endif\n\n")
@@ -277,7 +277,7 @@ def print_ir_allocator_helpers(ops, defines):
output_file.write("\tusing IRPair = Wrapper<T>;\n\n")
output_file.write("\tIRPair<IROp_Header> AllocateRawOp(size_t HeaderSize) {\n")
output_file.write("\t\tauto Op = reinterpret_cast<IROp_Header*>(DualListData.DataAllocate(HeaderSize));\n")
output_file.write("\t\tauto Op = reinterpret_cast<IROp_Header*>(Data.Allocate(HeaderSize));\n")
output_file.write("\t\tmemset(Op, 0, HeaderSize);\n")
output_file.write("\t\tOp->Op = IROps::OP_DUMMY;\n")
output_file.write("\t\treturn IRPair<IROp_Header>{Op, CreateNode(Op)};\n")
@@ -286,7 +286,7 @@ def print_ir_allocator_helpers(ops, defines):
output_file.write("\ttemplate<class T, IROps T2>\n")
output_file.write("\tT *AllocateOrphanOp() {\n")
output_file.write("\t\tsize_t Size = FEXCore::IR::GetSize(T2);\n")
output_file.write("\t\tauto Op = reinterpret_cast<T*>(DualListData.DataAllocate(Size));\n")
output_file.write("\t\tauto Op = reinterpret_cast<T*>(Data.Allocate(Size));\n")
output_file.write("\t\tmemset(Op, 0, Size);\n")
output_file.write("\t\tOp->Header.Op = T2;\n")
output_file.write("\t\treturn Op;\n")
@@ -295,25 +295,25 @@ def print_ir_allocator_helpers(ops, defines):
output_file.write("\ttemplate<class T, IROps T2>\n")
output_file.write("\tIRPair<T> AllocateOp() {\n")
output_file.write("\t\tsize_t Size = FEXCore::IR::GetSize(T2);\n")
output_file.write("\t\tauto Op = reinterpret_cast<T*>(DualListData.DataAllocate(Size));\n")
output_file.write("\t\tauto Op = reinterpret_cast<T*>(Data.Allocate(Size));\n")
output_file.write("\t\tmemset(Op, 0, Size);\n")
output_file.write("\t\tOp->Header.Op = T2;\n")
output_file.write("\t\treturn IRPair<T>{Op, CreateNode(&Op->Header)};\n")
output_file.write("\t}\n\n")
output_file.write("\tuint8_t GetOpSize(OrderedNode *Op) const {\n")
output_file.write("\t\tauto HeaderOp = Op->Header.Value.GetNode(DualListData.DataBegin());\n")
output_file.write("\t\tauto HeaderOp = Op->Header.Value.GetNode(Data.Begin());\n")
output_file.write("\t\treturn HeaderOp->Size;\n")
output_file.write("\t}\n\n")
output_file.write("\tuint8_t GetOpElements(OrderedNode *Op) const {\n")
output_file.write("\t\tauto HeaderOp = Op->Header.Value.GetNode(DualListData.DataBegin());\n")
output_file.write("\t\tLOGMAN_THROW_A(HeaderOp->HasDest, \"Op %s has no dest\\n\", GetName(HeaderOp->Op));\n")
output_file.write("\t\tauto HeaderOp = Op->Header.Value.GetNode(Data.Begin());\n")
output_file.write("\t\tLogMan::Throw::A(HeaderOp->HasDest, \"Op %s has no dest\\n\", GetName(HeaderOp->Op));\n")
output_file.write("\t\treturn HeaderOp->Size / HeaderOp->ElementSize;\n")
output_file.write("\t}\n\n")
output_file.write("\tbool OpHasDest(OrderedNode *Op) const {\n")
output_file.write("\t\tauto HeaderOp = Op->Header.Value.GetNode(DualListData.DataBegin());\n")
output_file.write("\t\tauto HeaderOp = Op->Header.Value.GetNode(Data.Begin());\n")
output_file.write("\t\treturn HeaderOp->HasDest;\n")
output_file.write("\t}\n\n")
@@ -387,14 +387,11 @@ def print_ir_allocator_helpers(ops, defines):
output_file.write(") {\n")
output_file.write("\t\tauto Op = AllocateOp<IROp_%s, IROps::OP_%s>();\n" % (op_key, op_key.upper()))
if (SSAArgs != 0):
output_file.write("\t\tauto ListDataBegin = DualListData.ListBegin();\n")
for i in range(0, SSAArgs):
output_file.write("\t\tOp.first->Header.Args[%d] = ssa%d->Wrapped(ListDataBegin);\n" % (i, i))
output_file.write("\t\tOp.first->Header.NumArgs = %d;\n" % (SSAArgs))
if (SSAArgs != 0):
for i in range(0, SSAArgs):
output_file.write("\t\tOp.first->Header.Args[%d] = ssa%d->Wrapped(ListData.Begin());\n" % (i, i))
output_file.write("\t\tssa%d->AddUse();\n" % (i))
if (HasArgs):
@@ -402,6 +399,11 @@ def print_ir_allocator_helpers(ops, defines):
data_name = op_vals["Args"][i]
output_file.write("\t\tOp.first->%s = %s;\n" % (data_name, data_name))
if (HasFixedDestSize):
output_file.write("\t\tOp.first->Header.Size = %d;\n" % FixedDestSize)
if (HasDestSize):
output_file.write("\t\tOp.first->Header.Size = %s;\n" % DestSize)
if (HasDest):
# We can only infer a size if we have arguments
if not (HasFixedDestSize or HasDestSize):
@@ -410,18 +412,10 @@ def print_ir_allocator_helpers(ops, defines):
if (SSAArgs != 0):
for i in range(0, SSAArgs):
output_file.write("\t\tuint8_t Size%d = GetOpSize(ssa%s);\n" % (i, i))
for i in range(0, SSAArgs):
output_file.write("\t\tInferSize = std::max(InferSize, Size%d);\n" % (i))
output_file.write("\t\tOp.first->Header.Size = InferSize;\n")
output_file.write("\t\tOp.first->Header.NumArgs = %d;\n" % (SSAArgs))
if (HasFixedDestSize):
output_file.write("\t\tOp.first->Header.Size = %d;\n" % FixedDestSize)
if (HasDestSize):
output_file.write("\t\tOp.first->Header.Size = %s;\n" % DestSize)
output_file.write("\t\tOp.first->Header.ElementSize = Op.first->Header.Size / (%s);\n" % NumElements)
if (HasDest):
@@ -505,7 +499,7 @@ def print_ir_parser_allocator_helpers(ops, defines):
if (SSAArgs != 0):
for i in range(0, SSAArgs):
output_file.write("\t\tOp.first->Header.Args[%d] = ssa%d->Wrapped(DualListData.ListBegin());\n" % (i, i))
output_file.write("\t\tOp.first->Header.Args[%d] = ssa%d->Wrapped(ListData.Begin());\n" % (i, i))
output_file.write("\t\tssa%d->AddUse();\n" % (i))
if (HasArgs):
+3 -22
View File
@@ -114,7 +114,6 @@ set (SRCS
Interface/IR/Passes/DeadContextStoreElimination.cpp
Interface/IR/Passes/IRCompaction.cpp
Interface/IR/Passes/IRValidation.cpp
Interface/IR/Passes/LongDivideRemovalPass.cpp
Interface/IR/Passes/ValueDominanceValidation.cpp
Interface/IR/Passes/PhiValidation.cpp
Interface/IR/Passes/RedundantFlagCalculationElimination.cpp
@@ -122,9 +121,7 @@ set (SRCS
Interface/IR/Passes/StaticRegisterAllocationPass.cpp
Interface/IR/Passes/RegisterAllocationPass.cpp
Interface/IR/Passes/SyscallOptimization.cpp
Utils/Allocator.cpp
Utils/Allocator/64BitAllocator.cpp
Utils/ELFContainer.cpp
Utils/ELFLoader.cpp
Utils/ELFSymbolDatabase.cpp
Utils/LogManager.cpp
Utils/Threads.cpp
@@ -267,11 +264,8 @@ function(AddObject Name Type)
add_dependencies(${Name} IR_INC)
add_dependencies(${Name} CONFIG_INC)
target_link_libraries(${Name} pthread vixl dl fmt::fmt xxhash FEX_jemalloc)
target_link_libraries(${Name} pthread rt vixl ${LINUX_LIBS} dl)
set_target_properties(${Name} PROPERTIES OUTPUT_NAME FEXCore)
set_target_properties(${Name} PROPERTIES C_VISIBILITY_PRESET hidden)
set_target_properties(${Name} PROPERTIES CXX_VISIBILITY_PRESET hidden)
set_target_properties(${Name} PROPERTIES VISIBILITY_INLINES_HIDDEN TRUE)
target_include_directories(${Name} PUBLIC "${CMAKE_CURRENT_BINARY_DIR}")
@@ -289,7 +283,6 @@ function(AddObject Name Type)
-Werror=implicit-fallthrough
-Wno-trigraphs
-ffunction-sections
)
if (GCC_COLOR)
@@ -306,24 +299,12 @@ endfunction()
function(AddLibrary Name Type)
add_library(${Name} ${Type} $<TARGET_OBJECTS:${PROJECT_NAME}_object>)
target_link_libraries(${Name} pthread vixl dl fmt::fmt xxhash FEX_jemalloc)
target_link_libraries(${Name} pthread rt vixl ${LINUX_LIBS} dl)
set_target_properties(${Name} PROPERTIES OUTPUT_NAME FEXCore)
set_target_properties(${Name} PROPERTIES C_VISIBILITY_PRESET hidden)
set_target_properties(${Name} PROPERTIES CXX_VISIBILITY_PRESET hidden)
set_target_properties(${Name} PROPERTIES VISIBILITY_INLINES_HIDDEN TRUE)
target_include_directories(${Name} PUBLIC "${CMAKE_CURRENT_BINARY_DIR}")
target_include_directories(${Name} PUBLIC "${PROJECT_SOURCE_DIR}/include/")
target_include_directories(${Name} PUBLIC "${CMAKE_BINARY_DIR}/include/")
if (CMAKE_BUILD_TYPE MATCHES "RELEASE")
target_link_options(${Name}
PRIVATE
"LINKER:--gc-sections"
"LINKER:--strip-all"
"LINKER:--as-needed"
)
endif()
endfunction()
AddObject(${PROJECT_NAME}_object OBJECT)
+6 -7
View File
@@ -1,6 +1,5 @@
#pragma once
#include "Common/MathUtils.h"
#include <FEXCore/Utils/Allocator.h>
#include <FEXCore/Utils/LogManager.h>
#include <cstring>
@@ -17,16 +16,16 @@ struct BitSet final {
ElementType *Memory;
void Allocate(size_t Elements) {
size_t AllocateSize = AlignUp(Elements, MinimumSizeBits) / MinimumSize;
LOGMAN_THROW_A((AllocateSize * MinimumSize) >= Elements, "Fail");
Memory = static_cast<ElementType*>(FEXCore::Allocator::malloc(AllocateSize));
LogMan::Throw::A((AllocateSize * MinimumSize) >= Elements, "Fail");
Memory = static_cast<ElementType*>(malloc(AllocateSize));
}
void Realloc(size_t Elements) {
size_t AllocateSize = AlignUp(Elements, MinimumSizeBits) / MinimumSize;
LOGMAN_THROW_A((AllocateSize * MinimumSize) >= Elements, "Fail");
Memory = static_cast<ElementType*>(FEXCore::Allocator::realloc(Memory, AllocateSize));
LogMan::Throw::A((AllocateSize * MinimumSize) >= Elements, "Fail");
Memory = static_cast<ElementType*>(realloc(Memory, AllocateSize));
}
void Free() {
FEXCore::Allocator::free(Memory);
free(Memory);
Memory = nullptr;
}
bool Get(T Element) {
@@ -61,7 +60,7 @@ struct BitSetView final {
ElementType *Memory;
void GetView(BitSet<T> &Set, uint64_t ElementOffset) {
LOGMAN_THROW_A((ElementOffset % MinimumSize) == 0,
LogMan::Throw::A((ElementOffset % MinimumSize) == 0,
"Bitset view offset needs to be aligned to size of backing element");
Memory = &Set.Memory[ElementOffset / MinimumSizeBits];
}
+11 -19
View File
@@ -6,13 +6,10 @@
#include <sys/stat.h>
namespace FEXCore::Paths {
std::unique_ptr<std::string> CachePath;
std::unique_ptr<std::string> EntryCache;
std::string CachePath;
std::string EntryCache;
void InitializePaths() {
CachePath = std::make_unique<std::string>();
EntryCache = std::make_unique<std::string>();
char const *HomeDir = getenv("HOME");
if (!HomeDir) {
@@ -25,34 +22,29 @@ namespace FEXCore::Paths {
char *XDGDataDir = getenv("XDG_DATA_DIR");
if (XDGDataDir) {
*CachePath = XDGDataDir;
CachePath = XDGDataDir;
}
else {
if (HomeDir) {
*CachePath = HomeDir;
CachePath = HomeDir;
}
}
*CachePath += "/.fex-emu/";
*EntryCache = *CachePath + "/EntryCache/";
CachePath += "/.fex-emu/";
EntryCache = CachePath + "/EntryCache/";
// Ensure the folder structure is created for our Data
if (!std::filesystem::exists(*EntryCache) &&
!std::filesystem::create_directories(*EntryCache)) {
LogMan::Msg::D("Couldn't create EntryCache directory: '%s'", EntryCache->c_str());
if (!std::filesystem::exists(EntryCache) &&
!std::filesystem::create_directories(EntryCache)) {
LogMan::Msg::D("Couldn't create EntryCache directory: '%s'", EntryCache.c_str());
}
}
void ShutdownPaths() {
CachePath.reset();
EntryCache.reset();
}
std::string GetCachePath() {
return *CachePath;
return CachePath;
}
std::string GetEntryCachePath() {
return *EntryCache;
return EntryCache;
}
}
-1
View File
@@ -3,7 +3,6 @@
namespace FEXCore::Paths {
void InitializePaths();
void ShutdownPaths();
std::string GetCachePath();
std::string GetEntryCachePath();
}
+6 -33
View File
@@ -48,15 +48,8 @@ namespace FEXCore::Config {
else {
char const *HomeDir = GetHomeDirectory();
char const *ConfigXDG = getenv("XDG_CONFIG_HOME");
char const *ConfigOverride = getenv("FEX_APP_CONFIG_LOCATION");
if (ConfigOverride) {
// Config override completely overrides the config directory
ConfigDir = ConfigOverride;
}
else {
ConfigDir = ConfigXDG ? ConfigXDG : HomeDir;
ConfigDir += "/.fex-emu/";
}
ConfigDir = ConfigXDG ? ConfigXDG : HomeDir;
ConfigDir += "/.fex-emu/";
// Ensure the folder structure is created for our configuration
if (!std::filesystem::exists(ConfigDir) &&
@@ -71,19 +64,11 @@ namespace FEXCore::Config {
}
std::string GetConfigFileLocation() {
std::string ConfigFile{};
const char *AppConfig = getenv("FEX_APP_CONFIG");
if (AppConfig) {
// App config environment variable overwrites only the config file
ConfigFile = AppConfig;
}
else {
ConfigFile = GetConfigDirectory(false) + "Config.json";
}
std::string ConfigFile = GetConfigDirectory(false) + "Config.json";
return ConfigFile;
}
std::string GetApplicationConfig(const std::string &Filename, bool Global) {
std::string GetApplicationConfig(std::string &Filename, bool Global) {
std::string ConfigFile = GetConfigDirectory(Global);
if (!Global &&
!std::filesystem::exists(ConfigFile) &&
@@ -102,15 +87,8 @@ namespace FEXCore::Config {
char const *HomeDir = GetHomeDirectory();
char const *DataXDG = getenv("XDG_DATA_HOME");
char const *DataOverride = getenv("FEX_APP_DATA_LOCATION");
if (DataOverride) {
// Data override will override the complete directory
DataDir = DataOverride;
}
else {
DataDir = DataXDG ?: HomeDir;
DataDir += "/.fex-emu/";
}
DataDir = DataXDG ?: HomeDir;
DataDir += "/.fex-emu/";
return DataDir;
}
@@ -348,10 +326,6 @@ namespace FEXCore::Config {
Meta->Set(Option, Data);
}
void Erase(ConfigOption Option) {
Meta->Erase(Option);
}
void EraseSet(ConfigOption Option, std::string Data) {
Meta->EraseSet(Option, Data);
}
@@ -415,6 +389,5 @@ namespace FEXCore::Config {
*List = **Value;
}
}
template void Value<std::string>::GetListIfExists(FEXCore::Config::ConfigOption Option, std::list<std::string> *List);
}
+1 -28
View File
@@ -129,18 +129,7 @@
"Desc": [
"Disables optimizations passes for debugging."
]
},
"Force32BitAllocator": {
"Type": "bool",
"Default": "false",
"Desc": [
"Forces use of the 32-bit allocator on 32-bit applications",
"Used to work around ulimit problems of CI runner",
"Potentially useful for debugging memory problems",
"32-bit allocator is always used if your host kernel is older than 4.17"
]
}
},
"Logging": {
"SilentLog": {
@@ -153,7 +142,7 @@
},
"OutputLog": {
"Type": "str",
"Default": "stderr",
"Default": "stdout",
"ShortArg": "o",
"Desc": [
"File to write FEX output to.",
@@ -199,14 +188,6 @@
"Removes the calculation of the parity flag from GPR instructions.",
"Assuming no uses rely on it"
]
},
"ParanoidTSO": {
"Type": "bool",
"Default": "false",
"Desc": [
"Makes TSO operations even more strict.",
"Forces vector loadstores to also become atomic."
]
}
},
"Misc": {
@@ -218,14 +199,6 @@
"Captures both the loaded executable and libraries it loads."
]
},
"AOTIRGenerate": {
"Type": "bool",
"Default": "false",
"Desc": [
"Scans file for executable code and generates an AOT IR cache.",
"Does not run the executable."
]
},
"AOTIRLoad": {
"Type": "bool",
"Default": "false",
+3 -18
View File
@@ -14,10 +14,6 @@ namespace FEXCore::Context {
IR::InstallOpcodeHandlers(Mode);
}
void ShutdownStaticTables() {
FEXCore::Paths::ShutdownPaths();
}
FEXCore::Context::Context *CreateNewContext() {
return new FEXCore::Context::Context{};
}
@@ -54,9 +50,6 @@ namespace FEXCore::Context {
CTX->Step();
}
void CompileRIP(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP) {
Thread->CTX->CompileBlock(Thread->CurrentFrame, GuestRIP);
}
FEXCore::Context::ExitReason RunUntilExit(FEXCore::Context::Context *CTX) {
return CTX->RunUntilExit();
@@ -149,20 +142,12 @@ namespace FEXCore::Context {
return CTX->CPUID.RunFunction(Function, Leaf);
}
void SetAOTIRLoader(FEXCore::Context::Context *CTX, std::function<int(const std::string&)> CacheReader) {
void SetAOTIRLoader(FEXCore::Context::Context *CTX, std::function<std::unique_ptr<std::istream>(const std::string&)> CacheReader) {
CTX->AOTIRLoader = CacheReader;
}
void SetAOTIRWriter(FEXCore::Context::Context *CTX, std::function<std::unique_ptr<std::ostream>(const std::string&)> CacheWriter) {
CTX->AOTIRWriter = CacheWriter;
}
void FinalizeAOTIRCache(FEXCore::Context::Context *CTX) {
CTX->FinalizeAOTIRCache();
}
void WriteFilesWithCode(FEXCore::Context::Context *CTX, std::function<void(const std::string& fileid, const std::string& filename)> Writer) {
CTX->WriteFilesWithCode(Writer);
bool WriteAOTIR(FEXCore::Context::Context *CTX, std::function<std::unique_ptr<std::ostream>(const std::string&)> CacheWriter) {
return CTX->WriteAOTIRCache(CacheWriter);
}
void AddNamedRegion(FEXCore::Context::Context *CTX, uintptr_t Base, uintptr_t Length, uintptr_t Offset, const std::string& Name) {
+24 -65
View File
@@ -1,5 +1,4 @@
#pragma once
#include "Common/JitSymbols.h"
#include "Interface/Core/CPUID.h"
#include "Interface/Core/Frontend.h"
@@ -10,7 +9,6 @@
#include "Interface/IR/Passes/RegisterAllocationPass.h"
#include <FEXCore/Config/Config.h>
#include <FEXCore/Core/CPUBackend.h>
#include <FEXCore/Utils/CompilerDefs.h>
#include <FEXCore/Utils/Event.h>
#include <stdint.h>
@@ -22,9 +20,7 @@
#include <optional>
#include <ostream>
#include <set>
#include <shared_mutex>
#include <unordered_map>
#include <queue>
namespace FEXCore {
class ThunkHandler;
@@ -56,38 +52,6 @@ namespace FEXCore::Context {
MODE_SINGLESTEP = 1,
};
struct AOTIRInlineEntry {
uint64_t GuestHash;
uint64_t GuestLength;
/* RAData followed by IRData */
uint8_t InlineData[0];
IR::RegisterAllocationData *GetRAData();
IR::IRListView *GetIRData();
};
struct AOTIRInlineIndexEntry {
uint64_t GuestStart;
uint64_t DataOffset;
};
struct AOTIRInlineIndex {
uint64_t Count;
uint64_t DataBase;
AOTIRInlineIndexEntry Entries[0];
AOTIRInlineEntry *Find(uint64_t GuestStart);
AOTIRInlineEntry *GetInlineEntry(uint64_t DataOffset);
};
struct AOTIRCaptureCacheEntry {
std::unique_ptr<std::ostream> Stream;
std::map<uint64_t, uint64_t> Index;
void AppendAOTIRCaptureCache(uint64_t GuestRIP, uint64_t Start, uint64_t Length, uint64_t Hash, FEXCore::IR::IRListView *IRList, FEXCore::IR::RegisterAllocationData *RAData);
};
struct Context {
friend class FEXCore::HLE::SyscallHandler;
#ifdef JIT_ARM64
@@ -114,7 +78,6 @@ namespace FEXCore::Context {
FEX_CONFIG_OPT(ABILocalFlags, ABILOCALFLAGS);
FEX_CONFIG_OPT(ABINoPF, ABINOPF);
FEX_CONFIG_OPT(AOTIRCapture, AOTIRCAPTURE);
FEX_CONFIG_OPT(AOTIRGenerate, AOTIRGENERATE);
FEX_CONFIG_OPT(AOTIRLoad, AOTIRLOAD);
FEX_CONFIG_OPT(SMCChecks, SMCCHECKS);
FEX_CONFIG_OPT(Core, CORE);
@@ -124,7 +87,7 @@ namespace FEXCore::Context {
FEX_CONFIG_OPT(DumpIR, DUMPIR);
} Config;
using IntCallbackReturn = FEX_NAKED void(*)(FEXCore::Core::InternalThreadState *Thread, volatile void *Host_RSP);
using IntCallbackReturn = __attribute__((naked)) void(*)(FEXCore::Core::InternalThreadState *Thread, volatile void *Host_RSP);
IntCallbackReturn InterpreterCallbackReturn;
FEXCore::HostFeatures HostFeatures;
@@ -150,28 +113,26 @@ namespace FEXCore::Context {
std::function<void(uint64_t ThreadId, FEXCore::Context::ExitReason)> CustomExitHandler;
struct AOTIRCacheEntry {
AOTIRInlineIndex *Array;
void *mapping;
size_t size;
uint64_t start;
uint64_t len;
uint64_t crc;
IR::IRListView *IR;
IR::RegisterAllocationData *RAData;
};
std::unordered_map<std::string, AOTIRCacheEntry> AOTIRCache;
std::function<int(const std::string&)> AOTIRLoader;
std::function<std::unique_ptr<std::ostream>(const std::string&)> AOTIRWriter;
std::unordered_map<std::string, AOTIRCaptureCacheEntry> AOTIRCaptureCache;
std::function<std::unique_ptr<std::istream>(const std::string&)> AOTIRLoader;
std::unordered_map<std::string, std::map<uint64_t, AOTIRCacheEntry>> AOTIRCache;
struct AddrToFileEntry {
uint64_t Start;
uint64_t Len;
uint64_t Offset;
std::string fileid;
std::string filename;
void *CachedFileEntry;
bool ContainsCode;
};
std::map<uint64_t, AddrToFileEntry> AddrToFile;
std::map<std::string, std::string> FilesWithCode;
#ifdef BLOCKSTATS
std::unique_ptr<FEXCore::BlockSamplingData> BlockData;
@@ -185,7 +146,7 @@ namespace FEXCore::Context {
bool InitCore(FEXCore::CodeLoader *Loader);
FEXCore::Context::ExitReason RunUntilExit();
int GetProgramStatus() const;
int GetProgramStatus();
bool IsPaused() const { return !Running; }
void Pause();
void Run();
@@ -196,7 +157,7 @@ namespace FEXCore::Context {
void StopThread(FEXCore::Core::InternalThreadState *Thread);
void SignalThread(FEXCore::Core::InternalThreadState *Thread, FEXCore::Core::SignalEvent Event);
bool GetGdbServerStatus() const { return DebugServer != nullptr; }
bool GetGdbServerStatus() { return (bool)DebugServer; }
void StartGdbServer();
void StopGdbServer();
void HandleCallback(uint64_t RIP);
@@ -217,18 +178,18 @@ namespace FEXCore::Context {
bool GetDebugDataForRIP(uint64_t RIP, FEXCore::Core::DebugData *Data);
bool FindHostCodeForRIP(uint64_t RIP, uint8_t **Code);
// XXX:
// bool FindIRForRIP(uint64_t RIP, FEXCore::IR::IntrusiveIRList **ir);
// void SetIRForRIP(uint64_t RIP, FEXCore::IR::IntrusiveIRList *const ir);
void LoadEntryList();
std::tuple<FEXCore::IR::IRListView *, FEXCore::IR::RegisterAllocationData *, uint64_t, uint64_t, uint64_t, uint64_t> GenerateIR(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP);
std::tuple<void *, FEXCore::IR::IRListView *, FEXCore::Core::DebugData *, FEXCore::IR::RegisterAllocationData *, bool, uint64_t, uint64_t> CompileCode(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP);
uintptr_t CompileBlock(FEXCore::Core::CpuStateFrame *Frame, uint64_t GuestRIP);
// same as CompileBlock, but aborts on failure
void CompileBlockJit(FEXCore::Core::CpuStateFrame *Frame, uint64_t GuestRIP);
bool LoadAOTIRCache(int streamfd);
void FinalizeAOTIRCache();
void WriteFilesWithCode(std::function<void(const std::string& fileid, const std::string& filename)> Writer);
bool LoadAOTIRCache(std::istream &stream);
bool WriteAOTIRCache(std::function<std::unique_ptr<std::ostream>(const std::string&)> CacheWriter);
// Used for thread creation from syscalls
void InitializeCompiler(FEXCore::Core::InternalThreadState* State, bool CompileThread);
FEXCore::Core::InternalThreadState* CreateThread(FEXCore::Core::CPUState *NewThreadState, uint64_t ParentTID);
@@ -261,21 +222,19 @@ namespace FEXCore::Context {
void NotifyPause();
void AddBlockMapping(FEXCore::Core::InternalThreadState *Thread, uint64_t Address, void *Ptr, uint64_t Start, uint64_t Length);
FEXCore::CodeLoader *LocalLoader{};
// Entry Cache
std::optional<std::string> GetFilenameHash(std::string const &Filename) const;
void AddThreadRIPsToEntryList(FEXCore::Core::InternalThreadState *Thread);
void SaveEntryList();
std::set<uint64_t> EntryList;
std::vector<uint64_t> InitLocations;
uint64_t StartingRIP;
std::mutex ExitMutex;
std::unique_ptr<GdbServer> DebugServer;
std::shared_mutex AOTIRCacheLock;
std::shared_mutex AOTIRCaptureCacheWriteoutLock;
std::atomic<bool> AOTIRCaptureCacheWriteoutFlusing;
std::queue<std::function<void()>> AOTIRCaptureCacheWriteoutQueue;
void AOTIRCaptureCacheWriteoutQueue_Flush();
void AOTIRCaptureCacheWriteoutQueue_Append(const std::function<void()> &fn);
bool StartPaused = false;
FEX_CONFIG_OPT(AppFilename, APP_FILENAME);
};
@@ -12,12 +12,6 @@ namespace FEXCore::ArchHelpers::Arm64 {
constexpr uint32_t ATOMIC_MEM_MASK = 0x3B200C00;
constexpr uint32_t ATOMIC_MEM_INST = 0x38200000;
constexpr uint32_t LDAXP_MASK = 0xBF'FF'80'00;
constexpr uint32_t LDAXP_INST = 0x88'7F'80'00;
constexpr uint32_t STLXP_MASK = 0xBF'E0'80'00;
constexpr uint32_t STLXP_INST = 0x88'20'80'00;
// Load ops are 4 bits
// Acquire and release bits are independent on the instruction
constexpr uint32_t ATOMIC_ADD_OP = 0b0000;
@@ -29,17 +29,6 @@ Arm64Emitter::Arm64Emitter(size_t size) : vixl::aarch64::Assembler(size, vixl::a
if (!SupportsAtomics) {
WARN_ONCE("Host CPU doesn't support atomics. Expect bad performance");
}
#ifdef _M_ARM_64
// We need to get the CPU's cache line size
// We expect sane targets that have correct cacheline sizes across clusters
uint64_t CTR;
__asm volatile ("mrs %[ctr], ctr_el0"
: [ctr] "=r"(CTR));
DCacheLineSize = 4 << ((CTR >> 16) & 0xF);
ICacheLineSize = 4 << (CTR & 0xF);
#endif
}
void Arm64Emitter::LoadConstant(vixl::aarch64::Register Reg, uint64_t Constant) {
@@ -69,9 +69,6 @@ protected:
void Align16B();
uint32_t SpillSlots{};
uint32_t DCacheLineSize{};
uint32_t ICacheLineSize{};
};
}
@@ -111,7 +111,7 @@ static inline void BackupContext(void* ucontext, T *Backup) {
// Host FPR state starts at _mcontext->reserved[0];
HostFPRState *HostState = reinterpret_cast<HostFPRState*>(&_mcontext->__reserved[0]);
LOGMAN_THROW_A(HostState->Head.Magic == FPR_MAGIC, "Wrong FPR Magic: 0x%08x", HostState->Head.Magic);
LogMan::Throw::A(HostState->Head.Magic == FPR_MAGIC, "Wrong FPR Magic: 0x%08x", HostState->Head.Magic);
Backup->FPSR = HostState->FPSR;
Backup->FPCR = HostState->FPCR;
memcpy(&Backup->FPRs[0], &HostState->FPRs[0], 32 * sizeof(__uint128_t));
@@ -126,7 +126,7 @@ static inline void RestoreContext(void* ucontext, T *Backup) {
auto _mcontext = GetMContext(ucontext);
HostFPRState *HostState = reinterpret_cast<HostFPRState*>(&_mcontext->__reserved[0]);
LOGMAN_THROW_A(HostState->Head.Magic == FPR_MAGIC, "Wrong FPR Magic: 0x%08x", HostState->Head.Magic);
LogMan::Throw::A(HostState->Head.Magic == FPR_MAGIC, "Wrong FPR Magic: 0x%08x", HostState->Head.Magic);
memcpy(&HostState->FPRs[0], &Backup->FPRs[0], 32 * sizeof(__uint128_t));
HostState->FPCR = Backup->FPCR;
HostState->FPSR = Backup->FPSR;
+5 -50
View File
@@ -90,7 +90,7 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_01h() {
(0 << 16) | // Reserved
(0 << 17) | // Process-context identifiers
(1 << 18) | // Prefetching from memory mapped device
(1 << 19) | // SSE4.1
(0 << 19) | // SSE4.1
(0 << 20) | // SSE4.2
(0 << 21) | // X2APIC
(1 << 22) | // MOVBE
@@ -124,7 +124,7 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_01h() {
(1 << 16) | // Page Attribute Table
(1 << 17) | // 36bit page size extension
(0 << 18) | // Processor serial number
(1 << 19) | // CLFLUSH
(0 << 19) | // CLFLUSH
(0 << 20) | // Reserved
(0 << 21) | // Debug store
(0 << 22) | // Thermal monitor and software controled clock
@@ -496,45 +496,6 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0007h() {
return Res;
}
// Virtual and physical address sizes
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0008h() {
FEXCore::CPUID::FunctionResults Res{};
Res.eax =
(48 << 0) | // PhysAddrSize = 48-bit
(48 << 8) | // LinAddrSize = 48-bit
(0 << 16); // GuestPhysAddrSize == PhysAddrSize
Res.ebx =
(0 << 2) | // XSaveErPtr: Saving and restoring error pointers
(0 << 1) | // IRPerf: Instructions retired count support
(0 << 0); // CLZERO support
uint32_t CoreCount = Cores() - 1;
Res.ecx =
(0 << 16) | // PerfTscSize: Performance timestamp count size
(0 << 12) | // ApicIdSize: Number of bits in ApicID
(CoreCount << 0); // Count count subtract one
return Res;
}
// TLB 1GB page identifiers
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0019h() {
FEXCore::CPUID::FunctionResults Res{};
Res.eax =
(0xF << 28) | // L1 DTLB associativity for 1GB pages
(64 << 16) | // L1 DTLB entry count for 1GB pages
(0xF << 12) | // L1 ITLB associativity for 1GB pages
(64 << 0); // L1 ITLB entry count for 1GB pages
Res.ebx =
(0 << 28) | // L2 DTLB associativity for 1GB pages
(0 << 16) | // L2 DTLB entry count for 1GB pages
(0 << 12) | // L2 ITLB associativity for 1GB pages
(0 << 0); // L2 ITLB entry count for 1GB pages
return Res;
}
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_Reserved() {
FEXCore::CPUID::FunctionResults Res{};
return Res;
@@ -581,22 +542,16 @@ void CPUIDEmu::Init(FEXCore::Context::Context *ctx) {
RegisterFunction(0x8000'0004, std::bind(&CPUIDEmu::Function_8000_0004h, this));
// 0x8000'0005: L1 Cache and TLB identifiers
#ifdef CPUID_AMD
RegisterFunction(0x8000'0005, std::bind(&CPUIDEmu::Function_8000_0005h, this));
#else
// This is full reserved on Intel platforms
RegisterFunction(0x8000'0005, std::bind(&CPUIDEmu::Function_Reserved, this));
RegisterFunction(0x8000'0005, std::bind(&CPUIDEmu::Function_8000_0005h, this));
#endif
// 0x8000'0006: L2 Cache identifiers
RegisterFunction(0x8000'0006, std::bind(&CPUIDEmu::Function_8000_0006h, this));
// Advanced power management information
RegisterFunction(0x8000'0007, std::bind(&CPUIDEmu::Function_8000_0007h, this));
// Virtual and physical address sizes
RegisterFunction(0x8000'0008, std::bind(&CPUIDEmu::Function_8000_0008h, this));
// 0x8000'0008: Virtual and physical address sizes
// 0x8000'000A: SVM Revision
// TLB 1GB page identifiers
RegisterFunction(0x8000'0019, std::bind(&CPUIDEmu::Function_8000_0019h, this));
// 0x8000'0019: TLB 1GB page identifiers
// 0x8000'001A: Performance optimization identifiers
// 0x8000'001B: Instruction based sampling identifiers
// 0x8000'001C: Lightweight profiling capabilities
+1 -5
View File
@@ -3,7 +3,6 @@
#include <unordered_map>
#include <FEXCore/Core/CPUID.h>
#include <FEXCore/Config/Config.h>
#include <FEXCore/Utils/LogManager.h>
namespace FEXCore {
@@ -38,7 +37,6 @@ public:
}
private:
FEXCore::Context::Context *CTX;
FEX_CONFIG_OPT(Cores, THREADS);
using FunctionHandler = std::function<FEXCore::CPUID::FunctionResults()>;
void RegisterFunction(uint32_t Function, FunctionHandler Handler) {
@@ -62,9 +60,7 @@ private:
FEXCore::CPUID::FunctionResults Function_8000_0005h();
FEXCore::CPUID::FunctionResults Function_8000_0006h();
FEXCore::CPUID::FunctionResults Function_8000_0007h();
FEXCore::CPUID::FunctionResults Function_8000_0008h();
FEXCore::CPUID::FunctionResults Function_8000_0009h();
FEXCore::CPUID::FunctionResults Function_8000_0019h();
FEXCore::CPUID::FunctionResults Function_Reserved();
};
}
+3 -3
View File
@@ -61,7 +61,7 @@ namespace FEXCore {
}
}
LOGMAN_THROW_A(CompileThreadData->LocalIRCache.size() == 0, "Compile service must never have LocalIRCache");
LogMan::Throw::A(CompileThreadData->LocalIRCache.size() == 0, "Compile service must never have LocalIRCache");
CompileMutex.unlock();
}
@@ -124,7 +124,7 @@ namespace FEXCore {
// If we had a work item then work on it
if (Item) {
// Make sure it's not in lookup cache by accident
LOGMAN_THROW_A(CompileThreadData->LookupCache->FindBlock(Item->RIP) == 0, "Compile Service must never have entries in the LookupCache");
LogMan::Throw::A(CompileThreadData->LookupCache->FindBlock(Item->RIP) == 0, "Compile Service must never have entries in the LookupCache");
// Code isn't in cache, compile now
// Set our thread state's RIP
@@ -132,7 +132,7 @@ namespace FEXCore {
auto [CodePtr, IRList, DebugData, RAData, Generated, StartAddr, Length] = CTX->CompileCode(CompileThreadData.get(), Item->RIP);
LOGMAN_THROW_A(Generated == true, "Compile Service doesn't have IR Cache");
LogMan::Throw::A(Generated == true, "Compile Service doesn't have IR Cache");
if (!CodePtr) {
// XXX: We currently have the expectation that compile service code will be significantly smaller than regular thread's code
+294 -320
View File
@@ -28,22 +28,59 @@ $end_info$
#include <FEXCore/Core/CPUBackend.h>
#include <FEXCore/Core/X86Enums.h>
#include <FEXCore/HLE/SyscallHandler.h>
#include <FEXCore/Utils/Allocator.h>
#include "Interface/HLE/Thunks/Thunks.h"
#include "FEXCore/Utils/Allocator.h"
#include <xxh3.h>
#include <fstream>
#include <unistd.h>
#include <filesystem>
#include <algorithm>
#include <sys/mman.h>
#include <unistd.h>
#include <sys/stat.h>
#include "Interface/Core/GdbServer.h"
namespace {
// Compression function for Merkle-Damgard construction.
// This function is generated using the framework provided.
#define mix(h) ({ \
(h) ^= (h) >> 23; \
(h) *= 0x2127599bf4325c37ULL; \
(h) ^= (h) >> 47; })
static uint64_t fasthash64(const void *buf, size_t len, uint64_t seed)
{
const uint64_t m = 0x880355f21e6d1965ULL;
const uint64_t *pos = (const uint64_t *)buf;
const uint64_t *end = pos + (len / 8);
const unsigned char *pos2;
uint64_t h = seed ^ (len * m);
uint64_t v;
while (pos != end) {
v = *pos++;
h ^= mix(v);
h *= m;
}
pos2 = (const unsigned char*)pos;
v = 0;
switch (len & 7) {
case 7: v ^= (uint64_t)pos2[6] << 48; [[fallthrough]];
case 6: v ^= (uint64_t)pos2[5] << 40; [[fallthrough]];
case 5: v ^= (uint64_t)pos2[4] << 32; [[fallthrough]];
case 4: v ^= (uint64_t)pos2[3] << 24; [[fallthrough]];
case 3: v ^= (uint64_t)pos2[2] << 16; [[fallthrough]];
case 2: v ^= (uint64_t)pos2[1] << 8; [[fallthrough]];
case 1: v ^= (uint64_t)pos2[0];
h ^= mix(v);
h *= m;
}
return mix(h);
}
#undef mix
}
namespace FEXCore::CPU {
bool CreateCPUCore(FEXCore::Context::Context *CTX) {
// This should be used for generating things that are shared between threads
@@ -52,6 +89,8 @@ namespace FEXCore::CPU {
}
}
static std::mutex AOTIRCacheLock;
namespace FEXCore::Core {
struct ThreadLocalData {
FEXCore::Core::InternalThreadState* Thread;
@@ -128,7 +167,7 @@ namespace DefaultFallbackCore {
void Initialize() override {}
bool NeedsOpDispatch() override { return false; }
void *CompileCode(uint64_t Entry, FEXCore::IR::IRListView const *IR, FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) override {
void *CompileCode(FEXCore::IR::IRListView const *IR, FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) override {
LogMan::Msg::E("Fell back to default code handler at RIP: 0x%lx", ThreadState->CurrentFrame->State.rip);
return nullptr;
}
@@ -145,53 +184,6 @@ namespace DefaultFallbackCore {
}
namespace FEXCore::Context {
void Context::AOTIRCaptureCacheWriteoutQueue_Flush() {
{
std::shared_lock lk{AOTIRCaptureCacheWriteoutLock};
if (AOTIRCaptureCacheWriteoutQueue.size() == 0) {
AOTIRCaptureCacheWriteoutFlusing.store(false);
return;
}
}
for (;;) {
AOTIRCaptureCacheWriteoutLock.lock();
std::function<void()> fn = std::move(AOTIRCaptureCacheWriteoutQueue.front());
bool MaybeEmpty = false;
AOTIRCaptureCacheWriteoutQueue.pop();
MaybeEmpty = AOTIRCaptureCacheWriteoutQueue.size() == 0;
AOTIRCaptureCacheWriteoutLock.unlock();
fn();
if (MaybeEmpty) {
std::shared_lock lk{AOTIRCaptureCacheWriteoutLock};
if (AOTIRCaptureCacheWriteoutQueue.size() == 0) {
AOTIRCaptureCacheWriteoutFlusing.store(false);
return;
}
}
}
LOGMAN_MSG_A("Must never get here");
}
void Context::AOTIRCaptureCacheWriteoutQueue_Append(const std::function<void()> &fn) {
bool Flush = false;
{
std::unique_lock lk{AOTIRCaptureCacheWriteoutLock};
AOTIRCaptureCacheWriteoutQueue.push(fn);
if (AOTIRCaptureCacheWriteoutQueue.size() > 10000) {
Flush = true;
}
}
bool test_val = false;
if (Flush && AOTIRCaptureCacheWriteoutFlusing.compare_exchange_strong(test_val, true)) {
AOTIRCaptureCacheWriteoutQueue_Flush();
}
}
Context::Context() {
#ifdef BLOCKSTATS
BlockData = std::make_unique<FEXCore::BlockSamplingData>();
@@ -204,6 +196,77 @@ namespace FEXCore::Context {
}
}
std::optional<std::string> Context::GetFilenameHash(std::string const &Filename) const {
// Calculate a hash for the input file
std::ifstream Input(Filename, std::ios::in | std::ios::binary | std::ios::ate);
if (!Input) {
return std::nullopt;
}
const auto Size = static_cast<size_t>(Input.tellg());
Input.seekg(0, std::ios::beg);
std::string Data(Size, '\0');
Input.read(Data.data(), Size);
Input.close();
std::hash<std::string> string_hash;
return std::to_string(string_hash(Data));
}
void Context::AddThreadRIPsToEntryList(FEXCore::Core::InternalThreadState *Thread) {
for (auto &IR : Thread->LocalIRCache) {
EntryList.insert(IR.first);
}
}
void Context::SaveEntryList() {
std::string const &Filename = AppFilename();
if (auto const hash = GetFilenameHash(Filename)) {
auto DataPath = FEXCore::Paths::GetEntryCachePath();
DataPath += "Entries_" + *hash;
std::ofstream Output(DataPath, std::ios::out | std::ios::binary);
if (!Output) {
return;
}
for (auto Entry : EntryList) {
Output.write(reinterpret_cast<char const*>(&Entry), sizeof(Entry));
}
}
}
void Context::LoadEntryList() {
std::string const &Filename = AppFilename();
if (auto const hash = GetFilenameHash(Filename)) {
auto DataPath = FEXCore::Paths::GetEntryCachePath();
DataPath += "Entries_" + *hash;
std::ifstream Input(DataPath, std::ios::in | std::ios::binary | std::ios::ate);
if (!Input) {
return;
}
auto const Size = static_cast<size_t>(Input.tellg());
Input.seekg(0, std::ios::beg);
std::string Data(Size, '\0');
if (!Input.read(Data.data(), Size)) {
return;
}
Input.close();
size_t const EntryCount = Size / sizeof(uint64_t);
for (size_t i = 0; i < EntryCount; ++i) {
uint64_t Entry = 0;
std::memcpy(&Entry, &Data[i * sizeof(Entry)], sizeof(Entry));
EntryList.insert(Entry);
}
}
}
Context::~Context() {
{
for (auto &Thread : Threads) {
@@ -212,6 +275,10 @@ namespace FEXCore::Context {
}
}
for (auto &Thread : Threads) {
AddThreadRIPsToEntryList(Thread);
}
for (auto &Thread : Threads) {
if (Thread->CompileService) {
@@ -222,8 +289,14 @@ namespace FEXCore::Context {
Threads.clear();
}
SaveEntryList();
// AOTIRCache needs manual clear
for (auto &Mod: AOTIRCache) {
FEXCore::Allocator::munmap(Mod.second.mapping, Mod.second.size);
for (auto &Entry: Mod.second) {
delete Entry.second.IR;
free(Entry.second.RAData);
}
}
}
@@ -254,7 +327,12 @@ namespace FEXCore::Context {
// We are the parent thread
ParentThread = Thread;
Thread->CurrentFrame->State.gregs[X86State::REG_RSP] = Loader->GetStackPointer();
Loader->MapMemoryRegion();
Thread->CurrentFrame->State.gregs[X86State::REG_RSP] = Loader->SetupStack();
Loader->LoadMemory();
Loader->GetInitLocations(&InitLocations);
Thread->CurrentFrame->State.rip = StartingRIP = Loader->DefaultRIP();
@@ -319,7 +397,7 @@ namespace FEXCore::Context {
// Tell all the threads that they should pause
std::lock_guard<std::mutex> lk(ThreadCreationMutex);
for (auto &Thread : Threads) {
Thread->SignalReason.store(FEXCore::Core::SignalEvent::Pause);
Thread->SignalReason.store(FEXCore::Core::SignalEvent::SIGNALEVENT_PAUSE);
if (Thread->RunningEvents.Running.load()) {
// Only attempt to stop this thread if it is running
tgkill(Thread->ThreadManager.PID, Thread->ThreadManager.TID, SignalDelegator::SIGNAL_FOR_PAUSE);
@@ -340,7 +418,7 @@ namespace FEXCore::Context {
// Spin up all the threads
std::lock_guard<std::mutex> lk(ThreadCreationMutex);
for (auto &Thread : Threads) {
Thread->SignalReason.store(FEXCore::Core::SignalEvent::Return);
Thread->SignalReason.store(FEXCore::Core::SignalEvent::SIGNALEVENT_RETURN);
Thread->RunningEvents.WaitingToStart.store(true);
}
@@ -420,7 +498,7 @@ namespace FEXCore::Context {
void Context::StopThread(FEXCore::Core::InternalThreadState *Thread) {
if (Thread->RunningEvents.Running.exchange(false)) {
Thread->SignalReason.store(FEXCore::Core::SignalEvent::Stop);
Thread->SignalReason.store(FEXCore::Core::SignalEvent::SIGNALEVENT_STOP);
tgkill(Thread->ThreadManager.PID, Thread->ThreadManager.TID, SignalDelegator::SIGNAL_FOR_PAUSE);
}
}
@@ -448,7 +526,7 @@ namespace FEXCore::Context {
}
}
int Context::GetProgramStatus() const {
int Context::GetProgramStatus() {
return ParentThread->StatusCode;
}
@@ -464,6 +542,12 @@ namespace FEXCore::Context {
LocalLoader->AddIR(IRHandler);
// Compile all of our cached entries
LogMan::Msg::D("Precompiling: %ld blocks...", EntryList.size());
for (auto Entry : EntryList) {
CompileRIP(Thread, Entry);
}
LogMan::Msg::D("Done", EntryList.size());
}
struct ExecutionThreadHandler {
@@ -474,13 +558,15 @@ namespace FEXCore::Context {
static void *ThreadHandler(void* Data) {
ExecutionThreadHandler *Handler = reinterpret_cast<ExecutionThreadHandler*>(Data);
Handler->This->ExecutionThread(Handler->Thread);
FEXCore::Allocator::free(Handler);
free(Handler);
return nullptr;
}
void Context::InitializeThread(FEXCore::Core::InternalThreadState *Thread) {
InitializeThreadData(Thread);
// This will create the execution thread but it won't actually start executing
ExecutionThreadHandler *Arg = reinterpret_cast<ExecutionThreadHandler*>(FEXCore::Allocator::malloc(sizeof(ExecutionThreadHandler)));
ExecutionThreadHandler *Arg = reinterpret_cast<ExecutionThreadHandler*>(malloc(sizeof(ExecutionThreadHandler)));
Arg->This = this;
Arg->Thread = Thread;
Thread->ExecutionThread = FEXCore::Threads::Thread::Create(ThreadHandler, Arg);
@@ -520,23 +606,21 @@ namespace FEXCore::Context {
// Create CPU backend
switch (Config.Core) {
case FEXCore::Config::CONFIG_INTERPRETER:
State->CPUBackend = FEXCore::CPU::CreateInterpreterCore(this, State, CompileThread);
State->CPUBackend.reset(FEXCore::CPU::CreateInterpreterCore(this, State, CompileThread));
break;
case FEXCore::Config::CONFIG_IRJIT:
State->PassManager->InsertRegisterAllocationPass(DoSRA);
#if (_M_X86_64 && JIT_X86_64)
State->CPUBackend = FEXCore::CPU::CreateX86JITCore(this, State, CompileThread);
State->CPUBackend.reset(FEXCore::CPU::CreateX86JITCore(this, State, CompileThread));
#elif (_M_ARM_64 && JIT_ARM64)
State->CPUBackend = FEXCore::CPU::CreateArm64JITCore(this, State, CompileThread);
State->CPUBackend.reset(FEXCore::CPU::CreateArm64JITCore(this, State, CompileThread));
#else
ERROR_AND_DIE("FEXCore has been compiled without a viable JIT core");
#endif
break;
case FEXCore::Config::CONFIG_CUSTOM:
State->CPUBackend = CustomCPUFactory(this, State);
break;
case FEXCore::Config::CONFIG_CUSTOM: State->CPUBackend.reset(CustomCPUFactory(this, State)); break;
default: ERROR_AND_DIE("Unknown core configuration");
}
}
@@ -559,7 +643,6 @@ namespace FEXCore::Context {
Thread->ThreadManager.parent_tid = ParentTID;
InitializeCompiler(Thread, false);
InitializeThreadData(Thread);
return Thread;
}
@@ -570,7 +653,7 @@ namespace FEXCore::Context {
std::lock_guard<std::mutex> lk(ThreadCreationMutex);
auto It = std::find(Threads.begin(), Threads.end(), Thread);
LOGMAN_THROW_A(It != Threads.end(), "Thread wasn't in Threads");
LogMan::Throw::A(It != Threads.end(), "Thread wasn't in Threads");
Threads.erase(It);
}
@@ -652,8 +735,6 @@ namespace FEXCore::Context {
Thread->OpDispatcher->BeginFunction(GuestRIP, CodeBlocks);
uint8_t GPRSize = Config.Is64BitMode ? 8 : 4;
for (size_t j = 0; j < CodeBlocks->size(); ++j) {
FEXCore::Frontend::Decoder::DecodedBlocks const &Block = CodeBlocks->at(j);
// Set the block entry point
@@ -668,7 +749,8 @@ namespace FEXCore::Context {
uint64_t InstsInBlock = Block.NumInstructions;
if (Block.HasInvalidInstruction) {
Thread->OpDispatcher->_ExitFunction(Thread->OpDispatcher->_EntrypointOffset(Block.Entry - GuestRIP, GPRSize));
uint8_t GPRSize = Config.Is64BitMode ? 8 : 4;
Thread->OpDispatcher->_ExitFunction(Thread->OpDispatcher->_Constant(GPRSize * 8, Block.Entry));
break;
}
@@ -693,7 +775,7 @@ namespace FEXCore::Context {
Thread->OpDispatcher->SetCurrentCodeBlock(CodeWasChangedBlock);
Thread->OpDispatcher->_RemoveCodeEntry();
Thread->OpDispatcher->_ExitFunction(Thread->OpDispatcher->_EntrypointOffset(Block.Entry + BlockInstructionsLength - GuestRIP, GPRSize));
Thread->OpDispatcher->_ExitFunction(Thread->OpDispatcher->_Constant(Block.Entry + BlockInstructionsLength));
auto NextOpBlock = Thread->OpDispatcher->CreateNewCodeBlockAfter(CurrentBlock);
@@ -704,16 +786,12 @@ namespace FEXCore::Context {
if (TableInfo->OpcodeDispatcher) {
auto Fn = TableInfo->OpcodeDispatcher;
Thread->OpDispatcher->HandledLock = false;
Thread->OpDispatcher->ResetDecodeFailure();
std::invoke(Fn, Thread->OpDispatcher, DecodedInfo);
if (Thread->OpDispatcher->HadDecodeFailure()) {
HadDispatchError = true;
}
else {
if (Thread->OpDispatcher->HandledLock != IsLocked) {
HadDispatchError = true;
LogMan::Msg::E("Missing LOCK HANDLER at 0x%lx{'%s'}", Block.Entry + BlockInstructionsLength, TableInfo->Name);
}
LogMan::Throw::A(Thread->OpDispatcher->HandledLock == IsLocked, "Missing LOCK HANDLER at 0x%lx{'%s'}\n", Block.Entry + BlockInstructionsLength, TableInfo->Name);
BlockInstructionsLength += DecodedInfo->InstSize;
TotalInstructionsLength += DecodedInfo->InstSize;
++TotalInstructions;
@@ -729,13 +807,13 @@ namespace FEXCore::Context {
if (TotalInstructions == 0) {
// Couldn't handle any instruction in op dispatcher
Thread->OpDispatcher->ResetWorkingList();
return { nullptr, nullptr, 0, 0, 0, 0 };
return { nullptr, nullptr, 0, 0, 0, 0};
}
else {
uint8_t GPRSize = Config.Is64BitMode ? 8 : 4;
// We had some instructions. Early exit
Thread->OpDispatcher->_ExitFunction(Thread->OpDispatcher->_EntrypointOffset(Block.Entry + BlockInstructionsLength - GuestRIP, GPRSize));
Thread->OpDispatcher->_ExitFunction(Thread->OpDispatcher->_Constant(GPRSize * 8, Block.Entry + BlockInstructionsLength));
break;
}
}
@@ -792,7 +870,7 @@ namespace FEXCore::Context {
out.seekg(0);
auto reparsed = IR::Parse(&out);
if (reparsed == nullptr) {
LOGMAN_MSG_A("Failed to parse ir\n");
LogMan::Msg::A("Failed to parse ir\n");
} else {
std::stringstream out2;
auto NewIR2 = reparsed->ViewIR();
@@ -800,8 +878,9 @@ namespace FEXCore::Context {
if (out.str() != out2.str()) {
LogMan::Msg::I("one:\n %s", out.str().c_str());
LogMan::Msg::I("two:\n %s", out2.str().c_str());
LOGMAN_MSG_A("Parsed ir doesn't match\n");
LogMan::Msg::A("Parsed ir doesn't match\n");
}
delete reparsed;
}
}
// Run the passmanager over the IR from the dispatcher
@@ -826,63 +905,6 @@ namespace FEXCore::Context {
return {IRList, RAData.release(), TotalInstructions, TotalInstructionsLength, Thread->FrontendDecoder->DecodedMinAddress, Thread->FrontendDecoder->DecodedMaxAddress - Thread->FrontendDecoder->DecodedMinAddress };
}
AOTIRInlineEntry *AOTIRInlineIndex::GetInlineEntry(uint64_t DataOffset) {
uintptr_t This = (uintptr_t)this;
return (AOTIRInlineEntry*)(This + DataBase + DataOffset);
}
AOTIRInlineEntry *AOTIRInlineIndex::Find(uint64_t GuestStart) {
ssize_t l = 0;
ssize_t r = Count - 1;
while (l <= r) {
size_t m = l + (r - l) / 2;
if (Entries[m].GuestStart == GuestStart)
return GetInlineEntry(Entries[m].DataOffset);
else if (Entries[m].GuestStart < GuestStart)
l = m + 1;
else
r = m - 1;
}
return nullptr;
}
IR::RegisterAllocationData *AOTIRInlineEntry::GetRAData() {
return (IR::RegisterAllocationData *)InlineData;
}
IR::IRListView *AOTIRInlineEntry::GetIRData() {
auto RAData = GetRAData();
auto Offset = RAData->Size(RAData->MapCount);
return (IR::IRListView *)&InlineData[Offset];
}
void AOTIRCaptureCacheEntry::AppendAOTIRCaptureCache(uint64_t GuestRIP, uint64_t Start, uint64_t Length, uint64_t Hash, FEXCore::IR::IRListView *IRList, FEXCore::IR::RegisterAllocationData *RAData) {
auto Inserted = Index.emplace(GuestRIP, Stream->tellp());
if (Inserted.second) {
//GuestHash
Stream->write((char*)&Hash, sizeof(Hash));
//GuestLength
Stream->write((char*)&Length, sizeof(Length));
// RAData (inline)
// In file, IsShared is always set
auto Shared = RAData->IsShared;
RAData->IsShared = true;
Stream->write((char*)RAData, RAData->Size(RAData->MapCount));
RAData->IsShared = Shared;
// IRData (inline)
IRList->Serialize(*Stream);
}
}
std::tuple<void *, FEXCore::IR::IRListView *, FEXCore::Core::DebugData *, FEXCore::IR::RegisterAllocationData *, bool, uint64_t, uint64_t> Context::CompileCode(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP) {
FEXCore::IR::IRListView *IRList {};
FEXCore::Core::DebugData *DebugData {};
@@ -906,53 +928,35 @@ namespace FEXCore::Context {
GeneratedIR = false;
}
{
std::shared_lock lk(AOTIRCacheLock);
auto file = AddrToFile.lower_bound(GuestRIP);
if (file != AddrToFile.begin()) {
--file;
if (!file->second.ContainsCode) {
file->second.ContainsCode = true;
FilesWithCode[file->second.fileid] = file->second.filename;
}
}
}
if (IRList == nullptr && Config.AOTIRLoad) {
std::shared_lock lk(AOTIRCacheLock);
std::lock_guard<std::mutex> lk(AOTIRCacheLock);
auto file = AddrToFile.lower_bound(GuestRIP);
if (file != AddrToFile.begin()) {
--file;
auto Mod = (AOTIRInlineIndex*)file->second.CachedFileEntry;
auto Mod = (decltype(AOTIRCache)::value_type::second_type*) file->second.CachedFileEntry;
if (Mod == nullptr) {
file->second.CachedFileEntry = Mod = AOTIRCache[file->second.fileid].Array;
file->second.CachedFileEntry = Mod = &AOTIRCache[file->second.fileid];
}
if (Mod != nullptr)
{
auto AOTEntry = Mod->Find(GuestRIP - file->second.Start + file->second.Offset);
auto AOTEntry = Mod->find(GuestRIP - file->second.Start + file->second.Offset);
if (AOTEntry) {
// verify hash
auto MappedStart = GuestRIP;
auto hash = XXH3_64bits((void*)MappedStart, AOTEntry->GuestLength);
if (hash == AOTEntry->GuestHash) {
IRList = AOTEntry->GetIRData();
//LogMan::Msg::D("using %s + %lx -> %lx\n", file->second.fileid.c_str(), AOTEntry->first, GuestRIP);
if (AOTEntry != Mod->end()) {
// verify hash
auto MappedStart = AOTEntry->second.start + file->second.Start - file->second.Offset;
auto hash = fasthash64((void*)MappedStart, AOTEntry->second.len, 0);
if (hash == AOTEntry->second.crc) {
IRList = AOTEntry->second.IR;
//LogMan::Msg::D("using %s + %lx -> %lx\n", file->second.fileid.c_str(), AOTEntry->first, GuestRIP);
// relocate
IRList->GetHeader()->Entry = GuestRIP;
RAData = AOTEntry->second.RAData;
DebugData = new FEXCore::Core::DebugData();
StartAddr = MappedStart;
Length = AOTEntry->second.len;
RAData = AOTEntry->GetRAData();;
DebugData = new FEXCore::Core::DebugData();
StartAddr = MappedStart;
Length = AOTEntry->GuestLength;
GeneratedIR = true;
} else {
LogMan::Msg::I("AOTIR: hash check failed %lx\n", MappedStart);
}
} else {
//LogMan::Msg::I("AOTIR: Failed to find %lx, %lx, %s\n", GuestRIP, GuestRIP - file->second.Start + file->second.Offset, file->second.fileid.c_str());
GeneratedIR = true;
}
}
}
@@ -980,129 +984,135 @@ namespace FEXCore::Context {
GeneratedIR = true;
}
if (IRList == nullptr) {
return { nullptr, nullptr, nullptr, nullptr, false, 0, 0 };
}
// Attempt to get the CPU backend to compile this code
return { Thread->CPUBackend->CompileCode(GuestRIP, IRList, DebugData, RAData), IRList, DebugData, RAData, GeneratedIR, StartAddr, Length};
return { Thread->CPUBackend->CompileCode(IRList, DebugData, RAData), IRList, DebugData, RAData, GeneratedIR, StartAddr, Length};
}
static bool readAll(int fd, void *data, size_t size) {
int rv = read(fd, data, size);
if (rv != size)
return false;
else
return true;
}
bool Context::LoadAOTIRCache(int streamfd) {
bool Context::LoadAOTIRCache(std::istream &stream) {
std::lock_guard<std::mutex> lk(AOTIRCacheLock);
uint64_t tag;
if (!readAll(streamfd, (char*)&tag, sizeof(tag)) || tag != 0xDEADBEEFC0D30004)
stream.read((char*)&tag, sizeof(tag));
if (!stream || tag != 0xDEADBEEFC0D30002)
return false;
std::string Module;
uint64_t ModSize;
uint64_t IndexSize;
lseek(streamfd, -sizeof(ModSize), SEEK_END);
if (!readAll(streamfd, (char*)&ModSize, sizeof(ModSize)))
uint64_t ModCount;
stream.read((char*)&ModCount, sizeof(ModCount));
if (!stream)
return false;
Module.resize(ModSize);
for (int ModIndex = 0; ModIndex < ModCount; ModIndex++) {
std::string Module;
uint64_t ModSize;
stream.read((char*)&ModSize, sizeof(ModSize));
if (!stream)
return false;
lseek(streamfd, -sizeof(ModSize) - ModSize, SEEK_END);
Module.resize(ModSize);
stream.read((char*)&Module[0], Module.size());
if (!stream)
return false;
if (!readAll(streamfd, (char*)&Module[0], Module.size()))
return false;
auto &Mod = AOTIRCache[Module];
lseek(streamfd, -sizeof(ModSize) - ModSize - sizeof(IndexSize), SEEK_END);
uint64_t FnCount;
stream.read((char*)&FnCount, sizeof(FnCount));
if (!stream)
return false;
if (!readAll(streamfd, (char*)&IndexSize, sizeof(IndexSize)))
return false;
LogMan::Msg::D("AOTIR: Module %s has %ld functions", Module.c_str(), FnCount);
for (int FnIndex = 0; FnIndex < FnCount; FnIndex++) {
uint64_t addr, start, crc, len;
stream.read((char*)&addr, sizeof(addr));
if (!stream)
return false;
struct stat fileinfo;
if (fstat(streamfd, &fileinfo) < 0)
return false;
size_t Size = (fileinfo.st_size + 4095) & ~4095;
stream.read((char*)&start, sizeof(start));
if (!stream)
return false;
stream.read((char*)&len, sizeof(len));
if (!stream)
return false;
stream.read((char*)&crc, sizeof(crc));
if (!stream)
return false;
auto IR = new IR::IRListView(stream);
if (!stream) {
delete IR;
return false;
}
uint64_t RASize;
stream.read((char*)&RASize, sizeof(RASize));
if (!stream) {
delete IR;
return false;
}
IR::RegisterAllocationData *RAData = (IR::RegisterAllocationData *)malloc(IR::RegisterAllocationData::Size(RASize));
RAData->MapCount = RASize;
size_t IndexOffset = fileinfo.st_size - IndexSize -sizeof(ModSize) - ModSize - sizeof(IndexSize);
stream.read((char*)&RAData->Map[0], sizeof(RAData->Map[0]) * RASize);
void *FilePtr = FEXCore::Allocator::mmap(nullptr, Size, PROT_READ, MAP_SHARED, streamfd, 0);
if (!stream) {
delete IR;
return false;
}
stream.read((char*)&RAData->SpillSlotCount, sizeof(RAData->SpillSlotCount));
if (!stream) {
delete IR;
return false;
}
if (FilePtr == MAP_FAILED)
return false;
IR->IsShared = true;
RAData->IsShared = true;
auto Array = (AOTIRInlineIndex *)((char*)FilePtr + IndexOffset);
AOTIRCache.insert({Module, {Array, FilePtr, Size}});
LogMan::Msg::D("AOTIR: Module %s has %ld functions", Module.c_str(), Array->Count);
Mod.insert({addr, {start, len, crc, IR, RAData}});
}
}
return true;
}
void Context::WriteFilesWithCode(std::function<void(const std::string& fileid, const std::string& filename)> Writer) {
std::shared_lock lk(AOTIRCacheLock);
for( const auto &File: FilesWithCode) {
Writer(File.first, File.second);
}
}
bool Context::WriteAOTIRCache(std::function<std::unique_ptr<std::ostream>(const std::string&)> CacheWriter) {
std::lock_guard<std::mutex> lk(AOTIRCacheLock);
void Context::FinalizeAOTIRCache() {
AOTIRCaptureCacheWriteoutQueue_Flush();
bool rv = true;
std::unique_lock lk(AOTIRCacheLock);
for (auto &AOTModule: AOTIRCaptureCache) {
if (!AOTModule.second.Stream) {
for (auto AOTModule: AOTIRCache) {
if (AOTModule.second.size() == 0) {
continue;
}
auto ModSize = AOTModule.first.size();
auto &stream = AOTModule.second.Stream;
// pad to 32 bytes
char Zero = 0;
while(stream->tellp() & 31)
stream->write(&Zero, 1);
// AOTIRInlineIndex
auto FnCount = AOTModule.second.Index.size();
size_t DataBase = -stream->tellp();
stream->write((char*)&FnCount, sizeof(FnCount));
stream->write((char*)&DataBase, sizeof(DataBase));
for (auto entry: AOTModule.second.Index) {
//AOTIRInlineIndexEntry
// GuestStart
stream->write((char*)&entry.first, sizeof(entry.first));
// DataOffset
stream->write((char*)&entry.second, sizeof(entry.second));
auto stream = CacheWriter(AOTModule.first);
if (!*stream) {
rv = false;
}
uint64_t tag = 0xDEADBEEFC0D30002;
stream->write((char*)&tag, sizeof(tag));
// End of file header
auto IndexSize = FnCount * sizeof(AOTIRInlineIndexEntry) + sizeof(DataBase) + sizeof(FnCount);
stream->write((char*)&IndexSize, sizeof(IndexSize));
stream->write((char*)&AOTModule.first[0], ModSize);
uint64_t ModCount = 1;
stream->write((char*)&ModCount, sizeof(ModCount));
auto ModSize = AOTModule.first.size();
stream->write((char*)&ModSize, sizeof(ModSize));
stream->write((char*)&AOTModule.first[0], ModSize);
auto FnCount = AOTModule.second.size();
stream->write((char*)&FnCount, sizeof(FnCount));
for (auto entry: AOTModule.second) {
stream->write((char*)&entry.first, sizeof(entry.first));
stream->write((char*)&entry.second.start, sizeof(entry.second.start));
stream->write((char*)&entry.second.len, sizeof(entry.second.len));
stream->write((char*)&entry.second.crc, sizeof(entry.second.crc));
entry.second.IR->Serialize(*stream);
uint64_t RASize = entry.second.RAData->MapCount;
stream->write((char*)&RASize, sizeof(RASize));
stream->write((char*)&entry.second.RAData->Map[0], sizeof(entry.second.RAData->Map[0]) * RASize);
stream->write((char*)&entry.second.RAData->SpillSlotCount, sizeof(entry.second.RAData->SpillSlotCount));
}
}
return rv;
}
void Context::CompileBlockJit(FEXCore::Core::CpuStateFrame *Frame, uint64_t GuestRIP) {
auto NewBlock = CompileBlock(Frame, GuestRIP);
if (NewBlock == 0) {
LogMan::Msg::E("CompileBlockJit: Failed to compile code %lX - aborting process", GuestRIP);
abort();
}
}
uintptr_t Context::CompileBlock(FEXCore::Core::CpuStateFrame *Frame, uint64_t GuestRIP) {
auto Thread = Frame->Thread;
@@ -1156,11 +1166,7 @@ namespace FEXCore::Context {
Length = _Length;
}
if (CodePtr == nullptr) {
if (DecrementRefCount)
--Thread->CompileBlockReentrantRefCount;
return 0;
}
LogMan::Throw::A(CodePtr != nullptr, "Failed to compile code %lX", GuestRIP);
// The core managed to compile the code.
#if ENABLE_JITSYMBOLS
@@ -1177,49 +1183,26 @@ namespace FEXCore::Context {
// Insert to caches if we generated IR
if (GeneratedIR) {
// Add to AOT cache if aot generation is enabled
if ((Config.AOTIRCapture() || Config.AOTIRGenerate()) && RAData) {
auto hash = XXH3_64bits((void*)StartAddr, Length);
Core::LocalIREntry Entry = {StartAddr, Length, decltype(Entry.IR)(IRList), decltype(Entry.RAData)(RAData), decltype(Entry.DebugData)(DebugData)};
Thread->LocalIRCache.insert({GuestRIP, std::move(Entry)});
std::shared_lock lk(AOTIRCacheLock);
// Add to AOT cache if aot generation is enabled
if (Config.AOTIRCapture && RAData) {
std::lock_guard<std::mutex> lk(AOTIRCacheLock);
RAData->IsShared = true;
IRList->IsShared = true;
auto hash = fasthash64((void*)StartAddr, Length, 0);
auto file = AddrToFile.lower_bound(StartAddr);
if (file != AddrToFile.begin()) {
--file;
if (file->second.Start <= StartAddr && (file->second.Start + file->second.Len) >= (StartAddr + Length)) {
auto LocalRIP = GuestRIP - file->second.Start + file->second.Offset;
auto LocalStartAddr = StartAddr - file->second.Start + file->second.Offset;
auto fileid = file->second.fileid;
AOTIRCaptureCacheWriteoutQueue_Append([this, LocalRIP, LocalStartAddr, Length, hash, IRList, RAData, fileid]() {
auto *AotFile = &AOTIRCaptureCache[fileid];
if (!AotFile->Stream) {
AotFile->Stream = AOTIRWriter(fileid);
uint64_t tag = 0xDEADBEEFC0D30004;
AotFile->Stream->write((char*)&tag, sizeof(tag));
}
AotFile->AppendAOTIRCaptureCache(LocalRIP, LocalStartAddr, Length, hash, IRList, RAData);
delete IRList;
FEXCore::Allocator::free(RAData);
});
AOTIRCache[file->second.fileid].insert({GuestRIP - file->second.Start + file->second.Offset, {StartAddr - file->second.Start + file->second.Offset, Length, hash, IRList, RAData}});
}
}
if (Config.AOTIRGenerate()) {
// cleanup memory and early exit here -- we're not running the application
if (DecrementRefCount)
--Thread->CompileBlockReentrantRefCount;
Thread->CPUBackend->ClearCache();
return (uintptr_t)CodePtr;
}
}
// Add to thread local ir cache
Core::LocalIREntry Entry = {StartAddr, Length, decltype(Entry.IR)(IRList), decltype(Entry.RAData)(RAData), decltype(Entry.DebugData)(DebugData)};
Thread->LocalIRCache.insert({GuestRIP, std::move(Entry)});
}
if (DecrementRefCount)
@@ -1358,7 +1341,7 @@ namespace FEXCore::Context {
auto base_filename = std::filesystem::path(filename).filename().string();
if (base_filename.size()) {
auto filename_hash = XXH3_64bits(filename.c_str(), filename.size());
auto filename_hash = fasthash64(filename.c_str(), filename.size(), 0xBAADF00D);
auto fileid = base_filename + "-" + std::to_string(filename_hash) + "-";
@@ -1368,28 +1351,19 @@ namespace FEXCore::Context {
fileid += Config.ABILocalFlags ? "L" : "l";
fileid += Config.ABINoPF ? "p" : "P";
std::unique_lock lk(AOTIRCacheLock);
AddrToFile.insert({ Base, { Base, Size, Offset, fileid, filename, nullptr, false} });
AddrToFile.insert({ Base, { Base, Size, Offset, fileid, nullptr } });
if (Config.AOTIRLoad && !AOTIRCache.contains(fileid) && AOTIRLoader) {
auto streamfd = AOTIRLoader(fileid);
if (streamfd != -1) {
LoadAOTIRCache(streamfd);
close(streamfd);
auto stream = AOTIRLoader(fileid);
if (*stream) {
LoadAOTIRCache(*stream);
}
}
}
}
void Context::RemoveNamedRegion(uintptr_t Base, uintptr_t Size) {
std::unique_lock lk(AOTIRCacheLock);
// TODO: Support partial removing
AddrToFile.erase(Base);
}
void ConfigureAOTGen(FEXCore::Core::InternalThreadState *Thread, std::set<uint64_t> *ExternalBranches, uint64_t SectionMaxAddress) {
Thread->FrontendDecoder->SetExternalBranches(ExternalBranches);
Thread->FrontendDecoder->SetSectionMaxAddress(SectionMaxAddress);
}
}
@@ -5,7 +5,6 @@
#include "Interface/Context/Context.h"
#include <FEXCore/Core/X86Enums.h>
#include <bit>
#include <cmath>
#include "aarch64/assembler-aarch64.h"
@@ -68,7 +67,6 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::
// We want to ensure that we are 16 byte aligned at the top of this loop
Align16B();
aarch64::Label FullLookup{};
aarch64::Label CallBlock{};
aarch64::Label LoopTop{};
aarch64::Label ExitSpillSRA{};
aarch64::Label ThreadPauseHandler{};
@@ -81,19 +79,16 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::
ldr(x2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.rip)));
auto RipReg = x2;
// L1 Cache
ldr(x0, &l_L1Ptr);
and_(x3, RipReg, LookupCache::L1_ENTRIES_MASK);
add(x0, x0, Operand(x3, Shift::LSL, 4));
ldp(x3, x0, MemOperand(x0));
cmp(x0, RipReg);
b(&FullLookup, Condition::ne);
if (!config.ExecuteBlocksWithCall) {
br(x3);
} else {
b(&CallBlock);
// L1 Cache
ldr(x0, &l_L1Ptr);
and_(x3, RipReg, LookupCache::L1_ENTRIES_MASK);
add(x0, x0, Operand(x3, Shift::LSL, 4));
ldp(x1, x0, MemOperand(x0));
cmp(x0, RipReg);
b(&FullLookup, Condition::ne);
br(x1);
}
// L1C check failed, do a full lookup
@@ -104,7 +99,7 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::
ldr(x0, &l_PagePtr);
// Mask the address by the virtual address size so we can check for aliases
if (std::popcount(VirtualMemorySize) == 1) {
if (__builtin_popcountl(VirtualMemorySize) == 1) {
and_(x3, RipReg, Thread->LookupCache->GetVirtualMemorySize() - 1);
}
else {
@@ -141,37 +136,40 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::
// If we've made it here then we have a real compiled block
{
// update L1 cache
ldr(x0, &l_L1Ptr);
and_(x1, RipReg, LookupCache::L1_ENTRIES_MASK);
add(x0, x0, Operand(x1, Shift::LSL, 4));
stp(x3, x2, MemOperand(x0));
// Jump to the block
if (!config.ExecuteBlocksWithCall) {
// update L1 cache
ldr(x0, &l_L1Ptr);
and_(x1, RipReg, LookupCache::L1_ENTRIES_MASK);
add(x0, x0, Operand(x1, Shift::LSL, 4));
stp(x3, x2, MemOperand(x0));
br(x3);
} else {
bind(&CallBlock);
mov(x0, STATE);
blr(x3);
}
}
if (CTX->GetGdbServerStatus()) {
// If we have a gdb server running then run in a less efficient mode that checks if we need to exit
// This happens when single stepping
if (config.ExecuteBlocksWithCall) {
// Interpreter continues execution here
if (CTX->GetGdbServerStatus()) {
// If we have a gdb server running then run in a less efficient mode that checks if we need to exit
// This happens when single stepping
static_assert(sizeof(CTX->Config.RunningMode) == 4, "This is expected to be size of 4");
ldr(x0, &l_CTX);
ldr(w0, MemOperand(x0, offsetof(FEXCore::Context::Context, Config.RunningMode)));
// If the value == 0 then branch to the top
cbz(x0, &LoopTop);
// Else we need to pause now
b(&ThreadPauseHandler);
} else {
// Unconditionally loop to the top
// We will only stop on error when compiling a block or signal
b(&LoopTop);
}
static_assert(sizeof(CTX->Config.RunningMode) == 4, "This is expected to be size of 4");
ldr(x0, &l_CTX);
ldr(w0, MemOperand(x0, offsetof(FEXCore::Context::Context, Config.RunningMode)));
// If the value == 0 then branch to the top
cbz(x0, &LoopTop);
// Else we need to pause now
b(&ThreadPauseHandler);
}
else {
// Unconditionally loop to the top
// We will only stop on error when compiling a block or signal
b(&LoopTop);
}
}
}
@@ -352,7 +350,7 @@ void InterpreterCore::CreateAsmDispatch(FEXCore::Context::Context *ctx, FEXCore:
DispatcherConfig config;
config.ExecuteBlocksWithCall = true;
Dispatcher = std::make_unique<Arm64Dispatcher>(ctx, Thread, config);
Dispatcher = new Arm64Dispatcher(ctx, Thread, config);
DispatchPtr = Dispatcher->DispatchPtr;
CallbackPtr = Dispatcher->CallbackPtr;
@@ -108,7 +108,7 @@ bool Dispatcher::HandleGuestSignal(int Signal, void *info, void *ucontext, Guest
if (GuestAction->sa_flags & SA_SIGINFO) {
if (SRAEnabled) {
if (!IsAddressInJITCode(ArchHelpers::Context::GetPc(ucontext), false)) {
LOGMAN_THROW_A(!IsAddressInJITCode(ArchHelpers::Context::GetPc(ucontext), true), "Signals in dispatcher have unsynchronized context");
LogMan::Throw::A(!IsAddressInJITCode(ArchHelpers::Context::GetPc(ucontext), true), "Signals in dispatcher have unsynchronized context");
} else {
// We are in jit, SRA must be spilled
SpillSRA(ucontext);
@@ -121,11 +121,7 @@ bool Dispatcher::HandleGuestSignal(int Signal, void *info, void *ucontext, Guest
NewGuestSP -= sizeof(FEXCore::x86_64::ucontext_t);
uint64_t UContextLocation = NewGuestSP;
NewGuestSP -= sizeof(siginfo_t);
uint64_t SigInfoLocation = NewGuestSP;
FEXCore::x86_64::ucontext_t *guest_uctx = reinterpret_cast<FEXCore::x86_64::ucontext_t*>(UContextLocation);
siginfo_t *guest_siginfo = reinterpret_cast<siginfo_t*>(SigInfoLocation);
// We have extended float information
guest_uctx->uc_flags |= FEXCore::x86_64::UC_FP_XSTATE;
@@ -173,28 +169,8 @@ bool Dispatcher::HandleGuestSignal(int Signal, void *info, void *ucontext, Guest
guest_uctx->uc_stack.ss_sp = GuestStack->ss_sp;
guest_uctx->uc_stack.ss_size = GuestStack->ss_size;
// siginfo_t
siginfo_t *HostSigInfo = reinterpret_cast<siginfo_t*>(info);
if (HostSigInfo->si_code == SI_USER) {
// If the signal was a user signal then we need to pass this struct through unaltered
// Guest might be doing something with it
*guest_siginfo = *HostSigInfo;
}
else {
guest_siginfo->si_signo = Signal;
switch (Signal) {
case SIGSEGV:
case SIGBUS:
guest_siginfo->si_code = HostSigInfo->si_code;
guest_siginfo->si_errno = HostSigInfo->si_errno;
// Macro expansion to get the si_addr
guest_siginfo->si_addr = HostSigInfo->si_addr;
break;
default: LogMan::Msg::D("Unhandled siginfo_t signal: %d", Signal); break;
}
}
Frame->State.gregs[X86State::REG_RSI] = SigInfoLocation;
// XXX: siginfo_t(RSI)
Frame->State.gregs[X86State::REG_RSI] = 0x4142434445460000;
Frame->State.gregs[X86State::REG_RDX] = UContextLocation;
}
else {
@@ -227,7 +203,7 @@ bool Dispatcher::HandleGuestSignal(int Signal, void *info, void *ucontext, Guest
else {
NewGuestSP -= 4;
*(uint32_t*)NewGuestSP = CTX->X86CodeGen.SignalReturn;
LOGMAN_THROW_A(CTX->X86CodeGen.SignalReturn < 0x1'0000'0000ULL, "This needs to be below 4GB");
LogMan::Throw::A(CTX->X86CodeGen.SignalReturn < 0x1'0000'0000ULL, "This needs to be below 4GB");
Frame->State.gregs[X86State::REG_RSP] = NewGuestSP;
}
@@ -261,7 +237,7 @@ bool Dispatcher::HandleSignalPause(int Signal, void *info, void *ucontext) {
FEXCore::Core::SignalEvent SignalReason = ThreadState->SignalReason.load();
auto Frame = ThreadState->CurrentFrame;
if (SignalReason == FEXCore::Core::SignalEvent::Pause) {
if (SignalReason == FEXCore::Core::SignalEvent::SIGNALEVENT_PAUSE) {
// Store our thread state so we can come back to this
StoreThreadState(Signal, ucontext);
@@ -271,7 +247,7 @@ bool Dispatcher::HandleSignalPause(int Signal, void *info, void *ucontext) {
} else {
if (SRAEnabled) {
// We are in non-jit, SRA is already spilled
LOGMAN_THROW_A(!IsAddressInJITCode(ArchHelpers::Context::GetPc(ucontext), true), "Signals in dispatcher have unsynchronized context");
LogMan::Throw::A(!IsAddressInJITCode(ArchHelpers::Context::GetPc(ucontext), true), "Signals in dispatcher have unsynchronized context");
}
ArchHelpers::Context::SetPc(ucontext, ThreadPauseHandlerAddress);
}
@@ -286,11 +262,11 @@ bool Dispatcher::HandleSignalPause(int Signal, void *info, void *ucontext) {
// We use this to track if it is safe to clear cache
++SignalHandlerRefCounter;
ThreadState->SignalReason.store(FEXCore::Core::SignalEvent::Nothing);
ThreadState->SignalReason.store(FEXCore::Core::SIGNALEVENT_NONE);
return true;
}
if (SignalReason == FEXCore::Core::SignalEvent::Stop) {
if (SignalReason == FEXCore::Core::SignalEvent::SIGNALEVENT_STOP) {
// Our thread is stopping
// We don't care about anything at this point
// Set the stack to our starting location when we entered the core and get out safely
@@ -306,23 +282,23 @@ bool Dispatcher::HandleSignalPause(int Signal, void *info, void *ucontext) {
} else {
if (SRAEnabled) {
// We are in non-jit, SRA is already spilled
LOGMAN_THROW_A(!IsAddressInJITCode(ArchHelpers::Context::GetPc(ucontext), true), "Signals in dispatcher have unsynchronized context");
LogMan::Throw::A(!IsAddressInJITCode(ArchHelpers::Context::GetPc(ucontext), true), "Signals in dispatcher have unsynchronized context");
}
ArchHelpers::Context::SetPc(ucontext, ThreadStopHandlerAddress);
}
ThreadState->SignalReason.store(FEXCore::Core::SignalEvent::Nothing);
ThreadState->SignalReason.store(FEXCore::Core::SIGNALEVENT_NONE);
return true;
}
if (SignalReason == FEXCore::Core::SignalEvent::Return) {
if (SignalReason == FEXCore::Core::SignalEvent::SIGNALEVENT_RETURN) {
RestoreThreadState(ucontext);
// Ref count our faults
// We use this to track if it is safe to clear cache
--SignalHandlerRefCounter;
ThreadState->SignalReason.store(FEXCore::Core::SignalEvent::Nothing);
ThreadState->SignalReason.store(FEXCore::Core::SIGNALEVENT_NONE);
return true;
}
@@ -330,14 +306,14 @@ bool Dispatcher::HandleSignalPause(int Signal, void *info, void *ucontext) {
}
uint64_t Dispatcher::GetCompileBlockPtr() {
using ClassPtrType = void (FEXCore::Context::Context::*)(FEXCore::Core::CpuStateFrame *, uint64_t);
using ClassPtrType = uintptr_t (FEXCore::Context::Context::*)(FEXCore::Core::CpuStateFrame *, uint64_t);
union PtrCast {
ClassPtrType ClassPtr;
uintptr_t Data;
};
PtrCast CompileBlockPtr;
CompileBlockPtr.ClassPtr = &FEXCore::Context::Context::CompileBlockJit;
CompileBlockPtr.ClassPtr = &FEXCore::Context::Context::CompileBlock;
return CompileBlockPtr.Data;
}
@@ -351,7 +327,7 @@ void Dispatcher::RemoveCodeBuffer(uint8_t* start_to_remove) {
}
}
bool Dispatcher::IsAddressInJITCode(uint64_t Address, bool IncludeDispatcher) const {
bool Dispatcher::IsAddressInJITCode(uint64_t Address, bool IncludeDispatcher) {
for (auto [start, end] : CodeBuffers) {
if (Address >= start && Address < end) {
return true;
@@ -18,7 +18,6 @@ struct DispatcherConfig {
class Dispatcher {
public:
virtual ~Dispatcher() = default;
CPUBackend::AsmDispatch DispatchPtr;
CPUBackend::JITCallback CallbackPtr;
FEXCore::Context::Context::IntCallbackReturn ReturnPtr;
@@ -54,8 +53,8 @@ public:
void RemoveCodeBuffer(uint8_t* start);
bool IsAddressInJITCode(uint64_t Address, bool IncludeDispatcher = true) const;
bool IsAddressInDispatcher(uint64_t Address) const {
bool IsAddressInJITCode(uint64_t Address, bool IncludeDispatcher = true);
bool IsAddressInDispatcher(uint64_t Address) {
return Address >= Start && Address < End;
}
@@ -82,4 +81,4 @@ private:
std::vector<std::tuple<uint64_t, uint64_t>> CodeBuffers; // Start, End
};
}
}
@@ -12,7 +12,7 @@ static constexpr size_t MAX_DISPATCHER_CODE_SIZE = 4096;
X86Dispatcher::X86Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, DispatcherConfig &config)
: Dispatcher(ctx, Thread)
, Xbyak::CodeGenerator(MAX_DISPATCHER_CODE_SIZE, nullptr, this) {
, Xbyak::CodeGenerator(MAX_DISPATCHER_CODE_SIZE) {
using namespace Xbyak;
using namespace Xbyak::util;
@@ -66,7 +66,6 @@ X86Dispatcher::X86Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::Inte
Label LoopTop;
Label FullLookup;
Label CallBlock;
Label NoBlock;
Label ExitBlock;
Label ThreadPauseHandler;
@@ -78,20 +77,17 @@ X86Dispatcher::X86Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::Inte
// Load our RIP
mov(rdx, qword [STATE + offsetof(FEXCore::Core::CPUState, rip)]);
// L1 Cache
mov(r13, Thread->LookupCache->GetL1Pointer());
mov(rax, rdx);
if (!config.ExecuteBlocksWithCall)
{
// L1 Cache
mov(r13, Thread->LookupCache->GetL1Pointer());
mov(rax, rdx);
and_(rax, LookupCache::L1_ENTRIES_MASK);
shl(rax, 4);
cmp(qword[r13 + rax + 8], rdx);
jne(FullLookup);
if (!config.ExecuteBlocksWithCall) {
and_(rax, LookupCache::L1_ENTRIES_MASK);
shl(rax, 4);
cmp(qword[r13 + rax + 8], rdx);
jne(FullLookup);
jmp(qword[r13 + rax + 0]);
} else {
mov(rax, qword[r13 + rax + 0]);
jmp(CallBlock);
}
L(FullLookup);
@@ -126,19 +122,19 @@ X86Dispatcher::X86Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::Inte
je(NoBlock);
// Update L1
mov(r13, Thread->LookupCache->GetL1Pointer());
mov(rcx, rdx);
and_(rcx, LookupCache::L1_ENTRIES_MASK);
shl(rcx, 1);
mov(qword[r13 + rcx*8 + 8], rdx);
mov(qword[r13 + rcx*8 + 0], rax);
if (config.ExecuteBlocksWithCall) {
mov(r13, Thread->LookupCache->GetL1Pointer());
mov(rcx, rdx);
and_(rcx, LookupCache::L1_ENTRIES_MASK);
shl(rcx, 1);
mov(qword[r13 + rcx*8 + 8], rdx);
mov(qword[r13 + rcx*8 + 0], rax);
}
// Real block if we made it here
if (!config.ExecuteBlocksWithCall) {
jmp(rax);
} else {
L(CallBlock);
mov(rdi, STATE);
call(rax);
@@ -181,10 +177,19 @@ X86Dispatcher::X86Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::Inte
{
L(NoBlock);
using ClassPtrType = uintptr_t (FEXCore::Context::Context::*)(FEXCore::Core::CpuStateFrame *, uint64_t);
union PtrCast {
ClassPtrType ClassPtr;
uintptr_t Data;
};
PtrCast Ptr;
Ptr.ClassPtr = &FEXCore::Context::Context::CompileBlock;
// {rdi, rsi, rdx}
mov(rdi, reinterpret_cast<uint64_t>(CTX));
mov(rsi, STATE);
mov(rax, GetCompileBlockPtr());
mov(rax, Ptr.Data);
call(rax);
@@ -266,7 +271,7 @@ X86Dispatcher::X86Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::Inte
{
ReturnPtr = getCurr<FEXCore::Context::Context::IntCallbackReturn>();
// using CallbackReturn = FEX_NAKED void(*)(FEXCore::Core::InternalThreadState *Thread, volatile void *Host_RSP);
// using CallbackReturn = __attribute__((naked)) void(*)(FEXCore::Core::InternalThreadState *Thread, volatile void *Host_RSP);
// rdi = thread
// rsi = rsp
@@ -293,7 +298,7 @@ X86Dispatcher::X86Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::Inte
#if ENABLE_JITSYMBOLS
std::string Name = "Dispatch_" + std::to_string(::gettid());
CTX->Symbols.Register(reinterpret_cast<void*>(Start), End-Start, Name);
CTX->Symbols.Register(Start, End-Start, Name);
#endif
}
@@ -307,7 +312,7 @@ void InterpreterCore::CreateAsmDispatch(FEXCore::Context::Context *ctx, FEXCore:
DispatcherConfig config;
config.ExecuteBlocksWithCall = true;
Dispatcher = std::make_unique<X86Dispatcher>(ctx, Thread, config);
Dispatcher = new X86Dispatcher(ctx, Thread, config);
DispatchPtr = Dispatcher->DispatchPtr;
CallbackPtr = Dispatcher->CallbackPtr;
@@ -2,26 +2,16 @@
#include "Interface/Core/Dispatcher/Dispatcher.h"
#include <FEXCore/Utils/Allocator.h>
#define XBYAK64
#include <xbyak/xbyak.h>
namespace FEXCore::CPU {
class X86Dispatcher final : public Dispatcher, public Xbyak::CodeGenerator, public Xbyak::Allocator {
class X86Dispatcher final : public Dispatcher, public Xbyak::CodeGenerator {
public:
X86Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, DispatcherConfig &config);
virtual ~X86Dispatcher() override;
// Xbyak::Allocator
Xbyak::uint8 *alloc(size_t size) override { Size = size; return reinterpret_cast<uint8_t*>(FEXCore::Allocator::mmap(nullptr, size, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)); }
void free(Xbyak::uint8 *p) override { FEXCore::Allocator::munmap(p, Size); }
bool useProtect() const override { return false; }
private:
size_t Size{};
};
}
}
+62 -94
View File
@@ -15,7 +15,6 @@ $end_info$
#include <FEXCore/Core/X86Enums.h>
#include <FEXCore/Debug/X86Tables.h>
#include <FEXCore/Utils/LogManager.h>
#include <set>
namespace FEXCore::Frontend {
using namespace FEXCore::X86Tables;
@@ -125,13 +124,13 @@ Decoder::Decoder(FEXCore::Context::Context *ctx)
uint8_t Decoder::ReadByte() {
uint8_t Byte = InstStream[InstructionSize];
LOGMAN_THROW_A(InstructionSize < MAX_INST_SIZE, "Max instruction size exceeded!");
LogMan::Throw::A(InstructionSize < MAX_INST_SIZE, "Max instruction size exceeded!");
Instruction[InstructionSize] = Byte;
InstructionSize++;
return Byte;
}
uint8_t Decoder::PeekByte(uint8_t Offset) const {
uint8_t Decoder::PeekByte(uint8_t Offset) {
uint8_t Byte = InstStream[InstructionSize + Offset];
return Byte;
}
@@ -142,7 +141,7 @@ uint64_t Decoder::ReadData(uint8_t Size) {
}
if (Size > sizeof(uint64_t)) {
LOGMAN_MSG_A("Unknown data size to read");
LogMan::Msg::A("Unknown data size to read");
return 0;
}
@@ -197,9 +196,9 @@ void Decoder::DecodeModRM_16(X86Tables::DecodedOperand *Operand, X86Tables::ModR
}
}
Operand->Type = DecodedOperand::OpType::SIB;
Operand->Data.SIB.Scale = 1;
Operand->Data.SIB.Offset = Literal;
Operand->TypeSIB.Type = DecodedOperand::TYPE_SIB;
Operand->TypeSIB.Scale = 1;
Operand->TypeSIB.Offset = Literal;
// Only called when ModRM.mod != 0b11
struct Encodings {
@@ -238,8 +237,8 @@ void Decoder::DecodeModRM_16(X86Tables::DecodedOperand *Operand, X86Tables::ModR
uint8_t LookupIndex = ModRM.mod << 3 | ModRM.rm;
auto it = Lookup[LookupIndex];
Operand->Data.SIB.Base = it.Base;
Operand->Data.SIB.Index = it.Index;
Operand->TypeSIB.Base = it.Base;
Operand->TypeSIB.Index = it.Index;
}
void Decoder::DecodeModRM_64(X86Tables::DecodedOperand *Operand, X86Tables::ModRMDecoded ModRM) {
@@ -277,21 +276,21 @@ void Decoder::DecodeModRM_64(X86Tables::DecodedOperand *Operand, X86Tables::ModR
}
// SIB
Operand->Type = DecodedOperand::OpType::SIB;
Operand->Data.SIB.Scale = 1 << SIB.scale;
Operand->TypeSIB.Type = DecodedOperand::TYPE_SIB;
Operand->TypeSIB.Scale = 1 << SIB.scale;
// The invalid encoding types are described at Table 1-12. "promoted nsigned is always non-zero"
Operand->Data.SIB.Index = MapModRMToReg(DecodeInst->Flags & DecodeFlags::FLAG_REX_XGPR_X ? 1 : 0, SIB.index, false, false, false, false, 0b100);
Operand->Data.SIB.Base = MapModRMToReg(DecodeInst->Flags & DecodeFlags::FLAG_REX_XGPR_B ? 1 : 0, SIB.base, false, false, false, false, ModRM.mod == 0 ? 0b101 : 16);
Operand->TypeSIB.Index = MapModRMToReg(DecodeInst->Flags & DecodeFlags::FLAG_REX_XGPR_X ? 1 : 0, SIB.index, false, false, false, false, 0b100);
Operand->TypeSIB.Base = MapModRMToReg(DecodeInst->Flags & DecodeFlags::FLAG_REX_XGPR_B ? 1 : 0, SIB.base, false, false, false, false, ModRM.mod == 0 ? 0b101 : 16);
uint64_t Literal {0};
LOGMAN_THROW_A(Displacement <= 4, "Number of bytes should be <= 4 for literal src");
LogMan::Throw::A(Displacement <= 4, "Number of bytes should be <= 4 for literal src");
Literal = ReadData(Displacement);
if (Displacement == 1) {
Literal = static_cast<int8_t>(Literal);
}
Operand->Data.SIB.Offset = Literal;
Operand->TypeSIB.Offset = Literal;
}
else if (ModRM.mod == 0) {
// Explained in Table 1-14. "Operand Addressing Using ModRM and SIB Bytes"
@@ -300,13 +299,13 @@ void Decoder::DecodeModRM_64(X86Tables::DecodedOperand *Operand, X86Tables::ModR
uint32_t Literal;
Literal = ReadData(4);
Operand->Type = DecodedOperand::OpType::RIPRelative;
Operand->Data.RIPLiteral.Value.u = Literal;
Operand->TypeRIPLiteral.Type = DecodedOperand::TYPE_RIP_RELATIVE;
Operand->TypeRIPLiteral.Literal.u = Literal;
}
else {
// Register-direct addressing
Operand->Type = DecodedOperand::OpType::GPRDirect;
Operand->Data.GPR.GPR = MapModRMToReg(DecodeInst->Flags & DecodeFlags::FLAG_REX_XGPR_B ? 1 : 0, ModRM.rm, false, false, false, false);
Operand->TypeGPR.Type = DecodedOperand::TYPE_GPR_DIRECT;
Operand->TypeGPR.GPR = MapModRMToReg(DecodeInst->Flags & DecodeFlags::FLAG_REX_XGPR_B ? 1 : 0, ModRM.rm, false, false, false, false);
}
}
else {
@@ -318,9 +317,9 @@ void Decoder::DecodeModRM_64(X86Tables::DecodedOperand *Operand, X86Tables::ModR
}
Displacement = DisplacementSize;
Operand->Type = DecodedOperand::OpType::GPRIndirect;
Operand->Data.GPRIndirect.GPR = MapModRMToReg(DecodeInst->Flags & DecodeFlags::FLAG_REX_XGPR_B ? 1 : 0, ModRM.rm, false, false, false, false);
Operand->Data.GPRIndirect.Displacement = Literal;
Operand->TypeGPRIndirect.Type = DecodedOperand::TYPE_GPR_INDIRECT;
Operand->TypeGPRIndirect.GPR = MapModRMToReg(DecodeInst->Flags & DecodeFlags::FLAG_REX_XGPR_B ? 1 : 0, ModRM.rm, false, false, false, false);
Operand->TypeGPRIndirect.Displacement = Literal;
}
}
@@ -344,7 +343,7 @@ bool Decoder::NormalOp(FEXCore::X86Tables::X86InstInfo const *Info, uint16_t Op)
return false;
}
LOGMAN_THROW_A(!(Info->Type >= FEXCore::X86Tables::TYPE_GROUP_1 && Info->Type <= FEXCore::X86Tables::TYPE_GROUP_P),
LogMan::Throw::A(!(Info->Type >= FEXCore::X86Tables::TYPE_GROUP_1 && Info->Type <= FEXCore::X86Tables::TYPE_GROUP_P),
"Group Ops should have been decoded before this!");
uint8_t DestSize{};
@@ -460,25 +459,22 @@ bool Decoder::NormalOp(FEXCore::X86Tables::X86InstInfo const *Info, uint16_t Op)
if (HAS_NON_XMM_SUBFLAG(Info->Flags, FEXCore::X86Tables::InstFlags::FLAGS_SF_DST_RAX) ||
HAS_NON_XMM_SUBFLAG(Info->Flags, FEXCore::X86Tables::InstFlags::FLAGS_SF_DST_RDX)) {
// Some instructions hardcode their destination as RAX
CurrentDest->Type = DecodedOperand::OpType::GPR;
CurrentDest->Data.GPR.HighBits = false;
CurrentDest->Data.GPR.GPR = HAS_NON_XMM_SUBFLAG(Info->Flags, FEXCore::X86Tables::InstFlags::FLAGS_SF_DST_RAX) ? FEXCore::X86State::REG_RAX : FEXCore::X86State::REG_RDX;
CurrentDest->TypeGPR.Type = DecodedOperand::TYPE_GPR;
CurrentDest->TypeGPR.HighBits = false;
CurrentDest->TypeGPR.GPR = HAS_NON_XMM_SUBFLAG(Info->Flags, FEXCore::X86Tables::InstFlags::FLAGS_SF_DST_RAX) ? FEXCore::X86State::REG_RAX : FEXCore::X86State::REG_RDX;
CurrentDest = &DecodeInst->Src[0];
}
if (HAS_NON_XMM_SUBFLAG(Info->Flags, FEXCore::X86Tables::InstFlags::FLAGS_SF_REX_IN_BYTE)) {
LOGMAN_THROW_A(!HasMODRM, "This instruction shouldn't have ModRM!");
LogMan::Throw::A(!HasMODRM, "This instruction shouldn't have ModRM!");
// If the REX is in the byte that means the lower nibble of the OP contains the destination GPR
// This also means that the destination is always a GPR on these ones
// ADDITIONALLY:
// If there is a REX prefix then that allows extended GPR usage
CurrentDest->Type = DecodedOperand::OpType::GPR;
DecodeInst->Dest.Data.GPR.HighBits = (Is8BitDest && !HasREX && (Op & 0b111) >= 0b100) || HasHighXMM;
CurrentDest->Data.GPR.GPR = MapModRMToReg(DecodeInst->Flags & DecodeFlags::FLAG_REX_XGPR_B ? 1 : 0, Op & 0b111, Is8BitDest, HasREX, false, false);
if (CurrentDest->Data.GPR.GPR == FEXCore::X86State::REG_INVALID)
return false;
CurrentDest->TypeGPR.Type = DecodedOperand::TYPE_GPR;
DecodeInst->Dest.TypeGPR.HighBits = (Is8BitDest && !HasREX && (Op & 0b111) >= 0b100) || HasHighXMM;
CurrentDest->TypeGPR.GPR = MapModRMToReg(DecodeInst->Flags & DecodeFlags::FLAG_REX_XGPR_B ? 1 : 0, Op & 0b111, Is8BitDest, HasREX, false, false);
}
uint8_t Bytes = Info->MoreBytes;
@@ -501,63 +497,55 @@ bool Decoder::NormalOp(FEXCore::X86Tables::X86InstInfo const *Info, uint16_t Op)
ModRM.Hex = DecodeInst->ModRM;
// Decode the GPR source first
GPR.Type = DecodedOperand::OpType::GPR;
GPR.Data.GPR.HighBits = (GPR8Bit && ModRM.reg >= 0b100 && !HasREX) || HasHighXMM;
GPR.Data.GPR.GPR = MapModRMToReg(DecodeInst->Flags & DecodeFlags::FLAG_REX_XGPR_R ? 1 : 0, ModRM.reg, GPR8Bit, HasREX, HasXMMGPR, HasMMGPR);
if (GPR.Data.GPR.GPR == FEXCore::X86State::REG_INVALID)
return false;
GPR.TypeGPR.Type = DecodedOperand::TYPE_GPR;
GPR.TypeGPR.HighBits = (GPR8Bit && ModRM.reg >= 0b100 && !HasREX) || HasHighXMM;
GPR.TypeGPR.GPR = MapModRMToReg(DecodeInst->Flags & DecodeFlags::FLAG_REX_XGPR_R ? 1 : 0, ModRM.reg, GPR8Bit, HasREX, HasXMMGPR, HasMMGPR);
// ModRM.mod == 0b11 == Register
// ModRM.Mod != 0b11 == Register-direct addressing
if (ModRM.mod == 0b11) {
NonGPR.Type = DecodedOperand::OpType::GPR;
NonGPR.Data.GPR.HighBits = (NonGPR8Bit && ModRM.rm >= 0b100 && !HasREX) || HasHighXMM;
NonGPR.Data.GPR.GPR = MapModRMToReg(DecodeInst->Flags & DecodeFlags::FLAG_REX_XGPR_B ? 1 : 0, ModRM.rm, NonGPR8Bit, HasREX, HasXMMNonGPR, HasMMNonGPR);
if (NonGPR.Data.GPR.GPR == FEXCore::X86State::REG_INVALID)
return false;
NonGPR.TypeGPR.Type = DecodedOperand::TYPE_GPR;
NonGPR.TypeGPR.HighBits = (NonGPR8Bit && ModRM.rm >= 0b100 && !HasREX) || HasHighXMM;
NonGPR.TypeGPR.GPR = MapModRMToReg(DecodeInst->Flags & DecodeFlags::FLAG_REX_XGPR_B ? 1 : 0, ModRM.rm, NonGPR8Bit, HasREX, HasXMMNonGPR, HasMMNonGPR);
}
else {
auto Disp = DecodeModRMs_Disp[Has16BitAddressing];
(this->*Disp)(&NonGPR, ModRM);
}
return true;
};
size_t CurrentSrc = 0;
if (Info->Flags & FEXCore::X86Tables::InstFlags::FLAGS_MODRM) {
if (Info->Flags & FEXCore::X86Tables::InstFlags::FLAGS_SF_MOD_DST) {
if (!ModRMOperand(DecodeInst->Src[CurrentSrc], DecodeInst->Dest, HasXMMSrc, HasXMMDst, HasMMSrc, HasMMDst, Is8BitSrc, Is8BitDest))
return false;
ModRMOperand(DecodeInst->Src[CurrentSrc], DecodeInst->Dest, HasXMMSrc, HasXMMDst, HasMMSrc, HasMMDst, Is8BitSrc, Is8BitDest);
}
else {
if (!ModRMOperand(DecodeInst->Dest, DecodeInst->Src[CurrentSrc], HasXMMDst, HasXMMSrc, HasMMDst, HasMMSrc, Is8BitDest, Is8BitSrc))
return false;
ModRMOperand(DecodeInst->Dest, DecodeInst->Src[CurrentSrc], HasXMMDst, HasXMMSrc, HasMMDst, HasMMSrc, Is8BitDest, Is8BitSrc);
}
++CurrentSrc;
}
if (HAS_NON_XMM_SUBFLAG(Info->Flags, FEXCore::X86Tables::InstFlags::FLAGS_SF_SRC_RAX)) {
DecodeInst->Src[CurrentSrc].Type = DecodedOperand::OpType::GPR;
DecodeInst->Src[CurrentSrc].Data.GPR.HighBits = false;
DecodeInst->Src[CurrentSrc].Data.GPR.GPR = FEXCore::X86State::REG_RAX;
DecodeInst->Src[CurrentSrc].TypeGPR.Type = DecodedOperand::TYPE_GPR;
DecodeInst->Src[CurrentSrc].TypeGPR.HighBits = false;
DecodeInst->Src[CurrentSrc].TypeGPR.GPR = FEXCore::X86State::REG_RAX;
++CurrentSrc;
}
else if (HAS_NON_XMM_SUBFLAG(Info->Flags, FEXCore::X86Tables::InstFlags::FLAGS_SF_SRC_RCX)) {
DecodeInst->Src[CurrentSrc].Type = DecodedOperand::OpType::GPR;
DecodeInst->Src[CurrentSrc].Data.GPR.HighBits = false;
DecodeInst->Src[CurrentSrc].Data.GPR.GPR = FEXCore::X86State::REG_RCX;
DecodeInst->Src[CurrentSrc].TypeGPR.Type = DecodedOperand::TYPE_GPR;
DecodeInst->Src[CurrentSrc].TypeGPR.HighBits = false;
DecodeInst->Src[CurrentSrc].TypeGPR.GPR = FEXCore::X86State::REG_RCX;
++CurrentSrc;
}
if (Bytes != 0) {
LOGMAN_THROW_A(Bytes <= 8, "Number of bytes should be <= 8 for literal src");
LogMan::Throw::A(Bytes <= 8, "Number of bytes should be <= 8 for literal src");
DecodeInst->Src[CurrentSrc].Data.Literal.Size = Bytes;
DecodeInst->Src[CurrentSrc].TypeLiteral.Size = Bytes;
uint64_t Literal = ReadData(Bytes);
uint64_t Literal {0};
Literal = ReadData(Bytes);
if ((Info->Flags & FEXCore::X86Tables::InstFlags::FLAGS_SRC_SEXT) ||
(DecodeFlags::GetSizeDstFlags(DecodeInst->Flags) == DecodeFlags::SIZE_64BIT && Info->Flags & FEXCore::X86Tables::InstFlags::FLAGS_SRC_SEXT64BIT)) {
@@ -570,15 +558,15 @@ bool Decoder::NormalOp(FEXCore::X86Tables::X86InstInfo const *Info, uint16_t Op)
else {
Literal = static_cast<int32_t>(Literal);
}
DecodeInst->Src[CurrentSrc].Data.Literal.Size = DestSize;
DecodeInst->Src[CurrentSrc].TypeLiteral.Size = DestSize;
}
Bytes = 0;
DecodeInst->Src[CurrentSrc].Type = DecodedOperand::OpType::Literal;
DecodeInst->Src[CurrentSrc].Data.Literal.Value = Literal;
DecodeInst->Src[CurrentSrc].TypeLiteral.Type = DecodedOperand::TYPE_LITERAL;
DecodeInst->Src[CurrentSrc].TypeLiteral.Literal = Literal;
}
LOGMAN_THROW_A(Bytes == 0, "Inst at 0x%lx: 0x%04x '%s' Had an instruction of size %d with %d remaining", DecodeInst->PC, DecodeInst->OP, DecodeInst->TableInfo->Name, InstructionSize, Bytes);
LogMan::Throw::A(Bytes == 0, "Inst at 0x%lx: 0x%04x '%s' Had an instruction of size %d with %d remaining", DecodeInst->PC, DecodeInst->OP, DecodeInst->TableInfo->Name, InstructionSize, Bytes);
DecodeInst->InstSize = InstructionSize;
return true;
}
@@ -603,7 +591,7 @@ bool Decoder::NormalOpHeader(FEXCore::X86Tables::X86InstInfo const *Info, uint16
return false;
}
LOGMAN_THROW_A(Info->Type != FEXCore::X86Tables::TYPE_REX_PREFIX, "REX PREFIX should have been decoded before this!");
LogMan::Throw::A(Info->Type != FEXCore::X86Tables::TYPE_REX_PREFIX, "REX PREFIX should have been decoded before this!");
if (Info->Type >= FEXCore::X86Tables::TYPE_GROUP_1 &&
Info->Type <= FEXCore::X86Tables::TYPE_GROUP_11) {
@@ -659,7 +647,7 @@ bool Decoder::NormalOpHeader(FEXCore::X86Tables::X86InstInfo const *Info, uint16
3,
};
uint8_t Field = RegToField[ModRM.reg];
LOGMAN_THROW_A(Field != 255, "Invalid field selected!");
LogMan::Throw::A(Field != 255, "Invalid field selected!");
LocalOp = (Field << 3) | ModRM.rm;
return NormalOp(&SecondModRMTableOps[LocalOp], LocalOp);
@@ -693,10 +681,7 @@ bool Decoder::NormalOpHeader(FEXCore::X86Tables::X86InstInfo const *Info, uint16
uint8_t Byte2 = ReadByte();
pp = Byte2 & 0b11;
map_select = Byte1 & 0b11111;
if (!(map_select >= 1 && map_select <= 3)) {
LogMan::Msg::E("We don't understand a map_select of: %d", map_select);
return false;
}
LogMan::Throw::A(map_select >= 1 && map_select <= 3, "We don't understand a map_select of: %d", map_select);
}
uint16_t VEXOp = ReadByte();
@@ -744,8 +729,6 @@ bool Decoder::DecodeInstruction(uint64_t PC) {
DecodeInst->PC = PC;
for(;;) {
if (InstructionSize >= MAX_INST_SIZE)
return false;
uint8_t Op = ReadByte();
switch (Op) {
case 0x0F: {// Escape Op
@@ -896,7 +879,7 @@ bool Decoder::DecodeInstruction(uint64_t PC) {
auto Info = &FEXCore::X86Tables::BaseOps[Op];
if (Info->Type == FEXCore::X86Tables::TYPE_REX_PREFIX) {
LOGMAN_THROW_A(CTX->Config.Is64BitMode, "Got REX prefix in 32bit mode");
LogMan::Throw::A(CTX->Config.Is64BitMode, "Got REX prefix in 32bit mode");
DecodeInst->Flags |= DecodeFlags::FLAG_REX_PREFIX;
// Widening displacement
@@ -926,10 +909,6 @@ bool Decoder::DecodeInstruction(uint64_t PC) {
}
if (DecodeInst->Dest.IsGPR()) {
assert(DecodeInst->Dest.Data.GPR.GPR != 255);
}
return true;
}
@@ -949,23 +928,19 @@ void Decoder::BranchTargetInMultiblockRange() {
// auto RIPOffset = LoadSource(Op, Op->Src[0], Op->Flags);
// auto RIPTargetConst = _Constant(Op->PC + Op->InstSize);
// Target offset is PC + InstSize + Literal
LOGMAN_THROW_A(DecodeInst->Src[0].IsLiteral(), "Had wrong operand type");
TargetRIP = DecodeInst->PC + DecodeInst->InstSize + DecodeInst->Src[0].Data.Literal.Value;
LogMan::Throw::A(DecodeInst->Src[0].TypeNone.Type == DecodedOperand::TYPE_LITERAL, "Had wrong operand type");
TargetRIP = DecodeInst->PC + DecodeInst->InstSize + DecodeInst->Src[0].TypeLiteral.Literal;
break;
}
case 0xE9:
case 0xEB: // Both are unconditional JMP instructions
LOGMAN_THROW_A(DecodeInst->Src[0].IsLiteral(), "Had wrong operand type");
TargetRIP = DecodeInst->PC + DecodeInst->InstSize + DecodeInst->Src[0].Data.Literal.Value;
LogMan::Throw::A(DecodeInst->Src[0].TypeNone.Type == DecodedOperand::TYPE_LITERAL, "Had wrong operand type");
TargetRIP = DecodeInst->PC + DecodeInst->InstSize + DecodeInst->Src[0].TypeLiteral.Literal;
Conditional = false;
break;
case 0xE8: // Call - Immediate target, We don't want to inline calls
if (ExternalBranches) {
ExternalBranches->insert(DecodeInst->PC + DecodeInst->InstSize);
}
[[fallthrough]];
case 0xC2: // RET imm
case 0xC3: // RET
case 0xE8: // Call - Immediate target, We don't want to inline calls
default:
return;
break;
@@ -995,10 +970,6 @@ void Decoder::BranchTargetInMultiblockRange() {
BlocksToDecode.find(TargetRIP) == BlocksToDecode.end()) {
BlocksToDecode.emplace(TargetRIP);
}
} else {
if (ExternalBranches) {
ExternalBranches->insert(TargetRIP);
}
}
}
@@ -1022,7 +993,7 @@ bool Decoder::DecodeInstructionsAtEntry(uint8_t const* _InstStream, uint64_t PC)
// If we don't have symbols available then we become a bit optimistic about multiblock ranges
if (!SymbolAvailable) {
// If we don't have a symbol available then assume all branches are valid for multiblock
SymbolMaxAddress = SectionMaxAddress;
SymbolMaxAddress = ~0ULL;
SymbolMinAddress = EntryPoint;
}
@@ -1052,10 +1023,7 @@ bool Decoder::DecodeInstructionsAtEntry(uint8_t const* _InstStream, uint64_t PC)
if (ErrorDuringDecoding) {
LogMan::Msg::D("Couldn't Decode something at 0x%lx, Started at 0x%lx", PC + PCOffset, PC);
if (Blocks.size() == 1) {
return false;
}
LOGMAN_THROW_A(Blocks.size() != 1, "Decode Error in entry block");
LogMan::Throw::A(Blocks.size() != 1, "Decode Error in entry block");
CurrentBlockDecoding.HasInvalidInstruction = true;
if (ErrorDuringDecoding && Blocks.size() != 1) {
+2 -6
View File
@@ -26,15 +26,13 @@ public:
Decoder(FEXCore::Context::Context *ctx);
bool DecodeInstructionsAtEntry(uint8_t const* InstStream, uint64_t PC);
std::vector<DecodedBlocks> const *GetDecodedBlocks() const {
std::vector<DecodedBlocks> const *GetDecodedBlocks() {
return &Blocks;
}
uint64_t DecodedMinAddress {};
uint64_t DecodedMaxAddress {~0ULL};
void SetSectionMaxAddress(uint64_t v) { SectionMaxAddress = v; }
void SetExternalBranches(std::set<uint64_t> *v) { ExternalBranches = v; }
private:
FEXCore::Context::Context *CTX;
@@ -43,7 +41,7 @@ private:
void BranchTargetInMultiblockRange();
uint8_t ReadByte();
uint8_t PeekByte(uint8_t Offset) const;
uint8_t PeekByte(uint8_t Offset);
uint64_t ReadData(uint8_t Size);
void SkipBytes(uint8_t Size) { InstructionSize += Size; }
bool NormalOp(FEXCore::X86Tables::X86InstInfo const *Info, uint16_t Op);
@@ -67,12 +65,10 @@ private:
uint64_t MaxCondBranchBackwards {~0ULL};
uint64_t SymbolMaxAddress {};
uint64_t SymbolMinAddress {~0ULL};
uint64_t SectionMaxAddress {~0ULL};
std::vector<DecodedBlocks> Blocks;
std::set<uint64_t> BlocksToDecode;
std::set<uint64_t> HasBlocks;
std::set<uint64_t> *ExternalBranches {nullptr};
// ModRM rm decoding
using DecodeModRMPtr = void (FEXCore::Frontend::Decoder::*)(X86Tables::DecodedOperand *Operand, X86Tables::ModRMDecoded ModRM);
+90 -74
View File
@@ -15,19 +15,15 @@ $end_info$
#include <optional>
#include "Common/NetStream.h"
#include "Common/SoftFloat.h"
#include <FEXCore/Utils/CompilerDefs.h>
#include <FEXCore/Utils/LogManager.h>
#include <cstring>
#include <fcntl.h>
#include <fmt/format.h>
#include <fstream>
#include <netdb.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netdb.h>
#include <string.h>
#include <fcntl.h>
#include <unistd.h>
#include <fstream>
#include "GdbServer.h"
#include <FEXCore/Core/CodeLoader.h>
@@ -38,12 +34,12 @@ namespace FEXCore
void GdbServer::Break(int signal) {
std::lock_guard lk(sendMutex);
if (!CommsStream) {
return;
}
const auto str = fmt::format("S{:02x}", signal);
SendPacket(*CommsStream, str);
std::ostringstream ss;
ss << "S" << std::setfill('0') << std::setw(2) << std::hex << signal;
if (CommsStream)
SendPacket(*CommsStream, ss.str());
}
GdbServer::GdbServer(FEXCore::Context::Context *ctx) : CTX(ctx) {
@@ -69,7 +65,7 @@ GdbServer::GdbServer(FEXCore::Context::Context *ctx) : CTX(ctx) {
StartThread();
}
static int calculateChecksum(const std::string &packet) {
static int calculateChecksum(std::string &packet) {
unsigned char checksum = 0;
for (const char &c : packet) {
checksum += c;
@@ -103,9 +99,11 @@ static std::string encodeHex(unsigned char *data, size_t length) {
}
static std::string getThreadName(uint32_t ThreadID) {
const auto ThreadFile = fmt::format("/proc/{}/task/{}/comm", getpid(), ThreadID);
std::fstream fs(ThreadFile, std::fstream::in | std::fstream::binary);
std::fstream fs;
std::ostringstream ThreadFile;
ThreadFile << "/proc/" << getpid() << "/task/" << ThreadID << "/comm";
fs.open(ThreadFile.str(), std::fstream::in | std::fstream::binary);
if (fs.is_open()) {
std::string ThreadName;
fs >> ThreadName;
@@ -137,7 +135,7 @@ std::string GdbServer::ReadPacket(std::iostream &stream) {
switch(c) {
case '$': // start of packet
if (packet.size() != 0)
LogMan::Msg::EFmt("Dropping unexpected data: \"{}\"", packet);
LogMan::Msg::E("Dropping unexpected data: \"%s\"", packet.c_str());
// clear any existing data, must have been a mistake.
packet = std::string();
@@ -158,7 +156,7 @@ std::string GdbServer::ReadPacket(std::iostream &stream) {
if (calculateChecksum(packet) == expected_checksum) {
return packet;
} else {
LogMan::Msg::EFmt("Received Invalid Packet: ${}#{:02x}", packet, expected_checksum);
LogMan::Msg::E("Received Invalid Packet: $%s#%02x %c%c", packet.c_str(), expected_checksum);
}
break;
}
@@ -171,10 +169,10 @@ std::string GdbServer::ReadPacket(std::iostream &stream) {
return "";
}
static std::string escapePacket(const std::string& packet) {
static std::string escapePacket(std::string packet) {
std::ostringstream ss;
for(const auto &c : packet) {
for(auto &c : packet) {
switch (c) {
case '$':
case '#':
@@ -193,11 +191,13 @@ static std::string escapePacket(const std::string& packet) {
return ss.str();
}
void GdbServer::SendPacket(std::ostream &stream, const std::string& packet) {
const auto escaped = escapePacket(packet);
const auto str = fmt::format("${}#{:02x}", escaped, calculateChecksum(escaped));
void GdbServer::SendPacket(std::ostream &stream, std::string packet) {
auto escaped = escapePacket(packet);
std::ostringstream ss;
stream << str << std::flush;
ss << '$' << escaped << '#';
ss << std::setfill('0') << std::setw(2) << std::hex << (int)calculateChecksum(escaped);
stream << ss.str() << std::flush;
}
void GdbServer::SendACK(std::ostream &stream, bool NACK) {
@@ -218,7 +218,7 @@ void GdbServer::SendACK(std::ostream &stream, bool NACK) {
}
}
struct FEX_PACKED GDBContextDefinition {
struct __attribute__((packed)) GDBContextDefinition {
uint64_t gregs[16];
uint64_t rip;
uint32_t eflags;
@@ -279,7 +279,7 @@ std::string GdbServer::readRegs() {
return encodeHex((unsigned char *)&GDB, sizeof(GDBContextDefinition));
}
GdbServer::HandledPacketType GdbServer::readReg(const std::string& packet) {
GdbServer::HandledPacketType GdbServer::readReg(std::string& packet) {
size_t addr;
auto ss = std::istringstream(packet);
ss.get(); // Drop first letter
@@ -357,7 +357,7 @@ GdbServer::HandledPacketType GdbServer::readReg(const std::string& packet) {
return {encodeHex((unsigned char *)(&Empty), sizeof(uint32_t)), HandledPacketType::TYPE_ACK};
}
LogMan::Msg::EFmt("Unknown GDB register 0x{:x}", addr);
LogMan::Msg::E("Unknown GDB register 0x%lx", addr);
return {"E00", HandledPacketType::TYPE_ACK};
}
@@ -462,7 +462,7 @@ std::string buildTargetXML() {
return xml.str();
}
GdbServer::HandledPacketType GdbServer::handleXfer(const std::string &packet) {
GdbServer::HandledPacketType GdbServer::handleXfer(std::string &packet) {
std::string object;
std::string rw;
std::string annex;
@@ -548,9 +548,10 @@ GdbServer::HandledPacketType GdbServer::handleXfer(const std::string &packet) {
static size_t CheckMemMapping(uint64_t Address, size_t Size) {
uint64_t AddressEnd = Address + Size;
std::fstream fs("/proc/self/maps", std::fstream::in | std::fstream::binary);
std::string Line;
std::fstream fs;
fs.open("/proc/self/maps", std::fstream::in | std::fstream::binary);
std::string Line;
while (std::getline(fs, Line)) {
if (fs.eof()) break;
uint64_t Begin, End;
@@ -567,29 +568,32 @@ static size_t CheckMemMapping(uint64_t Address, size_t Size) {
}
}
fs.close();
return 0;
}
GdbServer::HandledPacketType GdbServer::handleProgramOffsets() {
std::fstream fs("/proc/self/maps", std::fstream::in | std::fstream::binary);
std::fstream fs;
fs.open("/proc/self/maps", std::fstream::in | std::fstream::binary);
std::string Line;
std::string const &RuntimeExecutable = Filename();
while (std::getline(fs, Line)) {
uint64_t Begin, End;
char Filename[255];
if (sscanf(Line.c_str(), "%lx-%lx %*c%*c%*c%*c %*x %*x:%*x %*d%s", &Begin, &End, Filename) == 3) {
if (RuntimeExecutable == Filename) {
auto str = fmt::format("Text={:x};Data={:x};Bss={:x}", Begin, Begin, Begin);
return {std::move(str), HandledPacketType::TYPE_ACK};
std::ostringstream ss;
ss << "Text=" << std::hex << Begin << ";Data=" << std::hex << Begin << ";Bss=" << std::hex << Begin;
ss << std::flush;
return {ss.str(), HandledPacketType::TYPE_ACK};
}
}
}
fs.close();
return {"Text=0;Data=0;Bss=0", HandledPacketType::TYPE_ACK};
}
GdbServer::HandledPacketType GdbServer::handleMemory(const std::string &packet) {
GdbServer::HandledPacketType GdbServer::handleMemory(std::string &packet) {
bool write;
size_t addr;
size_t length;
@@ -630,8 +634,8 @@ GdbServer::HandledPacketType GdbServer::handleMemory(const std::string &packet)
}
GdbServer::HandledPacketType GdbServer::handleQuery(const std::string &packet) {
const auto match = [&](const char *str) -> bool { return packet.rfind(str, 0) == 0; };
GdbServer::HandledPacketType GdbServer::handleQuery(std::string &packet) {
auto match = [&](const char *str) -> bool { return packet.rfind(str, 0) == 0; };
if (match("qSupported")) {
return {"PacketSize=5000;xmlRegisters=i386;qXfer:exec-file:read+;qXfer:features:read+;", HandledPacketType::TYPE_ACK};
@@ -689,8 +693,8 @@ GdbServer::HandledPacketType GdbServer::handleQuery(const std::string &packet) {
return {"", HandledPacketType::TYPE_UNKNOWN};
}
GdbServer::HandledPacketType GdbServer::handleV(const std::string& packet) {
const auto match = [&](const std::string& str) -> std::optional<std::istringstream> {
GdbServer::HandledPacketType GdbServer::handleV(std::string& packet) {
auto match = [&](std::string str) -> std::optional<std::istringstream> {
if (packet.rfind(str, 0) == 0) {
auto ss = std::istringstream(packet);
ss.seekg(str.size());
@@ -699,11 +703,18 @@ GdbServer::HandledPacketType GdbServer::handleV(const std::string& packet) {
return std::nullopt;
};
const auto F = [](int result) { return fmt::format("F{:x}", result); };
const auto F_error = [] { return fmt::format("F-1,{:x}", errno); };
const auto F_data = [](int result, const std::string& data) {
return fmt::format("F{:x};{}", result, data);
};
auto F = [](int result) {
std::ostringstream ss;
ss << "F" << std::hex << result;
return ss.str(); };
auto F_error = [&]() {
std::ostringstream ss;
ss << "F-1," << std::hex << errno;
return ss.str(); };
auto F_data = [&](int result, std::string data) {
std::ostringstream ss;
ss << "F" << std::hex << result << ";" << data;
return ss.str(); };
std::optional<std::istringstream> ss;
if((ss = match("vFile:open:"))) {
@@ -725,11 +736,11 @@ GdbServer::HandledPacketType GdbServer::handleV(const std::string& packet) {
return {F(pid == 0 ? 0 : -1), HandledPacketType::TYPE_ACK}; // Only support the common filesystem
}
if((ss = match("vFile:close:"))) {
int fd;
*ss >> std::hex >> fd;
close(fd);
return {F(0), HandledPacketType::TYPE_ACK};
}
int fd;
*ss >> std::hex >> fd;
close(fd);
return {F(0), HandledPacketType::TYPE_ACK};
}
if((ss = match("vFile:pread:"))) {
int fd, count, offset;
@@ -766,7 +777,7 @@ GdbServer::HandledPacketType GdbServer::handleV(const std::string& packet) {
}
if (ss->fail()) {
return {"E00", HandledPacketType::TYPE_ACK};
return {"E00", HandledPacketType::TYPE_ACK};
}
switch (action) {
@@ -776,25 +787,27 @@ GdbServer::HandledPacketType GdbServer::handleV(const std::string& packet) {
}
case 's': {
CTX->Step();
SendPacketPair({"OK", HandledPacketType::TYPE_ACK});
auto str = fmt::format("T05thread:{:02x};core:2c;", getpid());
SendPacketPair({std::move(str), HandledPacketType::TYPE_ACK});
SendPacketPair({"OK", HandledPacketType::TYPE_ACK});
std::ostringstream ss;
ss << "T05thread:" << std::setfill('0') << std::setw(2) << std::hex << getpid() << ";core:2c;";
SendPacketPair({ss.str(), HandledPacketType::TYPE_ACK});
return {"OK", HandledPacketType::TYPE_ACK};
}
case 't':
// This thread isn't part of the thread pool
CTX->Stop(false /* Ignore current thread */);
return {"OK", HandledPacketType::TYPE_ACK};
return {"OK", HandledPacketType::TYPE_ACK};
default:
return {"E00", HandledPacketType::TYPE_ACK};
return {"E00", HandledPacketType::TYPE_ACK};
}
}
return {"", HandledPacketType::TYPE_ACK};
return {"", HandledPacketType::TYPE_ACK};
}
GdbServer::HandledPacketType GdbServer::handleThreadOp(const std::string &packet) {
const auto match = [&](const char *str) -> bool { return packet.rfind(str, 0) == 0; };
GdbServer::HandledPacketType GdbServer::handleThreadOp(std::string &packet) {
auto match = [&](const char *str) -> bool { return packet.rfind(str, 0) == 0; };
if (match("Hc")) {
// Sets thread to this ID for stepping
@@ -810,7 +823,7 @@ GdbServer::HandledPacketType GdbServer::handleThreadOp(const std::string &packet
if (match("Hg")) {
// Sets thread for "other" operations
auto ss = std::istringstream(packet);
ss.seekg(std::string_view("Hg").size());
ss.seekg(std::string("Hg").size());
ss >> std::hex >> CurrentDebuggingThread;
// This must return quick otherwise IDA complains
@@ -821,7 +834,7 @@ GdbServer::HandledPacketType GdbServer::handleThreadOp(const std::string &packet
return {"", HandledPacketType::TYPE_UNKNOWN};
}
GdbServer::HandledPacketType GdbServer::handleBreakpoint(const std::string &packet) {
GdbServer::HandledPacketType GdbServer::handleBreakpoint(std::string &packet) {
auto ss = std::istringstream(packet);
bool Set{};
@@ -837,15 +850,17 @@ GdbServer::HandledPacketType GdbServer::handleBreakpoint(const std::string &pack
return {"OK", HandledPacketType::TYPE_ACK};
}
GdbServer::HandledPacketType GdbServer::ProcessPacket(const std::string &packet) {
GdbServer::HandledPacketType GdbServer::ProcessPacket(std::string &packet) {
switch (packet[0]) {
case '?': {
// Indicates the reason that the thread has stopped
// Behaviour changes if the target is in non-stop mode
// Binja doesn't support S response here
//return {"S00", HandledPacketType::TYPE_ACK};
auto str = fmt::format("T00thread:{:02x};core:2c;", getpid());
return {std::move(str), HandledPacketType::TYPE_ACK};
std::ostringstream ss;
ss << "T00thread:" << std::setfill('0') << std::setw(2) << std::hex << getpid() << ";core:2c;";
return {ss.str(), HandledPacketType::TYPE_ACK};
}
case 'g':
return {readRegs(), HandledPacketType::TYPE_ACK};
@@ -875,14 +890,14 @@ GdbServer::HandledPacketType GdbServer::ProcessPacket(const std::string &packet)
}
}
void GdbServer::SendPacketPair(const HandledPacketType& response) {
void GdbServer::SendPacketPair(HandledPacketType response) {
std::lock_guard lk(sendMutex);
if (response.TypeResponse == HandledPacketType::TYPE_ACK ||
response.TypeResponse == HandledPacketType::TYPE_ONLYACK) {
SendACK(*CommsStream, false);
}
else if (response.TypeResponse == HandledPacketType::TYPE_NACK ||
response.TypeResponse == HandledPacketType::TYPE_ONLYNACK) {
response.TypeResponse == HandledPacketType::TYPE_ONLYNACK) {
SendACK(*CommsStream, true);
}
@@ -890,8 +905,8 @@ void GdbServer::SendPacketPair(const HandledPacketType& response) {
SendPacket(*CommsStream, "");
}
else if (response.TypeResponse != HandledPacketType::TYPE_ONLYNACK &&
response.TypeResponse != HandledPacketType::TYPE_ONLYACK &&
response.TypeResponse != HandledPacketType::TYPE_NONE) {
response.TypeResponse != HandledPacketType::TYPE_ONLYACK &&
response.TypeResponse != HandledPacketType::TYPE_NONE) {
SendPacket(*CommsStream, response.Response);
}
}
@@ -912,7 +927,7 @@ void GdbServer::GdbServerLoop() {
response = ProcessPacket(packet);
SendPacketPair(response);
if (response.TypeResponse == HandledPacketType::TYPE_UNKNOWN) {
LogMan::Msg::DFmt("Unknown packet {}", packet);
LogMan::Msg::D("Unknown packet %s", packet.c_str());
}
break;
}
@@ -928,12 +943,13 @@ void GdbServer::GdbServerLoop() {
break;
case '\x03': { // ASCII EOT
CTX->Pause();
auto str = fmt::format("T02thread:{:02x};core:2c;", getpid());
SendPacketPair({std::move(str), HandledPacketType::TYPE_ACK});
std::ostringstream ss;
ss << "T02thread:" << std::setfill('0') << std::setw(2) << std::hex << getpid() << ";core:2c;";
SendPacketPair({ss.str(), HandledPacketType::TYPE_ACK});
break;
}
default:
LogMan::Msg::DFmt("GdbServer: Unexpected byte {} ({:02x})", static_cast<char>(c), c);
LogMan::Msg::D("GdbServer: Unexpected byte %c (%02x)", c, c);
}
}
@@ -987,7 +1003,7 @@ std::unique_ptr<std::iostream> GdbServer::OpenSocket() {
// Block until a connection arrives
LogMan::Msg::IFmt("GdbServer, waiting for connection on localhost:8086");
LogMan::Msg::I("GdbServer, waiting for connection on localhost:8086");
listen(sockfd, 1);
new_fd = accept(sockfd, (struct sockaddr *)&their_addr, &addr_size);
+10 -10
View File
@@ -30,7 +30,7 @@ private:
std::unique_ptr<std::iostream> OpenSocket();
void StartThread();
std::string ReadPacket(std::iostream &stream);
void SendPacket(std::ostream &stream, const std::string& packet);
void SendPacket(std::ostream &stream, std::string packet);
void SendACK(std::ostream &stream, bool NACK);
@@ -47,18 +47,18 @@ private:
ResponseType TypeResponse{};
};
void SendPacketPair(const HandledPacketType& packetPair);
HandledPacketType ProcessPacket(const std::string &packet);
HandledPacketType handleQuery(const std::string &packet);
HandledPacketType handleXfer(const std::string &packet);
HandledPacketType handleMemory(const std::string &packet);
HandledPacketType handleV(const std::string& packet);
HandledPacketType handleThreadOp(const std::string &packet);
HandledPacketType handleBreakpoint(const std::string &packet);
void SendPacketPair(HandledPacketType packetPair);
HandledPacketType ProcessPacket(std::string &packet);
HandledPacketType handleQuery(std::string &packet);
HandledPacketType handleXfer(std::string &packet);
HandledPacketType handleMemory(std::string &packet);
HandledPacketType handleV(std::string& packet);
HandledPacketType handleThreadOp(std::string &packet);
HandledPacketType handleBreakpoint(std::string &packet);
HandledPacketType handleProgramOffsets();
std::string readRegs();
HandledPacketType readReg(const std::string& packet);
HandledPacketType readReg(std::string& packet);
FEXCore::Context::Context *CTX;
std::unique_ptr<FEXCore::Threads::Thread> gdbServerThread;
@@ -22,8 +22,9 @@ using DestMapType = std::vector<uint32_t>;
class InterpreterCore final : public CPUBackend {
public:
explicit InterpreterCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, bool CompileThread);
~InterpreterCore() override;
std::string GetName() override { return "Interpreter"; }
void *CompileCode(uint64_t Entry, FEXCore::IR::IRListView const *IR, FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) override;
void *CompileCode(FEXCore::IR::IRListView const *IR, FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) override;
void *MapRegion(void* HostPtr, uint64_t, uint64_t) override { return HostPtr; }
@@ -45,7 +46,7 @@ private:
template<typename Res>
Res GetSrc(void* SSAData, IR::OrderedNodeWrapper Src);
std::unique_ptr<Dispatcher> Dispatcher{};
Dispatcher *Dispatcher{};
};
}
@@ -31,7 +31,7 @@ static void InterpreterExecution(FEXCore::Core::CpuStateFrame *Frame) {
auto LocalEntry = Thread->LocalIRCache.find(Thread->CurrentFrame->State.rip);
InterpreterOps::InterpretIR(Thread, Thread->CurrentFrame->State.rip, LocalEntry->second.IR.get(), LocalEntry->second.DebugData.get());
InterpreterOps::InterpretIR(Thread, LocalEntry->second.IR.get(), LocalEntry->second.DebugData.get());
}
bool InterpreterCore::HandleSIGBUS(int Signal, void *info, void *ucontext) {
@@ -111,12 +111,18 @@ InterpreterCore::InterpreterCore(FEXCore::Context::Context *ctx, FEXCore::Core::
}
}
void *InterpreterCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IRListView const *IR, [[maybe_unused]] FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) {
InterpreterCore::~InterpreterCore() {
delete Dispatcher;
}
void *InterpreterCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const *IR, [[maybe_unused]] FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) {
return reinterpret_cast<void*>(InterpreterExecution);
}
std::unique_ptr<CPUBackend> CreateInterpreterCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, bool CompileThread) {
return std::make_unique<InterpreterCore>(ctx, Thread, CompileThread);
FEXCore::CPU::CPUBackend *CreateInterpreterCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, bool CompileThread) {
return new InterpreterCore(ctx, Thread, CompileThread);
}
}
@@ -1,7 +1,5 @@
#pragma once
#include <memory>
namespace FEXCore::Context {
struct Context;
}
@@ -13,6 +11,6 @@ namespace FEXCore::Core {
namespace FEXCore::CPU {
class CPUBackend;
std::unique_ptr<CPUBackend> CreateInterpreterCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, bool CompileThread);
FEXCore::CPU::CPUBackend *CreateInterpreterCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, bool CompileThread);
}
File diff suppressed because it is too large. Load diff
@@ -36,7 +36,7 @@ namespace FEXCore::CPU {
class InterpreterOps {
public:
static void InterpretIR(FEXCore::Core::InternalThreadState *Thread, uint64_t Entry, FEXCore::IR::IRListView *CurrentIR, FEXCore::Core::DebugData *DebugData);
static void InterpretIR(FEXCore::Core::InternalThreadState *Thread, FEXCore::IR::IRListView *CurrentIR, FEXCore::Core::DebugData *DebugData);
static bool GetFallbackHandler(IR::IROp_Header *IROp, FallbackInfo *Info);
};
};
+53 -65
View File
@@ -46,7 +46,7 @@ DEF_OP(TruncElementPair) {
mov(Dst.second, Src.second);
break;
}
default: LOGMAN_MSG_A("Unhandled Truncation size: %d", Op->Size); break;
default: LogMan::Msg::A("Unhandled Truncation size: %d", Op->Size); break;
}
}
@@ -59,7 +59,7 @@ DEF_OP(Constant) {
DEF_OP(EntrypointOffset) {
auto Op = IROp->C<IR::IROp_EntrypointOffset>();
auto Constant = Entry + Op->Offset;
auto Constant = IR->GetHeader()->Entry + Op->Offset;
auto Dst = GetReg<RA_64>(Node);
LoadConstant(Dst, Constant);
}
@@ -95,7 +95,7 @@ DEF_OP(Add) {
case 8:
add(GetReg<RA_64>(Node), GetReg<RA_64>(Op->Header.Args[0].ID()), Const);
break;
default: LOGMAN_MSG_A("Unsupported Add size: %d", OpSize);
default: LogMan::Msg::A("Unsupported Add size: %d", OpSize);
}
} else {
switch (OpSize) {
@@ -105,7 +105,7 @@ DEF_OP(Add) {
case 8:
add(GetReg<RA_64>(Node), GetReg<RA_64>(Op->Header.Args[0].ID()), GetReg<RA_64>(Op->Header.Args[1].ID()));
break;
default: LOGMAN_MSG_A("Unsupported Add size: %d", OpSize);
default: LogMan::Msg::A("Unsupported Add size: %d", OpSize);
}
}
}
@@ -121,7 +121,7 @@ DEF_OP(Sub) {
case 8:
sub(GRS(Node), GRS(Op->Header.Args[0].ID()), Const);
break;
default: LOGMAN_MSG_A("Unsupported Sub size: %d", OpSize);
default: LogMan::Msg::A("Unsupported Sub size: %d", OpSize);
}
} else {
switch (OpSize) {
@@ -131,7 +131,7 @@ DEF_OP(Sub) {
case 8:
sub(GetReg<RA_64>(Node), GetReg<RA_64>(Op->Header.Args[0].ID()), GetReg<RA_64>(Op->Header.Args[1].ID()));
break;
default: LOGMAN_MSG_A("Unsupported Sub size: %d", OpSize);
default: LogMan::Msg::A("Unsupported Sub size: %d", OpSize);
}
}
@@ -147,7 +147,7 @@ DEF_OP(Neg) {
case 8:
neg(GetReg<RA_64>(Node), GetReg<RA_64>(Op->Header.Args[0].ID()));
break;
default: LOGMAN_MSG_A("Unsupported Not size: %d", OpSize);
default: LogMan::Msg::A("Unsupported Not size: %d", OpSize);
}
}
@@ -159,11 +159,12 @@ DEF_OP(Mul) {
switch (OpSize) {
case 4:
mul(Dst.W(), GetReg<RA_32>(Op->Header.Args[0].ID()), GetReg<RA_32>(Op->Header.Args[1].ID()));
sxtw(Dst, Dst);
break;
case 8:
mul(Dst, GetReg<RA_64>(Op->Header.Args[0].ID()), GetReg<RA_64>(Op->Header.Args[1].ID()));
break;
default: LOGMAN_MSG_A("Unknown Sext size: %d", OpSize);
default: LogMan::Msg::A("Unknown Sext size: %d", OpSize);
}
}
@@ -179,7 +180,7 @@ DEF_OP(UMul) {
case 8:
mul(Dst, GetReg<RA_64>(Op->Header.Args[0].ID()), GetReg<RA_64>(Op->Header.Args[1].ID()));
break;
default: LOGMAN_MSG_A("Unknown Sext size: %d", OpSize);
default: LogMan::Msg::A("Unknown Sext size: %d", OpSize);
}
}
@@ -216,7 +217,7 @@ DEF_OP(Div) {
sdiv(GetReg<RA_64>(Node), GetReg<RA_64>(Op->Header.Args[0].ID()), GetReg<RA_64>(Op->Header.Args[1].ID()));
break;
}
default: LOGMAN_MSG_A("Unknown DIV Size: %d", Size); break;
default: LogMan::Msg::A("Unknown DIV Size: %d", Size); break;
}
}
@@ -243,7 +244,7 @@ DEF_OP(UDiv) {
udiv(GetReg<RA_64>(Node), GetReg<RA_64>(Op->Header.Args[0].ID()), GetReg<RA_64>(Op->Header.Args[1].ID()));
break;
}
default: LOGMAN_MSG_A("Unknown UDIV Size: %d", Size); break;
default: LogMan::Msg::A("Unknown UDIV Size: %d", Size); break;
}
}
@@ -290,7 +291,7 @@ DEF_OP(Rem) {
msub(GetReg<RA_64>(Node), TMP1, Divisor, Dividend);
break;
}
default: LOGMAN_MSG_A("Unknown REM Size: %d", OpSize); break;
default: LogMan::Msg::A("Unknown REM Size: %d", OpSize); break;
}
}
@@ -332,7 +333,7 @@ DEF_OP(URem) {
msub(GetReg<RA_64>(Node), TMP1, Divisor, Dividend);
break;
}
default: LOGMAN_MSG_A("Unknown UREM Size: %d", OpSize); break;
default: LogMan::Msg::A("Unknown UREM Size: %d", OpSize); break;
}
}
@@ -344,12 +345,12 @@ DEF_OP(MulH) {
sxtw(TMP1, GetReg<RA_64>(Op->Header.Args[0].ID()));
sxtw(TMP2, GetReg<RA_64>(Op->Header.Args[1].ID()));
mul(TMP1, TMP1, TMP2);
ubfx(GetReg<RA_64>(Node), TMP1, 32, 32);
sbfx(GetReg<RA_64>(Node), TMP1, 32, 32);
break;
case 8:
smulh(GetReg<RA_64>(Node), GetReg<RA_64>(Op->Header.Args[0].ID()), GetReg<RA_64>(Op->Header.Args[1].ID()));
break;
default: LOGMAN_MSG_A("Unknown Sext size: %d", OpSize);
default: LogMan::Msg::A("Unknown Sext size: %d", OpSize);
}
}
@@ -366,7 +367,7 @@ DEF_OP(UMulH) {
case 8:
umulh(GetReg<RA_64>(Node), GetReg<RA_64>(Op->Header.Args[0].ID()), GetReg<RA_64>(Op->Header.Args[1].ID()));
break;
default: LOGMAN_MSG_A("Unknown Sext size: %d", OpSize);
default: LogMan::Msg::A("Unknown Sext size: %d", OpSize);
}
}
@@ -462,7 +463,7 @@ DEF_OP(Ror) {
break;
}
default: LOGMAN_MSG_A("Unhandled ROR size: %d", OpSize);
default: LogMan::Msg::A("Unhandled ROR size: %d", OpSize);
}
} else {
switch (OpSize) {
@@ -475,7 +476,7 @@ DEF_OP(Ror) {
break;
}
default: LOGMAN_MSG_A("Unhandled ROR size: %d", OpSize);
default: LogMan::Msg::A("Unhandled ROR size: %d", OpSize);
}
}
}
@@ -494,7 +495,7 @@ DEF_OP(Extr) {
break;
}
default: LOGMAN_MSG_A("Unhandled EXTR size: %d", OpSize);
default: LogMan::Msg::A("Unhandled EXTR size: %d", OpSize);
}
}
@@ -539,7 +540,7 @@ DEF_OP(LDiv) {
mov(GetReg<RA_64>(Node), x0);
break;
}
default: LOGMAN_MSG_A("Unknown LDIV Size: %d", Size); break;
default: LogMan::Msg::A("Unknown LDIV Size: %d", Size); break;
}
}
@@ -582,7 +583,7 @@ DEF_OP(LUDiv) {
mov(GetReg<RA_64>(Node), x0);
break;
}
default: LOGMAN_MSG_A("Unknown LUDIV Size: %d", Size); break;
default: LogMan::Msg::A("Unknown LUDIV Size: %d", Size); break;
}
}
@@ -635,7 +636,7 @@ DEF_OP(LRem) {
mov(GetReg<RA_64>(Node), x0);
break;
}
default: LOGMAN_MSG_A("Unknown LREM Size: %d", Size); break;
default: LogMan::Msg::A("Unknown LREM Size: %d", Size); break;
}
}
@@ -685,7 +686,7 @@ DEF_OP(LURem) {
mov(GetReg<RA_64>(Node), x0);
break;
}
default: LOGMAN_MSG_A("Unknown LUREM Size: %d", OpSize); break;
default: LogMan::Msg::A("Unknown LUREM Size: %d", OpSize); break;
}
}
@@ -699,7 +700,7 @@ DEF_OP(Not) {
case 8:
mvn(GetReg<RA_64>(Node), GetReg<RA_64>(Op->Header.Args[0].ID()));
break;
default: LOGMAN_MSG_A("Unsupported Not size: %d", OpSize);
default: LogMan::Msg::A("Unsupported Not size: %d", OpSize);
}
}
@@ -730,7 +731,7 @@ DEF_OP(Popcount) {
// fmov has zero extended, unused bytes are zero
addv(VTMP1.B(), VTMP1.V8B());
break;
default: LOGMAN_MSG_A("Unsupported Popcount size: %d", OpSize);
default: LogMan::Msg::A("Unsupported Popcount size: %d", OpSize);
}
auto Dst = GetReg<RA_32>(Node);
@@ -779,7 +780,7 @@ DEF_OP(FindMSB) {
clz(Dst, GetReg<RA_64>(Op->Header.Args[0].ID()));
sub(Dst, TMP1, Dst);
break;
default: LOGMAN_MSG_A("Unknown REV size: %d", OpSize); break;
default: LogMan::Msg::A("Unknown REV size: %d", OpSize); break;
}
}
@@ -800,7 +801,7 @@ DEF_OP(FindTrailingZeros) {
rbit(GetReg<RA_64>(Node), GetReg<RA_64>(Op->Header.Args[0].ID()));
clz(GetReg<RA_64>(Node), GetReg<RA_64>(Node));
break;
default: LOGMAN_MSG_A("Unknown size: %d", OpSize); break;
default: LogMan::Msg::A("Unknown size: %d", OpSize); break;
}
}
@@ -819,7 +820,7 @@ DEF_OP(CountLeadingZeroes) {
case 8:
clz(GetReg<RA_64>(Node), GetReg<RA_64>(Op->Header.Args[0].ID()));
break;
default: LOGMAN_MSG_A("Unknown size: %d", OpSize); break;
default: LogMan::Msg::A("Unknown size: %d", OpSize); break;
}
}
@@ -837,7 +838,7 @@ DEF_OP(Rev) {
case 8:
rev(GetReg<RA_64>(Node), GetReg<RA_64>(Op->Header.Args[0].ID()));
break;
default: LOGMAN_MSG_A("Unknown REV size: %d", OpSize); break;
default: LogMan::Msg::A("Unknown REV size: %d", OpSize); break;
}
}
@@ -859,15 +860,15 @@ DEF_OP(Bfi) {
bfi(TMP1, GetReg<RA_64>(Op->Header.Args[1].ID()), Op->lsb, Op->Width);
mov(GetReg<RA_64>(Node), TMP1);
break;
default: LOGMAN_MSG_A("Unknown BFI size: %d", OpSize); break;
default: LogMan::Msg::A("Unknown BFI size: %d", OpSize); break;
}
}
DEF_OP(Bfe) {
auto Op = IROp->C<IR::IROp_Bfe>();
uint8_t OpSize = IROp->Size;
LOGMAN_THROW_A(OpSize <= 8, "OpSize is too large for BFE: %d", OpSize);
LOGMAN_THROW_A(Op->Width != 0, "Invalid BFE width of 0");
LogMan::Throw::A(OpSize <= 8, "OpSize is too large for BFE: %d", OpSize);
LogMan::Throw::A(Op->Width != 0, "Invalid BFE width of 0");
auto Dst = GetReg<RA_64>(Node);
ubfx(Dst, GetReg<RA_64>(Op->Header.Args[0].ID()), Op->lsb, Op->Width);
@@ -912,7 +913,7 @@ Condition MapSelectCC(IR::CondClassType Cond) {
case FEXCore::IR::COND_MI:
case FEXCore::IR::COND_PL:
default:
LOGMAN_MSG_A("Unsupported compare type");
LogMan::Msg::A("Unsupported compare type");
return Condition::nv;
}
}
@@ -930,7 +931,7 @@ DEF_OP(Select) {
} else if (IsFPR(Op->Cmp1.ID())) {
fcmp(GRFCMP(Op->Cmp1.ID()), GRFCMP(Op->Cmp2.ID()));
} else {
LOGMAN_MSG_A("Select: Expected GPR or FPR");
LogMan::Msg::A("Select: Expected GPR or FPR");
}
auto cc = MapSelectCC(Op->Cond);
@@ -941,7 +942,7 @@ DEF_OP(Select) {
if (is_const_true || is_const_false) {
if (is_const_false != true || is_const_true != true || const_true != 1 || const_false != 0) {
LOGMAN_MSG_A("Select: Unsupported compare inline parameters");
LogMan::Msg::A("Select: Unsupported compare inline parameters");
}
cset(GRS(Node), cc);
} else {
@@ -965,53 +966,38 @@ DEF_OP(VExtractToGPR) {
case 8:
umov(GetReg<RA_64>(Node), GetSrc(Op->Header.Args[0].ID()).V2D(), Op->Idx);
break;
default: LOGMAN_MSG_A("Unhandled ExtractElementSize: %d", OpSize);
default: LogMan::Msg::A("Unhandled ExtractElementSize: %d", OpSize);
}
}
DEF_OP(Float_ToGPR_ZU) {
LogMan::Msg::D("Unimplemented");
}
DEF_OP(Float_ToGPR_ZS) {
auto Op = IROp->C<IR::IROp_Float_ToGPR_ZS>();
aarch64::Register Dst{};
aarch64::VRegister Src{};
if (Op->SrcElementSize == 8) {
Src = GetSrc(Op->Header.Args[0].ID()).D();
if (Op->Header.ElementSize == 8) {
fcvtzs(GetReg<RA_64>(Node), GetSrc(Op->Header.Args[0].ID()).D());
}
else {
Src = GetSrc(Op->Header.Args[0].ID()).S();
fcvtzs(GetReg<RA_32>(Node), GetSrc(Op->Header.Args[0].ID()).S());
}
}
if (IROp->Size == 8) {
Dst = GetReg<RA_64>(Node);
}
else {
Dst = GetReg<RA_32>(Node);
}
fcvtzs(Dst, Src);
DEF_OP(Float_ToGPR_U) {
LogMan::Msg::D("Unimplemented");
}
DEF_OP(Float_ToGPR_S) {
auto Op = IROp->C<IR::IROp_Float_ToGPR_S>();
aarch64::Register Dst{};
aarch64::VRegister Src{};
if (Op->SrcElementSize == 8) {
if (Op->Header.ElementSize == 8) {
frinti(VTMP1.D(), GetSrc(Op->Header.Args[0].ID()).D());
Src = VTMP1.D();
fcvtzs(GetReg<RA_64>(Node), VTMP1.D());
}
else {
frinti(VTMP1.S(), GetSrc(Op->Header.Args[0].ID()).S());
Src = VTMP1.S();
fcvtzs(GetReg<RA_32>(Node), VTMP1.S());
}
if (IROp->Size == 8) {
Dst = GetReg<RA_64>(Node);
}
else {
Dst = GetReg<RA_32>(Node);
}
fcvtzs(Dst, Src);
}
DEF_OP(FCmp) {
@@ -1028,7 +1014,7 @@ DEF_OP(FCmp) {
bool set = false;
if (Op->Flags & (1 << IR::FCMP_FLAG_EQ)) {
LOGMAN_THROW_A(IR::FCMP_FLAG_EQ == 0, "IR::FCMP_FLAG_EQ must equal 0");
LogMan::Throw::A(IR::FCMP_FLAG_EQ == 0, "IR::FCMP_FLAG_EQ must equal 0");
// EQ or unordered
cset(Dst, Condition::eq); // Z = 1
csinc(Dst, Dst, xzr, Condition::vc); // IF !V ? Z : 1
@@ -1102,7 +1088,9 @@ void Arm64JITCore::RegisterALUHandlers() {
REGISTER_OP(SBFE, Sbfe);
REGISTER_OP(SELECT, Select);
REGISTER_OP(VEXTRACTTOGPR, VExtractToGPR);
REGISTER_OP(FLOAT_TOGPR_ZU, Float_ToGPR_ZU);
REGISTER_OP(FLOAT_TOGPR_ZS, Float_ToGPR_ZS);
REGISTER_OP(FLOAT_TOGPR_U, Float_ToGPR_U);
REGISTER_OP(FLOAT_TOGPR_S, Float_ToGPR_S);
REGISTER_OP(FCMP, FCmp);
@@ -34,7 +34,7 @@ DEF_OP(CASPair) {
mov(Dst.first, TMP3);
mov(Dst.second, TMP4);
break;
default: LOGMAN_MSG_A("Unsupported: %d", OpSize);
default: LogMan::Msg::A("Unsupported: %d", OpSize);
}
}
else {
@@ -89,7 +89,7 @@ DEF_OP(CASPair) {
bind(&LoopExpected);
break;
}
default: LOGMAN_MSG_A("Unsupported: %d", OpSize);
default: LogMan::Msg::A("Unsupported: %d", OpSize);
}
}
}
@@ -115,7 +115,7 @@ DEF_OP(CAS) {
case 2: casalh(TMP2.W(), Desired.W(), MemOperand(MemSrc)); break;
case 4: casal(TMP2.W(), Desired.W(), MemOperand(MemSrc)); break;
case 8: casal(TMP2.X(), Desired.X(), MemOperand(MemSrc)); break;
default: LOGMAN_MSG_A("Unsupported: %d", OpSize);
default: LogMan::Msg::A("Unsupported: %d", OpSize);
}
mov(GetReg<RA_64>(Node), TMP2);
}
@@ -206,7 +206,7 @@ DEF_OP(CAS) {
break;
}
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", OpSize);
default: LogMan::Msg::A("Unhandled Atomic size: %d", OpSize);
}
}
}
@@ -222,7 +222,7 @@ DEF_OP(AtomicAdd) {
case 2: staddlh(GetReg<RA_32>(Op->Header.Args[1].ID()), MemOperand(MemSrc)); break;
case 4: staddl(GetReg<RA_32>(Op->Header.Args[1].ID()), MemOperand(MemSrc)); break;
case 8: staddl(GetReg<RA_64>(Op->Header.Args[1].ID()), MemOperand(MemSrc)); break;
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
else {
@@ -264,7 +264,7 @@ DEF_OP(AtomicAdd) {
cbnz(TMP2, &LoopTop);
break;
}
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
}
@@ -281,7 +281,7 @@ DEF_OP(AtomicSub) {
case 2: staddlh(TMP2.W(), MemOperand(MemSrc)); break;
case 4: staddl(TMP2.W(), MemOperand(MemSrc)); break;
case 8: staddl(TMP2.X(), MemOperand(MemSrc)); break;
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
else {
@@ -323,7 +323,7 @@ DEF_OP(AtomicSub) {
cbnz(TMP2, &LoopTop);
break;
}
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
}
@@ -340,7 +340,7 @@ DEF_OP(AtomicAnd) {
case 2: stclrlh(TMP2.W(), MemOperand(MemSrc)); break;
case 4: stclrl(TMP2.W(), MemOperand(MemSrc)); break;
case 8: stclrl(TMP2.X(), MemOperand(MemSrc)); break;
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
else {
@@ -382,7 +382,7 @@ DEF_OP(AtomicAnd) {
cbnz(TMP2, &LoopTop);
break;
}
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
}
@@ -398,7 +398,7 @@ DEF_OP(AtomicOr) {
case 2: stsetlh(GetReg<RA_32>(Op->Header.Args[1].ID()), MemOperand(MemSrc)); break;
case 4: stsetl(GetReg<RA_32>(Op->Header.Args[1].ID()), MemOperand(MemSrc)); break;
case 8: stsetl(GetReg<RA_64>(Op->Header.Args[1].ID()), MemOperand(MemSrc)); break;
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
else {
@@ -440,7 +440,7 @@ DEF_OP(AtomicOr) {
cbnz(TMP2, &LoopTop);
break;
}
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
}
@@ -456,7 +456,7 @@ DEF_OP(AtomicXor) {
case 2: steorlh(GetReg<RA_32>(Op->Header.Args[1].ID()), MemOperand(MemSrc)); break;
case 4: steorl(GetReg<RA_32>(Op->Header.Args[1].ID()), MemOperand(MemSrc)); break;
case 8: steorl(GetReg<RA_64>(Op->Header.Args[1].ID()), MemOperand(MemSrc)); break;
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
else {
@@ -498,7 +498,7 @@ DEF_OP(AtomicXor) {
cbnz(TMP2, &LoopTop);
break;
}
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
}
@@ -515,7 +515,7 @@ DEF_OP(AtomicSwap) {
case 2: swplh(TMP2.W(), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
case 4: swpl(TMP2.W(), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
case 8: swpl(TMP2.X(), GetReg<RA_64>(Node), MemOperand(MemSrc)); break;
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
else {
@@ -558,7 +558,7 @@ DEF_OP(AtomicSwap) {
mov(GetReg<RA_64>(Node), TMP2.X());
break;
}
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
}
@@ -573,7 +573,7 @@ DEF_OP(AtomicFetchAdd) {
case 2: ldaddalh(GetReg<RA_32>(Op->Header.Args[1].ID()), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
case 4: ldaddal(GetReg<RA_32>(Op->Header.Args[1].ID()), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
case 8: ldaddal(GetReg<RA_64>(Op->Header.Args[1].ID()), GetReg<RA_64>(Node), MemOperand(MemSrc)); break;
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
else {
@@ -619,7 +619,7 @@ DEF_OP(AtomicFetchAdd) {
mov(GetReg<RA_64>(Node), TMP2);
break;
}
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
}
@@ -635,7 +635,7 @@ DEF_OP(AtomicFetchSub) {
case 2: ldaddalh(TMP2.W(), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
case 4: ldaddal(TMP2.W(), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
case 8: ldaddal(TMP2.X(), GetReg<RA_64>(Node), MemOperand(MemSrc)); break;
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
else {
@@ -681,7 +681,7 @@ DEF_OP(AtomicFetchSub) {
mov(GetReg<RA_64>(Node), TMP2);
break;
}
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
}
@@ -697,7 +697,7 @@ DEF_OP(AtomicFetchAnd) {
case 2: ldclralh(TMP2.W(), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
case 4: ldclral(TMP2.W(), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
case 8: ldclral(TMP2.X(), GetReg<RA_64>(Node), MemOperand(MemSrc)); break;
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
else {
@@ -743,7 +743,7 @@ DEF_OP(AtomicFetchAnd) {
mov(GetReg<RA_64>(Node), TMP2);
break;
}
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
}
@@ -758,7 +758,7 @@ DEF_OP(AtomicFetchOr) {
case 2: ldsetalh(GetReg<RA_32>(Op->Header.Args[1].ID()), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
case 4: ldsetal(GetReg<RA_32>(Op->Header.Args[1].ID()), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
case 8: ldsetal(GetReg<RA_64>(Op->Header.Args[1].ID()), GetReg<RA_64>(Node), MemOperand(MemSrc)); break;
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
else {
@@ -804,7 +804,7 @@ DEF_OP(AtomicFetchOr) {
mov(GetReg<RA_64>(Node), TMP2);
break;
}
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
}
@@ -819,7 +819,7 @@ DEF_OP(AtomicFetchXor) {
case 2: ldeoralh(GetReg<RA_32>(Op->Header.Args[1].ID()), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
case 4: ldeoral(GetReg<RA_32>(Op->Header.Args[1].ID()), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
case 8: ldeoral(GetReg<RA_64>(Op->Header.Args[1].ID()), GetReg<RA_64>(Node), MemOperand(MemSrc)); break;
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
else {
@@ -865,7 +865,7 @@ DEF_OP(AtomicFetchXor) {
mov(GetReg<RA_64>(Node), TMP2);
break;
}
default: LOGMAN_MSG_A("Unhandled Atomic size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled Atomic size: %d", Op->Size);
}
}
}
@@ -73,7 +73,7 @@ DEF_OP(ExitFunction) {
uint64_t NewRIP;
if (IsInlineConstant(Op->NewRIP, &NewRIP) || IsInlineEntrypointOffset(Op->NewRIP, &NewRIP)) {
Literal l_BranchHost{ThreadSharedData.Dispatcher->ExitFunctionLinkerAddress};
Literal l_BranchHost{Dispatcher->ExitFunctionLinkerAddress};
Literal l_BranchGuest{NewRIP};
ldr(x0, &l_BranchHost);
@@ -96,7 +96,7 @@ DEF_OP(ExitFunction) {
br(x1);
bind(&FullLookup);
LoadConstant(TMP1, ThreadSharedData.Dispatcher->AbsoluteLoopTopAddress);
LoadConstant(TMP1, Dispatcher->AbsoluteLoopTopAddress);
str(RipReg, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.rip)));
br(TMP1);
}
@@ -142,7 +142,7 @@ Condition MapBranchCC(IR::CondClassType Cond) {
case FEXCore::IR::COND_MI:
case FEXCore::IR::COND_PL:
default:
LOGMAN_MSG_A("Unsupported compare type");
LogMan::Msg::A("Unsupported compare type");
return Condition::nv;
}
}
@@ -169,10 +169,10 @@ DEF_OP(CondJump) {
bool isConst = IsInlineConstant(Op->Cmp2, &Const);
if (isConst && Const == 0 && Op->Cond.Val == FEXCore::IR::COND_EQ) {
LOGMAN_THROW_A(IsGPR(Op->Cmp1.ID()), "CondJump: Expected GPR");
LogMan::Throw::A(IsGPR(Op->Cmp1.ID()), "CondJump: Expected GPR");
cbz(GRCMP(Op->Cmp1.ID()), TrueTargetLabel);
} else if (isConst && Const == 0 && Op->Cond.Val == FEXCore::IR::COND_NEQ) {
LOGMAN_THROW_A(IsGPR(Op->Cmp1.ID()), "CondJump: Expected GPR");
LogMan::Throw::A(IsGPR(Op->Cmp1.ID()), "CondJump: Expected GPR");
cbnz(GRCMP(Op->Cmp1.ID()), TrueTargetLabel);
} else {
if (IsGPR(Op->Cmp1.ID())) {
@@ -183,7 +183,7 @@ DEF_OP(CondJump) {
} else if (IsFPR(Op->Cmp1.ID())) {
fcmp(GRFCMP(Op->Cmp1.ID()), GRFCMP(Op->Cmp2.ID()));
} else {
LOGMAN_MSG_A("CondJump: Expected GPR or FPR");
LogMan::Msg::A("CondJump: Expected GPR or FPR");
}
b(TrueTargetLabel, MapBranchCC(Op->Cond));
@@ -262,7 +262,7 @@ DEF_OP(ValidateCode) {
int idx = 0;
LoadConstant(GetReg<RA_64>(Node), 0);
LoadConstant(x0, Entry + Op->Offset);
LoadConstant(x0, IR->GetHeader()->Entry + Op->Offset);
LoadConstant(x1, 1);
while (len >= 8)
@@ -311,7 +311,7 @@ DEF_OP(RemoveCodeEntry) {
PushDynamicRegsAndLR();
mov(x0, STATE);
LoadConstant(x1, Entry);
LoadConstant(x1, IR->GetHeader()->Entry);
LoadConstant(x2, reinterpret_cast<uintptr_t>(&Context::Context::RemoveCodeEntryFromJit));
SpillStaticRegs();
@@ -31,7 +31,7 @@ DEF_OP(VInsGPR) {
ins(GetDst(Node).V2D(), Op->Index, GetReg<RA_64>(Op->Header.Args[1].ID()));
break;
}
default: LOGMAN_MSG_A("Unknown Element Size: %d", Op->Header.ElementSize); break;
default: LogMan::Msg::A("Unknown Element Size: %d", Op->Header.ElementSize); break;
}
}
@@ -52,10 +52,14 @@ DEF_OP(VCastFromGPR) {
case 8:
fmov(GetDst(Node).D(), GetReg<RA_64>(Op->Header.Args[0].ID()).X());
break;
default: LOGMAN_MSG_A("Unknown castGPR element size: %d", Op->Header.ElementSize);
default: LogMan::Msg::A("Unknown castGPR element size: %d", Op->Header.ElementSize);
}
}
DEF_OP(Float_FromGPR_U) {
LogMan::Msg::A("Unimplemented");
}
DEF_OP(Float_FromGPR_S) {
auto Op = IROp->C<IR::IROp_Float_FromGPR_S>();
uint16_t Conv = (Op->Header.ElementSize << 8) | Op->SrcElementSize;
@@ -91,7 +95,20 @@ DEF_OP(Float_FToF) {
fcvt(GetDst(Node).S(), GetSrc(Op->Header.Args[0].ID()).D());
break;
}
default: LOGMAN_MSG_A("Unknown FCVT sizes: 0x%x", Conv);
default: LogMan::Msg::A("Unknown FCVT sizes: 0x%x", Conv);
}
}
DEF_OP(Vector_UToF) {
auto Op = IROp->C<IR::IROp_Vector_UToF>();
switch (Op->Header.ElementSize) {
case 4:
ucvtf(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S());
break;
case 8:
ucvtf(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D());
break;
default: LogMan::Msg::A("Unknown castGPR element size: %d", Op->Header.ElementSize);
}
}
@@ -104,7 +121,20 @@ DEF_OP(Vector_SToF) {
case 8:
scvtf(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D());
break;
default: LOGMAN_MSG_A("Unknown castGPR element size: %d", Op->Header.ElementSize);
default: LogMan::Msg::A("Unknown castGPR element size: %d", Op->Header.ElementSize);
}
}
DEF_OP(Vector_FToZU) {
auto Op = IROp->C<IR::IROp_Vector_FToZU>();
switch (Op->Header.ElementSize) {
case 4:
fcvtzu(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S());
break;
case 8:
fcvtzu(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D());
break;
default: LogMan::Msg::A("Unknown castGPR element size: %d", Op->Header.ElementSize);
}
}
@@ -117,7 +147,22 @@ DEF_OP(Vector_FToZS) {
case 8:
fcvtzs(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D());
break;
default: LOGMAN_MSG_A("Unknown castGPR element size: %d", Op->Header.ElementSize);
default: LogMan::Msg::A("Unknown castGPR element size: %d", Op->Header.ElementSize);
}
}
DEF_OP(Vector_FToU) {
auto Op = IROp->C<IR::IROp_Vector_FToU>();
switch (Op->Header.ElementSize) {
case 4:
frinti(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S());
fcvtzu(GetDst(Node).V4S(), GetDst(Node).V4S());
break;
case 8:
frinti(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D());
fcvtzu(GetDst(Node).V2D(), GetDst(Node).V2D());
break;
default: LogMan::Msg::A("Unknown castGPR element size: %d", Op->Header.ElementSize);
}
}
@@ -132,7 +177,7 @@ DEF_OP(Vector_FToS) {
frinti(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D());
fcvtzs(GetDst(Node).V2D(), GetDst(Node).V2D());
break;
default: LOGMAN_MSG_A("Unknown castGPR element size: %d", Op->Header.ElementSize);
default: LogMan::Msg::A("Unknown castGPR element size: %d", Op->Header.ElementSize);
}
}
@@ -149,63 +194,7 @@ DEF_OP(Vector_FToF) {
fcvtn(GetDst(Node).V2S(), GetSrc(Op->Header.Args[0].ID()).V2D());
break;
}
default: LOGMAN_MSG_A("Unknown Conversion Type : 0%04x", Conv); break;
}
}
DEF_OP(Vector_FToI) {
auto Op = IROp->C<IR::IROp_Vector_FToI>();
switch (Op->Round) {
case FEXCore::IR::Round_Nearest.Val:
switch (Op->Header.ElementSize) {
case 4:
frintn(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S());
break;
case 8:
frintn(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D());
break;
}
break;
case FEXCore::IR::Round_Negative_Infinity.Val:
switch (Op->Header.ElementSize) {
case 4:
frintm(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S());
break;
case 8:
frintm(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D());
break;
}
break;
case FEXCore::IR::Round_Positive_Infinity.Val:
switch (Op->Header.ElementSize) {
case 4:
frintp(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S());
break;
case 8:
frintp(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D());
break;
}
break;
case FEXCore::IR::Round_Towards_Zero.Val:
switch (Op->Header.ElementSize) {
case 4:
frintz(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S());
break;
case 8:
frintz(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D());
break;
}
break;
case FEXCore::IR::Round_Host.Val:
switch (Op->Header.ElementSize) {
case 4:
frinti(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S());
break;
case 8:
frinti(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D());
break;
}
break;
default: LogMan::Msg::A("Unknown Conversion Type : 0%04x", Conv); break;
}
}
@@ -214,13 +203,16 @@ void Arm64JITCore::RegisterConversionHandlers() {
#define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &Arm64JITCore::Op_##x
REGISTER_OP(VINSGPR, VInsGPR);
REGISTER_OP(VCASTFROMGPR, VCastFromGPR);
REGISTER_OP(FLOAT_FROMGPR_U, Float_FromGPR_U);
REGISTER_OP(FLOAT_FROMGPR_S, Float_FromGPR_S);
REGISTER_OP(FLOAT_FTOF, Float_FToF);
REGISTER_OP(VECTOR_UTOF, Vector_UToF);
REGISTER_OP(VECTOR_STOF, Vector_SToF);
REGISTER_OP(VECTOR_FTOZU, Vector_FToZU);
REGISTER_OP(VECTOR_FTOZS, Vector_FToZS);
REGISTER_OP(VECTOR_FTOU, Vector_FToU);
REGISTER_OP(VECTOR_FTOS, Vector_FToS);
REGISTER_OP(VECTOR_FTOF, Vector_FToF);
REGISTER_OP(VECTOR_FTOI, Vector_FToI);
#undef REGISTER_OP
}
}
+49 -89
View File
@@ -22,8 +22,6 @@ $end_info$
#include <FEXCore/Core/X86Enums.h>
#include <FEXCore/Core/UContext.h>
#include <FEXCore/Utils/Allocator.h>
#include <FEXCore/Utils/CompilerDefs.h>
#include "Interface/Core/Interpreter/InterpreterOps.h"
#include <sys/mman.h>
@@ -45,7 +43,7 @@ void Arm64JITCore::Op_Unhandled(FEXCore::IR::IROp_Header *IROp, uint32_t Node) {
FallbackInfo Info;
if (!InterpreterOps::GetFallbackHandler(IROp, &Info)) {
auto Name = FEXCore::IR::GetName(IROp->Op);
LOGMAN_MSG_A("Unhandled IR Op: %s", std::string(Name).c_str());
LogMan::Msg::A("Unhandled IR Op: %s", std::string(Name).c_str());
} else {
switch(Info.ABI) {
case FABI_VOID_U16:{
@@ -293,7 +291,7 @@ void Arm64JITCore::Op_Unhandled(FEXCore::IR::IROp_Header *IROp, uint32_t Node) {
case FABI_UNKNOWN:
default:
auto Name = FEXCore::IR::GetName(IROp->Op);
LOGMAN_MSG_A("Unhandled IR Fallback abi: %s %d", std::string(Name).c_str(), Info.ABI);
LogMan::Msg::A("Unhandled IR Fallback abi: %s %d", std::string(Name).c_str(), Info.ABI);
}
}
}
@@ -305,18 +303,18 @@ Arm64JITCore::CodeBuffer Arm64JITCore::AllocateNewCodeBuffer(size_t Size) {
CodeBuffer Buffer;
Buffer.Size = Size;
Buffer.Ptr = static_cast<uint8_t*>(
FEXCore::Allocator::mmap(nullptr,
mmap(nullptr,
Buffer.Size,
PROT_READ | PROT_WRITE | PROT_EXEC,
MAP_PRIVATE | MAP_ANONYMOUS,
-1, 0));
LOGMAN_THROW_A(!!Buffer.Ptr, "Couldn't allocate code buffer");
LogMan::Throw::A(!!Buffer.Ptr, "Couldn't allocate code buffer");
Dispatcher->RegisterCodeBuffer(Buffer.Ptr, Buffer.Size);
return Buffer;
}
void Arm64JITCore::FreeCodeBuffer(CodeBuffer Buffer) {
FEXCore::Allocator::munmap(Buffer.Ptr, Buffer.Size);
munmap(Buffer.Ptr, Buffer.Size);
Dispatcher->RemoveCodeBuffer(Buffer.Ptr);
}
@@ -361,34 +359,6 @@ bool Arm64JITCore::HandleSIGBUS(int Signal, void *info, void *ucontext) {
// Back up one instruction and have another go
ArchHelpers::Context::SetPc(ucontext, ArchHelpers::Context::GetPc(ucontext) - 4);
}
else if ((Instr & FEXCore::ArchHelpers::Arm64::LDAXP_MASK) == FEXCore::ArchHelpers::Arm64::LDAXP_INST) { // LDAXP
uint32_t DataReg2 = (Instr >> 10) & 0x1F;
// Convert to LDP
uint32_t LDP = 0b0010'1001'0100'0000'0000'0000'0000'0000;
LDP |= Size << 31;
LDP |= DataReg2 << 10;
LDP |= AddrReg << 5;
LDP |= DataReg;
PC[-1] = DMB;
PC[0] = LDP;
PC[1] = DMB;
// Back up one instruction and have another go
ArchHelpers::Context::SetPc(ucontext, ArchHelpers::Context::GetPc(ucontext) - 4);
}
else if ((Instr & FEXCore::ArchHelpers::Arm64::STLXP_MASK) == FEXCore::ArchHelpers::Arm64::STLXP_INST) { // STLXP
uint32_t DataReg2 = (Instr >> 10) & 0x1F;
// Convert to STP
uint32_t STP = 0b0010'1001'0000'0000'0000'0000'0000'0000;
STP |= Size << 31;
STP |= DataReg2 << 10;
STP |= AddrReg << 5;
STP |= DataReg;
PC[-1] = DMB;
PC[0] = STP;
PC[1] = DMB;
// Back up one instruction and have another go
ArchHelpers::Context::SetPc(ucontext, ArchHelpers::Context::GetPc(ucontext) - 4);
}
else if ((Instr & FEXCore::ArchHelpers::Arm64::CASPAL_MASK) == FEXCore::ArchHelpers::Arm64::CASPAL_INST) { // CASPAL
if (FEXCore::ArchHelpers::Arm64::HandleCASPAL(ucontext, info, Instr)) {
// Skip this instruction now
@@ -442,7 +412,7 @@ Arm64JITCore::Arm64JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::Intern
config.ExitFunctionLinkThis = reinterpret_cast<uintptr_t>(this);
config.StaticRegisterAssignment = true;
Dispatcher = std::make_unique<Arm64Dispatcher>(CTX, ThreadState, config);
Dispatcher = new Arm64Dispatcher(CTX, ThreadState, config);
DispatchPtr = Dispatcher->DispatchPtr;
CallbackPtr = Dispatcher->CallbackPtr;
}
@@ -496,7 +466,6 @@ Arm64JITCore::Arm64JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::Intern
if (!CompileThread) {
ThreadSharedData.SignalHandlerRefCounterPtr = &Dispatcher->SignalHandlerRefCounter;
ThreadSharedData.SignalReturnInstruction = Dispatcher->SignalHandlerReturnAddress;
ThreadSharedData.Dispatcher = Dispatcher.get();
// This will register the host signal handler per thread, which is fine
CTX->SignalDelegation->RegisterHostSignalHandler(SIGILL, [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext) -> bool {
@@ -576,85 +545,77 @@ Arm64JITCore::~Arm64JITCore() {
FreeCodeBuffer(InitialCodeBuffer);
}
IR::PhysicalRegister Arm64JITCore::GetPhys(uint32_t Node) const {
static IR::PhysicalRegister GetPhys(IR::RegisterAllocationData *RAData, uint32_t Node) {
auto PhyReg = RAData->GetNodeRegister(Node);
LOGMAN_THROW_A(!PhyReg.IsInvalid(), "Couldn't Allocate register for node: ssa%d. Class: %d", Node, PhyReg.Class);
LogMan::Throw::A(!PhyReg.IsInvalid(), "Couldn't Allocate register for node: ssa%d. Class: %d", Node, PhyReg.Class);
return PhyReg;
}
template<>
aarch64::Register Arm64JITCore::GetReg<Arm64JITCore::RA_32>(uint32_t Node) const {
auto Reg = GetPhys(Node);
aarch64::Register Arm64JITCore::GetReg<Arm64JITCore::RA_32>(uint32_t Node) {
auto Reg = GetPhys(RAData, Node);
if (Reg.Class == IR::GPRFixedClass.Val) {
return SRA64[Reg.Reg].W();
} else if (Reg.Class == IR::GPRClass.Val) {
return RA64[Reg.Reg].W();
} else {
LOGMAN_THROW_A(false, "Unexpected Class: %d", Reg.Class);
LogMan::Throw::A(false, "Unexpected Class: %d", Reg.Class);
}
FEX_UNREACHABLE;
__builtin_unreachable();
}
template<>
aarch64::Register Arm64JITCore::GetReg<Arm64JITCore::RA_64>(uint32_t Node) const {
auto Reg = GetPhys(Node);
aarch64::Register Arm64JITCore::GetReg<Arm64JITCore::RA_64>(uint32_t Node) {
auto Reg = GetPhys(RAData, Node);
if (Reg.Class == IR::GPRFixedClass.Val) {
return SRA64[Reg.Reg];
} else if (Reg.Class == IR::GPRClass.Val) {
return RA64[Reg.Reg];
} else {
LOGMAN_THROW_A(false, "Unexpected Class: %d", Reg.Class);
LogMan::Throw::A(false, "Unexpected Class: %d", Reg.Class);
}
FEX_UNREACHABLE;
__builtin_unreachable();
}
template<>
std::pair<aarch64::Register, aarch64::Register> Arm64JITCore::GetSrcPair<Arm64JITCore::RA_32>(uint32_t Node) const {
uint32_t Reg = GetPhys(Node).Reg;
std::pair<aarch64::Register, aarch64::Register> Arm64JITCore::GetSrcPair<Arm64JITCore::RA_32>(uint32_t Node) {
uint32_t Reg = GetPhys(RAData, Node).Reg;
return RA32Pair[Reg];
}
template<>
std::pair<aarch64::Register, aarch64::Register> Arm64JITCore::GetSrcPair<Arm64JITCore::RA_64>(uint32_t Node) const {
uint32_t Reg = GetPhys(Node).Reg;
std::pair<aarch64::Register, aarch64::Register> Arm64JITCore::GetSrcPair<Arm64JITCore::RA_64>(uint32_t Node) {
uint32_t Reg = GetPhys(RAData, Node).Reg;
return RA64Pair[Reg];
}
aarch64::VRegister Arm64JITCore::GetSrc(uint32_t Node) const {
auto Reg = GetPhys(Node);
aarch64::VRegister Arm64JITCore::GetSrc(uint32_t Node) {
auto Reg = GetPhys(RAData, Node);
if (Reg.Class == IR::FPRFixedClass.Val) {
return SRAFPR[Reg.Reg];
} else if (Reg.Class == IR::FPRClass.Val) {
return RAFPR[Reg.Reg];
} else {
LOGMAN_THROW_A(false, "Unexpected Class: %d", Reg.Class);
LogMan::Throw::A(false, "Unexpected Class: %d", Reg.Class);
}
FEX_UNREACHABLE;
__builtin_unreachable();
}
aarch64::VRegister Arm64JITCore::GetDst(uint32_t Node) const {
auto Reg = GetPhys(Node);
aarch64::VRegister Arm64JITCore::GetDst(uint32_t Node) {
auto Reg = GetPhys(RAData, Node);
if (Reg.Class == IR::FPRFixedClass.Val) {
return SRAFPR[Reg.Reg];
} else if (Reg.Class == IR::FPRClass.Val) {
return RAFPR[Reg.Reg];
} else {
LOGMAN_THROW_A(false, "Unexpected Class: %d", Reg.Class);
LogMan::Throw::A(false, "Unexpected Class: %d", Reg.Class);
}
FEX_UNREACHABLE;
__builtin_unreachable();
}
bool Arm64JITCore::IsInlineConstant(const IR::OrderedNodeWrapper& WNode, uint64_t* Value) const {
bool Arm64JITCore::IsInlineConstant(const IR::OrderedNodeWrapper& WNode, uint64_t* Value) {
auto OpHeader = IR->GetOp<IR::IROp_Header>(WNode);
if (OpHeader->Op == IR::IROps::OP_INLINECONSTANT) {
@@ -668,13 +629,13 @@ bool Arm64JITCore::IsInlineConstant(const IR::OrderedNodeWrapper& WNode, uint64_
}
}
bool Arm64JITCore::IsInlineEntrypointOffset(const IR::OrderedNodeWrapper& WNode, uint64_t* Value) const {
bool Arm64JITCore::IsInlineEntrypointOffset(const IR::OrderedNodeWrapper& WNode, uint64_t* Value) {
auto OpHeader = IR->GetOp<IR::IROp_Header>(WNode);
if (OpHeader->Op == IR::IROps::OP_INLINEENTRYPOINTOFFSET) {
auto Op = OpHeader->C<IR::IROp_InlineEntrypointOffset>();
if (Value) {
*Value = Entry + Op->Offset;
*Value = IR->GetHeader()->Entry + Op->Offset;
}
return true;
} else {
@@ -682,35 +643,34 @@ bool Arm64JITCore::IsInlineEntrypointOffset(const IR::OrderedNodeWrapper& WNode,
}
}
FEXCore::IR::RegisterClassType Arm64JITCore::GetRegClass(uint32_t Node) const {
return FEXCore::IR::RegisterClassType {GetPhys(Node).Class};
FEXCore::IR::RegisterClassType Arm64JITCore::GetRegClass(uint32_t Node) {
return FEXCore::IR::RegisterClassType {GetPhys(RAData, Node).Class};
}
bool Arm64JITCore::IsFPR(uint32_t Node) const {
bool Arm64JITCore::IsFPR(uint32_t Node) {
auto Class = GetRegClass(Node);
return Class == IR::FPRClass || Class == IR::FPRFixedClass;
}
bool Arm64JITCore::IsGPR(uint32_t Node) const {
bool Arm64JITCore::IsGPR(uint32_t Node) {
auto Class = GetRegClass(Node);
return Class == IR::GPRClass || Class == IR::GPRFixedClass;
}
void *Arm64JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IRListView const *IR, [[maybe_unused]] FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) {
void *Arm64JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const *IR, [[maybe_unused]] FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) {
using namespace aarch64;
JumpTargets.clear();
uint32_t SSACount = IR->GetSSACount();
this->Entry = Entry;
this->RAData = RAData;
auto HeaderOp = IR->GetHeader();
#ifndef NDEBUG
LoadConstant(x0, Entry);
LoadConstant(x0, HeaderOp->Entry);
#endif
this->IR = IR;
@@ -742,7 +702,7 @@ void *Arm64JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IR
// X4-r18 = RA
auto Buffer = GetBuffer();
auto GuestEntry = Buffer->GetOffsetAddress<uint64_t>(GetCursorOffset());
auto Entry = Buffer->GetOffsetAddress<uint64_t>(GetCursorOffset());
if (CTX->GetGdbServerStatus()) {
aarch64::Label RunBlock;
@@ -759,17 +719,17 @@ void *Arm64JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IR
cbz(w0, &RunBlock);
{
// Make sure RIP is syncronized to the context
LoadConstant(x0, Entry);
LoadConstant(x0, HeaderOp->Entry);
str(x0, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.rip)));
// Stop the thread
LoadConstant(x0, ThreadSharedData.Dispatcher->ThreadPauseHandlerAddressSpillSRA);
LoadConstant(x0, Dispatcher->ThreadPauseHandlerAddressSpillSRA);
br(x0);
}
bind(&RunBlock);
}
//LOGMAN_THROW_A(RAData->HasFullRA(), "Arm64 JIT only works with RA");
//LogMan::Throw::A(RAData->HasFullRA(), "Arm64 JIT only works with RA");
SpillSlots = RAData->SpillSlots();
@@ -787,7 +747,7 @@ void *Arm64JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IR
for (auto [BlockNode, BlockHeader] : IR->GetBlocks()) {
using namespace FEXCore::IR;
auto BlockIROp = BlockHeader->CW<FEXCore::IR::IROp_CodeBlock>();
LOGMAN_THROW_A(BlockIROp->Header.Op == IR::OP_CODEBLOCK, "IR type failed to be a code block");
LogMan::Throw::A(BlockIROp->Header.Op == IR::OP_CODEBLOCK, "IR type failed to be a code block");
{
uint32_t Node = IR->GetID(BlockNode);
@@ -833,15 +793,15 @@ void *Arm64JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IR
FinalizeCode();
auto CodeEnd = Buffer->GetOffsetAddress<uint64_t>(GetCursorOffset());
CPU.EnsureIAndDCacheCoherency(reinterpret_cast<void*>(GuestEntry), CodeEnd - reinterpret_cast<uint64_t>(GuestEntry));
CPU.EnsureIAndDCacheCoherency(reinterpret_cast<void*>(Entry), CodeEnd - reinterpret_cast<uint64_t>(Entry));
if (DebugData) {
DebugData->HostCodeSize = reinterpret_cast<uintptr_t>(CodeEnd) - reinterpret_cast<uintptr_t>(GuestEntry);
DebugData->HostCodeSize = reinterpret_cast<uintptr_t>(CodeEnd) - reinterpret_cast<uintptr_t>(Entry);
}
this->IR = nullptr;
return reinterpret_cast<void*>(GuestEntry);
return reinterpret_cast<void*>(Entry);
}
uint64_t Arm64JITCore::ExitFunctionLink(Arm64JITCore *core, FEXCore::Core::CpuStateFrame *Frame, uint64_t *record) {
@@ -853,11 +813,11 @@ uint64_t Arm64JITCore::ExitFunctionLink(Arm64JITCore *core, FEXCore::Core::CpuSt
if (!HostCode) {
//printf("ExitFunctionLink: Aborting, %lX not in cache\n", GuestRip);
Frame->State.rip = GuestRip;
return core->ThreadSharedData.Dispatcher->AbsoluteLoopTopAddress;
return core->Dispatcher->AbsoluteLoopTopAddress;
}
uintptr_t branch = (uintptr_t)(record) - 8;
auto LinkerAddress = core->ThreadSharedData.Dispatcher->ExitFunctionLinkerAddress;
auto LinkerAddress = core->Dispatcher->ExitFunctionLinkerAddress;
auto offset = HostCode/4 - branch/4;
if (IsInt26(offset)) {
@@ -893,7 +853,7 @@ uint64_t Arm64JITCore::ExitFunctionLink(Arm64JITCore *core, FEXCore::Core::CpuSt
return HostCode;
}
std::unique_ptr<CPUBackend> CreateArm64JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, bool CompileThread) {
return std::make_unique<Arm64JITCore>(ctx, Thread, CompileThread);
FEXCore::CPU::CPUBackend *CreateArm64JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, bool CompileThread) {
return new Arm64JITCore(ctx, Thread, CompileThread);
}
}
+20 -32
View File
@@ -47,7 +47,7 @@ public:
~Arm64JITCore() override;
std::string GetName() override { return "JIT"; }
void *CompileCode(uint64_t Entry, FEXCore::IR::IRListView const *IR, FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) override;
void *CompileCode(FEXCore::IR::IRListView const *IR, FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) override;
void *MapRegion(void* HostPtr, uint64_t, uint64_t) override { return HostPtr; }
@@ -63,14 +63,11 @@ public:
void CopyNecessaryDataForCompileThread(CPUBackend *Original) override;
private:
FEX_CONFIG_OPT(ParanoidTSO, PARANOIDTSO);
std::unique_ptr<FEXCore::CPU::Dispatcher> Dispatcher;
Dispatcher *Dispatcher;
Label *PendingTargetLabel;
FEXCore::Context::Context *CTX;
FEXCore::Core::InternalThreadState *ThreadState;
FEXCore::IR::IRListView const *IR;
uint64_t Entry;
std::map<IR::OrderedNodeWrapper::NodeOffsetType, aarch64::Label> JumpTargets;
@@ -96,35 +93,33 @@ private:
constexpr static uint8_t RA_FPR = 2;
template<uint8_t RAType>
aarch64::Register GetReg(uint32_t Node) const;
aarch64::Register GetReg(uint32_t Node);
template<>
aarch64::Register GetReg<RA_32>(uint32_t Node) const;
aarch64::Register GetReg<RA_32>(uint32_t Node);
template<>
aarch64::Register GetReg<RA_64>(uint32_t Node) const;
aarch64::Register GetReg<RA_64>(uint32_t Node);
template<uint8_t RAType>
std::pair<aarch64::Register, aarch64::Register> GetSrcPair(uint32_t Node) const;
std::pair<aarch64::Register, aarch64::Register> GetSrcPair(uint32_t Node);
template<>
std::pair<aarch64::Register, aarch64::Register> GetSrcPair<RA_32>(uint32_t Node) const;
std::pair<aarch64::Register, aarch64::Register> GetSrcPair<RA_32>(uint32_t Node);
template<>
std::pair<aarch64::Register, aarch64::Register> GetSrcPair<RA_64>(uint32_t Node) const;
std::pair<aarch64::Register, aarch64::Register> GetSrcPair<RA_64>(uint32_t Node);
aarch64::VRegister GetSrc(uint32_t Node) const;
aarch64::VRegister GetDst(uint32_t Node) const;
aarch64::VRegister GetSrc(uint32_t Node);
aarch64::VRegister GetDst(uint32_t Node);
FEXCore::IR::RegisterClassType GetRegClass(uint32_t Node) const;
FEXCore::IR::RegisterClassType GetRegClass(uint32_t Node);
IR::PhysicalRegister GetPhys(uint32_t Node) const;
bool IsFPR(uint32_t Node) const;
bool IsGPR(uint32_t Node) const;
bool IsFPR(uint32_t Node);
bool IsGPR(uint32_t Node);
MemOperand GenerateMemOperand(uint8_t AccessSize, aarch64::Register Base, IR::OrderedNodeWrapper Offset, IR::MemOffsetType OffsetType, uint8_t OffsetScale);
bool IsInlineConstant(const IR::OrderedNodeWrapper& Node, uint64_t* Value = nullptr) const;
bool IsInlineEntrypointOffset(const IR::OrderedNodeWrapper& WNode, uint64_t* Value) const;
bool IsInlineConstant(const IR::OrderedNodeWrapper& Node, uint64_t* Value = nullptr);
bool IsInlineEntrypointOffset(const IR::OrderedNodeWrapper& WNode, uint64_t* Value);
struct LiveRange {
uint32_t Begin;
@@ -167,7 +162,6 @@ private:
uint64_t SignalReturnInstruction{};
uint32_t *SignalHandlerRefCounterPtr{};
FEXCore::CPU::Dispatcher *Dispatcher{};
};
CompilerSharedData ThreadSharedData;
@@ -240,6 +234,7 @@ private:
DEF_OP(VExtractToGPR);
DEF_OP(Float_ToGPR_ZU);
DEF_OP(Float_ToGPR_ZS);
DEF_OP(Float_ToGPR_U);
DEF_OP(Float_ToGPR_S);
DEF_OP(FCmp);
@@ -276,13 +271,16 @@ private:
///< Conversion ops
DEF_OP(VInsGPR);
DEF_OP(VCastFromGPR);
DEF_OP(Float_FromGPR_U);
DEF_OP(Float_FromGPR_S);
DEF_OP(Float_FToF);
DEF_OP(Vector_UToF);
DEF_OP(Vector_SToF);
DEF_OP(Vector_FToZU);
DEF_OP(Vector_FToZS);
DEF_OP(Vector_FToU);
DEF_OP(Vector_FToS);
DEF_OP(Vector_FToF);
DEF_OP(Vector_FToI);
///< Flag ops
DEF_OP(GetHostFlag);
@@ -302,11 +300,8 @@ private:
DEF_OP(StoreMem);
DEF_OP(LoadMemTSO);
DEF_OP(StoreMemTSO);
DEF_OP(ParanoidLoadMemTSO);
DEF_OP(ParanoidStoreMemTSO);
DEF_OP(VLoadMemElement);
DEF_OP(VStoreMemElement);
DEF_OP(CacheLineClear);
///< Misc ops
DEF_OP(EndBlock);
@@ -332,7 +327,6 @@ private:
DEF_OP(SplatVector4);
DEF_OP(VMov);
DEF_OP(VAnd);
DEF_OP(VBic);
DEF_OP(VOr);
DEF_OP(VXor);
DEF_OP(VAdd);
@@ -343,10 +337,8 @@ private:
DEF_OP(VSQSub);
DEF_OP(VAddP);
DEF_OP(VAddV);
DEF_OP(VUMinV);
DEF_OP(VURAvg);
DEF_OP(VAbs);
DEF_OP(VPopcount);
DEF_OP(VFAdd);
DEF_OP(VFAddP);
DEF_OP(VFSub);
@@ -366,8 +358,6 @@ private:
DEF_OP(VSMax);
DEF_OP(VZip);
DEF_OP(VZip2);
DEF_OP(VUnZip);
DEF_OP(VUnZip2);
DEF_OP(VBSL);
DEF_OP(VCMPEQ);
DEF_OP(VCMPEQZ);
@@ -390,7 +380,6 @@ private:
DEF_OP(VInsElement);
DEF_OP(VInsScalarElement);
DEF_OP(VExtractElement);
DEF_OP(VDupElement);
DEF_OP(VExtr);
DEF_OP(VSLI);
DEF_OP(VSRI);
@@ -413,7 +402,6 @@ private:
DEF_OP(VSMull);
DEF_OP(VUMull2);
DEF_OP(VSMull2);
DEF_OP(VUABDL);
DEF_OP(VTBL1);
///< Encryption ops
+61 -229
View File
@@ -5,7 +5,6 @@ $end_info$
*/
#include "Interface/Core/JIT/Arm64/JITClass.h"
#include <FEXCore/Utils/CompilerDefs.h>
namespace FEXCore::CPU {
@@ -30,7 +29,7 @@ DEF_OP(LoadContext) {
case 8:
ldr(GetReg<RA_64>(Node), MemOperand(STATE, Op->Offset));
break;
default: LOGMAN_MSG_A("Unhandled LoadContext size: %d", OpSize);
default: LogMan::Msg::A("Unhandled LoadContext size: %d", OpSize);
}
}
else {
@@ -51,7 +50,7 @@ DEF_OP(LoadContext) {
case 16:
ldr(Dst, MemOperand(STATE, Op->Offset));
break;
default: LOGMAN_MSG_A("Unhandled LoadContext size: %d", OpSize);
default: LogMan::Msg::A("Unhandled LoadContext size: %d", OpSize);
}
}
}
@@ -73,7 +72,7 @@ DEF_OP(StoreContext) {
case 8:
str(GetReg<RA_64>(Op->Header.Args[0].ID()), MemOperand(STATE, Op->Offset));
break;
default: LOGMAN_MSG_A("Unhandled StoreContext size: %d", OpSize);
default: LogMan::Msg::A("Unhandled StoreContext size: %d", OpSize);
}
}
else {
@@ -94,7 +93,7 @@ DEF_OP(StoreContext) {
case 16:
str(Src, MemOperand(STATE, Op->Offset));
break;
default: LOGMAN_MSG_A("Unhandled LoadContext size: %d", OpSize);
default: LogMan::Msg::A("Unhandled LoadContext size: %d", OpSize);
}
}
}
@@ -107,29 +106,29 @@ DEF_OP(LoadRegister) {
auto regId = (Op->Offset - offsetof(FEXCore::Core::CpuStateFrame, State.gregs[0])) / 8;
auto regOffs = Op->Offset & 7;
LOGMAN_THROW_A(regId < SRA64.size(), "out of range regId");
LogMan::Throw::A(regId < SRA64.size(), "out of range regId");
auto reg = SRA64[regId];
switch(Op->Header.Size) {
case 1:
LOGMAN_THROW_A(regOffs == 0 || regOffs == 1, "unexpected regOffs");
LogMan::Throw::A(regOffs == 0 || regOffs == 1, "unexpected regOffs");
ubfx(GetReg<RA_64>(Node), reg, regOffs * 8, 8);
break;
case 2:
LOGMAN_THROW_A(regOffs == 0, "unexpected regOffs");
LogMan::Throw::A(regOffs == 0, "unexpected regOffs");
ubfx(GetReg<RA_64>(Node), reg, 0, 16);
break;
case 4:
LOGMAN_THROW_A(regOffs == 0, "unexpected regOffs");
LogMan::Throw::A(regOffs == 0, "unexpected regOffs");
if (GetReg<RA_64>(Node).GetCode() != reg.GetCode())
mov(GetReg<RA_32>(Node), reg.W());
break;
case 8:
LOGMAN_THROW_A(regOffs == 0, "unexpected regOffs");
LogMan::Throw::A(regOffs == 0, "unexpected regOffs");
if (GetReg<RA_64>(Node).GetCode() != reg.GetCode())
mov(GetReg<RA_64>(Node), reg);
break;
@@ -138,24 +137,24 @@ DEF_OP(LoadRegister) {
auto regId = (Op->Offset - offsetof(FEXCore::Core::CpuStateFrame, State.xmm[0][0])) / 16;
auto regOffs = Op->Offset & 15;
LOGMAN_THROW_A(regId < SRAFPR.size(), "out of range regId");
LogMan::Throw::A(regId < SRAFPR.size(), "out of range regId");
auto guest = SRAFPR[regId];
auto host = GetSrc(Node);
switch(Op->Header.Size) {
case 1:
LOGMAN_THROW_A(regOffs == 0, "unexpected regOffs");
LogMan::Throw::A(regOffs == 0, "unexpected regOffs");
mov(host.B(), guest.B());
break;
case 2:
LOGMAN_THROW_A(regOffs == 0, "unexpected regOffs");
LogMan::Throw::A(regOffs == 0, "unexpected regOffs");
fmov(host.H(), guest.H());
break;
case 4:
LOGMAN_THROW_A((regOffs & 3) == 0, "unexpected regOffs");
LogMan::Throw::A((regOffs & 3) == 0, "unexpected regOffs");
if (regOffs == 0) {
if (host.GetCode() != guest.GetCode())
fmov(host.S(), guest.S());
@@ -165,7 +164,7 @@ DEF_OP(LoadRegister) {
break;
case 8:
LOGMAN_THROW_A((regOffs & 7) == 0, "unexpected regOffs");
LogMan::Throw::A((regOffs & 7) == 0, "unexpected regOffs");
if (regOffs == 0) {
if (host.GetCode() != guest.GetCode())
mov(host.D(), guest.D());
@@ -175,13 +174,13 @@ DEF_OP(LoadRegister) {
break;
case 16:
LOGMAN_THROW_A(regOffs == 0, "unexpected regOffs");
LogMan::Throw::A(regOffs == 0, "unexpected regOffs");
if (host.GetCode() != guest.GetCode())
mov(host.Q(), guest.Q());
break;
}
} else {
LOGMAN_THROW_A(false, "Unhandled Op->Class %d", Op->Class);
LogMan::Throw::A(false, "Unhandled Op->Class %d", Op->Class);
}
}
@@ -192,28 +191,28 @@ DEF_OP(StoreRegister) {
auto regId = Op->Offset / 8 - 1;
auto regOffs = Op->Offset & 7;
LOGMAN_THROW_A(regId < SRA64.size(), "out of range regId");
LogMan::Throw::A(regId < SRA64.size(), "out of range regId");
auto reg = SRA64[regId];
switch(Op->Header.Size) {
case 1:
LOGMAN_THROW_A(regOffs == 0 || regOffs == 1, "unexpected regOffs");
LogMan::Throw::A(regOffs == 0 || regOffs == 1, "unexpected regOffs");
bfi(reg, GetReg<RA_64>(Op->Value.ID()), regOffs * 8, 8);
break;
case 2:
LOGMAN_THROW_A(regOffs == 0, "unexpected regOffs");
LogMan::Throw::A(regOffs == 0, "unexpected regOffs");
bfi(reg, GetReg<RA_64>(Op->Value.ID()), 0, 16);
break;
case 4:
LOGMAN_THROW_A(regOffs == 0, "unexpected regOffs");
LogMan::Throw::A(regOffs == 0, "unexpected regOffs");
bfi(reg, GetReg<RA_64>(Op->Value.ID()), 0, 32);
break;
case 8:
LOGMAN_THROW_A(regOffs == 0, "unexpected regOffs");
LogMan::Throw::A(regOffs == 0, "unexpected regOffs");
if (GetReg<RA_64>(Op->Value.ID()).GetCode() != reg.GetCode())
mov(reg, GetReg<RA_64>(Op->Value.ID()));
break;
@@ -222,7 +221,7 @@ DEF_OP(StoreRegister) {
auto regId = (Op->Offset - offsetof(FEXCore::Core::CpuStateFrame, State.xmm[0][0])) / 16;
auto regOffs = Op->Offset & 15;
LOGMAN_THROW_A(regId < SRAFPR.size(), "regId out of range");
LogMan::Throw::A(regId < SRAFPR.size(), "regId out of range");
auto guest = SRAFPR[regId];
auto host = GetSrc(Op->Value.ID());
@@ -233,28 +232,28 @@ DEF_OP(StoreRegister) {
break;
case 2:
LOGMAN_THROW_A((regOffs & 1) == 0, "unexpected regOffs");
LogMan::Throw::A((regOffs & 1) == 0, "unexpected regOffs");
ins(guest.V8H(), regOffs/2, host.V8H(), 0);
break;
case 4:
LOGMAN_THROW_A((regOffs & 3) == 0, "unexpected regOffs");
LogMan::Throw::A((regOffs & 3) == 0, "unexpected regOffs");
ins(guest.V4S(), regOffs/4, host.V4S(), 0);
break;
case 8:
LOGMAN_THROW_A((regOffs & 7) == 0, "unexpected regOffs");
LogMan::Throw::A((regOffs & 7) == 0, "unexpected regOffs");
ins(guest.V2D(), regOffs / 8, host.V2D(), 0);
break;
case 16:
LOGMAN_THROW_A(regOffs == 0, "unexpected regOffs");
LogMan::Throw::A(regOffs == 0, "unexpected regOffs");
if (guest.GetCode() != host.GetCode())
mov(guest.Q(), host.Q());
break;
}
} else {
LOGMAN_THROW_A(false, "Unhandled Op->Class %d", Op->Class);
LogMan::Throw::A(false, "Unhandled Op->Class %d", Op->Class);
}
}
@@ -288,15 +287,15 @@ DEF_OP(LoadContextIndexed) {
ldr(GetReg<RA_64>(Node), MemOperand(TMP1, Op->BaseOffset));
break;
default:
LOGMAN_MSG_A("Unhandled LoadContextIndexed size: %d", Op->Size);
LogMan::Msg::A("Unhandled LoadContextIndexed size: %d", Op->Size);
}
break;
}
case 16:
LOGMAN_MSG_A("Invalid Class load of size 16");
LogMan::Msg::A("Invalid Class load of size 16");
break;
default:
LOGMAN_MSG_A("Unhandled LoadContextIndexed stride: %d", Op->Stride);
LogMan::Msg::A("Unhandled LoadContextIndexed stride: %d", Op->Stride);
}
}
else {
@@ -333,12 +332,12 @@ DEF_OP(LoadContextIndexed) {
}
break;
default:
LOGMAN_MSG_A("Unhandled LoadContextIndexed size: %d", Op->Size);
LogMan::Msg::A("Unhandled LoadContextIndexed size: %d", Op->Size);
}
break;
}
default:
LOGMAN_MSG_A("Unhandled LoadContextIndexed stride: %d", Op->Stride);
LogMan::Msg::A("Unhandled LoadContextIndexed stride: %d", Op->Stride);
}
}
}
@@ -374,15 +373,15 @@ DEF_OP(StoreContextIndexed) {
str(value, MemOperand(TMP1, Op->BaseOffset));
break;
default:
LOGMAN_MSG_A("Unhandled LoadContextIndexed size: %d", Op->Size);
LogMan::Msg::A("Unhandled LoadContextIndexed size: %d", Op->Size);
}
break;
}
case 16:
LOGMAN_MSG_A("Invalid Class load of size 16");
LogMan::Msg::A("Invalid Class load of size 16");
break;
default:
LOGMAN_MSG_A("Unhandled LoadContextIndexed stride: %d", Op->Stride);
LogMan::Msg::A("Unhandled LoadContextIndexed stride: %d", Op->Stride);
}
}
else {
@@ -421,12 +420,12 @@ DEF_OP(StoreContextIndexed) {
}
break;
default:
LOGMAN_MSG_A("Unhandled LoadContextIndexed size: %d", Op->Size);
LogMan::Msg::A("Unhandled LoadContextIndexed size: %d", Op->Size);
}
break;
}
default:
LOGMAN_MSG_A("Unhandled LoadContextIndexed stride: %d", Op->Stride);
LogMan::Msg::A("Unhandled LoadContextIndexed stride: %d", Op->Stride);
}
}
}
@@ -454,7 +453,7 @@ DEF_OP(SpillRegister) {
str(GetReg<RA_64>(Op->Header.Args[0].ID()), MemOperand(sp, SlotOffset));
break;
}
default: LOGMAN_MSG_A("Unhandled SpillRegister size: %d", OpSize);
default: LogMan::Msg::A("Unhandled SpillRegister size: %d", OpSize);
}
} else if (Op->Class == FEXCore::IR::FPRClass) {
switch (OpSize) {
@@ -470,10 +469,10 @@ DEF_OP(SpillRegister) {
str(GetSrc(Op->Header.Args[0].ID()), MemOperand(sp, SlotOffset));
break;
}
default: LOGMAN_MSG_A("Unhandled SpillRegister size: %d", OpSize);
default: LogMan::Msg::A("Unhandled SpillRegister size: %d", OpSize);
}
} else {
LOGMAN_MSG_A("Unhandled SpillRegister class: %d", Op->Class.Val);
LogMan::Msg::A("Unhandled SpillRegister class: %d", Op->Class.Val);
}
}
@@ -500,7 +499,7 @@ DEF_OP(FillRegister) {
ldr(GetReg<RA_64>(Node), MemOperand(sp, SlotOffset));
break;
}
default: LOGMAN_MSG_A("Unhandled SpillRegister size: %d", OpSize);
default: LogMan::Msg::A("Unhandled SpillRegister size: %d", OpSize);
}
} else if (Op->Class == FEXCore::IR::FPRClass) {
switch (OpSize) {
@@ -516,10 +515,10 @@ DEF_OP(FillRegister) {
ldr(GetDst(Node), MemOperand(sp, SlotOffset));
break;
}
default: LOGMAN_MSG_A("Unhandled SpillRegister size: %d", OpSize);
default: LogMan::Msg::A("Unhandled SpillRegister size: %d", OpSize);
}
} else {
LOGMAN_MSG_A("Unhandled FillRegister class: %d", Op->Class.Val);
LogMan::Msg::A("Unhandled FillRegister class: %d", Op->Class.Val);
}
}
@@ -539,7 +538,7 @@ MemOperand Arm64JITCore::GenerateMemOperand(uint8_t AccessSize, aarch64::Registe
return MemOperand(Base);
} else {
if (OffsetScale != 1 && OffsetScale != AccessSize) {
LOGMAN_MSG_A("Unhandled GenerateMemOperand OffsetScale: %d", OffsetScale);
LogMan::Msg::A("Unhandled GenerateMemOperand OffsetScale: %d", OffsetScale);
}
uint64_t Const;
if (IsInlineConstant(Offset, &Const)) {
@@ -551,12 +550,11 @@ MemOperand Arm64JITCore::GenerateMemOperand(uint8_t AccessSize, aarch64::Registe
case IR::MEM_OFFSET_UXTW.Val: return MemOperand(Base, RegOffset.W(), Extend::UXTW, (int)std::log2(OffsetScale) );
case IR::MEM_OFFSET_SXTW.Val: return MemOperand(Base, RegOffset.W(), Extend::SXTW, (int)std::log2(OffsetScale) );
default: LOGMAN_MSG_A("Unhandled GenerateMemOperand OffsetType: %d", OffsetType.Val); break;
default: LogMan::Msg::A("Unhandled GenerateMemOperand OffsetType: %d", OffsetType.Val); break;
}
}
}
FEX_UNREACHABLE;
__builtin_unreachable();
}
DEF_OP(LoadMem) {
@@ -580,7 +578,7 @@ DEF_OP(LoadMem) {
case 8:
ldr(Dst, MemSrc);
break;
default: LOGMAN_MSG_A("Unhandled LoadMem size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled LoadMem size: %d", Op->Size);
}
}
else {
@@ -601,7 +599,7 @@ DEF_OP(LoadMem) {
case 16:
ldr(Dst, MemSrc);
break;
default: LOGMAN_MSG_A("Unhandled LoadMem size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled LoadMem size: %d", Op->Size);
}
}
}
@@ -612,7 +610,7 @@ DEF_OP(LoadMemTSO) {
auto MemSrc = MemOperand(GetReg<RA_64>(Op->Header.Args[0].ID()));
if (!Op->Offset.IsInvalid()) {
LOGMAN_MSG_A("LoadMemTSO: No offset allowed");
LogMan::Msg::A("LoadMemTSO: No offset allowed");
}
if (SupportsRCPC && Op->Class == FEXCore::IR::GPRClass) {
@@ -635,7 +633,7 @@ DEF_OP(LoadMemTSO) {
case 8:
ldapr(Dst, MemSrc);
break;
default: LOGMAN_MSG_A("Unhandled LoadMem size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled LoadMem size: %d", Op->Size);
}
nop();
}
@@ -660,7 +658,7 @@ DEF_OP(LoadMemTSO) {
case 8:
ldar(Dst, MemSrc);
break;
default: LOGMAN_MSG_A("Unhandled LoadMem size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled LoadMem size: %d", Op->Size);
}
nop();
}
@@ -681,7 +679,7 @@ DEF_OP(LoadMemTSO) {
case 16:
ldr(Dst, MemSrc);
break;
default: LOGMAN_MSG_A("Unhandled LoadMem size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled LoadMem size: %d", Op->Size);
}
dmb(InnerShareable, BarrierAll);
}
@@ -708,7 +706,7 @@ DEF_OP(StoreMem) {
case 8:
str(GetReg<RA_64>(Op->Header.Args[1].ID()), MemSrc);
break;
default: LOGMAN_MSG_A("Unhandled StoreMem size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled StoreMem size: %d", Op->Size);
}
}
else {
@@ -729,7 +727,7 @@ DEF_OP(StoreMem) {
case 16:
str(Src, MemSrc);
break;
default: LOGMAN_MSG_A("Unhandled StoreMem size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled StoreMem size: %d", Op->Size);
}
}
}
@@ -739,7 +737,7 @@ DEF_OP(StoreMemTSO) {
auto MemSrc = MemOperand(GetReg<RA_64>(Op->Header.Args[0].ID()));
if (!Op->Offset.IsInvalid()) {
LOGMAN_MSG_A("StoreMemTSO: No offset allowed");
LogMan::Msg::A("StoreMemTSO: No offset allowed");
}
if (Op->Class == FEXCore::IR::GPRClass) {
@@ -759,7 +757,7 @@ DEF_OP(StoreMemTSO) {
case 8:
stlr(GetReg<RA_64>(Op->Header.Args[1].ID()), MemSrc);
break;
default: LOGMAN_MSG_A("Unhandled StoreMem size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled StoreMem size: %d", Op->Size);
}
nop();
}
@@ -783,177 +781,18 @@ DEF_OP(StoreMemTSO) {
case 16:
str(Src, MemSrc);
break;
default: LOGMAN_MSG_A("Unhandled StoreMem size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled StoreMem size: %d", Op->Size);
}
dmb(InnerShareable, BarrierAll);
}
}
DEF_OP(ParanoidLoadMemTSO) {
auto Op = IROp->C<IR::IROp_LoadMemTSO>();
auto MemSrc = MemOperand(GetReg<RA_64>(Op->Header.Args[0].ID()));
if (!Op->Offset.IsInvalid()) {
LOGMAN_MSG_A("LoadMemTSO: No offset allowed");
}
if (Op->Class == FEXCore::IR::GPRClass) {
if (Op->Size == 1) {
// 8bit load is always aligned to natural alignment
auto Dst = GetReg<RA_64>(Node);
ldarb(Dst, MemSrc);
}
else {
auto Dst = GetReg<RA_64>(Node);
nop();
switch (Op->Size) {
case 2:
ldarh(Dst, MemSrc);
break;
case 4:
ldar(Dst.W(), MemSrc);
break;
case 8:
ldar(Dst, MemSrc);
break;
default: LOGMAN_MSG_A("Unhandled LoadMem size: %d", Op->Size);
}
nop();
}
}
else {
auto Dst = GetDst(Node);
switch (Op->Size) {
case 2:
nop();
ldarh(TMP1, MemSrc);
nop();
fmov(Dst, TMP1);
break;
case 4:
nop();
ldar(TMP1.W(), MemSrc);
nop();
fmov(Dst, TMP1);
break;
case 8:
nop();
ldar(TMP1, MemSrc);
nop();
fmov(Dst, TMP1);
break;
case 16:
nop();
ldaxp(TMP1, TMP2, MemSrc);
clrex();
mov(Dst.V2D(), 0, TMP1);
mov(Dst.V2D(), 1, TMP2);
break;
default: LOGMAN_MSG_A("Unhandled LoadMem size: %d", Op->Size);
}
}
}
DEF_OP(ParanoidStoreMemTSO) {
auto Op = IROp->C<IR::IROp_StoreMemTSO>();
auto MemSrc = MemOperand(GetReg<RA_64>(Op->Header.Args[0].ID()));
if (!Op->Offset.IsInvalid()) {
LOGMAN_MSG_A("StoreMemTSO: No offset allowed");
}
if (Op->Class == FEXCore::IR::GPRClass) {
if (Op->Size == 1) {
// 8bit load is always aligned to natural alignment
stlrb(GetReg<RA_64>(Op->Header.Args[1].ID()), MemSrc);
}
else {
nop();
switch (Op->Size) {
case 2:
stlrh(GetReg<RA_64>(Op->Header.Args[1].ID()), MemSrc);
break;
case 4:
stlr(GetReg<RA_32>(Op->Header.Args[1].ID()), MemSrc);
break;
case 8:
stlr(GetReg<RA_64>(Op->Header.Args[1].ID()), MemSrc);
break;
default: LOGMAN_MSG_A("Unhandled StoreMem size: %d", Op->Size);
}
nop();
}
}
else {
auto Src = GetSrc(Op->Header.Args[1].ID());
if (Op->Size == 1) {
// 8bit load is always aligned to natural alignment
mov(TMP1, Src.V4S(), 0);
stlrb(TMP1, MemSrc);
}
else {
switch (Op->Size) {
case 2:
mov(TMP1, Src.V4S(), 0);
nop();
stlrh(TMP1, MemSrc);
nop();
break;
case 4:
mov(TMP1, Src.V4S(), 0);
nop();
stlr(TMP1.W(), MemSrc);
nop();
break;
case 8:
mov(TMP1, Src.V2D(), 0);
nop();
stlr(TMP1, MemSrc);
nop();
break;
case 16: {
// Move vector to GPRs
mov(TMP1, Src.V2D(), 0);
mov(TMP2, Src.V2D(), 1);
Label B;
bind(&B);
nop(); // < Overwritten with DMB
// ldaxp must not have both the destination registers be the same
ldaxp(xzr, TMP3, MemSrc); // <- Can hit SIGBUS
nop(); // < Overwritten with DMB
stlxp(TMP3, TMP1, TMP2, MemSrc); // <- Can also hit SIGBUS
cbnz(TMP3, &B); // < Overwritten with DMB
break;
}
default: LOGMAN_MSG_A("Unhandled StoreMem size: %d", Op->Size);
}
}
}
}
DEF_OP(VLoadMemElement) {
LOGMAN_MSG_A("Unimplemented");
LogMan::Msg::A("Unimplemented");
}
DEF_OP(VStoreMemElement) {
LOGMAN_MSG_A("Unimplemented");
}
DEF_OP(CacheLineClear) {
auto Op = IROp->C<IR::IROp_CacheLineClear>();
auto MemReg = GetReg<RA_64>(Op->Header.Args[0].ID());
// Clear dcache only
// icache doesn't matter here since the guest application shouldn't be calling clflush on JIT code.
mov(TMP1, MemReg);
for (size_t i = 0; i < std::max(1U, DCacheLineSize / 64U); ++i) {
dc(DataCacheOp::CVAU, TMP1);
add(TMP1, TMP1, DCacheLineSize);
}
dsb(InnerShareable, BarrierAll);
LogMan::Msg::A("Unimplemented");
}
#undef DEF_OP
@@ -971,17 +810,10 @@ void Arm64JITCore::RegisterMemoryHandlers() {
REGISTER_OP(STOREFLAG, StoreFlag);
REGISTER_OP(LOADMEM, LoadMem);
REGISTER_OP(STOREMEM, StoreMem);
if (ParanoidTSO()) {
REGISTER_OP(LOADMEMTSO, ParanoidLoadMemTSO);
REGISTER_OP(STOREMEMTSO, ParanoidStoreMemTSO);
}
else {
REGISTER_OP(LOADMEMTSO, LoadMemTSO);
REGISTER_OP(STOREMEMTSO, StoreMemTSO);
}
REGISTER_OP(LOADMEMTSO, LoadMemTSO);
REGISTER_OP(STOREMEMTSO, StoreMemTSO);
REGISTER_OP(VLOADMEMELEMENT, VLoadMemElement);
REGISTER_OP(VSTOREMEMELEMENT, VStoreMemElement);
REGISTER_OP(CACHELINECLEAR, CacheLineClear);
#undef REGISTER_OP
}
}
@@ -7,13 +7,6 @@ $end_info$
#include "Interface/Core/JIT/Arm64/JITClass.h"
namespace FEXCore::CPU {
static void PrintValue(uint64_t Value) {
LogMan::Msg::D("Value: 0x%lx", Value);
}
static void PrintVectorValue(uint64_t Value, uint64_t ValueUpper) {
LogMan::Msg::D("Value: 0x%016lx'%016lx", ValueUpper, Value);
}
using namespace vixl;
using namespace vixl::aarch64;
@@ -31,7 +24,7 @@ DEF_OP(Fence) {
case IR::Fence_Store.Val:
dmb(FullSystem, BarrierWrites);
break;
default: LOGMAN_MSG_A("Unknown Fence: %d", Op->Fence); break;
default: LogMan::Msg::A("Unknown Fence: %d", Op->Fence); break;
}
}
@@ -42,15 +35,6 @@ DEF_OP(Break) {
case 5: // Guest ud2
hlt(4);
break;
case 1: // Int <imm8>
hlt(4);
break;
case 2: // overflow
hlt(4);
break;
case 3: // int 1
hlt(4);
break;
case 4: { // HLT
// Time to quit
// Set our stack to the starting stack location
@@ -58,18 +42,18 @@ DEF_OP(Break) {
add(sp, TMP1, 0);
// Now we need to jump to the thread stop handler
LoadConstant(TMP1, ThreadSharedData.Dispatcher->ThreadStopHandlerAddressSpillSRA);
LoadConstant(TMP1, Dispatcher->ThreadStopHandlerAddressSpillSRA);
br(TMP1);
break;
}
case 6: { // INT3
ResetStack();
LoadConstant(TMP1, ThreadSharedData.Dispatcher->ThreadPauseHandlerAddressSpillSRA);
LoadConstant(TMP1, Dispatcher->ThreadPauseHandlerAddressSpillSRA);
br(TMP1);
break;
}
default: LOGMAN_MSG_A("Unknown Break reason: %d", Op->Reason);
default: LogMan::Msg::A("Unknown Break reason: %d", Op->Reason);
}
}
@@ -131,28 +115,6 @@ DEF_OP(SetRoundingMode) {
msr(FPCR, TMP1);
}
DEF_OP(Print) {
auto Op = IROp->C<IR::IROp_Print>();
PushDynamicRegsAndLR();
if (IsGPR(Op->Header.Args[0].ID())) {
mov(x0, GetReg<RA_64>(Op->Header.Args[0].ID()));
LoadConstant(x3, reinterpret_cast<uint64_t>(PrintValue));
}
else {
fmov(x0, GetSrc(Op->Header.Args[0].ID()).V1D());
// Bug in vixl that source vector needs to b V1D rather than V2D?
fmov(x1, GetSrc(Op->Header.Args[0].ID()).V1D(), 1);
LoadConstant(x3, reinterpret_cast<uint64_t>(PrintVectorValue));
}
SpillStaticRegs();
blr(x3);
FillStaticRegs();
PopDynamicRegsAndLR();
}
#undef DEF_OP
void Arm64JITCore::RegisterMiscHandlers() {
#define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &Arm64JITCore::Op_##x
@@ -165,7 +127,7 @@ void Arm64JITCore::RegisterMiscHandlers() {
REGISTER_OP(BREAK, Break);
REGISTER_OP(PHI, NoOp);
REGISTER_OP(PHIVALUE, NoOp);
REGISTER_OP(PRINT, Print);
REGISTER_OP(PRINT, Unhandled);
REGISTER_OP(GETROUNDINGMODE, GetRoundingMode);
REGISTER_OP(SETROUNDINGMODE, SetRoundingMode);
REGISTER_OP(INVALIDATEFLAGS, NoOp);
@@ -26,7 +26,7 @@ DEF_OP(ExtractElementPair) {
mov (GetReg<RA_64>(Node), Regs[Op->Element]);
break;
}
default: LOGMAN_MSG_A("Unknown Size"); break;
default: LogMan::Msg::A("Unknown Size"); break;
}
}
@@ -52,7 +52,7 @@ DEF_OP(CreateElementPair) {
RegTmp = x0;
break;
}
default: LOGMAN_MSG_A("Unknown Size"); break;
default: LogMan::Msg::A("Unknown Size"); break;
}
if (Dst.first.GetCode() != RegSecond.GetCode()) {
File diff suppressed because it is too large. Load diff
+2 -4
View File
@@ -1,7 +1,5 @@
#pragma once
#include <memory>
namespace FEXCore::Context {
struct Context;
}
@@ -13,6 +11,6 @@ struct InternalThreadState;
namespace FEXCore::CPU {
class CPUBackend;
std::unique_ptr<CPUBackend> CreateX86JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, bool CompileThread);
std::unique_ptr<CPUBackend> CreateArm64JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, bool CompileThread);
FEXCore::CPU::CPUBackend *CreateX86JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, bool CompileThread);
FEXCore::CPU::CPUBackend *CreateArm64JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, bool CompileThread);
}
+58 -77
View File
@@ -20,7 +20,7 @@ DEF_OP(TruncElementPair) {
mov(Dst.second, Src.second);
break;
}
default: LOGMAN_MSG_A("Unhandled Truncation size: %d", Op->Size); break;
default: LogMan::Msg::A("Unhandled Truncation size: %d", Op->Size); break;
}
}
@@ -32,7 +32,7 @@ DEF_OP(Constant) {
DEF_OP(EntrypointOffset) {
auto Op = IROp->C<IR::IROp_EntrypointOffset>();
auto Constant = Entry + Op->Offset;
auto Constant = IR->GetHeader()->Entry + Op->Offset;
mov(GetDst<RA_64>(Node), Constant);
}
@@ -70,7 +70,7 @@ DEF_OP(Add) {
case 8:
add(rax, Const);
break;
default: LOGMAN_MSG_A("Unhandled Add size: %d", OpSize);
default: LogMan::Msg::A("Unhandled Add size: %d", OpSize);
break;
}
} else {
@@ -81,7 +81,7 @@ DEF_OP(Add) {
case 8:
add(rax, GetSrc<RA_64>(Op->Header.Args[1].ID()));
break;
default: LOGMAN_MSG_A("Unhandled Add size: %d", OpSize);
default: LogMan::Msg::A("Unhandled Add size: %d", OpSize);
break;
}
}
@@ -103,7 +103,7 @@ DEF_OP(Sub) {
case 8:
sub(rax, Const);
break;
default: LOGMAN_MSG_A("Unhandled Sub size: %d", OpSize);
default: LogMan::Msg::A("Unhandled Sub size: %d", OpSize);
break;
}
} else {
@@ -114,7 +114,7 @@ DEF_OP(Sub) {
case 8:
sub(rax, GetSrc<RA_64>(Op->Header.Args[1].ID()));
break;
default: LOGMAN_MSG_A("Unhandled Sub size: %d", OpSize);
default: LogMan::Msg::A("Unhandled Sub size: %d", OpSize);
break;
}
}
@@ -136,7 +136,7 @@ DEF_OP(Neg) {
Src = GetSrc<RA_64>(Op->Header.Args[0].ID());
Dst = GetDst<RA_64>(Node);
break;
default: LOGMAN_MSG_A("Unhandled Neg size: %d", OpSize);
default: LogMan::Msg::A("Unhandled Neg size: %d", OpSize);
break;
}
mov(Dst, Src);
@@ -160,7 +160,7 @@ DEF_OP(Mul) {
imul(rax, GetSrc<RA_64>(Op->Header.Args[1].ID()));
mov(Dst, rax);
break;
default: LOGMAN_MSG_A("Unknown Sext size: %d", OpSize);
default: LogMan::Msg::A("Unknown Sext size: %d", OpSize);
}
}
@@ -179,7 +179,7 @@ DEF_OP(UMul) {
mul(GetSrc<RA_64>(Op->Header.Args[1].ID()));
mov(GetDst<RA_64>(Node), rax);
break;
default: LOGMAN_MSG_A("Unknown Sext size: %d", OpSize);
default: LogMan::Msg::A("Unknown Sext size: %d", OpSize);
}
}
@@ -218,7 +218,7 @@ DEF_OP(Div) {
mov(GetDst<RA_64>(Node), rax);
break;
}
default: LOGMAN_MSG_A("Unknown UDIV Size: %d", Size); break;
default: LogMan::Msg::A("Unknown UDIV Size: %d", Size); break;
}
}
@@ -261,7 +261,7 @@ DEF_OP(UDiv) {
mov(GetDst<RA_64>(Node), rax);
break;
}
default: LOGMAN_MSG_A("Unknown UDIV OpSize: %d", OpSize); break;
default: LogMan::Msg::A("Unknown UDIV OpSize: %d", OpSize); break;
}
}
@@ -298,7 +298,7 @@ DEF_OP(Rem) {
mov(GetDst<RA_64>(Node), rdx);
break;
}
default: LOGMAN_MSG_A("Unknown UDIV Size: %d", OpSize); break;
default: LogMan::Msg::A("Unknown UDIV Size: %d", OpSize); break;
}
}
@@ -341,7 +341,7 @@ DEF_OP(URem) {
mov(GetDst<RA_64>(Node), rdx);
break;
}
default: LOGMAN_MSG_A("Unknown UDIV OpSize: %d", OpSize); break;
default: LogMan::Msg::A("Unknown UDIV OpSize: %d", OpSize); break;
}
}
@@ -360,7 +360,7 @@ DEF_OP(MulH) {
imul(GetSrc<RA_64>(Op->Header.Args[1].ID()));
mov(GetDst<RA_64>(Node), rdx);
break;
default: LOGMAN_MSG_A("Unknown Sext size: %d", OpSize);
default: LogMan::Msg::A("Unknown Sext size: %d", OpSize);
}
}
@@ -379,7 +379,7 @@ DEF_OP(UMulH) {
mul(GetSrc<RA_64>(Op->Header.Args[1].ID()));
mov(GetDst<RA_64>(Node), rdx);
break;
default: LOGMAN_MSG_A("Unknown Sext size: %d", OpSize);
default: LogMan::Msg::A("Unknown Sext size: %d", OpSize);
}
}
@@ -441,7 +441,7 @@ DEF_OP(Lshl) {
mov(GetDst<RA_64>(Node), GetSrc<RA_64>(Op->Header.Args[0].ID()));
shl(GetDst<RA_64>(Node), Const);
break;
default: LOGMAN_MSG_A("Unknown LSHL Size: %d\n", OpSize); break;
default: LogMan::Msg::A("Unknown LSHL Size: %d\n", OpSize); break;
};
} else {
mov(rcx, GetSrc<RA_64>(Op->Header.Args[1].ID()));
@@ -456,7 +456,7 @@ DEF_OP(Lshl) {
mov(GetDst<RA_64>(Node), GetSrc<RA_64>(Op->Header.Args[0].ID()));
shl(GetDst<RA_64>(Node), cl);
break;
default: LOGMAN_MSG_A("Unknown LSHL Size: %d\n", OpSize); break;
default: LogMan::Msg::A("Unknown LSHL Size: %d\n", OpSize); break;
};
}
}
@@ -488,7 +488,7 @@ DEF_OP(Lshr) {
mov(GetDst<RA_64>(Node), GetSrc<RA_64>(Op->Header.Args[0].ID()));
shr(GetDst<RA_64>(Node), Const);
break;
default: LOGMAN_MSG_A("Unknown Size: %d\n", OpSize); break;
default: LogMan::Msg::A("Unknown Size: %d\n", OpSize); break;
};
} else {
@@ -512,7 +512,7 @@ DEF_OP(Lshr) {
mov(GetDst<RA_64>(Node), GetSrc<RA_64>(Op->Header.Args[0].ID()));
shr(GetDst<RA_64>(Node), cl);
break;
default: LOGMAN_MSG_A("Unknown Size: %d\n", OpSize); break;
default: LogMan::Msg::A("Unknown Size: %d\n", OpSize); break;
};
}
}
@@ -546,7 +546,7 @@ DEF_OP(Ashr) {
mov(GetDst<RA_64>(Node), GetSrc<RA_64>(Op->Header.Args[0].ID()));
sar(GetDst<RA_64>(Node), Const);
break;
default: LOGMAN_MSG_A("Unknown ASHR Size: %d\n", OpSize); break;
default: LogMan::Msg::A("Unknown ASHR Size: %d\n", OpSize); break;
};
} else {
@@ -571,7 +571,7 @@ DEF_OP(Ashr) {
mov(GetDst<RA_64>(Node), GetSrc<RA_64>(Op->Header.Args[0].ID()));
sar(GetDst<RA_64>(Node), cl);
break;
default: LOGMAN_MSG_A("Unknown ASHR Size: %d\n", OpSize); break;
default: LogMan::Msg::A("Unknown ASHR Size: %d\n", OpSize); break;
};
}
}
@@ -596,7 +596,7 @@ DEF_OP(Ror) {
ror(rax, Const);
break;
}
default: LOGMAN_MSG_A("Unknown ROR Size: %d\n", OpSize); break;
default: LogMan::Msg::A("Unknown ROR Size: %d\n", OpSize); break;
}
} else {
mov (rcx, GetSrc<RA_64>(Op->Header.Args[1].ID()));
@@ -612,7 +612,7 @@ DEF_OP(Ror) {
ror(rax, cl);
break;
}
default: LOGMAN_MSG_A("Unknown ROR Size: %d\n", OpSize); break;
default: LogMan::Msg::A("Unknown ROR Size: %d\n", OpSize); break;
}
}
mov(GetDst<RA_64>(Node), rax);
@@ -668,7 +668,7 @@ DEF_OP(LDiv) {
mov(GetDst<RA_64>(Node), rax);
break;
}
default: LOGMAN_MSG_A("Unknown LDIV OpSize: %d", OpSize); break;
default: LogMan::Msg::A("Unknown LDIV OpSize: %d", OpSize); break;
}
}
@@ -700,7 +700,7 @@ DEF_OP(LUDiv) {
mov(GetDst<RA_64>(Node), rax);
break;
}
default: LOGMAN_MSG_A("Unknown LUDIV OpSize: %d", OpSize); break;
default: LogMan::Msg::A("Unknown LUDIV OpSize: %d", OpSize); break;
}
}
@@ -732,7 +732,7 @@ DEF_OP(LRem) {
mov(GetDst<RA_64>(Node), rdx);
break;
}
default: LOGMAN_MSG_A("Unknown LREM OpSize: %d", OpSize); break;
default: LogMan::Msg::A("Unknown LREM OpSize: %d", OpSize); break;
}
}
@@ -764,7 +764,7 @@ DEF_OP(LURem) {
mov(GetDst<RA_64>(Node), rdx);
break;
}
default: LOGMAN_MSG_A("Unknown LUDIV OpSize: %d", OpSize); break;
default: LogMan::Msg::A("Unknown LUDIV OpSize: %d", OpSize); break;
}
}
@@ -829,7 +829,7 @@ DEF_OP(FindMSB) {
case 8:
bsr(GetDst<RA_64>(Node), GetSrc<RA_64>(Op->Header.Args[0].ID()));
break;
default: LOGMAN_MSG_A("Unknown OpSize: %d", OpSize);
default: LogMan::Msg::A("Unknown OpSize: %d", OpSize);
}
}
@@ -853,7 +853,7 @@ DEF_OP(FindTrailingZeros) {
mov(rax, 0x40);
cmovz(GetDst<RA_64>(Node), rax);
break;
default: LOGMAN_MSG_A("Unknown size: %d", OpSize); break;
default: LogMan::Msg::A("Unknown size: %d", OpSize); break;
}
}
@@ -876,7 +876,7 @@ DEF_OP(CountLeadingZeroes) {
lzcnt(GetDst<RA_64>(Node), GetSrc<RA_64>(Op->Header.Args[0].ID()));
break;
}
default: LOGMAN_MSG_A("Unknown size: %d", OpSize); break;
default: LogMan::Msg::A("Unknown size: %d", OpSize); break;
}
}
else {
@@ -915,7 +915,7 @@ DEF_OP(CountLeadingZeroes) {
mov(GetDst<RA_64>(Node), rax);
break;
}
default: LOGMAN_MSG_A("Unknown size: %d", OpSize); break;
default: LogMan::Msg::A("Unknown size: %d", OpSize); break;
}
}
}
@@ -937,7 +937,7 @@ DEF_OP(Rev) {
mov (GetDst<RA_64>(Node), GetSrc<RA_64>(Op->Header.Args[0].ID()));
bswap(GetDst<RA_64>(Node).cvt64());
break;
default: LOGMAN_MSG_A("Unknown REV size: %d", OpSize); break;
default: LogMan::Msg::A("Unknown REV size: %d", OpSize); break;
}
}
@@ -972,7 +972,7 @@ DEF_OP(Bfe) {
auto Op = IROp->C<IR::IROp_Bfe>();
uint8_t OpSize = IROp->Size;
LOGMAN_THROW_A(OpSize <= 8, "OpSize is too large for BFE: %d", OpSize);
LogMan::Throw::A(OpSize <= 8, "OpSize is too large for BFE: %d", OpSize);
auto Dst = GetDst<RA_64>(Node);
@@ -1073,7 +1073,7 @@ DEF_OP(Select) {
if (is_const_true || is_const_false) {
if (is_const_false != true || is_const_true != true || const_true != 1 || const_false != 0) {
LOGMAN_MSG_A("Select: Unsupported compare inline parameters");
LogMan::Msg::A("Select: Unsupported compare inline parameters");
}
(this->*SetCC)(al);
movzx(Dst, al);
@@ -1104,67 +1104,46 @@ DEF_OP(VExtractToGPR) {
pextrq(GetDst<RA_64>(Node), GetSrc(Op->Header.Args[0].ID()), Op->Idx);
break;
}
default: LOGMAN_MSG_A("Unknown Element Size: %d", Op->Header.ElementSize); break;
default: LogMan::Msg::A("Unknown Element Size: %d", Op->Header.ElementSize); break;
}
}
DEF_OP(Float_ToGPR_ZU) {
LogMan::Msg::D("Unimplemented");
}
DEF_OP(Float_ToGPR_ZS) {
auto Op = IROp->C<IR::IROp_Float_ToGPR_ZS>();
uint16_t Conv = (IROp->Size << 8) | Op->SrcElementSize;
switch (Conv) {
case 0x0804: // int64_t <- float
cvttss2si(GetDst<RA_64>(Node), GetSrc(Op->Header.Args[0].ID()));
break;
case 0x0808: // int64_t <- double
cvttsd2si(GetDst<RA_64>(Node), GetSrc(Op->Header.Args[0].ID()));
break;
case 0x0404: // int32_t <- float
cvttss2si(GetDst<RA_32>(Node), GetSrc(Op->Header.Args[0].ID()));
break;
case 0x0408: // int32_t <- double
cvttsd2si(GetDst<RA_32>(Node), GetSrc(Op->Header.Args[0].ID()));
break;
if (Op->Header.ElementSize == 8) {
cvttsd2si(GetDst<RA_64>(Node), GetSrc(Op->Header.Args[0].ID()));
}
else {
cvttss2si(GetDst<RA_32>(Node), GetSrc(Op->Header.Args[0].ID()));
}
}
DEF_OP(Float_ToGPR_U) {
LogMan::Msg::D("Unimplemented");
}
DEF_OP(Float_ToGPR_S) {
auto Op = IROp->C<IR::IROp_Float_ToGPR_S>();
uint16_t Conv = (IROp->Size << 8) | Op->SrcElementSize;
switch (Conv) {
case 0x0804: // int64_t <- float
cvtss2si(GetDst<RA_64>(Node), GetSrc(Op->Header.Args[0].ID()));
break;
case 0x0808: // int64_t <- double
cvtsd2si(GetDst<RA_64>(Node), GetSrc(Op->Header.Args[0].ID()));
break;
case 0x0404: // int32_t <- float
cvtss2si(GetDst<RA_32>(Node), GetSrc(Op->Header.Args[0].ID()));
break;
case 0x0408: // int32_t <- double
cvtsd2si(GetDst<RA_32>(Node), GetSrc(Op->Header.Args[0].ID()));
break;
if (Op->Header.ElementSize == 8) {
cvtsd2si(GetDst<RA_64>(Node), GetSrc(Op->Header.Args[0].ID()));
}
else {
cvtss2si(GetDst<RA_32>(Node), GetSrc(Op->Header.Args[0].ID()));
}
}
DEF_OP(FCmp) {
auto Op = IROp->C<IR::IROp_FCmp>();
if (Op->Flags & (1 << IR::FCMP_FLAG_UNORDERED)) {
if (Op->ElementSize == 4) {
ucomiss(GetSrc(Op->Header.Args[0].ID()), GetSrc(Op->Header.Args[1].ID()));
}
else {
ucomisd(GetSrc(Op->Header.Args[0].ID()), GetSrc(Op->Header.Args[1].ID()));
}
if (Op->ElementSize == 4) {
ucomiss(GetSrc(Op->Header.Args[0].ID()), GetSrc(Op->Header.Args[1].ID()));
}
else {
if (Op->ElementSize == 4) {
comiss(GetSrc(Op->Header.Args[0].ID()), GetSrc(Op->Header.Args[1].ID()));
}
else {
comisd(GetSrc(Op->Header.Args[0].ID()), GetSrc(Op->Header.Args[1].ID()));
}
ucomisd(GetSrc(Op->Header.Args[0].ID()), GetSrc(Op->Header.Args[1].ID()));
}
mov (rdx, 0);
@@ -1238,7 +1217,9 @@ void X86JITCore::RegisterALUHandlers() {
REGISTER_OP(SBFE, Sbfe);
REGISTER_OP(SELECT, Select);
REGISTER_OP(VEXTRACTTOGPR, VExtractToGPR);
REGISTER_OP(FLOAT_TOGPR_ZU, Float_ToGPR_ZU);
REGISTER_OP(FLOAT_TOGPR_ZS, Float_ToGPR_ZS);
REGISTER_OP(FLOAT_TOGPR_U, Float_ToGPR_U);
REGISTER_OP(FLOAT_TOGPR_S, Float_ToGPR_S);
REGISTER_OP(FCMP, FCmp);
#undef REGISTER_OP
@@ -55,7 +55,7 @@ DEF_OP(CASPair) {
mov(Dst.second, rdx);
break;
}
default: LOGMAN_MSG_A("Unsupported: %d", OpSize);
default: LogMan::Msg::A("Unsupported: %d", OpSize);
}
}
@@ -74,6 +74,7 @@ DEF_OP(CAS) {
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Header.Args[2].ID());
mov(rdx, GetSrc<RA_64>(Op->Header.Args[1].ID()));
mov(rax, GetSrc<RA_64>(Op->Header.Args[0].ID()));
// RCX now contains pointer
@@ -81,31 +82,31 @@ DEF_OP(CAS) {
// RDX contains our desired
lock();
switch (OpSize) {
case 1: {
cmpxchg(byte [MemReg], GetSrc<RA_8>(Op->Header.Args[1].ID()));
movzx(GetDst<RA_64>(Node), al);
cmpxchg(byte [MemReg], dl);
movzx(rax, al);
break;
}
case 2: {
cmpxchg(word [MemReg], GetSrc<RA_16>(Op->Header.Args[1].ID()));
movzx(GetDst<RA_64>(Node), ax);
cmpxchg(word [MemReg], dx);
movzx(rax, ax);
break;
}
case 4: {
cmpxchg(dword [MemReg], GetSrc<RA_32>(Op->Header.Args[1].ID()));
// RAX now contains the result
mov (GetDst<RA_64>(Node), eax);
cmpxchg(dword [MemReg], edx);
break;
}
case 8: {
cmpxchg(qword [MemReg], GetSrc<RA_64>(Op->Header.Args[1].ID()));
// RAX now contains the result
mov (GetDst<RA_64>(Node), rax);
cmpxchg(qword [MemReg], rdx);
break;
}
default: LOGMAN_MSG_A("Unsupported: %d", OpSize);
default: LogMan::Msg::A("Unsupported: %d", OpSize);
}
// RAX now contains the result
mov (GetDst<RA_64>(Node), rax);
}
DEF_OP(AtomicAdd) {
@@ -127,7 +128,7 @@ DEF_OP(AtomicAdd) {
case 8:
add(qword [MemReg], GetSrc<RA_64>(Op->Header.Args[1].ID()));
break;
default: LOGMAN_MSG_A("Unhandled AtomicAdd size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled AtomicAdd size: %d", Op->Size);
}
}
@@ -149,7 +150,7 @@ DEF_OP(AtomicSub) {
case 8:
sub(qword [MemReg], GetSrc<RA_64>(Op->Header.Args[1].ID()));
break;
default: LOGMAN_MSG_A("Unhandled AtomicAdd size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled AtomicAdd size: %d", Op->Size);
}
}
@@ -171,7 +172,7 @@ DEF_OP(AtomicAnd) {
case 8:
and_(qword [MemReg], GetSrc<RA_64>(Op->Header.Args[1].ID()));
break;
default: LOGMAN_MSG_A("Unhandled AtomicAdd size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled AtomicAdd size: %d", Op->Size);
}
}
@@ -193,7 +194,7 @@ DEF_OP(AtomicOr) {
case 8:
or_(qword [MemReg], GetSrc<RA_64>(Op->Header.Args[1].ID()));
break;
default: LOGMAN_MSG_A("Unhandled AtomicAdd size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled AtomicAdd size: %d", Op->Size);
}
}
@@ -215,7 +216,7 @@ DEF_OP(AtomicXor) {
case 8:
xor_(qword [MemReg], GetSrc<RA_64>(Op->Header.Args[1].ID()));
break;
default: LOGMAN_MSG_A("Unhandled AtomicAdd size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled AtomicAdd size: %d", Op->Size);
}
}
@@ -227,17 +228,17 @@ DEF_OP(AtomicSwap) {
switch (Op->Size) {
case 1:
movzx(GetDst<RA_64>(Node), GetSrc<RA_8>(Op->Header.Args[1].ID()));
mov(GetDst<RA_8>(Node), GetSrc<RA_8>(Op->Header.Args[1].ID()));
lock();
xchg(byte [MemReg], GetDst<RA_8>(Node));
break;
case 2:
movzx(GetDst<RA_64>(Node), GetSrc<RA_16>(Op->Header.Args[1].ID()));
mov(GetDst<RA_16>(Node), GetSrc<RA_16>(Op->Header.Args[1].ID()));
lock();
xchg(word [MemReg], GetDst<RA_16>(Node));
break;
case 4:
mov(GetDst<RA_64>(Node), GetSrc<RA_32>(Op->Header.Args[1].ID()));
mov(GetDst<RA_32>(Node), GetSrc<RA_32>(Op->Header.Args[1].ID()));
lock();
xchg(dword [MemReg], GetDst<RA_32>(Node));
break;
@@ -246,7 +247,7 @@ DEF_OP(AtomicSwap) {
lock();
xchg(qword [MemReg], GetDst<RA_64>(Node));
break;
default: LOGMAN_MSG_A("Unhandled AtomicAdd size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled AtomicAdd size: %d", Op->Size);
}
}
@@ -256,13 +257,13 @@ DEF_OP(AtomicFetchAdd) {
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Header.Args[0].ID());
switch (Op->Size) {
case 1:
movzx(rcx, GetSrc<RA_8>(Op->Header.Args[1].ID()));
mov(cl, GetSrc<RA_8>(Op->Header.Args[1].ID()));
lock();
xadd(byte [MemReg], cl);
movzx(GetDst<RA_32>(Node), cl);
break;
case 2:
movzx(rcx, GetSrc<RA_16>(Op->Header.Args[1].ID()));
mov(cx, GetSrc<RA_16>(Op->Header.Args[1].ID()));
lock();
xadd(word [MemReg], cx);
movzx(GetDst<RA_32>(Node), cx);
@@ -271,7 +272,7 @@ DEF_OP(AtomicFetchAdd) {
mov(ecx, GetSrc<RA_32>(Op->Header.Args[1].ID()));
lock();
xadd(dword [MemReg], ecx);
mov(GetDst<RA_64>(Node), ecx);
mov(GetDst<RA_32>(Node), ecx);
break;
case 8:
mov(rcx, GetSrc<RA_64>(Op->Header.Args[1].ID()));
@@ -279,7 +280,7 @@ DEF_OP(AtomicFetchAdd) {
xadd(qword [MemReg], rcx);
mov(GetDst<RA_64>(Node), rcx);
break;
default: LOGMAN_MSG_A("Unhandled AtomicFetchAdd size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled AtomicFetchAdd size: %d", Op->Size);
}
}
@@ -316,7 +317,7 @@ DEF_OP(AtomicFetchSub) {
xadd(qword [MemReg], rcx);
mov(GetDst<RA_64>(Node), rcx);
break;
default: LOGMAN_MSG_A("Unhandled AtomicFetchAdd size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled AtomicFetchAdd size: %d", Op->Size);
}
}
@@ -394,7 +395,7 @@ DEF_OP(AtomicFetchAnd) {
mov(GetDst<RA_64>(Node), TMP3.cvt64());
break;
}
default: LOGMAN_MSG_A("Unhandled AtomicFetchAdd size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled AtomicFetchAdd size: %d", Op->Size);
}
}
@@ -471,7 +472,7 @@ DEF_OP(AtomicFetchOr) {
mov(GetDst<RA_64>(Node), TMP3.cvt64());
break;
}
default: LOGMAN_MSG_A("Unhandled AtomicFetchAdd size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled AtomicFetchAdd size: %d", Op->Size);
}
}
@@ -548,7 +549,7 @@ DEF_OP(AtomicFetchXor) {
mov(GetDst<RA_64>(Node), TMP3.cvt64());
break;
}
default: LOGMAN_MSG_A("Unhandled AtomicFetchAdd size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled AtomicFetchAdd size: %d", Op->Size);
}
}
@@ -81,7 +81,7 @@ DEF_OP(ExitFunction) {
jmp(qword[rax]);
L(l_BranchHost);
dq(ThreadSharedData.Dispatcher->ExitFunctionLinkerAddress);
dq(Dispatcher->ExitFunctionLinkerAddress);
L(l_BranchGuest);
dq(NewRIP);
} else {
@@ -101,7 +101,7 @@ DEF_OP(ExitFunction) {
jmp(qword[LookupBase + 0]);
L(FullLookup);
mov(rax, ThreadSharedData.Dispatcher->AbsoluteLoopTopAddress);
mov(rax, Dispatcher->AbsoluteLoopTopAddress);
mov(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, State.rip)], RipReg);
jmp(rax);
}
@@ -253,7 +253,7 @@ DEF_OP(ValidateCode) {
int idx = 0;
xor_(GetDst<RA_64>(Node), GetDst<RA_64>(Node));
mov(rax, Entry + Op->Offset);
mov(rax, IR->GetHeader()->Entry + Op->Offset);
mov(rbx, 1);
while (len >= 4) {
cmp(dword[rax + idx], *(uint32_t*)(OldCode + idx));
@@ -286,7 +286,7 @@ DEF_OP(RemoveCodeEntry) {
sub(rsp, 8); // Align
mov(rdi, STATE);
mov(rax, Entry); // imm64 move
mov(rax, IR->GetHeader()->Entry); // imm64 move
mov(rsi, rax);
@@ -31,7 +31,7 @@ DEF_OP(VInsGPR) {
pinsrq(GetDst(Node), GetSrc<RA_64>(Op->Header.Args[1].ID()), Op->Index);
break;
}
default: LOGMAN_MSG_A("Unknown Element Size: %d", Op->Header.ElementSize); break;
default: LogMan::Msg::A("Unknown Element Size: %d", Op->Header.ElementSize); break;
}
}
@@ -52,10 +52,14 @@ DEF_OP(VCastFromGPR) {
case 8:
vmovq(GetDst(Node), GetSrc<RA_64>(Op->Header.Args[0].ID()).cvt64());
break;
default: LOGMAN_MSG_A("Unknown castGPR element size: %d", Op->Header.ElementSize);
default: LogMan::Msg::A("Unknown castGPR element size: %d", Op->Header.ElementSize);
}
}
DEF_OP(Float_FromGPR_U) {
LogMan::Msg::A("Unimplemented");
}
DEF_OP(Float_FromGPR_S) {
auto Op = IROp->C<IR::IROp_Float_FromGPR_S>();
uint16_t Conv = (Op->Header.ElementSize << 8) | Op->SrcElementSize;
@@ -91,10 +95,14 @@ DEF_OP(Float_FToF) {
cvtsd2ss(GetDst(Node), GetSrc(Op->Header.Args[0].ID()));
break;
}
default: LOGMAN_MSG_A("Unknown FCVT sizes: 0x%x", Conv);
default: LogMan::Msg::A("Unknown FCVT sizes: 0x%x", Conv);
}
}
DEF_OP(Vector_UToF) {
LogMan::Msg::A("Unimplemented");
}
DEF_OP(Vector_SToF) {
auto Op = IROp->C<IR::IROp_Vector_SToF>();
switch (Op->Header.ElementSize) {
@@ -113,10 +121,14 @@ DEF_OP(Vector_SToF) {
cvtsi2sd(xmm15, rax);
movlhps(GetDst(Node), xmm15);
break;
default: LOGMAN_MSG_A("Unknown castGPR element size: %d", Op->Header.ElementSize);
default: LogMan::Msg::A("Unknown castGPR element size: %d", Op->Header.ElementSize);
}
}
DEF_OP(Vector_FToZU) {
LogMan::Msg::A("Unimplemented");
}
DEF_OP(Vector_FToZS) {
auto Op = IROp->C<IR::IROp_Vector_FToZS>();
switch (Op->Header.ElementSize) {
@@ -126,10 +138,14 @@ DEF_OP(Vector_FToZS) {
case 8:
cvttpd2dq(GetDst(Node), GetSrc(Op->Header.Args[0].ID()));
break;
default: LOGMAN_MSG_A("Unknown castGPR element size: %d", Op->Header.ElementSize);
default: LogMan::Msg::A("Unknown castGPR element size: %d", Op->Header.ElementSize);
}
}
DEF_OP(Vector_FToU) {
LogMan::Msg::A("Unimplemented");
}
DEF_OP(Vector_FToS) {
auto Op = IROp->C<IR::IROp_Vector_FToS>();
switch (Op->Header.ElementSize) {
@@ -139,7 +155,7 @@ DEF_OP(Vector_FToS) {
case 8:
cvtpd2dq(GetDst(Node), GetSrc(Op->Header.Args[0].ID()));
break;
default: LOGMAN_MSG_A("Unknown castGPR element size: %d", Op->Header.ElementSize);
default: LogMan::Msg::A("Unknown castGPR element size: %d", Op->Header.ElementSize);
}
}
@@ -156,39 +172,7 @@ DEF_OP(Vector_FToF) {
cvtpd2ps(GetDst(Node), GetSrc(Op->Header.Args[0].ID()));
break;
}
default: LOGMAN_MSG_A("Unknown Conversion Type : 0%04x", Conv); break;
}
}
DEF_OP(Vector_FToI) {
auto Op = IROp->C<IR::IROp_Vector_FToI>();
uint8_t RoundMode{};
switch (Op->Round) {
case FEXCore::IR::Round_Nearest.Val:
RoundMode = 0b0000'0'0'00;
break;
case FEXCore::IR::Round_Negative_Infinity.Val:
RoundMode = 0b0000'0'0'01;
break;
case FEXCore::IR::Round_Positive_Infinity.Val:
RoundMode = 0b0000'0'0'10;
break;
case FEXCore::IR::Round_Towards_Zero.Val:
RoundMode = 0b0000'0'0'11;
break;
case FEXCore::IR::Round_Host.Val:
RoundMode = 0b0000'0'1'00;
break;
}
switch (Op->Header.ElementSize) {
case 4:
roundps(GetDst(Node), GetSrc(Op->Header.Args[0].ID()), RoundMode);
break;
case 8:
roundpd(GetDst(Node), GetSrc(Op->Header.Args[0].ID()), RoundMode);
break;
default: LogMan::Msg::A("Unknown Conversion Type : 0%04x", Conv); break;
}
}
@@ -197,13 +181,16 @@ void X86JITCore::RegisterConversionHandlers() {
#define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &X86JITCore::Op_##x
REGISTER_OP(VINSGPR, VInsGPR);
REGISTER_OP(VCASTFROMGPR, VCastFromGPR);
REGISTER_OP(FLOAT_FROMGPR_U, Float_FromGPR_U);
REGISTER_OP(FLOAT_FROMGPR_S, Float_FromGPR_S);
REGISTER_OP(FLOAT_FTOF, Float_FToF);
REGISTER_OP(VECTOR_UTOF, Vector_UToF);
REGISTER_OP(VECTOR_STOF, Vector_SToF);
REGISTER_OP(VECTOR_FTOZU, Vector_FToZU);
REGISTER_OP(VECTOR_FTOZS, Vector_FToZS);
REGISTER_OP(VECTOR_FTOU, Vector_FToU);
REGISTER_OP(VECTOR_FTOS, Vector_FToS);
REGISTER_OP(VECTOR_FTOF, Vector_FToF);
REGISTER_OP(VECTOR_FTOI, Vector_FToI);
#undef REGISTER_OP
}
}
+55 -57
View File
@@ -15,7 +15,6 @@ $end_info$
#include <FEXCore/Core/X86Enums.h>
#include <FEXCore/Core/UContext.h>
#include <FEXCore/Utils/Allocator.h>
#include <cmath>
#include <signal.h>
@@ -31,17 +30,17 @@ CodeBuffer AllocateNewCodeBuffer(size_t Size) {
CodeBuffer Buffer;
Buffer.Size = Size;
Buffer.Ptr = static_cast<uint8_t*>(
FEXCore::Allocator::mmap(nullptr,
mmap(nullptr,
Buffer.Size,
PROT_READ | PROT_WRITE | PROT_EXEC,
MAP_PRIVATE | MAP_ANONYMOUS,
-1, 0));
LOGMAN_THROW_A(Buffer.Ptr != reinterpret_cast<uint8_t*>(~0ULL), "Couldn't allocate code buffer");
LogMan::Throw::A(Buffer.Ptr != reinterpret_cast<uint8_t*>(~0ULL), "Couldn't allocate code buffer");
return Buffer;
}
void FreeCodeBuffer(CodeBuffer Buffer) {
FEXCore::Allocator::munmap(Buffer.Ptr, Buffer.Size);
munmap(Buffer.Ptr, Buffer.Size);
}
}
@@ -85,7 +84,7 @@ void X86JITCore::Op_Unhandled(FEXCore::IR::IROp_Header *IROp, uint32_t Node) {
FallbackInfo Info;
if (!InterpreterOps::GetFallbackHandler(IROp, &Info)) {
auto Name = FEXCore::IR::GetName(IROp->Op);
LOGMAN_MSG_A("Unhandled IR Op: %s", std::string(Name).c_str());
LogMan::Msg::A("Unhandled IR Op: %s", std::string(Name).c_str());
} else {
switch(Info.ABI) {
case FABI_VOID_U16: {
@@ -283,7 +282,7 @@ void X86JITCore::Op_Unhandled(FEXCore::IR::IROp_Header *IROp, uint32_t Node) {
case FABI_UNKNOWN:
default:
auto Name = FEXCore::IR::GetName(IROp->Op);
LOGMAN_MSG_A("Unhandled IR Fallback abi: %s %d", std::string(Name).c_str(), Info.ABI);
LogMan::Msg::A("Unhandled IR Fallback abi: %s %d", std::string(Name).c_str(), Info.ABI);
}
}
}
@@ -331,13 +330,13 @@ X86JITCore::X86JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalTh
config.ExitFunctionLink = reinterpret_cast<uintptr_t>(&ExitFunctionLink);
config.ExitFunctionLinkThis = reinterpret_cast<uintptr_t>(this);
Dispatcher = std::make_unique<X86Dispatcher>(CTX, ThreadState, config);
Dispatcher = new X86Dispatcher(CTX, ThreadState, config);
DispatchPtr = Dispatcher->DispatchPtr;
CallbackPtr = Dispatcher->CallbackPtr;
ThreadSharedData.SignalHandlerRefCounterPtr = &Dispatcher->SignalHandlerRefCounter;
ThreadSharedData.SignalHandlerReturnAddress = Dispatcher->SignalHandlerReturnAddress;
ThreadSharedData.Dispatcher = Dispatcher.get();
// This will register the host signal handler per thread, which is fine
CTX->SignalDelegation->RegisterHostSignalHandler(SIGILL, [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext) -> bool {
@@ -411,106 +410,106 @@ void X86JITCore::ClearCache() {
}
}
IR::PhysicalRegister X86JITCore::GetPhys(uint32_t Node) const {
IR::PhysicalRegister X86JITCore::GetPhys(uint32_t Node) {
auto PhyReg = RAData->GetNodeRegister(Node);
LOGMAN_THROW_A(PhyReg.Raw != 255, "Couldn't Allocate register for node: ssa%d. Class: %d", Node, PhyReg.Class);
LogMan::Throw::A(PhyReg.Raw != 255, "Couldn't Allocate register for node: ssa%d. Class: %d", Node, PhyReg.Class);
return PhyReg;
}
bool X86JITCore::IsFPR(uint32_t Node) const {
bool X86JITCore::IsFPR(uint32_t Node) {
return RAData->GetNodeRegister(Node).Class == IR::FPRClass.Val;
}
bool X86JITCore::IsGPR(uint32_t Node) const {
bool X86JITCore::IsGPR(uint32_t Node) {
return RAData->GetNodeRegister(Node).Class == IR::GPRClass.Val;
}
template<uint8_t RAType>
Xbyak::Reg X86JITCore::GetSrc(uint32_t Node) const {
Xbyak::Reg X86JITCore::GetSrc(uint32_t Node) {
// rax, rcx, rdx, rsi, r8, r9,
// r10
// Callee Saved
// rbx, rbp, r12, r13, r14, r15
auto PhyReg = GetPhys(Node);
if constexpr (RAType == RA_64)
if (RAType == RA_64)
return RA64[PhyReg.Reg].cvt64();
else if constexpr (RAType == RA_XMM)
else if (RAType == RA_XMM)
return RAXMM[PhyReg.Reg];
else if constexpr (RAType == RA_32)
else if (RAType == RA_32)
return RA64[PhyReg.Reg].cvt32();
else if constexpr (RAType == RA_16)
else if (RAType == RA_16)
return RA64[PhyReg.Reg].cvt16();
else if constexpr (RAType == RA_8)
else if (RAType == RA_8)
return RA64[PhyReg.Reg].cvt8();
}
template
Xbyak::Reg X86JITCore::GetSrc<X86JITCore::RA_64>(uint32_t Node) const;
Xbyak::Reg X86JITCore::GetSrc<X86JITCore::RA_64>(uint32_t Node);
template
Xbyak::Reg X86JITCore::GetSrc<X86JITCore::RA_32>(uint32_t Node) const;
Xbyak::Reg X86JITCore::GetSrc<X86JITCore::RA_32>(uint32_t Node);
template
Xbyak::Reg X86JITCore::GetSrc<X86JITCore::RA_16>(uint32_t Node) const;
Xbyak::Reg X86JITCore::GetSrc<X86JITCore::RA_16>(uint32_t Node);
template
Xbyak::Reg X86JITCore::GetSrc<X86JITCore::RA_8>(uint32_t Node) const;
Xbyak::Reg X86JITCore::GetSrc<X86JITCore::RA_8>(uint32_t Node);
Xbyak::Xmm X86JITCore::GetSrc(uint32_t Node) const {
Xbyak::Xmm X86JITCore::GetSrc(uint32_t Node) {
auto PhyReg = GetPhys(Node);
return RAXMM_x[PhyReg.Reg];
}
template<uint8_t RAType>
Xbyak::Reg X86JITCore::GetDst(uint32_t Node) const {
Xbyak::Reg X86JITCore::GetDst(uint32_t Node) {
auto PhyReg = GetPhys(Node);
if constexpr (RAType == RA_64)
if (RAType == RA_64)
return RA64[PhyReg.Reg].cvt64();
else if constexpr (RAType == RA_XMM)
else if (RAType == RA_XMM)
return RAXMM[PhyReg.Reg];
else if constexpr (RAType == RA_32)
else if (RAType == RA_32)
return RA64[PhyReg.Reg].cvt32();
else if constexpr (RAType == RA_16)
else if (RAType == RA_16)
return RA64[PhyReg.Reg].cvt16();
else if constexpr (RAType == RA_8)
else if (RAType == RA_8)
return RA64[PhyReg.Reg].cvt8();
}
template
Xbyak::Reg X86JITCore::GetDst<X86JITCore::RA_64>(uint32_t Node) const;
Xbyak::Reg X86JITCore::GetDst<X86JITCore::RA_64>(uint32_t Node);
template
Xbyak::Reg X86JITCore::GetDst<X86JITCore::RA_32>(uint32_t Node) const;
Xbyak::Reg X86JITCore::GetDst<X86JITCore::RA_32>(uint32_t Node);
template
Xbyak::Reg X86JITCore::GetDst<X86JITCore::RA_16>(uint32_t Node) const;
Xbyak::Reg X86JITCore::GetDst<X86JITCore::RA_16>(uint32_t Node);
template
Xbyak::Reg X86JITCore::GetDst<X86JITCore::RA_8>(uint32_t Node) const;
Xbyak::Reg X86JITCore::GetDst<X86JITCore::RA_8>(uint32_t Node);
template<uint8_t RAType>
std::pair<Xbyak::Reg, Xbyak::Reg> X86JITCore::GetSrcPair(uint32_t Node) const {
std::pair<Xbyak::Reg, Xbyak::Reg> X86JITCore::GetSrcPair(uint32_t Node) {
auto PhyReg = GetPhys(Node);
if constexpr (RAType == RA_64)
if (RAType == RA_64)
return RA64Pair[PhyReg.Reg];
else if constexpr (RAType == RA_32)
else if (RAType == RA_32)
return {RA64Pair[PhyReg.Reg].first.cvt32(), RA64Pair[PhyReg.Reg].second.cvt32()};
}
template
std::pair<Xbyak::Reg, Xbyak::Reg> X86JITCore::GetSrcPair<X86JITCore::RA_64>(uint32_t Node) const;
std::pair<Xbyak::Reg, Xbyak::Reg> X86JITCore::GetSrcPair<X86JITCore::RA_64>(uint32_t Node);
template
std::pair<Xbyak::Reg, Xbyak::Reg> X86JITCore::GetSrcPair<X86JITCore::RA_32>(uint32_t Node) const;
std::pair<Xbyak::Reg, Xbyak::Reg> X86JITCore::GetSrcPair<X86JITCore::RA_32>(uint32_t Node);
Xbyak::Xmm X86JITCore::GetDst(uint32_t Node) const {
Xbyak::Xmm X86JITCore::GetDst(uint32_t Node) {
auto PhyReg = GetPhys(Node);
return RAXMM_x[PhyReg.Reg];
}
bool X86JITCore::IsInlineConstant(const IR::OrderedNodeWrapper& WNode, uint64_t* Value) const {
bool X86JITCore::IsInlineConstant(const IR::OrderedNodeWrapper& WNode, uint64_t* Value) {
auto OpHeader = IR->GetOp<IR::IROp_Header>(WNode);
if (OpHeader->Op == IR::IROps::OP_INLINECONSTANT) {
@@ -524,13 +523,13 @@ bool X86JITCore::IsInlineConstant(const IR::OrderedNodeWrapper& WNode, uint64_t*
}
}
bool X86JITCore::IsInlineEntrypointOffset(const IR::OrderedNodeWrapper& WNode, uint64_t* Value) const {
bool X86JITCore::IsInlineEntrypointOffset(const IR::OrderedNodeWrapper& WNode, uint64_t* Value) {
auto OpHeader = IR->GetOp<IR::IROp_Header>(WNode);
if (OpHeader->Op == IR::IROps::OP_INLINEENTRYPOINTOFFSET) {
auto Op = OpHeader->C<IR::IROp_InlineEntrypointOffset>();
if (Value) {
*Value = Entry + Op->Offset;
*Value = IR->GetHeader()->Entry + Op->Offset;
}
return true;
} else {
@@ -563,7 +562,7 @@ std::tuple<X86JITCore::SetCC, X86JITCore::CMovCC, X86JITCore::JCC> X86JITCore::G
case FEXCore::IR::COND_VS:
case FEXCore::IR::COND_VC:
default:
LOGMAN_MSG_A("Unsupported compare type");
LogMan::Msg::A("Unsupported compare type");
break;
}
@@ -571,11 +570,10 @@ std::tuple<X86JITCore::SetCC, X86JITCore::CMovCC, X86JITCore::JCC> X86JITCore::G
return { &CodeGenerator::sete , &CodeGenerator::cmove , &CodeGenerator::je };
}
void *X86JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IRListView const *IR, [[maybe_unused]] FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) {
void *X86JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const *IR, [[maybe_unused]] FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) {
JumpTargets.clear();
uint32_t SSACount = IR->GetSSACount();
this->Entry = Entry;
this->RAData = RAData;
// Fairly excessive buffer range to make sure we don't overflow
@@ -584,7 +582,7 @@ void *X86JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IRLi
ThreadState->CTX->ClearCodeCache(ThreadState, false);
}
void *GuestEntry = getCurr<void*>();
void *Entry = getCurr<void*>();
this->IR = IR;
if (CTX->GetGdbServerStatus()) {
@@ -599,14 +597,14 @@ void *X86JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IRLi
cmp(dword [rax + (offsetof(FEXCore::Context::Context, Config.RunningMode))], 0);
je(RunBlock);
// Else we need to pause now
mov(rax, ThreadSharedData.Dispatcher->ThreadPauseHandlerAddress);
mov(rax, Dispatcher->ThreadPauseHandlerAddress);
jmp(rax);
ud2();
L(RunBlock);
}
LOGMAN_THROW_A(RAData != nullptr, "Needs RA");
LogMan::Throw::A(RAData != nullptr, "Needs RA");
SpillSlots = RAData->SpillSlots();
@@ -615,7 +613,7 @@ void *X86JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IRLi
}
#ifdef BLOCKSTATS
BlockSamplingData::BlockData *SamplingData = CTX->BlockData->GetBlockData(Entry);
BlockSamplingData::BlockData *SamplingData = CTX->BlockData->GetBlockData(HeaderOp->Entry);
if (GetSamplingData) {
mov(rcx, reinterpret_cast<uintptr_t>(SamplingData));
rdtsc();
@@ -663,7 +661,7 @@ void *X86JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IRLi
using namespace FEXCore::IR;
{
auto BlockIROp = BlockHeader->CW<IROp_CodeBlock>();
LOGMAN_THROW_A(BlockIROp->Header.Op == IR::OP_CODEBLOCK, "IR type failed to be a code block");
LogMan::Throw::A(BlockIROp->Header.Op == IR::OP_CODEBLOCK, "IR type failed to be a code block");
uint32_t Node = IR->GetID(BlockNode);
auto IsTarget = JumpTargets.find(Node);
@@ -732,15 +730,15 @@ void *X86JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IRLi
}
PendingTargetLabel = nullptr;
void *GuestExit = getCurr<void*>();
void *Exit = getCurr<void*>();
this->IR = nullptr;
ready();
if (DebugData) {
DebugData->HostCodeSize = reinterpret_cast<uintptr_t>(GuestExit) - reinterpret_cast<uintptr_t>(GuestEntry);
DebugData->HostCodeSize = reinterpret_cast<uintptr_t>(Exit) - reinterpret_cast<uintptr_t>(Entry);
}
return GuestEntry;
return Entry;
}
uint64_t X86JITCore::ExitFunctionLink(X86JITCore *core, FEXCore::Core::CpuStateFrame *Frame, uint64_t *record) {
@@ -751,10 +749,10 @@ uint64_t X86JITCore::ExitFunctionLink(X86JITCore *core, FEXCore::Core::CpuStateF
if (!HostCode) {
Thread->CurrentFrame->State.rip = GuestRip;
return core->ThreadSharedData.Dispatcher->AbsoluteLoopTopAddress;
return core->Dispatcher->AbsoluteLoopTopAddress;
}
auto LinkerAddress = core->ThreadSharedData.Dispatcher->ExitFunctionLinkerAddress;
auto LinkerAddress = core->Dispatcher->ExitFunctionLinkerAddress;
Thread->LookupCache->AddBlockLink(GuestRip, (uintptr_t)record, [record, LinkerAddress]{
// undo the link
record[0] = LinkerAddress;
@@ -764,7 +762,7 @@ uint64_t X86JITCore::ExitFunctionLink(X86JITCore *core, FEXCore::Core::CpuStateF
return HostCode;
}
std::unique_ptr<CPUBackend> CreateX86JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, bool CompileThread) {
return std::make_unique<X86JITCore>(ctx, Thread, AllocateNewCodeBuffer(CompileThread ? X86JITCore::MAX_CODE_SIZE : X86JITCore::INITIAL_CODE_SIZE), CompileThread);
FEXCore::CPU::CPUBackend *CreateX86JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, bool CompileThread) {
return new X86JITCore(ctx, Thread, AllocateNewCodeBuffer(CompileThread ? X86JITCore::MAX_CODE_SIZE : X86JITCore::INITIAL_CODE_SIZE), CompileThread);
}
}
+18 -24
View File
@@ -66,7 +66,7 @@ public:
explicit X86JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, CodeBuffer Buffer, bool CompileThread);
~X86JITCore() override;
std::string GetName() override { return "JIT"; }
void *CompileCode(uint64_t Entry, FEXCore::IR::IRListView const *IR, FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) override;
void *CompileCode(FEXCore::IR::IRListView const *IR, FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) override;
void *MapRegion(void* HostPtr, uint64_t, uint64_t) override { return HostPtr; }
@@ -83,8 +83,7 @@ private:
FEXCore::Context::Context *CTX;
FEXCore::Core::InternalThreadState *ThreadState;
FEXCore::IR::IRListView const *IR;
std::unique_ptr<FEXCore::CPU::Dispatcher> Dispatcher;
uint64_t Entry;
FEXCore::CPU::Dispatcher *Dispatcher;
std::unordered_map<IR::OrderedNodeWrapper::NodeOffsetType, Label> JumpTargets;
Xbyak::util::Cpu Features{};
@@ -112,26 +111,26 @@ private:
constexpr static uint8_t RA_64 = 3;
constexpr static uint8_t RA_XMM = 4;
IR::PhysicalRegister GetPhys(uint32_t Node) const;
IR::PhysicalRegister GetPhys(uint32_t Node);
bool IsFPR(uint32_t Node) const;
bool IsGPR(uint32_t Node) const;
bool IsFPR(uint32_t Node);
bool IsGPR(uint32_t Node);
template<uint8_t RAType>
Xbyak::Reg GetSrc(uint32_t Node) const;
Xbyak::Reg GetSrc(uint32_t Node);
template<uint8_t RAType>
std::pair<Xbyak::Reg, Xbyak::Reg> GetSrcPair(uint32_t Node) const;
std::pair<Xbyak::Reg, Xbyak::Reg> GetSrcPair(uint32_t Node);
template<uint8_t RAType>
Xbyak::Reg GetDst(uint32_t Node) const;
Xbyak::Reg GetDst(uint32_t Node);
Xbyak::Xmm GetSrc(uint32_t Node) const;
Xbyak::Xmm GetDst(uint32_t Node) const;
Xbyak::Xmm GetSrc(uint32_t Node);
Xbyak::Xmm GetDst(uint32_t Node);
Xbyak::RegExp GenerateModRM(Xbyak::Reg Base, IR::OrderedNodeWrapper Offset, IR::MemOffsetType OffsetType, uint8_t OffsetScale) const;
Xbyak::RegExp GenerateModRM(Xbyak::Reg Base, IR::OrderedNodeWrapper Offset, IR::MemOffsetType OffsetType, uint8_t OffsetScale);
bool IsInlineConstant(const IR::OrderedNodeWrapper& Node, uint64_t* Value = nullptr) const;
bool IsInlineEntrypointOffset(const IR::OrderedNodeWrapper& WNode, uint64_t* Value) const;
bool IsInlineConstant(const IR::OrderedNodeWrapper& Node, uint64_t* Value = nullptr);
bool IsInlineEntrypointOffset(const IR::OrderedNodeWrapper& WNode, uint64_t* Value);
IR::RegisterAllocationPass *RAPass;
FEXCore::IR::RegisterAllocationData *RAData;
@@ -162,7 +161,6 @@ private:
uint64_t SignalHandlerReturnAddress{};
uint32_t *SignalHandlerRefCounterPtr{};
FEXCore::CPU::Dispatcher *Dispatcher{};
};
CompilerSharedData ThreadSharedData;
@@ -241,7 +239,9 @@ private:
DEF_OP(Sbfe);
DEF_OP(Select);
DEF_OP(VExtractToGPR);
DEF_OP(Float_ToGPR_ZU);
DEF_OP(Float_ToGPR_ZS);
DEF_OP(Float_ToGPR_U);
DEF_OP(Float_ToGPR_S);
DEF_OP(FCmp);
DEF_OP(F80Cmp);
@@ -279,14 +279,16 @@ private:
///< Conversion ops
DEF_OP(VInsGPR);
DEF_OP(VCastFromGPR);
DEF_OP(Float_FromGPR_U);
DEF_OP(Float_FromGPR_S);
DEF_OP(Float_FToF);
DEF_OP(Vector_UToF);
DEF_OP(Vector_SToF);
DEF_OP(Vector_FToZU);
DEF_OP(Vector_FToZS);
DEF_OP(Vector_FToU);
DEF_OP(Vector_FToS);
DEF_OP(Vector_FToF);
DEF_OP(Vector_FToI);
///< Flag ops
DEF_OP(GetHostFlag);
@@ -304,7 +306,6 @@ private:
DEF_OP(StoreMem);
DEF_OP(VLoadMemElement);
DEF_OP(VStoreMemElement);
DEF_OP(CacheLineClear);
///< Misc ops
DEF_OP(EndBlock);
@@ -329,7 +330,6 @@ private:
DEF_OP(SplatVector);
DEF_OP(VMov);
DEF_OP(VAnd);
DEF_OP(VBic);
DEF_OP(VOr);
DEF_OP(VXor);
DEF_OP(VAdd);
@@ -340,10 +340,8 @@ private:
DEF_OP(VSQSub);
DEF_OP(VAddP);
DEF_OP(VAddV);
DEF_OP(VUMinV);
DEF_OP(VURAvg);
DEF_OP(VAbs);
DEF_OP(VPopcount);
DEF_OP(VFAdd);
DEF_OP(VFAddP);
DEF_OP(VFSub);
@@ -363,8 +361,6 @@ private:
DEF_OP(VSMax);
DEF_OP(VZip);
DEF_OP(VZip2);
DEF_OP(VUnZip);
DEF_OP(VUnZip2);
DEF_OP(VBSL);
DEF_OP(VCMPEQ);
DEF_OP(VCMPEQZ);
@@ -387,7 +383,6 @@ private:
DEF_OP(VInsElement);
DEF_OP(VInsScalarElement);
DEF_OP(VExtractElement);
DEF_OP(VDupElement);
DEF_OP(VExtr);
DEF_OP(VSLI);
DEF_OP(VSRI);
@@ -410,7 +405,6 @@ private:
DEF_OP(VSMull);
DEF_OP(VUMull2);
DEF_OP(VSMull2);
DEF_OP(VUABDL);
DEF_OP(VTBL1);
///< Encryption ops
@@ -36,10 +36,10 @@ DEF_OP(LoadContext) {
}
break;
case 16: {
LOGMAN_MSG_A("Invalid GPR load of size 16");
LogMan::Msg::A("Invalid GPR load of size 16");
}
break;
default: LOGMAN_MSG_A("Unhandled LoadContext size: %d", OpSize);
default: LogMan::Msg::A("Unhandled LoadContext size: %d", OpSize);
}
}
else {
@@ -69,7 +69,7 @@ DEF_OP(LoadContext) {
movups(GetDst(Node), xword [STATE + Op->Offset]);
}
break;
default: LOGMAN_MSG_A("Unhandled LoadContext size: %d", OpSize);
default: LogMan::Msg::A("Unhandled LoadContext size: %d", OpSize);
}
}
}
@@ -100,7 +100,7 @@ DEF_OP(StoreContext) {
case 16:
LogMan::Msg::D("Invalid store size of 16");
break;
default: LOGMAN_MSG_A("Unhandled StoreContext size: %d", OpSize);
default: LogMan::Msg::A("Unhandled StoreContext size: %d", OpSize);
}
}
else {
@@ -129,7 +129,7 @@ DEF_OP(StoreContext) {
movups(xword [STATE + Op->Offset], GetSrc(Op->Header.Args[0].ID()));
}
break;
default: LOGMAN_MSG_A("Unhandled StoreContext size: %d", OpSize);
default: LogMan::Msg::A("Unhandled StoreContext size: %d", OpSize);
}
}
}
@@ -160,15 +160,15 @@ DEF_OP(LoadContextIndexed) {
mov(GetDst<RA_64>(Node), qword [rax + index * Op->Stride]);
break;
default:
LOGMAN_MSG_A("Unhandled LoadContextIndexed size: %d", Op->Size);
LogMan::Msg::A("Unhandled LoadContextIndexed size: %d", Op->Size);
}
break;
}
case 16:
LOGMAN_MSG_A("Invalid Class load of size 16");
LogMan::Msg::A("Invalid Class load of size 16");
break;
default:
LOGMAN_MSG_A("Unhandled LoadContextIndexed stride: %d", Op->Stride);
LogMan::Msg::A("Unhandled LoadContextIndexed stride: %d", Op->Stride);
}
}
@@ -195,7 +195,7 @@ DEF_OP(LoadContextIndexed) {
vmovq(GetDst(Node), qword [rax + index * Op->Stride]);
break;
default:
LOGMAN_MSG_A("Unhandled LoadContextIndexed size: %d", Op->Size);
LogMan::Msg::A("Unhandled LoadContextIndexed size: %d", Op->Size);
}
break;
}
@@ -223,12 +223,12 @@ DEF_OP(LoadContextIndexed) {
movups(GetDst(Node), xword [STATE + rax]);
break;
default:
LOGMAN_MSG_A("Unhandled LoadContextIndexed size: %d", Op->Size);
LogMan::Msg::A("Unhandled LoadContextIndexed size: %d", Op->Size);
}
break;
}
default:
LOGMAN_MSG_A("Unhandled LoadContextIndexed stride: %d", Op->Stride);
LogMan::Msg::A("Unhandled LoadContextIndexed stride: %d", Op->Stride);
}
}
}
@@ -248,13 +248,13 @@ DEF_OP(StoreContextIndexed) {
case 4:
case 8: {
if (!(size == 1 || size == 2 || size == 4 || size == 8)) {
LOGMAN_MSG_A("Unhandled StoreContextIndexed size: %d", Op->Size);
LogMan::Msg::A("Unhandled StoreContextIndexed size: %d", Op->Size);
}
mov(AddressFrame(Op->Size * 8) [rax + index * Op->Stride], value);
break;
}
default:
LOGMAN_MSG_A("Unhandled StoreContextIndexed stride: %d", Op->Stride);
LogMan::Msg::A("Unhandled StoreContextIndexed stride: %d", Op->Stride);
}
}
else {
@@ -279,7 +279,7 @@ DEF_OP(StoreContextIndexed) {
vmovq(AddressFrame(Op->Size * 8) [rax + index * Op->Stride], value);
break;
default:
LOGMAN_MSG_A("Unhandled StoreContextIndexed size: %d", size);
LogMan::Msg::A("Unhandled StoreContextIndexed size: %d", size);
}
break;
}
@@ -307,12 +307,12 @@ DEF_OP(StoreContextIndexed) {
movups(xword [STATE + rax], value);
break;
default:
LOGMAN_MSG_A("Unhandled StoreContextIndexed size: %d", size);
LogMan::Msg::A("Unhandled StoreContextIndexed size: %d", size);
}
break;
}
default:
LOGMAN_MSG_A("Unhandled StoreContextIndexed stride: %d", Op->Stride);
LogMan::Msg::A("Unhandled StoreContextIndexed stride: %d", Op->Stride);
}
}
}
@@ -340,7 +340,7 @@ DEF_OP(SpillRegister) {
mov(qword [rsp + SlotOffset], GetSrc<RA_64>(Op->Header.Args[0].ID()));
break;
}
default: LOGMAN_MSG_A("Unhandled SpillRegister size: %d", OpSize);
default: LogMan::Msg::A("Unhandled SpillRegister size: %d", OpSize);
}
} else if (Op->Class == FEXCore::IR::FPRClass) {
switch (OpSize) {
@@ -356,10 +356,10 @@ DEF_OP(SpillRegister) {
movaps(xword [rsp + SlotOffset], GetSrc(Op->Header.Args[0].ID()));
break;
}
default: LOGMAN_MSG_A("Unhandled SpillRegister size: %d", OpSize);
default: LogMan::Msg::A("Unhandled SpillRegister size: %d", OpSize);
}
} else {
LOGMAN_MSG_A("Unhandled SpillRegister class: %d", Op->Class.Val);
LogMan::Msg::A("Unhandled SpillRegister class: %d", Op->Class.Val);
}
@@ -388,7 +388,7 @@ DEF_OP(FillRegister) {
mov(GetDst<RA_64>(Node), qword [rsp + SlotOffset]);
break;
}
default: LOGMAN_MSG_A("Unhandled FillRegister size: %d", OpSize);
default: LogMan::Msg::A("Unhandled FillRegister size: %d", OpSize);
}
} else if (Op->Class == FEXCore::IR::FPRClass) {
switch (OpSize) {
@@ -404,10 +404,10 @@ DEF_OP(FillRegister) {
movaps(GetDst(Node), xword [rsp + SlotOffset]);
break;
}
default: LOGMAN_MSG_A("Unhandled FillRegister size: %d", OpSize);
default: LogMan::Msg::A("Unhandled FillRegister size: %d", OpSize);
}
} else {
LOGMAN_MSG_A("Unhandled FillRegister class: %d", Op->Class.Val);
LogMan::Msg::A("Unhandled FillRegister class: %d", Op->Class.Val);
}
}
@@ -425,16 +425,16 @@ DEF_OP(StoreFlag) {
mov(byte [STATE + (offsetof(FEXCore::Core::CPUState, flags[0]) + Op->Flag)], al);
}
Xbyak::RegExp X86JITCore::GenerateModRM(Xbyak::Reg Base, IR::OrderedNodeWrapper Offset, IR::MemOffsetType OffsetType, uint8_t OffsetScale) const {
Xbyak::RegExp X86JITCore::GenerateModRM(Xbyak::Reg Base, IR::OrderedNodeWrapper Offset, IR::MemOffsetType OffsetType, uint8_t OffsetScale) {
if (Offset.IsInvalid()) {
return Base;
} else {
if (OffsetScale != 1 && OffsetScale != 2 && OffsetScale != 4 && OffsetScale != 8) {
LOGMAN_MSG_A("Unhandled GenerateModRM OffsetScale: %d", OffsetScale);
LogMan::Msg::A("Unhandled GenerateModRM OffsetScale: %d", OffsetScale);
}
if (OffsetType != IR::MEM_OFFSET_SXTX) {
LOGMAN_MSG_A("Unhandled GenerateModRM OffsetType: %d", OffsetType.Val);
LogMan::Msg::A("Unhandled GenerateModRM OffsetType: %d", OffsetType.Val);
}
uint64_t Const;
@@ -475,7 +475,7 @@ DEF_OP(LoadMem) {
mov(Dst, qword [MemPtr]);
}
break;
default: LOGMAN_MSG_A("Unhandled LoadMem size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled LoadMem size: %d", Op->Size);
}
}
else
@@ -511,7 +511,7 @@ DEF_OP(LoadMem) {
}
}
break;
default: LOGMAN_MSG_A("Unhandled LoadMem size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled LoadMem size: %d", Op->Size);
}
}
}
@@ -537,7 +537,7 @@ DEF_OP(StoreMem) {
case 8:
mov(qword [MemPtr], GetSrc<RA_64>(Op->Header.Args[1].ID()));
break;
default: LOGMAN_MSG_A("Unhandled StoreMem size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled StoreMem size: %d", Op->Size);
}
}
else {
@@ -560,25 +560,17 @@ DEF_OP(StoreMem) {
else
movups(xword [MemPtr], GetSrc(Op->Header.Args[1].ID()));
break;
default: LOGMAN_MSG_A("Unhandled StoreMem size: %d", Op->Size);
default: LogMan::Msg::A("Unhandled StoreMem size: %d", Op->Size);
}
}
}
DEF_OP(VLoadMemElement) {
LOGMAN_MSG_A("Unimplemented");
LogMan::Msg::A("Unimplemented");
}
DEF_OP(VStoreMemElement) {
LOGMAN_MSG_A("Unimplemented");
}
DEF_OP(CacheLineClear) {
auto Op = IROp->C<IR::IROp_CacheLineClear>();
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Addr.ID());
clflush(ptr [MemReg]);
LogMan::Msg::A("Unimplemented");
}
#undef DEF_OP
@@ -600,7 +592,6 @@ void X86JITCore::RegisterMemoryHandlers() {
REGISTER_OP(STOREMEMTSO, StoreMem);
REGISTER_OP(VLOADMEMELEMENT, VLoadMemElement);
REGISTER_OP(VSTOREMEMELEMENT, VStoreMemElement);
REGISTER_OP(CACHELINECLEAR, CacheLineClear);
#undef REGISTER_OP
}
}
+18 -29
View File
@@ -12,10 +12,6 @@ static void PrintValue(uint64_t Value) {
LogMan::Msg::D("Value: 0x%lx", Value);
}
static void PrintVectorValue(uint64_t Value, uint64_t ValueUpper) {
LogMan::Msg::D("Value: 0x%016lx'%016lx", ValueUpper, Value);
}
#define DEF_OP(x) void X86JITCore::Op_##x(FEXCore::IR::IROp_Header *IROp, uint32_t Node)
DEF_OP(Fence) {
@@ -30,7 +26,7 @@ DEF_OP(Fence) {
case IR::Fence_Store.Val:
sfence();
break;
default: LOGMAN_MSG_A("Unknown Fence: %d", Op->Fence); break;
default: LogMan::Msg::A("Unknown Fence: %d", Op->Fence); break;
}
}
@@ -41,22 +37,13 @@ DEF_OP(Break) {
case 5: // Guest ud2
ud2();
break;
case 1: // Int <imm8>
ud2();
break;
case 2: // overflow
ud2();
break;
case 3: // int 1
ud2();
break;
case 4: { // HLT
// Time to quit
// Set our stack to the starting stack location
mov(rsp, qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, ReturningStackLocation)]);
// Now we need to jump to the thread stop handler
mov(TMP1, ThreadSharedData.Dispatcher->ThreadStopHandlerAddress);
mov(TMP1, Dispatcher->ThreadStopHandlerAddress);
jmp(TMP1);
break;
}
@@ -69,7 +56,7 @@ DEF_OP(Break) {
}
// This jump target needs to be a constant offset here
mov(TMP1, ThreadSharedData.Dispatcher->ThreadPauseHandlerAddress);
mov(TMP1, Dispatcher->ThreadPauseHandlerAddress);
jmp(TMP1);
}
else {
@@ -78,12 +65,12 @@ DEF_OP(Break) {
mov(rsp, qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, ReturningStackLocation)]);
// Now we need to jump to the thread stop handler
mov(TMP1, ThreadSharedData.Dispatcher->ThreadStopHandlerAddress);
mov(TMP1, Dispatcher->ThreadStopHandlerAddress);
jmp(TMP1);
}
break;
}
default: LOGMAN_MSG_A("Unknown Break reason: %d", Op->Reason);
default: LogMan::Msg::A("Unknown Break reason: %d", Op->Reason);
}
}
@@ -123,22 +110,24 @@ DEF_OP(SetRoundingMode) {
DEF_OP(Print) {
auto Op = IROp->C<IR::IROp_Print>();
PushRegs();
if (IsGPR(Op->Header.Args[0].ID())) {
mov (rdi, GetSrc<RA_64>(Op->Header.Args[0].ID()));
for (auto &Reg : RA64)
push(Reg);
mov(rax, reinterpret_cast<uintptr_t>(PrintValue));
}
else {
pextrq(rdi, GetSrc(Op->Header.Args[0].ID()), 0);
pextrq(rsi, GetSrc(Op->Header.Args[0].ID()), 1);
auto NumPush = RA64.size();
if (NumPush & 1)
sub(rsp, 8); // Align
mov(rax, reinterpret_cast<uintptr_t>(PrintVectorValue));
}
mov (rdi, GetSrc<RA_64>(Op->Header.Args[0].ID()));
mov(rax, reinterpret_cast<uintptr_t>(PrintValue));
call(rax);
PopRegs();
if (NumPush & 1)
add(rsp, 8); // Align
for (uint32_t i = RA64.size(); i > 0; --i)
pop(RA64[i - 1]);
}
#undef DEF_OP
@@ -25,7 +25,7 @@ DEF_OP(ExtractElementPair) {
mov (GetDst<RA_64>(Node), Regs[Op->Element]);
break;
}
default: LOGMAN_MSG_A("Unknown Size"); break;
default: LogMan::Msg::A("Unknown Size"); break;
}
}
@@ -51,7 +51,7 @@ DEF_OP(CreateElementPair) {
RegTmp = rax;
break;
}
default: LOGMAN_MSG_A("Unknown Size"); break;
default: LogMan::Msg::A("Unknown Size"); break;
}
if (Dst.first != RegSecond) {
File diff suppressed because it is too large. Load diff
+8 -10
View File
@@ -8,8 +8,6 @@ $end_info$
#include "Interface/Context/Context.h"
#include "Interface/Core/Core.h"
#include "Interface/Core/LookupCache.h"
#include <FEXCore/Utils/Allocator.h>
#include <sys/mman.h>
namespace FEXCore {
@@ -28,27 +26,27 @@ LookupCache::LookupCache(FEXCore::Context::Context *CTX)
// Allocate a region of memory that we can use to back our block pointers
// We need one pointer per page of virtual memory
// At 64GB of virtual memory this will allocate 128MB of virtual memory space
PagePointer = reinterpret_cast<uintptr_t>(FEXCore::Allocator::mmap(nullptr, ctx->Config.VirtualMemSize / 4096 * 8, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0));
PagePointer = reinterpret_cast<uintptr_t>(mmap(nullptr, ctx->Config.VirtualMemSize / 4096 * 8, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0));
// Allocate our memory backing our pages
// We need 32KB per guest page (One pointer per byte)
// XXX: We can drop down to 16KB if we store 4byte offsets from the code base
// We currently limit to 128MB of real memory for caching for the total cache size.
// Can end up being inefficient if we compile a small number of blocks per page
PageMemory = reinterpret_cast<uintptr_t>(FEXCore::Allocator::mmap(nullptr, CODE_SIZE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0));
LOGMAN_THROW_A(PageMemory != -1ULL, "Failed to allocate page memory");
PageMemory = reinterpret_cast<uintptr_t>(mmap(nullptr, CODE_SIZE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0));
LogMan::Throw::A(PageMemory != -1ULL, "Failed to allocate page memory");
// L1 Cache
L1Pointer = reinterpret_cast<uintptr_t>(FEXCore::Allocator::mmap(nullptr, L1_SIZE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0));
LOGMAN_THROW_A(L1Pointer != -1ULL, "Failed to allocate L1Pointer");
L1Pointer = reinterpret_cast<uintptr_t>(mmap(nullptr, L1_SIZE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0));
LogMan::Throw::A(L1Pointer != -1ULL, "Failed to allocate L1Pointer");
VirtualMemSize = ctx->Config.VirtualMemSize;
}
LookupCache::~LookupCache() {
FEXCore::Allocator::munmap(reinterpret_cast<void*>(PagePointer), ctx->Config.VirtualMemSize / 4096 * 8);
FEXCore::Allocator::munmap(reinterpret_cast<void*>(PageMemory), CODE_SIZE);
FEXCore::Allocator::munmap(reinterpret_cast<void*>(L1Pointer), L1_SIZE);
munmap(reinterpret_cast<void*>(PagePointer), ctx->Config.VirtualMemSize / 4096 * 8);
munmap(reinterpret_cast<void*>(PageMemory), CODE_SIZE);
munmap(reinterpret_cast<void*>(L1Pointer), L1_SIZE);
}
void LookupCache::HintUsedRange(uint64_t Address, uint64_t Size) {
+7 -10
View File
@@ -39,17 +39,13 @@ public:
void AddBlockMapping(uint64_t Address, void *HostCode, uint64_t Start, uint64_t Length) {
auto InsertPoint = BlockList.emplace(Address, (uintptr_t)HostCode);
LOGMAN_THROW_A(InsertPoint.second == true, "Dupplicate block mapping added");
LogMan::Throw::A(InsertPoint.second == true, "Dupplicate block mapping added");
for (auto CurrentPage = Start >> 12, EndPage = (Start + Length) >> 12; CurrentPage <= EndPage; CurrentPage++) {
CodePages[CurrentPage].push_back(Address);
}
// There is no need to update L1 or L2, they will get updated on first lookup
// However, adding to L1 here increases performance
auto &L1Entry = reinterpret_cast<LookupCacheEntry*>(L1Pointer)[Address & L1_ENTRIES_MASK];
L1Entry.GuestCode = Address;
L1Entry.HostCode = (uintptr_t)HostCode;
// no need to update L1 or L2, they will get updated on first lookup
}
void Erase(uint64_t Address) {
@@ -98,8 +94,8 @@ public:
void HintUsedRange(uint64_t Address, uint64_t Size);
uintptr_t GetL1Pointer() const { return L1Pointer; }
uintptr_t GetPagePointer() const { return PagePointer; }
uintptr_t GetL1Pointer() { return L1Pointer; }
uintptr_t GetPagePointer() { return PagePointer; }
uintptr_t GetVirtualMemorySize() const { return VirtualMemSize; }
constexpr static size_t L1_ENTRIES = 1 * 1024 * 1024; // Must be a power of 2
@@ -109,8 +105,9 @@ private:
void CacheBlockMapping(uint64_t Address, uintptr_t HostCode) {
// Do L1
auto &L1Entry = reinterpret_cast<LookupCacheEntry*>(L1Pointer)[Address & L1_ENTRIES_MASK];
L1Entry.GuestCode = Address;
L1Entry.HostCode = HostCode;
if (L1Entry.GuestCode == Address) {
L1Entry.GuestCode = L1Entry.HostCode = 0;
}
// Do ful map
auto FullAddress = Address;
File diff suppressed because it is too large. Load diff
+32 -38
View File
@@ -49,7 +49,7 @@ public:
OrderedNode* GetNewJumpBlock(uint64_t RIP) {
auto it = JumpTargets.find(RIP);
LOGMAN_THROW_A(it != JumpTargets.end(), "Couldn't find block generated for 0x%lx", RIP);
LogMan::Throw::A(it != JumpTargets.end(), "Couldn't find block generated for 0x%lx", RIP);
return it->second.BlockEntry;
}
@@ -59,7 +59,7 @@ public:
it->second.HaveEmitted = true;
if (CurrentCodeBlock->Wrapped(DualListData.ListBegin()).ID() == it->second.BlockEntry->Wrapped(DualListData.ListBegin()).ID()) return;
if (CurrentCodeBlock->Wrapped(ListData.Begin()).ID() == it->second.BlockEntry->Wrapped(ListData.Begin()).ID()) return;
// We have hit a RIP that is a jump target
// Thus we need to end up in a new block
@@ -81,14 +81,14 @@ public:
// rdi, 0x8
// cmp qword [rdi-8], 0
// jne .label
if (LastOp && !BlockSetRIP) {
if (!BlockSetRIP) {
auto it = JumpTargets.find(NextRIP);
if (it == JumpTargets.end()) {
if (it == JumpTargets.end() && LastOp) {
uint8_t GPRSize = CTX->Config.Is64BitMode ? 8 : 4;
// If we don't have a jump target to a new block then we have to leave
// Set the RIP to the next instruction and leave
auto RelocatedNextRIP = _EntrypointOffset(NextRIP - Entry, GPRSize);
auto RelocatedNextRIP = _EntrypointOffset(NextRIP - Current_Header->Entry, GPRSize);
_ExitFunction(RelocatedNextRIP);
}
else if (it != JumpTargets.end()) {
@@ -104,8 +104,7 @@ public:
OpDispatchBuilder(FEXCore::Context::Context *ctx);
void ResetWorkingList();
void ResetDecodeFailure() { DecodeFailure = false; }
bool HadDecodeFailure() const { return DecodeFailure; }
bool HadDecodeFailure() { return DecodeFailure; }
void BeginFunction(uint64_t RIP, std::vector<FEXCore::Frontend::Decoder::DecodedBlocks> const *Blocks);
void Finalize();
@@ -260,6 +259,12 @@ public:
template<size_t ElementSize>
void PSUBQOp(OpcodeArgs);
template<size_t ElementSize>
void PMINUOp(OpcodeArgs);
template<size_t ElementSize>
void PMAXUOp(OpcodeArgs);
void PMINSWOp(OpcodeArgs);
void PMAXSWOp(OpcodeArgs);
template<size_t ElementSize>
void MOVMSKOp(OpcodeArgs);
void MOVMSKOpOne(OpcodeArgs);
template<size_t ElementSize>
@@ -269,6 +274,10 @@ public:
void PSHUFBOp(OpcodeArgs);
template<size_t ElementSize, bool HalfSize, bool Low>
void PSHUFDOp(OpcodeArgs);
template<size_t ElementSize>
void PCMPEQOp(OpcodeArgs);
template<size_t ElementSize>
void PCMPGTOp(OpcodeArgs);
void MOVDOp(OpcodeArgs);
template<size_t ElementSize, bool Scalar, uint32_t SrcIndex>
void PSRLDOp(OpcodeArgs);
@@ -287,21 +296,21 @@ public:
template<size_t ElementSize>
void PAVGOp(OpcodeArgs);
void MOVDDUPOp(OpcodeArgs);
template<size_t DstElementSize>
template<size_t DstElementSize, bool Signed>
void CVTGPR_To_FPR(OpcodeArgs);
template<size_t SrcElementSize, bool HostRoundingMode>
template<size_t SrcElementSize, bool Signed, bool HostRoundingMode>
void CVTFPR_To_GPR(OpcodeArgs);
template<size_t SrcElementSize, bool Widen>
template<size_t SrcElementSize, bool Signed, bool Widen>
void Vector_CVT_Int_To_Float(OpcodeArgs);
template<size_t DstElementSize, size_t SrcElementSize>
void Scalar_CVT_Float_To_Float(OpcodeArgs);
template<size_t DstElementSize, size_t SrcElementSize>
void Vector_CVT_Float_To_Float(OpcodeArgs);
template<size_t SrcElementSize, bool Narrow, bool HostRoundingMode>
template<size_t SrcElementSize, bool Signed, bool Narrow, bool HostRoundingMode>
void Vector_CVT_Float_To_Int(OpcodeArgs);
template<size_t SrcElementSize, bool Signed, bool Widen>
void MMX_To_XMM_Vector_CVT_Int_To_Float(OpcodeArgs);
template<size_t SrcElementSize, bool Narrow, bool HostRoundingMode>
template<size_t SrcElementSize, bool Signed, bool Narrow, bool HostRoundingMode>
void XMM_To_MMX_Vector_CVT_Float_To_Int(OpcodeArgs);
void MASKMOVOp(OpcodeArgs);
void MOVBetweenGPR_FPR(OpcodeArgs);
@@ -315,13 +324,17 @@ public:
void ANDNOp(OpcodeArgs);
template<size_t ElementSize>
void PINSROp(OpcodeArgs);
void InsertPSOp(OpcodeArgs);
template<size_t ElementSize>
void PExtrOp(OpcodeArgs);
template<size_t ElementSize, bool Signed>
void PMULOp(OpcodeArgs);
template<size_t ElementSize>
void PSIGN(OpcodeArgs);
template<size_t ElementSize>
void PABS(OpcodeArgs);
// X87 Ops
template<size_t width>
void FLD(OpcodeArgs);
@@ -445,8 +458,6 @@ public:
template<uint8_t FenceType>
void FenceOp(OpcodeArgs);
void StoreFenceOrCLFlush(OpcodeArgs);
void PSADBW(OpcodeArgs);
void AESImcOp(OpcodeArgs);
@@ -456,23 +467,6 @@ public:
void AESDecLastOp(OpcodeArgs);
void AESKeyGenAssist(OpcodeArgs);
template<size_t ElementSize, size_t DstElementSize, bool Signed>
void ExtendVectorElements(OpcodeArgs);
template<size_t ElementSize, bool Scalar>
void VectorRound(OpcodeArgs);
template<size_t ElementSize>
void VectorBlend(OpcodeArgs);
template<size_t ElementSize>
void VectorVariableBlend(OpcodeArgs);
void PTestOp(OpcodeArgs);
void PHMINPOSUWOp(OpcodeArgs);
template<size_t ElementSize>
void DPPOp(OpcodeArgs);
void MPSADBWOp(OpcodeArgs);
void UnimplementedOp(OpcodeArgs);
#undef OpcodeArgs
@@ -497,8 +491,8 @@ private:
void StoreResult(FEXCore::IR::RegisterClassType Class, FEXCore::X86Tables::DecodedOp Op, FEXCore::X86Tables::DecodedOperand const& Operand, OrderedNode *const Src, int8_t Align);
void StoreResult(FEXCore::IR::RegisterClassType Class, FEXCore::X86Tables::DecodedOp Op, OrderedNode *const Src, int8_t Align);
uint8_t GetDstSize(FEXCore::X86Tables::DecodedOp Op) const;
uint8_t GetSrcSize(FEXCore::X86Tables::DecodedOp Op) const;
uint8_t GetDstSize(FEXCore::X86Tables::DecodedOp Op);
uint8_t GetSrcSize(FEXCore::X86Tables::DecodedOp Op);
template<unsigned BitOffset>
void SetRFLAG(OrderedNode *Value);
@@ -528,12 +522,12 @@ private:
OrderedNode * GetX87Top();
void SetX87Top(OrderedNode *Value);
bool DestIsLockedMem(FEXCore::X86Tables::DecodedOp Op) const {
return DestIsMem(Op) && (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_LOCK) != 0;
bool DestIsLockedMem(FEXCore::X86Tables::DecodedOp Op) {
return Op->Dest.TypeNone.Type !=FEXCore::X86Tables::DecodedOperand::TYPE_GPR && (Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_LOCK);
}
bool DestIsMem(FEXCore::X86Tables::DecodedOp Op) const {
return !Op->Dest.IsGPR();
bool DestIsMem(FEXCore::X86Tables::DecodedOp Op) {
return Op->Dest.TypeNone.Type !=FEXCore::X86Tables::DecodedOperand::TYPE_GPR;
}
void CreateJumpBlocks(std::vector<FEXCore::Frontend::Decoder::DecodedBlocks> const *Blocks);
+4 -5
View File
@@ -6,7 +6,6 @@ $end_info$
*/
#include "Interface/Core/X86HelperGen.h"
#include <FEXCore/Utils/Allocator.h>
#include <cstring>
#include <stdlib.h>
@@ -32,7 +31,7 @@ X86GeneratedCode::X86GeneratedCode() {
}
X86GeneratedCode::~X86GeneratedCode() {
FEXCore::Allocator::munmap(CodePtr, CODE_SIZE);
munmap(CodePtr, CODE_SIZE);
}
void* X86GeneratedCode::AllocateGuestCodeSpace(size_t Size) {
@@ -40,7 +39,7 @@ void* X86GeneratedCode::AllocateGuestCodeSpace(size_t Size) {
if (Is64BitMode()) {
// 64bit mode can have its sigret handler anywhere
return FEXCore::Allocator::mmap(nullptr, Size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
return mmap(nullptr, Size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
}
// First 64bit page
@@ -50,14 +49,14 @@ void* X86GeneratedCode::AllocateGuestCodeSpace(size_t Size) {
// We need to have the sigret handler in the lower 32bits of memory space
// Scan top down and try to allocate a location
for (size_t Location = 0xFFFF'E000; Location != 0x0; Location -= 0x1000) {
void *Ptr = FEXCore::Allocator::mmap(reinterpret_cast<void*>(Location), Size, PROT_READ | PROT_WRITE, MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
void *Ptr = mmap(reinterpret_cast<void*>(Location), Size, PROT_READ | PROT_WRITE, MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (Ptr != MAP_FAILED &&
reinterpret_cast<uintptr_t>(Ptr) >= LOCATION_MAX) {
// Failed to map in the lower 32bits
// Try again
// Can happen in the case that host kernel ignores MAP_FIXED_NOREPLACE
FEXCore::Allocator::munmap(Ptr, Size);
munmap(Ptr, Size);
continue;
}
@@ -41,10 +41,10 @@ void InitializeH0F38Tables() {
{OPD(PF_38_NONE, 0x0B), 1, X86InstInfo{"PMULHRSW", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0, nullptr}},
{OPD(PF_38_66, 0x0B), 1, X86InstInfo{"PMULHRSW", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x10), 1, X86InstInfo{"PBLENDVB", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x14), 1, X86InstInfo{"BLENDVPS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x15), 1, X86InstInfo{"BLENDVPD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x17), 1, X86InstInfo{"PTEST", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x10), 1, X86InstInfo{"PBLENDVB", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x14), 1, X86InstInfo{"BLENDVPS", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x15), 1, X86InstInfo{"BLENDVPD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x17), 1, X86InstInfo{"PTEST", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_NONE, 0x1C), 1, X86InstInfo{"PABSB", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0, nullptr}},
{OPD(PF_38_66, 0x1C), 1, X86InstInfo{"PABSB", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_NONE, 0x1D), 1, X86InstInfo{"PABSW", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0, nullptr}},
@@ -52,34 +52,34 @@ void InitializeH0F38Tables() {
{OPD(PF_38_NONE, 0x1E), 1, X86InstInfo{"PABSD", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0, nullptr}},
{OPD(PF_38_66, 0x1E), 1, X86InstInfo{"PABSD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x20), 1, X86InstInfo{"PMOVSXBW", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x21), 1, X86InstInfo{"PMOVSXBD", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x22), 1, X86InstInfo{"PMOVSXBQ", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_16BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x23), 1, X86InstInfo{"PMOVSXWD", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x24), 1, X86InstInfo{"PMOVSXWQ", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x25), 1, X86InstInfo{"PMOVSXDQ", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x28), 1, X86InstInfo{"PMULDQ", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x29), 1, X86InstInfo{"PCMPEQQ", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x20), 1, X86InstInfo{"PMOVSXBW", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x21), 1, X86InstInfo{"PMOVSXBD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x22), 1, X86InstInfo{"PMOVSXBQ", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x23), 1, X86InstInfo{"PMOVSXWD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x24), 1, X86InstInfo{"PMOVSXWQ", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x25), 1, X86InstInfo{"PMOVSXDQ", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x28), 1, X86InstInfo{"PMULDQ", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x29), 1, X86InstInfo{"PCMPEQQ", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x2A), 1, X86InstInfo{"MOVNTDQA", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x2B), 1, X86InstInfo{"PACKUSDW", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x2B), 1, X86InstInfo{"PACKUSDW", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x30), 1, X86InstInfo{"PMOVZXBW", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x31), 1, X86InstInfo{"PMOVZXBD", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x32), 1, X86InstInfo{"PMOVZXBQ", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_16BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x33), 1, X86InstInfo{"PMOVZXWD", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x34), 1, X86InstInfo{"PMOVZXWQ", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x35), 1, X86InstInfo{"PMOVZXDQ", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x38), 1, X86InstInfo{"PMINSB", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x39), 1, X86InstInfo{"PMINSD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x3A), 1, X86InstInfo{"PMINUW", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x3B), 1, X86InstInfo{"PMINUD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x3C), 1, X86InstInfo{"PMAXSB", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x3D), 1, X86InstInfo{"PMAXSD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x3E), 1, X86InstInfo{"PMAXUW", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x3F), 1, X86InstInfo{"PMAXUD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x30), 1, X86InstInfo{"PMOVZXBW", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x31), 1, X86InstInfo{"PMOVZXBD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x32), 1, X86InstInfo{"PMOVZXBQ", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x33), 1, X86InstInfo{"PMOVZXWD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x34), 1, X86InstInfo{"PMOVZXWQ", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x35), 1, X86InstInfo{"PMOVZXDQ", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x38), 1, X86InstInfo{"PMINSB", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x39), 1, X86InstInfo{"PMINSD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x3A), 1, X86InstInfo{"PMINUW", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x3B), 1, X86InstInfo{"PMINUD", TYPE_INST, FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x3C), 1, X86InstInfo{"PMAXSB", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x3D), 1, X86InstInfo{"PMAXSD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x3E), 1, X86InstInfo{"PMAXUW", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x3F), 1, X86InstInfo{"PMAXUD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x40), 1, X86InstInfo{"PMULLD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x41), 1, X86InstInfo{"PHMINPOSUW", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0x40), 1, X86InstInfo{"PMULLD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0x41), 1, X86InstInfo{"PHMINPOSUW", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(PF_38_66, 0xDB), 1, X86InstInfo{"AESIMC", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
{OPD(PF_38_66, 0xDC), 1, X86InstInfo{"AESENC", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0, nullptr}},
@@ -16,26 +16,26 @@ void InitializeH0F3ATables(Context::OperatingMode Mode) {
const U16U8InfoStruct H0F3ATable[] = {
{OPD(0, PF_3A_NONE, 0x0F), 1, X86InstInfo{"PALIGNR", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 1, nullptr}},
{OPD(0, PF_3A_66, 0x08), 1, X86InstInfo{"ROUNDPS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(0, PF_3A_66, 0x09), 1, X86InstInfo{"ROUNDPD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(0, PF_3A_66, 0x0A), 1, X86InstInfo{"ROUNDSS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(0, PF_3A_66, 0x0B), 1, X86InstInfo{"ROUNDSD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(0, PF_3A_66, 0x0C), 1, X86InstInfo{"BLENDPS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(0, PF_3A_66, 0x0D), 1, X86InstInfo{"BLENDPD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(0, PF_3A_66, 0x0E), 1, X86InstInfo{"PBLENDW", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(0, PF_3A_66, 0x08), 1, X86InstInfo{"ROUNDPS", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(0, PF_3A_66, 0x09), 1, X86InstInfo{"ROUNDPD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(0, PF_3A_66, 0x0A), 1, X86InstInfo{"ROUNDSS", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(0, PF_3A_66, 0x0B), 1, X86InstInfo{"ROUNDSD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(0, PF_3A_66, 0x0C), 1, X86InstInfo{"BLENDPS", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(0, PF_3A_66, 0x0D), 1, X86InstInfo{"BLENDPD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(0, PF_3A_66, 0x0E), 1, X86InstInfo{"PBLENDW", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(0, PF_3A_66, 0x0F), 1, X86InstInfo{"PALIGNR", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(0, PF_3A_66, 0x14), 1, X86InstInfo{"PEXTRB", TYPE_INST, GenFlagsSizes(SIZE_8BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_DST_GPR | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(0, PF_3A_66, 0x15), 1, X86InstInfo{"PEXTRW", TYPE_INST, GenFlagsSizes(SIZE_16BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_DST_GPR | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(0, PF_3A_66, 0x16), 1, X86InstInfo{"PEXTRD", TYPE_INST, GenFlagsSizes(SIZE_32BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_DST_GPR | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(0, PF_3A_66, 0x17), 1, X86InstInfo{"EXTRACTPS", TYPE_INST, GenFlagsSizes(SIZE_32BIT, SIZE_128BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_DST_GPR | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(0, PF_3A_66, 0x17), 1, X86InstInfo{"EXTRACTPS", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(0, PF_3A_66, 0x20), 1, X86InstInfo{"PINSRB", TYPE_INST, GenFlagsDstSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_SRC_GPR, 1, nullptr}},
{OPD(0, PF_3A_66, 0x21), 1, X86InstInfo{"INSERTPS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(0, PF_3A_66, 0x20), 1, X86InstInfo{"PINSRB", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_8BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_SRC_GPR, 1, nullptr}},
{OPD(0, PF_3A_66, 0x21), 1, X86InstInfo{"INSERTPS", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(0, PF_3A_66, 0x22), 1, X86InstInfo{"PINSRD", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_SRC_GPR, 1, nullptr}},
{OPD(0, PF_3A_66, 0x40), 1, X86InstInfo{"DPPS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(0, PF_3A_66, 0x41), 1, X86InstInfo{"DPPD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(0, PF_3A_66, 0x42), 1, X86InstInfo{"MPSADBW", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 1, nullptr}},
{OPD(0, PF_3A_66, 0x40), 1, X86InstInfo{"DPPS", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(0, PF_3A_66, 0x41), 1, X86InstInfo{"DPPD", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(0, PF_3A_66, 0x42), 1, X86InstInfo{"MPSADBW", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(0, PF_3A_66, 0x44), 1, X86InstInfo{"PCLMULQDQ", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
{OPD(0, PF_3A_66, 0x60), 1, X86InstInfo{"PCMPESTRM", TYPE_UNDEC, FLAGS_NONE, 0, nullptr}},
@@ -33,7 +33,7 @@ static inline void GenerateTable(X86InstInfo *FinalTable, U8U8InfoStruct const *
auto OpNum = Op.first;
X86InstInfo const &Info = Op.Info;
for (uint32_t i = 0; i < Op.second; ++i) {
LOGMAN_THROW_A(FinalTable[OpNum + i].Type == TYPE_UNKNOWN, "Duplicate Entry %s->%s", FinalTable[OpNum + i].Name, Info.Name);
LogMan::Throw::A(FinalTable[OpNum + i].Type == TYPE_UNKNOWN, "Duplicate Entry %s->%s", FinalTable[OpNum + i].Name, Info.Name);
FinalTable[OpNum + i] = Info;
#ifndef NDEBUG
++Total;
@@ -50,7 +50,7 @@ static inline void GenerateTable(X86InstInfo *FinalTable, U16U8InfoStruct const
auto OpNum = Op.first;
X86InstInfo const &Info = Op.Info;
for (uint32_t i = 0; i < Op.second; ++i) {
LOGMAN_THROW_A(FinalTable[OpNum + i].Type == TYPE_UNKNOWN, "Duplicate Entry %s->%s", FinalTable[OpNum + i].Name, Info.Name);
LogMan::Throw::A(FinalTable[OpNum + i].Type == TYPE_UNKNOWN, "Duplicate Entry %s->%s", FinalTable[OpNum + i].Name, Info.Name);
FinalTable[OpNum + i] = Info;
#ifndef NDEBUG
++Total;
@@ -67,7 +67,7 @@ static inline void GenerateTableWithCopy(X86InstInfo *FinalTable, U8U8InfoStruct
auto OpNum = Op.first;
X86InstInfo const &Info = Op.Info;
for (uint32_t i = 0; i < Op.second; ++i) {
LOGMAN_THROW_A(FinalTable[OpNum + i].Type == TYPE_UNKNOWN, "Duplicate Entry %s->%s", FinalTable[OpNum + i].Name, Info.Name);
LogMan::Throw::A(FinalTable[OpNum + i].Type == TYPE_UNKNOWN, "Duplicate Entry %s->%s", FinalTable[OpNum + i].Name, Info.Name);
if (Info.Type == TYPE_COPY_OTHER) {
FinalTable[OpNum + i] = OtherLocal[OpNum + i];
}
@@ -89,7 +89,7 @@ static inline void GenerateX87Table(X86InstInfo *FinalTable, U16U8InfoStruct con
auto OpNum = Op.first;
X86InstInfo const &Info = Op.Info;
for (uint32_t i = 0; i < Op.second; ++i) {
LOGMAN_THROW_A(FinalTable[OpNum + i].Type == TYPE_UNKNOWN, "Duplicate Entry %s->%s", FinalTable[OpNum + i].Name, Info.Name);
LogMan::Throw::A(FinalTable[OpNum + i].Type == TYPE_UNKNOWN, "Duplicate Entry %s->%s", FinalTable[OpNum + i].Name, Info.Name);
if ((OpNum & 0b11'000'000) == 0b11'000'000) {
// If the mod field is 0b11 then it is a regular op
FinalTable[OpNum + i] = Info;
@@ -97,7 +97,7 @@ static inline void GenerateX87Table(X86InstInfo *FinalTable, U16U8InfoStruct con
else {
// If the mod field is !0b11 then this instruction is duplicated through the whole mod [0b00, 0b10] range
// and the modrm.rm space because that is used part of the instruction encoding
LOGMAN_THROW_A((OpNum & 0b11'000'000) == 0, "Only support mod field of zero in this path");
LogMan::Throw::A((OpNum & 0b11'000'000) == 0, "Only support mod field of zero in this path");
for (uint16_t mod = 0b00'000'000; mod < 0b11'000'000; mod += 0b01'000'000) {
for (uint16_t rm = 0b000; rm < 0b1'000; ++rm) {
FinalTable[(OpNum | mod | rm) + i] = Info;
+101 -179
View File
@@ -60,12 +60,6 @@
"constexpr static uint8_t ROUND_MODE_TOWARDS_ZERO = 3",
"constexpr static uint8_t ROUND_MODE_FLUSH_TO_ZERO = 1 << 2",
"static constexpr FEXCore::IR::RoundType Round_Nearest {ROUND_MODE_NEAREST}",
"static constexpr FEXCore::IR::RoundType Round_Negative_Infinity {ROUND_MODE_NEGATIVE_INFINITY}",
"static constexpr FEXCore::IR::RoundType Round_Positive_Infinity {ROUND_MODE_POSITIVE_INFINITY}",
"static constexpr FEXCore::IR::RoundType Round_Towards_Zero {ROUND_MODE_TOWARDS_ZERO} /* Truncate */",
"static constexpr FEXCore::IR::RoundType Round_Host {ROUND_MODE_TOWARDS_ZERO + 1}",
"constexpr static FEXCore::IR::MemOffsetType MEM_OFFSET_SXTX {0};",
"constexpr static FEXCore::IR::MemOffsetType MEM_OFFSET_UXTW {1};",
"constexpr static FEXCore::IR::MemOffsetType MEM_OFFSET_SXTW {2};"
@@ -85,6 +79,7 @@
"Blocks"
],
"Args": [
"uint64_t", "Entry",
"uint32_t", "BlockCount"
]
},
@@ -798,17 +793,6 @@
]
},
"CacheLineClear": {
"Desc": ["Does a 64 byte cacheline clear at the address specified"
],
"HasSideEffects": true,
"OpClass": "Memory",
"SSAArgs": "1",
"SSANames": [
"Addr"
]
},
"Add": {
"Desc": [ "Integer Add",
"Will truncate to 64 or 32bits"
@@ -1483,6 +1467,24 @@
]
},
"Float_ToGPR_U": {
"Desc": ["Moves the scalar element to a GPR with conversion",
"Converts the 32bit or 64bit float to an unsigned integer",
"Rounding mode determined by host flag's rounding mode"
],
"OpClass": "ALU",
"HasDest": true,
"DestClass": "GPR",
"DestSize": "ElementSize",
"SSAArgs": "1",
"SSANames": [
"Scalar"
],
"Args": [
"uint8_t", "ElementSize"
]
},
"Float_ToGPR_S": {
"Desc": ["Moves the scalar element to a GPR with conversion",
"Converts the 32bit or 64bit float to an signed integer",
@@ -1491,16 +1493,30 @@
"OpClass": "ALU",
"HasDest": true,
"DestClass": "GPR",
"DestSize": "DestElementSize",
"DestSize": "ElementSize",
"SSAArgs": "1",
"SSANames": [
"Scalar"
],
"HelperArgs": [
"uint8_t", "DestElementSize"
"Args": [
"uint8_t", "ElementSize"
]
},
"Float_ToGPR_ZU": {
"Desc": ["Moves the scalar element to a GPR with conversion",
"Converts the 32bit or 64bit float to an unsigned integer rounding towards zero (Truncating)"
],
"OpClass": "ALU",
"HasDest": true,
"DestClass": "GPR",
"DestSize": "ElementSize",
"SSAArgs": "1",
"SSANames": [
"Scalar"
],
"Args": [
"uint8_t", "SrcElementSize"
"uint8_t", "ElementSize"
]
},
@@ -1511,16 +1527,13 @@
"OpClass": "ALU",
"HasDest": true,
"DestClass": "GPR",
"DestSize": "DestElementSize",
"DestSize": "ElementSize",
"SSAArgs": "1",
"SSANames": [
"Scalar"
],
"HelperArgs": [
"uint8_t", "DestElementSize"
],
"Args": [
"uint8_t", "SrcElementSize"
"uint8_t", "ElementSize"
]
},
@@ -1550,6 +1563,7 @@
"Depending on backend, may only support GPR printing"
],
"OpClass": "Misc",
"DestSize": "GetOpSize(ssa0)",
"SSAArgs": "1",
"SSANames": [
"Value"
@@ -1641,23 +1655,6 @@
]
},
"VBic": {
"OpClass": "Vector",
"HasDest": true,
"DestClass": "FPR",
"DestSize": "RegisterSize",
"NumElements": "RegisterSize / ElementSize",
"SSAArgs": "2",
"SSANames": [
"Vector1",
"Vector2"
],
"HelperArgs": [
"uint8_t", "RegisterSize",
"uint8_t", "ElementSize"
]
},
"VOr": {
"OpClass": "Vector",
"HasDest": true,
@@ -1803,8 +1800,8 @@
"NumElements": "RegisterSize / ElementSize",
"SSAArgs": "2",
"SSANames": [
"VectorLower",
"VectorUpper"
"Vector1",
"Vector2"
],
"HelperArgs": [
"uint8_t", "RegisterSize",
@@ -1831,25 +1828,6 @@
]
},
"VUMinV": {
"OpClass": "Vector",
"Desc": ["Does a horizontal vector unsigned minimum of elements across the source vector",
"Result is a zero extended scalar"
],
"HasDest": true,
"DestClass": "FPR",
"DestSize": "RegisterSize",
"NumElements": "RegisterSize / ElementSize",
"SSAArgs": "1",
"SSANames": [
"Vector"
],
"HelperArgs": [
"uint8_t", "RegisterSize",
"uint8_t", "ElementSize"
]
},
"VURAvg": {
"OpClass": "Vector",
"Desc": ["Does an unsigned rounded average", "dst_elem = (src1_elem + src2_elem + 1) >> 1"],
@@ -1886,24 +1864,6 @@
]
},
"VPopcount": {
"OpClass": "Vector",
"Desc": ["Does a popcount for each element of the register"
],
"HasDest": true,
"DestClass": "FPR",
"DestSize": "RegisterSize",
"NumElements": "RegisterSize / ElementSize",
"SSAArgs": "1",
"SSANames": [
"Vector"
],
"HelperArgs": [
"uint8_t", "RegisterSize",
"uint8_t", "ElementSize"
]
},
"VFAdd": {
"OpClass": "Vector",
"HasDest": true,
@@ -1930,8 +1890,8 @@
"NumElements": "RegisterSize / ElementSize",
"SSAArgs": "2",
"SSANames": [
"VectorLow",
"VectorHigh"
"Vector1",
"Vector2"
],
"HelperArgs": [
"uint8_t", "RegisterSize",
@@ -2223,40 +2183,6 @@
]
},
"VUnZip": {
"OpClass": "Vector",
"HasDest": true,
"DestClass": "FPR",
"DestSize": "RegisterSize",
"NumElements": "RegisterSize / ElementSize",
"SSAArgs": "2",
"SSANames": [
"Lower",
"Upper"
],
"HelperArgs": [
"uint8_t", "RegisterSize",
"uint8_t", "ElementSize"
]
},
"VUnZip2": {
"OpClass": "Vector",
"HasDest": true,
"DestClass": "FPR",
"DestSize": "RegisterSize",
"NumElements": "RegisterSize / ElementSize",
"SSAArgs": "2",
"SSANames": [
"Lower",
"Upper"
],
"HelperArgs": [
"uint8_t", "RegisterSize",
"uint8_t", "ElementSize"
]
},
"VBSL": {
"Desc": ["Does a vector bitwise select.",
"If the bit in the field is 1 then the corresponding bit is pulled from VectorTrue",
@@ -2648,26 +2574,6 @@
]
},
"VDupElement": {
"Desc": ["Duplicates one element from the source register across the whole register"],
"OpClass": "Vector",
"HasDest": true,
"DestClass": "FPR",
"DestSize": "RegisterSize",
"NumElements": "RegisterSize / ElementSize",
"SSAArgs": "1",
"SSANames": [
"Vector"
],
"HelperArgs": [
"uint8_t", "RegisterSize",
"uint8_t", "ElementSize"
],
"Args": [
"uint8_t", "Index"
]
},
"VExtr": {
"Desc": ["Concats two vector registers together and extracts a full width register from the element index",
"Index is an element index. So it is offset by ElementSize argument",
@@ -3020,6 +2926,27 @@
]
},
"Float_FromGPR_U": {
"OpClass": "Conv",
"Desc": ["Scalar op: Converts unsigned GPR to Scalar float",
"Zeroes the upper bits of the vector register"
],
"HasDest": true,
"DestClass": "FPR",
"DestSize": "DstElementSize",
"NumElements": "1",
"SSAArgs": "1",
"SSANames": [
"GPR"
],
"HelperArgs": [
"uint8_t", "DstElementSize"
],
"Args": [
"uint8_t", "SrcElementSize"
]
},
"Float_FromGPR_S": {
"OpClass": "Conv",
"Desc": ["Scalar op: Converts signed GPR to Scalar float",
@@ -3096,6 +3023,25 @@
]
},
"Vector_FToU": {
"OpClass": "Conv",
"Desc": ["Vector op: Converts float to unsigned integer",
"Rounding mode determined by host rounding mode"
],
"HasDest": true,
"DestClass": "FPR",
"DestSize": "RegisterSize",
"NumElements": "RegisterSize / ElementSize",
"SSAArgs": "1",
"SSANames": [
"Vector"
],
"HelperArgs": [
"uint8_t", "RegisterSize",
"uint8_t", "ElementSize"
]
},
"Vector_FToS": {
"OpClass": "Conv",
"Desc": ["Vector op: Converts float to signed integer, rounding towards zero",
@@ -3115,6 +3061,23 @@
]
},
"Vector_FToZU": {
"OpClass": "Conv",
"Desc": "Vector op: Converts float to unsigned integer, rounding towards zero",
"HasDest": true,
"DestClass": "FPR",
"DestSize": "RegisterSize",
"NumElements": "RegisterSize / ElementSize",
"SSAArgs": "1",
"SSANames": [
"Vector"
],
"HelperArgs": [
"uint8_t", "RegisterSize",
"uint8_t", "ElementSize"
]
},
"Vector_FToZS": {
"OpClass": "Conv",
"Desc": "Vector op: Converts float to signed integer, rounding towards zero",
@@ -3152,28 +3115,6 @@
]
},
"Vector_FToI": {
"OpClass": "Conv",
"Desc": ["Vector op: Rounds float to integral",
"Rounding mode determined by argument"
],
"HasDest": true,
"DestClass": "FPR",
"DestSize": "RegisterSize",
"NumElements": "RegisterSize / ElementSize",
"SSAArgs": "1",
"SSANames": [
"Vector"
],
"HelperArgs": [
"uint8_t", "RegisterSize",
"uint8_t", "ElementSize"
],
"Args":[
"FEXCore::IR::RoundType", "Round"
]
},
"VUMul": {
"OpClass": "Vector",
"HasDest": true,
@@ -3281,25 +3222,6 @@
]
},
"VUABDL": {
"OpClass": "Vector",
"Desc": ["Unsigned Absolute Difference Long"
],
"HasDest": true,
"DestClass": "FPR",
"DestSize": "RegisterSize",
"NumElements": "RegisterSize / (ElementSize << 1)",
"SSAArgs": "2",
"SSANames": [
"Vector1",
"Vector2"
],
"HelperArgs": [
"uint8_t", "RegisterSize",
"uint8_t", "ElementSize"
]
},
"VTBL1": {
"Desc": ["Does a vector table lookup from one register in to the destination",
"Lookup is byte sized per byte element.",
+3 -13
View File
@@ -37,7 +37,7 @@ static void PrintArg(std::stringstream *out, [[maybe_unused]] IRListView const*
}
static void PrintArg(std::stringstream *out, [[maybe_unused]] IRListView const* IR, CondClassType Arg) {
static constexpr std::array<std::string_view, 22> CondNames = {
std::array<std::string, 22> CondNames = {
"EQ",
"NEQ",
"UGE",
@@ -66,7 +66,7 @@ static void PrintArg(std::stringstream *out, [[maybe_unused]] IRListView const*
}
static void PrintArg(std::stringstream *out, [[maybe_unused]] IRListView const* IR, MemOffsetType Arg) {
static constexpr std::array<std::string_view, 3> Names = {
std::array<std::string, 3> Names = {
"SXTX",
"UXTW",
"SXTW",
@@ -154,17 +154,6 @@ static void PrintArg(std::stringstream *out, [[maybe_unused]] IRListView const*
}
}
static void PrintArg(std::stringstream *out, [[maybe_unused]] IRListView const* IR, FEXCore::IR::RoundType Arg) {
switch (Arg) {
case FEXCore::IR::Round_Nearest: *out << "Nearest"; break;
case FEXCore::IR::Round_Negative_Infinity: *out << "-Inf"; break;
case FEXCore::IR::Round_Positive_Infinity: *out << "+Inf"; break;
case FEXCore::IR::Round_Towards_Zero: *out << "Towards Zero"; break;
case FEXCore::IR::Round_Host: *out << "Host"; break;
default: *out << "<Unknown Round Type>"; break;
}
}
void Dump(std::stringstream *out, IRListView const* IR, IR::RegisterAllocationData *RAData) {
auto HeaderOp = IR->GetHeader();
@@ -178,6 +167,7 @@ void Dump(std::stringstream *out, IRListView const* IR, IR::RegisterAllocationDa
++CurrentIndent;
AddIndent();
*out << "(%ssa0) " << "IRHeader ";
*out << "#0x" << std::hex << HeaderOp->Entry << ", ";
*out << "%ssa" << HeaderOp->Blocks.ID() << ", ";
*out << "#" << std::dec << HeaderOp->BlockCount << std::endl;
+17 -16
View File
@@ -9,17 +9,18 @@ $end_info$
namespace FEXCore::IR {
void IREmitter::ResetWorkingList() {
DualListData.Reset();
Data.Reset();
ListData.Reset();
CodeBlocks.clear();
CurrentWriteCursor = nullptr;
// This is necessary since we do "null" pointer checks
InvalidNode = reinterpret_cast<OrderedNode*>(DualListData.ListAllocate(sizeof(OrderedNode)));
InvalidNode = reinterpret_cast<OrderedNode*>(ListData.Allocate(sizeof(OrderedNode)));
memset(InvalidNode, 0, sizeof(OrderedNode));
CurrentCodeBlock = nullptr;
}
void IREmitter::ReplaceAllUsesWithRange(OrderedNode *Node, OrderedNode *NewNode, AllNodesIterator After, AllNodesIterator End) {
uintptr_t ListBegin = DualListData.ListBegin();
uintptr_t ListBegin = ListData.Begin();
auto NodeId = Node->Wrapped(ListBegin).ID();
while (After != End) {
@@ -44,8 +45,8 @@ void IREmitter::ReplaceAllUsesWithRange(OrderedNode *Node, OrderedNode *NewNode,
}
void IREmitter::ReplaceNodeArgument(OrderedNode *Node, uint8_t Arg, OrderedNode *NewArg) {
uintptr_t ListBegin = DualListData.ListBegin();
uintptr_t DataBegin = DualListData.DataBegin();
uintptr_t ListBegin = ListData.Begin();
uintptr_t DataBegin = Data.Begin();
FEXCore::IR::IROp_Header *IROp = Node->Op(DataBegin);
OrderedNodeWrapper OldArgWrapper = IROp->Args[Arg];
@@ -56,8 +57,8 @@ void IREmitter::ReplaceNodeArgument(OrderedNode *Node, uint8_t Arg, OrderedNode
}
void IREmitter::RemoveArgUses(OrderedNode *Node) {
uintptr_t ListBegin = DualListData.ListBegin();
uintptr_t DataBegin = DualListData.DataBegin();
uintptr_t ListBegin = ListData.Begin();
uintptr_t DataBegin = Data.Begin();
FEXCore::IR::IROp_Header *IROp = Node->Op(DataBegin);
@@ -71,7 +72,7 @@ void IREmitter::RemoveArgUses(OrderedNode *Node) {
void IREmitter::Remove(OrderedNode *Node) {
RemoveArgUses(Node);
Node->Unlink(DualListData.ListBegin());
Node->Unlink(ListData.Begin());
}
IREmitter::IRPair<IROp_CodeBlock> IREmitter::CreateNewCodeBlockAfter(OrderedNode* insertAfter) {
@@ -82,14 +83,14 @@ IREmitter::IRPair<IROp_CodeBlock> IREmitter::CreateNewCodeBlockAfter(OrderedNode
if (insertAfter) {
LinkCodeBlocks(insertAfter, CodeNode);
} else {
LOGMAN_THROW_A(CurrentCodeBlock != nullptr, "CurrentCodeBlock must not be null here");
LogMan::Throw::A(CurrentCodeBlock != nullptr, "CurrentCodeBlock must not be null here");
// Find last block
auto LastBlock = CurrentCodeBlock;
while (LastBlock->Header.Next.GetNode(DualListData.ListBegin()) != InvalidNode)
LastBlock = LastBlock->Header.Next.GetNode(DualListData.ListBegin());
while (LastBlock->Header.Next.GetNode(ListData.Begin()) != InvalidNode)
LastBlock = LastBlock->Header.Next.GetNode(ListData.Begin());
// Append it after the last block
LinkCodeBlocks(LastBlock, CodeNode);
}
@@ -101,12 +102,12 @@ IREmitter::IRPair<IROp_CodeBlock> IREmitter::CreateNewCodeBlockAfter(OrderedNode
void IREmitter::SetCurrentCodeBlock(OrderedNode *Node) {
CurrentCodeBlock = Node;
LOGMAN_THROW_A(Node->Op(DualListData.DataBegin())->Op == OP_CODEBLOCK, "Node wasn't codeblock. It was '%s'", std::string(IR::GetName(Node->Op(DualListData.DataBegin())->Op)).c_str());
SetWriteCursor(Node->Op(DualListData.DataBegin())->CW<IROp_CodeBlock>()->Begin.GetNode(DualListData.ListBegin()));
LogMan::Throw::A(Node->Op(Data.Begin())->Op == OP_CODEBLOCK, "Node wasn't codeblock. It was '%s'", std::string(IR::GetName(Node->Op(Data.Begin())->Op)).c_str());
SetWriteCursor(Node->Op(Data.Begin())->CW<IROp_CodeBlock>()->Begin.GetNode(ListData.Begin()));
}
void IREmitter::ReplaceWithConstant(OrderedNode *Node, uint64_t Value) {
auto Header = Node->Op(DualListData.DataBegin());
auto Header = Node->Op(Data.Begin());
if (IRSizes[Header->Op] >= sizeof(IROp_Constant)) {
// Unlink any arguments the node currently has
+117 -113
View File
@@ -66,8 +66,7 @@ std::string DecodeErrorToString(DecodeFailure Failure) {
case DecodeFailure::DECODE_INVALID_CONDFLAG: return "Invalid Conditional name";
case DecodeFailure::DECODE_INVALID_MEMOFFSETTYPE: return "Invalid Memory Offset Type";
case DecodeFailure::DECODE_INVALID_FENCETYPE: return "Invalid Fence Type";
}
return "Unknown Error";
};
}
std::unordered_map<std::string_view, FEXCore::IR::IROps> NameToOpMap;
@@ -75,22 +74,22 @@ std::unordered_map<std::string_view, FEXCore::IR::IROps> NameToOpMap;
class IRParser: public FEXCore::IR::IREmitter {
public:
template<typename Type>
std::pair<DecodeFailure, Type> DecodeValue(const std::string &Arg) {
std::pair<DecodeFailure, Type> DecodeValue(std::string &Arg) {
return {DecodeFailure::DECODE_UNKNOWN_TYPE, {}};
}
template<>
std::pair<DecodeFailure, uint8_t> DecodeValue(const std::string &Arg) {
std::pair<DecodeFailure, uint8_t> DecodeValue(std::string &Arg) {
if (Arg.at(0) != '#') return {DecodeFailure::DECODE_INVALIDCHAR, 0};
uint8_t Result = strtoul(&Arg.at(1), nullptr, 0);
if (errno == ERANGE) return {DecodeFailure::DECODE_INVALIDRANGE, 0};
return {DecodeFailure::DECODE_OKAY, Result};
}
}
template<>
std::pair<DecodeFailure, bool> DecodeValue(const std::string &Arg) {
std::pair<DecodeFailure, bool> DecodeValue(std::string &Arg) {
if (Arg.at(0) != '#') return {DecodeFailure::DECODE_INVALIDCHAR, 0};
uint8_t Result = strtoul(&Arg.at(1), nullptr, 0);
@@ -99,7 +98,7 @@ class IRParser: public FEXCore::IR::IREmitter {
}
template<>
std::pair<DecodeFailure, uint16_t> DecodeValue(const std::string &Arg) {
std::pair<DecodeFailure, uint16_t> DecodeValue(std::string &Arg) {
if (Arg.at(0) != '#') return {DecodeFailure::DECODE_INVALIDCHAR, 0};
uint16_t Result = strtoul(&Arg.at(1), nullptr, 0);
@@ -108,7 +107,7 @@ class IRParser: public FEXCore::IR::IREmitter {
}
template<>
std::pair<DecodeFailure, uint32_t> DecodeValue(const std::string &Arg) {
std::pair<DecodeFailure, uint32_t> DecodeValue(std::string &Arg) {
if (Arg.at(0) != '#') return {DecodeFailure::DECODE_INVALIDCHAR, 0};
uint32_t Result = strtoul(&Arg.at(1), nullptr, 0);
@@ -117,7 +116,7 @@ class IRParser: public FEXCore::IR::IREmitter {
}
template<>
std::pair<DecodeFailure, uint64_t> DecodeValue(const std::string &Arg) {
std::pair<DecodeFailure, uint64_t> DecodeValue(std::string &Arg) {
if (Arg.at(0) != '#') return {DecodeFailure::DECODE_INVALIDCHAR, 0};
uint64_t Result = strtoull(&Arg.at(1), nullptr, 0);
@@ -126,7 +125,7 @@ class IRParser: public FEXCore::IR::IREmitter {
}
template<>
std::pair<DecodeFailure, int64_t> DecodeValue(const std::string &Arg) {
std::pair<DecodeFailure, int64_t> DecodeValue(std::string &Arg) {
if (Arg.at(0) != '#') return {DecodeFailure::DECODE_INVALIDCHAR, 0};
int64_t Result = (int64_t)strtoull(&Arg.at(1), nullptr, 0);
@@ -135,7 +134,7 @@ class IRParser: public FEXCore::IR::IREmitter {
}
template<>
std::pair<DecodeFailure, IR::SHA256Sum> DecodeValue(const std::string &Arg) {
std::pair<DecodeFailure, IR::SHA256Sum> DecodeValue(std::string &Arg) {
IR::SHA256Sum Result;
if (Arg.at(0) != 's' || Arg.at(1) != 'h' || Arg.at(2) != 'a' || Arg.at(3) != '2' || Arg.at(4) != '5' || Arg.at(5) != '6' || Arg.at(6) != ':')
@@ -166,7 +165,7 @@ class IRParser: public FEXCore::IR::IREmitter {
}
template<>
std::pair<DecodeFailure, FEXCore::IR::RegisterClassType> DecodeValue(const std::string &Arg) {
std::pair<DecodeFailure, FEXCore::IR::RegisterClassType> DecodeValue(std::string &Arg) {
if (Arg == "GPR") {
return {DecodeFailure::DECODE_OKAY, FEXCore::IR::GPRClass};
}
@@ -184,7 +183,7 @@ class IRParser: public FEXCore::IR::IREmitter {
}
template<>
std::pair<DecodeFailure, FEXCore::IR::TypeDefinition> DecodeValue(const std::string &Arg) {
std::pair<DecodeFailure, FEXCore::IR::TypeDefinition> DecodeValue(std::string &Arg) {
uint8_t Size{}, Elements{1};
int NumArgs = sscanf(Arg.c_str(), "i%hhdv%hhd", &Size, &Elements);
@@ -196,8 +195,8 @@ class IRParser: public FEXCore::IR::IREmitter {
}
template<>
std::pair<DecodeFailure, FEXCore::IR::CondClassType> DecodeValue(const std::string &Arg) {
static constexpr std::array<std::string_view, 22> CondNames = {
std::pair<DecodeFailure, FEXCore::IR::CondClassType> DecodeValue(std::string &Arg) {
std::array<std::string, 22> CondNames = {
"EQ",
"NEQ",
"UGE",
@@ -231,8 +230,8 @@ class IRParser: public FEXCore::IR::IREmitter {
}
template<>
std::pair<DecodeFailure, FEXCore::IR::MemOffsetType> DecodeValue(const std::string &Arg) {
static constexpr std::array<std::string_view, 3> Names = {
std::pair<DecodeFailure, FEXCore::IR::MemOffsetType> DecodeValue(std::string &Arg) {
std::array<std::string, 3> Names = {
"SXTX",
"UXTW",
"SXTW",
@@ -247,8 +246,8 @@ class IRParser: public FEXCore::IR::IREmitter {
}
template<>
std::pair<DecodeFailure, FEXCore::IR::FenceType> DecodeValue(const std::string &Arg) {
static constexpr std::array<std::string_view, 3> Names = {
std::pair<DecodeFailure, FEXCore::IR::FenceType> DecodeValue(std::string &Arg) {
std::array<std::string, 3> Names = {
"Loads",
"Stores",
"LoadStores",
@@ -263,22 +262,23 @@ class IRParser: public FEXCore::IR::IREmitter {
}
template<>
std::pair<DecodeFailure, OrderedNode*> DecodeValue(const std::string &Arg) {
std::pair<DecodeFailure, OrderedNode*> DecodeValue(std::string &Arg) {
if (Arg.at(0) != '%') return {DecodeFailure::DECODE_INVALIDCHAR, 0};
// Strip off the type qualifier from the ssa value
size_t ArgEnd = std::string::npos;
std::string SSAName = trim(Arg);
const size_t ArgEnd = SSAName.find_first_of(' ');
ArgEnd = SSAName.find_first_of(" ");
if (ArgEnd != std::string::npos) {
SSAName = SSAName.substr(0, ArgEnd);
}
SSAName = SSAName.substr(0, ArgEnd);
}
// Forward declarations may make this not succed
// Forward declarations may make this not succed
auto Op = SSANameMapper.find(SSAName);
if (Op == SSANameMapper.end()) {
if (Op == SSANameMapper.end()) {
return {DecodeFailure::DECODE_UNKNOWN_SSA, nullptr};
}
}
return {DecodeFailure::DECODE_OKAY, Op->second};
}
@@ -302,21 +302,21 @@ class IRParser: public FEXCore::IR::IREmitter {
IRParser(std::istream *text) {
InitializeStaticTables();
std::string TmpLine;
while (!text->eof()) {
std::getline(*text, TmpLine);
if (text->eof()) {
break;
}
if (text->eof()) {
break;
}
if (text->fail()) {
LogMan::Msg::EFmt("Failed to getline on line: {}", Lines.size());
LogMan::Msg::E("Failed to getline on line: %ld", Lines.size());
return;
}
Lines.emplace_back(TmpLine);
}
ResetWorkingList();
ResetWorkingList();
Loaded = Parse();
}
@@ -327,11 +327,11 @@ class IRParser: public FEXCore::IR::IREmitter {
bool Parse() {
const auto CheckPrintError = [&](const LineDefinition &Def, DecodeFailure Failure) -> bool {
auto CheckPrintError = [&](LineDefinition &Def, DecodeFailure Failure) -> bool {
if (Failure != DecodeFailure::DECODE_OKAY) {
LogMan::Msg::EFmt("Error on Line: {}", Def.LineNumber);
LogMan::Msg::EFmt("{}", Lines[Def.LineNumber]);
LogMan::Msg::EFmt("Value Couldn't be decoded due to {}", DecodeErrorToString(Failure));
LogMan::Msg::E("Error on Line: %d", Def.LineNumber);
LogMan::Msg::E("%s", Lines[Def.LineNumber].c_str());
LogMan::Msg::E("Value Couldn't be decoded due to %s", DecodeErrorToString(Failure).c_str());
return false;
}
@@ -339,13 +339,13 @@ class IRParser: public FEXCore::IR::IREmitter {
};
// String parse every line for our definitions
for (size_t i = 0; i < Lines.size(); ++i) {
std::string Line = Lines[i];
for (size_t i = 0; i < Lines.size(); ++i) {
std::string Line = Lines[i];
LineDefinition Def{};
CurrentDef = &Def;
CurrentDef = &Def;
Def.LineNumber = i;
Line = trim(Line);
Line = trim(Line);
// Skip empty lines
if (Line.empty()) {
@@ -359,37 +359,35 @@ class IRParser: public FEXCore::IR::IREmitter {
}
size_t CurrentPos{};
// Let's see if this node is assigning something first
if (Line[0] == '%') {
// Let's see if this node is assigning something first
if (Line[0] == '%') {
size_t DefinitionEnd = std::string::npos;
if ((DefinitionEnd = Line.find_first_of('=', CurrentPos)) != std::string::npos) {
if ((DefinitionEnd = Line.find_first_of("=", CurrentPos)) != std::string::npos) {
Def.Definition = Line.substr(0, DefinitionEnd);
Def.Definition = trim(Def.Definition);
Def.HasDefinition = true;
CurrentPos = DefinitionEnd + 1; // +1 to ensure we go past then assignment
}
else {
LogMan::Msg::EFmt("Error on Line: {}", i);
LogMan::Msg::EFmt("{}", Lines[i]);
LogMan::Msg::EFmt("SSA declaration without assignment");
LogMan::Msg::E("Error on Line: %d", i);
LogMan::Msg::E("%s", Lines[i].c_str());
LogMan::Msg::E("SSA declaration without assignment");
return false;
}
}
}
// Check if we are pulling in some IR from the IR Printer
// Prints (%ssa%d) at the start of lines without a definition
if (Line[0] == '(') {
size_t DefinitionEnd = std::string::npos;
if ((DefinitionEnd = Line.find_first_of(')', CurrentPos)) != std::string::npos) {
if ((DefinitionEnd = Line.find_first_of(")", CurrentPos)) != std::string::npos) {
size_t SSAEnd = std::string::npos;
if ((SSAEnd = Line.find_last_of(' ', DefinitionEnd)) != std::string::npos) {
if ((SSAEnd = Line.find_last_of(" ", DefinitionEnd)) != std::string::npos) {
std::string Type = Line.substr(SSAEnd + 1, DefinitionEnd - SSAEnd - 1);
Type = trim(Type);
auto DefinitionSize = DecodeValue<FEXCore::IR::TypeDefinition>(Type);
if (!CheckPrintError(Def, DefinitionSize.first)) {
return false;
}
if (!CheckPrintError(Def, DefinitionSize.first)) return false;
Def.Size = DefinitionSize.second;
}
@@ -398,9 +396,9 @@ class IRParser: public FEXCore::IR::IREmitter {
CurrentPos = DefinitionEnd + 1;
}
else {
LogMan::Msg::EFmt("Error on Line: {}", i);
LogMan::Msg::EFmt("{}", Lines[i]);
LogMan::Msg::EFmt("SSA value with numbered SSA provided but no closing parentheses");
LogMan::Msg::E("Error on Line: %d", i);
LogMan::Msg::E("%s", Lines[i].c_str());
LogMan::Msg::E("SSA value with numbered SSA provided but no closing parentheses");
return false;
}
}
@@ -408,7 +406,7 @@ class IRParser: public FEXCore::IR::IREmitter {
if (Def.HasDefinition) {
// Let's check if we have a size declared with this variable
size_t NameEnd = std::string::npos;
if ((NameEnd = Def.Definition.find_first_of(' ')) != std::string::npos) {
if ((NameEnd = Def.Definition.find_first_of(" ")) != std::string::npos) {
std::string Type = Def.Definition.substr(NameEnd + 1);
Type = trim(Type);
Def.Definition = trim(Def.Definition.substr(0, NameEnd));
@@ -419,9 +417,9 @@ class IRParser: public FEXCore::IR::IREmitter {
}
if (Def.Definition == "%Invalid") {
LogMan::Msg::EFmt("Error on Line: {}", i);
LogMan::Msg::EFmt("{}", Lines[i]);
LogMan::Msg::EFmt("Definition tried to define reserved %Invalid ssa node");
LogMan::Msg::E("Error on Line: %d", i);
LogMan::Msg::E("%s", Lines[i].c_str());
LogMan::Msg::E("Definition tried to define reserved %Invalid ssa node");
return false;
}
}
@@ -438,9 +436,9 @@ class IRParser: public FEXCore::IR::IREmitter {
}
else {
if (RemainingLine.empty()) {
LogMan::Msg::EFmt("Error on Line: {}", i);
LogMan::Msg::EFmt("{}", Lines[i]);
LogMan::Msg::EFmt("Line without an IROp?");
LogMan::Msg::E("Error on Line: %d", i);
LogMan::Msg::E("%s", Lines[i].c_str());
LogMan::Msg::E("Line without an IROp?");
return false;
}
@@ -457,10 +455,12 @@ class IRParser: public FEXCore::IR::IREmitter {
}
else {
while (!RemainingLine.empty()) {
const size_t ArgEnd = RemainingLine.find(',');
std::string Arg = trim(RemainingLine.substr(0, ArgEnd));
size_t ArgEnd = std::string::npos;
ArgEnd = RemainingLine.find_first_of(",");
Def.Args.emplace_back(std::move(Arg));
std::string Arg = RemainingLine.substr(0, ArgEnd);
Arg = trim(Arg);
Def.Args.emplace_back(Arg);
RemainingLine.erase(0, ArgEnd+1); // +1 to ensure we go past the ','
if (ArgEnd == std::string::npos)
@@ -469,17 +469,17 @@ class IRParser: public FEXCore::IR::IREmitter {
}
}
CurrentDef = &Defs.emplace_back(std::move(Def));
}
Defs.emplace_back(Def);
}
// Ensure all of the ops are real ops
for(size_t i = 0; i < Defs.size(); ++i) {
auto &Def = Defs[i];
auto Op = NameToOpMap.find(Def.IROp);
if (Op == NameToOpMap.end()) {
LogMan::Msg::EFmt("Error on Line: {}", Def.LineNumber);
LogMan::Msg::EFmt("{}", Lines[Def.LineNumber]);
LogMan::Msg::EFmt("IROp '{}' doesn't exist", Def.IROp);
LogMan::Msg::E("Error on Line: %d", Def.LineNumber);
LogMan::Msg::E("%s", Lines[Def.LineNumber].c_str());
LogMan::Msg::E("IROp '%s' doesn't exist", Def.IROp.c_str());
return false;
}
Def.OpEnum = Op->second;
@@ -489,38 +489,40 @@ class IRParser: public FEXCore::IR::IREmitter {
IRPair<IROp_IRHeader> IRHeader;
{
auto &Def = Defs[0];
CurrentDef = &Def;
CurrentDef = &Def;
if (Def.OpEnum != FEXCore::IR::IROps::OP_IRHEADER) {
LogMan::Msg::EFmt("Error on Line: {}", Def.LineNumber);
LogMan::Msg::EFmt("{}", Lines[Def.LineNumber]);
LogMan::Msg::EFmt("First op needs to be IRHeader. Was '{}'", Def.IROp);
LogMan::Msg::E("Error on Line: %d", Def.LineNumber);
LogMan::Msg::E("%s", Lines[Def.LineNumber].c_str());
LogMan::Msg::E("First op needs to be IRHeader. Was '%s'", Def.IROp.c_str());
return false;
}
auto CodeBlockCount = DecodeValue<uint64_t>(Def.Args[1]);
auto Entry = DecodeValue<uint64_t>(Def.Args[0]);
auto CodeBlockCount = DecodeValue<uint64_t>(Def.Args[2]);
if (!CheckPrintError(Def, Entry.first)) return false;
if (!CheckPrintError(Def, CodeBlockCount.first)) return false;
IRHeader = _IRHeader(InvalidNode, CodeBlockCount.second);
IRHeader = _IRHeader(InvalidNode, Entry.second, CodeBlockCount.second);
}
SetWriteCursor(nullptr); // isolate the header from everything following
// Initialize SSANameMapper with Invalid value
SSANameMapper.insert_or_assign("%Invalid", Invalid());
SSANameMapper["%Invalid"] = Invalid();
// Spin through the blocks and generate basic block ops
for(size_t i = 0; i < Defs.size(); ++i) {
auto &Def = Defs[i];
if (Def.OpEnum == FEXCore::IR::IROps::OP_CODEBLOCK) {
auto CodeBlock = _CodeBlock(InvalidNode, InvalidNode);
SSANameMapper.insert_or_assign(Def.Definition, CodeBlock.Node);
SSANameMapper[Def.Definition] = CodeBlock.Node;
Def.Node = CodeBlock.Node;
if (i == 1) {
// First code block is the entry block
// Link the header to the first block
IRHeader.first->Blocks = CodeBlock.Node->Wrapped(DualListData.ListBegin());
IRHeader.first->Blocks = CodeBlock.Node->Wrapped(ListData.Begin());
}
CodeBlocks.emplace_back(CodeBlock.Node);
}
@@ -532,64 +534,63 @@ class IRParser: public FEXCore::IR::IREmitter {
FEXCore::IR::IROp_CodeBlock *CurrentBlockOp{};
for(size_t i = 1; i < Defs.size(); ++i) {
auto &Def = Defs[i];
CurrentDef = &Def;
CurrentDef = &Def;
switch (Def.OpEnum) {
// Special handled
case FEXCore::IR::IROps::OP_IRHEADER:
LogMan::Msg::EFmt("Error on Line: {}", Def.LineNumber);
LogMan::Msg::EFmt("{}", Lines[Def.LineNumber]);
LogMan::Msg::EFmt("IRHEADER used in the middle of the block!");
LogMan::Msg::E("Error on Line: %d", Def.LineNumber);
LogMan::Msg::E("%s", Lines[Def.LineNumber].c_str());
LogMan::Msg::E("IRHEADER used in the middle of the block!");
return false; // only one OP_IRHEADER allowed per block
case FEXCore::IR::IROps::OP_CODEBLOCK: {
SetWriteCursor(nullptr); // isolate from previous block
if (CurrentBlock != nullptr) {
LogMan::Msg::EFmt("Error on Line: {}", Def.LineNumber);
LogMan::Msg::EFmt("{}", Lines[Def.LineNumber]);
LogMan::Msg::EFmt("CodeBlock being used inside of already existing codeblock!");
LogMan::Msg::E("Error on Line: %d", Def.LineNumber);
LogMan::Msg::E("%s", Lines[Def.LineNumber].c_str());
LogMan::Msg::E("CodeBlock being used inside of already existing codeblock!");
return false;
}
CurrentBlock = Def.Node;
CurrentBlockOp = CurrentBlock->Op(DualListData.DataBegin())->CW<FEXCore::IR::IROp_CodeBlock>();
CurrentBlockOp = CurrentBlock->Op(Data.Begin())->CW<FEXCore::IR::IROp_CodeBlock>();
break;
}
case FEXCore::IR::IROps::OP_BEGINBLOCK: {
if (CurrentBlock == nullptr) {
LogMan::Msg::EFmt("Error on Line: {}", Def.LineNumber);
LogMan::Msg::EFmt("{}", Lines[Def.LineNumber]);
LogMan::Msg::EFmt("EndBlock being used outside of a block!");
LogMan::Msg::E("Error on Line: %d", Def.LineNumber);
LogMan::Msg::E("%s", Lines[Def.LineNumber].c_str());
LogMan::Msg::E("EndBlock being used outside of a block!");
return false;
}
auto Adjust = DecodeValue<OrderedNode*>(Def.Args[0]);
if (!CheckPrintError(Def, Adjust.first)) {
return false;
}
if (!CheckPrintError(Def, Adjust.first)) return false;
Def.Node = _BeginBlock(Adjust.second);
CurrentBlockOp->Begin = Def.Node->Wrapped(DualListData.ListBegin());
CurrentBlockOp->Begin = Def.Node->Wrapped(ListData.Begin());
break;
}
case FEXCore::IR::IROps::OP_ENDBLOCK: {
if (CurrentBlock == nullptr) {
LogMan::Msg::EFmt("Error on Line: {}", Def.LineNumber);
LogMan::Msg::EFmt("{}", Lines[Def.LineNumber]);
LogMan::Msg::EFmt("EndBlock being used outside of a block!");
LogMan::Msg::E("Error on Line: %d", Def.LineNumber);
LogMan::Msg::E("%s", Lines[Def.LineNumber].c_str());
LogMan::Msg::E("EndBlock being used outside of a block!");
return false;
}
auto Adjust = DecodeValue<OrderedNode*>(Def.Args[0]);
if (!CheckPrintError(Def, Adjust.first)) {
return false;
}
if (!CheckPrintError(Def, Adjust.first)) return false;
Def.Node = _EndBlock(Adjust.second);
CurrentBlockOp->Last = Def.Node->Wrapped(DualListData.ListBegin());
CurrentBlockOp->Last = Def.Node->Wrapped(ListData.Begin());
CurrentBlock = nullptr;
CurrentBlockOp = nullptr;
@@ -598,23 +599,25 @@ class IRParser: public FEXCore::IR::IREmitter {
}
case FEXCore::IR::IROps::OP_DUMMY: {
LogMan::Msg::EFmt("Error on Line: {}", Def.LineNumber);
LogMan::Msg::EFmt("{}", Lines[Def.LineNumber]);
LogMan::Msg::EFmt("Dummy op must not be used");
LogMan::Msg::E("Error on Line: %d", Def.LineNumber);
LogMan::Msg::E("%s", Lines[Def.LineNumber].c_str());
LogMan::Msg::E("Dummy op must not be used");
break;
}
#define IROP_PARSER_SWITCH_HELPERS
#include <FEXCore/IR/IRDefines.inc>
default: {
LogMan::Msg::EFmt("Error on Line: {}", Def.LineNumber);
LogMan::Msg::EFmt("{}", Lines[Def.LineNumber]);
LogMan::Msg::EFmt("Unhandled Op enum '{}' in parser", Def.IROp);
LogMan::Msg::E("Error on Line: %d", Def.LineNumber);
LogMan::Msg::E("%s", Lines[Def.LineNumber].c_str());
LogMan::Msg::E("Unhandled Op enum '%s' in parser", Def.IROp.c_str());
return false;
break;
}
}
if (Def.HasDefinition) {
auto IROp = Def.Node->Op(DualListData.DataBegin());
auto IROp = Def.Node->Op(Data.Begin());
if (Def.Size.Elements()) {
IROp->Size = Def.Size.Bytes() * Def.Size.Elements();
IROp->ElementSize = Def.Size.Bytes();
@@ -623,7 +626,7 @@ class IRParser: public FEXCore::IR::IREmitter {
IROp->Size = Def.Size.Bytes();
IROp->ElementSize = 0;
}
SSANameMapper.insert_or_assign(Def.Definition, Def.Node);
SSANameMapper[Def.Definition] = Def.Node;
}
}
@@ -631,11 +634,11 @@ class IRParser: public FEXCore::IR::IREmitter {
}
void InitializeStaticTables() {
if (NameToOpMap.empty()) {
if (NameToOpMap.size() == 0) {
for (FEXCore::IR::IROps Op = FEXCore::IR::IROps::OP_DUMMY;
Op <= FEXCore::IR::IROps::OP_LAST;
Op = static_cast<FEXCore::IR::IROps>(static_cast<uint32_t>(Op) + 1)) {
NameToOpMap.insert_or_assign(FEXCore::IR::GetName(Op), Op);
NameToOpMap[FEXCore::IR::GetName(Op)] = Op;
}
}
}
@@ -643,12 +646,13 @@ class IRParser: public FEXCore::IR::IREmitter {
} // anon namespace
std::unique_ptr<IREmitter> Parse(std::istream *in) {
auto parser = std::make_unique<IRParser>(in);
IREmitter* Parse(std::istream *in) {
auto parser = new IRParser(in);
if (parser->Loaded) {
return parser;
} else {
delete parser;
return nullptr;
}
}
-7
View File
@@ -19,13 +19,6 @@ void PassManager::AddDefaultPasses(bool InlineConstants, bool StaticRegisterAllo
if (!DisablePasses()) {
InsertPass(CreateContextLoadStoreElimination());
if (Is64BitMode()) {
// This needs to run after RCLSE
// This only matters for 64-bit code since these instructions don't exist in 32-bit
InsertPass(CreateLongDivideEliminationPass());
}
InsertPass(CreateDeadStoreElimination());
InsertPass(CreatePassDeadCodeElimination());
InsertPass(CreateConstProp(InlineConstants));
-3
View File
@@ -6,7 +6,6 @@ $end_info$
#pragma once
#include <FEXCore/Config/Config.h>
#include <FEXCore/IR/IntrusiveIRList.h>
#include <FEXCore/IR/IREmitter.h>
@@ -84,8 +83,6 @@ private:
ValidationPasses.emplace_back(Pass);
}
#endif
FEX_CONFIG_OPT(Is64BitMode, IS64BIT_MODE);
};
}
-1
View File
@@ -14,7 +14,6 @@ FEXCore::IR::Pass* CreatePassDeadCodeElimination();
FEXCore::IR::Pass* CreateIRCompaction();
FEXCore::IR::RegisterAllocationPass* CreateRegisterAllocationPass(FEXCore::IR::Pass* CompactionPass, bool OptimizeSRA);
FEXCore::IR::Pass* CreateStaticRegisterAllocationPass();
FEXCore::IR::Pass* CreateLongDivideEliminationPass();
namespace Validation {
FEXCore::IR::Pass* CreateIRValidation();
+320 -375
View File
@@ -19,6 +19,15 @@ $end_info$
namespace FEXCore::IR {
class ConstProp final : public FEXCore::IR::Pass {
std::unordered_map<uint64_t, OrderedNode*> ConstPool;
std::map<OrderedNode*, uint64_t> AddressgenConsts;
public:
bool Run(IREmitter *IREmit) override;
bool InlineConstants;
ConstProp(bool DoInlineConstants) : InlineConstants(DoInlineConstants) { }
};
template<typename T>
uint64_t getMask(T Op) {
uint64_t NumBits = Op->Header.Size * 8;
@@ -60,7 +69,8 @@ static bool IsImmMemory(uint64_t imm, uint8_t AccessSize) {
}
}
static std::tuple<MemOffsetType, uint8_t, OrderedNode*, OrderedNode*> MemExtendedAddressing(IREmitter *IREmit, uint8_t AccessSize, IROp_Header* AddressHeader) {
std::tuple<MemOffsetType, uint8_t, OrderedNode*, OrderedNode*> MemExtendedAddressing(IREmitter *IREmit, uint8_t AccessSize, IROp_Header* AddressHeader) {
auto Src0Header = IREmit->GetOpHeader(AddressHeader->Args[0]);
if (Src0Header->Size == 8) {
//Try to optimize: Base + MUL(Offset, Scale)
@@ -114,7 +124,7 @@ static std::tuple<MemOffsetType, uint8_t, OrderedNode*, OrderedNode*> MemExtende
return { MEM_OFFSET_SXTX, 1, IREmit->UnwrapNode(AddressHeader->Args[0]), IREmit->UnwrapNode(AddressHeader->Args[1]) };
}
static OrderedNodeWrapper RemoveUselessMasking(IREmitter *IREmit, OrderedNodeWrapper src, uint64_t mask) {
OrderedNodeWrapper RemoveUselessMasking(IREmitter *IREmit, OrderedNodeWrapper src, uint64_t mask) {
#if 1 // HOTFIX: We need to clear up the meaning of opsize and dest size. See #594
return src;
#else
@@ -141,7 +151,7 @@ static OrderedNodeWrapper RemoveUselessMasking(IREmitter *IREmit, OrderedNodeWra
#endif
}
static bool IsBfeAlreadyDone(IREmitter *IREmit, OrderedNodeWrapper src, uint64_t Width) {
bool IsBfeAlreadyDone(IREmitter *IREmit, OrderedNodeWrapper src, uint64_t Width) {
auto IROp = IREmit->GetOpHeader(src);
if (IROp->Op == OP_BFE) {
auto Op = IROp->C<IR::IROp_Bfe>();
@@ -152,55 +162,31 @@ static bool IsBfeAlreadyDone(IREmitter *IREmit, OrderedNodeWrapper src, uint64_t
return false;
}
class ConstProp final : public FEXCore::IR::Pass {
public:
explicit ConstProp(bool DoInlineConstants) : InlineConstants(DoInlineConstants) { }
bool ConstProp::Run(IREmitter *IREmit) {
bool Run(IREmitter *IREmit) override;
bool InlineConstants;
private:
bool HandleConstantPools(IREmitter *IREmit, const IRListView& CurrentIR);
void CodeMotionAroundSelects(IREmitter *IREmit, const IRListView& CurrentIR);
void FCMPOptimization(IREmitter *IREmit, const IRListView& CurrentIR);
void LoadMemStoreMemImmediatePooling(IREmitter *IREmit, const IRListView& CurrentIR);
bool ZextAndMaskingElimination(IREmitter *IREmit, const IRListView& CurrentIR,
OrderedNode* CodeNode, IROp_Header* IROp);
bool ConstantPropagation(IREmitter *IREmit, const IRListView& CurrentIR,
OrderedNode* CodeNode, IROp_Header* IROp);
bool ConstantInlining(IREmitter *IREmit, const IRListView& CurrentIR);
std::unordered_map<uint64_t, OrderedNode*> ConstPool;
std::map<OrderedNode*, uint64_t> AddressgenConsts;
};
bool ConstProp::HandleConstantPools(IREmitter *IREmit, const IRListView& CurrentIR) {
bool Changed = false;
auto CurrentIR = IREmit->ViewIR();
auto OriginalWriteCursor = IREmit->GetWriteCursor();
// constants are pooled per block
for (auto [BlockNode, BlockHeader] : CurrentIR.GetBlocks()) {
for (auto [CodeNode, IROp] : CurrentIR.GetCode(BlockNode)) {
if (IROp->Op == OP_CONSTANT) {
auto Op = IROp->C<IR::IROp_Constant>();
if (ConstPool.count(Op->Constant)) {
IREmit->ReplaceAllUsesWith(CodeNode, ConstPool[Op->Constant]);
Changed = true;
} else {
ConstPool[Op->Constant] = CodeNode;
{
// constants are pooled per block
for (auto [BlockNode, BlockHeader] : CurrentIR.GetBlocks()) {
for (auto [CodeNode, IROp] : CurrentIR.GetCode(BlockNode)) {
if (IROp->Op == OP_CONSTANT) {
auto Op = IROp->C<IR::IROp_Constant>();
if (ConstPool.count(Op->Constant)) {
IREmit->ReplaceAllUsesWith(CodeNode, ConstPool[Op->Constant]);
Changed = true;
} else {
ConstPool[Op->Constant] = CodeNode;
}
}
}
ConstPool.clear();
}
ConstPool.clear();
}
return Changed;
}
// Code motion around selects
// Moves unary ops that depend on a select before the select, if both inputs are constants
// assumes that unary ops without side effects on constants will be constprop'd
void ConstProp::CodeMotionAroundSelects(IREmitter *IREmit, const IRListView& CurrentIR) {
// Code motion around selects
// Moves unary ops that depend on a select before the select, if both inputs are constants
// assumes that unary ops without side effects on constants will be constprop'd
@@ -257,9 +243,9 @@ void ConstProp::CodeMotionAroundSelects(IREmitter *IREmit, const IRListView& Cur
}
}
}
}
void ConstProp::FCMPOptimization(IREmitter *IREmit, const IRListView& CurrentIR) {
// FCMP optimization
// Make all FCMPs set no flags
for (auto [CodeNode, IROp] : CurrentIR.GetAllCode()) {
if (IROp->Op == OP_FCMP) {
@@ -274,17 +260,16 @@ void ConstProp::FCMPOptimization(IREmitter *IREmit, const IRListView& CurrentIR)
auto ghf = IROp->CW<IR::IROp_GetHostFlag>();
auto fcmp = IREmit->GetOpHeader(ghf->GPR)->CW<IR::IROp_FCmp>();
LOGMAN_THROW_A(fcmp->Header.Op == OP_FCMP || fcmp->Header.Op == OP_F80CMP, "Unexpected OP_GETHOSTFLAG source");
LogMan::Throw::A(fcmp->Header.Op == OP_FCMP || fcmp->Header.Op == OP_F80CMP, "Unexpected OP_GETHOSTFLAG source");
if(fcmp->Header.Op == OP_FCMP) {
fcmp->Flags |= 1 << ghf->Flag;
}
}
}
}
// LoadMem / StoreMem imm pooling
// If imms are close by, use address gen to generate the values instead of using a new imm
void ConstProp::LoadMemStoreMemImmediatePooling(IREmitter *IREmit, const IRListView& CurrentIR) {
// LoadMem / StoreMem imm pooling
// If imms are close by, use address gen to generate the values instead of using a new imm
for (auto [BlockNode, BlockIROp] : CurrentIR.GetBlocks()) {
for (auto [CodeNode, IROp] : CurrentIR.GetCode(BlockNode)) {
if (IROp->Op == OP_LOADMEM || IROp->Op == OP_STOREMEM) {
@@ -308,163 +293,152 @@ void ConstProp::LoadMemStoreMemImmediatePooling(IREmitter *IREmit, const IRListV
}
AddressgenConsts.clear();
}
}
bool ConstProp::ZextAndMaskingElimination(IREmitter *IREmit, const IRListView& CurrentIR,
OrderedNode* CodeNode, IROp_Header* IROp) {
bool Changed = false;
switch (IROp->Op) {
// Generic handling
case OP_OR:
case OP_XOR:
case OP_NOT:
case OP_ADD:
case OP_SUB:
case OP_MUL:
case OP_UMUL:
case OP_DIV:
case OP_UDIV:
case OP_LSHR:
case OP_ASHR:
case OP_LSHL:
case OP_ROR: {
for (int i = 0; i < IROp->NumArgs; i++) {
auto newArg = RemoveUselessMasking(IREmit, IROp->Args[i], getMask(IROp));
if (newArg.ID() != IROp->Args[i].ID()) {
IREmit->ReplaceNodeArgument(CodeNode, i, IREmit->UnwrapNode(newArg));
Changed = true;
for (auto [CodeNode, IROp] : CurrentIR.GetAllCode()) {
// zext / masking elimination
switch (IROp->Op) {
// Generic handling
case OP_OR:
case OP_XOR:
case OP_NOT:
case OP_ADD:
case OP_SUB:
case OP_MUL:
case OP_UMUL:
case OP_DIV:
case OP_UDIV:
case OP_LSHR:
case OP_ASHR:
case OP_LSHL:
case OP_ROR: {
for (int i = 0; i < IROp->NumArgs; i++) {
auto newArg = RemoveUselessMasking(IREmit, IROp->Args[i], getMask(IROp));
if (newArg.ID() != IROp->Args[i].ID()) {
IREmit->ReplaceNodeArgument(CodeNode, i, IREmit->UnwrapNode(newArg));
Changed = true;
}
}
}
break;
}
case OP_AND: {
// if AND's arguments are imms, they are masking
for (int i = 0; i < IROp->NumArgs; i++) {
auto mask = getMask(IROp);
uint64_t imm = 0;
if (IREmit->IsValueConstant(IROp->Args[i^1], &imm))
mask = imm;
auto newArg = RemoveUselessMasking(IREmit, IROp->Args[i], imm);
if (newArg.ID() != IROp->Args[i].ID()) {
IREmit->ReplaceNodeArgument(CodeNode, i, IREmit->UnwrapNode(newArg));
Changed = true;
}
}
break;
}
case OP_BFE: {
auto Op = IROp->C<IR::IROp_Bfe>();
// Is this value already BFE'd?
if (IsBfeAlreadyDone(IREmit, IROp->Args[0], Op->Width)) {
IREmit->ReplaceAllUsesWith(CodeNode, CurrentIR.GetNode(IROp->Args[0]));
//printf("Removed BFE once \n");
break;
}
// Is this value already ZEXT'd?
if (Op->lsb == 0) {
//LoadMem, LoadMemTSO & LoadContext ZExt
case OP_AND: {
// if AND's arguments are imms, they are masking
for (int i = 0; i < IROp->NumArgs; i++) {
auto mask = getMask(IROp);
uint64_t imm = 0;
if (IREmit->IsValueConstant(IROp->Args[i^1], &imm))
mask = imm;
auto newArg = RemoveUselessMasking(IREmit, IROp->Args[i], imm);
if (newArg.ID() != IROp->Args[i].ID()) {
IREmit->ReplaceNodeArgument(CodeNode, i, IREmit->UnwrapNode(newArg));
Changed = true;
}
}
break;
}
case OP_BFE: {
auto Op = IROp->C<IR::IROp_Bfe>();
// Is this value already BFE'd?
if (IsBfeAlreadyDone(IREmit, IROp->Args[0], Op->Width)) {
IREmit->ReplaceAllUsesWith(CodeNode, CurrentIR.GetNode(IROp->Args[0]));
//printf("Removed BFE once \n");
break;
}
// Is this value already ZEXT'd?
if (Op->lsb == 0) {
//LoadMem, LoadMemTSO & LoadContext ZExt
auto source = IROp->Args[0];
auto sourceHeader = IREmit->GetOpHeader(source);
if (Op->Width >= (sourceHeader->Size*8) &&
(sourceHeader->Op == OP_LOADMEM || sourceHeader->Op == OP_LOADMEMTSO || sourceHeader->Op == OP_LOADCONTEXT)
) {
//printf("Eliminated needless zext bfe\n");
// Load mem / load ctx zexts, no need to vmem
IREmit->ReplaceAllUsesWith(CodeNode, CurrentIR.GetNode(source));
break;
}
}
// BFE does implicit masking, remove any masks leading to this, if possible
uint64_t imm = 1ULL << (Op->Width-1);
imm = (imm-1) *2 + 1;
imm <<= Op->lsb;
auto newArg = RemoveUselessMasking(IREmit, IROp->Args[0], imm);
if (newArg.ID() != IROp->Args[0].ID()) {
IREmit->ReplaceNodeArgument(CodeNode, 0, IREmit->UnwrapNode(newArg));
Changed = true;
}
break;
}
case OP_SBFE: {
auto Op = IROp->C<IR::IROp_Sbfe>();
// BFE does implicit masking
uint64_t imm = 1ULL << (Op->Width-1);
imm = (imm-1) *2 + 1;
imm <<= Op->lsb;
auto newArg = RemoveUselessMasking(IREmit, IROp->Args[0], imm);
if (newArg.ID() != IROp->Args[0].ID()) {
IREmit->ReplaceNodeArgument(CodeNode, 0, IREmit->UnwrapNode(newArg));
Changed = true;
}
break;
}
case OP_VFADD:
case OP_VFSUB:
case OP_VFMUL:
case OP_VFDIV:
case OP_FCMP: {
auto flopSize = IROp->Size;
for (int i = 0; i < IROp->NumArgs; i++) {
auto argHeader = IREmit->GetOpHeader(IROp->Args[i]);
if (argHeader->Op == OP_VMOV) {
auto source = argHeader->Args[0];
auto sourceHeader = IREmit->GetOpHeader(source);
if (sourceHeader->Size >= flopSize) {
IREmit->ReplaceNodeArgument(CodeNode, i, IREmit->UnwrapNode(source));
//printf("VMOV bypassed\n");
}
}
}
break;
}
case OP_VMOV: {
// elim from load mem
auto source = IROp->Args[0];
auto sourceHeader = IREmit->GetOpHeader(source);
if (Op->Width >= (sourceHeader->Size*8) &&
if (IROp->Size >= sourceHeader->Size &&
(sourceHeader->Op == OP_LOADMEM || sourceHeader->Op == OP_LOADMEMTSO || sourceHeader->Op == OP_LOADCONTEXT)
) {
//printf("Eliminated needless zext bfe\n");
) {
//printf("Eliminated needless zext VMOV\n");
// Load mem / load ctx zexts, no need to vmem
IREmit->ReplaceAllUsesWith(CodeNode, CurrentIR.GetNode(source));
break;
} else if (IROp->Size == sourceHeader->Size) {
// VMOV of same size
//printf("printf vmov of same size?!\n");
IREmit->ReplaceAllUsesWith(CodeNode, CurrentIR.GetNode(source));
}
break;
}
// BFE does implicit masking, remove any masks leading to this, if possible
uint64_t imm = 1ULL << (Op->Width-1);
imm = (imm-1) *2 + 1;
imm <<= Op->lsb;
auto newArg = RemoveUselessMasking(IREmit, IROp->Args[0], imm);
if (newArg.ID() != IROp->Args[0].ID()) {
IREmit->ReplaceNodeArgument(CodeNode, 0, IREmit->UnwrapNode(newArg));
Changed = true;
}
break;
default: break;
}
case OP_SBFE: {
auto Op = IROp->C<IR::IROp_Sbfe>();
// BFE does implicit masking
uint64_t imm = 1ULL << (Op->Width-1);
imm = (imm-1) *2 + 1;
imm <<= Op->lsb;
auto newArg = RemoveUselessMasking(IREmit, IROp->Args[0], imm);
if (newArg.ID() != IROp->Args[0].ID()) {
IREmit->ReplaceNodeArgument(CodeNode, 0, IREmit->UnwrapNode(newArg));
Changed = true;
}
break;
}
case OP_VFADD:
case OP_VFSUB:
case OP_VFMUL:
case OP_VFDIV:
case OP_FCMP: {
auto flopSize = IROp->Size;
for (int i = 0; i < IROp->NumArgs; i++) {
auto argHeader = IREmit->GetOpHeader(IROp->Args[i]);
if (argHeader->Op == OP_VMOV) {
auto source = argHeader->Args[0];
auto sourceHeader = IREmit->GetOpHeader(source);
if (sourceHeader->Size >= flopSize) {
IREmit->ReplaceNodeArgument(CodeNode, i, IREmit->UnwrapNode(source));
//printf("VMOV bypassed\n");
}
}
}
break;
}
case OP_VMOV: {
// elim from load mem
auto source = IROp->Args[0];
auto sourceHeader = IREmit->GetOpHeader(source);
if (IROp->Size >= sourceHeader->Size &&
(sourceHeader->Op == OP_LOADMEM || sourceHeader->Op == OP_LOADMEMTSO || sourceHeader->Op == OP_LOADCONTEXT)
) {
//printf("Eliminated needless zext VMOV\n");
// Load mem / load ctx zexts, no need to vmem
IREmit->ReplaceAllUsesWith(CodeNode, CurrentIR.GetNode(source));
} else if (IROp->Size == sourceHeader->Size) {
// VMOV of same size
//printf("printf vmov of same size?!\n");
IREmit->ReplaceAllUsesWith(CodeNode, CurrentIR.GetNode(source));
}
break;
}
default:
break;
}
return Changed;
}
// constprop + some more per instruction logic
bool ConstProp::ConstantPropagation(IREmitter *IREmit, const IRListView& CurrentIR,
OrderedNode* CodeNode, IROp_Header* IROp) {
bool Changed = false;
// constprop + some more per instruction logic
switch (IROp->Op) {
/*
case OP_UMUL:
@@ -489,7 +463,7 @@ bool ConstProp::ConstantPropagation(IREmitter *IREmit, const IRListView& Current
if (IREmit->IsValueConstant(IROp->Args[0], &Constant1) &&
IREmit->IsValueConstant(IROp->Args[1], &Constant2)) {
LOGMAN_MSG_A("Could const prop op: %s", std::string(IR::GetName(IROp->Op)).c_str());
LogMan::Msg::A("Could const prop op: %s", std::string(IR::GetName(IROp->Op)).c_str());
}
break;
}
@@ -505,7 +479,7 @@ bool ConstProp::ConstantPropagation(IREmitter *IREmit, const IRListView& Current
uint64_t Constant1;
if (IREmit->IsValueConstant(IROp->Args[0], &Constant1)) {
LOGMAN_MSG_A("Could const prop op: %s", std::string(IR::GetName(IROp->Op)).c_str());
LogMan::Msg::A("Could const prop op: %s", std::string(IR::GetName(IROp->Op)).c_str());
}
break;
}
@@ -516,6 +490,7 @@ bool ConstProp::ConstantPropagation(IREmitter *IREmit, const IRListView& Current
auto AddressHeader = IREmit->GetOpHeader(Op->Header.Args[0]);
if (AddressHeader->Op == OP_ADD && AddressHeader->Size == 8) {
auto [OffsetType, OffsetScale, Arg0, Arg1] = MemExtendedAddressing(IREmit, Op->Size, AddressHeader);
Op->OffsetType = OffsetType;
@@ -555,6 +530,7 @@ bool ConstProp::ConstantPropagation(IREmitter *IREmit, const IRListView& Current
uint64_t NewConstant = (Constant1 + Constant2) & getMask(Op) ;
IREmit->ReplaceWithConstant(CodeNode, NewConstant);
Changed = true;
continue;
}
break;
}
@@ -735,9 +711,9 @@ bool ConstProp::ConstantPropagation(IREmitter *IREmit, const IRListView& Current
uint64_t NewConstant = (Constant1 * Constant2) & getMask(Op);
IREmit->ReplaceWithConstant(CodeNode, NewConstant);
Changed = true;
} else if (IREmit->IsValueConstant(Op->Header.Args[1], &Constant2) && std::popcount(Constant2) == 1) {
} else if (IREmit->IsValueConstant(Op->Header.Args[1], &Constant2) && __builtin_popcountl(Constant2) == 1) {
if (IROp->Size == 4 || IROp->Size == 8) {
uint64_t amt = std::countr_zero(Constant2);
uint64_t amt = __builtin_ctzl(Constant2);
IREmit->SetWriteCursor(CodeNode);
auto shift = IREmit->_Lshl(CurrentIR.GetNode(Op->Header.Args[0]), IREmit->_Constant(amt));
shift.first->Header.Size = IROp->Size; // force Lshl to be the same size as the original Mul
@@ -777,218 +753,187 @@ bool ConstProp::ConstantPropagation(IREmitter *IREmit, const IRListView& Current
default:
break;
}
}
return Changed;
}
bool ConstProp::ConstantInlining(IREmitter *IREmit, const IRListView& CurrentIR) {
bool Changed = false;
for (auto [CodeNode, IROp] : CurrentIR.GetAllCode()) {
switch(IROp->Op) {
case OP_LSHR:
case OP_ASHR:
case OP_ROR:
case OP_LSHL:
{
auto Op = IROp->C<IR::IROp_Lshr>();
uint64_t Constant2{};
if (IREmit->IsValueConstant(Op->Header.Args[1], &Constant2)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Header.Args[1]));
// this shouldn't be here, but rather on the emitter themselves or the constprop transformation?
if (IROp->Size <=4)
Constant2 &= 31;
else
Constant2 &= 63;
IREmit->ReplaceNodeArgument(CodeNode, 1, IREmit->_InlineConstant(Constant2));
Changed = true;
}
break;
}
case OP_ADD:
case OP_SUB:
{
auto Op = IROp->C<IR::IROp_Add>();
uint64_t Constant2{};
if (IREmit->IsValueConstant(Op->Header.Args[1], &Constant2)) {
if (IsImmAddSub(Constant2)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Header.Args[1]));
IREmit->ReplaceNodeArgument(CodeNode, 1, IREmit->_InlineConstant(Constant2));
Changed = true;
}
}
break;
}
case OP_SELECT:
{
auto Op = IROp->C<IR::IROp_Select>();
uint64_t Constant1{};
if (IREmit->IsValueConstant(Op->Header.Args[1], &Constant1)) {
if (IsImmAddSub(Constant1)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Header.Args[1]));
IREmit->ReplaceNodeArgument(CodeNode, 1, IREmit->_InlineConstant(Constant1));
Changed = true;
}
}
uint64_t Constant2{};
uint64_t Constant3{};
if (IREmit->IsValueConstant(Op->Header.Args[2], &Constant2) &&
IREmit->IsValueConstant(Op->Header.Args[3], &Constant3) &&
Constant2 == 1 &&
Constant3 == 0)
// constant inlining
if (InlineConstants) {
for (auto [CodeNode, IROp] : CurrentIR.GetAllCode()) {
switch(IROp->Op) {
case OP_LSHR:
case OP_ASHR:
case OP_ROR:
case OP_LSHL:
{
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Header.Args[2]));
auto Op = IROp->C<IR::IROp_Lshr>();
IREmit->ReplaceNodeArgument(CodeNode, 2, IREmit->_InlineConstant(Constant2));
IREmit->ReplaceNodeArgument(CodeNode, 3, IREmit->_InlineConstant(Constant3));
}
break;
}
case OP_CONDJUMP:
{
auto Op = IROp->C<IR::IROp_CondJump>();
uint64_t Constant2{};
if (IREmit->IsValueConstant(Op->Header.Args[1], &Constant2)) {
if (IsImmAddSub(Constant2)) {
uint64_t Constant2{};
if (IREmit->IsValueConstant(Op->Header.Args[1], &Constant2)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Header.Args[1]));
// this shouldn't be here, but rather on the emitter themselves or the constprop transformation?
if (IROp->Size <=4)
Constant2 &= 31;
else
Constant2 &= 63;
IREmit->ReplaceNodeArgument(CodeNode, 1, IREmit->_InlineConstant(Constant2));
Changed = true;
}
break;
}
break;
}
case OP_EXITFUNCTION:
{
auto Op = IROp->C<IR::IROp_ExitFunction>();
case OP_ADD:
case OP_SUB:
{
auto Op = IROp->C<IR::IROp_Add>();
uint64_t Constant{};
if (IREmit->IsValueConstant(Op->NewRIP, &Constant)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->NewRIP));
uint64_t Constant2{};
if (IREmit->IsValueConstant(Op->Header.Args[1], &Constant2)) {
if (IsImmAddSub(Constant2)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Header.Args[1]));
IREmit->ReplaceNodeArgument(CodeNode, 0, IREmit->_InlineConstant(Constant));
IREmit->ReplaceNodeArgument(CodeNode, 1, IREmit->_InlineConstant(Constant2));
Changed = true;
} else {
auto NewRIP = IREmit->GetOpHeader(Op->NewRIP);
if (NewRIP->Op == OP_ENTRYPOINTOFFSET) {
auto EO = NewRIP->C<IR::IROp_EntrypointOffset>();
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->NewRIP));
IREmit->ReplaceNodeArgument(CodeNode, 0, IREmit->_InlineEntrypointOffset(EO->Offset, EO->Header.Size));
Changed = true;
Changed = true;
}
}
break;
}
break;
}
case OP_OR:
case OP_XOR:
case OP_AND:
{
auto Op = IROp->CW<IR::IROp_Or>();
case OP_SELECT:
{
auto Op = IROp->C<IR::IROp_Select>();
uint64_t Constant2{};
if (IREmit->IsValueConstant(Op->Header.Args[1], &Constant2)) {
if (IsImmLogical(Constant2, IROp->Size * 8)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Header.Args[1]));
uint64_t Constant1{};
if (IREmit->IsValueConstant(Op->Header.Args[1], &Constant1)) {
if (IsImmAddSub(Constant1)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Header.Args[1]));
IREmit->ReplaceNodeArgument(CodeNode, 1, IREmit->_InlineConstant(Constant2));
IREmit->ReplaceNodeArgument(CodeNode, 1, IREmit->_InlineConstant(Constant1));
Changed = true;
Changed = true;
}
}
}
break;
}
case OP_LOADMEM:
{
auto Op = IROp->CW<IR::IROp_LoadMem>();
uint64_t Constant2{};
if (Op->OffsetType == MEM_OFFSET_SXTX && IREmit->IsValueConstant(Op->Header.Args[1], &Constant2)) {
if (IsImmMemory(Constant2, Op->Size)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Header.Args[1]));
IREmit->ReplaceNodeArgument(CodeNode, 1, IREmit->_InlineConstant(Constant2));
Changed = true;
}
}
break;
}
case OP_STOREMEM:
{
auto Op = IROp->CW<IR::IROp_StoreMem>();
uint64_t Constant2{};
if (Op->OffsetType == MEM_OFFSET_SXTX && IREmit->IsValueConstant(Op->Header.Args[2], &Constant2)) {
if (IsImmMemory(Constant2, Op->Size)) {
uint64_t Constant2{};
uint64_t Constant3{};
if (IREmit->IsValueConstant(Op->Header.Args[2], &Constant2) &&
IREmit->IsValueConstant(Op->Header.Args[3], &Constant3) &&
Constant2 == 1 &&
Constant3 == 0)
{
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Header.Args[2]));
IREmit->ReplaceNodeArgument(CodeNode, 2, IREmit->_InlineConstant(Constant2));
IREmit->ReplaceNodeArgument(CodeNode, 3, IREmit->_InlineConstant(Constant3));
}
break;
}
case OP_CONDJUMP:
{
auto Op = IROp->C<IR::IROp_CondJump>();
uint64_t Constant2{};
if (IREmit->IsValueConstant(Op->Header.Args[1], &Constant2)) {
if (IsImmAddSub(Constant2)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Header.Args[1]));
IREmit->ReplaceNodeArgument(CodeNode, 1, IREmit->_InlineConstant(Constant2));
Changed = true;
}
}
break;
}
case OP_EXITFUNCTION:
{
auto Op = IROp->C<IR::IROp_ExitFunction>();
uint64_t Constant{};
if (IREmit->IsValueConstant(Op->NewRIP, &Constant)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->NewRIP));
IREmit->ReplaceNodeArgument(CodeNode, 0, IREmit->_InlineConstant(Constant));
Changed = true;
} else {
auto NewRIP = IREmit->GetOpHeader(Op->NewRIP);
if (NewRIP->Op == OP_ENTRYPOINTOFFSET) {
auto EO = NewRIP->C<IR::IROp_EntrypointOffset>();
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->NewRIP));
IREmit->ReplaceNodeArgument(CodeNode, 0, IREmit->_InlineEntrypointOffset(EO->Offset, EO->Header.Size));
Changed = true;
}
}
break;
}
break;
case OP_OR:
case OP_XOR:
case OP_AND:
{
auto Op = IROp->CW<IR::IROp_Or>();
uint64_t Constant2{};
if (IREmit->IsValueConstant(Op->Header.Args[1], &Constant2)) {
if (IsImmLogical(Constant2, IROp->Size * 8)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Header.Args[1]));
IREmit->ReplaceNodeArgument(CodeNode, 1, IREmit->_InlineConstant(Constant2));
Changed = true;
}
}
break;
}
case OP_LOADMEM:
{
auto Op = IROp->CW<IR::IROp_LoadMem>();
uint64_t Constant2{};
if (Op->OffsetType == MEM_OFFSET_SXTX && IREmit->IsValueConstant(Op->Header.Args[1], &Constant2)) {
if (IsImmMemory(Constant2, Op->Size)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Header.Args[1]));
IREmit->ReplaceNodeArgument(CodeNode, 1, IREmit->_InlineConstant(Constant2));
Changed = true;
}
}
break;
}
case OP_STOREMEM:
{
auto Op = IROp->CW<IR::IROp_StoreMem>();
uint64_t Constant2{};
if (Op->OffsetType == MEM_OFFSET_SXTX && IREmit->IsValueConstant(Op->Header.Args[2], &Constant2)) {
if (IsImmMemory(Constant2, Op->Size)) {
IREmit->SetWriteCursor(CurrentIR.GetNode(Op->Header.Args[2]));
IREmit->ReplaceNodeArgument(CodeNode, 2, IREmit->_InlineConstant(Constant2));
Changed = true;
}
}
break;
}
default: break;
}
default:
break;
}
}
return Changed;
}
bool ConstProp::Run(IREmitter *IREmit) {
bool Changed = false;
auto CurrentIR = IREmit->ViewIR();
auto OriginalWriteCursor = IREmit->GetWriteCursor();
if (HandleConstantPools(IREmit, CurrentIR)) {
Changed = true;
}
CodeMotionAroundSelects(IREmit, CurrentIR);
FCMPOptimization(IREmit, CurrentIR);
LoadMemStoreMemImmediatePooling(IREmit, CurrentIR);
for (auto [CodeNode, IROp] : CurrentIR.GetAllCode()) {
if (ZextAndMaskingElimination(IREmit, CurrentIR, CodeNode, IROp)) {
Changed = true;
}
if (ConstantPropagation(IREmit, CurrentIR, CodeNode, IROp)) {
Changed = true;
}
}
if (InlineConstants && ConstantInlining(IREmit, CurrentIR)) {
Changed = true;
}
IREmit->SetWriteCursor(OriginalWriteCursor);
return Changed;
}
@@ -211,18 +211,18 @@ namespace {
size_t ClassifiedStructSize{};
ContextClassificationInfo->Lookup.reserve(sizeof(FEXCore::Core::CPUState));
for (auto &it : *ContextClassification) {
LOGMAN_THROW_A(it.Class.Offset == ContextClassificationInfo->Lookup.size(), "Offset missmatch %d %d", it.Class.Offset == ContextClassificationInfo->Lookup.size());
LogMan::Throw::A(it.Class.Offset == ContextClassificationInfo->Lookup.size(), "Offset missmatch %d %d", it.Class.Offset == ContextClassificationInfo->Lookup.size());
for (int i = 0; i < it.Class.Size; i++) {
ContextClassificationInfo->Lookup.push_back(&it);
}
ClassifiedStructSize += it.Class.Size;
}
LOGMAN_THROW_A(ClassifiedStructSize == sizeof(FEXCore::Core::CPUState),
LogMan::Throw::A(ClassifiedStructSize == sizeof(FEXCore::Core::CPUState),
"Classified CPUStruct size doesn't match real CPUState struct size! %ld != %ld",
ClassifiedStructSize, sizeof(FEXCore::Core::CPUState));
LOGMAN_THROW_A(ContextClassificationInfo->Lookup.size() == sizeof(FEXCore::Core::CPUState),
LogMan::Throw::A(ContextClassificationInfo->Lookup.size() == sizeof(FEXCore::Core::CPUState),
"Classified CPUStruct size doesn't match real CPUState struct size! %ld != %ld",
ContextClassificationInfo->Lookup.size(), sizeof(FEXCore::Core::CPUState));
}
@@ -306,15 +306,15 @@ ContextMemberInfo *RCLSE::FindMemberInfo(ContextInfo *ContextClassificationInfo,
}
ContextMemberInfo *RCLSE::RecordAccess(ContextMemberInfo *Info, FEXCore::IR::RegisterClassType RegClass, uint32_t Offset, uint8_t Size, LastAccessType AccessType, FEXCore::IR::OrderedNode *Node, FEXCore::IR::OrderedNode *StoreNode) {
LOGMAN_THROW_A((Offset + Size) <= (Info->Class.Offset + Info->Class.Size), "Access to context item went over member size");
LOGMAN_THROW_A(Info->Accessed != ACCESS_INVALID, "Tried to access invalid member");
LogMan::Throw::A((Offset + Size) <= (Info->Class.Offset + Info->Class.Size), "Access to context item went over member size");
LogMan::Throw::A(Info->Accessed != ACCESS_INVALID, "Tried to access invalid member");
// If we aren't fully overwriting the member then it is a partial write that we need to track
if (Size < Info->Class.Size) {
AccessType = AccessType == ACCESS_WRITE ? ACCESS_PARTIAL_WRITE : ACCESS_PARTIAL_READ;
}
if (Size > Info->Class.Size) {
LOGMAN_MSG_A("Can't handle this");
LogMan::Msg::A("Can't handle this");
}
Info->Accessed = AccessType;
@@ -500,7 +500,7 @@ bool RCLSE::RedundantStoreLoadElimination(FEXCore::IR::IREmitter *IREmit) {
else if (LastClass == GPRClass) {
LastNode = IREmit->_Bfe(Info->AccessSize, TruncateSize * 8, 0, LastNode);
} else {
LOGMAN_MSG_A("Unhandled Register class");
LogMan::Msg::A("Unhandled Register class");
}
}
@@ -578,7 +578,7 @@ bool RCLSE::RedundantStoreLoadElimination(FEXCore::IR::IREmitter *IREmit) {
if (LastStoreNode != nullptr)
{
IREmit->SetWriteCursor(CodeNode);
RecordAccess(&LocalInfo, FEXCore::IR::GPRClass, offsetof(FEXCore::Core::CPUState, flags[0]) + F, 1, ACCESS_WRITE, IREmit->_Constant(0), CodeNode);
RecordAccess(&LocalInfo, FEXCore::IR::GPRClass, offsetof(FEXCore::Core::CPUState, flags[0]) + F, 1, ACCESS_WRITE, IREmit->_InlineConstant(0), CodeNode);
IREmit->Remove(LastStoreNode);
Changed = true;
@@ -106,7 +106,7 @@ uint64_t FPRBit(uint32_t Offset, uint32_t Size) {
else if (Size == 4)
return 1UL << (bitn);
else
LOGMAN_MSG_A("Unexpected FPR size %d", Size);
LogMan::Msg::A("Unexpected FPR size %d", Size);
return 7UL << (bitn); // Return maximum on failure case
}
@@ -66,7 +66,7 @@ bool IRCompaction::Run(IREmitter *IREmit) {
auto HeaderNode = CurrentIR.GetHeaderNode();
auto HeaderOp = CurrentIR.GetHeader();
LOGMAN_THROW_A(HeaderOp->Header.Op == OP_IRHEADER, "First op wasn't IRHeader");
LogMan::Throw::A(HeaderOp->Header.Op == OP_IRHEADER, "First op wasn't IRHeader");
// This compaction pass is something that we need to ensure correct ordering and distances between IROps
// Later on we assume that an IROp's SSA value live range is its Node locations
@@ -84,13 +84,13 @@ bool IRCompaction::Run(IREmitter *IREmit) {
// Zero is always zero(invalid)
OldToNewRemap[0].NodeID = 0;
auto LocalHeaderOp = LocalBuilder._IRHeader(OrderedNodeWrapper::WrapOffset(0).GetNode(ListBegin), HeaderOp->BlockCount);
auto LocalHeaderOp = LocalBuilder._IRHeader(OrderedNodeWrapper::WrapOffset(0).GetNode(ListBegin), HeaderOp->Entry, HeaderOp->BlockCount);
OldToNewRemap[CurrentIR.GetID(HeaderNode)].NodeID = LocalIR.GetID(LocalHeaderOp.Node);
{
// Generate our codeblocks and link them together
for (auto [BlockNode, BlockHeader] : CurrentIR.GetBlocks()) {
LOGMAN_THROW_A(BlockHeader->Op == OP_CODEBLOCK, "IR type failed to be a code block");
LogMan::Throw::A(BlockHeader->Op == OP_CODEBLOCK, "IR type failed to be a code block");
auto LocalBlockIRNode = LocalBuilder._CodeBlock(LocalHeaderOp, LocalHeaderOp); // Use LocalHeaderOp as a dummy arg for now
OldToNewRemap[CurrentIR.GetID(BlockNode)].NodeID = LocalIR.GetID(LocalBlockIRNode.Node);
@@ -154,7 +154,7 @@ bool IRCompaction::Run(IREmitter *IREmit) {
// Fixup the arguments of all the IROps
for (auto &Block : GeneratedCodeBlocks) {
auto BlockIROp = LocalIR.GetOp<FEXCore::IR::IROp_CodeBlock>(Block.NewNode);
LOGMAN_THROW_A(BlockIROp->Header.Op == OP_CODEBLOCK, "IR type failed to be a code block");
LogMan::Throw::A(BlockIROp->Header.Op == OP_CODEBLOCK, "IR type failed to be a code block");
for (auto [LocalNode, LocalIROp] : LocalIR.GetCode(Block.NewNode)) {
@@ -165,7 +165,7 @@ bool IRCompaction::Run(IREmitter *IREmit) {
for (uint8_t i = 0; i < NumArgs; ++i) {
uint32_t OldArg = LocalIROp->Args[i].ID();
#ifndef NDEBUG
LOGMAN_THROW_A(OldToNewRemap[OldArg].NodeID != ~0U, "Tried remapping unfound node %%ssa%d", OldArg);
LogMan::Throw::A(OldToNewRemap[OldArg].NodeID != ~0U, "Tried remapping unfound node %%ssa%d", OldArg);
#endif
LocalIROp->Args[i].NodeOffset = OldToNewRemap[OldArg].NodeID * sizeof(OrderedNode);
}
@@ -191,7 +191,7 @@ bool IRCompaction::Run(IREmitter *IREmit) {
// if (NewListSize > OldListSize ||
// NewDataSize > OldDataSize) {
// LOGMAN_MSG_A("Whoa. Compaction made the IR a different size when it shouldn't have. 0x%lx > 0x%lx or 0x%lx > 0x%lx",NewListSize, OldListSize, NewDataSize, OldDataSize);
// LogMan::Msg::A("Whoa. Compaction made the IR a different size when it shouldn't have. 0x%lx > 0x%lx or 0x%lx > 0x%lx",NewListSize, OldListSize, NewDataSize, OldDataSize);
// }
IREmit->CopyData(LocalBuilder);
@@ -55,7 +55,7 @@ bool IRValidation::Run(IREmitter *IREmit) {
std::vector<uint32_t> Uses(CurrentIR.GetSSACount(), 0);
auto HeaderOp = CurrentIR.GetHeader();
LOGMAN_THROW_A(HeaderOp->Header.Op == OP_IRHEADER, "First op wasn't IRHeader");
LogMan::Throw::A(HeaderOp->Header.Op == OP_IRHEADER, "First op wasn't IRHeader");
IR::RegisterAllocationData * RAData{};
if (Manager->HasRAPass()) {
@@ -66,7 +66,7 @@ bool IRValidation::Run(IREmitter *IREmit) {
for (auto [BlockNode, BlockHeader] : CurrentIR.GetBlocks()) {
auto BlockIROp = BlockHeader->CW<FEXCore::IR::IROp_CodeBlock>();
LOGMAN_THROW_A(BlockIROp->Header.Op == OP_CODEBLOCK, "IR type failed to be a code block");
LogMan::Throw::A(BlockIROp->Header.Op == OP_CODEBLOCK, "IR type failed to be a code block");
uint32_t BlockID = CurrentIR.GetID(BlockNode);
@@ -209,7 +209,7 @@ bool IRValidation::Run(IREmitter *IREmit) {
break;
}
default:
// LOGMAN_MSG_A("Unknown IR Op: %d(%s)", IROp->Op, FEXCore::IR::GetName(IROp->Op).data());
// LogMan::Msg::A("Unknown IR Op: %d(%s)", IROp->Op, FEXCore::IR::GetName(IROp->Op).data());
break;
}
}
@@ -1,112 +0,0 @@
/*
$info$
tags: ir|opts
desc: Long divide elimination pass
$end_info$
*/
#include "Interface/IR/PassManager.h"
#include <FEXCore/Utils/LogManager.h>
namespace FEXCore::IR {
class LongDivideEliminationPass final : public FEXCore::IR::Pass {
public:
bool Run(IREmitter *IREmit) override;
private:
bool IsZeroOp(IREmitter *IREmit, OrderedNodeWrapper Arg);
bool IsSextOp(IREmitter *IREmit, OrderedNodeWrapper Lower, OrderedNodeWrapper Upper);
};
bool LongDivideEliminationPass::IsZeroOp(IREmitter *IREmit, OrderedNodeWrapper Arg) {
auto IROp = IREmit->GetOpHeader(Arg);
uint64_t Value;
// XOR based zero
if (IROp->Op == OP_XOR) {
return IROp->Args[0] == IROp->Args[1];
}
else if (IREmit->IsValueConstant(Arg, &Value)) {
// Zero constant based zero op
return Value == 0;
}
return false;
}
bool LongDivideEliminationPass::IsSextOp(IREmitter *IREmit, OrderedNodeWrapper Lower, OrderedNodeWrapper Upper) {
// We need to check if the upper source is a sext of the lower source
auto UpperIROp = IREmit->GetOpHeader(Upper);
if (UpperIROp->Op == OP_SBFE) {
auto Op = UpperIROp->C<IR::IROp_Sbfe>();
if (Op->Width == 1 && Op->lsb == 63) {
// CQO: OrderedNode *Upper = _Sbfe(1, Size * 8 - 1, Src);
// If the lower is the upper in this case then it can be optimized
return Op->Header.Args[0] == Lower;
}
}
return false;
}
bool LongDivideEliminationPass::Run(IREmitter *IREmit) {
bool Changed = false;
auto CurrentIR = IREmit->ViewIR();
auto OriginalWriteCursor = IREmit->GetWriteCursor();
for (auto [BlockNode, BlockHeader] : CurrentIR.GetBlocks()) {
for (auto [CodeNode, IROp] : CurrentIR.GetCode(BlockNode)) {
if (IROp->Size == 8) {
if (IROp->Op == OP_LDIV ||
IROp->Op == OP_LREM) {
auto Op = IROp->C<IR::IROp_LDiv>();
// Check upper Op to see if it came from a CQO
// CQO: OrderedNode *Upper = _Sbfe(1, Size * 8 - 1, Src);
// If it does then it we only need a 64bit SDIV
if (IsSextOp(IREmit, Op->Lower, Op->Upper)) {
IREmit->SetWriteCursor(CodeNode);
OrderedNode *Lower = CurrentIR.GetNode(Op->Lower);
OrderedNode *Divisor = CurrentIR.GetNode(Op->Divisor);
OrderedNode *SDivOp{};
if (IROp->Op == OP_LDIV) {
SDivOp = IREmit->_Div(Lower, Divisor);
}
else {
SDivOp = IREmit->_Rem(Lower, Divisor);
}
IREmit->ReplaceAllUsesWith(CodeNode, SDivOp);
Changed = true;
}
}
else if (IROp->Op == OP_LUDIV ||
IROp->Op == OP_LUREM) {
auto Op = IROp->C<IR::IROp_LUDiv>();
// Check upper Op to see if it came from a xor zeroing op
// XOR: Result = _Xor(Dest, Src);
// If it does then it we only need a 64bit UDIV
if (IsZeroOp(IREmit, Op->Upper)) {
IREmit->SetWriteCursor(CodeNode);
OrderedNode *Lower = CurrentIR.GetNode(Op->Lower);
OrderedNode *Divisor = CurrentIR.GetNode(Op->Divisor);
OrderedNode *UDivOp{};
if (IROp->Op == OP_LUDIV) {
UDivOp = IREmit->_UDiv(Lower, Divisor);
}
else {
UDivOp = IREmit->_URem(Lower, Divisor);
}
IREmit->ReplaceAllUsesWith(CodeNode, UDivOp);
Changed = true;
}
}
}
}
}
IREmit->SetWriteCursor(OriginalWriteCursor);
return Changed;
}
FEXCore::IR::Pass* CreateLongDivideEliminationPass() {
return new LongDivideEliminationPass{};
}
}
@@ -8,7 +8,6 @@ $end_info$
#include "Interface/IR/Passes/RegisterAllocationPass.h"
#include "Interface/IR/Passes.h"
#include "Interface/Core/OpcodeDispatcher.h"
#include <FEXCore/Utils/Allocator.h>
#include <iterator>
#include <unordered_set>
@@ -66,7 +65,7 @@ namespace {
Enumerator(Item);
if (++i == Bucket->Size) {
LOGMAN_THROW_A(Bucket->Next != nullptr, "Interference bug");
LogMan::Throw::A(Bucket->Next != nullptr, "Interference bug");
Bucket = Bucket->Next.get();
i = 0;
}
@@ -87,7 +86,7 @@ namespace {
return true;
if (++i == Bucket->Size) {
LOGMAN_THROW_A(Bucket->Next != nullptr, "Bucket in bad state");
LogMan::Throw::A(Bucket->Next != nullptr, "Bucket in bad state");
Bucket = Bucket->Next.get();
i = 0;
}
@@ -131,7 +130,7 @@ namespace {
}
else if (++i == Size) {
i = 0;
LOGMAN_THROW_A(that->Next != nullptr, "Bucket::Erase but element not contained");
LogMan::Throw::A(that->Next != nullptr, "Bucket::Erase but element not contained");
that = that->Next.get();
}
}
@@ -263,7 +262,7 @@ namespace {
Graph->Nodes.resize(NodeCount);
Graph->VisitedNodePredecessors.clear();
Graph->AllocData.reset();
Graph->AllocData.reset((FEXCore::IR::RegisterAllocationData*)FEXCore::Allocator::malloc(FEXCore::IR::RegisterAllocationData::Size(NodeCount)));
Graph->AllocData.reset((FEXCore::IR::RegisterAllocationData*)malloc(FEXCore::IR::RegisterAllocationData::Size(NodeCount)));
memset(&Graph->AllocData->Map[0], INVALID_REGCLASS.Raw, NodeCount);
Graph->AllocData->MapCount = NodeCount;
Graph->AllocData->IsShared = false; // not shared by default
@@ -446,8 +445,8 @@ namespace FEXCore::IR {
}
void ConstrainedRAPass::AllocateRegisterSet(uint32_t RegisterCount, uint32_t ClassCount) {
LOGMAN_THROW_A(RegisterCount <= INVALID_REG, "Up to %d regs supported", INVALID_REG);
LOGMAN_THROW_A(ClassCount <= INVALID_CLASS, "Up to %d classes supported", INVALID_CLASS);
LogMan::Throw::A(RegisterCount <= INVALID_REG, "Up to %d regs supported", INVALID_REG);
LogMan::Throw::A(ClassCount <= INVALID_CLASS, "Up to %d classes supported", INVALID_CLASS);
Graph = AllocateRegisterGraph(ClassCount);
@@ -460,7 +459,7 @@ namespace FEXCore::IR {
}
void ConstrainedRAPass::AddRegisters(FEXCore::IR::RegisterClassType Class, uint32_t RegisterCount) {
LOGMAN_THROW_A(RegisterCount <= INVALID_REG, "Up to %d regs supported", INVALID_REG);
LogMan::Throw::A(RegisterCount <= INVALID_REG, "Up to %d regs supported", INVALID_REG);
AllocatePhysicalRegisters(Graph, Class, RegisterCount);
}
@@ -487,7 +486,7 @@ namespace FEXCore::IR {
auto Op = IROp->C<IROp_CodeBlock>();
LOGMAN_THROW_A(Op->Header.Op == OP_CODEBLOCK, "Block not defined by codeblock?");
LogMan::Throw::A(Op->Header.Op == OP_CODEBLOCK, "Block not defined by codeblock?");
LiveRange->Begin = std::min(LiveRange->Begin, Op->Begin.ID());
LiveRange->End = std::max(LiveRange->End, Op->Begin.ID());
@@ -515,7 +514,7 @@ namespace FEXCore::IR {
// If the destination hasn't yet been set then set it now
if (IROp->HasDest) {
LOGMAN_THROW_A(LiveRanges[Node].Begin == ~0U, "Node begin already defined?");
LogMan::Throw::A(LiveRanges[Node].Begin == ~0U, "Node begin already defined?");
LiveRanges[Node].Begin = Node;
// Default to ending right where after it starts
LiveRanges[Node].End = Node + 1;
@@ -547,7 +546,7 @@ namespace FEXCore::IR {
if (IR->GetOp<IROp_Header>(IROp->Args[i])->Op == OP_INLINEENTRYPOINTOFFSET) continue;
if (IR->GetOp<IROp_Header>(IROp->Args[i])->Op == OP_IRHEADER) continue;
uint32_t ArgNode = IROp->Args[i].ID();
LOGMAN_THROW_A(LiveRanges[ArgNode].Begin != ~0U, "%%ssa%d used by %%ssa%d before defined?", ArgNode, Node);
LogMan::Throw::A(LiveRanges[ArgNode].Begin != ~0U, "%%ssa%d used by %%ssa%d before defined?", ArgNode, Node);
auto ArgNodeBlockID = Graph->Nodes[ArgNode].Head.BlockID;
if (ArgNodeBlockID == BlockNodeID) {
@@ -601,7 +600,7 @@ namespace FEXCore::IR {
} else if (StaticClass == FPRFixedClass) {
return Size == 16;
} else {
LOGMAN_THROW_A(false, "Unexpected static class %d", StaticClass);
LogMan::Throw::A(false, "Unexpected static class %d", StaticClass);
}
return false; // Unknown
};
@@ -613,7 +612,7 @@ namespace FEXCore::IR {
} else if (StaticClass == FPRFixedClass) {
return (Size == 16 /*|| Size == 8 || Size == 4*/) && ((Offset & 15) == 0); // We need more meta info to support not-size-of-reg
} else {
LOGMAN_THROW_A(false, "Unexpected static class %d", StaticClass);
LogMan::Throw::A(false, "Unexpected static class %d", StaticClass);
}
return false; // Unknown
};
@@ -633,7 +632,7 @@ namespace FEXCore::IR {
auto reg = (Offset - beginFpr) / 16;
return PhysicalRegister(FPRFixedClass, reg);
} else {
LOGMAN_THROW_A(false, "Unexpected Offset %d", Offset);
LogMan::Throw::A(false, "Unexpected Offset %d", Offset);
return INVALID_REGCLASS;
}
};
@@ -657,7 +656,7 @@ namespace FEXCore::IR {
auto reg = (Offset - beginFpr) / 16;
return &StaticMaps[GprSize + reg];
} else {
LOGMAN_THROW_A(false, "Unexpected offset %d", Offset);
LogMan::Throw::A(false, "Unexpected offset %d", Offset);
return (LiveRange**)nullptr;
}
};
@@ -669,7 +668,7 @@ namespace FEXCore::IR {
} else if (PhyReg.Class == FPRFixedClass.Val) {
return &StaticMaps[GprSize + PhyReg.Reg];
} else {
LOGMAN_THROW_A(false, "Unexpected Class %d", PhyReg.Class);
LogMan::Throw::A(false, "Unexpected Class %d", PhyReg.Class);
return (LiveRange**)nullptr;
}
};
@@ -808,7 +807,7 @@ namespace FEXCore::IR {
for (auto [BlockNode, BlockHeader] : IR->GetBlocks()) {
auto BlockIROp = BlockHeader->CW<FEXCore::IR::IROp_CodeBlock>();
LOGMAN_THROW_A(BlockIROp->Header.Op == IR::OP_CODEBLOCK, "IR type failed to be a code block");
LogMan::Throw::A(BlockIROp->Header.Op == IR::OP_CODEBLOCK, "IR type failed to be a code block");
BlockInterferences *BlockInterferenceVector = &LocalBlockInterferences.try_emplace(IR->GetID(BlockNode)).first->second;
BlockInterferenceVector->reserve(BlockIROp->Last.ID() - BlockIROp->Begin.ID());
@@ -911,7 +910,7 @@ namespace FEXCore::IR {
SpanEnd.resize(NodeCount);
for (uint32_t i = 0; i < NodeCount; ++i) {
if (LiveRanges[i].Begin != ~0U) {
LOGMAN_THROW_A(LiveRanges[i].Begin < LiveRanges[i].End , "Span must Begin before Ending");
LogMan::Throw::A(LiveRanges[i].Begin < LiveRanges[i].End , "Span must Begin before Ending");
auto Class = GetClass(Graph->AllocData->Map[i]);
SpanStart[LiveRanges[i].Begin].Append(INFO_MAKE(i, Class));
@@ -939,7 +938,7 @@ namespace FEXCore::IR {
});
}
LOGMAN_THROW_A(Active.Items[0] == 0, "Interference bug");
LogMan::Throw::A(Active.Items[0] == 0, "Interference bug");
SpanStart.clear();
SpanEnd.clear();
}
@@ -958,7 +957,7 @@ namespace FEXCore::IR {
RegisterClass *RAClass = &Graph->Set.Classes[RegClass];
if (CurrentNode->Head.PhiPartner) {
LOGMAN_MSG_A("Phi nodes not supported");
LogMan::Msg::A("Phi nodes not supported");
#if 0
// In the case that we have a list of nodes that need the same register allocated we need to do something special
// We need to gather the data from the forward linked list and make sure they all match the virtual register
@@ -1157,7 +1156,7 @@ namespace FEXCore::IR {
// This would ensure something will spill earlier if its previous use and next use are farther away
auto InterferenceNodeNextUse = FindFirstUse(IREmit, InterferenceOrderedNode, NodeOpBeginIter, InterferenceNodeOpEndIter);
auto InterferenceNodePrevUse = FindLastUseBefore(IREmit, InterferenceOrderedNode, InterferenceNodeOpBeginIter, NodeOpBeginIter);
LOGMAN_THROW_A(InterferenceNodeNextUse != IR::NodeIterator::Invalid(), "Couldn't find next usage of op");
LogMan::Throw::A(InterferenceNodeNextUse != IR::NodeIterator::Invalid(), "Couldn't find next usage of op");
// If there is no use of the interference op prior to our op then it only has initial definition
if (InterferenceNodePrevUse == IR::NodeIterator::Invalid()) InterferenceNodePrevUse = InterferenceNodeOpBeginIter;
@@ -1323,7 +1322,7 @@ namespace FEXCore::IR {
LogMan::Msg::D("\tInt%d: %%ssa%d Remat: %d [%d, %d)", j++, InterferenceNode, InterferenceLiveRange->RematCost, InterferenceLiveRange->Begin, InterferenceLiveRange->End);
});
}
LOGMAN_THROW_A(InterferenceIdToSpill != 0, "Couldn't find Node to spill");
LogMan::Throw::A(InterferenceIdToSpill != 0, "Couldn't find Node to spill");
return InterferenceIdToSpill;
}
@@ -1358,7 +1357,7 @@ namespace FEXCore::IR {
auto LastCursor = IREmit->GetWriteCursor();
auto [CodeNode, IROp] = IR.at(SpillPointId)();
LOGMAN_THROW_A(IROp->HasDest, "Can't spill with no dest");
LogMan::Throw::A(IROp->HasDest, "Can't spill with no dest");
uint32_t Node = IR.GetID(CodeNode);
RegisterNode *CurrentNode = &Graph->Nodes[Node];
@@ -1382,7 +1381,7 @@ namespace FEXCore::IR {
// First op post Spill
auto NextIter = IR.at(CodeNode);
auto FirstUseLocation = FindFirstUse(IREmit, ConstantNode, NextIter, NodeIterator::Invalid());
LOGMAN_THROW_A(FirstUseLocation != IR::NodeIterator::Invalid(), "At %%ssa%d Spilling Op %%ssa%d but Failure to find op use", Node, InterferenceNode);
LogMan::Throw::A(FirstUseLocation != IR::NodeIterator::Invalid(), "At %%ssa%d Spilling Op %%ssa%d but Failure to find op use", Node, InterferenceNode);
if (FirstUseLocation != IR::NodeIterator::Invalid()) {
--FirstUseLocation;
auto [FirstUseOrderedNode, _] = FirstUseLocation();
@@ -1400,10 +1399,10 @@ namespace FEXCore::IR {
FEXCore::IR::RegisterClassType InterferenceRegClass = FEXCore::IR::RegisterClassType{Graph->AllocData->Map[InterferenceNode].Class};
uint32_t SpillSlot = FindSpillSlot(InterferenceNode, InterferenceRegClass);
RegisterNode *InterferenceRegisterNode = &Graph->Nodes[InterferenceNode];
LOGMAN_THROW_A(SpillSlot != ~0U, "Interference Node doesn't have a spill slot!");
//LOGMAN_THROW_A(InterferenceRegisterNode->Head.RegAndClass.Reg != INVALID_REG, "Interference node never assigned a register?");
LOGMAN_THROW_A(InterferenceRegClass != ~0U, "Interference node never assigned a register class?");
LOGMAN_THROW_A(InterferenceRegisterNode->Head.PhiPartner == nullptr, "We don't support spilling PHI nodes currently");
LogMan::Throw::A(SpillSlot != ~0U, "Interference Node doesn't have a spill slot!");
//LogMan::Throw::A(InterferenceRegisterNode->Head.RegAndClass.Reg != INVALID_REG, "Interference node never assigned a register?");
LogMan::Throw::A(InterferenceRegClass != ~0U, "Interference node never assigned a register class?");
LogMan::Throw::A(InterferenceRegisterNode->Head.PhiPartner == nullptr, "We don't support spilling PHI nodes currently");
// This is the op that we need to dump
auto [InterferenceOrderedNode, InterferenceIROp] = IR.at(InterferenceNode)();
@@ -1436,7 +1435,7 @@ namespace FEXCore::IR {
++FirstIter;
auto FirstUseLocation = FindFirstUse(IREmit, InterferenceOrderedNode, FirstIter, NodeIterator::Invalid());
LOGMAN_THROW_A(FirstUseLocation != NodeIterator::Invalid(), "At %%ssa%d Spilling Op %%ssa%d but Failure to find op use", Node, InterferenceNode);
LogMan::Throw::A(FirstUseLocation != NodeIterator::Invalid(), "At %%ssa%d Spilling Op %%ssa%d but Failure to find op use", Node, InterferenceNode);
if (FirstUseLocation != IR::NodeIterator::Invalid()) {
// We want to fill just before the first use
--FirstUseLocation;
@@ -22,7 +22,7 @@ bool IsStaticAllocGpr(uint32_t Offset, RegisterClassType Class) {
if (Offset >= begin && Offset < end) {
auto reg = (Offset - begin) / 8;
LOGMAN_THROW_A(Class == IR::GPRClass, "unexpected Class %d", Class);
LogMan::Throw::A(Class == IR::GPRClass, "unexpected Class %d", Class);
rv = reg < 16; // 0..15 -> 16 in total
}
@@ -37,7 +37,7 @@ bool IsStaticAllocFpr(uint32_t Offset, RegisterClassType Class, bool AllowGpr) {
if (Offset >= begin && Offset < end) {
auto reg = (Offset - begin)/16;
LOGMAN_THROW_A(Class == IR::FPRClass || (AllowGpr && Class == IR::GPRClass), "unexpected Class %d, AllowGpr %d", Class, AllowGpr);
LogMan::Throw::A(Class == IR::FPRClass || (AllowGpr && Class == IR::GPRClass), "unexpected Class %d, AllowGpr %d", Class, AllowGpr);
rv = reg < 16; // 0..15 -> 16 in total
}
-118
View File
@@ -1,118 +0,0 @@
#include "Utils/Allocator/HostAllocator.h"
#include <FEXCore/Utils/Allocator.h>
#include <sys/mman.h>
#include <jemalloc/jemalloc.h>
#include <memory>
#include <malloc.h>
extern "C" {
extern void *__libc_malloc(size_t size);
extern void *__libc_realloc(void *ptr, size_t size);
extern void __libc_free(void *ptr);
typedef void* (*mmap_hook_type)(
void *addr, size_t length, int prot, int flags,
int fd, off_t offset);
typedef int (*munmap_hook_type)(void *addr, size_t length);
extern mmap_hook_type __mmap_hook;
extern munmap_hook_type __munmap_hook;
static FEXCore::Allocator::MALLOC_Hook global_malloc {::__libc_malloc};
static FEXCore::Allocator::REALLOC_Hook global_realloc {::__libc_realloc};
static FEXCore::Allocator::FREE_Hook global_free {::__libc_free};
// Override the global functions
FEX_DEFAULT_VISIBILITY void *malloc(size_t size) { return global_malloc(size); }
FEX_DEFAULT_VISIBILITY void *realloc(void *ptr, size_t size) { return global_realloc(ptr, size); }
FEX_DEFAULT_VISIBILITY void free(void *ptr) { return global_free(ptr); }
}
namespace FEXCore::Allocator {
MMAP_Hook mmap {::mmap};
MUNMAP_Hook munmap {::munmap};
MALLOC_Hook malloc {::__libc_malloc};
REALLOC_Hook realloc {::__libc_realloc};
FREE_Hook free {::__libc_free};
using GLIBC_MALLOC_Hook = void*(*)(size_t, const void *caller);
using GLIBC_REALLOC_Hook = void*(*)(void*, size_t, const void *caller);
using GLIBC_FREE_Hook = void(*)(void*, const void *caller);
std::unique_ptr<Alloc::HostAllocator> Alloc64{};
void *FEX_mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset) {
void *Result = Alloc64->Mmap(addr, length, prot, flags, fd, offset);
if (Result >= (void*)-4096) {
errno = -(uint64_t)Result;
return (void*)-1;
}
return Result;
}
int FEX_munmap(void *addr, size_t length) {
int Result = Alloc64->Munmap(addr, length);
if (Result != 0) {
errno = -Result;
return -1;
}
return Result;
}
void *FEX_malloc_hook(size_t size, const void *caller) {
return ::je_malloc(size);
}
void *FEX_realloc_hook(void *ptr, size_t size, const void *caller) {
return ::je_realloc(ptr, size);
}
void FEX_free_hook(void *ptr, const void *caller) {
return ::je_free(ptr);
}
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
void SetupHooks() {
Alloc64.reset(Alloc::OSAllocator::Create64BitAllocator());
__mmap_hook = FEX_mmap;
__munmap_hook = FEX_munmap;
FEXCore::Allocator::mmap = FEX_mmap;
FEXCore::Allocator::munmap = FEX_munmap;
FEXCore::Allocator::malloc = ::je_malloc;
FEXCore::Allocator::realloc = ::je_realloc;
FEXCore::Allocator::free = ::je_free;
global_malloc = ::je_malloc;
global_realloc = ::je_realloc;
global_free = ::je_free;
__malloc_hook = FEXCore::Allocator::FEX_malloc_hook;
__realloc_hook = FEXCore::Allocator::FEX_realloc_hook;
__free_hook = FEXCore::Allocator::FEX_free_hook;
}
void ClearHooks() {
__mmap_hook = ::mmap;
__munmap_hook = ::munmap;
FEXCore::Allocator::mmap = ::mmap;
FEXCore::Allocator::munmap = ::munmap;
FEXCore::Allocator::malloc = ::__libc_malloc;
FEXCore::Allocator::realloc = ::__libc_realloc;
FEXCore::Allocator::free = ::__libc_free;
global_malloc = ::__libc_malloc;
global_realloc = ::__libc_realloc;
global_free = ::__libc_free;
// Reset's glibc hooks
__malloc_hook = 0;
__realloc_hook = 0;
__free_hook = 0;
}
#pragma GCC diagnostic pop
}
extern "C" {
}
@@ -1,715 +0,0 @@
#include "Utils/Allocator/FlexBitSet.h"
#include "Utils/Allocator/HostAllocator.h"
#include "Utils/Allocator/IntrusiveArenaAllocator.h"
#include <FEXCore/Utils/LogManager.h>
#include <array>
#include <bit>
#include <bitset>
#include <cassert>
#include <cmath>
#include <cstdint>
#include <cstring>
#include <list>
#include <malloc.h>
#include <mutex>
#include <stdio.h>
#include <set>
#include <sys/mman.h>
#include <sys/utsname.h>
#include <sys/resource.h>
#include <syscall.h>
#include <vector>
static constexpr uint64_t PAGE_SHIFT = 12;
static constexpr uint64_t PAGE_MASK = (1 << PAGE_SHIFT) - 1;
namespace Alloc::OSAllocator {
class OSAllocator_64Bit final : public Alloc::HostAllocator {
public:
OSAllocator_64Bit();
virtual ~OSAllocator_64Bit();
void *AllocateSlab(size_t Size) override { return nullptr; }
void DeallocateSlab(void *Ptr, size_t Size) override {}
void *Mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset) override;
int Munmap(void *addr, size_t length) override;
private:
constexpr static uint64_t PAGE_SIZE = 4096;
// Upper bound is the maximum virtual address space of the host processor
uintptr_t UPPER_BOUND = (1ULL << 57);
// Lower bound is the starting of the range just past the lower 32bits
constexpr static uintptr_t LOWER_BOUND = 0x1'0000'0000ULL;
uintptr_t UPPER_BOUND_PAGE = UPPER_BOUND / PAGE_SIZE;
constexpr static uintptr_t LOWER_BOUND_PAGE = LOWER_BOUND / PAGE_SIZE;
struct ReservedVMARegion {
uintptr_t Base;
// Could be number of pages if we want to pack this in to 12 bytes
uint64_t RegionSize;
};
bool MergeReservedRegionIfPossible(ReservedVMARegion *Region, uintptr_t NextPtr, uint64_t NextSize) {
constexpr uint64_t MaxReservedRegionSize = 64ULL * 1024 * 1024 * 1024; // 64GB
uintptr_t RegionEnd = Region->Base + Region->RegionSize;
uint64_t NewRegionSize = Region->RegionSize + NextSize;
if (RegionEnd == NextPtr &&
NewRegionSize <= MaxReservedRegionSize) {
// Append the contiguous region
Region->RegionSize = NewRegionSize;
return true;
}
return false;
}
struct LiveVMARegion {
ReservedVMARegion *SlabInfo;
uint64_t FreeSpace{};
uint32_t LastPageAllocation{};
FlexBitSet<uint64_t> UsedPages;
// This returns the size of the LiveVMARegion in addition to the flex set that tracks the used data
// The LiveVMARegion lives at the start of the VMA region which means on initialization we need to set that
// tracked ranged as used immediately
static size_t GetSizeWithFlexSet(size_t Size) {
// One element per page
// 0x10'0000'0000 bytes
// 0x100'0000 Pages
// 1 bit per page for tracking means 0x20'0000 (Pages / 8) bytes of flex space
// Which is 2MB of tracking
uint64_t NumElements = (Size >> PAGE_SHIFT) * sizeof(uint64_t);
return sizeof(LiveVMARegion) + FlexBitSet<uint64_t>::Size(NumElements);
}
static void InitializeVMARegionUsed(LiveVMARegion *Region, size_t AdditionalSize) {
size_t SizeOfLiveRegion = AlignUp(LiveVMARegion::GetSizeWithFlexSet(Region->SlabInfo->RegionSize), PAGE_SIZE);
size_t SizePlusManagedData = SizeOfLiveRegion + AdditionalSize;
Region->FreeSpace = Region->SlabInfo->RegionSize - SizePlusManagedData;
size_t NumPages = SizePlusManagedData >> PAGE_SHIFT;
// Memset the full tracking to zero to state nothing used
Region->UsedPages.MemSet(Region->SlabInfo->RegionSize >> PAGE_SHIFT);
// Set our reserved pages
for (size_t i = 0; i < NumPages; ++i) {
// Set our used pages
Region->UsedPages.Set(i);
}
Region->LastPageAllocation = NumPages;
}
};
static_assert(std::is_trivially_copyable<LiveVMARegion>::value, "Needs to be trivially copyable");
static_assert(offsetof(LiveVMARegion, UsedPages) == sizeof(LiveVMARegion), "FlexBitSet needs to be at the end");
using ReservedRegionListType = std::pmr::list<ReservedVMARegion*>;
using LiveRegionListType = std::pmr::list<LiveVMARegion*>;
ReservedRegionListType *ReservedRegions{};
LiveRegionListType *LiveRegions{};
Alloc::ForwardOnlyIntrusiveArenaAllocator *ObjectAlloc{};
std::mutex AllocationMutex{};
void DetermineVASize();
LiveVMARegion *MakeRegionActive(ReservedRegionListType::iterator ReservedIterator, uint64_t UsedSize) {
ReservedVMARegion *ReservedRegion = *ReservedIterator;
ReservedRegions->erase(ReservedIterator);
// mprotect the new region we've allocated
size_t SizeOfLiveRegion = AlignUp(LiveVMARegion::GetSizeWithFlexSet(ReservedRegion->RegionSize), PAGE_SIZE);
size_t SizePlusManagedData = UsedSize + SizeOfLiveRegion;
mprotect(reinterpret_cast<void*>(ReservedRegion->Base), SizePlusManagedData, PROT_READ | PROT_WRITE);
LiveVMARegion *LiveRange = new (reinterpret_cast<void*>(ReservedRegion->Base)) LiveVMARegion();
// Copy over the reserved data
LiveRange->SlabInfo = ReservedRegion;
// Initialize VMA
LiveVMARegion::InitializeVMARegionUsed(LiveRange, UsedSize);
// Add to our active tracked ranges
auto LiveIter = LiveRegions->emplace_back(LiveRange);
return LiveIter;
}
// 32-bit old kernel workarounds
struct PtrCache {
uint32_t Ptr;
uint32_t Size;
};
PtrCache *Steal32BitIfOldKernel();
void Clear32BitOnOldKernel(PtrCache *Base);
};
void OSAllocator_64Bit::DetermineVASize() {
const std::vector<uintptr_t> TLBSizes = {{
1ULL << 57,
1ULL << 52,
1ULL << 48,
1ULL << 47,
1ULL << 42,
1ULL << 39,
1ULL << 36,
}};
for (auto Size : TLBSizes) {
// Just try allocating
// We can't actually determine VA size on ARM safely
auto Find = [](uintptr_t Size) -> bool {
for (int i = 0; i < 64; ++i) {
// Try grabbing a some of the top pages of the range
// x86 allocates some high pages in the top end
void *Ptr = ::mmap(reinterpret_cast<void*>(Size - PAGE_SIZE * i), PAGE_SIZE, PROT_NONE, MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (Ptr != (void*)~0ULL) {
::munmap(Ptr, PAGE_SIZE);
if (Ptr == (void*)(Size - PAGE_SIZE * i)) {
return true;
}
}
}
return false;
};
if (Find(Size)) {
UPPER_BOUND = Size;
UPPER_BOUND_PAGE = UPPER_BOUND / PAGE_SIZE;
break;
}
}
}
void *OSAllocator_64Bit::Mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset) {
if (addr != 0 &&
addr < reinterpret_cast<void*>(LOWER_BOUND)) {
// If we are asked to allocate something outside of the 64-bit space
// Then we need to just hand this to the OS
return ::mmap(addr, length, prot, flags, fd, offset);
}
uint64_t Addr = reinterpret_cast<uint64_t>(addr);
// Addr must be page aligned
if (Addr & PAGE_MASK) {
return reinterpret_cast<void*>(-EINVAL);
}
// If FD is provided then offset must also be page aligned
if (fd != -1 &&
offset & PAGE_MASK) {
return reinterpret_cast<void*>(-EINVAL);
}
// 64bit address overflow
if (Addr + length < Addr) {
return reinterpret_cast<void*>(-EOVERFLOW);
}
bool Fixed = (flags & MAP_FIXED) || (flags & MAP_FIXED_NOREPLACE);
length = AlignUp(length, PAGE_SIZE);
uint64_t AddrEnd = Addr + length;
size_t NumberOfPages = length / PAGE_SIZE;
// This needs a mutex to be thread safe
std::scoped_lock<std::mutex> lk{AllocationMutex};
uint64_t AllocatedOffset{};
LiveVMARegion *LiveRegion{};
if (Fixed || Addr != 0) {
// Check active slabs to see if we can fit this
for (auto it = LiveRegions->begin(); it != LiveRegions->end(); ++it) {
uintptr_t RegionBegin = (*it)->SlabInfo->Base;
uintptr_t RegionEnd = RegionBegin + (*it)->SlabInfo->RegionSize;
if (Addr >= RegionBegin &&
Addr < RegionEnd) {
LiveRegion = *it;
// Leave our loop
break;
}
}
// Couldn't find an active region that fit
// Check reserved regions
if (!LiveRegion) {
// Didn't have a slab that fit this range
// Check our reserved regions to see if we have one that fits
for (auto it = ReservedRegions->begin(); it != ReservedRegions->end(); ++it) {
ReservedVMARegion *ReservedRegion = *it;
uintptr_t RegionEnd = ReservedRegion->Base + ReservedRegion->RegionSize;
if (Addr >= ReservedRegion->Base &&
AddrEnd < RegionEnd) {
// Found one, let's make it active
LiveRegion = MakeRegionActive(it, 0);
break;
}
}
}
}
again:
auto CheckIfRangeFits = [&AllocatedOffset](LiveVMARegion *Region, uint64_t length, int prot, int flags, int fd, off_t offset, uint64_t StartingPosition = 0) -> std::pair<LiveVMARegion*, void*> {
uint64_t AllocatedPage{};
uint64_t NumberOfPages = length >> PAGE_SHIFT;
if (Region->FreeSpace >= length) {
uint64_t LastAllocation =
StartingPosition ?
(StartingPosition - Region->SlabInfo->Base) >> PAGE_SHIFT
: Region->LastPageAllocation;
size_t RegionNumberOfPages = Region->SlabInfo->RegionSize >> PAGE_SHIFT;
try_again:
for (size_t CurrentPage = LastAllocation;
CurrentPage < (RegionNumberOfPages - NumberOfPages);) {
// If we have enough free space, check if we have enough free pages that are contiguous
size_t Remaining = NumberOfPages;
assert((CurrentPage + Remaining - 1) < RegionNumberOfPages);
while (Remaining) {
if (Region->UsedPages[CurrentPage + Remaining - 1]) {
// Has an intersecting range
break;
}
--Remaining;
}
if (Remaining) {
// Didn't find a slab range
CurrentPage += Remaining;
}
else {
// We have a slab range
AllocatedPage = CurrentPage;
break;
}
}
if (!AllocatedPage && LastAllocation != 0) {
// Try again but starting from the beginning
LastAllocation = 0;
// Using goto so we don't have recursive mutex shenanigans
goto try_again;
}
if (AllocatedPage) {
AllocatedOffset = Region->SlabInfo->Base + AllocatedPage * PAGE_SIZE;
// We need to setup protections for this
void *MMapResult = ::mmap(reinterpret_cast<void*>(AllocatedOffset),
length,
prot,
(flags & ~MAP_FIXED_NOREPLACE) | MAP_FIXED,
fd, offset);
if (MMapResult == MAP_FAILED) {
return std::make_pair(Region, reinterpret_cast<void*>(-errno));
}
return std::make_pair(Region, MMapResult);
}
}
return std::make_pair(nullptr, nullptr);
};
if (Fixed) {
// Found a region let's allocate to it
if (LiveRegion) {
// Found a slab that fits this
if (flags & MAP_FIXED_NOREPLACE) {
auto Fits = CheckIfRangeFits(LiveRegion, length, prot, flags, fd, offset, Addr);
if (Fits.first && Fits.second == reinterpret_cast<void*>(Addr)) {
// We fit correctly
AllocatedOffset = Addr;
}
else {
// Intersected with something that already existed
return reinterpret_cast<void*>(-EEXIST);
}
}
else {
// We need to mmap the file to this location
void *MMapResult = ::mmap(reinterpret_cast<void*>(Addr),
length,
prot,
(flags & ~MAP_FIXED_NOREPLACE) | MAP_FIXED,
fd, offset);
if (MMapResult == MAP_FAILED) {
return reinterpret_cast<void*>(-errno);
}
AllocatedOffset = Addr;
}
// Fall through to live region tracking
}
}
else {
// Check our active slabs to see if we can fit the allocation
// Slightly different than fixed since it doesn't need exact placement
if (LiveRegion && Addr != 0) {
// We found a LiveRegion that could hold this address. Let's try to place it
// Check if this area is free
auto Fits = CheckIfRangeFits(LiveRegion, length, prot, flags, fd, offset, Addr);
if (Fits.first && Fits.second == reinterpret_cast<void*>(Addr)) {
// We fit correctly
AllocatedOffset = Addr;
}
else {
// Couldn't fit
// We can continue past this point still
LiveRegion = nullptr;
}
}
if (!LiveRegion) {
for (auto it = LiveRegions->begin(); it != LiveRegions->end(); ++it) {
auto Fits = CheckIfRangeFits(*it, length, prot, flags, fd, offset);
if (Fits.first && Fits.second == reinterpret_cast<void*>(AllocatedOffset)) {
// We fit correctly
LiveRegion = Fits.first;
break;
}
// Couldn't fit but mmap gave us an error
if (!Fits.first && Fits.second) {
return Fits.second;
}
// nullptr on both means no error and couldn't fit
}
}
if (!LiveRegion) {
// Couldn't find a fit in the live regions
// Allocate a new reserved region
size_t lengthOfLiveRegion = AlignUp(LiveVMARegion::GetSizeWithFlexSet(length), PAGE_SIZE);
size_t lengthPlusManagedData = length + lengthOfLiveRegion;
for (auto it = ReservedRegions->begin(); it != ReservedRegions->end(); ++it) {
if ((*it)->RegionSize >= lengthPlusManagedData) {
MakeRegionActive(it, 0);
goto again;
}
}
}
}
if (LiveRegion) {
// Mark the pages as used
uintptr_t RegionBegin = LiveRegion->SlabInfo->Base;
uintptr_t MappedBegin = (AllocatedOffset - RegionBegin) >> PAGE_SHIFT;
for (size_t i = 0; i < NumberOfPages; ++i) {
LiveRegion->UsedPages.Set(MappedBegin + i);
}
// Change our last allocation region
LiveRegion->LastPageAllocation = MappedBegin + NumberOfPages;
LiveRegion->FreeSpace -= length;
}
if (!AllocatedOffset) {
AllocatedOffset = -ENOMEM;
}
return reinterpret_cast<void*>(AllocatedOffset);
}
int OSAllocator_64Bit::Munmap(void *addr, size_t length) {
if (addr < reinterpret_cast<void*>(LOWER_BOUND)) {
// If we are asked to allocate something outside of the 64-bit space
// Then we need to just hand this to the OS
return ::munmap(addr, length);
}
uint64_t Addr = reinterpret_cast<uint64_t>(addr);
if (Addr & PAGE_MASK) {
return -EINVAL;
}
if (length & PAGE_MASK) {
return -EINVAL;
}
if (Addr + length < Addr) {
return -EOVERFLOW;
}
// This needs a mutex to be thread safe
std::scoped_lock<std::mutex> lk{AllocationMutex};
length = AlignUp(length, PAGE_SIZE);
uintptr_t PtrBegin = reinterpret_cast<uintptr_t>(addr);
uintptr_t PtrEnd = PtrBegin + length;
// Walk all of the live ranges and find this slab then delete it
for (auto it = LiveRegions->begin(); it != LiveRegions->end(); ++it) {
uintptr_t RegionBegin = (*it)->SlabInfo->Base;
uintptr_t RegionEnd = RegionBegin + (*it)->SlabInfo->RegionSize;
if (RegionBegin <= PtrBegin &&
RegionEnd > PtrEnd) {
// Live region fully encompasses slab range
uint64_t FreedPages{};
uint64_t SlabPageBegin = (PtrBegin - RegionBegin) >> PAGE_SHIFT;
uint64_t PagesToFree = length >> PAGE_SHIFT;
for (size_t i = 0; i < PagesToFree; ++i) {
FreedPages += (*it)->UsedPages.TestAndClear(SlabPageBegin + i) ? 1 : 0;
}
if (FreedPages != 0)
{
// If we were contiuous freeing then make sure to give back the physical address space
// If the region was locked then madvise won't remove the physical backing
// This woul be a bug in the frontend application
// So be careful with mlock/munlock
::madvise(addr, length, MADV_DONTNEED);
::mmap(addr, length, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0);
}
(*it)->FreeSpace += FreedPages * 4096;
// XXX: Move region back to reserved list
return 0;
}
}
// If it didn't match at all then no error
return 0;
}
OSAllocator_64Bit::PtrCache *OSAllocator_64Bit::Steal32BitIfOldKernel() {
// First calculate kernel version
struct utsname buf{};
if (uname(&buf) == -1) {
return nullptr;
}
int32_t Major{};
int32_t Minor{};
int32_t Patch{};
char Tmp{};
std::istringstream ss{buf.release};
ss >> Major;
ss.read(&Tmp, 1);
ss >> Minor;
ss.read(&Tmp, 1);
ss >> Patch;
ss.read(&Tmp, 1);
uint32_t Version = (Major << 24) | (Minor << 16) | Patch;
if (Version >= ((4 << 24) | (17 << 16) | 0)) {
// If the kernel is >= 4.17 then it supports MAP_FIXED_NOREPLACE
return nullptr;
}
OSAllocator_64Bit::PtrCache *Cache{};
uint32_t CacheSize{};
uint32_t CurrentCacheOffset = 0;
constexpr std::array<size_t, 6> ReservedVMARegionSizes = {{
1ULL * 1024 * 1024 * 1024, // 1GB
512ULL * 1024 * 1024, // 512MB
128ULL * 1024 * 1024, // 128MB
32ULL * 1024 * 1024, // 32MB
1ULL * 1024 * 1024, // 1MB
4096ULL // One page
}};
constexpr size_t AllocationSizeMaxIndex = ReservedVMARegionSizes.size() - 1;
uint64_t CurrentSizeIndex = 0;
constexpr size_t LOWER_BOUND_32 = 0x1'0000;
constexpr size_t UPPER_BOUND_32 = LOWER_BOUND;
for (size_t MemoryOffset = LOWER_BOUND_32; MemoryOffset < UPPER_BOUND_32;) {
size_t AllocationSize = ReservedVMARegionSizes[CurrentSizeIndex];
size_t MemoryOffsetUpper = MemoryOffset + AllocationSize;
// If we would go above the upper bound on size then try the next size
if (MemoryOffsetUpper > UPPER_BOUND_32) {
++CurrentSizeIndex;
continue;
}
void *Ptr = ::mmap(reinterpret_cast<void*>(MemoryOffset), AllocationSize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0);
// If we managed to allocate and not get the address we want then unmap it
// This happens with kernels older than 4.17
if (reinterpret_cast<uintptr_t>(Ptr) + AllocationSize > UPPER_BOUND_32) {
munmap(Ptr, AllocationSize);
Ptr = reinterpret_cast<void*>(~0ULL);
}
// If we failed to allocate and we are on the smallest allocation size then just continue onward
// This page was unmappable
if (reinterpret_cast<uintptr_t>(Ptr) == ~0ULL && CurrentSizeIndex == AllocationSizeMaxIndex) {
CurrentSizeIndex = 0;
MemoryOffset += AllocationSize;
continue;
}
// Congratulations we were able to map this bit
// Reset and claim it was available
if (reinterpret_cast<uintptr_t>(Ptr) != ~0ULL) {
if (!Cache) {
Cache = reinterpret_cast<OSAllocator_64Bit::PtrCache *>(Ptr);
CacheSize = AllocationSize;
}
else {
Cache[CurrentCacheOffset] = {
.Ptr = static_cast<uint32_t>(reinterpret_cast<uint64_t>(Ptr)),
.Size = static_cast<uint32_t>(AllocationSize)
};
++CurrentCacheOffset;
}
CurrentSizeIndex = 0;
MemoryOffset += AllocationSize;
continue;
}
// Couldn't allocate at this size
// Increase and continue
++CurrentSizeIndex;
}
Cache[CurrentCacheOffset] = {
.Ptr = static_cast<uint32_t>(reinterpret_cast<uint64_t>(Cache)),
.Size = CacheSize,
};
return Cache;
}
void OSAllocator_64Bit::Clear32BitOnOldKernel(OSAllocator_64Bit::PtrCache *Base) {
if (Base == nullptr) {
return;
}
for (size_t i = 0;; ++i) {
void *Ptr = reinterpret_cast<void*>(Base[i].Ptr);
size_t Size = Base[i].Size;
munmap(Ptr, Size);
if (Ptr == Base) {
break;
}
}
}
OSAllocator_64Bit::OSAllocator_64Bit() {
malloc_trim(0);
DetermineVASize();
auto ArrayPtr = Steal32BitIfOldKernel();
// On allocation try and steal the entire upper 64bits of address space for mapping
constexpr std::array<size_t, 8> ReservedVMARegionSizes = {{
// Anything larger than 64GB fails out
64ULL * 1024 * 1024 * 1024, // 64GB
32ULL * 1024 * 1024 * 1024, // 32GB
16ULL * 1024 * 1024 * 1024, // 16GB
4ULL * 1024 * 1024 * 1024, // 4GB
1ULL * 1024 * 1024 * 1024, // 1GB
512ULL * 1024 * 1024, // 512MB
128ULL * 1024 * 1024, // 128MB
4096ULL // One page
}};
constexpr size_t AllocationSizeMaxIndex = ReservedVMARegionSizes.size() - 1;
// Have the first region only be 4GB VMA
// Avoids conflicts with some tests
uint64_t CurrentSizeIndex = 3;
ReservedVMARegion *PrevReserved{};
for (size_t MemoryOffset = LOWER_BOUND; MemoryOffset < UPPER_BOUND;) {
size_t AllocationSize = ReservedVMARegionSizes[CurrentSizeIndex];
size_t MemoryOffsetUpper = MemoryOffset + AllocationSize;
// If we would go above the upper bound on size then try the next size
if (MemoryOffsetUpper > UPPER_BOUND) {
++CurrentSizeIndex;
continue;
}
void *Ptr = ::mmap(reinterpret_cast<void*>(MemoryOffset), AllocationSize, PROT_NONE, MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0);
// If we managed to allocate and not get the address we want then unmap it
// This happens with kernels older than 4.17
if (reinterpret_cast<uintptr_t>(Ptr) != MemoryOffset &&
reinterpret_cast<uintptr_t>(Ptr) < LOWER_BOUND) {
munmap(Ptr, AllocationSize);
Ptr = reinterpret_cast<void*>(~0ULL);
}
// If we failed to allocate and we are on the smallest allocation size then just continue onward
// This page was unmappable
if (reinterpret_cast<uintptr_t>(Ptr) == ~0ULL && CurrentSizeIndex == AllocationSizeMaxIndex) {
CurrentSizeIndex = 0;
MemoryOffset += AllocationSize;
continue;
}
// Congratulations we were able to map this bit
// Reset and claim it was available
if (reinterpret_cast<uintptr_t>(Ptr) != ~0ULL) {
if (!ObjectAlloc) {
// Steal the first allocation for an intrusive allocator
// Will be mprotected correctly already
int Result = mprotect(Ptr, AllocationSize, PROT_READ | PROT_WRITE);
LogMan::Throw::A(Result == 0, "mprotect(%p, 0x%lx) -> %d (%s)", Ptr, AllocationSize, Result, strerror(errno));
ObjectAlloc = new (Ptr) Alloc::ForwardOnlyIntrusiveArenaAllocator(Ptr, AllocationSize);
ReservedRegions = ObjectAlloc->new_construct(ReservedRegions, ObjectAlloc);
LiveRegions = ObjectAlloc->new_construct(LiveRegions, ObjectAlloc);
}
else {
bool Merged = false;
if (PrevReserved) {
Merged = MergeReservedRegionIfPossible(PrevReserved, reinterpret_cast<uint64_t>(Ptr), AllocationSize);
}
if (!Merged) {
ReservedVMARegion *Region = ObjectAlloc->new_construct<ReservedVMARegion>();
Region->Base = reinterpret_cast<uint64_t>(Ptr);
Region->RegionSize = AllocationSize;
ReservedRegions->emplace_back(Region);
PrevReserved = Region;
}
}
CurrentSizeIndex = 0;
MemoryOffset += AllocationSize;
continue;
}
// Couldn't allocate at this size
// Increase and continue
++CurrentSizeIndex;
}
Clear32BitOnOldKernel(ArrayPtr);
}
OSAllocator_64Bit::~OSAllocator_64Bit() {
// For consistency, pull the mutex
std::scoped_lock<std::mutex> lk{AllocationMutex};
// Walk the pages and deallocate
// First walk the live regions
for (auto it = LiveRegions->begin(); it != LiveRegions->end(); ++it) {
::munmap(reinterpret_cast<void*>((*it)->SlabInfo->Base), (*it)->SlabInfo->RegionSize);
}
// Now walk the reserved regions
for (auto it = ReservedRegions->begin(); it != ReservedRegions->end(); ++it) {
::munmap(reinterpret_cast<void*>((*it)->Base), (*it)->RegionSize);
}
}
Alloc::HostAllocator *Create64BitAllocator() {
return new OSAllocator_64Bit{};
}
}
-50
View File
@@ -1,50 +0,0 @@
#pragma once
#include "HostAllocator.h"
#include <cstddef>
#include <cstdint>
#include <cstring>
#include <type_traits>
template<typename T>
struct FlexBitSet final {
using ElementType = T;
constexpr static size_t MinimumSize = sizeof(ElementType);
constexpr static size_t MinimumSizeBits = sizeof(ElementType) * 8;
T Memory[];
bool Get(T Element) {
return (Memory[Element / MinimumSizeBits] & (1ULL << (Element % MinimumSizeBits))) != 0;
}
bool TestAndClear(T Element) {
bool Value = Get(Element);
Memory[Element / MinimumSizeBits] &= ~(1ULL << (Element % MinimumSizeBits));
return Value;
}
void Set(T Element) {
Memory[Element / MinimumSizeBits] |= (1ULL << (Element % MinimumSizeBits));
}
void Clear(T Element) {
Memory[Element / MinimumSizeBits] &= ~(1ULL << (Element % MinimumSizeBits));
}
void MemClear(size_t Elements) {
memset(Memory, 0, Alloc::AlignUp(Elements / MinimumSizeBits, MinimumSizeBits));
}
void MemSet(size_t Elements) {
memset(Memory, 0xFF, Alloc::AlignUp(Elements / MinimumSizeBits, MinimumSizeBits));
}
// This very explicitly doesn't let you take an address
// Is only a getter
bool operator[](T Element) {
return Get(Element);
}
static size_t Size(T Elements) {
return Alloc::AlignUp(Elements / MinimumSizeBits, MinimumSizeBits);
}
};
static_assert(sizeof(FlexBitSet<uint64_t>) == 0, "This needs to be a flex member");
static_assert(std::is_trivially_copyable<FlexBitSet<uint64_t>>::value, "Needsto be trivially copyable");
-44
View File
@@ -1,44 +0,0 @@
#pragma once
#include <cstddef>
#include <cstdint>
#include <sys/types.h>
constexpr static uint64_t PAGE_SIZE = 4096;
namespace Alloc {
static inline uint64_t AlignUp(uint64_t value, uint64_t size) {
return value + (size - value % size) % size;
};
// HostAllocator is just a page pased slab allocator
// Similar to mmap and munmap only mapping at the page level
class HostAllocator {
public:
HostAllocator() = default;
virtual ~HostAllocator() = default;
virtual void *AllocateSlab(size_t Size) = 0;
virtual void DeallocateSlab(void *Ptr, size_t Size) = 0;
virtual void *Mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset) { return nullptr; }
virtual int Munmap(void *addr, size_t length) { return -1; }
};
class GlobalAllocator {
public:
HostAllocator *Alloc{};
GlobalAllocator(HostAllocator *_Alloc)
: Alloc {_Alloc} {}
virtual void *malloc(size_t Size) = 0;
virtual void *calloc(size_t num, size_t size) = 0;
virtual void *realloc(void *ptr, size_t size) = 0;
virtual void *memalign(size_t alignment, size_t size) = 0;
virtual void free(void *ptr) = 0;
};
GlobalAllocator *CreateBasicAllocator(HostAllocator *Alloc);
}
namespace Alloc::OSAllocator {
Alloc::HostAllocator *Create64BitAllocator();
}
@@ -1,186 +0,0 @@
#pragma once
#include "FlexBitSet.h"
#include "HostAllocator.h"
#include <bitset>
#include <cstddef>
#include <memory_resource>
#include <mutex>
#include <vector>
namespace Alloc {
class ForwardOnlyIntrusiveArenaAllocator final : public std::pmr::memory_resource {
public:
ForwardOnlyIntrusiveArenaAllocator(void* Ptr, size_t _Size)
: Begin {reinterpret_cast<uintptr_t>(Ptr)}
, Size {_Size} {
LastAllocation = sizeof(ForwardOnlyIntrusiveArenaAllocator);
}
~ForwardOnlyIntrusiveArenaAllocator() = default;
template<class U, class... Args>
U *new_construct(Args&&... args) {
void *Ptr = do_allocate(sizeof(U), std::alignment_of<U>::value);
return new (Ptr) U(args...);
}
template<class U, class... Args>
U *new_construct(U *Class, Args&&... args) {
void *Ptr = do_allocate(sizeof(U), std::alignment_of<U>::value);
return new (Ptr) U(args...);
}
size_t AmountAllocated() const { return LastAllocation; }
private:
void *do_allocate(std::size_t bytes, std::size_t alignment) override {
size_t PreviousAligned = Alloc::AlignUp(LastAllocation, alignment);
size_t NewOffset = PreviousAligned + bytes;
if (NewOffset > Size) {
return nullptr;
}
LastAllocation = NewOffset;
return reinterpret_cast<void*>(Begin + PreviousAligned);
}
void do_deallocate(void*, std::size_t, std::size_t) override {
// Do nothing
}
bool do_is_equal(const std::pmr::memory_resource& other) const noexcept override {
// Only if the allocator pointers are the same are they equal
if (this == &other) {
return true;
}
// We don't share state with another allocator so we can't share anything
return false;
}
uintptr_t Begin;
size_t Size;
size_t LastAllocation{};
};
class IntrusiveArenaAllocator final : public std::pmr::memory_resource {
public:
IntrusiveArenaAllocator(void* Ptr, size_t _Size)
: Begin {reinterpret_cast<uintptr_t>(Ptr)}
, Size {_Size} {
uint64_t NumberOfPages = _Size / PAGE_SIZE;
uint64_t UsedBits = Alloc::AlignUp(sizeof(IntrusiveArenaAllocator) +
Size / PAGE_SIZE / 8, PAGE_SIZE);
for (size_t i = 0; i < UsedBits; ++i) {
UsedPages.Set(i);
}
FreePages = NumberOfPages - UsedBits;
}
template<class U, class... Args>
U *new_construct(Args&&... args) {
void *Ptr = do_allocate(sizeof(U), std::alignment_of<U>::value);
return new (Ptr) U(args...);
}
template<class U, class... Args>
U *new_construct(U *Class, Args&&... args) {
void *Ptr = do_allocate(sizeof(U), std::alignment_of<U>::value);
return new (Ptr) U(args...);
}
uintptr_t GetSlabBase() const { return Begin; }
uint64_t GetSlabSize() const { return Size; }
uint64_t GetFreePages() const { return FreePages; }
private:
void *do_allocate(std::size_t bytes, std::size_t alignment) override {
std::scoped_lock<std::mutex> lk{AllocationMutex};
size_t NumberPages = Alloc::AlignUp(bytes, PAGE_SIZE) / PAGE_SIZE;
uintptr_t AllocatedOffset{};
try_again:
for (uintptr_t CurrentPage = LastAllocatedPageOffset; CurrentPage <= (Size - NumberPages);) {
size_t Remaining = NumberPages;
while (Remaining) {
if (UsedPages[CurrentPage + Remaining - 1]) {
// Has an intersecting range
break;
}
--Remaining;
}
if (Remaining) {
// Didn't find an allocation range
CurrentPage += Remaining;
}
else {
// We have a range to allocate
AllocatedOffset = CurrentPage;
break;
}
}
if (!AllocatedOffset && LastAllocatedPageOffset != 0) {
// Try again but starting from the beginning
LastAllocatedPageOffset = 0;
// Using goto so we don't have recursive mutex shenanigans
goto try_again;
}
// Allocated offset must be valid or zero at this point
if (AllocatedOffset) {
// Map the range as no longer available
for (size_t i = 0; i < NumberPages; ++i) {
UsedPages.Set(AllocatedOffset + i);
}
LastAllocatedPageOffset = AllocatedOffset + NumberPages;
// Now convert this base page to a pointer and return it
return reinterpret_cast<void*>(Begin + AllocatedOffset * PAGE_SIZE);
}
return nullptr;
}
void do_deallocate(void* p, std::size_t bytes, std::size_t alignment) override {
std::scoped_lock<std::mutex> lk{AllocationMutex};
uintptr_t PageOffset = (reinterpret_cast<uintptr_t>(p) - Begin) / PAGE_SIZE;
size_t NumPages = AlignUp(bytes, PAGE_SIZE) / PAGE_SIZE;
// Walk the allocation list and deallocate
uint64_t FreedPages{};
for (size_t i = 0; i < NumPages; ++i) {
FreedPages += UsedPages.TestAndClear(PageOffset + i) ? 1 : 0;
}
FreePages += FreedPages;
}
bool do_is_equal(const std::pmr::memory_resource& other) const noexcept override {
// Only if the allocator pointers are the same are they equal
if (this == &other) {
return true;
}
// We don't share state with another allocator so we can't share anything
return false;
}
uintptr_t Begin;
size_t Size;
uint64_t FreePages{};
size_t LastAllocatedPageOffset{};
std::mutex AllocationMutex{};
// For up to 64GB regions this will require up to 2MB tracking
// Needs to be the last element
FlexBitSet<uint64_t> UsedPages;
};
}
@@ -6,7 +6,7 @@ $end_info$
*/
#include <FEXCore/Utils/Common/MathUtils.h>
#include <FEXCore/Utils/ELFContainer.h>
#include <FEXCore/Utils/ELFLoader.h>
#include <FEXCore/Utils/LogManager.h>
#include <cstring>
#include <elf.h>
@@ -134,7 +134,7 @@ ELFContainer::ELFContainer(std::string const &Filename, std::string const &RootF
//PrintInitArray();
//PrintDynamicTable();
//LOGMAN_THROW_A(InterpreterHeader == nullptr, "Can only handle static programs");
//LogMan::Throw::A(InterpreterHeader == nullptr, "Can only handle static programs");
}
ELFContainer::~ELFContainer() {
@@ -196,8 +196,8 @@ bool ELFContainer::LoadELF_32() {
memcpy(&Header, reinterpret_cast<Elf32_Ehdr *>(&RawFile.at(0)),
sizeof(Elf32_Ehdr));
LOGMAN_THROW_A(Header._32.e_phentsize == sizeof(Elf32_Phdr), "PH Entry size wasn't correct size");
LOGMAN_THROW_A(Header._32.e_shentsize == sizeof(Elf32_Shdr), "PH Entry size wasn't correct size");
LogMan::Throw::A(Header._32.e_phentsize == sizeof(Elf32_Phdr), "PH Entry size wasn't correct size");
LogMan::Throw::A(Header._32.e_shentsize == sizeof(Elf32_Shdr), "PH Entry size wasn't correct size");
if (Header._32.e_machine != EM_386) {
LogMan::Msg::D("32bit ELF wasn't x86 based");
@@ -237,8 +237,8 @@ bool ELFContainer::LoadELF_64() {
memcpy(&Header, reinterpret_cast<Elf64_Ehdr *>(&RawFile.at(0)),
sizeof(Elf64_Ehdr));
LOGMAN_THROW_A(Header._64.e_phentsize == 56, "PH Entry size wasn't 56");
LOGMAN_THROW_A(Header._64.e_shentsize == 64, "PH Entry size wasn't 64");
LogMan::Throw::A(Header._64.e_phentsize == 56, "PH Entry size wasn't 56");
LogMan::Throw::A(Header._64.e_shentsize == 64, "PH Entry size wasn't 64");
if (Header._64.e_machine != EM_X86_64) {
LogMan::Msg::D("64bit ELF wasn't x86-64 based");
@@ -408,9 +408,9 @@ void ELFContainer::CalculateSymbols() {
uint64_t NumSymTabSymbols = 0;
uint64_t NumDynSymSymbols = 0;
if (SymTabHeader) {
LOGMAN_THROW_A(SymTabHeader->sh_link < SectionHeaders.size(),
LogMan::Throw::A(SymTabHeader->sh_link < SectionHeaders.size(),
"Symbol table string table section is wrong");
LOGMAN_THROW_A(SymTabHeader->sh_entsize == sizeof(Elf32_Sym),
LogMan::Throw::A(SymTabHeader->sh_entsize == sizeof(Elf32_Sym),
"Entry size doesn't match symbol entry");
StringTableHeader = SectionHeaders.at(SymTabHeader->sh_link)._32;
@@ -419,9 +419,9 @@ void ELFContainer::CalculateSymbols() {
}
if (DynSymTabHeader) {
LOGMAN_THROW_A(DynSymTabHeader->sh_link < SectionHeaders.size(),
LogMan::Throw::A(DynSymTabHeader->sh_link < SectionHeaders.size(),
"Symbol table string table section is wrong");
LOGMAN_THROW_A(DynSymTabHeader->sh_entsize == sizeof(Elf32_Sym),
LogMan::Throw::A(DynSymTabHeader->sh_entsize == sizeof(Elf32_Sym),
"Entry size doesn't match symbol entry");
DynStringTableHeader = SectionHeaders.at(DynSymTabHeader->sh_link)._32;
@@ -477,36 +477,6 @@ void ELFContainer::CalculateSymbols() {
}
}
}
Elf32_Shdr const *StrHeader = SectionHeaders.at(Header._32.e_shstrndx)._32;
char const *SHStrings = &RawFile.at(StrHeader->sh_offset);
for (uint32_t i = 0; i < SectionHeaders.size(); ++i) {
Elf32_Shdr const *hdr = SectionHeaders.at(i)._32;
if (strcmp(&SHStrings[hdr->sh_name], ".eh_frame_hdr") == 0) {
auto eh_frame_hdr = &RawFile.at(hdr->sh_offset);
// we only handle this specific unwind table encoding
if (eh_frame_hdr[0] == 1 && eh_frame_hdr[1] == 0x1B && eh_frame_hdr[2] == 0x3 && eh_frame_hdr[3] == 0x3b) {
// ptr enc : 4 bytes, signed, pcrel
// fde count : 4 bytes udata
// table enc : 4 bytes, signed, datarel
int fde_count = *(int*)(eh_frame_hdr + 8);
UnwindEntries.clear();
UnwindEntries.reserve(fde_count);
struct entry {
int32_t pc;
int32_t fde;
};
entry *Table = (entry*)(eh_frame_hdr+12);
for (int f = 0; f < fde_count; f++) {
uintptr_t Entry = (uintptr_t)(Table[f].pc + hdr->sh_offset);
UnwindEntries.push_back(Entry);
}
}
break;
}
}
}
else {
Elf64_Shdr const *SymTabHeader{nullptr};
@@ -541,9 +511,9 @@ void ELFContainer::CalculateSymbols() {
uint64_t NumSymTabSymbols = 0;
uint64_t NumDynSymSymbols = 0;
if (SymTabHeader) {
LOGMAN_THROW_A(SymTabHeader->sh_link < SectionHeaders.size(),
LogMan::Throw::A(SymTabHeader->sh_link < SectionHeaders.size(),
"Symbol table string table section is wrong");
LOGMAN_THROW_A(SymTabHeader->sh_entsize == sizeof(Elf64_Sym),
LogMan::Throw::A(SymTabHeader->sh_entsize == sizeof(Elf64_Sym),
"Entry size doesn't match symbol entry");
StringTableHeader = SectionHeaders.at(SymTabHeader->sh_link)._64;
@@ -552,9 +522,9 @@ void ELFContainer::CalculateSymbols() {
}
if (DynSymTabHeader) {
LOGMAN_THROW_A(DynSymTabHeader->sh_link < SectionHeaders.size(),
LogMan::Throw::A(DynSymTabHeader->sh_link < SectionHeaders.size(),
"Symbol table string table section is wrong");
LOGMAN_THROW_A(DynSymTabHeader->sh_entsize == sizeof(Elf64_Sym),
LogMan::Throw::A(DynSymTabHeader->sh_entsize == sizeof(Elf64_Sym),
"Entry size doesn't match symbol entry");
DynStringTableHeader = SectionHeaders.at(DynSymTabHeader->sh_link)._64;
@@ -610,36 +580,6 @@ void ELFContainer::CalculateSymbols() {
}
}
}
Elf64_Shdr const *StrHeader = SectionHeaders.at(Header._64.e_shstrndx)._64;
char const *SHStrings = &RawFile.at(StrHeader->sh_offset);
for (uint32_t i = 0; i < SectionHeaders.size(); ++i) {
Elf64_Shdr const *hdr = SectionHeaders.at(i)._64;
if (strcmp(&SHStrings[hdr->sh_name], ".eh_frame_hdr") == 0) {
auto eh_frame_hdr = &RawFile.at(hdr->sh_offset);
// we only handle this specific unwind table encoding
if (eh_frame_hdr[0] == 1 && eh_frame_hdr[1] == 0x1B && eh_frame_hdr[2] == 0x3 && eh_frame_hdr[3] == 0x3b) {
// ptr enc : 4 bytes, signed, pcrel
// fde count : 4 bytes udata
// table enc : 4 bytes, signed, datarel
int fde_count = *(int*)(eh_frame_hdr + 8);
UnwindEntries.clear();
UnwindEntries.reserve(fde_count);
struct entry {
int32_t pc;
int32_t fde;
};
entry *Table = (entry*)(eh_frame_hdr+12);
for (int f = 0; f < fde_count; f++) {
uintptr_t Entry = (uintptr_t)(Table[f].pc + hdr->sh_offset);
UnwindEntries.push_back(Entry);
}
}
break;
}
}
}
}
@@ -689,11 +629,6 @@ void ELFContainer::AddSymbols(SymbolAdder Adder) {
}
}
}
void ELFContainer::AddUnwindEntries(UnwindAdder Adder) {
for (auto Entry : UnwindEntries) {
Adder(Entry);
}
}
void ELFContainer::PrintHeader() const {
if (Mode == MODE_32BIT) {
@@ -730,7 +665,7 @@ void ELFContainer::PrintHeader() const {
void ELFContainer::PrintSectionHeaders() const {
if (Mode == MODE_32BIT) {
LOGMAN_THROW_A(Header._32.e_shstrndx < SectionHeaders.size(),
LogMan::Throw::A(Header._32.e_shstrndx < SectionHeaders.size(),
"String index section is wrong index!");
Elf32_Shdr const *StrHeader = SectionHeaders.at(Header._32.e_shstrndx)._32;
char const *SHStrings = &RawFile.at(StrHeader->sh_offset);
@@ -750,7 +685,7 @@ void ELFContainer::PrintSectionHeaders() const {
}
}
else {
LOGMAN_THROW_A(Header._64.e_shstrndx < SectionHeaders.size(),
LogMan::Throw::A(Header._64.e_shstrndx < SectionHeaders.size(),
"String index section is wrong index!");
Elf64_Shdr const *StrHeader = SectionHeaders.at(Header._64.e_shstrndx)._64;
char const *SHStrings = &RawFile.at(StrHeader->sh_offset);
@@ -773,7 +708,7 @@ void ELFContainer::PrintSectionHeaders() const {
void ELFContainer::PrintProgramHeaders() const {
if (Mode == MODE_32BIT) {
LOGMAN_THROW_A(Header._32.e_shstrndx < SectionHeaders.size(),
LogMan::Throw::A(Header._32.e_shstrndx < SectionHeaders.size(),
"String index section is wrong index!");
for (uint32_t i = 0; i < ProgramHeaders.size(); ++i) {
Elf32_Phdr const *hdr = ProgramHeaders.at(i)._32;
@@ -788,7 +723,7 @@ void ELFContainer::PrintProgramHeaders() const {
}
}
else {
LOGMAN_THROW_A(Header._64.e_shstrndx < SectionHeaders.size(),
LogMan::Throw::A(Header._64.e_shstrndx < SectionHeaders.size(),
"String index section is wrong index!");
for (uint32_t i = 0; i < ProgramHeaders.size(); ++i) {
Elf64_Phdr const *hdr = ProgramHeaders.at(i)._64;
@@ -822,9 +757,9 @@ void ELFContainer::PrintSymbolTable() const {
return;
}
LOGMAN_THROW_A(SymTabHeader->sh_link < SectionHeaders.size(),
LogMan::Throw::A(SymTabHeader->sh_link < SectionHeaders.size(),
"Symbol table string table section is wrong");
LOGMAN_THROW_A(SymTabHeader->sh_entsize == sizeof(Elf32_Sym),
LogMan::Throw::A(SymTabHeader->sh_entsize == sizeof(Elf32_Sym),
"Entry size doesn't match symbol entry");
StringTableHeader = SectionHeaders.at(SymTabHeader->sh_link)._32;
@@ -860,9 +795,9 @@ void ELFContainer::PrintSymbolTable() const {
return;
}
LOGMAN_THROW_A(SymTabHeader->sh_link < SectionHeaders.size(),
LogMan::Throw::A(SymTabHeader->sh_link < SectionHeaders.size(),
"Symbol table string table section is wrong");
LOGMAN_THROW_A(SymTabHeader->sh_entsize == sizeof(Elf64_Sym),
LogMan::Throw::A(SymTabHeader->sh_entsize == sizeof(Elf64_Sym),
"Entry size doesn't match symbol entry");
StringTableHeader = SectionHeaders.at(SymTabHeader->sh_link)._64;
@@ -907,12 +842,12 @@ void ELFContainer::PrintRelocationTable() const {
LogMan::Msg::D("Relocation Section: '%s'", &SHStrings[RelaHeader->sh_name]);
if (RelaHeader->sh_info != 0) {
LOGMAN_THROW_A(RelaHeader->sh_info < SectionHeaders.size(), "Rela header pointers to invalid GOT header");
LogMan::Throw::A(RelaHeader->sh_info < SectionHeaders.size(), "Rela header pointers to invalid GOT header");
GOTHeader = SectionHeaders.at(RelaHeader->sh_info)._64;
}
if (RelaHeader->sh_link != 0) {
LOGMAN_THROW_A(RelaHeader->sh_link < SectionHeaders.size(), "Rela header pointers to invalid dyndym header");
LogMan::Throw::A(RelaHeader->sh_link < SectionHeaders.size(), "Rela header pointers to invalid dyndym header");
DynSymHeader = SectionHeaders.at(RelaHeader->sh_link)._64;
StringTableHeader = SectionHeaders.at(DynSymHeader->sh_link)._64;
@@ -930,7 +865,7 @@ void ELFContainer::PrintRelocationTable() const {
LogMan::Msg::D("\toffset: 0x%lx", Entry->r_offset);
LogMan::Msg::D("\tSym: 0x%lx", Sym);
if (DynSymHeader && Sym != 0) {
LOGMAN_THROW_A(DynSymHeader->sh_entsize == sizeof(Elf64_Sym), "Oops, entry size doesn't match");
LogMan::Throw::A(DynSymHeader->sh_entsize == sizeof(Elf64_Sym), "Oops, entry size doesn't match");
uint64_t offset = DynSymHeader->sh_offset + Sym * DynSymHeader->sh_entsize;
Elf64_Sym const *Symbol =
@@ -993,12 +928,12 @@ void ELFContainer::FixupRelocations(void *ELFBase, uint64_t GuestELFBase, Symbol
RelaHeader = hdr;
if (RelaHeader->sh_info != 0) {
LOGMAN_THROW_A(RelaHeader->sh_info < SectionHeaders.size(), "Rela header pointers to invalid GOT header");
LogMan::Throw::A(RelaHeader->sh_info < SectionHeaders.size(), "Rela header pointers to invalid GOT header");
GOTHeader = SectionHeaders.at(RelaHeader->sh_info)._64;
}
if (RelaHeader->sh_link != 0) {
LOGMAN_THROW_A(RelaHeader->sh_link < SectionHeaders.size(), "Rela header pointers to invalid dyndym header");
LogMan::Throw::A(RelaHeader->sh_link < SectionHeaders.size(), "Rela header pointers to invalid dyndym header");
DynSymHeader = SectionHeaders.at(RelaHeader->sh_link)._64;
StringTableHeader = SectionHeaders.at(DynSymHeader->sh_link)._64;
@@ -1015,7 +950,7 @@ void ELFContainer::FixupRelocations(void *ELFBase, uint64_t GuestELFBase, Symbol
Elf64_Sym const *EntrySymbol {nullptr};
char const *EntrySymbolName {nullptr};
if (DynSymHeader && Sym != 0) {
LOGMAN_THROW_A(DynSymHeader->sh_entsize == sizeof(Elf64_Sym), "Oops, entry size doesn't match");
LogMan::Throw::A(DynSymHeader->sh_entsize == sizeof(Elf64_Sym), "Oops, entry size doesn't match");
uint64_t offset = DynSymHeader->sh_offset + Sym * DynSymHeader->sh_entsize;
EntrySymbol =
@@ -1351,4 +1286,5 @@ void ELFContainer::GetInitLocations(uint64_t GuestELFBase, std::vector<uint64_t>
}
}
} // namespace ELFLoader
+4 -5
View File
@@ -6,7 +6,6 @@ $end_info$
*/
#include <FEXCore/Utils/ELFSymbolDatabase.h>
#include <FEXCore/Utils/Allocator.h>
#include <FEXCore/Utils/LogManager.h>
#include <FEXCore/Utils/Common/MathUtils.h>
@@ -83,7 +82,7 @@ ELFSymbolDatabase::ELFSymbolDatabase(::ELFLoader::ELFContainer *file)
if (NameToELF.find(Lib) == NameToELF.end()) {
std::string LibraryPath;
bool Found = FindLibraryFile(&LibraryPath, Lib.c_str());
LOGMAN_THROW_A(Found, "Couldn't find library '%s'", Lib.c_str());
LogMan::Throw::A(Found, "Couldn't find library '%s'", Lib.c_str());
auto Info = DynamicELFInfo.emplace_back(new ELFInfo{});
Info->Name = Lib;
Info->Container = new ::ELFLoader::ELFContainer(LibraryPath, {}, true);
@@ -111,7 +110,7 @@ ELFSymbolDatabase::ELFSymbolDatabase(::ELFLoader::ELFContainer *file)
FillSymbols();
if (LocalInfo.Container->WasDynamic() && File->GetMode() == ELFContainer::MODE_64BIT) {
ELFBase = FEXCore::Allocator::mmap(nullptr, ELFMemorySize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
ELFBase = mmap(nullptr, ELFMemorySize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
FillMemoryLayouts(reinterpret_cast<uintptr_t>(ELFBase));
FillInitializationOrder();
FillSymbols();
@@ -122,7 +121,7 @@ ELFSymbolDatabase::ELFSymbolDatabase(::ELFLoader::ELFContainer *file)
ELFSymbolDatabase::~ELFSymbolDatabase() {
if (ELFBase) {
FEXCore::Allocator::munmap(ELFBase, ELFMemorySize);
munmap(ELFBase, ELFMemorySize);
ELFBase = nullptr;
}
}
@@ -165,7 +164,7 @@ void ELFSymbolDatabase::FillMemoryLayouts(uint64_t DefinedBase) {
uint64_t CurrentELFAlignedSize = AlignUp(std::get<2>(LocalInfo.CustomLayout), 4096);
if (CurrentELFBase < 0x10000) {
// We can't allocate memory in the first 16KB, Hopefully no elfs require this.
LOGMAN_MSG_A("Elf requires memory mapped in the first 16kb");
LogMan::Msg::A("Elf requires memory mapped in the first 16kb");
}
std::get<2>(LocalInfo.CustomLayout) = CurrentELFAlignedSize;
+1 -19
View File
@@ -37,17 +37,7 @@ void UnInstallHandlers() { Handlers.clear(); }
Handler(Buffer);
}
FEX_TRAP_EXECUTION;
}
void MFmt(const char *fmt, const fmt::format_args& args) {
auto msg = fmt::vformat(fmt, args);
for (auto& Handler : Handlers) {
Handler(msg.c_str());
}
FEX_TRAP_EXECUTION;
__builtin_trap();
}
} // namespace Throw
@@ -77,13 +67,5 @@ void M(DebugLevels Level, const char *fmt, va_list args) {
}
}
void MFmtImpl(DebugLevels level, const char* fmt, const fmt::format_args& args) {
const auto msg = fmt::vformat(fmt, args);
for (auto& Handler : Handlers) {
Handler(level, msg.c_str());
}
}
} // namespace Msg
} // namespace LogMan
+1 -68
View File
@@ -1,59 +1,15 @@
#include <FEXCore/Utils/Allocator.h>
#include <FEXCore/Utils/LogManager.h>
#include <FEXCore/Utils/Threads.h>
#include <cstring>
#include <mutex>
#include <pthread.h>
#include <sys/mman.h>
#include <deque>
namespace FEXCore::Threads {
// Stack pool handling
struct StackPoolItem {
void *Ptr;
size_t Size;
};
std::mutex StackPoolMutex{};
std::deque<StackPoolItem> StackPool;
void *AllocateStackObject(size_t Size) {
std::unique_lock<std::mutex> lk{StackPoolMutex};
if (StackPool.size() == 0) {
// Nothing in the pool, just allocate
return FEXCore::Allocator::mmap(nullptr, Size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_GROWSDOWN, -1, 0);
}
// Keep the first item in the stack pool
auto Result = StackPool.front().Ptr;
StackPool.pop_front();
// Erase the rest as a garbage collection step
for (auto &Item : StackPool) {
FEXCore::Allocator::munmap(Item.Ptr, Item.Size);
}
return Result;
}
void AddStackToPool(void *Ptr, size_t Size) {
std::unique_lock<std::mutex> lk{StackPoolMutex};
StackPool.emplace_back(StackPoolItem{Ptr, Size});
}
void *InitializeThread(void *Ptr);
class PThread final : public Thread {
public:
PThread(FEXCore::Threads::ThreadFunc Func, void *Arg)
: UserFunc {Func}
, UserArg {Arg} {
PThread(FEXCore::Threads::ThreadFunc Func, void *Arg) {
pthread_attr_t Attr{};
Stack = AllocateStackObject(STACK_SIZE);
pthread_attr_init(&Attr);
pthread_attr_setstack(&Attr, Stack, STACK_SIZE);
pthread_create(&Thread, &Attr, Func, Arg);
pthread_attr_destroy(&Attr);
}
bool joinable() override {
@@ -82,33 +38,10 @@ namespace FEXCore::Threads {
return self == Thread;
}
void *Execute() {
return UserFunc(UserArg);
}
void FreeStack() {
AddStackToPool(Stack, STACK_SIZE);
}
private:
pthread_t Thread;
FEXCore::Threads::ThreadFunc UserFunc;
void *UserArg;
void *Stack{};
constexpr static size_t STACK_SIZE = 8 * 1024 * 1024;
};
void *InitializeThread(void *Ptr) {
PThread *Thread{reinterpret_cast<PThread*>(Ptr)};
// Run the user function
void *Result = Thread->Execute();
// Put the stack back in to the stack pool
Thread->FreeStack();
return Result;
}
std::unique_ptr<FEXCore::Threads::Thread> CreateThread_PThread(
ThreadFunc Func,
void* Arg) {
+31 -34
View File
@@ -1,7 +1,5 @@
#pragma once
#include <FEXCore/Core/Context.h>
#include <FEXCore/Utils/CompilerDefs.h>
#include <FEXCore/Utils/LogManager.h>
#include <list>
@@ -56,15 +54,15 @@ namespace Type {
#undef P
}
FEX_DEFAULT_VISIBILITY std::string GetDataDirectory();
FEX_DEFAULT_VISIBILITY std::string GetConfigDirectory(bool Global);
FEX_DEFAULT_VISIBILITY std::string GetConfigFileLocation();
FEX_DEFAULT_VISIBILITY std::string GetApplicationConfig(const std::string &Filename, bool Global);
std::string GetDataDirectory();
std::string GetConfigDirectory(bool Global);
std::string GetConfigFileLocation();
std::string GetApplicationConfig(std::string &Filename, bool Global);
using LayerValue = std::list<std::string>;
using LayerOptions = std::unordered_map<ConfigOption, LayerValue>;
class FEX_DEFAULT_VISIBILITY Layer {
class Layer {
public:
explicit Layer(const LayerType _Type);
virtual ~Layer();
@@ -96,57 +94,56 @@ namespace Type {
}
void Set(ConfigOption Option, std::string Data) {
OptionMap[Option].emplace_back(std::move(Data));
OptionMap[Option].emplace_back(Data);
}
void EraseSet(ConfigOption Option, std::string Data) {
Erase(Option);
Set(Option, std::move(Data));
}
void Erase(ConfigOption Option) {
OptionMap.erase(Option);
OptionMap[Option].emplace_back(Data);
}
LayerType GetLayerType() const { return Type; }
const LayerOptions &GetOptionMap() const { return OptionMap; }
const LayerType GetLayerType() const { return Type; }
const LayerOptions &GetOptionMap() { return OptionMap; }
protected:
const LayerType Type;
LayerOptions OptionMap;
void Erase(ConfigOption Option) {
OptionMap.erase(Option);
}
};
FEX_DEFAULT_VISIBILITY void Initialize();
FEX_DEFAULT_VISIBILITY void Shutdown();
void Initialize();
void Shutdown();
FEX_DEFAULT_VISIBILITY void Load();
FEX_DEFAULT_VISIBILITY void ReloadMetaLayer();
void Load();
void ReloadMetaLayer();
FEX_DEFAULT_VISIBILITY void AddLayer(std::unique_ptr<FEXCore::Config::Layer> _Layer);
void AddLayer(std::unique_ptr<FEXCore::Config::Layer> _Layer);
FEX_DEFAULT_VISIBILITY bool Exists(ConfigOption Option);
FEX_DEFAULT_VISIBILITY std::optional<LayerValue*> All(ConfigOption Option);
FEX_DEFAULT_VISIBILITY std::optional<std::string*> Get(ConfigOption Option);
bool Exists(ConfigOption Option);
std::optional<LayerValue*> All(ConfigOption Option);
std::optional<std::string*> Get(ConfigOption Option);
FEX_DEFAULT_VISIBILITY void Set(ConfigOption Option, std::string Data);
FEX_DEFAULT_VISIBILITY void Erase(ConfigOption Option);
FEX_DEFAULT_VISIBILITY void EraseSet(ConfigOption Option, std::string Data);
void Set(ConfigOption Option, std::string Data);
void EraseSet(ConfigOption Option, std::string Data);
template<typename T>
class FEX_DEFAULT_VISIBILITY Value {
class Value {
public:
template <typename TT = T,
typename std::enable_if<!std::is_same<TT, std::string>::value, int>::type = 0>
Value(FEXCore::Config::ConfigOption _Option, T Default)
: Option {_Option} {
ValueData = GetIfExists(Option, Default);
ValueData = FEXCore::Config::Value<T>::GetIfExists(Option, Default);
}
template <typename TT = T,
typename std::enable_if<std::is_same<TT, std::string>::value, int>::type = 0>
Value(FEXCore::Config::ConfigOption _Option, T Default)
: Option {_Option} {
ValueData = GetIfExists(Option, Default);
ValueData = FEXCore::Config::Value<T>::GetIfExists(Option, Default);
GetListIfExists(Option, &AppendList);
}
@@ -158,7 +155,7 @@ namespace Type {
ERROR_AND_DIE("FEXCore::Config::Value has no value");
}
ValueData = Get(Option);
ValueData = FEXCore::Config::Value<T>::Get(Option);
}
template <typename TT = T,
@@ -169,13 +166,13 @@ namespace Type {
ERROR_AND_DIE("FEXCore::Config::Value has no value");
}
ValueData = GetIfExists(Option);
ValueData = FEXCore::Config::Value<T>::GetIfExists(Option);
GetListIfExists(Option, &AppendList);
}
operator T() const { return ValueData; }
T operator()() const { return ValueData; }
Value<T>(T Value) { ValueData = std::move(Value); }
operator T() { return ValueData; }
T operator()() { return ValueData; }
Value<T>(T Value) { ValueData = Value; }
std::list<T> &All() { return AppendList; }
private:
+4 -7
View File
@@ -6,10 +6,7 @@ $end_info$
*/
#pragma once
#include <FEXCore/Utils/CompilerDefs.h>
#include <cstdint>
#include <stdint.h>
#include <string>
namespace FEXCore {
@@ -54,7 +51,7 @@ class LLVMCore;
* @return An executable function pointer that is theoretically compiled from this point.
* Is actually a function pointer of type `void (FEXCore::Core::ThreadState *Thread)
*/
virtual void *CompileCode(uint64_t Entry, FEXCore::IR::IRListView const *IR, FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) = 0;
virtual void *CompileCode(FEXCore::IR::IRListView const *IR, FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) = 0;
/**
* @brief Function for mapping memory in to the CPUBackend's visible space. Allows setting up virtual mappings if required
@@ -88,8 +85,8 @@ class LLVMCore;
virtual void ClearCache() {}
virtual void CopyNecessaryDataForCompileThread(CPUBackend *Original) {}
using AsmDispatch = FEX_NAKED void(*)(FEXCore::Core::CpuStateFrame *Frame);
using JITCallback = FEX_NAKED void(*)(FEXCore::Core::CpuStateFrame *Frame, uint64_t RIP);
using AsmDispatch = __attribute__((naked)) void(*)(FEXCore::Core::CpuStateFrame *Frame);
using JITCallback = __attribute__((naked)) void(*)(FEXCore::Core::CpuStateFrame *Frame, uint64_t RIP);
JITCallback CallbackPtr{};
protected:
+38 -13
View File
@@ -1,7 +1,6 @@
#pragma once
#include <cstdint>
#include <functional>
#include <string>
#include <vector>
namespace FEXCore {
@@ -16,32 +15,58 @@ class IREmitter;
*/
class CodeLoader {
public:
using MapperFn = std::function<void *(void *addr, size_t length, int prot, int flags, int fd, off_t offset)>;
using UnmapperFn = std::function<int(void *addr, size_t length)>;
virtual ~CodeLoader() = default;
/**
* @brief CPU Core uses this to choose what the stack size should be for this code
*/
virtual uint64_t StackSize() const = 0;
/**
* Returns the initial stack pointer
* @brief Allows the code loader to set up the stack the way it wants
*
* @param HostPtr The host facing pointer to the base of the stack.
* Size of memory will be at least the size that StackSize() returns
*
* @param GuestPtr The guest facing memory location where the base of the stack lives
*
* @return The location that the guest stack pointer register should be set to
*
* Probably will be GuestPtr + StackSize() - <Some amount>
*/
virtual uint64_t GetStackPointer() = 0;
virtual uint64_t SetupStack() = 0;
/**
* @brief Function to return the guest RIP that the code should start out at
*/
virtual uint64_t DefaultRIP() const = 0;
/**
* @brief Maps and copies the executable, also sets up stack
*/
virtual bool MapMemory(const MapperFn& Mapper, const UnmapperFn& Unmapper) { return false; }
virtual void GetInitLocations(std::vector<uint64_t> *Locations) {}
virtual std::vector<std::string> const *GetApplicationArguments() { return nullptr; }
/**
* @brief Allows the loader to map memory regions that it needs
*
* Code loader is expected to call the Mapper function with a memory offset and size for mapping
*
* @param Mapper Returns the host facing pointer for memory setup if the codfe loader needs to do things to it
*/
virtual void MapMemoryRegion() {}
/**
* @brief Memory writer function for loading code in to guest memory
*
* First argument = Data to write
* Second argument = Guest memory data location
* Third argument = Guest memory size
*/
virtual void LoadMemory() = 0;
/**
* @brief Get the final RIP we are supposed to end up on in a debugger
*
* @return When the debugger reaches this RIP then we know that we have completed
*/
virtual uint64_t GetFinalRIP() { return ~0ULL; }
virtual char const *FindSymbolNameInRange(uint64_t Address) { return nullptr; }
virtual void GetExecveArguments(std::vector<char const*> *Args) {}
virtual void GetAuxv(uint64_t& addr, uint64_t& size) {}
+38 -47
View File
@@ -5,12 +5,10 @@
#include <FEXCore/Core/SignalDelegator.h>
#include <FEXCore/Core/CPUID.h>
#include <FEXCore/Utils/CompilerDefs.h>
#include <istream>
#include <ostream>
#include <memory>
#include <set>
namespace FEXCore {
class CodeLoader;
@@ -46,13 +44,12 @@ namespace FEXCore::Context {
MODE_32BIT,
MODE_64BIT,
};
using CustomCPUFactoryType = std::function<std::unique_ptr<FEXCore::CPU::CPUBackend> (FEXCore::Context::Context*, FEXCore::Core::InternalThreadState *Thread)>;
using CustomCPUFactoryType = std::function<FEXCore::CPU::CPUBackend* (FEXCore::Context::Context*, FEXCore::Core::InternalThreadState *Thread)>;
/**
* @brief This initializes internal FEXCore state that is shared between contexts and requires overhead to setup
*/
FEX_DEFAULT_VISIBILITY void InitializeStaticTables(OperatingMode Mode = MODE_64BIT);
FEX_DEFAULT_VISIBILITY void ShutdownStaticTables();
void InitializeStaticTables(OperatingMode Mode = MODE_64BIT);
/**
* @brief [[threadsafe]] Create a new FEXCore context object
@@ -61,7 +58,7 @@ namespace FEXCore::Context {
*
* @return a new context object
*/
FEX_DEFAULT_VISIBILITY FEXCore::Context::Context *CreateNewContext();
FEXCore::Context::Context *CreateNewContext();
/**
* @brief Post creation context initialization
@@ -71,14 +68,14 @@ namespace FEXCore::Context {
*
* @return true if we managed to initialize correctly
*/
FEX_DEFAULT_VISIBILITY bool InitializeContext(FEXCore::Context::Context *CTX);
bool InitializeContext(FEXCore::Context::Context *CTX);
/**
* @brief Destroy the context object
*
* @param CTX
*/
FEX_DEFAULT_VISIBILITY void DestroyContext(FEXCore::Context::Context *CTX);
void DestroyContext(FEXCore::Context::Context *CTX);
/**
* @brief Allows setting up in memory code and other things prior to launchign code execution
@@ -88,17 +85,17 @@ namespace FEXCore::Context {
*
* @return true if we loaded code
*/
FEX_DEFAULT_VISIBILITY bool InitCore(FEXCore::Context::Context *CTX, FEXCore::CodeLoader *Loader);
bool InitCore(FEXCore::Context::Context *CTX, FEXCore::CodeLoader *Loader);
FEX_DEFAULT_VISIBILITY void SetExitHandler(FEXCore::Context::Context *CTX, std::function<void(uint64_t ThreadId, FEXCore::Context::ExitReason)> handler);
FEX_DEFAULT_VISIBILITY std::function<void(uint64_t ThreadId, FEXCore::Context::ExitReason)> GetExitHandler(FEXCore::Context::Context *CTX);
void SetExitHandler(FEXCore::Context::Context *CTX, std::function<void(uint64_t ThreadId, FEXCore::Context::ExitReason)> handler);
std::function<void(uint64_t ThreadId, FEXCore::Context::ExitReason)> GetExitHandler(FEXCore::Context::Context *CTX);
/**
* @brief Pauses execution on the CPU core
*
* Blocks until all threads have paused.
*/
FEX_DEFAULT_VISIBILITY void Pause(FEXCore::Context::Context *CTX);
void Pause(FEXCore::Context::Context *CTX);
/**
* @brief Starts (or continues) the CPU core
@@ -107,7 +104,7 @@ namespace FEXCore::Context {
* Use RunUntilExit() for synchonous executions
*
*/
FEX_DEFAULT_VISIBILITY void Run(FEXCore::Context::Context *CTX);
void Run(FEXCore::Context::Context *CTX);
/**
* @brief Runs the CPU core until it exits
@@ -119,9 +116,7 @@ namespace FEXCore::Context {
*
* @return The ExitReason for the parentthread.
*/
FEX_DEFAULT_VISIBILITY ExitReason RunUntilExit(FEXCore::Context::Context *CTX);
FEX_DEFAULT_VISIBILITY void CompileRIP(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP);
ExitReason RunUntilExit(FEXCore::Context::Context *CTX);
/**
* @brief Gets the program exit status
@@ -131,21 +126,21 @@ namespace FEXCore::Context {
*
* @return The program exit status
*/
FEX_DEFAULT_VISIBILITY int GetProgramStatus(FEXCore::Context::Context *CTX);
int GetProgramStatus(FEXCore::Context::Context *CTX);
/**
* @brief Tells the core to shutdown
*
* Blocks until shutdown
*/
FEX_DEFAULT_VISIBILITY void Stop(FEXCore::Context::Context *CTX);
void Stop(FEXCore::Context::Context *CTX);
/**
* @brief Executes one instruction
*
* Returns once execution is complete.
*/
FEX_DEFAULT_VISIBILITY void Step(FEXCore::Context::Context *CTX);
void Step(FEXCore::Context::Context *CTX);
/**
* @brief [[threadsafe]] Returns the ExitReason of the parent thread. Typically used for async result status
@@ -154,7 +149,7 @@ namespace FEXCore::Context {
*
* @return The ExitReason for the parentthread
*/
FEX_DEFAULT_VISIBILITY ExitReason GetExitReason(FEXCore::Context::Context *CTX);
ExitReason GetExitReason(FEXCore::Context::Context *CTX);
/**
* @brief [[theadsafe]] Checks if the Context is either done working or paused(in the case of single stepping)
@@ -165,7 +160,7 @@ namespace FEXCore::Context {
*
* @return true if the core is done or paused
*/
FEX_DEFAULT_VISIBILITY bool IsDone(FEXCore::Context::Context *CTX);
bool IsDone(FEXCore::Context::Context *CTX);
/**
* @brief Gets a copy the CPUState of the parent thread
@@ -173,7 +168,7 @@ namespace FEXCore::Context {
* @param CTX The context that we created
* @param State The state object to populate
*/
FEX_DEFAULT_VISIBILITY void GetCPUState(FEXCore::Context::Context *CTX, FEXCore::Core::CPUState *State);
void GetCPUState(FEXCore::Context::Context *CTX, FEXCore::Core::CPUState *State);
/**
* @brief Copies the CPUState provided to the parent thread
@@ -181,7 +176,7 @@ namespace FEXCore::Context {
* @param CTX The context that we created
* @param State The satate object to copy from
*/
FEX_DEFAULT_VISIBILITY void SetCPUState(FEXCore::Context::Context *CTX, FEXCore::Core::CPUState *State);
void SetCPUState(FEXCore::Context::Context *CTX, FEXCore::Core::CPUState *State);
/**
* @brief Allows the frontend to pass in a custom CPUBackend creation factory
@@ -191,7 +186,7 @@ namespace FEXCore::Context {
* @param CTX The context that we created
* @param Factory The factory that the context will call if the DefaultCore config ise set to CUSTOM
*/
FEX_DEFAULT_VISIBILITY void SetCustomCPUBackendFactory(FEXCore::Context::Context *CTX, CustomCPUFactoryType Factory);
void SetCustomCPUBackendFactory(FEXCore::Context::Context *CTX, CustomCPUFactoryType Factory);
/**
* @brief Sets up memory regions on the guest for mirroring within the guest's VM space
@@ -202,7 +197,7 @@ namespace FEXCore::Context {
*
* @return true when successfully mapped. false if there was an error adding
*/
FEX_DEFAULT_VISIBILITY bool AddVirtualMemoryMapping(FEXCore::Context::Context *CTX, uint64_t VirtualAddress, uint64_t PhysicalAddress, uint64_t Size);
bool AddVirtualMemoryMapping(FEXCore::Context::Context *CTX, uint64_t VirtualAddress, uint64_t PhysicalAddress, uint64_t Size);
/**
* @brief Allows the frontend to set a custom syscall handler
@@ -212,30 +207,26 @@ namespace FEXCore::Context {
* @param Syscall Which syscall ID to install a visitor to
* @param Visitor The Visitor to install
*/
FEX_DEFAULT_VISIBILITY void RegisterExternalSyscallVisitor(FEXCore::Context::Context *CTX, uint64_t Syscall, FEXCore::HLE::SyscallVisitor *Visitor);
void RegisterExternalSyscallVisitor(FEXCore::Context::Context *CTX, uint64_t Syscall, FEXCore::HLE::SyscallVisitor *Visitor);
FEX_DEFAULT_VISIBILITY void HandleCallback(FEXCore::Context::Context *CTX, uint64_t RIP);
void HandleCallback(FEXCore::Context::Context *CTX, uint64_t RIP);
FEX_DEFAULT_VISIBILITY void RegisterHostSignalHandler(FEXCore::Context::Context *CTX, int Signal, HostSignalDelegatorFunction Func);
FEX_DEFAULT_VISIBILITY void RegisterFrontendHostSignalHandler(FEXCore::Context::Context *CTX, int Signal, HostSignalDelegatorFunction Func);
void RegisterHostSignalHandler(FEXCore::Context::Context *CTX, int Signal, HostSignalDelegatorFunction Func);
void RegisterFrontendHostSignalHandler(FEXCore::Context::Context *CTX, int Signal, HostSignalDelegatorFunction Func);
FEX_DEFAULT_VISIBILITY FEXCore::Core::InternalThreadState* CreateThread(FEXCore::Context::Context *CTX, FEXCore::Core::CPUState *NewThreadState, uint64_t ParentTID);
FEX_DEFAULT_VISIBILITY void InitializeThread(FEXCore::Context::Context *CTX, FEXCore::Core::InternalThreadState *Thread);
FEX_DEFAULT_VISIBILITY void RunThread(FEXCore::Context::Context *CTX, FEXCore::Core::InternalThreadState *Thread);
FEX_DEFAULT_VISIBILITY void StopThread(FEXCore::Context::Context *CTX, FEXCore::Core::InternalThreadState *Thread);
FEX_DEFAULT_VISIBILITY void DestroyThread(FEXCore::Context::Context *CTX, FEXCore::Core::InternalThreadState *Thread);
FEX_DEFAULT_VISIBILITY void CleanupAfterFork(FEXCore::Context::Context *CTX, FEXCore::Core::InternalThreadState *Thread);
FEX_DEFAULT_VISIBILITY void SetSignalDelegator(FEXCore::Context::Context *CTX, FEXCore::SignalDelegator *SignalDelegation);
FEX_DEFAULT_VISIBILITY void SetSyscallHandler(FEXCore::Context::Context *CTX, FEXCore::HLE::SyscallHandler *Handler);
FEX_DEFAULT_VISIBILITY FEXCore::CPUID::FunctionResults RunCPUIDFunction(FEXCore::Context::Context *CTX, uint32_t Function, uint32_t Leaf);
FEXCore::Core::InternalThreadState* CreateThread(FEXCore::Context::Context *CTX, FEXCore::Core::CPUState *NewThreadState, uint64_t ParentTID);
void InitializeThread(FEXCore::Context::Context *CTX, FEXCore::Core::InternalThreadState *Thread);
void RunThread(FEXCore::Context::Context *CTX, FEXCore::Core::InternalThreadState *Thread);
void StopThread(FEXCore::Context::Context *CTX, FEXCore::Core::InternalThreadState *Thread);
void DestroyThread(FEXCore::Context::Context *CTX, FEXCore::Core::InternalThreadState *Thread);
void CleanupAfterFork(FEXCore::Context::Context *CTX, FEXCore::Core::InternalThreadState *Thread);
void SetSignalDelegator(FEXCore::Context::Context *CTX, FEXCore::SignalDelegator *SignalDelegation);
void SetSyscallHandler(FEXCore::Context::Context *CTX, FEXCore::HLE::SyscallHandler *Handler);
FEXCore::CPUID::FunctionResults RunCPUIDFunction(FEXCore::Context::Context *CTX, uint32_t Function, uint32_t Leaf);
FEX_DEFAULT_VISIBILITY void AddNamedRegion(FEXCore::Context::Context *CTX, uintptr_t Base, uintptr_t Length, uintptr_t Offset, const std::string& Name);
FEX_DEFAULT_VISIBILITY void RemoveNamedRegion(FEXCore::Context::Context *CTX, uintptr_t Base, uintptr_t Length);
FEX_DEFAULT_VISIBILITY void SetAOTIRLoader(FEXCore::Context::Context *CTX, std::function<int(const std::string&)> CacheReader);
FEX_DEFAULT_VISIBILITY void SetAOTIRWriter(FEXCore::Context::Context *CTX, std::function<std::unique_ptr<std::ostream>(const std::string&)> CacheWriter);
FEX_DEFAULT_VISIBILITY void FinalizeAOTIRCache(FEXCore::Context::Context *CTX);
FEX_DEFAULT_VISIBILITY void WriteFilesWithCode(FEXCore::Context::Context *CTX, std::function<void(const std::string& fileid, const std::string& filename)> Writer);
FEX_DEFAULT_VISIBILITY void FlushCodeRange(FEXCore::Core::InternalThreadState *Thread, uint64_t Start, uint64_t Length);
FEX_DEFAULT_VISIBILITY void ConfigureAOTGen(FEXCore::Core::InternalThreadState *Thread, std::set<uint64_t> *ExternalBranches, uint64_t SectionMaxAddress);
void AddNamedRegion(FEXCore::Context::Context *CTX, uintptr_t Base, uintptr_t Length, uintptr_t Offset, const std::string& Name);
void RemoveNamedRegion(FEXCore::Context::Context *CTX, uintptr_t Base, uintptr_t Length);
void SetAOTIRLoader(FEXCore::Context::Context *CTX, std::function<std::unique_ptr<std::istream>(const std::string&)> CacheReader);
bool WriteAOTIR(FEXCore::Context::Context *CTX, std::function<std::unique_ptr<std::ostream>(const std::string&)> CacheWriter);
void FlushCodeRange(FEXCore::Core::InternalThreadState *Thread, uint64_t Start, uint64_t Length);
}
+3 -6
View File
@@ -1,15 +1,12 @@
#pragma once
#include <FEXCore/HLE/Linux/ThreadManagement.h>
#include <FEXCore/Utils/CompilerDefs.h>
#include <atomic>
#include <cstddef>
#include <stdint.h>
#include <string_view>
namespace FEXCore::Core {
struct FEX_PACKED CPUState {
struct __attribute__((packed)) CPUState {
uint64_t rip; ///< Current core's RIP. May not be entirely accurate while JIT is active
uint64_t gregs[16];
uint64_t : 64;
@@ -54,6 +51,6 @@ namespace FEXCore::Core {
constexpr uint64_t PAGE_SIZE = 4096;
FEX_DEFAULT_VISIBILITY std::string_view const& GetFlagName(unsigned Flag);
FEX_DEFAULT_VISIBILITY std::string_view const& GetGRegName(unsigned Reg);
std::string_view const& GetFlagName(unsigned Flag);
std::string_view const& GetGRegName(unsigned Reg);
}
+2 -7
View File
@@ -1,7 +1,4 @@
#pragma once
#include <FEXCore/Utils/CompilerDefs.h>
#include <cstdint>
#include <functional>
#include <signal.h>
@@ -10,11 +7,11 @@ namespace FEXCore {
namespace Core {
struct InternalThreadState;
}
struct FEX_PACKED GuestSAMask {
struct __attribute__((packed)) GuestSAMask {
uint64_t Val;
};
struct FEX_PACKED GuestSigAction {
struct __attribute__((packed)) GuestSigAction {
union {
void (*handler)(int);
void (*sigaction)(int, siginfo_t *, void*);
@@ -30,8 +27,6 @@ namespace Core {
class SignalDelegator {
public:
virtual ~SignalDelegator() = default;
/**
* @brief Registers an emulated thread's object to a TLS object
*
+7 -10
View File
@@ -1,7 +1,4 @@
#pragma once
#include <FEXCore/Utils/CompilerDefs.h>
#include <cstddef>
#include <cstdint>
@@ -16,7 +13,7 @@ namespace FEXCore {
constexpr uint64_t UC_STRICT_RESTORE_SS = (1ULL << 2);
///< Describes the signal stack
struct FEX_PACKED stack_t {
struct __attribute__((packed)) stack_t {
void *ss_sp;
int32_t ss_flags;
uint32_t : 32;
@@ -24,7 +21,7 @@ namespace FEXCore {
};
static_assert(sizeof(FEXCore::x86_64::stack_t) == 24, "This needs to be the right size");
struct FEX_PACKED _libc_fpstate {
struct __attribute__((packed)) _libc_fpstate {
// This is in FXSAVE format
uint16_t fcw;
uint16_t fsw;
@@ -68,19 +65,19 @@ namespace FEXCore {
};
static_assert(FEX_REG_CR2 == 22, "Oops");
struct FEX_PACKED mcontext_t {
struct __attribute__((packed)) mcontext_t {
uint64_t gregs[23];
FEXCore::x86_64::_libc_fpstate *fpregs;
uint64_t __reserved[8];
};
static_assert(sizeof(FEXCore::x86_64::mcontext_t) == 256, "This needs to be the right size");
struct FEX_PACKED sigset_t {
struct __attribute__((packed)) sigset_t {
uint64_t val[16];
};
static_assert(sizeof(FEXCore::x86_64::sigset_t) == 128, "This needs to be the right size");
struct FEX_PACKED ucontext_t {
struct __attribute__((packed)) ucontext_t {
uint64_t uc_flags;
FEXCore::x86_64::ucontext_t *uc_link;
FEXCore::x86_64::stack_t uc_stack;
@@ -95,12 +92,12 @@ namespace FEXCore {
}
namespace x86 {
struct FEX_PACKED siginfo_t {
struct __attribute__((packed)) siginfo_t {
uint32_t pad[32];
};
static_assert(sizeof(FEXCore::x86::siginfo_t) == 128, "This needs to be the right size");
struct FEX_PACKED ucontext_t {
struct __attribute__((packed)) ucontext_t {
uint32_t pad[91];
};
static_assert(sizeof(FEXCore::x86::ucontext_t) == 364, "This needs to be the right size");
@@ -54,11 +54,11 @@ namespace FEXCore::Core {
std::vector<DebugDataSubblock> Subblocks;
};
enum class SignalEvent {
Nothing, // If the guest uses our signal we need to know it was errant on our end
Pause,
Stop,
Return,
enum SignalEvent {
SIGNALEVENT_NONE, // If the guest uses our signal we need to know it was errant on our end
SIGNALEVENT_PAUSE,
SIGNALEVENT_STOP,
SIGNALEVENT_RETURN,
};
struct LocalIREntry {
@@ -78,7 +78,7 @@ namespace FEXCore::Core {
} RunningEvents;
FEXCore::Context::Context *CTX;
std::atomic<SignalEvent> SignalReason{SignalEvent::Nothing};
std::atomic<SignalEvent> SignalReason {SignalEvent::SIGNALEVENT_NONE};
std::unique_ptr<FEXCore::Threads::Thread> ExecutionThread;
Event StartRunning;
@@ -107,7 +107,7 @@ namespace FEXCore::Core {
alignas(16) FEXCore::Core::CpuStateFrame BaseFrameState{};
};
// static_assert(std::is_standard_layout<InternalThreadState>::value, "This needs to be standard layout");
static_assert(std::is_standard_layout<InternalThreadState>::value, "This needs to be standard layout");
}
+58 -80
View File
@@ -1,7 +1,6 @@
#pragma once
#include <FEXCore/Core/Context.h>
#include <FEXCore/Utils/CompilerDefs.h>
#include <cstdint>
#include <cstring>
@@ -100,72 +99,54 @@ inline void PopOpAddrIf(uint32_t *Flags, uint32_t Flag) {
}
struct DecodedOperand {
enum class OpType : uint8_t {
Nothing,
GPR,
GPRDirect,
GPRIndirect,
RIPRelative,
Literal,
SIB,
union DecodedOperand {
enum {
TYPE_NONE,
TYPE_GPR,
TYPE_GPR_DIRECT,
TYPE_GPR_INDIRECT,
TYPE_RIP_RELATIVE,
TYPE_LITERAL,
TYPE_SIB,
};
bool IsNone() const {
return Type == OpType::Nothing;
}
bool IsGPR() const {
return Type == OpType::GPR;
}
bool IsGPRDirect() const {
return Type == OpType::GPRDirect;
}
bool IsGPRIndirect() const {
return Type == OpType::GPRIndirect;
}
bool IsRIPRelative() const {
return Type == OpType::RIPRelative;
}
bool IsLiteral() const {
return Type == OpType::Literal;
}
bool IsSIB() const {
return Type == OpType::SIB;
}
struct {
uint8_t Type;
} TypeNone;
union TypeUnion {
struct {
bool HighBits;
uint8_t GPR;
} GPR;
struct {
uint8_t Type;
bool HighBits;
uint8_t GPR;
} TypeGPR;
struct {
uint8_t GPR;
int32_t Displacement;
} GPRIndirect;
struct {
uint8_t Type;
uint8_t GPR;
int32_t Displacement;
} TypeGPRIndirect;
struct {
union {
int32_t s;
uint32_t u;
} Value;
} RIPLiteral;
struct {
uint8_t Size;
uint64_t Value;
struct {
uint8_t Type;
union {
int32_t s;
uint32_t u;
} Literal;
} TypeRIPLiteral;
struct {
uint8_t Index; // ~0 invalid
uint8_t Base; // ~0 invalid
uint32_t Scale : 8;
int32_t Offset;
} SIB;
};
struct {
uint8_t Type;
uint8_t Size;
uint64_t Literal;
} TypeLiteral;
OpType Type;
TypeUnion Data;
struct {
uint8_t Type;
uint8_t Index; // ~0 invalid
uint8_t Base; // ~0 invalid
uint32_t Scale : 8;
int32_t Offset;
} TypeSIB;
};
struct DecodedInst {
@@ -437,9 +418,6 @@ struct X86InstInfo {
// We don't care if the opcode dispatcher differs
return true;
}
bool operator!=(const X86InstInfo &b) const {
return !operator==(b);
}
};
static_assert(std::is_trivial<X86InstInfo>::value, "X86InstInfo needs to be trivial");
@@ -477,29 +455,29 @@ constexpr size_t MAX_XOP_GROUP_TABLE_SIZE = (1 << 6);
constexpr size_t MAX_EVEX_TABLE_SIZE = 256;
extern FEX_DEFAULT_VISIBILITY X86InstInfo BaseOps[MAX_PRIMARY_TABLE_SIZE];
extern FEX_DEFAULT_VISIBILITY X86InstInfo SecondBaseOps[MAX_SECOND_TABLE_SIZE];
extern FEX_DEFAULT_VISIBILITY X86InstInfo RepModOps[MAX_REP_MOD_TABLE_SIZE];
extern FEX_DEFAULT_VISIBILITY X86InstInfo RepNEModOps[MAX_REPNE_MOD_TABLE_SIZE];
extern FEX_DEFAULT_VISIBILITY X86InstInfo OpSizeModOps[MAX_OPSIZE_MOD_TABLE_SIZE];
extern FEX_DEFAULT_VISIBILITY X86InstInfo PrimaryInstGroupOps[MAX_INST_GROUP_TABLE_SIZE];
extern FEX_DEFAULT_VISIBILITY X86InstInfo SecondInstGroupOps[MAX_INST_SECOND_GROUP_TABLE_SIZE];
extern FEX_DEFAULT_VISIBILITY X86InstInfo SecondModRMTableOps[MAX_SECOND_MODRM_TABLE_SIZE];
extern FEX_DEFAULT_VISIBILITY X86InstInfo X87Ops[MAX_X87_TABLE_SIZE];
extern FEX_DEFAULT_VISIBILITY X86InstInfo DDDNowOps[MAX_3DNOW_TABLE_SIZE];
extern FEX_DEFAULT_VISIBILITY X86InstInfo H0F38TableOps[MAX_0F_38_TABLE_SIZE];
extern FEX_DEFAULT_VISIBILITY X86InstInfo H0F3ATableOps[MAX_0F_3A_TABLE_SIZE];
extern X86InstInfo BaseOps[MAX_PRIMARY_TABLE_SIZE];
extern X86InstInfo SecondBaseOps[MAX_SECOND_TABLE_SIZE];
extern X86InstInfo RepModOps[MAX_REP_MOD_TABLE_SIZE];
extern X86InstInfo RepNEModOps[MAX_REPNE_MOD_TABLE_SIZE];
extern X86InstInfo OpSizeModOps[MAX_OPSIZE_MOD_TABLE_SIZE];
extern X86InstInfo PrimaryInstGroupOps[MAX_INST_GROUP_TABLE_SIZE];
extern X86InstInfo SecondInstGroupOps[MAX_INST_SECOND_GROUP_TABLE_SIZE];
extern X86InstInfo SecondModRMTableOps[MAX_SECOND_MODRM_TABLE_SIZE];
extern X86InstInfo X87Ops[MAX_X87_TABLE_SIZE];
extern X86InstInfo DDDNowOps[MAX_3DNOW_TABLE_SIZE];
extern X86InstInfo H0F38TableOps[MAX_0F_38_TABLE_SIZE];
extern X86InstInfo H0F3ATableOps[MAX_0F_3A_TABLE_SIZE];
// VEX
extern FEX_DEFAULT_VISIBILITY X86InstInfo VEXTableOps[MAX_VEX_TABLE_SIZE];
extern FEX_DEFAULT_VISIBILITY X86InstInfo VEXTableGroupOps[MAX_VEX_GROUP_TABLE_SIZE];
extern X86InstInfo VEXTableOps[MAX_VEX_TABLE_SIZE];
extern X86InstInfo VEXTableGroupOps[MAX_VEX_GROUP_TABLE_SIZE];
// XOP
extern FEX_DEFAULT_VISIBILITY X86InstInfo XOPTableOps[MAX_XOP_TABLE_SIZE];
extern FEX_DEFAULT_VISIBILITY X86InstInfo XOPTableGroupOps[MAX_XOP_GROUP_TABLE_SIZE];
extern X86InstInfo XOPTableOps[MAX_XOP_TABLE_SIZE];
extern X86InstInfo XOPTableGroupOps[MAX_XOP_GROUP_TABLE_SIZE];
// EVEX
extern FEX_DEFAULT_VISIBILITY X86InstInfo EVEXTableOps[MAX_EVEX_TABLE_SIZE];
extern X86InstInfo EVEXTableOps[MAX_EVEX_TABLE_SIZE];
FEX_DEFAULT_VISIBILITY void InitializeInfoTables(Context::OperatingMode Mode);
void InitializeInfoTables(Context::OperatingMode Mode);
}
@@ -7,12 +7,12 @@ namespace FEXCore::HLE {
// Tracking relationships between thread IDs and such
class ThreadManagement {
public:
uint64_t GetUID() const { return UID; }
uint64_t GetGID() const { return GID; }
uint64_t GetEUID() const { return EUID; }
uint64_t GetEGID() const { return EGID; }
uint64_t GetTID() const { return TID; }
uint64_t GetPID() const { return PID; }
uint64_t GetUID() { return UID; }
uint64_t GetGID() { return GID; }
uint64_t GetEUID() { return EUID; }
uint64_t GetEGID() { return EGID; }
uint64_t GetTID() { return TID; }
uint64_t GetPID() { return PID; }
uint64_t UID{1000};
uint64_t GID{1000};
+1 -1
View File
@@ -3,7 +3,7 @@
#include <stdint.h>
namespace FEXCore::HLE {
#define INVALID_OP { LOGMAN_MSG_A("Tried to syscall with unknown number of registers"); return 0; }
#define INVALID_OP { LogMan::Msg::A("Tried to syscall with unknown number of registers"); return 0; }
class SyscallVisitor {
public:
SyscallVisitor(uint32_t Mask) : SyscallVisitor(Mask, false) {}
+31 -37
View File
@@ -1,12 +1,8 @@
#pragma once
#include <FEXCore/Utils/CompilerDefs.h>
#include <array>
#include <cassert>
#include <cstdint>
#include <cstring>
#include <memory>
#include <string.h>
#include <sstream>
#include <tuple>
@@ -74,10 +70,11 @@ struct NodeWrapperBase final {
Type const *GetNode(uintptr_t Base) const { return reinterpret_cast<Type*>(Base + NodeOffset); }
void SetOffset(uintptr_t Base, uintptr_t Value) { NodeOffset = Value - Base; }
friend constexpr bool operator==(const NodeWrapperBase<Type>&, const NodeWrapperBase<Type>&) = default;
constexpr bool operator==(NodeWrapperBase<Type> const &rhs) const { return NodeOffset == rhs.NodeOffset; }
constexpr bool operator!=(NodeWrapperBase<Type> const &rhs) const { return !operator==(rhs); }
};
static_assert(std::is_trivial_v<NodeWrapperBase<OrderedNode>>);
static_assert(std::is_trivial<NodeWrapperBase<OrderedNode>>::value);
static_assert(sizeof(NodeWrapperBase<OrderedNode>) == sizeof(uint32_t));
@@ -243,11 +240,13 @@ class OrderedNode final {
}
static void SetPrevious(uintptr_t Base, value_type Node, value_type New) {
if (Node.NodeOffset == 0) return;
OrderedNode *RealNode = Node.GetNode(Base);
RealNode->Header.Previous = New;
}
static void SetNext(uintptr_t Base, value_type Node, value_type New) {
if (Node.NodeOffset == 0) return;
OrderedNode *RealNode = Node.GetNode(Base);
RealNode->Header.Next = New;
}
@@ -255,81 +254,76 @@ class OrderedNode final {
void SetUses(uint32_t Uses) { NumUses = Uses; }
};
static_assert(std::is_trivial_v<OrderedNode>);
static_assert(std::is_trivially_copyable_v<OrderedNode>);
static_assert(std::is_trivial<OrderedNode>::value);
static_assert(std::is_trivially_copyable<OrderedNode>::value);
static_assert(offsetof(OrderedNode, Header) == 0);
static_assert(sizeof(OrderedNode) == (sizeof(OrderedNodeHeader) + sizeof(uint32_t)));
struct RegisterClassType final {
uint32_t Val;
constexpr operator uint32_t() const {
operator uint32_t() {
return Val;
}
friend constexpr bool operator==(const RegisterClassType&, const RegisterClassType&) = default;
constexpr bool operator==(RegisterClassType const &rhs) const { return Val == rhs.Val; }
constexpr bool operator!=(RegisterClassType const &rhs) const { return !operator==(rhs); }
};
struct CondClassType final {
uint8_t Val;
constexpr operator uint8_t() const {
operator uint8_t() {
return Val;
}
friend constexpr bool operator==(const CondClassType&, const CondClassType&) = default;
};
struct MemOffsetType final {
uint8_t Val;
constexpr operator uint8_t() const {
operator uint8_t() {
return Val;
}
friend constexpr bool operator==(const MemOffsetType&, const MemOffsetType&) = default;
int operator ==(const MemOffsetType other) {
return Val == other.Val;
}
int operator !=(const MemOffsetType other) {
return Val != other.Val;
}
};
struct TypeDefinition final {
uint16_t Val;
constexpr operator uint16_t() const {
operator uint16_t() const {
return Val;
}
static constexpr TypeDefinition Create(uint8_t Bytes) {
static TypeDefinition Create(uint8_t Bytes) {
TypeDefinition Type{};
Type.Val = Bytes << 8;
return Type;
}
static constexpr TypeDefinition Create(uint8_t Bytes, uint8_t Elements) {
static TypeDefinition Create(uint8_t Bytes, uint8_t Elements) {
TypeDefinition Type{};
Type.Val = (Bytes << 8) | (Elements & 255);
return Type;
}
constexpr uint8_t Bytes() const {
uint8_t Bytes() const {
return Val >> 8;
}
constexpr uint8_t Elements() const {
uint8_t Elements() const {
return Val & 255;
}
friend constexpr bool operator==(const TypeDefinition&, const TypeDefinition&) = default;
};
static_assert(std::is_trivial_v<TypeDefinition>);
static_assert(std::is_trivial<TypeDefinition>::value);
struct FenceType final {
uint8_t Val;
constexpr operator uint8_t() const {
operator uint8_t() const {
return Val;
}
friend constexpr bool operator==(const FenceType&, const FenceType&) = default;
};
struct RoundType final {
uint8_t Val;
constexpr operator uint8_t() const {
return Val;
}
friend constexpr bool operator==(const RoundType&, const RoundType&) = default;
constexpr bool operator==(FenceType const &rhs) const { return Val == rhs.Val; }
constexpr bool operator!=(FenceType const &rhs) const { return !operator==(rhs); }
};
struct SHA256Sum final {
@@ -391,7 +385,7 @@ public:
return { RealNode, RealNode->Op(IRList) };
}
uint32_t ID() const {
uint32_t ID() {
return Node.ID();
}
@@ -471,8 +465,8 @@ public:
class IRListView;
class IREmitter;
FEX_DEFAULT_VISIBILITY void Dump(std::stringstream *out, IRListView const* IR, IR::RegisterAllocationData *RAData);
FEX_DEFAULT_VISIBILITY std::unique_ptr<IREmitter> Parse(std::istream *in);
void Dump(std::stringstream *out, IRListView const* IR, IR::RegisterAllocationData *RAData);
IREmitter* Parse(std::istream *in);
template<typename Type>
inline uint32_t NodeWrapperBase<Type>::ID() const { return NodeOffset / sizeof(IR::OrderedNode); }
Loaded 100 of 289 files, more files were not shown because too many files have changed in this diff. Show more