mirror of
https://github.com/FEX-Emu/FEX.git
synced 2026-10-07 13:00:15 +02:00
Whole-tree reformat
This follows discussions from #3413. Followup commits add clang-format file, script and blame ignore lists.
This commit is contained in:
1 parent
028c220041
commit
2b4ec88dae
447 files changed
+78590
-73070
No files matched your search
+281
-287
@@ -44,16 +44,16 @@ $end_info$
|
||||
#include <wine/unixlib.h>
|
||||
|
||||
namespace ControlBits {
|
||||
// When this is unset, a thread can be safely interrupted and have its context recovered
|
||||
// IMPORTANT: This can only safely be written by the owning thread
|
||||
static constexpr uint32_t IN_JIT{1U << 0};
|
||||
// When this is unset, a thread can be safely interrupted and have its context recovered
|
||||
// IMPORTANT: This can only safely be written by the owning thread
|
||||
static constexpr uint32_t IN_JIT {1U << 0};
|
||||
|
||||
// JIT entry polls this bit until it is unset, at which point CONTROL_IN_JIT will be set
|
||||
static constexpr uint32_t PAUSED{1U << 1};
|
||||
// JIT entry polls this bit until it is unset, at which point CONTROL_IN_JIT will be set
|
||||
static constexpr uint32_t PAUSED {1U << 1};
|
||||
|
||||
// When this is set, the CPU context stored in the CPU area has not yet been flushed to the FEX TLS
|
||||
static constexpr uint32_t WOW_CPU_AREA_DIRTY{1U << 2};
|
||||
};
|
||||
// When this is set, the CPU context stored in the CPU area has not yet been flushed to the FEX TLS
|
||||
static constexpr uint32_t WOW_CPU_AREA_DIRTY {1U << 2};
|
||||
}; // namespace ControlBits
|
||||
|
||||
struct TLS {
|
||||
enum class Slot : size_t {
|
||||
@@ -62,299 +62,295 @@ struct TLS {
|
||||
THREAD_STATE = WOW64_TLS_MAX_NUMBER - 2,
|
||||
};
|
||||
|
||||
_TEB *TEB;
|
||||
_TEB* TEB;
|
||||
|
||||
explicit TLS(_TEB *TEB) : TEB(TEB) {}
|
||||
explicit TLS(_TEB* TEB)
|
||||
: TEB(TEB) {}
|
||||
|
||||
std::atomic<uint32_t> &ControlWord() const {
|
||||
std::atomic<uint32_t>& ControlWord() const {
|
||||
// TODO: Change this when libc++ gains std::atomic_ref support
|
||||
return reinterpret_cast<std::atomic<uint32_t> &>(TEB->TlsSlots[FEXCore::ToUnderlying(Slot::CONTROL_WORD)]);
|
||||
return reinterpret_cast<std::atomic<uint32_t>&>(TEB->TlsSlots[FEXCore::ToUnderlying(Slot::CONTROL_WORD)]);
|
||||
}
|
||||
|
||||
CONTEXT *&EntryContext() const {
|
||||
return reinterpret_cast<CONTEXT *&>(TEB->TlsSlots[FEXCore::ToUnderlying(Slot::ENTRY_CONTEXT)]);
|
||||
CONTEXT*& EntryContext() const {
|
||||
return reinterpret_cast<CONTEXT*&>(TEB->TlsSlots[FEXCore::ToUnderlying(Slot::ENTRY_CONTEXT)]);
|
||||
}
|
||||
|
||||
FEXCore::Core::InternalThreadState *&ThreadState() const {
|
||||
return reinterpret_cast<FEXCore::Core::InternalThreadState *&>(TEB->TlsSlots[FEXCore::ToUnderlying(Slot::THREAD_STATE)]);
|
||||
FEXCore::Core::InternalThreadState*& ThreadState() const {
|
||||
return reinterpret_cast<FEXCore::Core::InternalThreadState*&>(TEB->TlsSlots[FEXCore::ToUnderlying(Slot::THREAD_STATE)]);
|
||||
}
|
||||
};
|
||||
|
||||
class WowSyscallHandler;
|
||||
|
||||
namespace {
|
||||
namespace BridgeInstrs {
|
||||
// These directly jumped to by the guest to make system calls
|
||||
uint16_t Syscall{0x2ecd};
|
||||
uint16_t UnixCall{0x2ecd};
|
||||
}
|
||||
namespace BridgeInstrs {
|
||||
// These directly jumped to by the guest to make system calls
|
||||
uint16_t Syscall {0x2ecd};
|
||||
uint16_t UnixCall {0x2ecd};
|
||||
} // namespace BridgeInstrs
|
||||
|
||||
fextl::unique_ptr<FEXCore::Context::Context> CTX;
|
||||
fextl::unique_ptr<FEX::DummyHandlers::DummySignalDelegator> SignalDelegator;
|
||||
fextl::unique_ptr<WowSyscallHandler> SyscallHandler;
|
||||
fextl::unique_ptr<FEXCore::Context::Context> CTX;
|
||||
fextl::unique_ptr<FEX::DummyHandlers::DummySignalDelegator> SignalDelegator;
|
||||
fextl::unique_ptr<WowSyscallHandler> SyscallHandler;
|
||||
|
||||
FEX::Windows::InvalidationTracker InvalidationTracker;
|
||||
std::optional<FEX::Windows::CPUFeatures> CPUFeatures;
|
||||
FEX::Windows::InvalidationTracker InvalidationTracker;
|
||||
std::optional<FEX::Windows::CPUFeatures> CPUFeatures;
|
||||
|
||||
std::mutex ThreadSuspendLock;
|
||||
std::unordered_set<DWORD> InitializedWOWThreads; // Set of TIDs, `ThreadSuspendLock` must be locked when accessing
|
||||
std::mutex ThreadSuspendLock;
|
||||
std::unordered_set<DWORD> InitializedWOWThreads; // Set of TIDs, `ThreadSuspendLock` must be locked when accessing
|
||||
|
||||
std::pair<NTSTATUS, TLS> GetThreadTLS(HANDLE Thread) {
|
||||
THREAD_BASIC_INFORMATION Info;
|
||||
const NTSTATUS Err = NtQueryInformationThread(Thread, ThreadBasicInformation, &Info, sizeof(Info), nullptr);
|
||||
return {Err, TLS{reinterpret_cast<_TEB *>(Info.TebBaseAddress)}};
|
||||
}
|
||||
|
||||
TLS GetTLS() {
|
||||
return TLS{NtCurrentTeb()};
|
||||
}
|
||||
|
||||
uint64_t GetWowTEB(void *TEB) {
|
||||
static constexpr size_t WowTEBOffsetMemberOffset{0x180c};
|
||||
return static_cast<uint64_t>(*reinterpret_cast<LONG *>(reinterpret_cast<uintptr_t>(TEB) + WowTEBOffsetMemberOffset)
|
||||
+ reinterpret_cast<uint64_t>(TEB));
|
||||
}
|
||||
|
||||
bool IsAddressInJit(uint64_t Address) {
|
||||
auto Thread = GetTLS().ThreadState();
|
||||
return Thread->CTX->IsAddressInCodeBuffer(Thread, Address);
|
||||
}
|
||||
std::pair<NTSTATUS, TLS> GetThreadTLS(HANDLE Thread) {
|
||||
THREAD_BASIC_INFORMATION Info;
|
||||
const NTSTATUS Err = NtQueryInformationThread(Thread, ThreadBasicInformation, &Info, sizeof(Info), nullptr);
|
||||
return {Err, TLS {reinterpret_cast<_TEB*>(Info.TebBaseAddress)}};
|
||||
}
|
||||
|
||||
TLS GetTLS() {
|
||||
return TLS {NtCurrentTeb()};
|
||||
}
|
||||
|
||||
uint64_t GetWowTEB(void* TEB) {
|
||||
static constexpr size_t WowTEBOffsetMemberOffset {0x180c};
|
||||
return static_cast<uint64_t>(
|
||||
*reinterpret_cast<LONG*>(reinterpret_cast<uintptr_t>(TEB) + WowTEBOffsetMemberOffset) + reinterpret_cast<uint64_t>(TEB));
|
||||
}
|
||||
|
||||
bool IsAddressInJit(uint64_t Address) {
|
||||
auto Thread = GetTLS().ThreadState();
|
||||
return Thread->CTX->IsAddressInCodeBuffer(Thread, Address);
|
||||
}
|
||||
} // namespace
|
||||
|
||||
namespace Context {
|
||||
void LoadStateFromWowContext(FEXCore::Core::InternalThreadState *Thread, uint64_t WowTEB, WOW64_CONTEXT *Context) {
|
||||
auto &State = Thread->CurrentFrame->State;
|
||||
void LoadStateFromWowContext(FEXCore::Core::InternalThreadState* Thread, uint64_t WowTEB, WOW64_CONTEXT* Context) {
|
||||
auto& State = Thread->CurrentFrame->State;
|
||||
|
||||
// General register state
|
||||
// General register state
|
||||
|
||||
State.gregs[FEXCore::X86State::REG_RAX] = Context->Eax;
|
||||
State.gregs[FEXCore::X86State::REG_RBX] = Context->Ebx;
|
||||
State.gregs[FEXCore::X86State::REG_RCX] = Context->Ecx;
|
||||
State.gregs[FEXCore::X86State::REG_RDX] = Context->Edx;
|
||||
State.gregs[FEXCore::X86State::REG_RSI] = Context->Esi;
|
||||
State.gregs[FEXCore::X86State::REG_RDI] = Context->Edi;
|
||||
State.gregs[FEXCore::X86State::REG_RBP] = Context->Ebp;
|
||||
State.gregs[FEXCore::X86State::REG_RSP] = Context->Esp;
|
||||
State.gregs[FEXCore::X86State::REG_RAX] = Context->Eax;
|
||||
State.gregs[FEXCore::X86State::REG_RBX] = Context->Ebx;
|
||||
State.gregs[FEXCore::X86State::REG_RCX] = Context->Ecx;
|
||||
State.gregs[FEXCore::X86State::REG_RDX] = Context->Edx;
|
||||
State.gregs[FEXCore::X86State::REG_RSI] = Context->Esi;
|
||||
State.gregs[FEXCore::X86State::REG_RDI] = Context->Edi;
|
||||
State.gregs[FEXCore::X86State::REG_RBP] = Context->Ebp;
|
||||
State.gregs[FEXCore::X86State::REG_RSP] = Context->Esp;
|
||||
|
||||
State.rip = Context->Eip;
|
||||
CTX->SetFlagsFromCompactedEFLAGS(Thread, Context->EFlags);
|
||||
State.rip = Context->Eip;
|
||||
CTX->SetFlagsFromCompactedEFLAGS(Thread, Context->EFlags);
|
||||
|
||||
State.es_idx = Context->SegEs & 0xffff;
|
||||
State.cs_idx = Context->SegCs & 0xffff;
|
||||
State.ss_idx = Context->SegSs & 0xffff;
|
||||
State.ds_idx = Context->SegDs & 0xffff;
|
||||
State.fs_idx = Context->SegFs & 0xffff;
|
||||
State.gs_idx = Context->SegGs & 0xffff;
|
||||
State.es_idx = Context->SegEs & 0xffff;
|
||||
State.cs_idx = Context->SegCs & 0xffff;
|
||||
State.ss_idx = Context->SegSs & 0xffff;
|
||||
State.ds_idx = Context->SegDs & 0xffff;
|
||||
State.fs_idx = Context->SegFs & 0xffff;
|
||||
State.gs_idx = Context->SegGs & 0xffff;
|
||||
|
||||
// The TEB is the only populated GDT entry by default
|
||||
State.gdt[(Context->SegFs & 0xffff) >> 3].base = WowTEB;
|
||||
State.fs_cached = WowTEB;
|
||||
State.es_cached = 0;
|
||||
State.cs_cached = 0;
|
||||
State.ss_cached = 0;
|
||||
State.ds_cached = 0;
|
||||
// The TEB is the only populated GDT entry by default
|
||||
State.gdt[(Context->SegFs & 0xffff) >> 3].base = WowTEB;
|
||||
State.fs_cached = WowTEB;
|
||||
State.es_cached = 0;
|
||||
State.cs_cached = 0;
|
||||
State.ss_cached = 0;
|
||||
State.ds_cached = 0;
|
||||
|
||||
// Floating-point register state
|
||||
const auto *XSave = reinterpret_cast<XSAVE_FORMAT*>(Context->ExtendedRegisters);
|
||||
// Floating-point register state
|
||||
const auto* XSave = reinterpret_cast<XSAVE_FORMAT*>(Context->ExtendedRegisters);
|
||||
|
||||
memcpy(State.xmm.sse.data, XSave->XmmRegisters, sizeof(State.xmm.sse.data));
|
||||
memcpy(State.mm, XSave->FloatRegisters, sizeof(State.mm));
|
||||
memcpy(State.xmm.sse.data, XSave->XmmRegisters, sizeof(State.xmm.sse.data));
|
||||
memcpy(State.mm, XSave->FloatRegisters, sizeof(State.mm));
|
||||
|
||||
State.FCW = XSave->ControlWord;
|
||||
State.flags[FEXCore::X86State::X87FLAG_C0_LOC] = (XSave->StatusWord >> 8) & 1;
|
||||
State.flags[FEXCore::X86State::X87FLAG_C1_LOC] = (XSave->StatusWord >> 9) & 1;
|
||||
State.flags[FEXCore::X86State::X87FLAG_C2_LOC] = (XSave->StatusWord >> 10) & 1;
|
||||
State.flags[FEXCore::X86State::X87FLAG_C3_LOC] = (XSave->StatusWord >> 14) & 1;
|
||||
State.flags[FEXCore::X86State::X87FLAG_TOP_LOC] = (XSave->StatusWord >> 11) & 0b111;
|
||||
State.AbridgedFTW = XSave->TagWord;
|
||||
State.FCW = XSave->ControlWord;
|
||||
State.flags[FEXCore::X86State::X87FLAG_C0_LOC] = (XSave->StatusWord >> 8) & 1;
|
||||
State.flags[FEXCore::X86State::X87FLAG_C1_LOC] = (XSave->StatusWord >> 9) & 1;
|
||||
State.flags[FEXCore::X86State::X87FLAG_C2_LOC] = (XSave->StatusWord >> 10) & 1;
|
||||
State.flags[FEXCore::X86State::X87FLAG_C3_LOC] = (XSave->StatusWord >> 14) & 1;
|
||||
State.flags[FEXCore::X86State::X87FLAG_TOP_LOC] = (XSave->StatusWord >> 11) & 0b111;
|
||||
State.AbridgedFTW = XSave->TagWord;
|
||||
}
|
||||
|
||||
void StoreWowContextFromState(FEXCore::Core::InternalThreadState* Thread, WOW64_CONTEXT* Context) {
|
||||
auto& State = Thread->CurrentFrame->State;
|
||||
|
||||
// General register state
|
||||
|
||||
Context->Eax = State.gregs[FEXCore::X86State::REG_RAX];
|
||||
Context->Ebx = State.gregs[FEXCore::X86State::REG_RBX];
|
||||
Context->Ecx = State.gregs[FEXCore::X86State::REG_RCX];
|
||||
Context->Edx = State.gregs[FEXCore::X86State::REG_RDX];
|
||||
Context->Esi = State.gregs[FEXCore::X86State::REG_RSI];
|
||||
Context->Edi = State.gregs[FEXCore::X86State::REG_RDI];
|
||||
Context->Ebp = State.gregs[FEXCore::X86State::REG_RBP];
|
||||
Context->Esp = State.gregs[FEXCore::X86State::REG_RSP];
|
||||
|
||||
Context->Eip = State.rip;
|
||||
Context->EFlags = CTX->ReconstructCompactedEFLAGS(Thread, false, nullptr, 0);
|
||||
|
||||
Context->SegEs = State.es_idx;
|
||||
Context->SegCs = State.cs_idx;
|
||||
Context->SegSs = State.ss_idx;
|
||||
Context->SegDs = State.ds_idx;
|
||||
Context->SegFs = State.fs_idx;
|
||||
Context->SegGs = State.gs_idx;
|
||||
|
||||
// Floating-point register state
|
||||
|
||||
auto* XSave = reinterpret_cast<XSAVE_FORMAT*>(Context->ExtendedRegisters);
|
||||
|
||||
memcpy(XSave->XmmRegisters, State.xmm.sse.data, sizeof(State.xmm.sse.data));
|
||||
memcpy(XSave->FloatRegisters, State.mm, sizeof(State.mm));
|
||||
|
||||
XSave->ControlWord = State.FCW;
|
||||
XSave->StatusWord = (State.flags[FEXCore::X86State::X87FLAG_TOP_LOC] << 11) | (State.flags[FEXCore::X86State::X87FLAG_C0_LOC] << 8) |
|
||||
(State.flags[FEXCore::X86State::X87FLAG_C1_LOC] << 9) | (State.flags[FEXCore::X86State::X87FLAG_C2_LOC] << 10) |
|
||||
(State.flags[FEXCore::X86State::X87FLAG_C3_LOC] << 14);
|
||||
XSave->TagWord = State.AbridgedFTW;
|
||||
|
||||
Context->FloatSave.ControlWord = XSave->ControlWord;
|
||||
Context->FloatSave.StatusWord = XSave->StatusWord;
|
||||
Context->FloatSave.TagWord = FEXCore::FPState::ConvertFromAbridgedFTW(XSave->StatusWord, State.mm, XSave->TagWord);
|
||||
Context->FloatSave.ErrorOffset = XSave->ErrorOffset;
|
||||
Context->FloatSave.ErrorSelector = XSave->ErrorSelector | (XSave->ErrorOpcode << 16);
|
||||
Context->FloatSave.DataOffset = XSave->DataOffset;
|
||||
Context->FloatSave.DataSelector = XSave->DataSelector;
|
||||
Context->FloatSave.Cr0NpxState = XSave->StatusWord | 0xffff0000;
|
||||
}
|
||||
|
||||
NTSTATUS FlushThreadStateContext(HANDLE Thread) {
|
||||
const auto [Err, TLS] = GetThreadTLS(Thread);
|
||||
if (Err) {
|
||||
return Err;
|
||||
}
|
||||
|
||||
void StoreWowContextFromState(FEXCore::Core::InternalThreadState *Thread, WOW64_CONTEXT *Context) {
|
||||
auto &State = Thread->CurrentFrame->State;
|
||||
WOW64_CONTEXT TmpWowContext {.ContextFlags = WOW64_CONTEXT_FULL | WOW64_CONTEXT_EXTENDED_REGISTERS};
|
||||
|
||||
// General register state
|
||||
Context::StoreWowContextFromState(TLS.ThreadState(), &TmpWowContext);
|
||||
return RtlWow64SetThreadContext(Thread, &TmpWowContext);
|
||||
}
|
||||
|
||||
Context->Eax = State.gregs[FEXCore::X86State::REG_RAX];
|
||||
Context->Ebx = State.gregs[FEXCore::X86State::REG_RBX];
|
||||
Context->Ecx = State.gregs[FEXCore::X86State::REG_RCX];
|
||||
Context->Edx = State.gregs[FEXCore::X86State::REG_RDX];
|
||||
Context->Esi = State.gregs[FEXCore::X86State::REG_RSI];
|
||||
Context->Edi = State.gregs[FEXCore::X86State::REG_RDI];
|
||||
Context->Ebp = State.gregs[FEXCore::X86State::REG_RBP];
|
||||
Context->Esp = State.gregs[FEXCore::X86State::REG_RSP];
|
||||
void ReconstructThreadState(CONTEXT* Context) {
|
||||
const auto& Config = SignalDelegator->GetConfig();
|
||||
auto* Thread = GetTLS().ThreadState();
|
||||
auto& State = Thread->CurrentFrame->State;
|
||||
|
||||
Context->Eip = State.rip;
|
||||
Context->EFlags = CTX->ReconstructCompactedEFLAGS(Thread, false, nullptr, 0);
|
||||
State.rip = CTX->RestoreRIPFromHostPC(Thread, Context->Pc);
|
||||
|
||||
Context->SegEs = State.es_idx;
|
||||
Context->SegCs = State.cs_idx;
|
||||
Context->SegSs = State.ss_idx;
|
||||
Context->SegDs = State.ds_idx;
|
||||
Context->SegFs = State.fs_idx;
|
||||
Context->SegGs = State.gs_idx;
|
||||
|
||||
// Floating-point register state
|
||||
|
||||
auto *XSave = reinterpret_cast<XSAVE_FORMAT*>(Context->ExtendedRegisters);
|
||||
|
||||
memcpy(XSave->XmmRegisters, State.xmm.sse.data, sizeof(State.xmm.sse.data));
|
||||
memcpy(XSave->FloatRegisters, State.mm, sizeof(State.mm));
|
||||
|
||||
XSave->ControlWord = State.FCW;
|
||||
XSave->StatusWord =
|
||||
(State.flags[FEXCore::X86State::X87FLAG_TOP_LOC] << 11) |
|
||||
(State.flags[FEXCore::X86State::X87FLAG_C0_LOC] << 8) |
|
||||
(State.flags[FEXCore::X86State::X87FLAG_C1_LOC] << 9) |
|
||||
(State.flags[FEXCore::X86State::X87FLAG_C2_LOC] << 10) |
|
||||
(State.flags[FEXCore::X86State::X87FLAG_C3_LOC] << 14);
|
||||
XSave->TagWord = State.AbridgedFTW;
|
||||
|
||||
Context->FloatSave.ControlWord = XSave->ControlWord;
|
||||
Context->FloatSave.StatusWord = XSave->StatusWord;
|
||||
Context->FloatSave.TagWord = FEXCore::FPState::ConvertFromAbridgedFTW(XSave->StatusWord, State.mm, XSave->TagWord);
|
||||
Context->FloatSave.ErrorOffset = XSave->ErrorOffset;
|
||||
Context->FloatSave.ErrorSelector = XSave->ErrorSelector | (XSave->ErrorOpcode << 16);
|
||||
Context->FloatSave.DataOffset = XSave->DataOffset;
|
||||
Context->FloatSave.DataSelector = XSave->DataSelector;
|
||||
Context->FloatSave.Cr0NpxState = XSave->StatusWord | 0xffff0000;
|
||||
// Spill all SRA GPRs
|
||||
for (size_t i = 0; i < Config.SRAGPRCount; i++) {
|
||||
State.gregs[i] = Context->X[Config.SRAGPRMapping[i]];
|
||||
}
|
||||
|
||||
NTSTATUS FlushThreadStateContext(HANDLE Thread) {
|
||||
const auto [Err, TLS] = GetThreadTLS(Thread);
|
||||
if (Err) {
|
||||
return Err;
|
||||
}
|
||||
|
||||
WOW64_CONTEXT TmpWowContext{
|
||||
.ContextFlags = WOW64_CONTEXT_FULL | WOW64_CONTEXT_EXTENDED_REGISTERS
|
||||
};
|
||||
|
||||
Context::StoreWowContextFromState(TLS.ThreadState(), &TmpWowContext);
|
||||
return RtlWow64SetThreadContext(Thread, &TmpWowContext);
|
||||
}
|
||||
|
||||
void ReconstructThreadState(CONTEXT *Context) {
|
||||
const auto &Config = SignalDelegator->GetConfig();
|
||||
auto *Thread = GetTLS().ThreadState();
|
||||
auto &State = Thread->CurrentFrame->State;
|
||||
|
||||
State.rip = CTX->RestoreRIPFromHostPC(Thread, Context->Pc);
|
||||
|
||||
// Spill all SRA GPRs
|
||||
for (size_t i = 0; i < Config.SRAGPRCount; i++) {
|
||||
State.gregs[i] = Context->X[Config.SRAGPRMapping[i]];
|
||||
}
|
||||
|
||||
// Spill all SRA FPRs
|
||||
for (size_t i = 0; i < Config.SRAFPRCount; i++) {
|
||||
memcpy(State.xmm.sse.data[i], &Context->V[Config.SRAFPRMapping[i]], sizeof(__uint128_t));
|
||||
}
|
||||
}
|
||||
|
||||
WOW64_CONTEXT ReconstructWowContext(CONTEXT *Context) {
|
||||
ReconstructThreadState(Context);
|
||||
|
||||
WOW64_CONTEXT WowContext{
|
||||
.ContextFlags = WOW64_CONTEXT_ALL,
|
||||
};
|
||||
|
||||
auto *XSave = reinterpret_cast<XSAVE_FORMAT *>(WowContext.ExtendedRegisters);
|
||||
XSave->ControlWord = 0x27f;
|
||||
XSave->MxCsr = 0x1f80;
|
||||
|
||||
Context::StoreWowContextFromState(GetTLS().ThreadState(), &WowContext);
|
||||
return WowContext;
|
||||
}
|
||||
|
||||
bool HandleUnalignedAccess(CONTEXT *Context) {
|
||||
auto Thread = GetTLS().ThreadState();
|
||||
if (!Thread->CTX->IsAddressInCodeBuffer(Thread, Context->Pc)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
FEX_CONFIG_OPT(ParanoidTSO, PARANOIDTSO);
|
||||
const auto Result = FEXCore::ArchHelpers::Arm64::HandleUnalignedAccess(Thread, ParanoidTSO(), Context->Pc, &Context->X0);
|
||||
if (!Result.first) {
|
||||
return false;
|
||||
}
|
||||
|
||||
Context->Pc += Result.second;
|
||||
return true;
|
||||
}
|
||||
|
||||
void LockJITContext() {
|
||||
uint32_t Expected = GetTLS().ControlWord().load(), New;
|
||||
|
||||
// Spin until PAUSED is unset, setting IN_JIT when that occurs
|
||||
do {
|
||||
Expected = Expected & ~ControlBits::PAUSED;
|
||||
New = (Expected | ControlBits::IN_JIT) & ~ControlBits::WOW_CPU_AREA_DIRTY;
|
||||
} while (!GetTLS().ControlWord().compare_exchange_weak(Expected, New, std::memory_order::relaxed));
|
||||
std::atomic_signal_fence(std::memory_order::seq_cst);
|
||||
|
||||
// If the CPU area is dirty, flush it to the JIT context before reentry
|
||||
if (Expected & ControlBits::WOW_CPU_AREA_DIRTY) {
|
||||
WOW64_CONTEXT *WowContext;
|
||||
RtlWow64GetCurrentCpuArea(nullptr, reinterpret_cast<void **>(&WowContext), nullptr);
|
||||
Context::LoadStateFromWowContext(GetTLS().ThreadState(), GetWowTEB(NtCurrentTeb()), WowContext);
|
||||
}
|
||||
}
|
||||
|
||||
void UnlockJITContext() {
|
||||
std::atomic_signal_fence(std::memory_order::seq_cst);
|
||||
GetTLS().ControlWord().fetch_and(~ControlBits::IN_JIT, std::memory_order::relaxed);
|
||||
}
|
||||
|
||||
bool HandleSuspendInterrupt(CONTEXT *Context, uint64_t FaultAddress) {
|
||||
if (FaultAddress != reinterpret_cast<uint64_t>(&GetTLS().ThreadState()->InterruptFaultPage)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
void *TmpAddress = reinterpret_cast<void *>(FaultAddress);
|
||||
SIZE_T TmpSize = FEXCore::Utils::FEX_PAGE_SIZE;
|
||||
ULONG TmpProt;
|
||||
NtProtectVirtualMemory(NtCurrentProcess(), &TmpAddress, &TmpSize, PAGE_READWRITE, &TmpProt);
|
||||
|
||||
// Since interrupts only happen at the start of blocks, the reconstructed state should be entirely accurate
|
||||
ReconstructThreadState(Context);
|
||||
|
||||
// Yield to the suspender
|
||||
UnlockJITContext();
|
||||
LockJITContext();
|
||||
|
||||
// Adjust context to return to the dispatcher, reloading SRA from thread state
|
||||
const auto &Config = SignalDelegator->GetConfig();
|
||||
Context->Pc = Config.AbsoluteLoopTopAddressFillSRA;
|
||||
return true;
|
||||
// Spill all SRA FPRs
|
||||
for (size_t i = 0; i < Config.SRAFPRCount; i++) {
|
||||
memcpy(State.xmm.sse.data[i], &Context->V[Config.SRAFPRMapping[i]], sizeof(__uint128_t));
|
||||
}
|
||||
}
|
||||
|
||||
WOW64_CONTEXT ReconstructWowContext(CONTEXT* Context) {
|
||||
ReconstructThreadState(Context);
|
||||
|
||||
WOW64_CONTEXT WowContext {
|
||||
.ContextFlags = WOW64_CONTEXT_ALL,
|
||||
};
|
||||
|
||||
auto* XSave = reinterpret_cast<XSAVE_FORMAT*>(WowContext.ExtendedRegisters);
|
||||
XSave->ControlWord = 0x27f;
|
||||
XSave->MxCsr = 0x1f80;
|
||||
|
||||
Context::StoreWowContextFromState(GetTLS().ThreadState(), &WowContext);
|
||||
return WowContext;
|
||||
}
|
||||
|
||||
bool HandleUnalignedAccess(CONTEXT* Context) {
|
||||
auto Thread = GetTLS().ThreadState();
|
||||
if (!Thread->CTX->IsAddressInCodeBuffer(Thread, Context->Pc)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
FEX_CONFIG_OPT(ParanoidTSO, PARANOIDTSO);
|
||||
const auto Result = FEXCore::ArchHelpers::Arm64::HandleUnalignedAccess(Thread, ParanoidTSO(), Context->Pc, &Context->X0);
|
||||
if (!Result.first) {
|
||||
return false;
|
||||
}
|
||||
|
||||
Context->Pc += Result.second;
|
||||
return true;
|
||||
}
|
||||
|
||||
void LockJITContext() {
|
||||
uint32_t Expected = GetTLS().ControlWord().load(), New;
|
||||
|
||||
// Spin until PAUSED is unset, setting IN_JIT when that occurs
|
||||
do {
|
||||
Expected = Expected & ~ControlBits::PAUSED;
|
||||
New = (Expected | ControlBits::IN_JIT) & ~ControlBits::WOW_CPU_AREA_DIRTY;
|
||||
} while (!GetTLS().ControlWord().compare_exchange_weak(Expected, New, std::memory_order::relaxed));
|
||||
std::atomic_signal_fence(std::memory_order::seq_cst);
|
||||
|
||||
// If the CPU area is dirty, flush it to the JIT context before reentry
|
||||
if (Expected & ControlBits::WOW_CPU_AREA_DIRTY) {
|
||||
WOW64_CONTEXT* WowContext;
|
||||
RtlWow64GetCurrentCpuArea(nullptr, reinterpret_cast<void**>(&WowContext), nullptr);
|
||||
Context::LoadStateFromWowContext(GetTLS().ThreadState(), GetWowTEB(NtCurrentTeb()), WowContext);
|
||||
}
|
||||
}
|
||||
|
||||
void UnlockJITContext() {
|
||||
std::atomic_signal_fence(std::memory_order::seq_cst);
|
||||
GetTLS().ControlWord().fetch_and(~ControlBits::IN_JIT, std::memory_order::relaxed);
|
||||
}
|
||||
|
||||
bool HandleSuspendInterrupt(CONTEXT* Context, uint64_t FaultAddress) {
|
||||
if (FaultAddress != reinterpret_cast<uint64_t>(&GetTLS().ThreadState()->InterruptFaultPage)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
void* TmpAddress = reinterpret_cast<void*>(FaultAddress);
|
||||
SIZE_T TmpSize = FEXCore::Utils::FEX_PAGE_SIZE;
|
||||
ULONG TmpProt;
|
||||
NtProtectVirtualMemory(NtCurrentProcess(), &TmpAddress, &TmpSize, PAGE_READWRITE, &TmpProt);
|
||||
|
||||
// Since interrupts only happen at the start of blocks, the reconstructed state should be entirely accurate
|
||||
ReconstructThreadState(Context);
|
||||
|
||||
// Yield to the suspender
|
||||
UnlockJITContext();
|
||||
LockJITContext();
|
||||
|
||||
// Adjust context to return to the dispatcher, reloading SRA from thread state
|
||||
const auto& Config = SignalDelegator->GetConfig();
|
||||
Context->Pc = Config.AbsoluteLoopTopAddressFillSRA;
|
||||
return true;
|
||||
}
|
||||
} // namespace Context
|
||||
|
||||
namespace Logging {
|
||||
void MsgHandler(LogMan::DebugLevels Level, char const *Message) {
|
||||
const auto Output = fextl::fmt::format("[{}][{:X}] {}\n", LogMan::DebugLevelStr(Level), GetCurrentThreadId(), Message);
|
||||
__wine_dbg_output(Output.c_str());
|
||||
}
|
||||
|
||||
void AssertHandler(char const *Message) {
|
||||
const auto Output = fextl::fmt::format("[ASSERT] {}\n", Message);
|
||||
__wine_dbg_output(Output.c_str());
|
||||
}
|
||||
|
||||
void Init() {
|
||||
LogMan::Throw::InstallHandler(AssertHandler);
|
||||
LogMan::Msg::InstallHandler(MsgHandler);
|
||||
}
|
||||
void MsgHandler(LogMan::DebugLevels Level, const char* Message) {
|
||||
const auto Output = fextl::fmt::format("[{}][{:X}] {}\n", LogMan::DebugLevelStr(Level), GetCurrentThreadId(), Message);
|
||||
__wine_dbg_output(Output.c_str());
|
||||
}
|
||||
|
||||
void AssertHandler(const char* Message) {
|
||||
const auto Output = fextl::fmt::format("[ASSERT] {}\n", Message);
|
||||
__wine_dbg_output(Output.c_str());
|
||||
}
|
||||
|
||||
void Init() {
|
||||
LogMan::Throw::InstallHandler(AssertHandler);
|
||||
LogMan::Msg::InstallHandler(MsgHandler);
|
||||
}
|
||||
} // namespace Logging
|
||||
|
||||
class WowSyscallHandler : public FEXCore::HLE::SyscallHandler, public FEXCore::Allocator::FEXAllocOperators {
|
||||
public:
|
||||
WowSyscallHandler() {
|
||||
OSABI = FEXCore::HLE::SyscallOSABI::OS_WIN32;
|
||||
}
|
||||
|
||||
uint64_t HandleSyscall(FEXCore::Core::CpuStateFrame *Frame, FEXCore::HLE::SyscallArguments *Args) override {
|
||||
const uint64_t ReturnRIP = *(uint32_t *)(Frame->State.gregs[FEXCore::X86State::REG_RSP]); // Return address from the stack
|
||||
uint64_t ReturnRSP = Frame->State.gregs[FEXCore::X86State::REG_RSP] + 4; // Stack pointer after popping return address
|
||||
uint64_t HandleSyscall(FEXCore::Core::CpuStateFrame* Frame, FEXCore::HLE::SyscallArguments* Args) override {
|
||||
const uint64_t ReturnRIP = *(uint32_t*)(Frame->State.gregs[FEXCore::X86State::REG_RSP]); // Return address from the stack
|
||||
uint64_t ReturnRSP = Frame->State.gregs[FEXCore::X86State::REG_RSP] + 4; // Stack pointer after popping return address
|
||||
uint64_t ReturnRAX = 0;
|
||||
|
||||
if (Frame->State.rip == (uint64_t)&BridgeInstrs::UnixCall) {
|
||||
@@ -362,7 +358,7 @@ public:
|
||||
unixlib_handle_t Handle;
|
||||
UINT32 ID;
|
||||
ULONG32 Args;
|
||||
} *StackArgs = reinterpret_cast<StackLayout *>(ReturnRSP);
|
||||
}* StackArgs = reinterpret_cast<StackLayout*>(ReturnRSP);
|
||||
|
||||
ReturnRSP += sizeof(StackLayout);
|
||||
|
||||
@@ -374,14 +370,11 @@ public:
|
||||
|
||||
Context::UnlockJITContext();
|
||||
Wow64ProcessPendingCrossProcessItems();
|
||||
ReturnRAX = static_cast<uint64_t>(Wow64SystemServiceEx(static_cast<UINT>(EntryRAX),
|
||||
reinterpret_cast<UINT *>(ReturnRSP + 4)));
|
||||
ReturnRAX = static_cast<uint64_t>(Wow64SystemServiceEx(static_cast<UINT>(EntryRAX), reinterpret_cast<UINT*>(ReturnRSP + 4)));
|
||||
Context::LockJITContext();
|
||||
|
||||
}
|
||||
// If a new context has been set, use it directly and don't return to the syscall caller
|
||||
if (Frame->State.rip == (uint64_t)&BridgeInstrs::Syscall ||
|
||||
Frame->State.rip == (uint64_t)&BridgeInstrs::UnixCall) {
|
||||
if (Frame->State.rip == (uint64_t)&BridgeInstrs::Syscall || Frame->State.rip == (uint64_t)&BridgeInstrs::UnixCall) {
|
||||
Frame->State.gregs[FEXCore::X86State::REG_RAX] = ReturnRAX;
|
||||
Frame->State.gregs[FEXCore::X86State::REG_RSP] = ReturnRSP;
|
||||
Frame->State.rip = ReturnRIP;
|
||||
@@ -392,14 +385,14 @@ public:
|
||||
}
|
||||
|
||||
FEXCore::HLE::SyscallABI GetSyscallABI(uint64_t Syscall) override {
|
||||
return { .NumArgs = 0, .HasReturn = false, .HostSyscallNumber = -1 };
|
||||
return {.NumArgs = 0, .HasReturn = false, .HostSyscallNumber = -1};
|
||||
}
|
||||
|
||||
FEXCore::HLE::AOTIRCacheEntryLookupResult LookupAOTIRCacheEntry(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestAddr) override {
|
||||
FEXCore::HLE::AOTIRCacheEntryLookupResult LookupAOTIRCacheEntry(FEXCore::Core::InternalThreadState* Thread, uint64_t GuestAddr) override {
|
||||
return {0, 0};
|
||||
}
|
||||
|
||||
void MarkGuestExecutableRange(FEXCore::Core::InternalThreadState *Thread, uint64_t Start, uint64_t Length) override {
|
||||
void MarkGuestExecutableRange(FEXCore::Core::InternalThreadState* Thread, uint64_t Start, uint64_t Length) override {
|
||||
InvalidationTracker.ReprotectRWXIntervals(Start, Length);
|
||||
}
|
||||
};
|
||||
@@ -461,15 +454,15 @@ NTSTATUS BTCpuThreadTerm(HANDLE Thread) {
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
void *BTCpuGetBopCode() {
|
||||
void* BTCpuGetBopCode() {
|
||||
return &BridgeInstrs::Syscall;
|
||||
}
|
||||
|
||||
void *__wine_get_unix_opcode() {
|
||||
void* __wine_get_unix_opcode() {
|
||||
return &BridgeInstrs::UnixCall;
|
||||
}
|
||||
|
||||
NTSTATUS BTCpuGetContext(HANDLE Thread, HANDLE Process, void *Unknown, WOW64_CONTEXT *Context) {
|
||||
NTSTATUS BTCpuGetContext(HANDLE Thread, HANDLE Process, void* Unknown, WOW64_CONTEXT* Context) {
|
||||
auto [Err, TLS] = GetThreadTLS(Thread);
|
||||
if (Err) {
|
||||
return Err;
|
||||
@@ -484,7 +477,7 @@ NTSTATUS BTCpuGetContext(HANDLE Thread, HANDLE Process, void *Unknown, WOW64_CON
|
||||
return RtlWow64GetThreadContext(Thread, Context);
|
||||
}
|
||||
|
||||
NTSTATUS BTCpuSetContext(HANDLE Thread, HANDLE Process, void *Unknown, WOW64_CONTEXT *Context) {
|
||||
NTSTATUS BTCpuSetContext(HANDLE Thread, HANDLE Process, void* Unknown, WOW64_CONTEXT* Context) {
|
||||
auto [Err, TLS] = GetThreadTLS(Thread);
|
||||
if (Err) {
|
||||
return Err;
|
||||
@@ -521,7 +514,7 @@ void BTCpuSimulate() {
|
||||
|
||||
// APC handling calls BTCpuSimulate from syscalls and then use NtContinue to return to the previous context,
|
||||
// to avoid the saved context being clobbered in this case only save the entry context highest in the stack
|
||||
if (!GetTLS().EntryContext() || GetTLS().EntryContext()->Sp <= entry_context.Sp) {
|
||||
if (!GetTLS().EntryContext() || GetTLS().EntryContext()->Sp <= entry_context.Sp) {
|
||||
GetTLS().EntryContext() = &entry_context;
|
||||
}
|
||||
|
||||
@@ -530,7 +523,7 @@ void BTCpuSimulate() {
|
||||
Context::UnlockJITContext();
|
||||
}
|
||||
|
||||
NTSTATUS BTCpuSuspendLocalThread(HANDLE Thread, ULONG *Count) {
|
||||
NTSTATUS BTCpuSuspendLocalThread(HANDLE Thread, ULONG* Count) {
|
||||
THREAD_BASIC_INFORMATION Info;
|
||||
if (NTSTATUS Err = NtQueryInformationThread(Thread, ThreadBasicInformation, &Info, sizeof(Info), nullptr); Err) {
|
||||
return Err;
|
||||
@@ -541,8 +534,7 @@ NTSTATUS BTCpuSuspendLocalThread(HANDLE Thread, ULONG *Count) {
|
||||
LogMan::Msg::DFmt("Suspending self");
|
||||
// Mark the CPU area as dirty, to force the JIT context to be restored from it on entry as it may be changed using
|
||||
// SetThreadContext (which doesn't use the BTCpu API)
|
||||
if (!(GetTLS().ControlWord().fetch_or(ControlBits::WOW_CPU_AREA_DIRTY, std::memory_order::relaxed) &
|
||||
ControlBits::WOW_CPU_AREA_DIRTY)) {
|
||||
if (!(GetTLS().ControlWord().fetch_or(ControlBits::WOW_CPU_AREA_DIRTY, std::memory_order::relaxed) & ControlBits::WOW_CPU_AREA_DIRTY)) {
|
||||
if (NTSTATUS Err = Context::FlushThreadStateContext(Thread); Err) {
|
||||
return Err;
|
||||
}
|
||||
@@ -561,8 +553,9 @@ NTSTATUS BTCpuSuspendLocalThread(HANDLE Thread, ULONG *Count) {
|
||||
std::scoped_lock Lock(ThreadSuspendLock);
|
||||
|
||||
// If the thread hasn't yet been initialized, suspend it without special handling as it wont yet have entered the JIT
|
||||
if (!InitializedWOWThreads.contains(ThreadTID))
|
||||
if (!InitializedWOWThreads.contains(ThreadTID)) {
|
||||
return NtSuspendThread(Thread, Count);
|
||||
}
|
||||
|
||||
// If CONTROL_IN_JIT is unset at this point, then it can never be set (and thus the JIT cannot be reentered) as
|
||||
// CONTROL_PAUSED has been set, as such, while this may redundantly request interrupts in rare cases it will never
|
||||
@@ -571,13 +564,14 @@ NTSTATUS BTCpuSuspendLocalThread(HANDLE Thread, ULONG *Count) {
|
||||
LogMan::Msg::DFmt("Thread {:X} is in JIT, polling for interrupt", ThreadTID);
|
||||
|
||||
ULONG TmpProt;
|
||||
void *TmpAddress = &TLS.ThreadState()->InterruptFaultPage;
|
||||
void* TmpAddress = &TLS.ThreadState()->InterruptFaultPage;
|
||||
SIZE_T TmpSize = FEXCore::Utils::FEX_PAGE_SIZE;
|
||||
NtProtectVirtualMemory(NtCurrentProcess(), &TmpAddress, &TmpSize, PAGE_READONLY, &TmpProt);
|
||||
}
|
||||
|
||||
// Spin until the JIT is interrupted
|
||||
while (TLS.ControlWord().load() & ControlBits::IN_JIT);
|
||||
while (TLS.ControlWord().load() & ControlBits::IN_JIT)
|
||||
;
|
||||
|
||||
// The JIT has now been interrupted and the context stored in the thread's CPU area is up-to-date
|
||||
if (Err = NtSuspendThread(Thread, Count); Err) {
|
||||
@@ -585,7 +579,7 @@ NTSTATUS BTCpuSuspendLocalThread(HANDLE Thread, ULONG *Count) {
|
||||
return Err;
|
||||
}
|
||||
|
||||
CONTEXT TmpContext{
|
||||
CONTEXT TmpContext {
|
||||
.ContextFlags = CONTEXT_INTEGER,
|
||||
};
|
||||
|
||||
@@ -610,9 +604,9 @@ NTSTATUS BTCpuSuspendLocalThread(HANDLE Thread, ULONG *Count) {
|
||||
return Err;
|
||||
}
|
||||
|
||||
NTSTATUS BTCpuResetToConsistentState(EXCEPTION_POINTERS *Ptrs) {
|
||||
auto *Context = Ptrs->ContextRecord;
|
||||
const auto *Exception = Ptrs->ExceptionRecord;
|
||||
NTSTATUS BTCpuResetToConsistentState(EXCEPTION_POINTERS* Ptrs) {
|
||||
auto* Context = Ptrs->ContextRecord;
|
||||
const auto* Exception = Ptrs->ExceptionRecord;
|
||||
if (Exception->ExceptionCode == EXCEPTION_DATATYPE_MISALIGNMENT && Context::HandleUnalignedAccess(Context)) {
|
||||
LogMan::Msg::DFmt("Handled unaligned atomic: new pc: {:X}", Context->Pc);
|
||||
NtContinue(Context, FALSE);
|
||||
@@ -650,21 +644,21 @@ NTSTATUS BTCpuResetToConsistentState(EXCEPTION_POINTERS *Ptrs) {
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
void BTCpuFlushInstructionCache2(const void *Address, SIZE_T Size) {
|
||||
void BTCpuFlushInstructionCache2(const void* Address, SIZE_T Size) {
|
||||
InvalidationTracker.InvalidateAlignedInterval(GetTLS().ThreadState(), reinterpret_cast<uint64_t>(Address), static_cast<uint64_t>(Size), false);
|
||||
}
|
||||
|
||||
void BTCpuNotifyMemoryAlloc(void *Address, SIZE_T Size, ULONG Type, ULONG Prot) {
|
||||
InvalidationTracker.HandleMemoryProtectionNotification(GetTLS().ThreadState(), reinterpret_cast<uint64_t>(Address), static_cast<uint64_t>(Size),
|
||||
Prot);
|
||||
void BTCpuNotifyMemoryAlloc(void* Address, SIZE_T Size, ULONG Type, ULONG Prot) {
|
||||
InvalidationTracker.HandleMemoryProtectionNotification(GetTLS().ThreadState(), reinterpret_cast<uint64_t>(Address),
|
||||
static_cast<uint64_t>(Size), Prot);
|
||||
}
|
||||
|
||||
void BTCpuNotifyMemoryProtect(void *Address, SIZE_T Size, ULONG NewProt) {
|
||||
InvalidationTracker.HandleMemoryProtectionNotification(GetTLS().ThreadState(), reinterpret_cast<uint64_t>(Address), static_cast<uint64_t>(Size),
|
||||
NewProt);
|
||||
void BTCpuNotifyMemoryProtect(void* Address, SIZE_T Size, ULONG NewProt) {
|
||||
InvalidationTracker.HandleMemoryProtectionNotification(GetTLS().ThreadState(), reinterpret_cast<uint64_t>(Address),
|
||||
static_cast<uint64_t>(Size), NewProt);
|
||||
}
|
||||
|
||||
void BTCpuNotifyMemoryFree(void *Address, SIZE_T Size, ULONG FreeType) {
|
||||
void BTCpuNotifyMemoryFree(void* Address, SIZE_T Size, ULONG FreeType) {
|
||||
if (!Size) {
|
||||
InvalidationTracker.InvalidateContainingSection(GetTLS().ThreadState(), reinterpret_cast<uint64_t>(Address), true);
|
||||
} else if (FreeType & MEM_DECOMMIT) {
|
||||
@@ -672,7 +666,7 @@ void BTCpuNotifyMemoryFree(void *Address, SIZE_T Size, ULONG FreeType) {
|
||||
}
|
||||
}
|
||||
|
||||
void BTCpuNotifyUnmapViewOfSection(void *Address, ULONG Flags) {
|
||||
void BTCpuNotifyUnmapViewOfSection(void* Address, ULONG Flags) {
|
||||
InvalidationTracker.InvalidateContainingSection(GetTLS().ThreadState(), reinterpret_cast<uint64_t>(Address), true);
|
||||
}
|
||||
|
||||
@@ -680,7 +674,7 @@ BOOLEAN WINAPI BTCpuIsProcessorFeaturePresent(UINT Feature) {
|
||||
return CPUFeatures->IsFeaturePresent(Feature) ? TRUE : FALSE;
|
||||
}
|
||||
|
||||
BOOLEAN BTCpuUpdateProcessorInformation(SYSTEM_CPU_INFORMATION *Info) {
|
||||
BOOLEAN BTCpuUpdateProcessorInformation(SYSTEM_CPU_INFORMATION* Info) {
|
||||
CPUFeatures->UpdateInformation(Info);
|
||||
return TRUE;
|
||||
}
|
||||
Reference in new issue
Block a user