Whole-tree reformat

This follows discussions from #3413.
Followup commits add clang-format file, script and blame ignore lists.
This commit is contained in:
Paulo Matos committed 2024-04-12 16:26:02 +02:00
1 parent 028c220041
commit 2b4ec88dae
447 files changed
+78590 -73070

No files matched your search

+281 -287
View File
@@ -44,16 +44,16 @@ $end_info$
#include <wine/unixlib.h>
namespace ControlBits {
// When this is unset, a thread can be safely interrupted and have its context recovered
// IMPORTANT: This can only safely be written by the owning thread
static constexpr uint32_t IN_JIT{1U << 0};
// When this is unset, a thread can be safely interrupted and have its context recovered
// IMPORTANT: This can only safely be written by the owning thread
static constexpr uint32_t IN_JIT {1U << 0};
// JIT entry polls this bit until it is unset, at which point CONTROL_IN_JIT will be set
static constexpr uint32_t PAUSED{1U << 1};
// JIT entry polls this bit until it is unset, at which point CONTROL_IN_JIT will be set
static constexpr uint32_t PAUSED {1U << 1};
// When this is set, the CPU context stored in the CPU area has not yet been flushed to the FEX TLS
static constexpr uint32_t WOW_CPU_AREA_DIRTY{1U << 2};
};
// When this is set, the CPU context stored in the CPU area has not yet been flushed to the FEX TLS
static constexpr uint32_t WOW_CPU_AREA_DIRTY {1U << 2};
}; // namespace ControlBits
struct TLS {
enum class Slot : size_t {
@@ -62,299 +62,295 @@ struct TLS {
THREAD_STATE = WOW64_TLS_MAX_NUMBER - 2,
};
_TEB *TEB;
_TEB* TEB;
explicit TLS(_TEB *TEB) : TEB(TEB) {}
explicit TLS(_TEB* TEB)
: TEB(TEB) {}
std::atomic<uint32_t> &ControlWord() const {
std::atomic<uint32_t>& ControlWord() const {
// TODO: Change this when libc++ gains std::atomic_ref support
return reinterpret_cast<std::atomic<uint32_t> &>(TEB->TlsSlots[FEXCore::ToUnderlying(Slot::CONTROL_WORD)]);
return reinterpret_cast<std::atomic<uint32_t>&>(TEB->TlsSlots[FEXCore::ToUnderlying(Slot::CONTROL_WORD)]);
}
CONTEXT *&EntryContext() const {
return reinterpret_cast<CONTEXT *&>(TEB->TlsSlots[FEXCore::ToUnderlying(Slot::ENTRY_CONTEXT)]);
CONTEXT*& EntryContext() const {
return reinterpret_cast<CONTEXT*&>(TEB->TlsSlots[FEXCore::ToUnderlying(Slot::ENTRY_CONTEXT)]);
}
FEXCore::Core::InternalThreadState *&ThreadState() const {
return reinterpret_cast<FEXCore::Core::InternalThreadState *&>(TEB->TlsSlots[FEXCore::ToUnderlying(Slot::THREAD_STATE)]);
FEXCore::Core::InternalThreadState*& ThreadState() const {
return reinterpret_cast<FEXCore::Core::InternalThreadState*&>(TEB->TlsSlots[FEXCore::ToUnderlying(Slot::THREAD_STATE)]);
}
};
class WowSyscallHandler;
namespace {
namespace BridgeInstrs {
// These directly jumped to by the guest to make system calls
uint16_t Syscall{0x2ecd};
uint16_t UnixCall{0x2ecd};
}
namespace BridgeInstrs {
// These directly jumped to by the guest to make system calls
uint16_t Syscall {0x2ecd};
uint16_t UnixCall {0x2ecd};
} // namespace BridgeInstrs
fextl::unique_ptr<FEXCore::Context::Context> CTX;
fextl::unique_ptr<FEX::DummyHandlers::DummySignalDelegator> SignalDelegator;
fextl::unique_ptr<WowSyscallHandler> SyscallHandler;
fextl::unique_ptr<FEXCore::Context::Context> CTX;
fextl::unique_ptr<FEX::DummyHandlers::DummySignalDelegator> SignalDelegator;
fextl::unique_ptr<WowSyscallHandler> SyscallHandler;
FEX::Windows::InvalidationTracker InvalidationTracker;
std::optional<FEX::Windows::CPUFeatures> CPUFeatures;
FEX::Windows::InvalidationTracker InvalidationTracker;
std::optional<FEX::Windows::CPUFeatures> CPUFeatures;
std::mutex ThreadSuspendLock;
std::unordered_set<DWORD> InitializedWOWThreads; // Set of TIDs, `ThreadSuspendLock` must be locked when accessing
std::mutex ThreadSuspendLock;
std::unordered_set<DWORD> InitializedWOWThreads; // Set of TIDs, `ThreadSuspendLock` must be locked when accessing
std::pair<NTSTATUS, TLS> GetThreadTLS(HANDLE Thread) {
THREAD_BASIC_INFORMATION Info;
const NTSTATUS Err = NtQueryInformationThread(Thread, ThreadBasicInformation, &Info, sizeof(Info), nullptr);
return {Err, TLS{reinterpret_cast<_TEB *>(Info.TebBaseAddress)}};
}
TLS GetTLS() {
return TLS{NtCurrentTeb()};
}
uint64_t GetWowTEB(void *TEB) {
static constexpr size_t WowTEBOffsetMemberOffset{0x180c};
return static_cast<uint64_t>(*reinterpret_cast<LONG *>(reinterpret_cast<uintptr_t>(TEB) + WowTEBOffsetMemberOffset)
+ reinterpret_cast<uint64_t>(TEB));
}
bool IsAddressInJit(uint64_t Address) {
auto Thread = GetTLS().ThreadState();
return Thread->CTX->IsAddressInCodeBuffer(Thread, Address);
}
std::pair<NTSTATUS, TLS> GetThreadTLS(HANDLE Thread) {
THREAD_BASIC_INFORMATION Info;
const NTSTATUS Err = NtQueryInformationThread(Thread, ThreadBasicInformation, &Info, sizeof(Info), nullptr);
return {Err, TLS {reinterpret_cast<_TEB*>(Info.TebBaseAddress)}};
}
TLS GetTLS() {
return TLS {NtCurrentTeb()};
}
uint64_t GetWowTEB(void* TEB) {
static constexpr size_t WowTEBOffsetMemberOffset {0x180c};
return static_cast<uint64_t>(
*reinterpret_cast<LONG*>(reinterpret_cast<uintptr_t>(TEB) + WowTEBOffsetMemberOffset) + reinterpret_cast<uint64_t>(TEB));
}
bool IsAddressInJit(uint64_t Address) {
auto Thread = GetTLS().ThreadState();
return Thread->CTX->IsAddressInCodeBuffer(Thread, Address);
}
} // namespace
namespace Context {
void LoadStateFromWowContext(FEXCore::Core::InternalThreadState *Thread, uint64_t WowTEB, WOW64_CONTEXT *Context) {
auto &State = Thread->CurrentFrame->State;
void LoadStateFromWowContext(FEXCore::Core::InternalThreadState* Thread, uint64_t WowTEB, WOW64_CONTEXT* Context) {
auto& State = Thread->CurrentFrame->State;
// General register state
// General register state
State.gregs[FEXCore::X86State::REG_RAX] = Context->Eax;
State.gregs[FEXCore::X86State::REG_RBX] = Context->Ebx;
State.gregs[FEXCore::X86State::REG_RCX] = Context->Ecx;
State.gregs[FEXCore::X86State::REG_RDX] = Context->Edx;
State.gregs[FEXCore::X86State::REG_RSI] = Context->Esi;
State.gregs[FEXCore::X86State::REG_RDI] = Context->Edi;
State.gregs[FEXCore::X86State::REG_RBP] = Context->Ebp;
State.gregs[FEXCore::X86State::REG_RSP] = Context->Esp;
State.gregs[FEXCore::X86State::REG_RAX] = Context->Eax;
State.gregs[FEXCore::X86State::REG_RBX] = Context->Ebx;
State.gregs[FEXCore::X86State::REG_RCX] = Context->Ecx;
State.gregs[FEXCore::X86State::REG_RDX] = Context->Edx;
State.gregs[FEXCore::X86State::REG_RSI] = Context->Esi;
State.gregs[FEXCore::X86State::REG_RDI] = Context->Edi;
State.gregs[FEXCore::X86State::REG_RBP] = Context->Ebp;
State.gregs[FEXCore::X86State::REG_RSP] = Context->Esp;
State.rip = Context->Eip;
CTX->SetFlagsFromCompactedEFLAGS(Thread, Context->EFlags);
State.rip = Context->Eip;
CTX->SetFlagsFromCompactedEFLAGS(Thread, Context->EFlags);
State.es_idx = Context->SegEs & 0xffff;
State.cs_idx = Context->SegCs & 0xffff;
State.ss_idx = Context->SegSs & 0xffff;
State.ds_idx = Context->SegDs & 0xffff;
State.fs_idx = Context->SegFs & 0xffff;
State.gs_idx = Context->SegGs & 0xffff;
State.es_idx = Context->SegEs & 0xffff;
State.cs_idx = Context->SegCs & 0xffff;
State.ss_idx = Context->SegSs & 0xffff;
State.ds_idx = Context->SegDs & 0xffff;
State.fs_idx = Context->SegFs & 0xffff;
State.gs_idx = Context->SegGs & 0xffff;
// The TEB is the only populated GDT entry by default
State.gdt[(Context->SegFs & 0xffff) >> 3].base = WowTEB;
State.fs_cached = WowTEB;
State.es_cached = 0;
State.cs_cached = 0;
State.ss_cached = 0;
State.ds_cached = 0;
// The TEB is the only populated GDT entry by default
State.gdt[(Context->SegFs & 0xffff) >> 3].base = WowTEB;
State.fs_cached = WowTEB;
State.es_cached = 0;
State.cs_cached = 0;
State.ss_cached = 0;
State.ds_cached = 0;
// Floating-point register state
const auto *XSave = reinterpret_cast<XSAVE_FORMAT*>(Context->ExtendedRegisters);
// Floating-point register state
const auto* XSave = reinterpret_cast<XSAVE_FORMAT*>(Context->ExtendedRegisters);
memcpy(State.xmm.sse.data, XSave->XmmRegisters, sizeof(State.xmm.sse.data));
memcpy(State.mm, XSave->FloatRegisters, sizeof(State.mm));
memcpy(State.xmm.sse.data, XSave->XmmRegisters, sizeof(State.xmm.sse.data));
memcpy(State.mm, XSave->FloatRegisters, sizeof(State.mm));
State.FCW = XSave->ControlWord;
State.flags[FEXCore::X86State::X87FLAG_C0_LOC] = (XSave->StatusWord >> 8) & 1;
State.flags[FEXCore::X86State::X87FLAG_C1_LOC] = (XSave->StatusWord >> 9) & 1;
State.flags[FEXCore::X86State::X87FLAG_C2_LOC] = (XSave->StatusWord >> 10) & 1;
State.flags[FEXCore::X86State::X87FLAG_C3_LOC] = (XSave->StatusWord >> 14) & 1;
State.flags[FEXCore::X86State::X87FLAG_TOP_LOC] = (XSave->StatusWord >> 11) & 0b111;
State.AbridgedFTW = XSave->TagWord;
State.FCW = XSave->ControlWord;
State.flags[FEXCore::X86State::X87FLAG_C0_LOC] = (XSave->StatusWord >> 8) & 1;
State.flags[FEXCore::X86State::X87FLAG_C1_LOC] = (XSave->StatusWord >> 9) & 1;
State.flags[FEXCore::X86State::X87FLAG_C2_LOC] = (XSave->StatusWord >> 10) & 1;
State.flags[FEXCore::X86State::X87FLAG_C3_LOC] = (XSave->StatusWord >> 14) & 1;
State.flags[FEXCore::X86State::X87FLAG_TOP_LOC] = (XSave->StatusWord >> 11) & 0b111;
State.AbridgedFTW = XSave->TagWord;
}
void StoreWowContextFromState(FEXCore::Core::InternalThreadState* Thread, WOW64_CONTEXT* Context) {
auto& State = Thread->CurrentFrame->State;
// General register state
Context->Eax = State.gregs[FEXCore::X86State::REG_RAX];
Context->Ebx = State.gregs[FEXCore::X86State::REG_RBX];
Context->Ecx = State.gregs[FEXCore::X86State::REG_RCX];
Context->Edx = State.gregs[FEXCore::X86State::REG_RDX];
Context->Esi = State.gregs[FEXCore::X86State::REG_RSI];
Context->Edi = State.gregs[FEXCore::X86State::REG_RDI];
Context->Ebp = State.gregs[FEXCore::X86State::REG_RBP];
Context->Esp = State.gregs[FEXCore::X86State::REG_RSP];
Context->Eip = State.rip;
Context->EFlags = CTX->ReconstructCompactedEFLAGS(Thread, false, nullptr, 0);
Context->SegEs = State.es_idx;
Context->SegCs = State.cs_idx;
Context->SegSs = State.ss_idx;
Context->SegDs = State.ds_idx;
Context->SegFs = State.fs_idx;
Context->SegGs = State.gs_idx;
// Floating-point register state
auto* XSave = reinterpret_cast<XSAVE_FORMAT*>(Context->ExtendedRegisters);
memcpy(XSave->XmmRegisters, State.xmm.sse.data, sizeof(State.xmm.sse.data));
memcpy(XSave->FloatRegisters, State.mm, sizeof(State.mm));
XSave->ControlWord = State.FCW;
XSave->StatusWord = (State.flags[FEXCore::X86State::X87FLAG_TOP_LOC] << 11) | (State.flags[FEXCore::X86State::X87FLAG_C0_LOC] << 8) |
(State.flags[FEXCore::X86State::X87FLAG_C1_LOC] << 9) | (State.flags[FEXCore::X86State::X87FLAG_C2_LOC] << 10) |
(State.flags[FEXCore::X86State::X87FLAG_C3_LOC] << 14);
XSave->TagWord = State.AbridgedFTW;
Context->FloatSave.ControlWord = XSave->ControlWord;
Context->FloatSave.StatusWord = XSave->StatusWord;
Context->FloatSave.TagWord = FEXCore::FPState::ConvertFromAbridgedFTW(XSave->StatusWord, State.mm, XSave->TagWord);
Context->FloatSave.ErrorOffset = XSave->ErrorOffset;
Context->FloatSave.ErrorSelector = XSave->ErrorSelector | (XSave->ErrorOpcode << 16);
Context->FloatSave.DataOffset = XSave->DataOffset;
Context->FloatSave.DataSelector = XSave->DataSelector;
Context->FloatSave.Cr0NpxState = XSave->StatusWord | 0xffff0000;
}
NTSTATUS FlushThreadStateContext(HANDLE Thread) {
const auto [Err, TLS] = GetThreadTLS(Thread);
if (Err) {
return Err;
}
void StoreWowContextFromState(FEXCore::Core::InternalThreadState *Thread, WOW64_CONTEXT *Context) {
auto &State = Thread->CurrentFrame->State;
WOW64_CONTEXT TmpWowContext {.ContextFlags = WOW64_CONTEXT_FULL | WOW64_CONTEXT_EXTENDED_REGISTERS};
// General register state
Context::StoreWowContextFromState(TLS.ThreadState(), &TmpWowContext);
return RtlWow64SetThreadContext(Thread, &TmpWowContext);
}
Context->Eax = State.gregs[FEXCore::X86State::REG_RAX];
Context->Ebx = State.gregs[FEXCore::X86State::REG_RBX];
Context->Ecx = State.gregs[FEXCore::X86State::REG_RCX];
Context->Edx = State.gregs[FEXCore::X86State::REG_RDX];
Context->Esi = State.gregs[FEXCore::X86State::REG_RSI];
Context->Edi = State.gregs[FEXCore::X86State::REG_RDI];
Context->Ebp = State.gregs[FEXCore::X86State::REG_RBP];
Context->Esp = State.gregs[FEXCore::X86State::REG_RSP];
void ReconstructThreadState(CONTEXT* Context) {
const auto& Config = SignalDelegator->GetConfig();
auto* Thread = GetTLS().ThreadState();
auto& State = Thread->CurrentFrame->State;
Context->Eip = State.rip;
Context->EFlags = CTX->ReconstructCompactedEFLAGS(Thread, false, nullptr, 0);
State.rip = CTX->RestoreRIPFromHostPC(Thread, Context->Pc);
Context->SegEs = State.es_idx;
Context->SegCs = State.cs_idx;
Context->SegSs = State.ss_idx;
Context->SegDs = State.ds_idx;
Context->SegFs = State.fs_idx;
Context->SegGs = State.gs_idx;
// Floating-point register state
auto *XSave = reinterpret_cast<XSAVE_FORMAT*>(Context->ExtendedRegisters);
memcpy(XSave->XmmRegisters, State.xmm.sse.data, sizeof(State.xmm.sse.data));
memcpy(XSave->FloatRegisters, State.mm, sizeof(State.mm));
XSave->ControlWord = State.FCW;
XSave->StatusWord =
(State.flags[FEXCore::X86State::X87FLAG_TOP_LOC] << 11) |
(State.flags[FEXCore::X86State::X87FLAG_C0_LOC] << 8) |
(State.flags[FEXCore::X86State::X87FLAG_C1_LOC] << 9) |
(State.flags[FEXCore::X86State::X87FLAG_C2_LOC] << 10) |
(State.flags[FEXCore::X86State::X87FLAG_C3_LOC] << 14);
XSave->TagWord = State.AbridgedFTW;
Context->FloatSave.ControlWord = XSave->ControlWord;
Context->FloatSave.StatusWord = XSave->StatusWord;
Context->FloatSave.TagWord = FEXCore::FPState::ConvertFromAbridgedFTW(XSave->StatusWord, State.mm, XSave->TagWord);
Context->FloatSave.ErrorOffset = XSave->ErrorOffset;
Context->FloatSave.ErrorSelector = XSave->ErrorSelector | (XSave->ErrorOpcode << 16);
Context->FloatSave.DataOffset = XSave->DataOffset;
Context->FloatSave.DataSelector = XSave->DataSelector;
Context->FloatSave.Cr0NpxState = XSave->StatusWord | 0xffff0000;
// Spill all SRA GPRs
for (size_t i = 0; i < Config.SRAGPRCount; i++) {
State.gregs[i] = Context->X[Config.SRAGPRMapping[i]];
}
NTSTATUS FlushThreadStateContext(HANDLE Thread) {
const auto [Err, TLS] = GetThreadTLS(Thread);
if (Err) {
return Err;
}
WOW64_CONTEXT TmpWowContext{
.ContextFlags = WOW64_CONTEXT_FULL | WOW64_CONTEXT_EXTENDED_REGISTERS
};
Context::StoreWowContextFromState(TLS.ThreadState(), &TmpWowContext);
return RtlWow64SetThreadContext(Thread, &TmpWowContext);
}
void ReconstructThreadState(CONTEXT *Context) {
const auto &Config = SignalDelegator->GetConfig();
auto *Thread = GetTLS().ThreadState();
auto &State = Thread->CurrentFrame->State;
State.rip = CTX->RestoreRIPFromHostPC(Thread, Context->Pc);
// Spill all SRA GPRs
for (size_t i = 0; i < Config.SRAGPRCount; i++) {
State.gregs[i] = Context->X[Config.SRAGPRMapping[i]];
}
// Spill all SRA FPRs
for (size_t i = 0; i < Config.SRAFPRCount; i++) {
memcpy(State.xmm.sse.data[i], &Context->V[Config.SRAFPRMapping[i]], sizeof(__uint128_t));
}
}
WOW64_CONTEXT ReconstructWowContext(CONTEXT *Context) {
ReconstructThreadState(Context);
WOW64_CONTEXT WowContext{
.ContextFlags = WOW64_CONTEXT_ALL,
};
auto *XSave = reinterpret_cast<XSAVE_FORMAT *>(WowContext.ExtendedRegisters);
XSave->ControlWord = 0x27f;
XSave->MxCsr = 0x1f80;
Context::StoreWowContextFromState(GetTLS().ThreadState(), &WowContext);
return WowContext;
}
bool HandleUnalignedAccess(CONTEXT *Context) {
auto Thread = GetTLS().ThreadState();
if (!Thread->CTX->IsAddressInCodeBuffer(Thread, Context->Pc)) {
return false;
}
FEX_CONFIG_OPT(ParanoidTSO, PARANOIDTSO);
const auto Result = FEXCore::ArchHelpers::Arm64::HandleUnalignedAccess(Thread, ParanoidTSO(), Context->Pc, &Context->X0);
if (!Result.first) {
return false;
}
Context->Pc += Result.second;
return true;
}
void LockJITContext() {
uint32_t Expected = GetTLS().ControlWord().load(), New;
// Spin until PAUSED is unset, setting IN_JIT when that occurs
do {
Expected = Expected & ~ControlBits::PAUSED;
New = (Expected | ControlBits::IN_JIT) & ~ControlBits::WOW_CPU_AREA_DIRTY;
} while (!GetTLS().ControlWord().compare_exchange_weak(Expected, New, std::memory_order::relaxed));
std::atomic_signal_fence(std::memory_order::seq_cst);
// If the CPU area is dirty, flush it to the JIT context before reentry
if (Expected & ControlBits::WOW_CPU_AREA_DIRTY) {
WOW64_CONTEXT *WowContext;
RtlWow64GetCurrentCpuArea(nullptr, reinterpret_cast<void **>(&WowContext), nullptr);
Context::LoadStateFromWowContext(GetTLS().ThreadState(), GetWowTEB(NtCurrentTeb()), WowContext);
}
}
void UnlockJITContext() {
std::atomic_signal_fence(std::memory_order::seq_cst);
GetTLS().ControlWord().fetch_and(~ControlBits::IN_JIT, std::memory_order::relaxed);
}
bool HandleSuspendInterrupt(CONTEXT *Context, uint64_t FaultAddress) {
if (FaultAddress != reinterpret_cast<uint64_t>(&GetTLS().ThreadState()->InterruptFaultPage)) {
return false;
}
void *TmpAddress = reinterpret_cast<void *>(FaultAddress);
SIZE_T TmpSize = FEXCore::Utils::FEX_PAGE_SIZE;
ULONG TmpProt;
NtProtectVirtualMemory(NtCurrentProcess(), &TmpAddress, &TmpSize, PAGE_READWRITE, &TmpProt);
// Since interrupts only happen at the start of blocks, the reconstructed state should be entirely accurate
ReconstructThreadState(Context);
// Yield to the suspender
UnlockJITContext();
LockJITContext();
// Adjust context to return to the dispatcher, reloading SRA from thread state
const auto &Config = SignalDelegator->GetConfig();
Context->Pc = Config.AbsoluteLoopTopAddressFillSRA;
return true;
// Spill all SRA FPRs
for (size_t i = 0; i < Config.SRAFPRCount; i++) {
memcpy(State.xmm.sse.data[i], &Context->V[Config.SRAFPRMapping[i]], sizeof(__uint128_t));
}
}
WOW64_CONTEXT ReconstructWowContext(CONTEXT* Context) {
ReconstructThreadState(Context);
WOW64_CONTEXT WowContext {
.ContextFlags = WOW64_CONTEXT_ALL,
};
auto* XSave = reinterpret_cast<XSAVE_FORMAT*>(WowContext.ExtendedRegisters);
XSave->ControlWord = 0x27f;
XSave->MxCsr = 0x1f80;
Context::StoreWowContextFromState(GetTLS().ThreadState(), &WowContext);
return WowContext;
}
bool HandleUnalignedAccess(CONTEXT* Context) {
auto Thread = GetTLS().ThreadState();
if (!Thread->CTX->IsAddressInCodeBuffer(Thread, Context->Pc)) {
return false;
}
FEX_CONFIG_OPT(ParanoidTSO, PARANOIDTSO);
const auto Result = FEXCore::ArchHelpers::Arm64::HandleUnalignedAccess(Thread, ParanoidTSO(), Context->Pc, &Context->X0);
if (!Result.first) {
return false;
}
Context->Pc += Result.second;
return true;
}
void LockJITContext() {
uint32_t Expected = GetTLS().ControlWord().load(), New;
// Spin until PAUSED is unset, setting IN_JIT when that occurs
do {
Expected = Expected & ~ControlBits::PAUSED;
New = (Expected | ControlBits::IN_JIT) & ~ControlBits::WOW_CPU_AREA_DIRTY;
} while (!GetTLS().ControlWord().compare_exchange_weak(Expected, New, std::memory_order::relaxed));
std::atomic_signal_fence(std::memory_order::seq_cst);
// If the CPU area is dirty, flush it to the JIT context before reentry
if (Expected & ControlBits::WOW_CPU_AREA_DIRTY) {
WOW64_CONTEXT* WowContext;
RtlWow64GetCurrentCpuArea(nullptr, reinterpret_cast<void**>(&WowContext), nullptr);
Context::LoadStateFromWowContext(GetTLS().ThreadState(), GetWowTEB(NtCurrentTeb()), WowContext);
}
}
void UnlockJITContext() {
std::atomic_signal_fence(std::memory_order::seq_cst);
GetTLS().ControlWord().fetch_and(~ControlBits::IN_JIT, std::memory_order::relaxed);
}
bool HandleSuspendInterrupt(CONTEXT* Context, uint64_t FaultAddress) {
if (FaultAddress != reinterpret_cast<uint64_t>(&GetTLS().ThreadState()->InterruptFaultPage)) {
return false;
}
void* TmpAddress = reinterpret_cast<void*>(FaultAddress);
SIZE_T TmpSize = FEXCore::Utils::FEX_PAGE_SIZE;
ULONG TmpProt;
NtProtectVirtualMemory(NtCurrentProcess(), &TmpAddress, &TmpSize, PAGE_READWRITE, &TmpProt);
// Since interrupts only happen at the start of blocks, the reconstructed state should be entirely accurate
ReconstructThreadState(Context);
// Yield to the suspender
UnlockJITContext();
LockJITContext();
// Adjust context to return to the dispatcher, reloading SRA from thread state
const auto& Config = SignalDelegator->GetConfig();
Context->Pc = Config.AbsoluteLoopTopAddressFillSRA;
return true;
}
} // namespace Context
namespace Logging {
void MsgHandler(LogMan::DebugLevels Level, char const *Message) {
const auto Output = fextl::fmt::format("[{}][{:X}] {}\n", LogMan::DebugLevelStr(Level), GetCurrentThreadId(), Message);
__wine_dbg_output(Output.c_str());
}
void AssertHandler(char const *Message) {
const auto Output = fextl::fmt::format("[ASSERT] {}\n", Message);
__wine_dbg_output(Output.c_str());
}
void Init() {
LogMan::Throw::InstallHandler(AssertHandler);
LogMan::Msg::InstallHandler(MsgHandler);
}
void MsgHandler(LogMan::DebugLevels Level, const char* Message) {
const auto Output = fextl::fmt::format("[{}][{:X}] {}\n", LogMan::DebugLevelStr(Level), GetCurrentThreadId(), Message);
__wine_dbg_output(Output.c_str());
}
void AssertHandler(const char* Message) {
const auto Output = fextl::fmt::format("[ASSERT] {}\n", Message);
__wine_dbg_output(Output.c_str());
}
void Init() {
LogMan::Throw::InstallHandler(AssertHandler);
LogMan::Msg::InstallHandler(MsgHandler);
}
} // namespace Logging
class WowSyscallHandler : public FEXCore::HLE::SyscallHandler, public FEXCore::Allocator::FEXAllocOperators {
public:
WowSyscallHandler() {
OSABI = FEXCore::HLE::SyscallOSABI::OS_WIN32;
}
uint64_t HandleSyscall(FEXCore::Core::CpuStateFrame *Frame, FEXCore::HLE::SyscallArguments *Args) override {
const uint64_t ReturnRIP = *(uint32_t *)(Frame->State.gregs[FEXCore::X86State::REG_RSP]); // Return address from the stack
uint64_t ReturnRSP = Frame->State.gregs[FEXCore::X86State::REG_RSP] + 4; // Stack pointer after popping return address
uint64_t HandleSyscall(FEXCore::Core::CpuStateFrame* Frame, FEXCore::HLE::SyscallArguments* Args) override {
const uint64_t ReturnRIP = *(uint32_t*)(Frame->State.gregs[FEXCore::X86State::REG_RSP]); // Return address from the stack
uint64_t ReturnRSP = Frame->State.gregs[FEXCore::X86State::REG_RSP] + 4; // Stack pointer after popping return address
uint64_t ReturnRAX = 0;
if (Frame->State.rip == (uint64_t)&BridgeInstrs::UnixCall) {
@@ -362,7 +358,7 @@ public:
unixlib_handle_t Handle;
UINT32 ID;
ULONG32 Args;
} *StackArgs = reinterpret_cast<StackLayout *>(ReturnRSP);
}* StackArgs = reinterpret_cast<StackLayout*>(ReturnRSP);
ReturnRSP += sizeof(StackLayout);
@@ -374,14 +370,11 @@ public:
Context::UnlockJITContext();
Wow64ProcessPendingCrossProcessItems();
ReturnRAX = static_cast<uint64_t>(Wow64SystemServiceEx(static_cast<UINT>(EntryRAX),
reinterpret_cast<UINT *>(ReturnRSP + 4)));
ReturnRAX = static_cast<uint64_t>(Wow64SystemServiceEx(static_cast<UINT>(EntryRAX), reinterpret_cast<UINT*>(ReturnRSP + 4)));
Context::LockJITContext();
}
// If a new context has been set, use it directly and don't return to the syscall caller
if (Frame->State.rip == (uint64_t)&BridgeInstrs::Syscall ||
Frame->State.rip == (uint64_t)&BridgeInstrs::UnixCall) {
if (Frame->State.rip == (uint64_t)&BridgeInstrs::Syscall || Frame->State.rip == (uint64_t)&BridgeInstrs::UnixCall) {
Frame->State.gregs[FEXCore::X86State::REG_RAX] = ReturnRAX;
Frame->State.gregs[FEXCore::X86State::REG_RSP] = ReturnRSP;
Frame->State.rip = ReturnRIP;
@@ -392,14 +385,14 @@ public:
}
FEXCore::HLE::SyscallABI GetSyscallABI(uint64_t Syscall) override {
return { .NumArgs = 0, .HasReturn = false, .HostSyscallNumber = -1 };
return {.NumArgs = 0, .HasReturn = false, .HostSyscallNumber = -1};
}
FEXCore::HLE::AOTIRCacheEntryLookupResult LookupAOTIRCacheEntry(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestAddr) override {
FEXCore::HLE::AOTIRCacheEntryLookupResult LookupAOTIRCacheEntry(FEXCore::Core::InternalThreadState* Thread, uint64_t GuestAddr) override {
return {0, 0};
}
void MarkGuestExecutableRange(FEXCore::Core::InternalThreadState *Thread, uint64_t Start, uint64_t Length) override {
void MarkGuestExecutableRange(FEXCore::Core::InternalThreadState* Thread, uint64_t Start, uint64_t Length) override {
InvalidationTracker.ReprotectRWXIntervals(Start, Length);
}
};
@@ -461,15 +454,15 @@ NTSTATUS BTCpuThreadTerm(HANDLE Thread) {
return STATUS_SUCCESS;
}
void *BTCpuGetBopCode() {
void* BTCpuGetBopCode() {
return &BridgeInstrs::Syscall;
}
void *__wine_get_unix_opcode() {
void* __wine_get_unix_opcode() {
return &BridgeInstrs::UnixCall;
}
NTSTATUS BTCpuGetContext(HANDLE Thread, HANDLE Process, void *Unknown, WOW64_CONTEXT *Context) {
NTSTATUS BTCpuGetContext(HANDLE Thread, HANDLE Process, void* Unknown, WOW64_CONTEXT* Context) {
auto [Err, TLS] = GetThreadTLS(Thread);
if (Err) {
return Err;
@@ -484,7 +477,7 @@ NTSTATUS BTCpuGetContext(HANDLE Thread, HANDLE Process, void *Unknown, WOW64_CON
return RtlWow64GetThreadContext(Thread, Context);
}
NTSTATUS BTCpuSetContext(HANDLE Thread, HANDLE Process, void *Unknown, WOW64_CONTEXT *Context) {
NTSTATUS BTCpuSetContext(HANDLE Thread, HANDLE Process, void* Unknown, WOW64_CONTEXT* Context) {
auto [Err, TLS] = GetThreadTLS(Thread);
if (Err) {
return Err;
@@ -521,7 +514,7 @@ void BTCpuSimulate() {
// APC handling calls BTCpuSimulate from syscalls and then use NtContinue to return to the previous context,
// to avoid the saved context being clobbered in this case only save the entry context highest in the stack
if (!GetTLS().EntryContext() || GetTLS().EntryContext()->Sp <= entry_context.Sp) {
if (!GetTLS().EntryContext() || GetTLS().EntryContext()->Sp <= entry_context.Sp) {
GetTLS().EntryContext() = &entry_context;
}
@@ -530,7 +523,7 @@ void BTCpuSimulate() {
Context::UnlockJITContext();
}
NTSTATUS BTCpuSuspendLocalThread(HANDLE Thread, ULONG *Count) {
NTSTATUS BTCpuSuspendLocalThread(HANDLE Thread, ULONG* Count) {
THREAD_BASIC_INFORMATION Info;
if (NTSTATUS Err = NtQueryInformationThread(Thread, ThreadBasicInformation, &Info, sizeof(Info), nullptr); Err) {
return Err;
@@ -541,8 +534,7 @@ NTSTATUS BTCpuSuspendLocalThread(HANDLE Thread, ULONG *Count) {
LogMan::Msg::DFmt("Suspending self");
// Mark the CPU area as dirty, to force the JIT context to be restored from it on entry as it may be changed using
// SetThreadContext (which doesn't use the BTCpu API)
if (!(GetTLS().ControlWord().fetch_or(ControlBits::WOW_CPU_AREA_DIRTY, std::memory_order::relaxed) &
ControlBits::WOW_CPU_AREA_DIRTY)) {
if (!(GetTLS().ControlWord().fetch_or(ControlBits::WOW_CPU_AREA_DIRTY, std::memory_order::relaxed) & ControlBits::WOW_CPU_AREA_DIRTY)) {
if (NTSTATUS Err = Context::FlushThreadStateContext(Thread); Err) {
return Err;
}
@@ -561,8 +553,9 @@ NTSTATUS BTCpuSuspendLocalThread(HANDLE Thread, ULONG *Count) {
std::scoped_lock Lock(ThreadSuspendLock);
// If the thread hasn't yet been initialized, suspend it without special handling as it wont yet have entered the JIT
if (!InitializedWOWThreads.contains(ThreadTID))
if (!InitializedWOWThreads.contains(ThreadTID)) {
return NtSuspendThread(Thread, Count);
}
// If CONTROL_IN_JIT is unset at this point, then it can never be set (and thus the JIT cannot be reentered) as
// CONTROL_PAUSED has been set, as such, while this may redundantly request interrupts in rare cases it will never
@@ -571,13 +564,14 @@ NTSTATUS BTCpuSuspendLocalThread(HANDLE Thread, ULONG *Count) {
LogMan::Msg::DFmt("Thread {:X} is in JIT, polling for interrupt", ThreadTID);
ULONG TmpProt;
void *TmpAddress = &TLS.ThreadState()->InterruptFaultPage;
void* TmpAddress = &TLS.ThreadState()->InterruptFaultPage;
SIZE_T TmpSize = FEXCore::Utils::FEX_PAGE_SIZE;
NtProtectVirtualMemory(NtCurrentProcess(), &TmpAddress, &TmpSize, PAGE_READONLY, &TmpProt);
}
// Spin until the JIT is interrupted
while (TLS.ControlWord().load() & ControlBits::IN_JIT);
while (TLS.ControlWord().load() & ControlBits::IN_JIT)
;
// The JIT has now been interrupted and the context stored in the thread's CPU area is up-to-date
if (Err = NtSuspendThread(Thread, Count); Err) {
@@ -585,7 +579,7 @@ NTSTATUS BTCpuSuspendLocalThread(HANDLE Thread, ULONG *Count) {
return Err;
}
CONTEXT TmpContext{
CONTEXT TmpContext {
.ContextFlags = CONTEXT_INTEGER,
};
@@ -610,9 +604,9 @@ NTSTATUS BTCpuSuspendLocalThread(HANDLE Thread, ULONG *Count) {
return Err;
}
NTSTATUS BTCpuResetToConsistentState(EXCEPTION_POINTERS *Ptrs) {
auto *Context = Ptrs->ContextRecord;
const auto *Exception = Ptrs->ExceptionRecord;
NTSTATUS BTCpuResetToConsistentState(EXCEPTION_POINTERS* Ptrs) {
auto* Context = Ptrs->ContextRecord;
const auto* Exception = Ptrs->ExceptionRecord;
if (Exception->ExceptionCode == EXCEPTION_DATATYPE_MISALIGNMENT && Context::HandleUnalignedAccess(Context)) {
LogMan::Msg::DFmt("Handled unaligned atomic: new pc: {:X}", Context->Pc);
NtContinue(Context, FALSE);
@@ -650,21 +644,21 @@ NTSTATUS BTCpuResetToConsistentState(EXCEPTION_POINTERS *Ptrs) {
return STATUS_SUCCESS;
}
void BTCpuFlushInstructionCache2(const void *Address, SIZE_T Size) {
void BTCpuFlushInstructionCache2(const void* Address, SIZE_T Size) {
InvalidationTracker.InvalidateAlignedInterval(GetTLS().ThreadState(), reinterpret_cast<uint64_t>(Address), static_cast<uint64_t>(Size), false);
}
void BTCpuNotifyMemoryAlloc(void *Address, SIZE_T Size, ULONG Type, ULONG Prot) {
InvalidationTracker.HandleMemoryProtectionNotification(GetTLS().ThreadState(), reinterpret_cast<uint64_t>(Address), static_cast<uint64_t>(Size),
Prot);
void BTCpuNotifyMemoryAlloc(void* Address, SIZE_T Size, ULONG Type, ULONG Prot) {
InvalidationTracker.HandleMemoryProtectionNotification(GetTLS().ThreadState(), reinterpret_cast<uint64_t>(Address),
static_cast<uint64_t>(Size), Prot);
}
void BTCpuNotifyMemoryProtect(void *Address, SIZE_T Size, ULONG NewProt) {
InvalidationTracker.HandleMemoryProtectionNotification(GetTLS().ThreadState(), reinterpret_cast<uint64_t>(Address), static_cast<uint64_t>(Size),
NewProt);
void BTCpuNotifyMemoryProtect(void* Address, SIZE_T Size, ULONG NewProt) {
InvalidationTracker.HandleMemoryProtectionNotification(GetTLS().ThreadState(), reinterpret_cast<uint64_t>(Address),
static_cast<uint64_t>(Size), NewProt);
}
void BTCpuNotifyMemoryFree(void *Address, SIZE_T Size, ULONG FreeType) {
void BTCpuNotifyMemoryFree(void* Address, SIZE_T Size, ULONG FreeType) {
if (!Size) {
InvalidationTracker.InvalidateContainingSection(GetTLS().ThreadState(), reinterpret_cast<uint64_t>(Address), true);
} else if (FreeType & MEM_DECOMMIT) {
@@ -672,7 +666,7 @@ void BTCpuNotifyMemoryFree(void *Address, SIZE_T Size, ULONG FreeType) {
}
}
void BTCpuNotifyUnmapViewOfSection(void *Address, ULONG Flags) {
void BTCpuNotifyUnmapViewOfSection(void* Address, ULONG Flags) {
InvalidationTracker.InvalidateContainingSection(GetTLS().ThreadState(), reinterpret_cast<uint64_t>(Address), true);
}
@@ -680,7 +674,7 @@ BOOLEAN WINAPI BTCpuIsProcessorFeaturePresent(UINT Feature) {
return CPUFeatures->IsFeaturePresent(Feature) ? TRUE : FALSE;
}
BOOLEAN BTCpuUpdateProcessorInformation(SYSTEM_CPU_INFORMATION *Info) {
BOOLEAN BTCpuUpdateProcessorInformation(SYSTEM_CPU_INFORMATION* Info) {
CPUFeatures->UpdateInformation(Info);
return TRUE;
}