// SPDX-License-Identifier: MIT #include "VDSO_Emulation.h" #include "LinuxSyscalls/Syscalls.h" #include "LinuxSyscalls/x32/Types.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include namespace FEX::VDSO { static VDSOEntrypoints VDSOPointers {}; namespace VDSOHandlers { using TimeType = decltype(::time)*; using GetTimeOfDayType = decltype(::gettimeofday)*; using ClockGetTimeType = decltype(::clock_gettime)*; using ClockGetResType = decltype(::clock_getres)*; using GetCPUType = decltype(FHU::Syscalls::getcpu)*; using GetRandomType = ssize_t (*)(void*, size_t, uint32_t, void*, size_t); static TimeType TimePtr; static GetTimeOfDayType GetTimeOfDayPtr; static ClockGetTimeType ClockGetTimePtr; static ClockGetResType ClockGetResPtr; static GetCPUType GetCPUPtr; static GetRandomType GetRandomPtr; } // namespace VDSOHandlers using HandlerPtr = void (*)(void*); namespace x64 { static uint64_t SyscallRet(uint64_t Result) { if (Result == -1) { return -errno; } return Result; } // glibc handlers namespace glibc { static void time(void* ArgsRV) { struct __attribute__((packed)) ArgsRV_t { time_t* a_0; uint64_t rv; }* args = reinterpret_cast(ArgsRV); uint64_t Result = ::time(args->a_0); args->rv = SyscallRet(Result); } static void gettimeofday(void* ArgsRV) { struct __attribute__((packed)) ArgsRV_t { struct timeval* tv; struct timezone* tz; int rv; }* args = reinterpret_cast(ArgsRV); int Result = ::gettimeofday(args->tv, args->tz); args->rv = SyscallRet(Result); } static void clock_gettime(void* ArgsRV) { struct __attribute__((packed)) ArgsRV_t { clockid_t clk_id; struct timespec* tp; int rv; }* args = reinterpret_cast(ArgsRV); int Result = ::clock_gettime(args->clk_id, args->tp); args->rv = SyscallRet(Result); } static void clock_getres(void* ArgsRV) { struct __attribute__((packed)) ArgsRV_t { clockid_t clk_id; struct timespec* tp; int rv; }* args = reinterpret_cast(ArgsRV); int Result = ::clock_getres(args->clk_id, args->tp); args->rv = SyscallRet(Result); } static void getcpu(void* ArgsRV) { struct __attribute__((packed)) ArgsRV_t { uint32_t* cpu; uint32_t* node; int rv; }* args = reinterpret_cast(ArgsRV); int Result = FHU::Syscalls::getcpu(args->cpu, args->node); args->rv = SyscallRet(Result); } static void getrandom(void* ArgsRV) { struct vgetrandom_opaque_params { uint32_t size_of_opaque_state; uint32_t mmap_prot; uint32_t mmap_flags; uint32_t reserved[13]; }; static_assert(sizeof(vgetrandom_opaque_params) == sizeof(uint32_t[16])); struct __attribute__((packed)) ArgsRV_t { void* buffer; size_t len; uint32_t flags; vgetrandom_opaque_params* opaque_state; size_t opaque_len; ssize_t rv; }* args = reinterpret_cast(ArgsRV); if (args->buffer == nullptr && args->len == 0 && args->flags == 0 && args->opaque_len == ~0ULL) [[unlikely]] { // Special case querying for flags // Since this is the syscall implementation, we need to return valid but unused data. // This will cause glibc to allocate a page of memory, but it ends up being unused. args->opaque_state->size_of_opaque_state = FEXCore::Utils::FEX_PAGE_SIZE; args->opaque_state->mmap_prot = PROT_NONE; args->opaque_state->mmap_flags = MAP_NORESERVE | MAP_ANONYMOUS | MAP_PRIVATE; args->rv = 0; return; } int Result = ::syscall(SYS_getrandom, args->buffer, args->len, args->flags); args->rv = SyscallRet(Result); } } // namespace glibc namespace VDSO { // VDSO handlers static void time(void* ArgsRV) { struct __attribute__((packed)) ArgsRV_t { time_t* a_0; uint64_t rv; }* args = reinterpret_cast(ArgsRV); args->rv = VDSOHandlers::TimePtr(args->a_0); } static void gettimeofday(void* ArgsRV) { struct __attribute__((packed)) ArgsRV_t { struct timeval* tv; struct timezone* tz; int rv; }* args = reinterpret_cast(ArgsRV); args->rv = VDSOHandlers::GetTimeOfDayPtr(args->tv, args->tz); } static void clock_gettime(void* ArgsRV) { struct __attribute__((packed)) ArgsRV_t { clockid_t clk_id; struct timespec* tp; int rv; }* args = reinterpret_cast(ArgsRV); args->rv = VDSOHandlers::ClockGetTimePtr(args->clk_id, args->tp); } static void clock_getres(void* ArgsRV) { struct __attribute__((packed)) ArgsRV_t { clockid_t clk_id; struct timespec* tp; int rv; }* args = reinterpret_cast(ArgsRV); args->rv = VDSOHandlers::ClockGetResPtr(args->clk_id, args->tp); } static void getcpu(void* ArgsRV) { struct __attribute__((packed)) ArgsRV_t { uint32_t* cpu; uint32_t* node; int rv; }* args = reinterpret_cast(ArgsRV); args->rv = VDSOHandlers::GetCPUPtr(args->cpu, args->node); } static void getrandom(void* ArgsRV) { struct __attribute__((packed)) ArgsRV_t { void* buffer; size_t len; uint32_t flags; void* opaque_state; size_t opaque_len; ssize_t rv; }* args = reinterpret_cast(ArgsRV); args->rv = VDSOHandlers::GetRandomPtr(args->buffer, args->len, args->flags, args->opaque_state, args->opaque_len); } } // namespace VDSO static HandlerPtr Handler_time = FEX::VDSO::x64::glibc::time; static HandlerPtr Handler_gettimeofday = FEX::VDSO::x64::glibc::gettimeofday; static HandlerPtr Handler_clock_gettime = FEX::VDSO::x64::glibc::clock_gettime; static HandlerPtr Handler_clock_getres = FEX::VDSO::x64::glibc::clock_getres; static HandlerPtr Handler_getcpu = FEX::VDSO::x64::glibc::getcpu; static HandlerPtr Handler_getrandom = FEX::VDSO::x64::glibc::getrandom; } // namespace x64 namespace x32 { namespace glibc { static int SyscallRet(int Result) { if (Result == -1) { return -errno; } return Result; } // glibc handlers static void time(void* ArgsRV) { struct __attribute__((packed)) ArgsRV_t { HLE::x32::compat_ptr a_0; int rv; }* args = reinterpret_cast(ArgsRV); time_t Host {}; int Result = ::time(&Host); args->rv = SyscallRet(Result); if (Result != -1 && args->a_0) { *args->a_0 = Host; } } static void gettimeofday(void* ArgsRV) { struct __attribute__((packed)) ArgsRV_t { HLE::x32::compat_ptr tv; HLE::x32::compat_ptr tz; int rv; }* args = reinterpret_cast(ArgsRV); struct timeval tv64 {}; struct timeval* tv_ptr {}; if (args->tv) { tv_ptr = &tv64; } int Result = ::gettimeofday(tv_ptr, args->tz); args->rv = SyscallRet(Result); if (Result != -1 && args->tv) { *args->tv = tv64; } } static void clock_gettime(void* ArgsRV) { struct __attribute__((packed)) ArgsRV_t { clockid_t clk_id; HLE::x32::compat_ptr tp; int rv; }* args = reinterpret_cast(ArgsRV); struct timespec tp64 {}; int Result = ::clock_gettime(args->clk_id, &tp64); args->rv = SyscallRet(Result); if (Result != -1 && args->tp) { *args->tp = tp64; } } static void clock_gettime64(void* ArgsRV) { struct __attribute__((packed)) ArgsRV_t { clockid_t clk_id; HLE::x32::compat_ptr tp; int rv; }* args = reinterpret_cast(ArgsRV); int Result = ::clock_gettime(args->clk_id, args->tp); args->rv = SyscallRet(Result); } static void clock_getres(void* ArgsRV) { struct __attribute__((packed)) ArgsRV_t { clockid_t clk_id; HLE::x32::compat_ptr tp; int rv; }* args = reinterpret_cast(ArgsRV); struct timespec tp64 {}; int Result = ::clock_getres(args->clk_id, &tp64); args->rv = SyscallRet(Result); if (Result != -1 && args->tp) { *args->tp = tp64; } } static void getcpu(void* ArgsRV) { struct __attribute__((packed)) ArgsRV_t { HLE::x32::compat_ptr cpu; HLE::x32::compat_ptr node; int rv; }* args = reinterpret_cast(ArgsRV); int Result = FHU::Syscalls::getcpu(args->cpu, args->node); args->rv = SyscallRet(Result); } } // namespace glibc namespace VDSO { static bool SyscallErr(uint64_t Result) { return Result >= -4095; } // VDSO handlers static void time(void* ArgsRV) { struct __attribute__((packed)) ArgsRV_t { HLE::x32::compat_ptr a_0; int rv; }* args = reinterpret_cast(ArgsRV); time_t Host {}; uint64_t Result = VDSOHandlers::TimePtr(&Host); args->rv = Result; if (!SyscallErr(Result) && args->a_0) { *args->a_0 = Host; } } static void gettimeofday(void* ArgsRV) { struct __attribute__((packed)) ArgsRV_t { HLE::x32::compat_ptr tv; HLE::x32::compat_ptr tz; int rv; }* args = reinterpret_cast(ArgsRV); struct timeval tv64 {}; struct timeval* tv_ptr {}; if (args->tv) { tv_ptr = &tv64; } uint64_t Result = VDSOHandlers::GetTimeOfDayPtr(tv_ptr, args->tz); args->rv = Result; if (!SyscallErr(Result) && args->tv) { *args->tv = tv64; } } static void clock_gettime(void* ArgsRV) { struct __attribute__((packed)) ArgsRV_t { clockid_t clk_id; HLE::x32::compat_ptr tp; int rv; }* args = reinterpret_cast(ArgsRV); struct timespec tp64 {}; uint64_t Result = VDSOHandlers::ClockGetTimePtr(args->clk_id, &tp64); args->rv = Result; if (!SyscallErr(Result) && args->tp) { *args->tp = tp64; } } static void clock_gettime64(void* ArgsRV) { struct __attribute__((packed)) ArgsRV_t { clockid_t clk_id; HLE::x32::compat_ptr tp; int rv; }* args = reinterpret_cast(ArgsRV); args->rv = VDSOHandlers::ClockGetTimePtr(args->clk_id, args->tp); } static void clock_getres(void* ArgsRV) { struct __attribute__((packed)) ArgsRV_t { clockid_t clk_id; HLE::x32::compat_ptr tp; int rv; }* args = reinterpret_cast(ArgsRV); struct timespec tp64 {}; uint64_t Result = VDSOHandlers::ClockGetResPtr(args->clk_id, &tp64); args->rv = Result; if (!SyscallErr(Result) && args->tp) { *args->tp = tp64; } } static void getcpu(void* ArgsRV) { struct __attribute__((packed)) ArgsRV_t { HLE::x32::compat_ptr cpu; HLE::x32::compat_ptr node; int rv; }* args = reinterpret_cast(ArgsRV); args->rv = VDSOHandlers::GetCPUPtr(args->cpu, args->node); } } // namespace VDSO static HandlerPtr Handler_time = FEX::VDSO::x32::glibc::time; static HandlerPtr Handler_gettimeofday = FEX::VDSO::x32::glibc::gettimeofday; static HandlerPtr Handler_clock_gettime = FEX::VDSO::x32::glibc::clock_gettime; static HandlerPtr Handler_clock_gettime64 = FEX::VDSO::x32::glibc::clock_gettime64; static HandlerPtr Handler_clock_getres = FEX::VDSO::x32::glibc::clock_getres; static HandlerPtr Handler_getcpu = FEX::VDSO::x32::glibc::getcpu; } // namespace x32 class VDSOParser final { public: VDSOParser(const uint8_t* HeaderBase); void* FindSymbol(std::string_view Name) const { auto it = Symbols.find(Name); if (it == Symbols.end()) { return nullptr; } return it->second; } private: fextl::map Symbols; }; VDSOParser::VDSOParser(const uint8_t* HeaderBase) { // Minimal ELF parser that only knows how to scan for dynamic symbols from VDSO. auto Header = reinterpret_cast(HeaderBase); auto SectionHeaderOffset = Header->e_shoff; auto SectionHeaderCount = Header->e_shnum; auto SectionHeaders = reinterpret_cast(&HeaderBase[SectionHeaderOffset]); // Scan for the symbol and string headers. const Elf64_Shdr* DynamicSymbolHeader {}; const Elf64_Shdr* DynamicStringHeader {}; for (size_t i = 0; i < SectionHeaderCount; ++i) { if (DynamicSymbolHeader && DynamicStringHeader) { // Found both headers. break; } if (SectionHeaders[i].sh_type == SHT_DYNSYM) { // Dynamic symbol header found. DynamicSymbolHeader = &SectionHeaders[i]; } if (SectionHeaders[i].sh_type == SHT_STRTAB && SectionHeaders[i].sh_addr) { // Dynamic string header found. DynamicStringHeader = &SectionHeaders[i]; } } if (!DynamicSymbolHeader || !DynamicStringHeader) { LogMan::Msg::DFmt("Couldn't parse host VDSO symbols. Falling back to glibc implementations."); return; } auto NumberOfDynamicSymbols = DynamicSymbolHeader->sh_size / DynamicSymbolHeader->sh_entsize; const char* DynamicStringTable = reinterpret_cast(&HeaderBase[DynamicStringHeader->sh_offset]); // Scan all the symbols and populate the look-up table. for (size_t i = 0; i < NumberOfDynamicSymbols; ++i) { auto Offset = DynamicSymbolHeader->sh_offset + (i * DynamicSymbolHeader->sh_entsize); auto Symbol = reinterpret_cast(&HeaderBase[Offset]); if (Symbol->st_info != 0) { // Save the symbol. const char* Name = &DynamicStringTable[Symbol->st_name]; auto SymbolPtr = HeaderBase + Symbol->st_value; Symbols[Name] = const_cast(static_cast(SymbolPtr)); } } } static void LoadHostVDSO() { // Linux gives the VDSO ELF header base in the auxv value AT_SYSINFO_EHDR. auto VDSOHeader = ::getauxval(AT_SYSINFO_EHDR); if (!VDSOHeader) { // We couldn't load VDSO, fallback to C implementations. Which will still be faster than emulated libc versions. LogMan::Msg::IFmt("linux-vdso implementation falling back to libc. Consider enabling VDSO in your kernel."); return; } auto VDSO = VDSOParser(reinterpret_cast(VDSOHeader)); auto SymbolPtr = VDSO.FindSymbol("__kernel_time"); if (!SymbolPtr) { SymbolPtr = VDSO.FindSymbol("__vdso_time"); } if (SymbolPtr) { VDSOHandlers::TimePtr = reinterpret_cast(SymbolPtr); x64::Handler_time = x64::VDSO::time; x32::Handler_time = x32::VDSO::time; } SymbolPtr = VDSO.FindSymbol("__kernel_gettimeofday"); if (!SymbolPtr) { SymbolPtr = VDSO.FindSymbol("__vdso_gettimeofday"); } if (SymbolPtr) { VDSOHandlers::GetTimeOfDayPtr = reinterpret_cast(SymbolPtr); x64::Handler_gettimeofday = x64::VDSO::gettimeofday; x32::Handler_gettimeofday = x32::VDSO::gettimeofday; } SymbolPtr = VDSO.FindSymbol("__kernel_clock_gettime"); if (!SymbolPtr) { SymbolPtr = VDSO.FindSymbol("__vdso_clock_gettime"); } if (SymbolPtr) { VDSOHandlers::ClockGetTimePtr = reinterpret_cast(SymbolPtr); x64::Handler_clock_gettime = x64::VDSO::clock_gettime; x32::Handler_clock_gettime = x32::VDSO::clock_gettime; x32::Handler_clock_gettime64 = x32::VDSO::clock_gettime64; } SymbolPtr = VDSO.FindSymbol("__kernel_clock_getres"); if (!SymbolPtr) { SymbolPtr = VDSO.FindSymbol("__vdso_clock_getres"); } if (SymbolPtr) { VDSOHandlers::ClockGetResPtr = reinterpret_cast(SymbolPtr); x64::Handler_clock_getres = x64::VDSO::clock_getres; x32::Handler_clock_getres = x32::VDSO::clock_getres; } SymbolPtr = VDSO.FindSymbol("__kernel_getcpu"); if (!SymbolPtr) { SymbolPtr = VDSO.FindSymbol("__vdso_getcpu"); } if (SymbolPtr) { VDSOHandlers::GetCPUPtr = reinterpret_cast(SymbolPtr); x64::Handler_getcpu = x64::VDSO::getcpu; x32::Handler_getcpu = x32::VDSO::getcpu; } SymbolPtr = VDSO.FindSymbol("__kernel_getrandom"); if (!SymbolPtr) { SymbolPtr = VDSO.FindSymbol("__vdso_getrandom"); } if (SymbolPtr) { VDSOHandlers::GetRandomPtr = reinterpret_cast(SymbolPtr); x64::Handler_getrandom = x64::VDSO::getrandom; // 32-bit doesn't have getrandom vdso } } static std::array VDSODefinitions = {{ { // sha256(libVDSO:time) {0x37, 0x63, 0x46, 0xb0, 0x79, 0x06, 0x5f, 0x9d, 0x00, 0xb6, 0x8d, 0xfd, 0x9e, 0x4a, 0x62, 0xcd, 0x1e, 0x6c, 0xcc, 0x22, 0xcd, 0xb2, 0xc0, 0x17, 0x7d, 0x42, 0x6a, 0x40, 0xd1, 0xeb, 0xfa, 0xe0}, nullptr, }, { // sha256(libVDSO:gettimeofday) {0x77, 0x2a, 0xde, 0x1c, 0x13, 0x2d, 0xe9, 0x48, 0xaf, 0xe0, 0xba, 0xcc, 0x6a, 0x89, 0xff, 0xca, 0x4a, 0xdc, 0xd5, 0x63, 0x2c, 0xc5, 0x62, 0x8b, 0x5d, 0xde, 0x0b, 0x15, 0x35, 0xc6, 0xc7, 0x14}, nullptr, }, { // sha256(libVDSO:clock_gettime) {0x3c, 0x96, 0x9b, 0x2d, 0xc3, 0xad, 0x2b, 0x3b, 0x9c, 0x4e, 0x4d, 0xca, 0x1c, 0xe8, 0x18, 0x4a, 0x12, 0x8a, 0xe4, 0xc1, 0x56, 0x92, 0x73, 0xce, 0x65, 0x85, 0x5f, 0x65, 0x7e, 0x94, 0x26, 0xbe}, nullptr, }, { // sha256(libVDSO:clock_gettime64) {0xba, 0xe9, 0x6d, 0x30, 0xc0, 0x68, 0xc6, 0xd7, 0x59, 0x04, 0xf7, 0x10, 0x06, 0x72, 0x88, 0xfd, 0x4c, 0x57, 0x0f, 0x31, 0xa5, 0xea, 0xa9, 0xb9, 0xd3, 0x8d, 0x03, 0x81, 0x50, 0x16, 0x22, 0x71}, nullptr, }, { // sha256(libVDSO:clock_getres) {0xe4, 0xa1, 0xf6, 0x23, 0x35, 0xae, 0xb7, 0xb6, 0xb0, 0x37, 0xc5, 0xc3, 0xa3, 0xfd, 0xbf, 0xa2, 0xa1, 0xc8, 0x95, 0x78, 0xe5, 0x76, 0x86, 0xdb, 0x3e, 0x6c, 0x54, 0xd5, 0x02, 0x60, 0xd8, 0x6d}, nullptr, }, { // sha256(libVDSO:getcpu) {0x39, 0x83, 0x39, 0x36, 0x0f, 0x68, 0xd6, 0xfc, 0xc2, 0x3a, 0x97, 0x11, 0x85, 0x09, 0xc7, 0x25, 0xbb, 0x50, 0x49, 0x55, 0x6b, 0x0c, 0x9f, 0x50, 0x37, 0xf5, 0x9d, 0xb0, 0x38, 0x58, 0x57, 0x12}, nullptr, }, { // sha256(libVDSO:getrandom) {0xf8, 0x03, 0xe2, 0x70, 0xe3, 0xf1, 0xbb, 0xc1, 0x7d, 0xa7, 0x8b, 0xb3, 0x1f, 0x3e, 0xbd, 0xc6, 0x8a, 0x50, 0xd3, 0x4a, 0x1f, 0xb3, 0x4b, 0x7e, 0x32, 0xcb, 0x1e, 0x18, 0x3b, 0x7c, 0xeb, 0x4b}, nullptr, }, }}; template static void LoadGuestVDSOSymbols(char* VDSOBase) { using ELFHeaderType = std::conditional_t; using ELFSHeaderType = std::conditional_t; using ELFSymbolType = std::conditional_t; constexpr auto ELFClass = Is64Bit ? ELFCLASS64 : ELFCLASS32; constexpr auto ELFMachine = Is64Bit ? EM_X86_64 : EM_386; // We need to load symbols we care about. auto Header = reinterpret_cast(VDSOBase); // Check ELF magic. if (Header->e_ident[EI_MAG0] != ELFMAG0 || Header->e_ident[EI_MAG1] != ELFMAG1 || Header->e_ident[EI_MAG2] != ELFMAG2 || Header->e_ident[EI_MAG3] != ELFMAG3) { return; } // Check ELF class and Machine. if (Header->e_ident[EI_CLASS] != ELFClass || Header->e_machine != ELFMachine) { return; } // First walk the section headers to find the symbol table. auto RawShdrs = reinterpret_cast(VDSOBase + Header->e_shoff); const auto StrHeader = &RawShdrs[Header->e_shstrndx]; const char* SHStrings = VDSOBase + StrHeader->sh_offset; struct SymbolTypes { const char* name; int sh_type; }; constexpr std::array symbol_table_names = {{{".dynsym", SHT_DYNSYM}, {".symtab", SHT_SYMTAB}}}; for (auto sym_table : symbol_table_names) { const ELFSHeaderType* SymTableHeader {}; const ELFSHeaderType* StringTableHeader {}; for (size_t i = 0; i < Header->e_shnum; ++i) { const auto& Header = RawShdrs[i]; if (Header.sh_type == sym_table.sh_type && strcmp(&SHStrings[Header.sh_name], sym_table.name) == 0) { SymTableHeader = &Header; StringTableHeader = &RawShdrs[SymTableHeader->sh_link]; break; } } if (!SymTableHeader) { // Couldn't find symbol table continue; } const char* StrTab = VDSOBase + StringTableHeader->sh_offset; size_t NumSymbols = SymTableHeader->sh_size / SymTableHeader->sh_entsize; for (size_t i = 0; i < NumSymbols; ++i) { uint64_t offset = SymTableHeader->sh_offset + i * SymTableHeader->sh_entsize; auto Symbol = reinterpret_cast(VDSOBase + offset); if (ELF32_ST_VISIBILITY(Symbol->st_other) != STV_HIDDEN && Symbol->st_value != 0) { const char* Name = &StrTab[Symbol->st_name]; if (Name[0] != '\0') { if (strcmp(Name, "__kernel_sigreturn") == 0) { VDSOPointers.VDSO_kernel_sigreturn = VDSOBase + Symbol->st_value; } else if (strcmp(Name, "__kernel_rt_sigreturn") == 0) { VDSOPointers.VDSO_kernel_rt_sigreturn = VDSOBase + Symbol->st_value; } else if (strcmp(Name, "__fex_callback_ret") == 0) { VDSOPointers.VDSO_FEX_CallbackRET = VDSOBase + Symbol->st_value; } } } } } } static void LoadFEXGeneratedCode(FEXCore::Core::InternalThreadState* Thread, bool Is64Bit, VDSOMapping* Mapping, FEX::HLE::SyscallHandler* const Handler) { if (VDSOPointers.VDSO_FEX_CallbackRET && (!Is64Bit || (VDSOPointers.VDSO_kernel_sigreturn && VDSOPointers.VDSO_kernel_rt_sigreturn))) { // Unnecessary if all VDSO paths have already been loaded. return; } // Hardcoded to one page for now auto PageSize = sysconf(_SC_PAGESIZE); PageSize = PageSize > 0 ? PageSize : FEXCore::Utils::FEX_PAGE_SIZE; Mapping->X86GeneratedCodeSize = PageSize; if (Is64Bit) { // 64bit mode can have its code anywhere auto Result = Handler->GuestMmap(Is64Bit, Thread, nullptr, Mapping->X86GeneratedCodeSize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if (!FEX::HLE::HasSyscallError(Result)) { Mapping->X86GeneratedCodePtr = Result; } } else { // We need to have the sigret handler in the lower 32bits of memory space // Scan top down and try to allocate a location for (size_t Location = 0xFFFF'E000; Location != 0x0; Location -= PageSize) { auto Ptr = Handler->GuestMmap(Is64Bit, Thread, reinterpret_cast(Location), PageSize, PROT_READ | PROT_WRITE, MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if (!FEX::HLE::HasSyscallError(Ptr)) { Mapping->X86GeneratedCodePtr = Ptr; break; } } } // Can't do anything about this // Here's hoping the application doesn't use signals if (!Mapping->X86GeneratedCodePtr) { return; } FEXCore::Allocator::VirtualName("FEXMem_Misc", Mapping->X86GeneratedCodePtr, Mapping->X86GeneratedCodeSize); size_t CurrentCodeOffset {}; if (!Is64Bit) { // Signal return handlers need to be bit-exact to what the Linux kernel provides in VDSO. // GDB and unwinding libraries key off of these instructions to understand if the stack frame is a signal frame or not. // This two code sections match exactly what libSegFault expects. // // Typically this handlers are provided by the 32-bit VDSO thunk library, but that isn't available in all cases. // Falling back to this generated code segment still allows a backtrace to work, just might not show // the symbol as VDSO since there is no ELF to parse. constexpr std::array sigreturn_32_code = { 0x58, // pop eax 0xb8, 0x77, 0x00, 0x00, 0x00, // mov eax, 0x77 0xcd, 0x80, // int 0x80 0x90, // nop }; constexpr std::array rt_sigreturn_32_code = { 0xb8, 0xad, 0x00, 0x00, 0x00, // mov eax, 0xad 0xcd, 0x80, // int 0x80 }; if (!VDSOPointers.VDSO_kernel_sigreturn) { VDSOPointers.VDSO_kernel_sigreturn = reinterpret_cast(reinterpret_cast(Mapping->X86GeneratedCodePtr) + CurrentCodeOffset); memcpy(VDSOPointers.VDSO_kernel_sigreturn, sigreturn_32_code.data(), sigreturn_32_code.size()); CurrentCodeOffset += sigreturn_32_code.size(); } if (!VDSOPointers.VDSO_kernel_rt_sigreturn) { VDSOPointers.VDSO_kernel_rt_sigreturn = reinterpret_cast(reinterpret_cast(Mapping->X86GeneratedCodePtr) + CurrentCodeOffset); memcpy(VDSOPointers.VDSO_kernel_rt_sigreturn, rt_sigreturn_32_code.data(), rt_sigreturn_32_code.size()); CurrentCodeOffset += rt_sigreturn_32_code.size(); } } if (!VDSOPointers.VDSO_FEX_CallbackRET) { constexpr std::array CallbackRetCode = { 0x0F, 0x3E, // CALLBACKRET FEX Instruction }; VDSOPointers.VDSO_FEX_CallbackRET = reinterpret_cast(reinterpret_cast(Mapping->X86GeneratedCodePtr) + CurrentCodeOffset); memcpy(VDSOPointers.VDSO_FEX_CallbackRET, CallbackRetCode.data(), CallbackRetCode.size()); CurrentCodeOffset += CallbackRetCode.size(); } Handler->GuestMprotect(Thread, Mapping->X86GeneratedCodePtr, Mapping->X86GeneratedCodeSize, PROT_READ | PROT_EXEC); } void UnloadVDSOMapping(FEXCore::Core::InternalThreadState* Thread, FEX::HLE::SyscallHandler* const Handler, const VDSOMapping& Mapping) { if (Mapping.VDSOBase) { Handler->GuestMunmap(Thread, Mapping.VDSOBase, Mapping.VDSOSize); } if (Mapping.X86GeneratedCodePtr) { Handler->GuestMunmap(Thread, Mapping.X86GeneratedCodePtr, Mapping.X86GeneratedCodeSize); } } VDSOMapping LoadVDSOThunks(FEXCore::Core::InternalThreadState* Thread, bool Is64Bit, FEX::HLE::SyscallHandler* const Handler) { VDSOMapping Mapping {}; FEX_CONFIG_OPT(ThunkGuestLibs, THUNKGUESTLIBS); fextl::string ThunkGuestPath = ThunkGuestLibs(); while (ThunkGuestPath.ends_with('/')) { ThunkGuestPath.pop_back(); } ThunkGuestPath = fextl::fmt::format("{}{}/libVDSO-guest.so", ThunkGuestPath, Is64Bit ? "" : "_32"); // Load VDSO if we can int VDSOFD = ::open(ThunkGuestPath.c_str(), O_RDONLY); if (VDSOFD != -1) { // Get file size Mapping.VDSOSize = lseek(VDSOFD, 0, SEEK_END); if (Mapping.VDSOSize >= std::min(sizeof(Elf32_Ehdr), sizeof(Elf64_Ehdr))) { // Reset to beginning lseek(VDSOFD, 0, SEEK_SET); Mapping.VDSOSize = FEXCore::AlignUp(Mapping.VDSOSize, FEXCore::Utils::FEX_PAGE_SIZE); auto VABits = FEXCore::Allocator::GetHostVABits(); uint64_t VDSOHint {}; if (Is64Bit) { if (VABits > 47) { // If VA size is at least as large as minimum x86 specification, then set to max. VABits = 47; } // Calculate the highest point the vdso could go. VDSOHint = (1ULL << VABits) - Mapping.VDSOSize; } else { VDSOHint = 0x1'0000'0000ULL - Mapping.VDSOSize; } auto PageSize = sysconf(_SC_PAGESIZE); PageSize = PageSize > 0 ? PageSize : FEXCore::Utils::FEX_PAGE_SIZE; // Scan top down and try to allocate a location void* VDSOPointerBase {}; do { VDSOPointerBase = Handler->GuestMmap(Is64Bit, Thread, reinterpret_cast(VDSOHint), Mapping.VDSOSize, PROT_READ | PROT_EXEC, MAP_FIXED_NOREPLACE | MAP_SHARED, VDSOFD, 0); // Scan-downward until we fit. VDSOHint -= PageSize; } while (FEX::HLE::HasSyscallError(VDSOPointerBase) && static_cast(VDSOHint) > 0); if (FEX::HLE::HasSyscallError(VDSOPointerBase)) { LogMan::Msg::EFmt("Couldn't Map VDSO"); close(VDSOFD); return {}; } Mapping.VDSOBase = VDSOPointerBase; // Since we found our VDSO thunk library, find our host VDSO function implementations. LoadHostVDSO(); } close(VDSOFD); if (!Mapping.VDSOBase) { return {}; } if (Is64Bit) { LoadGuestVDSOSymbols(reinterpret_cast(Mapping.VDSOBase)); } else { LoadGuestVDSOSymbols(reinterpret_cast(Mapping.VDSOBase)); } } // If VDSO couldn't find sigreturn then FEX needs to provide unique implementations. LoadFEXGeneratedCode(Thread, Is64Bit, &Mapping, Handler); if (Is64Bit) { // Set the Thunk definition pointers for x86-64 VDSODefinitions[0].ThunkFunction = FEX::VDSO::x64::Handler_time; VDSODefinitions[1].ThunkFunction = FEX::VDSO::x64::Handler_gettimeofday; VDSODefinitions[2].ThunkFunction = FEX::VDSO::x64::Handler_clock_gettime; VDSODefinitions[3].ThunkFunction = FEX::VDSO::x64::Handler_clock_gettime; VDSODefinitions[4].ThunkFunction = FEX::VDSO::x64::Handler_clock_getres; VDSODefinitions[5].ThunkFunction = FEX::VDSO::x64::Handler_getcpu; VDSODefinitions[6].ThunkFunction = FEX::VDSO::x64::Handler_getrandom; } else { // Set the Thunk definition pointers for x86 VDSODefinitions[0].ThunkFunction = FEX::VDSO::x32::Handler_time; VDSODefinitions[1].ThunkFunction = FEX::VDSO::x32::Handler_gettimeofday; VDSODefinitions[2].ThunkFunction = FEX::VDSO::x32::Handler_clock_gettime; VDSODefinitions[3].ThunkFunction = FEX::VDSO::x32::Handler_clock_gettime64; VDSODefinitions[4].ThunkFunction = FEX::VDSO::x32::Handler_clock_getres; VDSODefinitions[5].ThunkFunction = FEX::VDSO::x32::Handler_getcpu; // getrandom doesn't exist on 32-bit, so leave VDSODefinitions[6] unfilled } return Mapping; } uint64_t GetVSyscallEntry(const void* VDSOBase) { if (!VDSOBase) { return 0; } // Extract the vsyscall location from the VDSO header. auto Header = reinterpret_cast(VDSOBase); if (Header->e_entry) { return reinterpret_cast(VDSOBase) + Header->e_entry; } return 0; } const std::span GetVDSOThunkDefinitions(bool Is64Bit) { return std::span(VDSODefinitions.begin(), VDSODefinitions.end() - (Is64Bit ? 0 : 1)); } const VDSOEntrypoints& GetVDSOSymbols() { return VDSOPointers; } } // namespace FEX::VDSO