diff --git a/FEXCore/Source/Interface/Context/Context.h b/FEXCore/Source/Interface/Context/Context.h index b3868ca22..9bc21d8d5 100644 --- a/FEXCore/Source/Interface/Context/Context.h +++ b/FEXCore/Source/Interface/Context/Context.h @@ -118,12 +118,14 @@ public: * Note that FEX relocations are unrelated to ELF/PE relocations. * * @param GuestDelta Guest address offset to apply to RIP-relative data + * @param RelocationOffset Offset to subtract from relocation target offsets * @param ForStorage True for serializing data (producing deterministic output); false for de-serializing it (resolving dynamic symbols) * * @return Returns true on success */ [[nodiscard]] - bool ApplyCodeRelocations(uint64_t GuestDelta, std::span Code, std::span Relocations, bool ForStorage); + bool ApplyCodeRelocations(uint64_t GuestDelta, std::span Code, std::span Relocations, + uint32_t RelocationOffset, bool ForStorage); }; class ContextImpl final : public FEXCore::Context::Context, public CPU::CodeBufferManager { diff --git a/FEXCore/Source/Interface/Core/CodeCache.cpp b/FEXCore/Source/Interface/Core/CodeCache.cpp index 9d23b311f..f7c47e122 100644 --- a/FEXCore/Source/Interface/Core/CodeCache.cpp +++ b/FEXCore/Source/Interface/Core/CodeCache.cpp @@ -45,6 +45,12 @@ MappedCodeCacheFile::~MappedCodeCacheFile() { if (CacheManager) { CacheManager->UnregisterMappedCodeBuffer(*this); } + +#ifndef _WIN32 + if (CodeBuffer.empty()) { + FEXCore::Allocator::munmap(CodeBuffer.data(), CodeBuffer.size_bytes()); + } +#endif } void AbstractCodeCache::RegisterMappedCodeBuffer(MappedCodeCacheFile& Code) { @@ -346,7 +352,7 @@ bool CodeCache::SaveData(Core::InternalThreadState& Thread, int fd, const Execut // Dump the host code (relocated for position-independent serialization) std::span CodeBufferData(reinterpret_cast(CodeBuffer->Ptr), reinterpret_cast(CodeBuffer->Ptr) + CTX.LatestOffset); - if (!ApplyCodeRelocations(SerializedBaseAddress, CodeBufferData, Relocations, true)) { + if (!ApplyCodeRelocations(SerializedBaseAddress, CodeBufferData, Relocations, 0, true)) { LOGMAN_THROW_A_FMT(false, "Failed to apply code relocations"); return false; } @@ -428,7 +434,7 @@ void CodeCache::Validate(const ExecutableFileSectionInfo& Section, fextl::setLatestOffset <= CodeBufferRangeRef.size()) { // Reference compilation produced fewer bytes than our cache, so validation is going to fail. @@ -490,11 +496,13 @@ void CodeCache::Validate(const ExecutableFileSectionInfo& Section, fextl::set Code, - std::span EntryRelocations, bool ForStorage) { + std::span EntryRelocations, uint32_t RelocationOffset, bool ForStorage) { CPU::Arm64Emitter Emitter(&CTX, Code.data(), Code.size_bytes()); for (size_t j = 0; j < EntryRelocations.size(); ++j) { const FEXCore::CPU::Relocation& Reloc = EntryRelocations[j]; - Emitter.SetCursorOffset(Reloc.Header.Offset); + LOGMAN_THROW_A_FMT(Reloc.Header.Offset >= RelocationOffset, "Invalid relocation offset"); + LOGMAN_THROW_A_FMT(Reloc.Header.Offset - RelocationOffset < Code.size_bytes(), "Invalid relocation offset"); + Emitter.SetCursorOffset(Reloc.Header.Offset - RelocationOffset); switch (Reloc.Header.Type) { case FEXCore::CPU::RelocationTypes::RELOC_NAMED_SYMBOL_LITERAL: { @@ -582,8 +590,20 @@ CodeCache::LoadCache(std::span CacheFile, const ExecutableFileInfo& F Cursor = reinterpret_cast(AlignUp(reinterpret_cast(Cursor), Utils::FEX_PAGE_SIZE)); auto CodeDataInFile = std::span {Cursor, header.CodeBufferSize}; - // Make code data inaccessible until finalized - Allocator::VirtualProtect(CodeDataInFile.data(), header.CodeBufferSize, Allocator::ProtectOptions::None); +#ifndef _WIN32 + // Allocate target memory for post-relocation code. This is PROT_NONE until + // the first execution, so that contents can be lazily populated in a + // frontend-provided segfault handler. + void* CodeBufferAllocation = Allocator::mmap(nullptr, header.CodeBufferSize, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (CodeBufferAllocation == MAP_FAILED) { + LogMan::Msg::EFmt("Failed to reserve target memory for code cache"); + return nullptr; + } + auto CodeBuffer = std::span {static_cast(CodeBufferAllocation), header.CodeBufferSize}; +#else + // TODO: Implement lazy mapping on Windows + auto CodeBuffer = CodeDataInFile; +#endif // Group relocations by page size_t NumPages = header.CodeBufferSize / Utils::FEX_PAGE_SIZE; @@ -600,7 +620,7 @@ CodeCache::LoadCache(std::span CacheFile, const ExecutableFileInfo& F auto Storage = FEXCore::Allocator::aligned_alloc(alignof(MappedCodeCacheFile), sizeof(MappedCodeCacheFile)); return fextl::unique_ptr( - new (Storage) MappedCodeCacheFile {this, CacheFile, CodeDataInFile, BlockListStart, header.NumBlocks, header.NumCodePages, + new (Storage) MappedCodeCacheFile {this, CacheFile, CodeDataInFile, CodeBuffer, BlockListStart, header.NumBlocks, header.NumCodePages, std::move(PageRelocationRanges), fextl::vector(NumPages), FileStartVA}); } @@ -674,7 +694,7 @@ bool CodeCache::EnableLoadedSection(Core::InternalThreadState* Thread, MappedCod } // Guest code pages - auto* Cursor = Code.CodeBuffer.data() + Code.CodeBuffer.size_bytes(); + auto* Cursor = Code.CodeBufferInFile.data() + Code.CodeBufferInFile.size_bytes(); fextl::vector Entrypoints; for (uint32_t i = 0; i < Code.NumCodePages; ++i) { uint64_t CodePage; @@ -699,7 +719,12 @@ bool CodeCache::EnableLoadedSection(Core::InternalThreadState* Thread, MappedCod } } +#ifndef _WIN32 if (!EnableLazyCodeCaching || EnableCodeCacheValidation) { +#else + // TODO: Implement lazy mapping on Windows + if (true) { +#endif auto Range = SelectCodeRangeToFinalize(Code, 0, Code.CodeBuffer.size_bytes() / Utils::FEX_PAGE_SIZE); FinalizeCodePages(Code, Range); } @@ -792,6 +817,7 @@ void CodeCache::FinalizeCodePages(MappedCodeCacheFile& Code, std::span(Allocator::VirtualAlloc(nullptr, Size, true)); + if (!Staging) { + ERROR_AND_DIE_FMT("Failed to allocate {} bytes of staging memory for code-cache finalization", Size); + } + + // Copy code from the cache file to the staging buffer + memcpy(Staging, Code.CodeBufferInFile.data() + StartOffset, Size); // Apply relocations + auto StagingSpan = std::span {Staging, Size}; for (size_t i = StartPage; i < EndPage; ++i) { auto PageRelocations = SpanPageRelocations(Code, i); - (void)ApplyCodeRelocations(Code.GuestBase, Code.CodeBuffer, PageRelocations, false); + (void)ApplyCodeRelocations(Code.GuestBase, StagingSpan, PageRelocations, static_cast(StartOffset), false); Code.LoadedPages[i] = true; } - ARMEmitter::Emitter::ClearICache(CodeRange.data(), CodeRange.size_bytes()); - if (!Allocator::VirtualProtect(CodeRange.data(), CodeRange.size_bytes(), - Allocator::ProtectOptions::Read | Allocator::ProtectOptions::Write | Allocator::ProtectOptions::Exec)) { - ERROR_AND_DIE_FMT("VirtualProtect failed"); + // Atomically make the finalized code data visible by remapping the staging + // buffer onto the requested CodeBuffer window. MREMAP_DONTUNMAP is used to + // leave the old VA range reserved so that we can cleanly deallocate it + // through Allocator. + void* RemapResult = ::mremap(Staging, Size, Size, MREMAP_FIXED | MREMAP_MAYMOVE | MREMAP_DONTUNMAP, CodeRange.data()); + if (RemapResult == MAP_FAILED) { + ERROR_AND_DIE_FMT("{}: mremap failed: {}", __FUNCTION__, errno); } + Allocator::VirtualFree(Staging, Size); + + // Release resident file pages that will no longer be needed. The VA range is left allocated to allow cleanup with a single VirtualFree. + Allocator::VirtualDontNeed(Code.CodeBufferInFile.data() + StartOffset, Size); +#else + // TODO: Implement lazy mapping on Windows + for (size_t i = StartPage; i < EndPage; ++i) { + auto PageRelocations = SpanPageRelocations(Code, i); + (void)ApplyCodeRelocations(Code.GuestBase, Code.CodeBuffer, PageRelocations, 0, false); + Code.LoadedPages[i] = true; + } +#endif + + ARMEmitter::Emitter::ClearICache(CodeRange.data(), Size); } } // namespace FEXCore::Context diff --git a/FEXCore/include/FEXCore/Core/CodeCache.h b/FEXCore/include/FEXCore/Core/CodeCache.h index eaa20a4ac..bd99d4bd2 100644 --- a/FEXCore/include/FEXCore/Core/CodeCache.h +++ b/FEXCore/include/FEXCore/Core/CodeCache.h @@ -194,11 +194,12 @@ struct MappedCodeCacheFile { // If not nullptr, the MappedCodeCacheFile will be unregistered from this on destruction AbstractCodeCache* CacheManager; - std::span MappedFile; // Mapped data of the whole cache file - std::span CodeBuffer; // Subspan of cached ARM64 data within MappedFile - std::byte* BlockListInFile; // Pointer to BlockListEntry data within MappedFile - uint32_t NumBlocks; // Number of BlockListEntry objects - uint32_t NumCodePages; // Number of code page entrypoint mappings + std::span MappedFile; // Mapped data of the whole cache file + std::span CodeBufferInFile; // Subspan of cached ARM64 data within MappedFile (pre-relocation) + std::span CodeBuffer; // Cached ARM64 data used for execution (post-relocation; owned by MappedCodeCacheFile) + std::byte* BlockListInFile; // Pointer to BlockListEntry data within MappedFile + uint32_t NumBlocks; // Number of BlockListEntry objects + uint32_t NumCodePages; // Number of code page entrypoint mappings struct PageRelocationRange { uint32_t Offset; // In bytes from start of file diff --git a/Source/Tools/LinuxEmulation/LinuxSyscalls/SyscallsSMCTracking.cpp b/Source/Tools/LinuxEmulation/LinuxSyscalls/SyscallsSMCTracking.cpp index 69fe6a163..12ad90772 100644 --- a/Source/Tools/LinuxEmulation/LinuxSyscalls/SyscallsSMCTracking.cpp +++ b/Source/Tools/LinuxEmulation/LinuxSyscalls/SyscallsSMCTracking.cpp @@ -280,7 +280,7 @@ LoadCodeCache(FEXCore::Core::InternalThreadState& Thread, VMATracking::VMATracki } auto CacheFileSize = static_cast(buf.st_size); - auto MappedCache = (std::byte*)FEXCore::Allocator::mmap(nullptr, CacheFileSize, PROT_READ | PROT_WRITE, MAP_PRIVATE, CacheFD, 0); + auto MappedCache = (std::byte*)FEXCore::Allocator::mmap(nullptr, CacheFileSize, PROT_READ, MAP_PRIVATE, CacheFD, 0); close(CacheFD); if (!MappedCache || MappedCache == MAP_FAILED) { LogMan::Msg::EFmt("Failed to map code cache into memory");