diff --git a/FEXCore/Source/Interface/Core/CodeCache.cpp b/FEXCore/Source/Interface/Core/CodeCache.cpp index 9ca86c3ac..3866fa552 100644 --- a/FEXCore/Source/Interface/Core/CodeCache.cpp +++ b/FEXCore/Source/Interface/Core/CodeCache.cpp @@ -111,16 +111,22 @@ fextl::map CodeMap::ParseCodeMap(std::if break; } Ret[Info.ExternalFileId].Filename = std::move(Filename); - } else if (Entry.FileId == SetExecutableFileId {}.Marker.FileId && Entry.BlockOffset == SetExecutableFileId {}.Marker.BlockOffset) { + } else if ((Entry.FileId == SetExecutableFileId::Marker32.FileId && Entry.BlockOffset == SetExecutableFileId::Marker32.BlockOffset) || + (Entry.FileId == SetExecutableFileId::Marker64.FileId && Entry.BlockOffset == SetExecutableFileId::Marker64.BlockOffset)) { CodeMapFileId ExecutableFileId; File.read(reinterpret_cast(&ExecutableFileId), sizeof(ExecutableFileId)); if (!File) { break; } - Ret[ExecutableFileId].IsExecutable = true; + Ret[ExecutableFileId].ExecutableBitness = + (Entry.FileId == SetExecutableFileId::Marker32.FileId && Entry.BlockOffset == SetExecutableFileId::Marker32.BlockOffset) ? 32 : 64; } else { if (!Ret.contains(Entry.FileId)) { - LogMan::Msg::EFmt("Code map referenced unknown file id {:016x}", Entry.FileId); + if (Entry.FileId == 0xffff'ffff'ffff'ffff) { + ERROR_AND_DIE_FMT("Malformed code map"); + } else { + LogMan::Msg::EFmt("Code map referenced unknown file id {:016x}", Entry.FileId); + } } else { Ret[Entry.FileId].Blocks.insert(Entry.BlockOffset); } @@ -226,8 +232,8 @@ void CodeMapWriter::AppendLibraryLoad(const FEXCore::ExecutableFileInfo& FileInf AppendData(std::as_bytes(std::span {Data, TotalSize})); } -void CodeMapWriter::AppendSetMainExecutable(const FEXCore::ExecutableFileInfo& FileInfo) { - CodeMap::SetExecutableFileId Data {.ExecutableFileId = FileInfo.FileId}; +void CodeMapWriter::AppendSetMainExecutable(const FEXCore::ExecutableFileInfo& FileInfo, bool Is64Bit) { + CodeMap::SetExecutableFileId Data {Is64Bit ? CodeMap::SetExecutableFileId::Marker64 : CodeMap::SetExecutableFileId::Marker32, FileInfo.FileId}; AppendData(std::span {reinterpret_cast(&Data), sizeof(Data)}); } diff --git a/FEXCore/include/FEXCore/Core/CodeCache.h b/FEXCore/include/FEXCore/Core/CodeCache.h index bd99d4bd2..09b1a1245 100644 --- a/FEXCore/include/FEXCore/Core/CodeCache.h +++ b/FEXCore/include/FEXCore/Core/CodeCache.h @@ -103,14 +103,17 @@ struct CodeMap { static constexpr Entry LoadExternalLibrary = {0xffff'ffff'ffff'ffff, 0xffff'ffff}; struct FEX_PACKED SetExecutableFileId { - Entry Marker = {0xffff'ffff'ffff'ffff, 0xffff'fffe}; + static constexpr Entry Marker32 = {0xffff'ffff'ffff'ffff, 0xffff'ffee}; + static constexpr Entry Marker64 = {0xffff'ffff'ffff'ffff, 0xffff'ffef}; + Entry Marker; CodeMapFileId ExecutableFileId; }; struct ParsedContents { fextl::string Filename; fextl::set Blocks; - bool IsExecutable = false; + // 32/64 for executables, nullopt for non-executables (libraries) + std::optional ExecutableBitness; }; // Follows scheme fileid[-nomb] @@ -152,7 +155,7 @@ public: void AppendBlock(const FEXCore::ExecutableFileSectionInfo&, uint64_t Entry); void AppendLibraryLoad(const FEXCore::ExecutableFileInfo&); - void AppendSetMainExecutable(const FEXCore::ExecutableFileInfo&); + void AppendSetMainExecutable(const FEXCore::ExecutableFileInfo&, bool Is64Bit); // Thread-safely commit any pending data to disk void Flush(size_t Offset); diff --git a/Source/Tools/FEXOfflineCompiler/CMakeLists.txt b/Source/Tools/FEXOfflineCompiler/CMakeLists.txt index 33f337043..5ce74795d 100644 --- a/Source/Tools/FEXOfflineCompiler/CMakeLists.txt +++ b/Source/Tools/FEXOfflineCompiler/CMakeLists.txt @@ -9,6 +9,11 @@ target_link_libraries(FEXOfflineCompiler PRIVATE fmt::fmt) if (MINGW) + if (ARCHITECTURE_arm64ec) + set_target_properties(FEXOfflineCompiler PROPERTIES OUTPUT_NAME "FEXOfflineCompiler64") + else() + set_target_properties(FEXOfflineCompiler PROPERTIES OUTPUT_NAME "FEXOfflineCompiler32") + endif() patch_library_wine(FEXOfflineCompiler) target_include_directories(FEXOfflineCompiler PRIVATE "${CMAKE_SOURCE_DIR}/Source/Windows/include/" diff --git a/Source/Tools/FEXOfflineCompiler/Main.cpp b/Source/Tools/FEXOfflineCompiler/Main.cpp index a8681c877..e969af057 100644 --- a/Source/Tools/FEXOfflineCompiler/Main.cpp +++ b/Source/Tools/FEXOfflineCompiler/Main.cpp @@ -51,6 +51,14 @@ static std::unique_ptr OvercommitTracker; #endif static FEXCore::Core::InternalThreadState* Thread = nullptr; +#ifdef _WIN32 +#ifdef _M_ARM64EC +const bool Is64BitCompiler = true; +#else +const bool Is64BitCompiler = false; +#endif +#endif + #ifdef _WIN32 class AOTSyscallHandler : public FEXCore::HLE::SyscallHandler { #else @@ -396,10 +404,8 @@ static std::optional GenerateSingleCache(FEXCore::ExecutableFileInf return std::nullopt; } const bool Is64Bit = Loader.Is64BitMode(); -#elif defined(_M_ARM64EC) - const bool Is64Bit = true; #else - const bool Is64Bit = false; + const bool Is64Bit = Is64BitCompiler; #endif FEXCore::Config::Set(FEXCore::Config::CONFIG_IS64BIT_MODE, Is64Bit ? "1" : "0"); @@ -572,7 +578,7 @@ static int GenerateCache(int argc, const char** argv) { } for (auto& [FileId, Contents] : Parsed) { - if (!ExplicitFileId && (Contents.IsExecutable || Parsed.size() == 1)) { + if (!ExplicitFileId && (Contents.ExecutableBitness || Parsed.size() == 1)) { ProgramName.FileId = FileId; ProgramName.Filename = Contents.Filename; } @@ -621,7 +627,7 @@ static int GenerateCache(int argc, const char** argv) { * Writes aggregated code map data into a single code map file that is ready to be used for cache generation */ static void WriteNewCodeMap(const FEXCore::ExecutableFileInfo& File, const std::string& OutputName, const fextl::set& Blocks, - bool IsExecutable, const std::set& Dependencies) { + std::optional ExecutableBitness, const std::set& Dependencies) { fmt::print("Writing {} blocks to {}\n", Blocks.size(), OutputName); struct CodeMapOpener : FEXCore::CodeMapOpener { @@ -638,9 +644,9 @@ static void WriteNewCodeMap(const FEXCore::ExecutableFileInfo& File, const std:: CodeMapOpener CodeMapOpener(OutputName); FEXCore::CodeMapWriter OutputCodeMap(CodeMapOpener, true); - if (IsExecutable) { + if (ExecutableBitness) { // List the main executable and all used libraries - OutputCodeMap.AppendSetMainExecutable(File); + OutputCodeMap.AppendSetMainExecutable(File, ExecutableBitness == 64); for (auto& Dependency : Dependencies) { OutputCodeMap.AppendLibraryLoad(Dependency); @@ -658,7 +664,7 @@ static void WriteNewCodeMap(const FEXCore::ExecutableFileInfo& File, const std:: struct ParsedContentsAndDependencies { fextl::string Filename; fextl::set Blocks; - bool IsExecutable = false; + std::optional ExecutableBitness; std::set Dependencies; }; @@ -688,13 +694,13 @@ static std::map ImportPen std::set Dependencies; std::optional ExecutableFileId; for (auto& [FileId, Contents] : FEXCore::CodeMap::ParseCodeMap(Incoming)) { - auto& [Filename, Blocks, IsExecutable, _] = + auto& [Filename, Blocks, ExecutableBitness, _] = Result.emplace(std::piecewise_construct, std::forward_as_tuple(FileId), std::tuple {}).first->second; Filename = std::move(Contents.Filename); Blocks.merge(std::move(Contents.Blocks)); - IsExecutable = Contents.IsExecutable; - if (IsExecutable) { - LOGMAN_THROW_A_FMT(!ExecutableFileId, "Expected a unique executable identifiers per code map"); + ExecutableBitness = Contents.ExecutableBitness; + if (ExecutableBitness) { + LOGMAN_THROW_A_FMT(!ExecutableFileId, "Expected a unique executable identifier per code map"); ExecutableFileId = FileId; } else { Dependencies.insert(FileId); @@ -745,7 +751,7 @@ static void AggregateCodeMaps(const std::string& NewCodeMapDirectory, const std: for (auto& Dependency : Contents.Dependencies) { Dependencies.emplace(nullptr, Dependency, IncomingCodeMap.at(Dependency).Filename); } - WriteNewCodeMap(File, OutputName, Contents.Blocks, Contents.IsExecutable, Dependencies); + WriteNewCodeMap(File, OutputName, Contents.Blocks, Contents.ExecutableBitness, Dependencies); } } @@ -767,12 +773,22 @@ static int ProcessAll() { for (auto& Entry : std::filesystem::directory_iterator(ReadyCodeMapDirectory)) { std::ifstream CodeMap(Entry.path(), std::ios_base::binary); auto Parsed = FEXCore::CodeMap::ParseCodeMap(CodeMap); - auto ExecutableIt = std::ranges::find_if(Parsed, [](const auto& Entry) { return Entry.second.IsExecutable; }); + auto ExecutableIt = std::ranges::find_if(Parsed, [](const auto& Entry) { return Entry.second.ExecutableBitness.has_value(); }); if (ExecutableIt == Parsed.end()) { // Skip libraries; they're only processed as dependencies of a main executable continue; } +#ifdef _WIN32 + // For WoA, spawn a subprocess for each cache generation run. + // This ensures robustness and allows for switching FEXOfflineCompiler between WoW64 and ARM64EC. + char SelfPathRaw[PATH_MAX]; + GetModuleFileNameA(nullptr, SelfPathRaw, sizeof(SelfPathRaw)); + std::string SelfPath = SelfPathRaw; + auto NewExecName = fmt::format("FEXOfflineCompiler{}.exe", ExecutableIt->second.ExecutableBitness.value()); + SelfPath.replace(SelfPath.size() - NewExecName.size(), NewExecName.size(), NewExecName); +#endif + fmt::println("\nChecking caches for executable {}", ExecutableIt->second.Filename); // TODO: Compute the cache config id from the active FEX configuration @@ -808,9 +824,18 @@ static int ProcessAll() { std::vector GenerateArgs { "generate", "--fileid", FileIdArg.c_str(), "--outdir", OutDir.c_str(), MergedCodeMapFilename.c_str(), }; +#ifndef _WIN32 if (GenerateCache(GenerateArgs.size(), GenerateArgs.data()) != 0) { fmt::println("ERROR: Cache generation failed for {}", BinaryName); } +#else + GenerateArgs.insert(GenerateArgs.begin(), SelfPath.c_str()); + GenerateArgs.push_back(nullptr); + auto Status = _spawnv(_P_WAIT, SelfPath.c_str(), GenerateArgs.data()); + if (Status) { + fmt::println("ERROR: Cache generation failed for {}", BinaryName); + } +#endif } } diff --git a/Source/Tools/FEXServer/ProcessPipe.cpp b/Source/Tools/FEXServer/ProcessPipe.cpp index 16289ed17..f0762ea37 100644 --- a/Source/Tools/FEXServer/ProcessPipe.cpp +++ b/Source/Tools/FEXServer/ProcessPipe.cpp @@ -307,7 +307,7 @@ void SendFDSuccessPacket(fasio::tcp_socket& Socket, int FD) { // Discovers any pending code maps, parses their contents into a runtime data structure, and deletes them static std::map> -ImportPendingCodeMaps(const FEXCore::ExecutableFileInfo& MainFileId, bool HasMultiblock) { +ImportPendingCodeMaps(const FEXCore::ExecutableFileInfo& MainFileId, bool HasMultiblock, std::optional& ExecutableBitness) { // Detect code maps by checking file name suffixes by counting up an index. // Code maps that are ready for reading must be non-empty and flock(FLOCK_EX) must succeed: // - If empty, we tried generating the cache before the client could even lock it @@ -334,7 +334,7 @@ ImportPendingCodeMaps(const FEXCore::ExecutableFileInfo& MainFileId, bool HasMul fmt::print("Found code map {}, queuing for merge\n", CodeMap); } close(FD); - CodeMaps.push_back(CodeMap); + CodeMaps.push_back(std::move(CodeMap)); } // Update merged code map @@ -348,6 +348,11 @@ ImportPendingCodeMaps(const FEXCore::ExecutableFileInfo& MainFileId, bool HasMul for (auto& [FileId, Contents] : NewBlocks) { ImportedCodeMaps.emplace(std::piecewise_construct, std::forward_as_tuple(nullptr, FileId, std::move(Contents.Filename)), std::forward_as_tuple(std::move(Contents.Blocks))); + if (FileId == MainFileId.FileId) { + LOGMAN_THROW_A_FMT(Contents.ExecutableBitness && (!ExecutableBitness.has_value() || ExecutableBitness == Contents.ExecutableBitness), + "Inconsistent executable bitness"); + ExecutableBitness = Contents.ExecutableBitness; + } } } } @@ -365,7 +370,7 @@ ImportPendingCodeMaps(const FEXCore::ExecutableFileInfo& MainFileId, bool HasMul * Writes aggregated code map data into a single code map file that is ready to be used for cache generation */ static void WriteNewCodeMap(const FEXCore::ExecutableFileInfo& File, const std::string& OutputName, const fextl::set& Blocks, - bool IsMainFile, const auto& Dependencies) { + std::optional ExecutableBitness, const auto& Dependencies) { fmt::print("Writing {} blocks to {}\n", Blocks.size(), OutputName); struct CodeMapOpener : FEXCore::CodeMapOpener { @@ -382,9 +387,9 @@ static void WriteNewCodeMap(const FEXCore::ExecutableFileInfo& File, const std:: CodeMapOpener CodeMapOpener(OutputName); FEXCore::CodeMapWriter OutputCodeMap(CodeMapOpener, true); - if (IsMainFile) { + if (ExecutableBitness) { // List the main executable and all used libraries - OutputCodeMap.AppendSetMainExecutable(File); + OutputCodeMap.AppendSetMainExecutable(File, ExecutableBitness == 64); for (auto& [Dependency, _] : Dependencies) { OutputCodeMap.AppendLibraryLoad(Dependency); @@ -426,11 +431,13 @@ static std::map AggregateCodeMap } // Accumulate information from new code maps - auto IncomingCodeMap = ImportPendingCodeMaps(MainFileId, HasMultiblock); + std::optional ExecutableBitness; + auto IncomingCodeMap = ImportPendingCodeMaps(MainFileId, HasMultiblock, ExecutableBitness); for (auto& [File, _] : IncomingCodeMap) { Result.emplace(std::piecewise_construct, std::forward_as_tuple(nullptr, File.FileId, File.Filename), std::forward_as_tuple(NeedsCacheRefresh::No)); } + LOGMAN_THROW_A_FMT(ExecutableBitness, "New code maps did not contain executable marker"); // For each referenced library, add referenced offsets to that library's reference code map for (auto& [File, Blocks] : IncomingCodeMap) { @@ -452,7 +459,7 @@ static std::map AggregateCodeMap // Update code map and queue for cache generation std::map Empty; - WriteNewCodeMap(File, OutputName, Blocks, true, File.FileId == MainFileId.FileId ? Result : Empty); + WriteNewCodeMap(File, OutputName, Blocks, ExecutableBitness, File.FileId == MainFileId.FileId ? Result : Empty); Result.at(File) = NeedsCacheRefresh::Yes; } diff --git a/Source/Windows/Common/ImageTracker.cpp b/Source/Windows/Common/ImageTracker.cpp index 8bff21826..c459c446f 100644 --- a/Source/Windows/Common/ImageTracker.cpp +++ b/Source/Windows/Common/ImageTracker.cpp @@ -164,7 +164,11 @@ FEXCore::ExecutableFileSectionInfo ImageTracker::HandleImageMap(std::string_view ActiveCodeMapPath = fmt::format("{}{}.{}.bin", CodeMapDir, ID, Time.QuadPart); auto Writer = fextl::make_unique(*this, false); - Writer->AppendSetMainExecutable(ImageInfo->Info); +#ifdef _M_ARM64EC + Writer->AppendSetMainExecutable(ImageInfo->Info, true); +#else + Writer->AppendSetMainExecutable(ImageInfo->Info, false); +#endif CTX.SetCodeMapWriter(std::move(Writer)); } } @@ -223,8 +227,9 @@ int ImageTracker::OpenCodeMapFile() { void ImageTracker::LoadAOTImages(MappedImageInfo& ImageInfo) { // Don't attempt cache loading for FEXOfflineCompiler itself - static bool IsFOC = ImageInfo.Info.Filename.ends_with("fexofflinecompiler.exe"); - if (IsFOC) { + static bool IsFOC32 = ImageInfo.Info.Filename.ends_with("fexofflinecompiler32.exe"); + static bool IsFOC64 = ImageInfo.Info.Filename.ends_with("fexofflinecompiler64.exe"); + if (IsFOC32 || IsFOC64) { return; }