Merge pull request #5547 from neobrain/feature_fexofflinecompiler_process_all

FEXOfflineCompiler: Add "process-all" verb

Source/Tools/FEXOfflineCompiler/Main.cpp

@@ -92,6 +92,11 @@ struct std::hash<FEXCore::ExecutableFileInfo> {
   }
 };
 
+// Windows requires O_BINARY, whereas on Linux it's implicit
+#ifndef O_BINARY
+#define O_BINARY 0
+#endif
+
 // Placeholder data to ensure the compile thread doesn't de-reference nullptr data
 static FEXCore::Core::CPUState::gdt_segment gdt[32] {};
 
@@ -319,6 +324,206 @@ static int GenerateCache(int argc, const char** argv) {
   return GeneratedCache ? 0 : 1;
 }
 
+/**
+ * Writes aggregated code map data into a single code map file that is ready to be used for cache generation
+ */
+static void WriteNewCodeMap(const FEXCore::ExecutableFileInfo& File, const std::string& OutputName, const fextl::set<uintptr_t>& Blocks,
+                            bool IsExecutable, const std::set<FEXCore::ExecutableFileInfo>& Dependencies) {
+  fmt::print("Writing {} blocks to {}\n", Blocks.size(), OutputName);
+
+  struct CodeMapOpener : FEXCore::CodeMapOpener {
+    CodeMapOpener(const std::string& Filename) {
+      FD = open(Filename.c_str(), O_CREAT | O_TRUNC | O_WRONLY | O_BINARY, 0644);
+    }
+
+    int OpenCodeMapFile() override {
+      return FD;
+    }
+
+    int FD;
+  };
+
+  CodeMapOpener CodeMapOpener(OutputName);
+  FEXCore::CodeMapWriter OutputCodeMap(CodeMapOpener, true);
+  if (IsExecutable) {
+    // List the main executable and all used libraries
+    OutputCodeMap.AppendSetMainExecutable(File);
+
+    for (auto& Dependency : Dependencies) {
+      OutputCodeMap.AppendLibraryLoad(Dependency);
+    }
+  } else {
+    // List only the library itself
+    OutputCodeMap.AppendLibraryLoad(File);
+  }
+
+  for (auto& Block : Blocks) {
+    OutputCodeMap.AppendBlock(FEXCore::ExecutableFileSectionInfo {File, 0}, Block);
+  }
+}
+
+struct ParsedContentsAndDependencies {
+  fextl::string Filename;
+  fextl::set<uint64_t> Blocks;
+  bool IsExecutable = false;
+  std::set<FEXCore::CodeMapFileId> Dependencies;
+};
+
+/**
+ * Discovers any pending code maps, parses their contents into a runtime data structure, and deletes them
+ */
+static std::map<FEXCore::CodeMapFileId, ParsedContentsAndDependencies> ImportPendingCodeMaps(const std::string& NewCodeMapDirectory) {
+  // TODO: Handle nomb code maps
+  std::map<FEXCore::CodeMapFileId, ParsedContentsAndDependencies> Result;
+  for (auto& Entry : std::filesystem::directory_iterator(NewCodeMapDirectory)) {
+    if (!Entry.is_regular_file()) {
+      continue;
+    }
+    const auto Name = Entry.path().filename().string();
+    if (!Name.ends_with(".bin")) {
+      continue;
+    }
+
+    if (std::filesystem::file_size(Entry.path()) == 0) {
+      fmt::println("Found zero-size code map {}, deleting", Name);
+      std::filesystem::remove(Entry.path());
+      continue;
+    }
+
+    fmt::print("Importing new code map {}\n", Name);
+    std::ifstream Incoming(Entry.path(), std::ios_base::binary);
+    std::set<FEXCore::CodeMapFileId> Dependencies;
+    std::optional<FEXCore::CodeMapFileId> ExecutableFileId;
+    for (auto& [FileId, Contents] : FEXCore::CodeMap::ParseCodeMap(Incoming)) {
+      auto& [Filename, Blocks, IsExecutable, _] =
+        Result.emplace(std::piecewise_construct, std::forward_as_tuple(FileId), std::tuple {}).first->second;
+      Filename = std::move(Contents.Filename);
+      Blocks.merge(std::move(Contents.Blocks));
+      IsExecutable = Contents.IsExecutable;
+      if (IsExecutable) {
+        LOGMAN_THROW_A_FMT(!ExecutableFileId, "Expected a unique executable identifiers per code map");
+        ExecutableFileId = FileId;
+      } else {
+        Dependencies.insert(FileId);
+      }
+    }
+
+    // Every imported code map should have had exactly one executable marker
+    LOGMAN_THROW_A_FMT(ExecutableFileId, "Could not find an executable identifer in the code map");
+    Result.at(*ExecutableFileId).Dependencies = std::move(Dependencies);
+
+    // Delete imported code map
+    Incoming.close();
+    std::filesystem::remove(Entry.path());
+  }
+
+  return Result;
+}
+
+/**
+ * Checks and processes new code maps generated by FEX
+ *
+ * Processed code maps are merged into the reference ("ready") code maps
+ */
+static void AggregateCodeMaps(const std::string& NewCodeMapDirectory, const std::string& ReadyCodeMapDirectory) {
+  auto IncomingCodeMap = ImportPendingCodeMaps(NewCodeMapDirectory);
+
+  for (auto& [FileId, Contents] : IncomingCodeMap) {
+    // For each referenced binary, add the newly referenced offsets to that binary's reference code map
+    const FEXCore::ExecutableFileInfo File {nullptr, FileId, Contents.Filename};
+    const auto BinaryName = std::string {FEXCore::CodeMap::GetBaseFilename(File, false)};
+    auto OutputName = fmt::format("{}/{}", ReadyCodeMapDirectory, BinaryName);
+
+    if (auto ReferenceCodeMap = std::ifstream(OutputName, std::ios_base::binary)) {
+      auto PreviousBlocks = FEXCore::CodeMap::ParseCodeMap(ReferenceCodeMap).at(File.FileId).Blocks;
+      auto NumPreviousBlocks = PreviousBlocks.size();
+      Contents.Blocks.merge(std::move(PreviousBlocks));
+      if (Contents.Blocks.size() == NumPreviousBlocks) {
+        // No new blocks => skip updating
+        continue;
+      } else {
+        fmt::println("  Found {} new blocks ({} total) in code map {} for {}", Contents.Blocks.size() - NumPreviousBlocks,
+                     Contents.Blocks.size(), BinaryName, File.Filename);
+      }
+    }
+
+    // Update code map
+    std::set<FEXCore::ExecutableFileInfo> Dependencies;
+    for (auto& Dependency : Contents.Dependencies) {
+      Dependencies.emplace(nullptr, Dependency, IncomingCodeMap.at(Dependency).Filename);
+    }
+    WriteNewCodeMap(File, OutputName, Contents.Blocks, Contents.IsExecutable, Dependencies);
+  }
+}
+
+static int ProcessAll() {
+  const auto CacheDirectory = FEX::Config::GetCacheDirectory();
+  const std::string NewCodeMapDirectory = fmt::format("{}codemap/new", CacheDirectory);
+  const std::string ReadyCodeMapDirectory = fmt::format("{}codemap/ready", CacheDirectory);
+
+  // Import new code maps and aggregate them into ready code maps
+  std::filesystem::create_directories(ReadyCodeMapDirectory);
+  AggregateCodeMaps(NewCodeMapDirectory, ReadyCodeMapDirectory);
+
+  // Generate caches
+  fextl::string OutDir = CacheDirectory + "cache/";
+  std::filesystem::create_directories(OutDir);
+
+  // Iterate over all executables (.exe).
+  // These determine the emulator configuration to use when compiling dependencies.
+  for (auto& Entry : std::filesystem::directory_iterator(ReadyCodeMapDirectory)) {
+    std::ifstream CodeMap(Entry.path(), std::ios_base::binary);
+    auto Parsed = FEXCore::CodeMap::ParseCodeMap(CodeMap);
+    auto ExecutableIt = std::ranges::find_if(Parsed, [](const auto& Entry) { return Entry.second.IsExecutable; });
+    if (ExecutableIt == Parsed.end()) {
+      // Skip libraries; they're only processed as dependencies of a main executable
+      continue;
+    }
+
+    fmt::println("\nChecking caches for executable {}", ExecutableIt->second.Filename);
+
+    // TODO: Compute the cache config id from the active FEX configuration
+    uint64_t CodeCacheConfigId = 0;
+
+    auto GetCacheFilename = [&](const FEXCore::ExecutableFileInfo& File) {
+      return fmt::format("{}{}-{:016x}", OutDir, FEXCore::CodeMap::GetBaseFilename(File, false), CodeCacheConfigId);
+    };
+
+    // Check the main binary and all of its dependencies
+    for (auto& [FileId, Contents] : Parsed) {
+      const FEXCore::ExecutableFileInfo File {nullptr, FileId, Contents.Filename};
+      std::error_code ec;
+      const auto BinaryName = FEXCore::CodeMap::GetBaseFilename(File, false);
+      const auto MergedCodeMapFilename = fmt::format("{}/{}", ReadyCodeMapDirectory, BinaryName);
+      const auto LastCodeMapUpdate = std::filesystem::last_write_time(MergedCodeMapFilename, ec);
+      if (ec) {
+        // No reference code map exists for this dependency yet, so there's nothing to generate a cache from
+        continue;
+      }
+
+      if (std::filesystem::last_write_time(GetCacheFilename(File), ec) > LastCodeMapUpdate && !ec) {
+        fmt::println("  Cache up to date: {}", BinaryName);
+        continue;
+      }
+
+      // TODO: Also check for matching FEX version from cache header
+
+      fmt::println("  {} cache: {}", ec ? "Generating" : "Updating outdated", BinaryName);
+
+      // Defer to GenerateCache
+      const auto FileIdArg = fmt::format("{:016x}", FileId);
+      std::vector<const char*> GenerateArgs {
+        "generate", "--fileid", FileIdArg.c_str(), "--outdir", OutDir.c_str(), MergedCodeMapFilename.c_str(),
+      };
+      if (GenerateCache(GenerateArgs.size(), GenerateArgs.data()) != 0) {
+        fmt::println("ERROR: Cache generation failed for {}", BinaryName);
+      }
+    }
+  }
+
+  return 0;
+}
+
 int main(int argc, char** argv) {
   LogMan::Throw::InstallHandler(AssertHandler);
   LogMan::Msg::InstallHandler(MsgHandler);
@@ -329,10 +534,13 @@ int main(int argc, char** argv) {
 
   if (argc >= 2 && argv[1] == std::string_view {"generate"}) {
     return GenerateCache(argc - 1, Args.data());
+  } else if (argc >= 2 && argv[1] == std::string_view {"process-all"}) {
+    return ProcessAll();
   } else {
     fmt::print("Usage: {} <command>\n\n", basename(argv[0]));
     fmt::print("Commands:\n");
     fmt::print("  generate\tTrigger cache generation from combined code map\n");
+    fmt::print("  process-all\tProcess all new code maps and update all caches\n");
     return EXIT_FAILURE;
   }
 }