From 3e0e60161632dfdb1eae1341117a57532d9e1e96 Mon Sep 17 00:00:00 2001 From: Ryan Houdek Date: Mon, 4 Jul 2022 20:51:13 -0700 Subject: [PATCH 1/4] FileFormatCheck: Add support for EroFS --- Source/Common/FileFormatCheck.cpp | 50 +++++++++++++++++++++++++++++++ Source/Common/FileFormatCheck.h | 1 + 2 files changed, 51 insertions(+) diff --git a/Source/Common/FileFormatCheck.cpp b/Source/Common/FileFormatCheck.cpp index 0b4f53919..5703f2856 100644 --- a/Source/Common/FileFormatCheck.cpp +++ b/Source/Common/FileFormatCheck.cpp @@ -63,4 +63,54 @@ namespace FEX::FormatCheck { } return false; } + + bool IsEroFS(std::string const &Filename) { + // v1 of EroFS has a 128byte header + // This lives within a fixed offset inside of the first superblock of the file + // Each superblock is 4096bytes + // + // We only care about the uint32_t at the start of this offset which is the cookie + struct EroFSHeader { + uint32_t Magic; + // Additional data after this if necessary in the future. + }; + + constexpr size_t SUPERBLOCK_SIZE = 4096; + constexpr size_t HEADER_OFFSET = 1024; + constexpr uint32_t COOKIE_MAGIC_V1 = 0xE0F5E1E2; + + EroFSHeader Header{}; + std::fstream File(Filename, std::ios::in); + + if (!File.is_open()) { + return false; + } + + if (!File.seekg(0, std::fstream::end)) { + return false; + } + + auto FileSize = File.tellg(); + if (File.fail()) { + return false; + } + + if (FileSize <= 0) { + return false; + } + + if (FileSize < SUPERBLOCK_SIZE) { + return false; + } + + if (!File.seekg(HEADER_OFFSET, std::fstream::beg)) { + return false; + } + + if (!File.read(reinterpret_cast(&Header), sizeof(EroFSHeader))) { + return false; + } + + return Header.Magic == COOKIE_MAGIC_V1; + } } diff --git a/Source/Common/FileFormatCheck.h b/Source/Common/FileFormatCheck.h index 25de61315..8885f9226 100644 --- a/Source/Common/FileFormatCheck.h +++ b/Source/Common/FileFormatCheck.h @@ -4,4 +4,5 @@ namespace FEX::FormatCheck { bool IsSquashFS(std::string const &Filename); + bool IsEroFS(std::string const &Filename); } From 0de2558ef7c706c19468f4e880ebebc691520490 Mon Sep 17 00:00:00 2001 From: Ryan Houdek Date: Mon, 4 Jul 2022 20:51:34 -0700 Subject: [PATCH 2/4] FEXConfig: Add support for erofs --- Source/Tools/FEXConfig/Main.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Source/Tools/FEXConfig/Main.cpp b/Source/Tools/FEXConfig/Main.cpp index 2b06e46ea..b002bb4c8 100644 --- a/Source/Tools/FEXConfig/Main.cpp +++ b/Source/Tools/FEXConfig/Main.cpp @@ -117,6 +117,10 @@ namespace { FEX::FormatCheck::IsSquashFS(it.path().string())) { NamedRootFS.emplace_back(it.path().filename()); } + else if (it.path().extension() == ".ero" && + FEX::FormatCheck::IsEroFS(it.path().string())) { + NamedRootFS.emplace_back(it.path().filename()); + } } } } From 9ad1d5548ddb73056533e45b725c430fb31873e9 Mon Sep 17 00:00:00 2001 From: Ryan Houdek Date: Mon, 4 Jul 2022 20:51:59 -0700 Subject: [PATCH 3/4] FEXServer: Add support for EroFS --- Source/Tools/FEXServer/SquashFS.cpp | 30 ++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/Source/Tools/FEXServer/SquashFS.cpp b/Source/Tools/FEXServer/SquashFS.cpp index 61901f7f1..306cc64b4 100644 --- a/Source/Tools/FEXServer/SquashFS.cpp +++ b/Source/Tools/FEXServer/SquashFS.cpp @@ -97,7 +97,7 @@ namespace SquashFS { return true; } - bool MountSquashFSPath(std::string SquashFS) { + bool MountRootFSImagePath(std::string SquashFS, bool EroFS) { pid_t ParentTID = ::getpid(); MountFolder = fmt::format("{}/.FEXMount{}-XXXXXX", std::filesystem::temp_directory_path().string(), ParentTID); char *MountFolderStr = MountFolder.data(); @@ -124,16 +124,16 @@ namespace SquashFS { // Child close(fds[0]); // Close read side const char *argv[4]; - argv[0] = "squashfuse"; + argv[0] = EroFS ? "erofsfuse" : "squashfuse"; argv[1] = SquashFS.c_str(); argv[2] = MountFolder.c_str(); argv[3] = nullptr; - // Try and execute squashfuse to mount our rootfs + // Try and execute {erofsfuse, squashfuse} to mount our rootfs if (execvpe(argv[0], (char * const*)argv, environ) == -1) { // Give a hopefully helpful error message for users LogMan::Msg::EFmt("[FEXServer] '{}' Couldn't execute for some reason: {} {}\n", argv[0], errno, strerror(errno)); - LogMan::Msg::EFmt("[FEXServer] To mount squashfs rootfs files you need squashfuse installed\n"); + LogMan::Msg::EFmt("[FEXServer] To mount squashfs rootfs files you need {} installed\n", argv[0]); LogMan::Msg::EFmt("[FEXServer] Check your FUSE setup.\n"); // Let the parent know that we couldn't execute for some reason @@ -182,12 +182,12 @@ namespace SquashFS { void UnmountRootFS() { FEX_CONFIG_OPT(LDPath, ROOTFS); - if (!FEX::FormatCheck::IsSquashFS(LDPath())) { + if (!FEX::FormatCheck::IsSquashFS(LDPath()) && !FEX::FormatCheck::IsEroFS(LDPath())) { return; } // Handle final mount removal - // fusermount for unmounting the mountpoint, then the squashfuse will exit automatically + // fusermount for unmounting the mountpoint, then the {erfsfuse, squashfuse} will exit automatically int pid = fork(); if (pid == 0) { @@ -221,8 +221,20 @@ namespace SquashFS { FEX_CONFIG_OPT(LDPath, ROOTFS); MountFolder = LDPath(); - if (!FEX::FormatCheck::IsSquashFS(MountFolder)) { - // If this isn't a squashfs mount then we have nothing to do here + + bool IsSquashFS {false}; + bool IsEroFS {false}; + + // Check if the image is an EroFS + IsEroFS = FEX::FormatCheck::IsEroFS(MountFolder); + + if (!IsEroFS) { + // Check if the image is an SquashFS + IsSquashFS = FEX::FormatCheck::IsSquashFS(MountFolder); + } + + if (!IsSquashFS && !IsEroFS) { + // If this isn't a rootfs image then we have nothing to do here return true; } @@ -232,7 +244,7 @@ namespace SquashFS { } // Setup rootfs here - if (!MountSquashFSPath(LDPath())) { + if (!MountRootFSImagePath(LDPath(), IsEroFS)) { LogMan::Msg::EFmt("[FEXServer] Couldn't mount squashfs path"); return false; } From a1ed54adc78bf3aa5cf9061771b08cc4f13ad505 Mon Sep 17 00:00:00 2001 From: Ryan Houdek Date: Mon, 4 Jul 2022 20:52:17 -0700 Subject: [PATCH 4/4] FEXRootFSFetcher: Add support for EroFS Fixes a bug in `ExecAndWaitForResponse` where results > 1024 bytes would overwrite data. Switches from a custom format txt file to a json file. JSON file now has a "Type" field to specify squashfs versus erofs. JSON is now versioned so we don't need to move the file around, just append to a new versioned segment. Only shows erofs files if you have the bleeding edge `erofsfuse` application. This application was available starting with erofs-utils v1.5 which was released on 2022-06-13, so it isn't available pretty much everywhere. --- Source/Common/FileFormatCheck.cpp | 18 --- Source/Tools/FEXRootFSFetcher/Main.cpp | 207 +++++++++++++++++++------ 2 files changed, 159 insertions(+), 66 deletions(-) diff --git a/Source/Common/FileFormatCheck.cpp b/Source/Common/FileFormatCheck.cpp index 5703f2856..9c709cf4e 100644 --- a/Source/Common/FileFormatCheck.cpp +++ b/Source/Common/FileFormatCheck.cpp @@ -75,7 +75,6 @@ namespace FEX::FormatCheck { // Additional data after this if necessary in the future. }; - constexpr size_t SUPERBLOCK_SIZE = 4096; constexpr size_t HEADER_OFFSET = 1024; constexpr uint32_t COOKIE_MAGIC_V1 = 0xE0F5E1E2; @@ -86,23 +85,6 @@ namespace FEX::FormatCheck { return false; } - if (!File.seekg(0, std::fstream::end)) { - return false; - } - - auto FileSize = File.tellg(); - if (File.fail()) { - return false; - } - - if (FileSize <= 0) { - return false; - } - - if (FileSize < SUPERBLOCK_SIZE) { - return false; - } - if (!File.seekg(HEADER_OFFSET, std::fstream::beg)) { return false; } diff --git a/Source/Tools/FEXRootFSFetcher/Main.cpp b/Source/Tools/FEXRootFSFetcher/Main.cpp index 553222056..79672d798 100644 --- a/Source/Tools/FEXRootFSFetcher/Main.cpp +++ b/Source/Tools/FEXRootFSFetcher/Main.cpp @@ -13,6 +13,8 @@ #include #include +#include + namespace Exec { int32_t ExecAndWaitForResponse(const char *path, char* const* args) { pid_t pid = fork(); @@ -98,16 +100,19 @@ namespace Exec { // Nothing larger than this char Buffer[1024]{}; + std::string Output{}; // Read the pipe until it closes - while (read(fd[0], Buffer, sizeof(Buffer))); + while (size_t Size = read(fd[0], Buffer, sizeof(Buffer))) { + Output += std::string_view(Buffer, Size); + } int32_t Status{}; waitpid(pid, &Status, 0); if (WIFEXITED(Status)) { // Return what we've read close(fd[0]); - return Buffer; + return Output; } } @@ -120,6 +125,9 @@ namespace WorkingAppsTester { static bool Has_Squashfuse {false}; static bool Has_Unsquashfs {false}; + // EroFS specific + static bool Has_EroFSFuse {false}; + void CheckCurl() { // Check if curl exists on the host std::vector ExecveArgs = { @@ -190,10 +198,23 @@ namespace WorkingAppsTester { close(fd); } + // EroFS specific tests + void CheckEroFSFuse() { + std::vector ExecveArgs = { + "erofsfuse", + "--help", + nullptr, + }; + + int32_t Result = Exec::ExecAndWaitForResponseRedirect(ExecveArgs[0], const_cast(ExecveArgs.data()), -1, -1); + Has_EroFSFuse = Result != -1; + } + void Init() { CheckCurl(); CheckSquashfuse(); CheckUnsquashfs(); + CheckEroFSFuse(); } } @@ -344,9 +365,17 @@ namespace WebFileFetcher { // This is the hash of the file std::string Hash; + + // FileType + enum class FileType { + TYPE_UNKNOWN, + TYPE_SQUASHFS, + TYPE_EROFS, + }; + FileType Type; }; - const static std::string DownloadURL = "https://rootfs.fex-emu.org/file/fex-rootfs/RootFS_links_XXH3.txt"; + const static std::string DownloadURL = "https://rootfs.fex-emu.org/file/fex-rootfs/RootFS_links.json"; std::string DownloadToString(const std::string &URL) { std::string BigArgs = @@ -402,27 +431,96 @@ namespace WebFileFetcher { return Exec::ExecAndWaitForResponse(ExecveArgs[0], const_cast(ExecveArgs.data())) == 0; } + struct JsonAllocator { + jsonPool_t PoolObject; + std::unique_ptr> json_objects; + }; + static_assert(offsetof(JsonAllocator, PoolObject) == 0, "This needs to be at offset zero"); + + json_t* PoolInit(jsonPool_t* Pool) { + JsonAllocator* alloc = reinterpret_cast(Pool); + alloc->json_objects = std::make_unique>(); + return &*alloc->json_objects->emplace(alloc->json_objects->end()); + } + + json_t* PoolAlloc(jsonPool_t* Pool) { + JsonAllocator* alloc = reinterpret_cast(Pool); + return &*alloc->json_objects->emplace(alloc->json_objects->end()); + } + std::vector GetRootFSLinks() { // Decode the filetargets std::string Data = DownloadToString(DownloadURL); - std::stringstream ss(Data); + + JsonAllocator Pool { + .PoolObject = { + .init = PoolInit, + .alloc = PoolAlloc, + }, + }; + + json_t const *json = json_createWithPool(&Data.at(0), &Pool.PoolObject); + if (!json) { + fprintf(stderr, "Couldn't create json"); + return {}; + } + + json_t const* RootList = json_getProperty(json, "v1"); + + if (!RootList) { + fprintf(stderr, "Couldn't get root list"); + return {}; + } + std::vector Targets; - while (!ss.eof()) { - FileTargets Target; - if (!std::getline(ss, Target.DistroMatch)) - break; - if (!std::getline(ss, Target.VersionMatch)) - break; - if (!std::getline(ss, Target.DistroName)) - break; - if (!std::getline(ss, Target.URL)) - break; - if (!std::getline(ss, Target.Hash)) - break; + for (json_t const* RootItem = json_getChild(RootList); + RootItem != nullptr; + RootItem = json_getSibling(RootItem)) { - Targets.emplace_back(Target); + FileTargets Target; + Target.DistroName = json_getName(RootItem); + + for (json_t const* DataItem = json_getChild(RootItem); + DataItem != nullptr; + DataItem = json_getSibling(DataItem)) { + auto DataName = std::string_view {json_getName(DataItem)}; + + if (DataName == "DistroMatch") { + Target.DistroMatch = json_getValue(DataItem); + } + else if (DataName == "DistroVersion") { + Target.VersionMatch = json_getValue(DataItem); + } + else if (DataName == "URL") { + Target.URL = json_getValue(DataItem); + } + else if (DataName == "Hash") { + Target.Hash = json_getValue(DataItem); + } + else if (DataName == "Type") { + auto DataValue = std::string_view {json_getValue(DataItem)}; + if (DataValue == "squashfs") { + Target.Type = FileTargets::FileType::TYPE_SQUASHFS; + } + else if (DataValue == "erofs") { + Target.Type = FileTargets::FileType::TYPE_EROFS; + } + else { + Target.Type = FileTargets::FileType::TYPE_UNKNOWN; + } + } + } + bool SupportsSquashFS = WorkingAppsTester::Has_Squashfuse || WorkingAppsTester::Has_Unsquashfs; + bool SupportsEroFS = WorkingAppsTester::Has_EroFSFuse; + if ((Target.Type == FileTargets::FileType::TYPE_SQUASHFS && SupportsSquashFS) || + (Target.Type == FileTargets::FileType::TYPE_EROFS && SupportsEroFS)) { + // If we don't understand the type, then we can't use this. + // Additionally if the type is erofs but the user doesn't have erofsfuse, then we can't use this + Targets.emplace_back(Target); + } } + return Targets; } } @@ -940,6 +1038,11 @@ int main(int argc, char **argv, char **const envp) { if (AskForConfirmation(Question)) { auto Targets = WebFileFetcher::GetRootFSLinks(); + if (Targets.empty()) { + ExecWithInfo("Couldn't parse rootfs definition URL."); + return -1; + } + int32_t DistroIndex = AskForDistroSelection(Targets); if (DistroIndex != -1) { const auto &Target = Targets[DistroIndex]; @@ -1004,42 +1107,50 @@ int main(int argc, char **argv, char **const envp) { "As-Is", }; - int32_t Result{}; - if (WorkingAppsTester::Has_Unsquashfs) { - if (WorkingAppsTester::Has_Squashfuse) { - Result = AskForConfirmationList("Do you wish to extract the squashfs file or use it as-is?", Args); - } - else { - Args.pop_back(); - Result = AskForConfirmationList("Squashfuse doesn't work. Do you wish to extract the squashfs file?", Args); - } - } - else { - if (WorkingAppsTester::Has_Squashfuse) { - Args.erase(Args.begin()); - Result = AskForConfirmationList("Unsquashfs doesn't work. Do you want to use the squashfs file as-is?", Args); - if (Result == 0) { - // We removed an argument, Just change "As-Is" from 0 to 1 for later logic to work - Result = 1; + if (Target.Type == WebFileFetcher::FileTargets::FileType::TYPE_SQUASHFS) { + int32_t Result{}; + if (WorkingAppsTester::Has_Unsquashfs) { + if (WorkingAppsTester::Has_Squashfuse) { + Result = AskForConfirmationList("Do you wish to extract the squashfs file or use it as-is?", Args); + } + else { + Args.pop_back(); + Result = AskForConfirmationList("Squashfuse doesn't work. Do you wish to extract the squashfs file?", Args); } } else { - Args.erase(Args.begin()); - ExecWithInfo("Unsquashfs and squashfuse isn't working. Leaving rootfs as-is"); - Result = -1; + if (WorkingAppsTester::Has_Squashfuse) { + Args.erase(Args.begin()); + Result = AskForConfirmationList("Unsquashfs doesn't work. Do you want to use the squashfs file as-is?", Args); + if (Result == 0) { + // We removed an argument, Just change "As-Is" from 0 to 1 for later logic to work + Result = 1; + } + } + else { + Args.erase(Args.begin()); + ExecWithInfo("Unsquashfs and squashfuse isn't working. Leaving rootfs as-is"); + Result = -1; + } } - } - if (Result == -1 || - Result == 1) { - // Nothing - // As-Is - } - else if (Result == 0) { - auto FolderName = filename.substr(0, filename.find_last_of('.')); - if (UnSquash::UnsquashRootFS(RootFS, PathName, FolderName)) { - // Remove the .sqsh suffix since we extracted to that - filename = FolderName; + + if (Result == -1 || + Result == 1) { + // Nothing + // As-Is } + else if (Result == 0) { + auto FolderName = filename.substr(0, filename.find_last_of('.')); + if (UnSquash::UnsquashRootFS(RootFS, PathName, FolderName)) { + // Remove the .sqsh suffix since we extracted to that + filename = FolderName; + } + } + + } + else if (Target.Type == WebFileFetcher::FileTargets::FileType::TYPE_EROFS) { + // Once erofs tooling is available for easy extraction, offer the same settings to extract as squashfs. + // Currently this is unavailable, would need a mount + copy + unmount dance. } if (AskForConfirmation("Do you wish to set this RootFS as default?")) {