From bc2840e4a741ddf3088f46548b6cb74bbfcb30de Mon Sep 17 00:00:00 2001 From: Ryan Houdek Date: Thu, 9 Jun 2022 20:10:11 -0700 Subject: [PATCH] FEXRootFSFetcher: Update and fix xxhash file hashing The final tail of the file reading was incorrect, so our hashing was "correct" but it was using stale data from the previous block size read. Noticed this while wiring up the CI rootfs fetching since the hashing is a lot simpler there. Now instead of reading a tail, just attempt to read the full block size and use the resulting data size instead. Confirmed it matches expected results now. In the process we are going to need to update hyperlinks and hashes anyway, change the hash to XXH3 so it is faster to run. --- Scripts/InstallFEX.py | 4 +-- Source/Tools/FEXRootFSFetcher/Main.cpp | 2 +- Source/Tools/FEXRootFSFetcher/XXFileHash.cpp | 27 ++++++-------------- docs/ReleaseProcess.md | 6 ++--- 4 files changed, 14 insertions(+), 25 deletions(-) diff --git a/Scripts/InstallFEX.py b/Scripts/InstallFEX.py index 0cf361b17..99c9119d9 100755 --- a/Scripts/InstallFEX.py +++ b/Scripts/InstallFEX.py @@ -306,12 +306,12 @@ def GetRootFSPath(): return _RootFSPath def CheckRootFSInstallStatus(): - # Matches what is available on https://rootfs.fex-emu.org/file/fex-rootfs/RootFS_links.txt + # Matches what is available on https://rootfs.fex-emu.org/file/fex-rootfs/RootFS_links_XXH3.txt UbuntuVersionToRootFS = { "20.04": "Ubuntu_21_04.sqsh", "21.04": "Ubuntu_21_04.sqsh", "21.10": "Ubuntu_21_10.sqsh", - "22.04": "Ubuntu_21_10.sqsh", + "22.04": "Ubuntu_22_04.sqsh", } return os.path.exists(GetRootFSPath() + UbuntuVersionToRootFS[GetDistro()[1]]) diff --git a/Source/Tools/FEXRootFSFetcher/Main.cpp b/Source/Tools/FEXRootFSFetcher/Main.cpp index d403cb970..553222056 100644 --- a/Source/Tools/FEXRootFSFetcher/Main.cpp +++ b/Source/Tools/FEXRootFSFetcher/Main.cpp @@ -346,7 +346,7 @@ namespace WebFileFetcher { std::string Hash; }; - const static std::string DownloadURL = "https://rootfs.fex-emu.org/file/fex-rootfs/RootFS_links.txt"; + const static std::string DownloadURL = "https://rootfs.fex-emu.org/file/fex-rootfs/RootFS_links_XXH3.txt"; std::string DownloadToString(const std::string &URL) { std::string BigArgs = diff --git a/Source/Tools/FEXRootFSFetcher/XXFileHash.cpp b/Source/Tools/FEXRootFSFetcher/XXFileHash.cpp index 87a37adec..6f5b39747 100644 --- a/Source/Tools/FEXRootFSFetcher/XXFileHash.cpp +++ b/Source/Tools/FEXRootFSFetcher/XXFileHash.cpp @@ -28,32 +28,31 @@ namespace XXFileHash { lseek(fd, 0, SEEK_SET); // Set up XXHash state - XXH64_state_t* const State = XXH64_createState(); + XXH3_state_t* const State = XXH3_createState(); XXH64_hash_t const Seed = 0; if (!State) { return HadError(); } - if (XXH64_reset(State, Seed) == XXH_ERROR) { + if (XXH3_64bits_reset_withSeed(State, Seed) == XXH_ERROR) { return HadError(); } std::vector Data(BLOCK_SIZE); - off_t DataRemaining = Size - BLOCK_SIZE; - off_t DataTail = Size - DataRemaining; off_t CurrentOffset = 0; auto Now = std::chrono::high_resolution_clock::now(); // Let the kernel know that we will be reading linearly posix_fadvise(fd, 0, Size, POSIX_FADV_SEQUENTIAL); - while (CurrentOffset < DataRemaining) { + while (CurrentOffset < Size) { + ssize_t Result = pread(fd, Data.data(), BLOCK_SIZE, CurrentOffset); if (Result == -1) { return HadError(); } - if (XXH64_update(State, Data.data(), BLOCK_SIZE) == XXH_ERROR) { + if (XXH3_64bits_update(State, Data.data(), Result) == XXH_ERROR) { return HadError(); } auto Cur = std::chrono::high_resolution_clock::now(); @@ -62,21 +61,11 @@ namespace XXFileHash { fmt::print("{}% hashed\n", (double)CurrentOffset / SizeD); Now = Cur; } - CurrentOffset += BLOCK_SIZE; + CurrentOffset += Result; } - // Finish the tail - ssize_t Result = pread(fd, Data.data(), DataTail, CurrentOffset); - if (Result == -1) { - return HadError(); - } - - if (XXH64_update(State, Data.data(), DataTail) == XXH_ERROR) { - return HadError(); - } - - XXH64_hash_t const Hash = XXH64_digest(State); - XXH64_freeState(State); + XXH64_hash_t const Hash = XXH3_64bits_digest(State); + XXH3_freeState(State); close(fd); return {true, Hash}; diff --git a/docs/ReleaseProcess.md b/docs/ReleaseProcess.md index aad3d2c8d..cc8c60a31 100644 --- a/docs/ReleaseProcess.md +++ b/docs/ReleaseProcess.md @@ -80,7 +80,7 @@ Follow the steps in: https://github.com/FEX-Emu/FEX-ppa/blob/main/README_ppa.md * Follow the Build_Data file's information for how to generate an image using `build_image.py` * This gives a squashfs image for the rootfs * Use FEXRootFSFetcher to generate the xxhash for the image -* Update `https://rootfs.fex-emu.org/file/fex-rootfs/RootFS_links.txt` with the new rootfs image and hash +* Update `https://rootfs.fex-emu.org/file/fex-rootfs/RootFS_links_XXH3.txt` with the new rootfs image and hash * This currently lives in a private FEX-Emu backblaze bucket with cloudflare servicing it. * Never publically give the direct backblaze link to the file. Will cause BW costs to skyrocket * Always pass through cloudflare @@ -88,9 +88,9 @@ Follow the steps in: https://github.com/FEX-Emu/FEX-ppa/blob/main/README_ppa.md * Upload new image to Backblaze using the b2 upload tool * b2 upload-file / -* Upload the new RootFS_links.txt +* Upload the new RootFS_links_XXH3.txt * Lives in the root of the bucket - * b2 upload-file RootFS_links.txt RootFS_links.txt + * b2 upload-file RootFS_links_XXH3.txt RootFS_links_XXH3.txt * Once uploaded it should propagate immediately * Might be worth thinking about the coherency problem of updating the hash versus image independently if overwriting an image