From 0bf2675db16e9f97eca0dc91f2fc196f4ae56eaa Mon Sep 17 00:00:00 2001 From: iChris4 Date: Thu, 17 Sep 2026 20:33:43 +0200 Subject: [PATCH] Introduced translator_host C program to start the WiiCompiled translator on the headset, managing heap pointer tagging and dynamic library loading. --- THIRD-PARTY-NOTICES.md | 23 + android/Build-Quest.ps1 | 4 +- android/Prepare-QuestToolchain.ps1 | 341 ++++++++++++ android/QuestGameKit.psm1 | 124 ++++- android/README.md | 6 +- android/app/build.gradle.kts | 44 ++ .../wiicompiled/quest/launcher/BuildRecipe.kt | 89 ++++ .../wiicompiled/quest/launcher/GameBuild.kt | 493 ++++++++++++++++++ .../wiicompiled/quest/launcher/GameSetup.kt | 37 +- .../quest/launcher/GameSetupService.kt | 31 +- .../quest/launcher/LauncherActivity.kt | 81 ++- .../wiicompiled/quest/launcher/RemoteZip.kt | 126 +++++ .../quest/launcher/SettingsPage.kt | 7 + .../wiicompiled/quest/launcher/ToolProcess.kt | 93 ++++ android/app/src/main/res/values/strings.xml | 26 +- .../quest/launcher/BuildRecipeTest.kt | 97 ++++ .../quest/launcher/RemoteZipTest.kt | 77 +++ android/toolchain/translator_host.c | 61 +++ docs/quest-port.md | 96 +++- 19 files changed, 1796 insertions(+), 60 deletions(-) create mode 100644 android/Prepare-QuestToolchain.ps1 create mode 100644 android/app/src/main/java/org/wiicompiled/quest/launcher/BuildRecipe.kt create mode 100644 android/app/src/main/java/org/wiicompiled/quest/launcher/GameBuild.kt create mode 100644 android/app/src/main/java/org/wiicompiled/quest/launcher/RemoteZip.kt create mode 100644 android/app/src/main/java/org/wiicompiled/quest/launcher/ToolProcess.kt create mode 100644 android/app/src/test/java/org/wiicompiled/quest/launcher/BuildRecipeTest.kt create mode 100644 android/app/src/test/java/org/wiicompiled/quest/launcher/RemoteZipTest.kt create mode 100644 android/toolchain/translator_host.c diff --git a/THIRD-PARTY-NOTICES.md b/THIRD-PARTY-NOTICES.md index 2badf0b..d606612 100644 --- a/THIRD-PARTY-NOTICES.md +++ b/THIRD-PARTY-NOTICES.md @@ -154,6 +154,28 @@ Unicode-3.0, BSL-1.0 or Unlicense licensed. | bzip2 (via bzip2-sys) | 1.0.8 | bzip2 license (BSD-style) | | | jni-rs | 0.21.1 | MIT OR Apache-2.0 | | +For building the game on the headset, the APK also carries a toolchain +(`android/Prepare-QuestToolchain.ps1`): the translator published with the .NET 10 runtime for +`linux-bionic-arm64`, and Termux's Android builds of clang and lld with the libraries they load, +redistributed unmodified from the Termux package repository (pins and license names in the script +and in the toolchain's `llvm/licenses/`). + +| Component (Quest APK toolchain) | Version | License | Upstream | +| --- | --- | --- | --- | +| .NET runtime (Mono, linux-bionic-arm64) | 10.0 | MIT | | +| YamlDotNet | 15.1.2 | MIT | | +| LLVM clang, lld, libLLVM, libc++ (Termux clang/lld/libllvm 21.1.8-3, libc++ 29) | 21.1.8 | Apache-2.0 WITH LLVM-exception | | +| libffi | 3.8.0 | MIT | | +| libxml2 | 2.15.4 | MIT | | +| GNU libiconv | 1.19 | LGPL-2.1-or-later | | +| zlib | 1.3.2 | Zlib | | +| Zstandard | 1.5.7 | BSD-3-Clause | | +| OpenSSL | 3.6.3 | Apache-2.0 | | + +libiconv is LGPL-licensed and ships as the unmodified shared library `libiconv.so`, which can be +replaced; its complete corresponding source is available from the Termux package repository and +from GNU, and this project will supply it on request for the version shipped. + ### Dual-licensed components - elections made by this project - **FreeType** is offered under the FreeType License (FTL) or GPL-2.0. **This project elects the @@ -192,6 +214,7 @@ unmodified, with their license texts, in the installer's `licenses/` folder. | Component | License | Upstream | | --- | --- | --- | | Android NDK r29 for Windows (clang, lld, sysroot), fetched by `--build-quest` from Google with a pinned SHA-1 | Android Software Development Kit License Agreement | | +| Android NDK r29 aarch64 sysroot, compiler-rt builtins, libunwind and libatomic, fetched by the Quest app's Build on this Quest from Google's Linux NDK zip with pinned SHA-256s | Android Software Development Kit License Agreement | | --- diff --git a/android/Build-Quest.ps1 b/android/Build-Quest.ps1 index f951e6d..e773c45 100644 --- a/android/Build-Quest.ps1 +++ b/android/Build-Quest.ps1 @@ -94,14 +94,16 @@ if ($Flavor -eq 'base') { try { $gameLibraries = @($archive.Entries | Where-Object { $_.FullName -match '^lib/[^/]+/libmain[^/]*\.so$' }) $hasKit = @($archive.Entries | Where-Object { $_.FullName -eq 'assets/game_kit/kit.json' }).Count -gt 0 + $hasToolchain = @($archive.Entries | Where-Object { $_.FullName -eq 'assets/quest_toolchain/files.zip' }).Count -gt 0 } finally { $archive.Dispose() } if ($gameLibraries.Count -gt 0) { throw "The APK contains a translated game library: $($gameLibraries.FullName -join ', ')" } if (-not $hasKit) { throw 'The APK has no game kit (assets/game_kit/kit.json)' } + if (-not $hasToolchain) { throw 'The APK has no build toolchain (assets/quest_toolchain/files.zip)' } } if ($Install) { $adb = Join-Path $sdkRoot 'platform-tools\adb.exe' & $adb install -r $apk.FullName if ($LASTEXITCODE -ne 0) { throw "adb install failed ($LASTEXITCODE)" } - Write-Host 'Installed. Build and push the game with android/Build-QuestGame.ps1 -Install.' + Write-Host 'Installed. Build the game with Build on this Quest in the launcher, or on this PC with android/Build-QuestGame.ps1 -Install.' } diff --git a/android/Prepare-QuestToolchain.ps1 b/android/Prepare-QuestToolchain.ps1 new file mode 100644 index 0000000..fe9c279 --- /dev/null +++ b/android/Prepare-QuestToolchain.ps1 @@ -0,0 +1,341 @@ +# Builds the toolchain the Quest app carries so a player can build the game on the headset +# ("Build on this Quest"), without a PC: +# +# * translator/ the WiiCompiled translator published for linux-bionic-arm64 (.NET 10, which runs +# it on Mono), started through translator_host (android/toolchain/translator_host.c); +# * llvm/ Termux's Android build of clang and lld 21.1.8 with exactly the shared libraries +# they load, OpenSSL for the translator's hashing (Android only has BoringSSL), and +# clang's own builtin headers; +# * ndk.json the Android NDK files a build also needs (the aarch64 sysroot, libunwind, libatomic, +# compiler-rt builtins). The NDK is Google's to distribute, so the headset downloads +# them from dl.google.com with HTTP range requests (about 8 MB of the 784 MB zip) and +# checks each file against the SHA-256 recorded here; +# * toolchain.json every file of translator/ and llvm/ with its SHA-256, and the fingerprint over +# them. Those two directories travel packed in files.zip. +# +# powershell -ExecutionPolicy Bypass -File android/Prepare-QuestToolchain.ps1 -OutputDir -NdkLlvm \toolchains\llvm\prebuilt\windows-x86_64 +# +# The Gradle build runs it (prepareQuestToolchain). Downloads are cached in android/.dependencies. +# Needs the .NET 10 SDK: the linux-bionic runtime packs only exist for .NET 10. +[CmdletBinding()] +param( + [Parameter(Mandatory)] [string]$OutputDir, + [Parameter(Mandatory)] [string]$NdkLlvm, + [string]$CacheDir = '', + [string]$Dotnet = 'dotnet' +) +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 +Add-Type -AssemblyName System.IO.Compression, System.IO.Compression.FileSystem +# Windows PowerShell leaves $PSScriptRoot empty while binding parameter defaults. +if (-not $CacheDir) { $CacheDir = Join-Path $PSScriptRoot '.dependencies\quest-toolchain' } + +$repo = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path +$termuxRoot = 'https://packages.termux.dev/apt/termux-main/' +$termuxPrefix = './data/data/com.termux/files/usr/' + +# Termux packages (aarch64), pinned by the SHA-256 of their .deb. Files maps a path inside the +# package to its name in llvm/: the libraries are stored under the SONAME their users load. +$termuxPackages = @( + @{ Name = 'clang'; File = 'pool/main/c/clang/clang_21.1.8-3_aarch64.deb'; Sha256 = '9fb565013b96ce10abd8f8f217aaebb48f72873f53acbf8886329bcea841a21c' + License = 'Apache-2.0 WITH LLVM-exception' + Files = @{ 'bin/clang-21' = 'bin/clang-21'; 'lib/libclang-cpp.so' = 'lib/libclang-cpp.so'; 'lib/clang/21/include/' = 'lib/clang/21/include/' } } + @{ Name = 'lld'; File = 'pool/main/l/lld/lld_21.1.8-3_aarch64.deb'; Sha256 = '6299c8fdff59dcff972f6b444805bcdb211c2bd797a64d2ddb3875e699f30630' + License = 'Apache-2.0 WITH LLVM-exception' + Files = @{ 'bin/lld' = 'bin/ld.lld' } } + @{ Name = 'libllvm'; File = 'pool/main/libl/libllvm/libllvm_21.1.8-3_aarch64.deb'; Sha256 = '5056a73f2645fc9c0759e6f94a1bd6edcf36305acaef58dc685211ced9cc2a02' + License = 'Apache-2.0 WITH LLVM-exception' + Files = @{ 'lib/libLLVM.so' = 'lib/libLLVM.so'; 'share/doc/libllvm/LICENSE.TXT' = 'licenses/llvm/LICENSE.TXT' } } + @{ Name = 'libc++'; File = 'pool/main/libc/libc++/libc++_29_aarch64.deb'; Sha256 = 'bb9f12113c137aa0e8513bb51cc49fe77a5ce3ca39ab9e92c57d228ecdf00222' + License = 'Apache-2.0 WITH LLVM-exception' + Files = @{ 'lib/libc++_shared.so' = 'lib/libc++_shared.so' } } + @{ Name = 'libffi'; File = 'pool/main/libf/libffi/libffi_3.8.0_aarch64.deb'; Sha256 = '4f255badf74cd31f6a2801c17fa1444199c84c834b517b7c843e2fe9ebe91d77' + License = 'MIT' + Files = @{ 'lib/libffi.so' = 'lib/libffi.so'; 'share/doc/libffi/copyright' = 'licenses/libffi/copyright' } } + @{ Name = 'libxml2'; File = 'pool/main/libx/libxml2/libxml2_2.15.4-1_aarch64.deb'; Sha256 = 'd315861bbcf716cfb088f53cddab9fb6a96430cc66769f79774cc69c2f53bcff' + License = 'MIT' + Files = @{ 'lib/libxml2.so.16.1.4' = 'lib/libxml2.so.16' } } + @{ Name = 'libiconv'; File = 'pool/main/libi/libiconv/libiconv_1.19_aarch64.deb'; Sha256 = 'fe9481b1dc101c6c3552943f25435109fd522aecc615ae49594f9fbee863bb37' + License = 'LGPL-2.1-or-later' + Files = @{ 'lib/libiconv.so' = 'lib/libiconv.so' } } + @{ Name = 'zlib'; File = 'pool/main/z/zlib/zlib_1.3.2_aarch64.deb'; Sha256 = '75e7d0af17fcc3b40004309fdc00a1ddb9ae08346dce5e269902c34ac3966ac9' + License = 'Zlib' + Files = @{ 'lib/libz.so.1.3.2' = 'lib/libz.so.1' } } + @{ Name = 'zstd'; File = 'pool/main/z/zstd/zstd_1.5.7-1_aarch64.deb'; Sha256 = 'e1b4a5113648da8de189620ba1fce74c48b2d0833d9043391b9a1c91fb606fd3' + License = 'BSD-3-Clause' + Files = @{ 'lib/libzstd.so.1.5.7' = 'lib/libzstd.so.1' } } + @{ Name = 'openssl'; File = 'pool/main/o/openssl/openssl_1:3.6.3_aarch64.deb'; Sha256 = '86760e9ce736f463236f2c15b1eb3a3fdcfc5778d0fd7077a917448dcc90f3aa' + License = 'Apache-2.0' + # .NET's OpenSSL shim on Android loads "libssl.so" by that name; libssl needs libcrypto.so.3. + Files = @{ 'lib/libssl.so.3' = 'lib/libssl.so'; 'lib/libcrypto.so.3' = 'lib/libcrypto.so.3' } } +) + +# The NDK files the headset downloads: the r29 Linux zip, pinned by size and by the digest of the +# selected files (Get-ListDigest over them). The Linux zip is used because Windows file systems +# cannot hold its sysroot headers whose names differ only in case. +$ndkUrl = 'https://dl.google.com/android/repository/android-ndk-r29-linux.zip' +$ndkSize = 783549481 +$ndkPrefix = 'android-ndk-r29/toolchains/llvm/prebuilt/linux-x86_64/' +$ndkSubsetDigest = '4221ed656e864cb3434d922e8e9eb916b08833718380847d0923527394041d0f' + +function Get-Sha256Hex([string]$Path) { + $sha = [Security.Cryptography.SHA256]::Create() + try { + $stream = [IO.File]::OpenRead($Path) + try { $bytes = $sha.ComputeHash($stream) } finally { $stream.Dispose() } + } finally { $sha.Dispose() } + return (($bytes | ForEach-Object { $_.ToString('x2') }) -join '') +} + +function Get-BytesSha256Hex([byte[]]$Bytes) { + $sha = [Security.Cryptography.SHA256]::Create() + try { $hash = $sha.ComputeHash($Bytes) } finally { $sha.Dispose() } + return (($hash | ForEach-Object { $_.ToString('x2') }) -join '') +} + +# SHA-256 over " " lines sorted by path, joined with \n. The headset computes the +# same over the files it downloaded. +function Get-ListDigest([object[]]$Files) { + [string[]]$lines = @($Files | ForEach-Object { "$($_.path) $($_.sha256)" }) + [Array]::Sort($lines, [StringComparer]::Ordinal) + return Get-BytesSha256Hex ([Text.Encoding]::UTF8.GetBytes(($lines -join "`n"))) +} + +function Invoke-Tar([string[]]$Arguments) { + & (Join-Path $env:SystemRoot 'System32\tar.exe') @Arguments + if ($LASTEXITCODE -ne 0) { throw "tar $($Arguments -join ' ') failed ($LASTEXITCODE)" } +} + +function Save-TermuxPackage($Package) { + $directory = Join-Path $CacheDir 'termux' + New-Item -ItemType Directory -Force $directory | Out-Null + # Termux file names can contain ':', which Windows does not allow. + $local = Join-Path $directory ($Package.File.Split('/')[-1].Replace(':', '_')) + if (-not (Test-Path $local) -or (Get-Sha256Hex $local) -ne $Package.Sha256) { + $url = $termuxRoot + $Package.File.Replace(':', '%3A') + Write-Host "Downloading $url" + Invoke-WebRequest -UseBasicParsing -Uri $url -OutFile "$local.partial" + $actual = Get-Sha256Hex "$local.partial" + if ($actual -ne $Package.Sha256) { + Remove-Item "$local.partial" + throw "$($Package.Name) package digest mismatch: $actual" + } + Move-Item -Force "$local.partial" $local + } + return $local +} + +function Expand-TermuxPackage($Package, [string]$Destination, [string]$Work) { + $deb = Save-TermuxPackage $Package + $unpack = Join-Path $Work $Package.Name + if (Test-Path $unpack) { Remove-Item -Recurse -Force $unpack } + New-Item -ItemType Directory -Force $unpack | Out-Null + Invoke-Tar @('-xf', $deb, '-C', $unpack) + $data = Get-ChildItem $unpack -Filter 'data.tar.*' | Select-Object -First 1 + if (-not $data) { throw "$($Package.Name) has no data archive" } + # Only the listed regular files: the packages also hold symlinks, which Windows cannot create. + $members = @($Package.Files.Keys | ForEach-Object { $termuxPrefix + $_ }) + Invoke-Tar (@('-xf', $data.FullName, '-C', $unpack) + $members) + foreach ($source in $Package.Files.Keys) { + $from = Join-Path $unpack ($termuxPrefix.Substring(2) + $source) + $to = Join-Path $Destination $Package.Files[$source] + if ($source.EndsWith('/')) { + New-Item -ItemType Directory -Force $to | Out-Null + Copy-Item -Recurse -Force (Join-Path $from '*') $to + } else { + if (-not (Test-Path $from -PathType Leaf)) { throw "$($Package.Name) has no regular file $source" } + New-Item -ItemType Directory -Force (Split-Path -Parent $to) | Out-Null + Copy-Item -Force $from $to + } + } + Remove-Item -Recurse -Force $unpack +} + +function Read-RemoteRange([string]$Url, [long]$Start, [long]$End) { + $request = [Net.HttpWebRequest]::Create($Url) + $request.AddRange($Start, $End) + $response = $request.GetResponse() + try { + if ([int]$response.StatusCode -ne 206) { throw "$Url ignored the range request" } + $stream = $response.GetResponseStream() + $buffer = New-Object byte[] ([int]($End - $Start + 1)) + $read = 0 + while ($read -lt $buffer.Length) { + $count = $stream.Read($buffer, $read, $buffer.Length - $read) + if ($count -le 0) { throw "Short read from $Url" } + $read += $count + } + return , $buffer + } finally { $response.Close() } +} + +# Whether a file below the NDK's prebuilt directory belongs to what the headset downloads. +function Test-NdkSubsetFile([string]$Path) { + if ($Path.StartsWith('sysroot/usr/include/')) { + return $Path -notmatch '^sysroot/usr/include/(arm-linux-androideabi|i686-linux-android|x86_64-linux-android|riscv64-linux-android)/' + } + if ($Path.StartsWith('sysroot/usr/lib/aarch64-linux-android/29/')) { return $true } + if ($Path -in 'sysroot/usr/lib/aarch64-linux-android/libc++_shared.so', 'sysroot/usr/lib/aarch64-linux-android/libc++.so') { return $true } + return $Path -match '^lib/clang/[^/]+/lib/linux/(libclang_rt\.builtins-aarch64-android\.a|aarch64/(libunwind|libatomic)\.a)$' +} + +# Lists the NDK subset with each file's SHA-256, reading only the zip's central directory and the +# selected entries over HTTP. Cached, since the pinned zip never changes. +function Get-NdkSubset { + $cached = Join-Path $CacheDir 'ndk-r29-subset.json' + if (Test-Path $cached) { return [IO.File]::ReadAllText($cached) | ConvertFrom-Json } + + $tail = Read-RemoteRange $ndkUrl ($ndkSize - 65557) ($ndkSize - 1) + $eocd = -1 + for ($i = $tail.Length - 22; $i -ge 0; $i--) { + if ([BitConverter]::ToUInt32($tail, $i) -eq 0x06054b50) { $eocd = $i; break } + } + if ($eocd -lt 0) { throw 'No end of central directory in the NDK zip' } + $entryCount = [BitConverter]::ToUInt16($tail, $eocd + 10) + $directorySize = [BitConverter]::ToUInt32($tail, $eocd + 12) + $directoryOffset = [BitConverter]::ToUInt32($tail, $eocd + 16) + if ($entryCount -eq 0xFFFF -or $directoryOffset -eq 0xFFFFFFFF) { throw 'The NDK zip needs ZIP64, which this reader does not support' } + $directory = Read-RemoteRange $ndkUrl $directoryOffset ($directoryOffset + $directorySize - 1) + + $entries = New-Object System.Collections.Generic.List[object] + $at = 0 + for ($n = 0; $n -lt $entryCount; $n++) { + if ([BitConverter]::ToUInt32($directory, $at) -ne 0x02014b50) { throw 'Corrupt NDK zip central directory' } + $nameLength = [BitConverter]::ToUInt16($directory, $at + 28) + $extraLength = [BitConverter]::ToUInt16($directory, $at + 30) + $commentLength = [BitConverter]::ToUInt16($directory, $at + 32) + $name = [Text.Encoding]::UTF8.GetString($directory, $at + 46, $nameLength) + $entries.Add([pscustomobject]@{ + name = $name + method = [BitConverter]::ToUInt16($directory, $at + 10) + compressedSize = [long][BitConverter]::ToUInt32($directory, $at + 20) + size = [long][BitConverter]::ToUInt32($directory, $at + 24) + offset = [long][BitConverter]::ToUInt32($directory, $at + 42) + }) + $at += 46 + $nameLength + $extraLength + $commentLength + } + $sorted = @($entries | Sort-Object offset) + $selected = New-Object System.Collections.Generic.List[object] + for ($i = 0; $i -lt $sorted.Count; $i++) { + $entry = $sorted[$i] + if (-not $entry.name.StartsWith($ndkPrefix) -or $entry.name.EndsWith('/')) { continue } + $path = $entry.name.Substring($ndkPrefix.Length) + if (-not (Test-NdkSubsetFile $path)) { continue } + $end = if ($i + 1 -lt $sorted.Count) { $sorted[$i + 1].offset } else { $directoryOffset } + $selected.Add([pscustomobject]@{ path = $path; entry = $entry; end = $end }) + } + + # Neighbouring entries are fetched together. + $files = New-Object System.Collections.Generic.List[object] + $i = 0 + while ($i -lt $selected.Count) { + $j = $i + while ($j + 1 -lt $selected.Count -and $selected[$j + 1].entry.offset - $selected[$j].end -lt 1MB) { $j++ } + $chunkStart = $selected[$i].entry.offset + $chunk = Read-RemoteRange $ndkUrl $chunkStart ($selected[$j].end - 1) + for ($k = $i; $k -le $j; $k++) { + $item = $selected[$k] + $local = [int]($item.entry.offset - $chunkStart) + if ([BitConverter]::ToUInt32($chunk, $local) -ne 0x04034b50) { throw "Corrupt NDK zip entry $($item.path)" } + $dataStart = $local + 30 + [BitConverter]::ToUInt16($chunk, $local + 26) + [BitConverter]::ToUInt16($chunk, $local + 28) + $compressed = New-Object IO.MemoryStream($chunk, $dataStart, [int]$item.entry.compressedSize) + $content = New-Object IO.MemoryStream + switch ($item.entry.method) { + 0 { $compressed.CopyTo($content) } + 8 { + $inflate = New-Object IO.Compression.DeflateStream($compressed, [IO.Compression.CompressionMode]::Decompress) + try { $inflate.CopyTo($content) } finally { $inflate.Dispose() } + } + default { throw "NDK zip entry $($item.path) uses compression method $($item.entry.method)" } + } + $bytes = $content.ToArray() + if ($bytes.Length -ne $item.entry.size) { throw "NDK zip entry $($item.path) inflated to the wrong size" } + $files.Add([pscustomobject]@{ path = $item.path; size = $item.entry.size; sha256 = Get-BytesSha256Hex $bytes }) + } + $i = $j + 1 + } + $subset = [pscustomobject]@{ url = $ndkUrl; size = $ndkSize; prefix = $ndkPrefix; digest = Get-ListDigest $files.ToArray(); files = $files.ToArray() } + New-Item -ItemType Directory -Force $CacheDir | Out-Null + [IO.File]::WriteAllText($cached, ($subset | ConvertTo-Json -Depth 4 -Compress), (New-Object Text.UTF8Encoding $false)) + return $subset +} + +function Publish-Translator([string]$Destination, [string]$Work) { + $sdks = (& $Dotnet --list-sdks) -join "`n" + if ($sdks -notmatch '(?m)^10\.') { throw 'Publishing the translator for Android needs the .NET 10 SDK.' } + # A staged copy retargeted to net10.0, so the checkout's own net8.0 build outputs stay untouched. + $stage = Join-Path $Work 'translator-stage' + if (Test-Path $stage) { Remove-Item -Recurse -Force $stage } + foreach ($tree in 'translator\src', 'Launcher\WiiCompiled.Setup.Common') { + & (Join-Path $env:SystemRoot 'System32\robocopy.exe') (Join-Path $repo $tree) (Join-Path $stage $tree) /E /XD bin obj /NFL /NDL /NJH /NJS /NP | Out-Null + if ($LASTEXITCODE -ge 8) { throw "Staging $tree failed (robocopy $LASTEXITCODE)" } + } + foreach ($project in Get-ChildItem -Recurse -Filter *.csproj $stage) { + $text = [IO.File]::ReadAllText($project.FullName) + $retargeted = $text.Replace('net8.0', 'net10.0') + if ($retargeted -eq $text) { throw "$($project.Name) does not target net8.0" } + [IO.File]::WriteAllText($project.FullName, $retargeted) + } + if (Test-Path $Destination) { Remove-Item -Recurse -Force $Destination } + & $Dotnet publish (Join-Path $stage 'translator\src\Translator.Cli\Translator.Cli.csproj') -c Release ` + -r linux-bionic-arm64 --self-contained true -p:PublishSingleFile=false -p:InvariantGlobalization=true ` + -p:DebugType=none -o $Destination | Out-Host + if ($LASTEXITCODE -ne 0) { throw "Publishing the translator failed ($LASTEXITCODE)" } + Get-ChildItem -Recurse -Filter *.pdb $Destination | Remove-Item + if (-not (Test-Path (Join-Path $Destination 'libhostfxr.so'))) { throw 'The translator was not published self-contained' } +} + +$work = Join-Path $CacheDir 'work' +New-Item -ItemType Directory -Force $work | Out-Null +# The tree is staged in the cache and packed into one deflated files.zip: Android's asset packaging +# never compresses .so files, and libLLVM alone is 134 MB. +$stage = Join-Path $work 'toolchain' +if (Test-Path $stage) { Remove-Item -Recurse -Force $stage } +New-Item -ItemType Directory -Force $stage | Out-Null +if (Test-Path $OutputDir) { Remove-Item -Recurse -Force $OutputDir } +New-Item -ItemType Directory -Force $OutputDir | Out-Null + +$llvm = Join-Path $stage 'llvm' +foreach ($package in $termuxPackages) { Expand-TermuxPackage $package $llvm $work } +$licenses = @($termuxPackages | ForEach-Object { "$($_.Name) $($_.File.Split('/')[-1].Replace('.deb', '')) - $($_.License) - https://github.com/termux/termux-packages" }) +New-Item -ItemType Directory -Force (Join-Path $llvm 'licenses') | Out-Null +[IO.File]::WriteAllText((Join-Path $llvm 'licenses\PACKAGES.txt'), (($licenses -join "`n") + "`n")) + +$translator = Join-Path $stage 'translator' +Publish-Translator $translator $work +# Static archives for embedding the runtime in a native app; the translator runs on the shared one. +Get-ChildItem -Filter *.a $translator | Remove-Item +& (Join-Path $NdkLlvm 'bin\clang.exe') --target=aarch64-linux-android29 -O2 -Wall -Wextra -Werror -pie -fPIE ` + -o (Join-Path $translator 'translator_host') (Join-Path $PSScriptRoot 'toolchain\translator_host.c') +if ($LASTEXITCODE -ne 0) { throw "Building translator_host failed ($LASTEXITCODE)" } + +$ndk = Get-NdkSubset +if ($ndkSubsetDigest -and $ndk.digest -ne $ndkSubsetDigest) { + throw "The NDK subset digest $($ndk.digest) does not match the pinned $ndkSubsetDigest" +} +[IO.File]::WriteAllText((Join-Path $OutputDir 'ndk.json'), ($ndk | ConvertTo-Json -Depth 4 -Compress), (New-Object Text.UTF8Encoding $false)) + +# .NET enumeration keeps the root as given (Get-ChildItem can expand an 8.3 TEMP name). +$root = [IO.Path]::GetFullPath($stage).TrimEnd('\') +$files = @([IO.Directory]::EnumerateFiles($root, '*', [IO.SearchOption]::AllDirectories) | ForEach-Object { + [pscustomobject]@{ path = $_.Substring($root.Length + 1).Replace('\', '/'); size = (New-Object IO.FileInfo $_).Length; sha256 = Get-Sha256Hex $_ } +} | Sort-Object path) +$zip = [IO.Compression.ZipFile]::Open((Join-Path $OutputDir 'files.zip'), 'Create') +try { + foreach ($file in $files) { + [void][IO.Compression.ZipFileExtensions]::CreateEntryFromFile($zip, (Join-Path $root $file.path), $file.path, 'Optimal') + } +} finally { $zip.Dispose() } + +$toolchain = [ordered]@{ + schema = 1 + fingerprint = Get-ListDigest $files + archive = 'files.zip' + ndk = 'ndk.json' + files = $files +} +[IO.File]::WriteAllText((Join-Path $OutputDir 'toolchain.json'), ($toolchain | ConvertTo-Json -Depth 4), (New-Object Text.UTF8Encoding $false)) +$unpacked = [math]::Round((($files | Measure-Object size -Sum).Sum) / 1MB) +$packed = [math]::Round((New-Object IO.FileInfo (Join-Path $OutputDir 'files.zip')).Length / 1MB) +Write-Host "Quest toolchain $($toolchain.fingerprint) ($unpacked MB, $packed MB packed; NDK subset $($ndk.digest))" diff --git a/android/QuestGameKit.psm1 b/android/QuestGameKit.psm1 index de295f2..9db4b9d 100644 --- a/android/QuestGameKit.psm1 +++ b/android/QuestGameKit.psm1 @@ -81,13 +81,22 @@ function Get-StringSha256Hex([string]$Text) { # Identity of a runtime/include tree. Translated code compiles against these headers, so a game is # only built from a translation whose runtime headers are the kit's own. function Get-RuntimeIncludeFingerprint([string]$Directory) { - $root = (Resolve-Path $Directory).Path.TrimEnd('\', '/') - $lines = Get-ChildItem -Recurse -File $root | Sort-Object FullName | ForEach-Object { - $_.FullName.Substring($root.Length + 1).Replace('\', '/') + ' ' + (Get-Sha256Hex $_.FullName) + $lines = Get-RelativeFiles $Directory | Sort-Object FullName | ForEach-Object { + $_.Relative + ' ' + (Get-Sha256Hex $_.FullName) } return Get-StringSha256Hex ($lines -join "`n") } +# The files below a directory with their paths relative to it. .NET enumeration keeps the root +# exactly as given; Resolve-Path and Get-ChildItem can disagree about an 8.3 short name (TEMP), +# which would shift every relative path. +function Get-RelativeFiles([string]$Directory) { + $root = [IO.Path]::GetFullPath($Directory).TrimEnd('\', '/') + foreach ($path in [IO.Directory]::EnumerateFiles($root, '*', [IO.SearchOption]::AllDirectories)) { + [pscustomobject]@{ FullName = $path; Relative = $path.Substring($root.Length + 1).Replace('\', '/') } + } +} + # compile_commands.json as {file, command} objects. Windows PowerShell's ConvertFrom-Json refuses # documents over 2 MB, so 5.1 uses the serializer underneath it with the limit lifted. function Read-CompileCommands([string]$Path) { @@ -133,6 +142,91 @@ function ConvertTo-KitCompileFlags { return ,$flags.ToArray() } +# The arguments of a `clang -###` command line: double-quoted, with backslash escapes. +function Split-DriverCommand([string]$Line) { + $arguments = New-Object System.Collections.Generic.List[string] + $current = New-Object System.Text.StringBuilder + $inQuotes = $false + for ($i = 0; $i -lt $Line.Length; $i++) { + $c = $Line[$i] + if ($inQuotes) { + if ($c -eq '\' -and $i + 1 -lt $Line.Length) { [void]$current.Append($Line[++$i]) } + elseif ($c -eq '"') { $arguments.Add($current.ToString()); [void]$current.Clear(); $inQuotes = $false } + else { [void]$current.Append($c) } + } elseif ($c -eq '"') { $inQuotes = $true } + } + return , $arguments.ToArray() +} + +# The kit's link as a raw ld.lld command, for the headset. Android lets the app start its tools +# only through the system linker, so clang there cannot start lld itself and the builder runs lld +# directly. The NDK's own driver expands the link here, with the kit and marker objects standing in +# as real files. Placeholders: {kit}, {ndk} (the NDK's prebuilt toolchain directory, which holds +# sysroot/ and lib/clang/), {output}, and {game:runtime}, {game:product}, {game:translated}, each +# replaced by its object files (the translated ones already sit inside --start-lib/--end-lib). +function Get-KitLldArguments { + param([string]$ClangCxx, [string]$KitDir, [string[]]$Flags, [string[]]$Inputs) + $kit = ConvertTo-ForwardPath $KitDir + $ndk = ConvertTo-ForwardPath (Join-Path (Split-Path -Parent $ClangCxx) '..') + # Forward slashes throughout: clang reads response files with GNU quoting, where '\' escapes. + $probe = ConvertTo-ForwardPath (Join-Path ([IO.Path]::GetTempPath()) ('mkw-kit-link-' + [Guid]::NewGuid().ToString('N'))) + New-Item -ItemType Directory $probe | Out-Null + try { + $markers = [ordered]@{} + foreach ($slot in 'runtime', 'product', 'translated') { + $markers[$slot] = ConvertTo-ForwardPath (Join-Path $probe "game_$slot.o") + [IO.File]::WriteAllBytes($markers[$slot], [byte[]]@()) + } + $real = { param([string]$s) $s.Replace('{kit}', $kit).Replace('{sysroot}', "$ndk/sysroot") } + $arguments = New-Object System.Collections.Generic.List[string] + foreach ($flag in $Flags) { $arguments.Add((& $real $flag)) } + $arguments.Add('-o'); $arguments.Add("$probe/libmain.so") + foreach ($linkInput in $Inputs) { + switch ($linkInput) { + '{game:runtime}' { $arguments.Add($markers.runtime) } + '{game:product}' { $arguments.Add($markers.product) } + '{game:translated}' { $arguments.Add('-Wl,--start-lib'); $arguments.Add($markers.translated); $arguments.Add('-Wl,--end-lib') } + default { $arguments.Add((& $real $linkInput)) } + } + } + $rsp = Join-Path $probe 'link.rsp' + [IO.File]::WriteAllText($rsp, (($arguments | ForEach-Object { ConvertTo-QuotedArgument $_ }) -join "`n"), (New-Object Text.UTF8Encoding $false)) + + # -### prints the command on stderr; Windows PowerShell turns redirected native stderr into errors. + $start = New-Object Diagnostics.ProcessStartInfo $ClangCxx, "-### `"@$rsp`"" + $start.UseShellExecute = $false + $start.RedirectStandardError = $true + $start.RedirectStandardOutput = $true + $process = [Diagnostics.Process]::Start($start) + $stdout = $process.StandardOutput.ReadToEndAsync() + $stderr = $process.StandardError.ReadToEnd() + $process.WaitForExit() + [void]$stdout.Result + if ($process.ExitCode -ne 0) { throw "clang++ -### failed for the kit link:`n$stderr" } + $line = @($stderr -split "`r?`n" | Where-Object { $_ -match '^\s*"[^"]*ld(\.lld)?(\.exe)?"' }) | Select-Object -Last 1 + if (-not $line) { throw "clang++ -### printed no linker command:`n$stderr" } + + $lld = New-Object System.Collections.Generic.List[string] + $driverArguments = Split-DriverCommand $line + for ($i = 1; $i -lt $driverArguments.Length; $i++) { + $argument = $driverArguments[$i].Replace('\', '/') + if ($argument -eq "$probe/libmain.so") { $lld.Add('{output}'); continue } + $slot = @($markers.Keys | Where-Object { $markers[$_] -eq $argument }) + if ($slot.Count -eq 1) { $lld.Add("{game:$($slot[0])}"); continue } + $argument = [regex]::Replace($argument, [regex]::Escape($kit), '{kit}', 'IgnoreCase') + $argument = [regex]::Replace($argument, [regex]::Escape($ndk), '{ndk}', 'IgnoreCase') + if ($argument -match '[A-Za-z]:/') { throw "The kit link names a path outside the kit and the NDK: $argument" } + $lld.Add($argument) + } + foreach ($required in '{output}', '{game:runtime}', '{game:product}', '{game:translated}') { + if (-not $lld.Contains($required)) { throw "The expanded kit link lacks $required" } + } + return , $lld.ToArray() + } finally { + Remove-Item -Recurse -Force $probe + } +} + function Export-QuestGameKit { [CmdletBinding()] param( @@ -239,6 +333,16 @@ function Export-QuestGameKit { } } $linkFlags.Add('-Wl,-soname,libmain.so') + $lld = Get-KitLldArguments -ClangCxx (Join-Path (Split-Path -Parent $LlvmStrip) 'clang++.exe') -KitDir $OutputDir ` + -Flags $linkFlags.ToArray() -Inputs $inputs.ToArray() + + # What the headset's translator needs besides the disc: the game manifest and symbol map, and + # the runtime sources it indexes for native registrations, which must be the ones compiled into + # this kit. + $translation = Join-Path $OutputDir 'translation' + New-Item -ItemType Directory -Force (Join-Path $translation 'projects/mkwii'), (Join-Path $translation 'runtime') | Out-Null + Copy-Item (Join-Path $RepoRoot 'projects/mkwii/recomp.yml'), (Join-Path $RepoRoot 'projects/mkwii/MAP.txt') (Join-Path $translation 'projects/mkwii') + Copy-Item -Recurse (Join-Path $RepoRoot 'runtime/src') (Join-Path $translation 'runtime/src') $recipe = [ordered]@{ schema = $script:KitSchema @@ -247,12 +351,11 @@ function Export-QuestGameKit { target = $target runtimeIncludeFingerprint = Get-RuntimeIncludeFingerprint (Join-Path $OutputDir 'include') compile = $compile - link = [ordered]@{ flags = $linkFlags.ToArray(); inputs = $inputs.ToArray() } + link = [ordered]@{ flags = $linkFlags.ToArray(); inputs = $inputs.ToArray(); lld = $lld } } $recipeJson = $recipe | ConvertTo-Json -Depth 8 -Compress - $files = Get-ChildItem -Recurse -File $OutputDir | Sort-Object FullName | ForEach-Object { - $relative = $_.FullName.Substring((Resolve-Path $OutputDir).Path.Length).Replace('\', '/') - "$relative $(Get-Sha256Hex $_.FullName)" + $files = Get-RelativeFiles $OutputDir | Sort-Object FullName | ForEach-Object { + "/$($_.Relative) $(Get-Sha256Hex $_.FullName)" } $recipe.fingerprint = Get-StringSha256Hex ($recipeJson + "`n" + ($files -join "`n")) [IO.File]::WriteAllText((Join-Path $OutputDir 'kit.json'), ($recipe | ConvertTo-Json -Depth 8), (New-Object Text.UTF8Encoding $false)) @@ -404,6 +507,7 @@ function New-QuestGamePackage { } Add-Type -AssemblyName System.IO.Compression, System.IO.Compression.FileSystem if (Test-Path $OutputPath) { Remove-Item -Force $OutputPath } + New-Item -ItemType Directory -Force (Split-Path -Parent ([IO.Path]::GetFullPath($OutputPath))) | Out-Null $zip = [IO.Compression.ZipFile]::Open($OutputPath, 'Create') try { $entry = $zip.CreateEntry('game.json') @@ -411,10 +515,8 @@ function New-QuestGamePackage { try { $writer.Write(($game | ConvertTo-Json)) } finally { $writer.Dispose() } [void][IO.Compression.ZipFileExtensions]::CreateEntryFromFile($zip, $Library, $recipe.output, 'Optimal') if ($DataDir) { - $dataRoot = (Resolve-Path $DataDir).Path.TrimEnd('\', '/') - foreach ($file in Get-ChildItem -Recurse -File $dataRoot) { - $name = 'DATA/' + $file.FullName.Substring($dataRoot.Length + 1).Replace('\', '/') - [void][IO.Compression.ZipFileExtensions]::CreateEntryFromFile($zip, $file.FullName, $name, 'NoCompression') + foreach ($file in Get-RelativeFiles $DataDir) { + [void][IO.Compression.ZipFileExtensions]::CreateEntryFromFile($zip, $file.FullName, "DATA/$($file.Relative)", 'NoCompression') } } } finally { $zip.Dispose() } diff --git a/android/README.md b/android/README.md index 9d3dd66..bc64a23 100644 --- a/android/README.md +++ b/android/README.md @@ -4,7 +4,8 @@ Standalone Android/OpenXR build of the Mario Kart Wii recompilation for Quest 2, Quest 3, Quest 3S and Quest Pro. The full design, build walkthrough and current status live in [docs/quest-port.md](../docs/quest-port.md); this directory only holds the Gradle project, its helper scripts, the game kit tooling -(`QuestGameKit.psm1`, `Build-QuestGame.ps1`) and `nod-jni`. +(`QuestGameKit.psm1`, `Build-QuestGame.ps1`), the on-headset build toolchain +(`Prepare-QuestToolchain.ps1`, `toolchain/`) and `nod-jni`. ```powershell powershell -ExecutionPolicy Bypass -File android/Prepare-QuestDependencies.ps1 # stages the SDL3 3.4.4 AAR once @@ -16,7 +17,8 @@ The translated game (the translator's `generated/` tree for **your own** PAL `RMCP01` disc) is passed with `-Generated `; it defaults to the installer workspace next to this checkout. No game data is ever part of the APK, and neither is any translated game code: the base APK carries a game kit, -and your `libmain.so` is built from your own disc and imported +and your `libmain.so` is built from your own disc, either on the headset +(**Build on this Quest**, about half an hour) or on a PC and imported (`Build-QuestGame.ps1 -Install`, or **Import from computer** with its `.wcgame`). Copy your disc image to the headset (for example `adb push MarioKart.iso /sdcard/Download/`) and press **Select disc image** in the launcher, which checks diff --git a/android/app/build.gradle.kts b/android/app/build.gradle.kts index b9eb5c7..393c7bc 100644 --- a/android/app/build.gradle.kts +++ b/android/app/build.gradle.kts @@ -229,8 +229,52 @@ abstract class ExportQuestGameKit : DefaultTask() { } } +/** + * The toolchain for building the game on the headset (android/Prepare-QuestToolchain.ps1): the + * translator for Android, clang/lld, and the pin of the NDK files the headset downloads. Packaged + * into the base variants' assets as quest_toolchain/. + */ +abstract class PrepareQuestToolchain : DefaultTask() { + @get:OutputDirectory + abstract val outputDir: DirectoryProperty + + @get:InputFiles + @get:PathSensitive(PathSensitivity.RELATIVE) + abstract val sources: ConfigurableFileCollection + + @get:Internal + abstract val script: RegularFileProperty + + @get:Internal + abstract val ndkLlvm: DirectoryProperty + + @get:Inject + abstract val execOperations: ExecOperations + + @TaskAction + fun prepare() { + execOperations.exec { + commandLine( + "powershell", "-NoProfile", "-ExecutionPolicy", "Bypass", "-File", script.get().asFile.path, + "-OutputDir", File(outputDir.get().asFile, "quest_toolchain").path, "-NdkLlvm", ndkLlvm.get().asFile.path, + ) + } + } +} + androidComponents { onVariants(selector().withFlavor("profile" to "base")) { variant -> + val prepareToolchain = tasks.register( + "prepare${variant.name.replaceFirstChar { it.uppercase() }}QuestToolchain" + ) { + script.set(rootProject.layout.projectDirectory.file("Prepare-QuestToolchain.ps1")) + sources.from(script, rootProject.layout.projectDirectory.dir("toolchain")) + sources.from(fileTree(File(mkwRepoRoot, "translator/src")) { exclude("**/bin/**", "**/obj/**") }) + sources.from(fileTree(File(mkwRepoRoot, "Launcher/WiiCompiled.Setup.Common")) { exclude("**/bin/**", "**/obj/**") }) + val host = if (System.getProperty("os.name").startsWith("Windows")) "windows-x86_64" else "linux-x86_64" + ndkLlvm.set(sdkComponents.ndkDirectory.map { it.dir("toolchains/llvm/prebuilt/$host") }) + } + variant.sources.assets?.addGeneratedSourceDirectory(prepareToolchain, PrepareQuestToolchain::outputDir) val capitalized = variant.name.replaceFirstChar { it.uppercase() } // AGP packages every library in the CMake output directory, so a game library left over // from a build that still linked the game would otherwise ship in the APK. diff --git a/android/app/src/main/java/org/wiicompiled/quest/launcher/BuildRecipe.kt b/android/app/src/main/java/org/wiicompiled/quest/launcher/BuildRecipe.kt new file mode 100644 index 0000000..969a715 --- /dev/null +++ b/android/app/src/main/java/org/wiicompiled/quest/launcher/BuildRecipe.kt @@ -0,0 +1,89 @@ +package org.wiicompiled.quest.launcher + +import java.security.MessageDigest + +/** + * The text-in, text-out parts of building the game on the headset ([GameBuild]): reading the + * translator's outputs and turning the game kit's recipe (kit.json) into compiler and linker + * command lines. They mirror android/QuestGameKit.psm1, which does the same on a PC. + */ +object BuildRecipe { + + /** The translated source lists emit-build-shards writes into shards.cmake, in link order. */ + val TRANSLATED_LISTS = listOf("MKW_BASE_COMMON_SHARDS", "MKW_BASE_PORTABLE_SENSITIVE_SHARDS") + const val REGISTRATION_LIST = "MKW_BASE_REGISTRATION_SOURCES" + + /** + * One argument of a response file for clang or lld on Android, which read them with GNU + * quoting: a backslash escapes the next character. + */ + fun responseFileArgument(argument: String): String = buildString { + for (c in argument) { + if (c == '\\' || c == '"' || c == '\'' || c.isWhitespace()) append('\\') + append(c) + } + } + + fun responseFile(arguments: List): String = + arguments.joinToString(separator = "\n", postfix = "\n") { responseFileArgument(it) } + + /** Replaces the kit recipe's {name} placeholders; an unknown placeholder is an error. */ + fun expand(argument: String, values: Map): String { + var result = argument + for ((name, value) in values) result = result.replace("{$name}", value) + PLACEHOLDER.find(result)?.let { throw IllegalArgumentException("Unexpanded ${it.value} in the game kit recipe") } + return result + } + + /** + * The link.lld arguments with every {game:slot} replaced by that slot's object files + * (runtime, product, translated) and the other placeholders expanded. + */ + fun linkArguments(template: List, values: Map, objects: Map>): List { + val arguments = ArrayList(template.size + objects.values.sumOf { it.size }) + for (argument in template) { + val slot = GAME_SLOT.matchEntire(argument)?.groupValues?.get(1) + if (slot != null) { + arguments += objects[slot] ?: throw IllegalArgumentException("The game kit recipe names an unknown slot {game:$slot}") + } else { + arguments += expand(argument, values) + } + } + return arguments + } + + /** The quoted entries of `set(NAME ...)` in a shards.cmake, or an empty list without it. */ + fun sourceList(shardsCmake: String, name: String): List { + val body = Regex("""set\(${Regex.escape(name)}\s(.*?)\)""", RegexOption.DOT_MATCHES_ALL).find(shardsCmake) + ?.groupValues?.get(1) ?: return emptyList() + return Regex("\"([^\"]+)\"").findAll(body).map { it.groupValues[1].replace('\\', '/') }.toList() + } + + /** The first translation entry point in recomp.yml, as the translator expects it (0x...). */ + fun entryPoint(manifest: String): String = + Regex("""\n\s*entry_points:\s*\n\s*-\s*(0x[0-9A-Fa-f]{8})""").find(manifest)?.groupValues?.get(1) + ?: throw IllegalArgumentException("recomp.yml names no translation entry point") + + /** + * The blob assembly for ELF, as runtime/cmake/PublicProducts.cmake rewrites a Windows-generated + * one. The headset's translator already writes ELF sections, so this normally changes nothing. + */ + fun elfBlobAssembly(text: String): String { + val elf = text.replace(".section .rdata,\"dr\"", ".section .rodata,\"a\",@progbits") + return if (elf == text) text else elf + "\n.section .note.GNU-stack,\"\",@progbits\n" + } + + /** An object file name for a generated source: its stem, which the translator keeps unique. */ + fun objectName(source: String): String = source.substringAfterLast('/').substringBeforeLast('.') + ".o" + + /** SHA-256 over "path sha256" lines sorted by path (ordinal), joined with \n. */ + fun listDigest(files: Map): String { + val lines = files.entries.map { "${it.key} ${it.value}" }.sorted() + return hex(MessageDigest.getInstance("SHA-256").digest(lines.joinToString("\n").toByteArray(Charsets.UTF_8))) + } + + fun hex(bytes: ByteArray): String = bytes.joinToString("") { "%02x".format(it) } + + private val PLACEHOLDER = Regex("""\{[a-z:]+\}""") + private val GAME_SLOT = Regex("""\{game:(\w+)\}""") +} diff --git a/android/app/src/main/java/org/wiicompiled/quest/launcher/GameBuild.kt b/android/app/src/main/java/org/wiicompiled/quest/launcher/GameBuild.kt new file mode 100644 index 0000000..79f85c7 --- /dev/null +++ b/android/app/src/main/java/org/wiicompiled/quest/launcher/GameBuild.kt @@ -0,0 +1,493 @@ +package org.wiicompiled.quest.launcher + +import android.app.ActivityManager +import android.content.Context +import android.os.StatFs +import android.util.Log +import java.io.File +import java.io.IOException +import java.io.InterruptedIOException +import java.net.HttpURLConnection +import java.net.URL +import java.security.MessageDigest +import java.text.SimpleDateFormat +import java.util.Date +import java.util.Locale +import java.util.TimeZone +import java.util.concurrent.Executors +import java.util.concurrent.Future +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicReference +import java.util.zip.ZipInputStream +import org.json.JSONArray +import org.json.JSONObject +import org.wiicompiled.quest.BuildConfig +import org.wiicompiled.quest.GameLibrary +import org.wiicompiled.quest.GameStorage + +/** + * Builds the game on the headset from the player's extracted disc, the way the PC launcher's + * Build for Quest does (android/Build-QuestGame.ps1), for players without a PC: + * + * 1. unpack the toolchain (assets/quest_toolchain, android/Prepare-QuestToolchain.ps1) and the + * game kit (assets/game_kit) into private storage; + * 2. download the Android NDK files the build needs from Google, checked against the pins the + * toolchain carries (ndk.json); + * 3. translate main.dol and StaticR.rel: translate-recursive, generate-data-init, emit-build-shards; + * 4. compile the generated sources with the kit's flags, several at a time; + * 5. link them with the kit's objects and archives (kit.json link.lld); + * 6. install libmain.so and its game.json like an imported game. + * + * About half an hour on a Quest 3. Work survives a cancelled or failed run: a translation whose + * inputs have not changed is reused and finished objects are not compiled again. A successful + * build removes everything it unpacked and generated. + */ +object GameBuild { + + enum class Step { Prepare, Download, Translate, Compile, Link, Install } + + /** [permille] of the whole build; [done] of [total] items in [step]. False stops the build. */ + fun interface Reporter { + fun update(permille: Int, step: Step, done: Int, total: Int): Boolean + } + + private const val TAG = "WiiCompiledLauncher" + private const val TOOLCHAIN_ASSETS = "quest_toolchain" + private const val KIT_ASSETS = "game_kit" + private const val MARKER = ".complete" + private const val REQUIRED_SPACE = 2_500_000_000L + // Translation peaks near 3.1 GB of memory with the runtime's default heap; this limit keeps it + // near 2.1 GB for the same output, at some cost in time. + private const val MONO_GC_PARAMS = "soft-heap-limit=1200m" + private const val TRANSLATOR_THREADS = 4 + private const val EXPECTED_TRANSLATION_SECONDS = 600.0 + private const val COMPILE_MEMORY_BYTES = 700L * 1024 * 1024 + + /** Null on success, otherwise the message to show. */ + fun run(context: Context, reporter: Reporter, cancelled: () -> Boolean, finishing: () -> Unit): String? { + val disc = GameStorage.discDirectory(context) + if (GameStorage.discStatus(context) != GameStorage.DiscStatus.Ready) { + return "The game files (DATA) are needed to build the game. Select your disc image first." + } + GameFiles.validateData(disc)?.let { return it } + val root = File(context.filesDir, "build") + root.mkdirs() + val available = StatFs(root.absolutePath).availableBytes + if (available < REQUIRED_SPACE) { + return "Not enough free space: building needs ${GameFiles.gigabytes(REQUIRED_SPACE)}, and ${GameFiles.gigabytes(available)} is free." + } + + val stamp = SimpleDateFormat("yyyyMMdd_HHmmss", Locale.US).format(Date()) + val log = BuildLog(File(GameStorage.logsDirectory(context), "build_$stamp.log")) + try { + return Build(context, root, disc, reporter, cancelled, finishing, log).run() + } catch (e: InterruptedIOException) { + throw e + } catch (e: Exception) { + Log.e(TAG, "Game build failed", e) + log.line("Build failed: $e") + return "${e.message ?: e.toString()}\n\nThe build log is ${log.file.absolutePath}" + } finally { + log.close() + } + } + + private class Build( + val context: Context, + val root: File, + val disc: File, + val reporter: Reporter, + val cancelled: () -> Boolean, + val finishing: () -> Unit, + val log: BuildLog, + ) { + val toolchain = File(root, "toolchain") + val ndk = File(root, "ndk") + val kit = File(root, "kit") + val work = File(root, GameLibrary.directory(context).name) + val workspace = File(work, "ws") + val objects = File(work, "obj") + val temporary = File(root, "tmp") + val failure = AtomicReference(null) + + fun run(): String? { + log.line("WiiCompiled Quest ${BuildConfig.VERSION_NAME} building ${BuildConfig.PROFILE} on this headset") + report(0, Step.Prepare) + val toolchainManifest = JSONObject(context.assets.open("$TOOLCHAIN_ASSETS/toolchain.json").reader().use { it.readText() }) + unpackToolchain(toolchainManifest) + report(30, Step.Prepare) + val kitFingerprint = GameLibrary.kitFingerprint(context) ?: return "This app carries no game kit." + unpackKit(kitFingerprint) + val recipe = JSONObject(File(kit, "kit.json").readText()) + report(40, Step.Prepare) + downloadNdk(JSONObject(context.assets.open("$TOOLCHAIN_ASSETS/${toolchainManifest.getString("ndk")}").reader().use { it.readText() })) + + temporary.mkdirs() + val tools = ToolProcess( + mapOf( + "LD_LIBRARY_PATH" to File(toolchain, "llvm/lib").absolutePath, + "HOME" to temporary.absolutePath, + "TMPDIR" to temporary.absolutePath, + "MONO_GC_PARAMS" to MONO_GC_PARAMS, + ), + log, + cancelled = { cancelled() || failure.get() != null }, + ) + translate(tools, "$kitFingerprint ${toolchainManifest.getString("fingerprint")}")?.let { return it } + val objectsBySlot = compile(tools, recipe, toolchainManifest.getString("fingerprint")) + failure.get()?.let { return it } + val library = link(tools, recipe, objectsBySlot) ?: return failure.get() ?: "Linking the game failed." + + finishing() + report(990, Step.Install) + install(library, kitFingerprint)?.let { return it } + log.line("Installed the game built on this headset") + // Everything above is only needed again after an app update, which changes the kit. + if (!root.deleteRecursively()) Log.w(TAG, "Could not remove all of ${root.absolutePath}") + return null + } + + fun report(permille: Int, step: Step, done: Int = 0, total: Int = 0) { + if (!reporter.update(permille, step, done, total) || cancelled()) throw InterruptedIOException("Build cancelled") + } + + /** Unpacks files.zip, which holds exactly the files toolchain.json lists. */ + fun unpackToolchain(manifest: JSONObject) { + val fingerprint = manifest.getString("fingerprint") + if (File(toolchain, MARKER).takeIf { it.isFile }?.readText() == fingerprint) return + toolchain.deleteRecursively() + val listed = manifest.getJSONArray("files") + val sizes = HashMap() + for (i in 0 until listed.length()) { + val file = listed.getJSONObject(i) + sizes[file.getString("path")] = file.getLong("size") + } + var unpacked = 0 + ZipInputStream(context.assets.open("$TOOLCHAIN_ASSETS/${manifest.getString("archive")}").buffered(1 shl 20)).use { zip -> + while (true) { + val entry = zip.nextEntry ?: break + if (entry.isDirectory) continue + val expected = sizes[entry.name] ?: throw IOException("The app's toolchain holds an unlisted file ${entry.name}.") + val target = File(toolchain, entry.name) + target.parentFile?.mkdirs() + target.outputStream().use { zip.copyTo(it, 1 shl 20) } + if (target.length() != expected) throw IOException("The app's toolchain file ${entry.name} is damaged.") + unpacked++ + report(30 * unpacked / sizes.size, Step.Prepare) + } + } + if (unpacked != sizes.size) throw IOException("The app's toolchain is incomplete.") + File(toolchain, MARKER).writeText(fingerprint) + } + + fun unpackKit(fingerprint: String) { + if (File(kit, MARKER).takeIf { it.isFile }?.readText() == fingerprint) return + kit.deleteRecursively() + copyAssetTree(KIT_ASSETS, kit) + File(kit, MARKER).writeText(fingerprint) + } + + fun copyAssetTree(asset: String, target: File) { + val children = context.assets.list(asset).orEmpty() + if (children.isEmpty()) { + copyAsset(asset, target) + return + } + for (child in children) copyAssetTree("$asset/$child", File(target, child)) + } + + fun copyAsset(asset: String, target: File) { + target.parentFile?.mkdirs() + context.assets.open(asset).use { input -> target.outputStream().use { input.copyTo(it, 1 shl 20) } } + } + + /** The NDK subset ndk.json lists, from Google's zip, each file checked against its SHA-256. */ + fun downloadNdk(subset: JSONObject) { + val digest = subset.getString("digest") + if (File(ndk, MARKER).takeIf { it.isFile }?.readText() == digest) return + report(40, Step.Download) + ndk.deleteRecursively() + val url = subset.getString("url") + val prefix = subset.getString("prefix") + val expected = HashMap() + val listed: JSONArray = subset.getJSONArray("files") + for (i in 0 until listed.length()) { + val file = listed.getJSONObject(i) + expected[prefix + file.getString("path")] = file.getString("sha256") + } + log.line("Downloading ${expected.size} Android NDK files from $url") + val zip = RemoteZip(subset.getLong("size")) { start, end -> fetchRange(url, start, end) } + val wanted = zip.entries().filter { it.name in expected } + if (wanted.size != expected.size) throw IOException("Google's Android NDK download no longer has the expected files.") + val totalBytes = wanted.sumOf { it.end - it.offset } + var doneBytes = 0L + val actual = HashMap() + zip.read(wanted) { entry, bytes -> + val sha256 = BuildRecipe.hex(MessageDigest.getInstance("SHA-256").digest(bytes)) + if (sha256 != expected[entry.name]) throw IOException("The downloaded Android NDK file ${entry.name} is damaged. Try again.") + val path = entry.name.removePrefix(prefix) + File(ndk, path).apply { parentFile?.mkdirs() }.writeBytes(bytes) + actual[path] = sha256 + doneBytes += entry.end - entry.offset + report(40 + (10 * doneBytes / totalBytes).toInt(), Step.Download, (doneBytes shr 20).toInt(), (totalBytes shr 20).toInt()) + } + if (BuildRecipe.listDigest(actual) != digest) throw IOException("The downloaded Android NDK files do not match this app's pins.") + File(ndk, MARKER).writeText(digest) + } + + fun fetchRange(url: String, start: Long, end: Long): ByteArray { + if (cancelled()) throw InterruptedIOException("Build cancelled") + val connection = URL(url).openConnection() as HttpURLConnection + try { + connection.connectTimeout = 30_000 + connection.readTimeout = 60_000 + connection.setRequestProperty("Range", "bytes=$start-$end") + // Android asks for gzip by default, and Google's server then serves a gzip-encoded + // zip whose ranges are not the file's (HTTP 416 near the end). + connection.setRequestProperty("Accept-Encoding", "identity") + val code = try { + connection.responseCode + } catch (e: IOException) { + throw IOException("The Android NDK files could not be downloaded from Google. Check the headset's internet connection.", e) + } + if (code != HttpURLConnection.HTTP_PARTIAL) throw IOException("Google's server answered $code to the Android NDK download.") + val bytes = connection.inputStream.use { it.readBytes() } + if (bytes.size.toLong() != end - start + 1) throw IOException("The Android NDK download was cut short. Try again.") + return bytes + } finally { + connection.disconnect() + } + } + + /** Translates the disc unless this workspace already holds a translation of the same inputs. */ + fun translate(tools: ToolProcess, identity: String): String? { + val generated = File(workspace, "generated") + val provenance = File(generated, "translation-provenance.txt") + val expected = "$identity ${BuildConfig.DISC_DOL_SHA256} ${BuildConfig.DISC_REL_SHA256}" + val shards = File(generated, "build_shards/shards.cmake") + if (provenance.isFile && provenance.readText() == expected && shards.isFile) { + log.line("Reusing the translation of an earlier build") + report(400, Step.Translate) + return null + } + provenance.delete() + val project = File(workspace, "projects/mkwii") + File(kit, "translation/projects/mkwii").copyRecursively(project, overwrite = true) + File(workspace, "runtime").deleteRecursively() + File(kit, "translation/runtime").copyRecursively(File(workspace, "runtime"), overwrite = true) + File(disc, DiscChecks.DOL_PATH).copyTo(File(workspace, "Assets/main.dol"), overwrite = true) + File(disc, DiscChecks.REL_PATH).copyTo(File(workspace, "Assets/StaticR.rel"), overwrite = true) + val manifest = "projects/mkwii/recomp.yml" + val entryPoint = BuildRecipe.entryPoint(File(workspace, manifest).readText()) + + report(50, Step.Translate, 1, 3) + val timing = Regex("""\(t=([0-9.]+)s\)""") + translator( + tools, "translating the game", + "translate-recursive", entryPoint, "--project", manifest, "--outdir", "generated/functions", + "--output-metadata", "generated/base_translation_output.json", + "--production-source-bundle", "generated/base_translation_sources.bin", + "--no-function-files", "--prune-stale", "--threads", TRANSLATOR_THREADS.toString(), + ) { line -> + timing.find(line)?.groupValues?.get(1)?.toDoubleOrNull()?.let { seconds -> + val fraction = minOf(seconds / EXPECTED_TRANSLATION_SECONDS, 0.97) + reporter.update(50 + (320 * fraction).toInt(), Step.Translate, 1, 3) + } + }?.let { return it } + report(370, Step.Translate, 2, 3) + translator(tools, "generating the game data", "generate-data-init", "--project", manifest, "--target-os", "android")?.let { return it } + report(385, Step.Translate, 3, 3) + translator( + tools, "preparing the build", + "emit-build-shards", "--project", manifest, "--base-metadata", "generated/base_translation_output.json", + "--base-functions-dir", "generated/functions", "--native-source-dir", "runtime/src", "--out", "generated/build_shards", + )?.let { return it } + provenance.writeText(expected) + report(400, Step.Translate, 3, 3) + return null + } + + fun translator(tools: ToolProcess, what: String, vararg arguments: String, onLine: (String) -> Unit = {}): String? { + log.line("== translator ${arguments.first()}") + val result = tools.run( + workspace, + File(toolchain, "translator/translator_host"), + listOf(File(toolchain, "translator").absolutePath, "Translator.Cli") + arguments, + onLine, + ) + if (result.exitCode == 0) return null + return if (result.killed) { + "The headset ran out of memory while $what. Close other apps, then build again." + } else { + "The translator failed while $what (exit ${result.exitCode}).\n${result.tail.takeLast(6).joinToString("\n")}" + } + } + + /** Compiles every generated source; returns their objects by link slot (runtime, product, translated). */ + fun compile(tools: ToolProcess, recipe: JSONObject, toolchainFingerprint: String): Map> { + val generated = File(workspace, "generated") + val shards = File(generated, "build_shards/shards.cmake").readText() + val stamp = File(objects, ".stamp") + val identity = "${recipe.getString("fingerprint")} $toolchainFingerprint" + if (stamp.takeIf { it.isFile }?.readText() != identity) { + objects.deleteRecursively() + objects.mkdirs() + stamp.writeText(identity) + } + + val blob = File(generated, "data_sections_init_blobs.S") + val elfBlob = File(work, "data_sections_init_blobs_android.S") + val blobText = blob.readText() + val elfText = BuildRecipe.elfBlobAssembly(blobText) + if (!elfBlob.isFile || elfBlob.readText() != elfText) elfBlob.writeText(elfText) + + data class Job(val kind: String, val source: File, val slot: String) + val jobs = ArrayList() + jobs += Job("runtime", File(generated, "data_sections_init.cpp"), "runtime") + jobs += Job("runtime", File(generated, "guest_symbol_table.cpp"), "runtime") + jobs += Job("asm", elfBlob, "runtime") + BuildRecipe.sourceList(shards, BuildRecipe.REGISTRATION_LIST).forEach { jobs += Job("product", File(it), "product") } + val translated = BuildRecipe.TRANSLATED_LISTS.flatMap { BuildRecipe.sourceList(shards, it) } + if (translated.isEmpty()) throw IOException("The translation produced no game code.") + translated.forEach { jobs += Job("translated", File(it), "translated") } + if (jobs.map { BuildRecipe.objectName(it.source.path) }.toSet().size != jobs.size) { + throw IOException("Two generated sources share an object name.") + } + + val values = mapOf( + "kit" to kit.absolutePath, + "sysroot" to File(ndk, "sysroot").absolutePath, + "workspace" to workspace.absolutePath, + ) + val compileFlags = recipe.getJSONObject("compile") + for (kind in listOf("runtime", "asm", "product", "translated")) { + val flags = compileFlags.getJSONArray(kind).let { array -> (0 until array.length()).map { BuildRecipe.expand(array.getString(it), values) } } + File(work, "$kind.rsp").writeText(BuildRecipe.responseFile(flags)) + } + + val clang = File(toolchain, "llvm/bin/clang-21") + val resourceDir = File(toolchain, "llvm/lib/clang/21").absolutePath + val pending = jobs.filter { job -> + val obj = File(objects, BuildRecipe.objectName(job.source.path)) + !(obj.isFile && obj.length() > 0 && obj.lastModified() >= job.source.lastModified()) + } + val finished = AtomicInteger(jobs.size - pending.size) + log.line("== compiling ${pending.size} of ${jobs.size} sources") + val pool = Executors.newFixedThreadPool(compileJobs()) + try { + val futures: List> = pending.map { job -> + pool.submit { + if (failure.get() != null || cancelled()) return@submit + val obj = File(objects, BuildRecipe.objectName(job.source.path)) + val partial = File(objects, obj.name + ".partial") + // The blob assembly needs no preprocessing, and preprocessing would make clang + // start itself again, which Android does not allow here. + val language = if (job.kind == "asm") listOf("--driver-mode=gcc", "-x", "assembler") else listOf("--driver-mode=g++") + val result = tools.run( + work, clang, + language + listOf( + "-resource-dir", resourceDir, "@" + File(work, "${job.kind}.rsp").absolutePath, + "-c", job.source.absolutePath, "-o", partial.absolutePath, + ), + ) + if (result.exitCode != 0) { + partial.delete() + failure.compareAndSet( + null, + if (result.killed) "The headset ran out of memory while compiling the game. Close other apps, then build again." + else "Compiling ${job.source.name} failed.\n${result.tail.takeLast(6).joinToString("\n")}", + ) + return@submit + } + if (!partial.renameTo(obj)) failure.compareAndSet(null, "Could not store ${obj.name}.") + finished.incrementAndGet() + } + } + while (futures.any { !it.isDone }) { + report(400 + 570 * finished.get() / jobs.size, Step.Compile, finished.get(), jobs.size) + TimeUnit.MILLISECONDS.sleep(500) + } + futures.forEach { future -> + try { + future.get() + } catch (e: java.util.concurrent.ExecutionException) { + val cause = e.cause + // A failed compile stops the others, which is not the player cancelling. + if (cause is InterruptedIOException && failure.get() == null) throw cause + failure.compareAndSet(null, cause?.message ?: cause.toString()) + } + } + } finally { + pool.shutdownNow() + } + if (failure.get() == null) report(970, Step.Compile, jobs.size, jobs.size) + return jobs.groupBy({ it.slot }, { File(objects, BuildRecipe.objectName(it.source.path)).absolutePath }) + } + + fun compileJobs(): Int { + val memory = ActivityManager.MemoryInfo() + context.getSystemService(ActivityManager::class.java)?.getMemoryInfo(memory) + val byMemory = (memory.availMem / COMPILE_MEMORY_BYTES).toInt() + return minOf(4, Runtime.getRuntime().availableProcessors(), byMemory).coerceAtLeast(1) + } + + fun link(tools: ToolProcess, recipe: JSONObject, objectsBySlot: Map>): File? { + report(970, Step.Link) + val library = File(work, recipe.getString("output")) + library.delete() + val template = recipe.getJSONObject("link").getJSONArray("lld").let { array -> (0 until array.length()).map { array.getString(it) } } + val arguments = BuildRecipe.linkArguments( + template, + mapOf("kit" to kit.absolutePath, "ndk" to ndk.absolutePath, "output" to library.absolutePath), + objectsBySlot, + ) + val rsp = File(work, "link.rsp").apply { writeText(BuildRecipe.responseFile(arguments)) } + log.line("== linking") + val result = tools.run(work, File(toolchain, "llvm/bin/ld.lld"), listOf("@" + rsp.absolutePath)) + if (result.exitCode != 0 || !library.isFile) { + failure.compareAndSet(null, "Linking the game failed.\n${result.tail.takeLast(6).joinToString("\n")}") + return null + } + return library + } + + fun install(library: File, kitFingerprint: String): String? { + val destination = GameLibrary.directory(context) + val staging = File(destination.parentFile, "${destination.name}.building") + staging.deleteRecursively() + staging.mkdirs() + try { + val installed = File(staging, GameLibrary.LIBRARY_NAME) + library.copyTo(installed) + val digest = MessageDigest.getInstance("SHA-256") + installed.inputStream().use { input -> + val buffer = ByteArray(1 shl 20) + while (true) { + val read = input.read(buffer) + if (read < 0) break + digest.update(buffer, 0, read) + } + } + val builtAt = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US).apply { timeZone = TimeZone.getTimeZone("UTC") }.format(Date()) + val manifest = JSONObject() + .put("schema", 1) + .put("profile", BuildConfig.PROFILE) + .put("gameId", BuildConfig.DISC_GAME_ID) + .put("dolSha256", BuildConfig.DISC_DOL_SHA256) + .put("relSha256", BuildConfig.DISC_REL_SHA256) + .put("kitFingerprint", kitFingerprint) + .put("library", GameLibrary.LIBRARY_NAME) + .put("librarySha256", BuildRecipe.hex(digest.digest())) + .put("includesData", false) + .put("builtBy", "this headset (WiiCompiled Quest ${BuildConfig.VERSION_NAME})") + .put("builtAt", builtAt) + File(staging, GameLibrary.MANIFEST_NAME).writeText(manifest.toString(2)) + destination.parentFile?.mkdirs() + return GameFiles.replace(destination, staging) + } finally { + staging.deleteRecursively() + } + } + } +} diff --git a/android/app/src/main/java/org/wiicompiled/quest/launcher/GameSetup.kt b/android/app/src/main/java/org/wiicompiled/quest/launcher/GameSetup.kt index 3676c02..ce7722f 100644 --- a/android/app/src/main/java/org/wiicompiled/quest/launcher/GameSetup.kt +++ b/android/app/src/main/java/org/wiicompiled/quest/launcher/GameSetup.kt @@ -15,18 +15,27 @@ import java.io.InterruptedIOException * - [Task.ExtractDisc]: the player's disc image becomes DATA ([DiscExtraction]). * - [Task.ImportPackage]: a .wcgame built on a PC becomes the game library, and DATA too when * the package carries it ([GamePackageImport]). + * - [Task.BuildGame]: the game library is built on the headset from DATA ([GameBuild]). * * Every task stages its output next to the destination and swaps it in only after it has been * checked, so a failed, cancelled or killed run never costs working files. */ object GameSetup { - enum class Task { ExtractDisc, ImportPackage } + enum class Task { ExtractDisc, ImportPackage, BuildGame } sealed interface State { data object Idle : State data class Checking(val task: Task) : State - data class Working(val task: Task, val done: Long, val total: Long) : State + /** A build also reports its [step] and how many of that step's items are [stepDone]. */ + data class Working( + val task: Task, + val done: Long, + val total: Long, + val step: GameBuild.Step? = null, + val stepDone: Int = 0, + val stepTotal: Int = 0, + ) : State data class Finishing(val task: Task) : State data class Done(val task: Task) : State data class Failed(val task: Task, val message: String) : State @@ -77,25 +86,33 @@ object GameSetup { } } - /** The whole task, on a worker thread. Always ends in Done, Failed or Cancelled. */ - fun run(context: Context, task: Task, uri: Uri, deleteSource: Boolean) { + /** The whole task, on a worker thread. Always ends in Done, Failed or Cancelled. [uri] is null only for a build. */ + fun run(context: Context, task: Task, uri: Uri?, deleteSource: Boolean) { val progress = Progress { done, total -> publish(State.Working(task, done, total)) !cancelRequested } val finishing = { publish(State.Finishing(task)) } try { - val failure = context.contentResolver.openFileDescriptor(uri, "r")?.use { descriptor -> - when (task) { - Task.ExtractDisc -> DiscExtraction.run(context, descriptor.fd, displayName(context, uri), progress, finishing) - Task.ImportPackage -> GamePackageImport.run(context, descriptor, progress, finishing) + val failure = if (task == Task.BuildGame) { + val reporter = GameBuild.Reporter { permille, step, done, total -> + publish(State.Working(task, permille.toLong(), 1000, step, done, total)) + !cancelRequested } - } ?: "The selected file could not be opened." + GameBuild.run(context, reporter, cancelled = { cancelRequested }, finishing) + } else { + uri?.let { context.contentResolver.openFileDescriptor(it, "r") }?.use { descriptor -> + when (task) { + Task.ExtractDisc -> DiscExtraction.run(context, descriptor.fd, displayName(context, uri), progress, finishing) + else -> GamePackageImport.run(context, descriptor, progress, finishing) + } + } ?: "The selected file could not be opened." + } if (failure != null) { publish(State.Failed(task, failure)) return } - if (deleteSource && uri.scheme == "file") { + if (deleteSource && uri?.scheme == "file") { uri.path?.let { File(it).delete() } } publish(State.Done(task)) diff --git a/android/app/src/main/java/org/wiicompiled/quest/launcher/GameSetupService.kt b/android/app/src/main/java/org/wiicompiled/quest/launcher/GameSetupService.kt index cbede93..9a6cd69 100644 --- a/android/app/src/main/java/org/wiicompiled/quest/launcher/GameSetupService.kt +++ b/android/app/src/main/java/org/wiicompiled/quest/launcher/GameSetupService.kt @@ -17,8 +17,8 @@ import org.wiicompiled.quest.R /** * Runs a [GameSetup] task as a foreground service. Extracting a disc or importing a game takes - * minutes, and a panel app's process is otherwise fair game once the panel is closed; taking the - * headset off also sleeps the CPU, hence the partial wake lock. + * minutes and building one half an hour, and a panel app's process is otherwise fair game once the + * panel is closed; taking the headset off also sleeps the CPU, hence the partial wake lock. */ class GameSetupService : Service() { @@ -41,7 +41,7 @@ class GameSetupService : Service() { ?: GameSetup.Task.ExtractDisc startForeground(NOTIFICATION_ID, notification(GameSetup.State.Checking(task)), ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC) val uri = intent?.data - if (uri == null || !GameSetup.begin(task)) { + if ((uri == null && task != GameSetup.Task.BuildGame) || !GameSetup.begin(task)) { // A task already going keeps the service, and ends it itself. if (!GameSetup.isRunning) { stopForeground(STOP_FOREGROUND_REMOVE) @@ -50,16 +50,16 @@ class GameSetupService : Service() { return START_NOT_STICKY } GameSetup.addListener(listener) - val deleteSource = intent.getBooleanExtra(EXTRA_DELETE_SOURCE, false) + val deleteSource = intent?.getBooleanExtra(EXTRA_DELETE_SOURCE, false) ?: false val wakeLock = getSystemService(PowerManager::class.java) .newWakeLock(PowerManager.PARTIAL_WAKE_LOCK, "WiiCompiled:GameSetup") - wakeLock.acquire(WAKE_LOCK_TIMEOUT_MS) + wakeLock.acquire(if (task == GameSetup.Task.BuildGame) BUILD_WAKE_LOCK_TIMEOUT_MS else WAKE_LOCK_TIMEOUT_MS) thread(name = "GameSetup") { try { GameSetup.run(applicationContext, task, uri, deleteSource) } finally { - releaseReadPermission(uri) + uri?.let(::releaseReadPermission) if (wakeLock.isHeld) wakeLock.release() mainExecutor.execute { GameSetup.removeListener(listener) @@ -88,17 +88,23 @@ class GameSetupService : Service() { } private fun notification(state: GameSetup.State): Notification { + val building = state is GameSetup.State.Checking && state.task == GameSetup.Task.BuildGame || + state is GameSetup.State.Working && state.task == GameSetup.Task.BuildGame val builder = Notification.Builder(this, CHANNEL_ID) .setSmallIcon(R.drawable.ic_wheel) - .setContentTitle(getString(R.string.disc_setup_notification_title)) + .setContentTitle(getString(if (building) R.string.build_notification_title else R.string.disc_setup_notification_title)) .setOngoing(true) .setOnlyAlertOnce(true) if (state is GameSetup.State.Working) { val percent = if (state.total > 0) (state.done * 100 / state.total).toInt() else 0 - val text = if (state.task == GameSetup.Task.ImportPackage) R.string.disc_setup_importing else R.string.disc_setup_extracting + val text = when (state.task) { + GameSetup.Task.ImportPackage -> R.string.disc_setup_importing + GameSetup.Task.BuildGame -> R.string.disc_setup_building + GameSetup.Task.ExtractDisc -> R.string.disc_setup_extracting + } builder.setContentText(getString(text, percent)).setProgress(100, percent, false) } else { - builder.setContentText(getString(R.string.disc_setup_checking)).setProgress(0, 0, true) + builder.setContentText(getString(if (building) R.string.build_preparing else R.string.disc_setup_checking)).setProgress(0, 0, true) } return builder.build() } @@ -112,11 +118,18 @@ class GameSetupService : Service() { private const val EXTRA_DELETE_SOURCE = "deleteSource" // Longer than any real task; only a hung run would reach it. private const val WAKE_LOCK_TIMEOUT_MS = 60L * 60 * 1000 + private const val BUILD_WAKE_LOCK_TIMEOUT_MS = 3L * 60 * 60 * 1000 /** * Starts [task] on [uri]: a document the player picked, or a file in the Import folder, * which [deleteSource] removes once it has been imported. */ + /** Builds the game from DATA on this headset ([GameBuild]). */ + fun startBuild(context: Context) { + val intent = Intent(context, GameSetupService::class.java).putExtra(EXTRA_TASK, GameSetup.Task.BuildGame.name) + context.startForegroundService(intent) + } + fun start(context: Context, task: GameSetup.Task, uri: Uri, deleteSource: Boolean = false) { if (uri.scheme == "content") { try { diff --git a/android/app/src/main/java/org/wiicompiled/quest/launcher/LauncherActivity.kt b/android/app/src/main/java/org/wiicompiled/quest/launcher/LauncherActivity.kt index 30a90e9..4ad2608 100644 --- a/android/app/src/main/java/org/wiicompiled/quest/launcher/LauncherActivity.kt +++ b/android/app/src/main/java/org/wiicompiled/quest/launcher/LauncherActivity.kt @@ -27,8 +27,8 @@ import org.wiicompiled.quest.R * * The APK carries no game code. Playing needs two things the player owns: the game files (DATA, * extracted from their disc image here or on a PC) and the game itself (libmain.so, built from - * their disc on a PC and brought over with Import from computer). Home's main button is always - * the next of those steps, and Play once both are there. + * their disc on this headset, or on a PC and brought over with Import from computer). Home's main + * button is always the next of those steps, and Play once both are there. * * The game runs as [QuestActivity], an immersive activity in its own `:game` process. SDL and the * runtime cannot start twice in one process, so that process ends with every game session, and @@ -39,7 +39,7 @@ class LauncherActivity : Activity() { private enum class Page { Home, Settings } /** What Home's main and secondary buttons do. */ - private enum class Action { Play, Resume, SelectDisc, ImportGame } + private enum class Action { Play, Resume, SelectDisc, ImportGame, BuildGame } private lateinit var navHome: View private lateinit var navSettings: View @@ -111,6 +111,7 @@ class LauncherActivity : Activity() { gameRunning = ::isGameRunning, selectDiscImage = ::selectDiscImage, importGame = ::importGame, + buildGame = ::buildGame, ) savedInstanceState?.getString(KEY_TAB)?.let { name -> SettingsPage.Tab.entries.firstOrNull { it.name == name }?.let(settings::select) @@ -124,6 +125,14 @@ class LauncherActivity : Activity() { val restored = savedInstanceState?.getString(KEY_PAGE)?.let { name -> Page.entries.firstOrNull { it.name == name } } showPage(restored ?: Page.Home) + + // Unattended headset tests (docs/quest-port.md): debug builds start a build from adb. + if (BuildConfig.DEBUG && savedInstanceState == null && intent.getBooleanExtra(EXTRA_DEBUG_BUILD_GAME, false) && + !GameSetup.isRunning && GameStorage.discStatus(this) == GameStorage.DiscStatus.Ready + ) { + Log.i(TAG, "Starting a game build requested over adb") + GameSetupService.startBuild(this) + } } override fun onResume() { @@ -193,27 +202,42 @@ class LauncherActivity : Activity() { val running = isGameRunning() val setup = GameSetup.state val settingUp = GameSetup.isRunning - val importing = setupTask(setup) == GameSetup.Task.ImportPackage + val task = setupTask(setup) + val importing = task == GameSetup.Task.ImportPackage + val building = task == GameSetup.Task.BuildGame + // Without anything yet, a player with only a headset starts from their disc image, and + // builds the game once its files are there; a PC-built game can always be imported instead. mainAction = when { running -> Action.Resume - gameStatus != GameLibrary.Status.Ready -> Action.ImportGame discStatus != GameStorage.DiscStatus.Ready -> Action.SelectDisc + gameStatus != GameLibrary.Status.Ready -> Action.BuildGame else -> Action.Play } - // Without anything yet, a player with only a headset starts from their disc image. secondaryAction = when { settingUp || running -> null - mainAction == Action.ImportGame && discStatus != GameStorage.DiscStatus.Ready -> Action.SelectDisc - mainAction == Action.SelectDisc -> Action.ImportGame + gameStatus != GameLibrary.Status.Ready || mainAction == Action.SelectDisc -> Action.ImportGame else -> null } playButton.isEnabled = !settingUp playIcon.setImageResource(if (!settingUp && (mainAction == Action.Play || mainAction == Action.Resume)) R.drawable.ic_play else R.drawable.ic_disc) playText.text = when { - setup is GameSetup.State.Checking -> getString(if (importing) R.string.home_checking_package else R.string.home_checking) - setup is GameSetup.State.Working -> getString(if (importing) R.string.home_importing else R.string.home_extracting, percent(setup)) + setup is GameSetup.State.Checking -> getString( + when { + building -> R.string.home_build_preparing + importing -> R.string.home_checking_package + else -> R.string.home_checking + }, + ) + setup is GameSetup.State.Working -> getString( + when { + building -> R.string.home_building + importing -> R.string.home_importing + else -> R.string.home_extracting + }, + percent(setup), + ) setup is GameSetup.State.Finishing -> getString(R.string.home_finishing) else -> getString(label(mainAction)) } @@ -228,22 +252,31 @@ class LauncherActivity : Activity() { cancel.visibility = if (settingUp && setup !is GameSetup.State.Finishing) View.VISIBLE else View.GONE homeStatus.text = when { + setup is GameSetup.State.Working && building -> buildStatus(setup) setup is GameSetup.State.Working -> getString( R.string.home_extract_progress, Formatter.formatShortFileSize(this, setup.done), Formatter.formatShortFileSize(this, setup.total), ) - setup is GameSetup.State.Checking -> getString(if (importing) R.string.home_checking_package_status else R.string.home_checking_status) + setup is GameSetup.State.Checking -> getString( + when { + building -> R.string.build_step_prepare + importing -> R.string.home_checking_package_status + else -> R.string.home_checking_status + }, + ) settingUp || setup is GameSetup.State.Failed -> "" - setup is GameSetup.State.Cancelled -> getString(R.string.home_setup_cancelled) + setup is GameSetup.State.Cancelled -> getString(if (building) R.string.home_build_cancelled else R.string.home_setup_cancelled) running -> getString(R.string.home_running) - mainAction == Action.Play && setup is GameSetup.State.Done -> getString(R.string.home_setup_done) + mainAction == Action.Play && setup is GameSetup.State.Done -> getString(if (building) R.string.home_build_done else R.string.home_setup_done) mainAction == Action.Play -> getString(R.string.home_put_on_headset) else -> "" } when { + settingUp && building -> showBanner(getString(R.string.home_build_running), warning = false) settingUp -> showBanner(null) + setup is GameSetup.State.Failed && building -> showBanner(getString(R.string.home_build_failed, setup.message), warning = true) setup is GameSetup.State.Failed -> showBanner(getString(R.string.home_setup_failed, setup.message), warning = true) discStatus == GameStorage.DiscStatus.Incomplete -> showBanner(getString(R.string.home_data_incomplete, disc), warning = true) gameStatus == GameLibrary.Status.Stale -> showBanner(getString(R.string.home_game_stale), warning = true) @@ -265,6 +298,7 @@ class LauncherActivity : Activity() { Action.Resume -> R.string.home_resume Action.SelectDisc -> R.string.home_select_disc Action.ImportGame -> R.string.home_import + Action.BuildGame -> R.string.home_build } private fun perform(action: Action) { @@ -273,6 +307,26 @@ class LauncherActivity : Activity() { Action.Play, Action.Resume -> play() Action.SelectDisc -> selectDiscImage() Action.ImportGame -> importGame() + Action.BuildGame -> buildGame() + } + } + + private fun buildStatus(state: GameSetup.State.Working): String = when (state.step) { + GameBuild.Step.Download -> getString(R.string.build_step_download, state.stepDone, state.stepTotal) + GameBuild.Step.Translate -> getString(R.string.build_step_translate, state.stepDone, state.stepTotal) + GameBuild.Step.Compile -> getString(R.string.build_step_compile, state.stepDone, state.stepTotal) + GameBuild.Step.Link -> getString(R.string.build_step_link) + GameBuild.Step.Install -> getString(R.string.build_step_install) + GameBuild.Step.Prepare, null -> getString(R.string.build_step_prepare) + } + + /** Builds the game on this headset from DATA, after saying what that takes. */ + private fun buildGame() { + if (GameSetup.isRunning || GameStorage.discStatus(this) != GameStorage.DiscStatus.Ready) return + val message = if (GameLibrary.status(this) == GameLibrary.Status.Ready) R.string.home_build_replace_message else R.string.home_build_message + confirm(R.string.home_build_title, message, R.string.home_build) { + GameSetupService.startBuild(this) + showPage(Page.Home) } } @@ -403,5 +457,6 @@ class LauncherActivity : Activity() { const val REQUEST_GAME_PACKAGE = 2 const val PREFERENCES = "launcher" const val KEY_LAST_DROPPED_IMPORT = "lastDroppedImport" + const val EXTRA_DEBUG_BUILD_GAME = "org.wiicompiled.quest.debug.BUILD_GAME" } } diff --git a/android/app/src/main/java/org/wiicompiled/quest/launcher/RemoteZip.kt b/android/app/src/main/java/org/wiicompiled/quest/launcher/RemoteZip.kt new file mode 100644 index 0000000..4f7f7bd --- /dev/null +++ b/android/app/src/main/java/org/wiicompiled/quest/launcher/RemoteZip.kt @@ -0,0 +1,126 @@ +package org.wiicompiled.quest.launcher + +import java.io.ByteArrayOutputStream +import java.io.IOException +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.util.zip.Inflater + +/** + * Reads a few entries of a large zip on a web server without downloading the rest: the central + * directory from the end of the file, then only the wanted entries, neighbouring ones in one + * request. [fetch] returns the bytes from start to endInclusive (an HTTP range request). + * + * Only what the Android NDK zip needs: no ZIP64, stored or deflated entries. + */ +class RemoteZip(private val size: Long, private val fetch: (start: Long, endInclusive: Long) -> ByteArray) { + + /** [end] is where the next entry (or the central directory) starts, which bounds this one's bytes. */ + data class Entry(val name: String, val method: Int, val compressedSize: Long, val size: Long, val offset: Long, val end: Long) + + fun entries(): List { + val tailStart = maxOf(0L, size - EOCD_SEARCH) + val tail = fetch(tailStart, size - 1) + var eocd = -1 + for (i in tail.size - EOCD_SIZE downTo 0) { + if (u32(tail, i) == EOCD_SIGNATURE) { eocd = i; break } + } + if (eocd < 0) throw IOException("No end of central directory record") + val count = u16(tail, eocd + 10) + val directorySize = u32(tail, eocd + 12) + val directoryOffset = u32(tail, eocd + 16) + if (count == 0xFFFF || directoryOffset == 0xFFFFFFFFL) throw IOException("ZIP64 archives are not supported") + val directory = fetch(directoryOffset, directoryOffset + directorySize - 1) + + data class Raw(val name: String, val method: Int, val compressedSize: Long, val size: Long, val offset: Long) + val raw = ArrayList(count) + var at = 0 + repeat(count) { + if (u32(directory, at) != CENTRAL_SIGNATURE) throw IOException("Corrupt central directory") + val nameLength = u16(directory, at + 28) + val extraLength = u16(directory, at + 30) + val commentLength = u16(directory, at + 32) + raw += Raw( + name = String(directory, at + 46, nameLength, Charsets.UTF_8), + method = u16(directory, at + 10), + compressedSize = u32(directory, at + 20), + size = u32(directory, at + 24), + offset = u32(directory, at + 42), + ) + at += 46 + nameLength + extraLength + commentLength + } + val sorted = raw.sortedBy { it.offset } + return sorted.mapIndexed { i, entry -> + val end = if (i + 1 < sorted.size) sorted[i + 1].offset else directoryOffset + Entry(entry.name, entry.method, entry.compressedSize, entry.size, entry.offset, end) + } + } + + /** + * Calls [onEntry] with each of [wanted]'s contents, in offset order. Entries less than + * [maxGap] apart are fetched in one request of at most [maxRequest] bytes. + */ + fun read(wanted: List, maxGap: Long = 1L shl 20, maxRequest: Long = 16L shl 20, onEntry: (Entry, ByteArray) -> Unit) { + val entries = wanted.sortedBy { it.offset } + var first = 0 + while (first < entries.size) { + var last = first + while (last + 1 < entries.size && + entries[last + 1].offset - entries[last].end < maxGap && + entries[last + 1].end - entries[first].offset <= maxRequest + ) { + last++ + } + val start = entries[first].offset + val chunk = fetch(start, entries[last].end - 1) + for (i in first..last) { + val entry = entries[i] + val local = (entry.offset - start).toInt() + if (u32(chunk, local) != LOCAL_SIGNATURE) throw IOException("Corrupt entry ${entry.name}") + val dataStart = local + 30 + u16(chunk, local + 26) + u16(chunk, local + 28) + onEntry(entry, content(entry, chunk, dataStart)) + } + first = last + 1 + } + } + + private fun content(entry: Entry, chunk: ByteArray, dataStart: Int): ByteArray { + if (dataStart + entry.compressedSize > chunk.size) throw IOException("Truncated entry ${entry.name}") + val bytes = when (entry.method) { + 0 -> chunk.copyOfRange(dataStart, dataStart + entry.compressedSize.toInt()) + 8 -> { + val inflater = Inflater(true) + try { + inflater.setInput(chunk, dataStart, entry.compressedSize.toInt()) + val out = ByteArrayOutputStream(entry.size.toInt()) + val buffer = ByteArray(64 * 1024) + while (!inflater.finished()) { + val n = inflater.inflate(buffer) + if (n == 0 && (inflater.needsInput() || inflater.needsDictionary())) break + out.write(buffer, 0, n) + } + out.toByteArray() + } finally { + inflater.end() + } + } + else -> throw IOException("Entry ${entry.name} uses compression method ${entry.method}") + } + if (bytes.size.toLong() != entry.size) throw IOException("Entry ${entry.name} has the wrong size") + return bytes + } + + private fun u16(bytes: ByteArray, at: Int): Int = + ByteBuffer.wrap(bytes, at, 2).order(ByteOrder.LITTLE_ENDIAN).short.toInt() and 0xFFFF + + private fun u32(bytes: ByteArray, at: Int): Long = + ByteBuffer.wrap(bytes, at, 4).order(ByteOrder.LITTLE_ENDIAN).int.toLong() and 0xFFFFFFFFL + + private companion object { + const val EOCD_SIGNATURE = 0x06054b50L + const val CENTRAL_SIGNATURE = 0x02014b50L + const val LOCAL_SIGNATURE = 0x04034b50L + const val EOCD_SIZE = 22 + const val EOCD_SEARCH = EOCD_SIZE + 0xFFFFL + } +} diff --git a/android/app/src/main/java/org/wiicompiled/quest/launcher/SettingsPage.kt b/android/app/src/main/java/org/wiicompiled/quest/launcher/SettingsPage.kt index c7dfedb..c31ea4c 100644 --- a/android/app/src/main/java/org/wiicompiled/quest/launcher/SettingsPage.kt +++ b/android/app/src/main/java/org/wiicompiled/quest/launcher/SettingsPage.kt @@ -38,6 +38,7 @@ class SettingsPage( private val gameRunning: () -> Boolean, private val selectDiscImage: () -> Unit, private val importGame: () -> Unit, + private val buildGame: () -> Unit, ) { enum class Tab(val label: Int) { @@ -281,6 +282,12 @@ class SettingsPage( GameLibrary.Status.Missing -> activity.getString(R.string.about_game_missing) } info(R.string.about_game, gameText, stacked = true) + action( + R.string.home_build, R.string.about_build_helper, R.string.home_build, + enabled = !GameSetup.isRunning && GameStorage.discStatus(activity) == GameStorage.DiscStatus.Ready, + ) { + buildGame() + } action(R.string.home_import, R.string.about_import_helper, R.string.home_import, enabled = !GameSetup.isRunning) { importGame() } diff --git a/android/app/src/main/java/org/wiicompiled/quest/launcher/ToolProcess.kt b/android/app/src/main/java/org/wiicompiled/quest/launcher/ToolProcess.kt new file mode 100644 index 0000000..081a35f --- /dev/null +++ b/android/app/src/main/java/org/wiicompiled/quest/launcher/ToolProcess.kt @@ -0,0 +1,93 @@ +package org.wiicompiled.quest.launcher + +import java.io.File +import java.io.InterruptedIOException +import java.util.concurrent.TimeUnit +import kotlin.concurrent.thread + +/** + * Runs a build tool from the app's private storage. Android refuses to exec files there, but lets + * the system linker load them (the same permission that lets the game load libmain.so), so every + * tool starts as `/system/bin/linker64 `. + * + * Output goes to [log] and to the caller line by line. [cancelled] is polled while the tool runs; + * when it turns true the tool is stopped and [run] throws [InterruptedIOException]. + */ +class ToolProcess( + private val environment: Map, + private val log: BuildLog, + private val cancelled: () -> Boolean, +) { + + class Result(val exitCode: Int, val tail: List) { + /** Android's low memory killer ends a process with SIGKILL, which reads as 128 + 9. */ + val killed: Boolean get() = exitCode == 137 + } + + fun run(workingDirectory: File, executable: File, arguments: List, onLine: (String) -> Unit = {}): Result { + val command = listOf(LINKER, executable.absolutePath) + arguments + val process = ProcessBuilder(command) + .directory(workingDirectory) + .redirectErrorStream(true) + .apply { environment().putAll(this@ToolProcess.environment) } + .start() + val tail = ArrayDeque() + val reader = thread(name = "BuildOutput") { + process.inputStream.bufferedReader().useLines { lines -> + for (line in lines) { + log.line(line) + synchronized(tail) { + tail.addLast(line) + if (tail.size > TAIL_LINES) tail.removeFirst() + } + onLine(line) + } + } + } + try { + while (!process.waitFor(POLL_MS, TimeUnit.MILLISECONDS)) { + if (cancelled()) { + stop(process) + throw InterruptedIOException("Build cancelled") + } + } + } catch (e: InterruptedException) { + stop(process) + throw InterruptedIOException("Build interrupted") + } + reader.join(READER_JOIN_MS) + return Result(process.exitValue(), synchronized(tail) { tail.toList() }) + } + + private fun stop(process: Process) { + process.destroy() + if (!process.waitFor(STOP_GRACE_MS, TimeUnit.MILLISECONDS)) process.destroyForcibly() + } + + private companion object { + const val LINKER = "/system/bin/linker64" + const val POLL_MS = 250L + const val STOP_GRACE_MS = 2000L + const val READER_JOIN_MS = 5000L + const val TAIL_LINES = 40 + } +} + +/** The build's log file, which several tools write to at once. */ +class BuildLog(val file: File) { + init { + file.parentFile?.mkdirs() + } + + private val writer = file.bufferedWriter() + + @Synchronized + fun line(text: String) { + writer.write(text) + writer.newLine() + writer.flush() + } + + @Synchronized + fun close() = writer.close() +} diff --git a/android/app/src/main/res/values/strings.xml b/android/app/src/main/res/values/strings.xml index 1f94aac..6db2a99 100644 --- a/android/app/src/main/res/values/strings.xml +++ b/android/app/src/main/res/values/strings.xml @@ -21,9 +21,25 @@ The game opens in the headset. The game is ready, but its game files (DATA) are missing. Select your own PAL (RMCP01) disc image to extract them, or import a game file from your computer that includes them.\n\nThe game files go to:\n%1$s Import from computer - Nothing from Mario Kart Wii comes with this app: the game is built from your own PAL (RMCP01) disc.\n\nWith a computer: in the WiiCompiled VR launcher, use Build for Quest with your disc image, copy the .wcgame file it makes to this headset, then press Import from computer.\n\nOn this headset: copy your disc image (ISO, WBFS, RVZ…) here, for example into Download, and press Select disc image. - The game files are ready. The game itself is built from your disc: in the WiiCompiled VR launcher on your computer, use Build for Quest, copy the .wcgame file to this headset, then press Import from computer. - This app was updated, so the game must be built again for it: use Build for Quest in the WiiCompiled VR launcher again, then Import from computer. + Nothing from Mario Kart Wii comes with this app: the game is built from your own PAL (RMCP01) disc.\n\nOn this headset: copy your disc image (ISO, WBFS, RVZ…) here, for example into Download, press Select disc image, then Build on this Quest.\n\nWith a computer: in the WiiCompiled VR launcher, use Build for Quest with your disc image, copy the .wcgame file it makes to this headset, then press Import from computer. + The game files are ready. Now build the game from them: Build on this Quest takes about 30–40 minutes on a Quest 3. With a computer, use Build for Quest in the WiiCompiled VR launcher instead, copy the .wcgame file to this headset and press Import from computer. + This app was updated, so the game must be built again for it: press Build on this Quest, or use Build for Quest in the WiiCompiled VR launcher again and Import from computer. + Build on this Quest + Preparing… + Building %1$d%% + Build the game on this Quest? + The game is built on this headset from your own disc files. It takes about 30–40 minutes on a Quest 3 and uses most of its memory, so close other apps and keep the headset on its charger.\n\nThe first build downloads about 10 MB of Android build files from Google. + The installed game stays until the new build has finished.\n\nBuilding takes about 30–40 minutes on a Quest 3 and uses most of its memory, so close other apps and keep the headset on its charger. + Building the game from your disc files: about 30–40 minutes on a Quest 3. Keep the headset on its charger. The build carries on if you close this panel. + The game is built. + Build cancelled. Building again picks up where it stopped. + The game could not be built.\n%1$s + Unpacking the build tools + Downloading Android build files (%1$d of %2$d MB) + Translating the game (step %1$d of %2$d) + Compiling %1$d of %2$d + Linking the game + Installing the game Checking game file… Checking the game file was built from your disc for this version of the app. Importing %1$d%% @@ -159,6 +175,7 @@ Built by %1$s on %2$s Not installed yet Built for another version of this app; build it again + Builds the game from DATA on this headset: about 30–40 minutes on a Quest 3. Installs a .wcgame made by Build for Quest in the WiiCompiled VR launcher on a computer. Diagnostics OpenXR diagnostic logging @@ -169,6 +186,9 @@ Setting up the game files Checking the disc image… Extracting: %1$d%% + Building: %1$d%% + Building the game + Preparing the build… Importing: %1$d%% The game is not installed. Open WiiCompiled VR and set it up from Home. diff --git a/android/app/src/test/java/org/wiicompiled/quest/launcher/BuildRecipeTest.kt b/android/app/src/test/java/org/wiicompiled/quest/launcher/BuildRecipeTest.kt new file mode 100644 index 0000000..e411ec0 --- /dev/null +++ b/android/app/src/test/java/org/wiicompiled/quest/launcher/BuildRecipeTest.kt @@ -0,0 +1,97 @@ +package org.wiicompiled.quest.launcher + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertThrows +import org.junit.Test + +/** The headset must turn the game kit's recipe into the same commands android/QuestGameKit.psm1 runs. */ +class BuildRecipeTest { + + @Test + fun quotesResponseFileArgumentsForGnuTokenizers() { + assertEquals("-DIMGUI_USER_CONFIG=\\\"aurora/imgui_config.h\\\"", BuildRecipe.responseFileArgument("-DIMGUI_USER_CONFIG=\"aurora/imgui_config.h\"")) + assertEquals("/data/with\\ space/a\\\\b", BuildRecipe.responseFileArgument("/data/with space/a\\b")) + assertEquals("-O2\n-c\n", BuildRecipe.responseFile(listOf("-O2", "-c"))) + } + + @Test + fun expandsPlaceholdersAndRejectsUnknownOnes() { + val values = mapOf("kit" to "/k", "sysroot" to "/n/sysroot") + assertEquals("-I/k/include", BuildRecipe.expand("-I{kit}/include", values)) + assertEquals("--sysroot=/n/sysroot", BuildRecipe.expand("--sysroot={sysroot}", values)) + assertThrows(IllegalArgumentException::class.java) { BuildRecipe.expand("-I{workspace}", values) } + } + + @Test + fun expandsTheLinkSlotsInPlace() { + val template = listOf("-o", "{output}", "{kit}/objects/001_a.o", "{game:runtime}", "--start-lib", "{game:translated}", "--end-lib", "-lm") + val arguments = BuildRecipe.linkArguments( + template, + mapOf("kit" to "/k", "output" to "/w/libmain.so"), + mapOf("runtime" to listOf("/o/data.o", "/o/blob.o"), "translated" to listOf("/o/s1.o", "/o/s2.o")), + ) + assertEquals( + listOf("-o", "/w/libmain.so", "/k/objects/001_a.o", "/o/data.o", "/o/blob.o", "--start-lib", "/o/s1.o", "/o/s2.o", "--end-lib", "-lm"), + arguments, + ) + assertThrows(IllegalArgumentException::class.java) { + BuildRecipe.linkArguments(listOf("{game:product}"), emptyMap(), emptyMap()) + } + } + + @Test + fun readsSourceListsFromShardsCmake() { + val shards = """ + set(MKW_TRANSLATED_SHARD_ROOT "/w/generated/build_shards") + set(MKW_BASE_COMMON_SHARDS + "/w/generated/build_shards/base_common/shard_1.cpp" + "/w/generated/build_shards/base_common/shard_2.cpp" + ) + set(MKW_BASE_PORTABLE_SENSITIVE_SHARDS + ) + set(MKW_BASE_REGISTRATION_SOURCES + "C:\w\generated\build_shards\base_registration\registration_00.cpp" + ) + """.trimIndent() + assertEquals( + listOf("/w/generated/build_shards/base_common/shard_1.cpp", "/w/generated/build_shards/base_common/shard_2.cpp"), + BuildRecipe.sourceList(shards, "MKW_BASE_COMMON_SHARDS"), + ) + assertEquals(emptyList(), BuildRecipe.sourceList(shards, "MKW_BASE_PORTABLE_SENSITIVE_SHARDS")) + assertEquals(emptyList(), BuildRecipe.sourceList(shards, "MKW_RETRO_MOD_SHARDS")) + assertEquals(listOf("C:/w/generated/build_shards/base_registration/registration_00.cpp"), BuildRecipe.sourceList(shards, BuildRecipe.REGISTRATION_LIST)) + } + + @Test + fun readsTheEntryPointFromTheGameManifest() { + val manifest = "translation:\n entry_points:\n - 0x800060A4\n function_map:\n path: MAP.txt\n" + assertEquals("0x800060A4", BuildRecipe.entryPoint(manifest)) + assertThrows(IllegalArgumentException::class.java) { BuildRecipe.entryPoint("translation:\n") } + } + + @Test + fun rewritesWindowsBlobSectionsForElfOnly() { + val windows = "// blobs\n.section .rdata,\"dr\"\n.incbin \"a.bin\"\n" + assertEquals( + "// blobs\n.section .rodata,\"a\",@progbits\n.incbin \"a.bin\"\n\n.section .note.GNU-stack,\"\",@progbits\n", + BuildRecipe.elfBlobAssembly(windows), + ) + val elf = "// blobs\n.section .rodata,\"a\",@progbits\n" + assertEquals(elf, BuildRecipe.elfBlobAssembly(elf)) + } + + @Test + fun namesObjectsAfterTheSourceStem() { + assertEquals("shard_8b98.o", BuildRecipe.objectName("/w/generated/build_shards/base_common/shard_8b98.cpp")) + assertEquals("data_sections_init_blobs_android.o", BuildRecipe.objectName("/w/data_sections_init_blobs_android.S")) + } + + @Test + fun digestsTheFileListLikePrepareQuestToolchain() { + // SHA-256 of "a/x.h 11\nb.so 22", the lines sorted by path and joined with \n. + assertEquals( + java.security.MessageDigest.getInstance("SHA-256").digest("a/x.h 11\nb.so 22".toByteArray()).joinToString("") { "%02x".format(it) }, + BuildRecipe.listDigest(mapOf("b.so" to "22", "a/x.h" to "11")), + ) + } +} diff --git a/android/app/src/test/java/org/wiicompiled/quest/launcher/RemoteZipTest.kt b/android/app/src/test/java/org/wiicompiled/quest/launcher/RemoteZipTest.kt new file mode 100644 index 0000000..b2d493f --- /dev/null +++ b/android/app/src/test/java/org/wiicompiled/quest/launcher/RemoteZipTest.kt @@ -0,0 +1,77 @@ +package org.wiicompiled.quest.launcher + +import java.io.ByteArrayOutputStream +import java.util.zip.CRC32 +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +/** The NDK download reads only the wanted entries of a zip, through range requests. */ +class RemoteZipTest { + + private val contents = linkedMapOf( + "ndk/sysroot/usr/include/stdio.h" to "int printf(const char *, ...);\n".repeat(200).toByteArray(), + "ndk/sysroot/usr/lib/aarch64-linux-android/29/libc.so" to ByteArray(70_000) { (it * 31).toByte() }, + "ndk/prebuilt/bin/clang" to ByteArray(300_000) { (it * 7).toByte() }, + "ndk/lib/clang/21/lib/linux/libclang_rt.builtins-aarch64-android.a" to "!\n".toByteArray(), + ) + + private val zip: ByteArray = ByteArrayOutputStream().also { bytes -> + ZipOutputStream(bytes).use { out -> + for ((name, data) in contents) { + val entry = ZipEntry(name) + if (name.endsWith(".a")) { + // A stored entry: sizes and CRC must be known up front. + entry.method = ZipEntry.STORED + entry.size = data.size.toLong() + entry.crc = CRC32().apply { update(data) }.value + } + out.putNextEntry(entry) + out.write(data) + out.closeEntry() + } + out.putNextEntry(ZipEntry("ndk/empty-dir/")) + out.closeEntry() + } + }.toByteArray() + + @Test + fun readsWantedEntriesWithoutTheRest() { + val requests = mutableListOf() + val remote = RemoteZip(zip.size.toLong()) { start, end -> + requests += start..end + zip.copyOfRange(start.toInt(), end.toInt() + 1) + } + val entries = remote.entries() + assertEquals(contents.keys + "ndk/empty-dir/", entries.map { it.name }.toSet()) + + val wanted = entries.filter { it.name.contains("sysroot") || it.name.endsWith(".a") } + val read = LinkedHashMap() + requests.clear() + remote.read(wanted, maxGap = 64) { entry, bytes -> read[entry.name] = bytes } + + assertEquals(wanted.map { it.name }.toSet(), read.keys) + for ((name, bytes) in read) assertArrayEquals(name, contents.getValue(name), bytes) + // The two sysroot entries sit next to each other and share a request; clang, 300 KB, is skipped. + assertEquals(2, requests.size) + assertTrue(requests.sumOf { it.last - it.first + 1 } < 150_000) + } + + @Test + fun keepsRequestsWithinTheSizeLimit() { + val requests = mutableListOf() + val remote = RemoteZip(zip.size.toLong()) { start, end -> + requests += start..end + zip.copyOfRange(start.toInt(), end.toInt() + 1) + } + val files = remote.entries().filter { !it.name.endsWith("/") } + requests.clear() + var count = 0 + remote.read(files, maxGap = 1L shl 20, maxRequest = 1) { _, _ -> count++ } + assertEquals(files.size, count) + assertEquals(files.size, requests.size) + } +} diff --git a/android/toolchain/translator_host.c b/android/toolchain/translator_host.c new file mode 100644 index 0000000..4b66c9d --- /dev/null +++ b/android/toolchain/translator_host.c @@ -0,0 +1,61 @@ +/* + * Starts the WiiCompiled translator, a self-contained .NET app, on the headset. + * + * Android refuses to exec files in an app's private storage, so the on-device builder starts its + * tools through /system/bin/linker64. Under the linker, the .NET apphost reads its own location from + * /proc/self/exe, gets the linker's, and cannot find the app. This host is told the app's directory + * and name instead and hands them to hostfxr exactly as the apphost would: + * + * linker64 translator_host [app arguments...] + * + * It also switches off bionic's heap pointer tagging first. The .NET runtime for Android (Mono) + * crashes at startup on tagged heap pointers. + */ +#include +#include +#include +#include + +#ifndef M_BIONIC_SET_HEAP_TAGGING_LEVEL +#define M_BIONIC_SET_HEAP_TAGGING_LEVEL (-204) +#endif +#ifndef M_HEAP_TAGGING_LEVEL_NONE +#define M_HEAP_TAGGING_LEVEL_NONE 0 +#endif + +typedef int (*hostfxr_main_startupinfo_fn)(int argc, const char **argv, const char *host_path, + const char *dotnet_root, const char *app_path); + +int main(int argc, char **argv) { + if (argc < 3) { + fprintf(stderr, "usage: translator_host [arguments...]\n"); + return 2; + } + /* Only lowering the level is allowed, and bionic accepts pointers tagged before this call. */ + mallopt(M_BIONIC_SET_HEAP_TAGGING_LEVEL, M_HEAP_TAGGING_LEVEL_NONE); + + const char *directory = argv[1]; + const char *name = argv[2]; + char hostfxr[PATH_MAX], apphost[PATH_MAX], assembly[PATH_MAX]; + if (snprintf(hostfxr, sizeof hostfxr, "%s/libhostfxr.so", directory) >= (int)sizeof hostfxr || + snprintf(apphost, sizeof apphost, "%s/%s", directory, name) >= (int)sizeof apphost || + snprintf(assembly, sizeof assembly, "%s/%s.dll", directory, name) >= (int)sizeof assembly) { + fprintf(stderr, "translator_host: path too long\n"); + return 2; + } + + void *library = dlopen(hostfxr, RTLD_NOW | RTLD_LOCAL); + if (library == NULL) { + fprintf(stderr, "translator_host: %s\n", dlerror()); + return 3; + } + hostfxr_main_startupinfo_fn start = (hostfxr_main_startupinfo_fn)dlsym(library, "hostfxr_main_startupinfo"); + if (start == NULL) { + fprintf(stderr, "translator_host: %s\n", dlerror()); + return 3; + } + + /* The app sees the apphost path as argv[0], then its own arguments. */ + argv[2] = apphost; + return start(argc - 2, (const char **)argv + 2, apphost, directory, assembly); +} diff --git a/docs/quest-port.md b/docs/quest-port.md index de9a12d..81c759e 100644 --- a/docs/quest-port.md +++ b/docs/quest-port.md @@ -281,13 +281,70 @@ if the game files pass the same checks as an extraction. A package built for ano is refused, and an installed game whose kit fingerprint no longer matches the APK shows as stale on Home. -Home's main button is always the next step: **Import from computer** until a game is installed -(with **Select disc image** beside it while there are no game files), **Select disc image** when -only the game files are missing, and **Play** once both are present. Building the game on the -headset itself is the next step of this work. The translator (a self-contained -`linux-bionic-arm64` .NET build, which needs heap pointer tagging disabled) and Termux's clang -21.1.8 both ran on a Quest 3. A base translation took 6.7 minutes with a 3.1 GB memory peak, and -the heaviest shard compiled in 65 seconds with a 480 MB peak. +Home's main button is always the next step: **Select disc image** while there are no game files, +**Build on this Quest** once they are there and no game is installed (or the installed one is +stale), and **Play** once both are present. **Import from computer** sits beside the first two. + +### Building the game on the headset + +**Build on this Quest** (`GameBuild`, a `GameSetup` task in `GameSetupService`) does on the +headset what `Build-QuestGame.ps1` does on a PC, from `DATA`, in about 28 minutes on a Quest 3: + +1. It unpacks `assets/quest_toolchain` and `assets/game_kit` into `files/build/`. +2. It downloads the NDK files a build needs (below) into `files/build/ndk`. +3. It translates the disc: `translate-recursive`, `generate-data-init --target-os android` and + `emit-build-shards`, in a workspace made of the kit's `translation/` copy of `recomp.yml`, + `MAP.txt` and `runtime/src`, plus `main.dol` and `StaticR.rel` from `DATA`. A translation of the + same kit, toolchain and disc is reused. +4. It compiles the generated sources with the kit's flags, up to four at a time (fewer when the + available memory allows less than 700 MB each). The blob assembly is compiled as plain + `-x assembler`: preprocessing a `.S` makes clang start itself, which the linker trick below + cannot do. Finished objects survive a cancelled or failed run. +5. It links with `kit.json`'s `link.lld` and installs `libmain.so` and `game.json` through the same + staging swap as an import. `builtBy` says the headset built it. A successful build deletes + `files/build`. + +The log goes to `Logs/build_.log` next to `DATA`. + +The toolchain (`android/Prepare-QuestToolchain.ps1`, run by the `prepareBase*QuestToolchain` +Gradle tasks) is 249 MB unpacked. It travels as one deflated 80 MB `files.zip`, because asset +packaging never compresses `.so` files, next to `toolchain.json`, which lists every file. It holds: + +- the translator published for `linux-bionic-arm64` from a copy of the sources retargeted to + net10.0, which is the first .NET with those runtime packs. That runtime is Mono. +- `translator_host` (`android/toolchain/translator_host.c`). +- Termux's clang/lld 21.1.8 and the ten shared libraries they and the translator load, pinned by + package SHA-256 and stored under the names their users load. OpenSSL is stored as `libssl.so`, + the name .NET's shim opens on Android; Android's own BoringSSL lacks symbols it needs. +- `ndk.json`: the pin of the NDK files. + +Android facts this design rests on, all measured on a Quest 3: + +- An app cannot `exec` files in its private storage, but `/system/bin/linker64 ` + runs them (`PrivateCodeExecutionTest`). Every tool starts that way (`ToolProcess`), with + `LD_LIBRARY_PATH` at the toolchain's `llvm/lib`. +- Under the linker, the .NET apphost reads `/proc/self/exe`, gets the linker, and cannot find the + app. `translator_host` hands hostfxr the app directory instead. It also turns off bionic's heap + pointer tagging, which crashes the runtime at startup. +- Termux's clang driver compiles with the same `cc1` arguments as the NDK's. Its link line is + patched for Termux (`-rpath`, `-L/system/lib64`), and clang could not start lld anyway. So the kit + export expands the link with the NDK's own driver (`clang++ -###`) into `link.lld`, a raw lld + command with `{kit}`, `{ndk}`, `{output}` and `{game:*}` placeholders, and the headset runs + `ld.lld` directly. The compile uses Termux's clang resource headers, which match that compiler. +- The NDK files come from Google's `android-ndk-r29-linux.zip`, not from the APK. `RemoteZip` + reads the zip's central directory and then only the 3,499 wanted entries, with HTTP range + requests: the aarch64 sysroot, `libc++_shared.so`, the API 29 stubs and CRT objects, + compiler-rt builtins, `libunwind.a` and `libatomic.a`. That is about 8 MB compressed of 784 MB. + Each file must match the SHA-256 in `ndk.json`, and the list must match the digest pinned in the + script. The requests say `Accept-Encoding: identity`: Android's HTTP stack asks for gzip by + default, and Google's server then serves a gzip-encoded zip whose byte ranges are not the file's + (HTTP 416). The Linux zip is used because its sysroot holds headers whose names differ only in case + (`xt_TCPMSS.h`, `xt_tcpmss.h`), which a Windows copy of the NDK loses. +- Translation peaks at 3.1 GB with Mono's default heap and 2.1 GB with + `MONO_GC_PARAMS=soft-heap-limit=1200m` and four threads, for byte-identical output. The builder + uses the latter. + +`BuildRecipeTest` and `RemoteZipTest` cover the command lines and the zip reader. ### Build system @@ -318,13 +375,14 @@ the heaviest shard compiled in 65 seconds with a 480 MB peak. Prerequisites on the Windows host (all already present on the machine this was developed on): JDK 17, Android SDK with platform 34+, NDK `29.0.14206865`, -SDK CMake `3.22.1`, `adb`, Rust 1.85+ with `rustup target add aarch64-linux-android`; -a translated graph for your own disc (the installer's `BuildWorkspace/generated`, -produced by the normal Windows pipeline). +SDK CMake `3.22.1`, `adb`, Rust 1.85+ with `rustup target add aarch64-linux-android`, +the .NET 10 SDK (for the headset's translator; the base APK build downloads ~70 MB of Termux +packages into `android/.dependencies` the first time); a translated graph for your own disc (the +installer's `BuildWorkspace/generated`, produced by the normal Windows pipeline). ```powershell powershell -ExecutionPolicy Bypass -File android/Prepare-QuestDependencies.ps1 # SDL3 3.4.4 AAR into android/app/libs -powershell -ExecutionPolicy Bypass -File android/Build-Quest.ps1 -Install # the app and its game kit, debug-signed +powershell -ExecutionPolicy Bypass -File android/Build-Quest.ps1 -Install # the app, its game kit and toolchain, debug-signed powershell -ExecutionPolicy Bypass -File android/Build-QuestGame.ps1 -Install # your game, against that kit, into Import (or WheelWizard VR's Build for Quest) powershell -ExecutionPolicy Bypass -File android/Build-Quest.ps1 -Flavor retroRewind # Retro Rewind (needs translate-mod output) adb push MarioKart.iso /sdcard/Download/ # then Select disc image in the launcher @@ -379,6 +437,15 @@ What has been verified on the development machine (September 2026): all 189 perspective draws replayed per eye, the first projection layer is submitted, and the menus return to the virtual screen. No WebGPU or OpenXR errors over a 90 second session. +- **Device, 2026-09-17: the headset built its own game.** Build on this Quest ran inside the app + in 27.8 minutes on a Quest 3: unpacking the toolchain, 43 MB of NDK files downloaded from Google + and checked in about 6 seconds, translation in 646 s (2.1 GB peak), 92 sources compiled four at a + time in 16 minutes (about 250 MB each, 3 GB still available), `ld.lld` in under a second, then + the install and the cleanup of everything it unpacked. The game ran from the result: session + `FOCUSED`, past 1,000 frames, the intro movie on the virtual screen. That `libmain.so` is + byte-identical to one the same toolchain built from a shell, and against the PC-built library it + has the same soname, `NEEDED` list, 785 undefined symbols and 29,995 translated functions + (61,974 defined against 61,975: the PC's older clang keeps one inline helper out of line). Bring-up fixes that only a device could reveal: @@ -438,6 +505,13 @@ screen is up, reach Grand Prix character select. A value left over from an earlier run is ignored on the first read. Presses only land while the XR session is `FOCUSED`. +A debug APK also builds the game on the headset without a press, once `DATA` is there; the +build log is the newest `Logs/build_*.log`: + +```powershell +adb shell am start -n org.wiicompiled.quest/.launcher.LauncherActivity --ez org.wiicompiled.quest.debug.BUILD_GAME true +``` + Performance, measured 2026-09-16 on a 50cc Luigi Circuit start with the player idle, over 40 s, with an optimized build (`-O3`, translated code `-O2`, `-mcpu=cortex-a77`):