From 150062280db99847769c8101e8bb1db017f6925d Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 08:27:12 +0000 Subject: [PATCH 01/14] Add a Steam Frame flavour of the Android app with Frame controller bindings The Steam Frame runs Android apps through Lepton with SteamVR's OpenXR runtime. A third headset flavour, steamFrame, targets it: - -mcpu=cortex-x4+nosve for the Snapdragon 8 Gen 3 (its firmware does not expose SVE, which clang otherwise auto-vectorises with), from one flavour-to-CPU map that the kit export now reads instead of guessing from the variant name. - MKW_ANDROID_HEADSET=steam_frame defines MKW_HEADSET_STEAM_FRAME for the runtime's own targets, for Frame-specific defaults. - A manifest without the Horizon OS entries, and FrameEntryActivity as the single real MAIN/LAUNCHER activity with the Khronos and Oculus VR categories, which Lepton needs to start an app in VR. It opens the setup panel and, when the selected game can start, the game on top. - XR_VALVE_frame_controller_interaction: the Frame controller profile with its left D-pad (new dpad_* actions, the Wii Remote's D-pad or the gamepad's), View as menu and the left shoulder as the panel button. Also requested on Windows for SteamVR streaming to a Frame. - Build-Quest.ps1, Build-QuestGame.ps1 and Run-Quest.ps1 take -Headset frame. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_019HBRGKTE1GnN2ah8gcZKr3 --- android/Build-Quest.ps1 | 19 +++++-- android/Build-QuestGame.ps1 | 6 +-- android/Run-Quest.ps1 | 6 +-- android/app/build.gradle.kts | 46 +++++++++++++--- .../app/src/steamFrame/AndroidManifest.xml | 44 +++++++++++++++ .../wiicompiled/quest/FrameEntryActivity.kt | 52 ++++++++++++++++++ runtime/CMakeLists.txt | 15 ++++++ runtime/cmake/PublicProducts.cmake | 1 + runtime/include/vr/openxr_input.h | 5 ++ runtime/include/vr/openxr_wii_remote.h | 12 +++++ runtime/src/vr/openxr_input.cpp | 53 ++++++++++++++++++- runtime/src/vr/openxr_integration.cpp | 7 ++- runtime/tests/vr_wii_remote_tests.cpp | 13 +++++ 13 files changed, 257 insertions(+), 22 deletions(-) create mode 100644 android/app/src/steamFrame/AndroidManifest.xml create mode 100644 android/app/src/steamFrame/java/org/wiicompiled/quest/FrameEntryActivity.kt diff --git a/android/Build-Quest.ps1 b/android/Build-Quest.ps1 index 6e86b77..340796e 100644 --- a/android/Build-Quest.ps1 +++ b/android/Build-Quest.ps1 @@ -1,7 +1,10 @@ # Builds the standalone Meta Quest APK on a Windows host. # # powershell -ExecutionPolicy Bypass -File android/Build-Quest.ps1 [-Generated ] -# [-Headset modern|quest1] [-Configuration debug|release] [-Install] +# [-Headset modern|quest1|frame] [-Configuration debug|release] [-Install] +# +# -Headset frame builds the Steam Frame flavour (steamFrame), which runs under Lepton, SteamOS's +# Android layer, with SteamVR's OpenXR runtime (docs/quest-port.md, "Steam Frame"). # # One app offers both games: the APK carries a kit for the base game, and for Retro Rewind too # when -Generated holds a translation that includes the mod. @@ -25,7 +28,7 @@ param( [string]$Generated = '', [string]$Dependencies = '', [string]$CMakeDir = '', - [ValidateSet('modern', 'quest1')] [string]$Headset = 'modern', + [ValidateSet('modern', 'quest1', 'frame')] [string]$Headset = 'modern', [ValidateSet('debug', 'release')] [string]$Configuration = 'debug', [switch]$Install, [switch]$StockDawn @@ -80,8 +83,13 @@ if (-not $Dependencies) { } $variant = (Get-Culture).TextInfo.ToTitleCase($Configuration) -$flavour = if ($Headset -eq 'quest1') { 'Quest1' } else { 'ModernQuest' } -$expectedCpu = if ($Headset -eq 'quest1') { 'kryo' } else { 'cortex-a77' } +# The Gradle flavour and the -mcpu target its kit must record (headsetCpus in app/build.gradle.kts). +$flavourDir, $expectedCpu = switch ($Headset) { + 'quest1' { 'quest1', 'kryo' } + 'frame' { 'steamFrame', 'cortex-x4+nosve' } + default { 'modernQuest', 'cortex-a77' } +} +$flavour = $flavourDir.Substring(0, 1).ToUpperInvariant() + $flavourDir.Substring(1) $task = "app:assemble$flavour$variant" $gradleArgs = @( '--project-dir', $root, @@ -98,7 +106,6 @@ Write-Host "gradlew $($gradleArgs -join ' ')" & (Join-Path $root 'gradlew.bat') @gradleArgs if ($LASTEXITCODE -ne 0) { throw "Gradle failed ($LASTEXITCODE)" } -$flavourDir = if ($Headset -eq 'quest1') { 'quest1' } else { 'modernQuest' } $apkDir = Join-Path $root "app\build\outputs\apk\$flavourDir\$Configuration" $apk = Get-ChildItem -Path $apkDir -Filter '*.apk' | Select-Object -First 1 if (-not $apk) { throw "No APK under $apkDir" } @@ -128,6 +135,8 @@ if ($Install) { if ($LASTEXITCODE -ne 0) { throw "adb install failed ($LASTEXITCODE)" } if ($Headset -eq 'quest1') { Write-Host 'Installed. Use WiiCompiled Settings for setup, then launch WiiCompiled VR directly from the library.' + } elseif ($Headset -eq 'frame') { + Write-Host 'Installed. Build a game on this PC with android/Build-QuestGame.ps1 -Headset frame -Install, then start WiiCompiled VR from the Steam library.' } else { Write-Host 'Installed. Build a game with Build on this Quest in the launcher, or on this PC with android/Build-QuestGame.ps1 -Install.' } diff --git a/android/Build-QuestGame.ps1 b/android/Build-QuestGame.ps1 index 3dbc754..c63877b 100644 --- a/android/Build-QuestGame.ps1 +++ b/android/Build-QuestGame.ps1 @@ -6,7 +6,7 @@ # SDK's NDK and ninja: # # powershell -ExecutionPolicy Bypass -File android/Build-QuestGame.ps1 [-Generated ] [-Kit ] -# [-Headset modern|quest1] [-Configuration debug|release] +# [-Headset modern|quest1|frame] [-Configuration debug|release] # [-Data ] [-Output ] [-Install] # # WiiCompiled Setup's --build-quest runs the copy staged in an installation's BuildWorkspace\android, @@ -33,7 +33,7 @@ param( [string]$Sysroot = '', [string]$Ninja = '', [string]$BuiltBy = 'android/Build-QuestGame.ps1', - [ValidateSet('modern', 'quest1')] [string]$Headset = 'modern', + [ValidateSet('modern', 'quest1', 'frame')] [string]$Headset = 'modern', [ValidateSet('debug', 'release')] [string]$Configuration = 'debug', [ValidateSet('base', 'retro_rewind')] [string]$Product = 'base', [string]$Mod = '', @@ -47,7 +47,7 @@ $root = $PSScriptRoot $workspace = (Resolve-Path (Join-Path $root '..')).Path if (-not $Generated) { $Generated = Join-Path $workspace '.scratch\vr-build-workspace\BuildWorkspace\generated' } if (-not $Manifest) { $Manifest = Join-Path $workspace 'projects\mkwii\recomp.yml' } -$variant = if ($Headset -eq 'quest1') { 'quest1' } else { 'modernQuest' } +$variant = switch ($Headset) { 'quest1' { 'quest1' } 'frame' { 'steamFrame' } default { 'modernQuest' } } $variant += (Get-Culture).TextInfo.ToTitleCase($Configuration) if (-not $BuildDir) { $BuildDir = Join-Path $root "app\build\questGame\$variant\$Product" } if (-not $Kit) { diff --git a/android/Run-Quest.ps1 b/android/Run-Quest.ps1 index f2c3c40..5a193bd 100644 --- a/android/Run-Quest.ps1 +++ b/android/Run-Quest.ps1 @@ -3,7 +3,7 @@ # diagnostics for one session. # # powershell -ExecutionPolicy Bypass -File android/Run-Quest.ps1 [-Apk ] [-Data ] -# [-Headset modern|quest1] [-Configuration debug|release] +# [-Headset modern|quest1|frame] [-Configuration debug|release] # [-Seconds 60] [-NoLaunch] [-SkipInstall] # # -Data names the extracted disc partition (the directory holding sys/, files/, @@ -25,7 +25,7 @@ param( [string]$Apk = '', [string]$Data = '', - [ValidateSet('modern', 'quest1')] [string]$Headset = 'modern', + [ValidateSet('modern', 'quest1', 'frame')] [string]$Headset = 'modern', [ValidateSet('debug', 'release')] [string]$Configuration = 'debug', [int]$Seconds = 60, [switch]$NoLaunch, @@ -52,7 +52,7 @@ if (-not $devices) { throw 'No device in "device" state; check the Quest is conn if (-not $SkipInstall) { if (-not $Apk) { - $flavour = if ($Headset -eq 'quest1') { 'quest1' } else { 'modernQuest' } + $flavour = switch ($Headset) { 'quest1' { 'quest1' } 'frame' { 'steamFrame' } default { 'modernQuest' } } $apkDir = Join-Path $root "app\build\outputs\apk\$flavour\$Configuration" $Apk = Get-ChildItem -Path $apkDir -Filter '*.apk' -ErrorAction SilentlyContinue | Select-Object -First 1 | ForEach-Object FullName diff --git a/android/app/build.gradle.kts b/android/app/build.gradle.kts index b22638a..e71493b 100644 --- a/android/app/build.gradle.kts +++ b/android/app/build.gradle.kts @@ -53,6 +53,19 @@ val hasRetroRewindShards: Boolean = run { shards.isFile && shards.readText().contains("set(MKW_HAVE_RETRO_REWIND_SHARDS ON)") } +// Each headset flavour's AArch64 -mcpu target. The game kit records it, and the kit export, the APK +// audit (Build-Quest.ps1), the PC build (QuestGameKit.psm1), the on-headset build and the package +// import all refuse a game built for another one. +val headsetCpus = mapOf( + // Snapdragon XR2 and newer. Keep this as the default build target. + "modernQuest" to "cortex-a77", + // Snapdragon 835. cortex-a77 binaries terminate with SIGILL on Quest 1. + "quest1" to "kryo", + // Snapdragon 8 Gen 3 (Cortex-X4, A720 and A520, all ARMv9.2). Its firmware does not expose SVE, + // which clang would otherwise auto-vectorise with for a cortex-x4. + "steamFrame" to "cortex-x4+nosve", +) + // android/nod-jni: nod, the disc image library the PC installer runs as nodtool, // cross-compiled with cargo for the launcher's "Select disc image". Needs a Rust // toolchain with the aarch64-linux-android target (see docs/quest-port.md). @@ -152,11 +165,11 @@ android { dimension = "headset" manifestPlaceholders["mkwQuestSupportedDevices"] = "quest2|quest3|quest3s|questpro" buildConfigField("boolean", "QUEST1_DIRECT_LAUNCH", "false") - buildConfigField("String", "ANDROID_CPU", "\"cortex-a77\"") + buildConfigField("boolean", "STEAM_FRAME", "false") + buildConfigField("String", "ANDROID_CPU", "\"${headsetCpus.getValue("modernQuest")}\"") externalNativeBuild { cmake { - // Snapdragon XR2 and newer. Keep this as the default build target. - arguments += "-DMKW_ANDROID_CPU=cortex-a77" + arguments += "-DMKW_ANDROID_CPU=${headsetCpus.getValue("modernQuest")}" } } } @@ -164,11 +177,30 @@ android { dimension = "headset" manifestPlaceholders["mkwQuestSupportedDevices"] = "quest|quest2" buildConfigField("boolean", "QUEST1_DIRECT_LAUNCH", "true") - buildConfigField("String", "ANDROID_CPU", "\"kryo\"") + buildConfigField("boolean", "STEAM_FRAME", "false") + buildConfigField("String", "ANDROID_CPU", "\"${headsetCpus.getValue("quest1")}\"") externalNativeBuild { cmake { - // Snapdragon 835. cortex-a77 binaries terminate with SIGILL on Quest 1. - arguments += "-DMKW_ANDROID_CPU=kryo" + arguments += "-DMKW_ANDROID_CPU=${headsetCpus.getValue("quest1")}" + } + } + } + // Valve's Steam Frame, through Lepton (SteamOS's Android layer) and SteamVR's Android + // OpenXR runtime. Its manifest (src/steamFrame) drops the Horizon OS entries, and + // MKW_ANDROID_HEADSET gives the runtime the Frame's defaults (runtime_config.h). + create("steamFrame") { + dimension = "headset" + // Horizon OS only; src/steamFrame/AndroidManifest.xml removes the entry. + manifestPlaceholders["mkwQuestSupportedDevices"] = "" + buildConfigField("boolean", "QUEST1_DIRECT_LAUNCH", "false") + buildConfigField("boolean", "STEAM_FRAME", "true") + buildConfigField("String", "ANDROID_CPU", "\"${headsetCpus.getValue("steamFrame")}\"") + externalNativeBuild { + cmake { + arguments += listOf( + "-DMKW_ANDROID_CPU=${headsetCpus.getValue("steamFrame")}", + "-DMKW_ANDROID_HEADSET=steam_frame", + ) } } } @@ -324,7 +356,7 @@ androidComponents { dependsOn("merge${capitalized}NativeLibs") appDir.set(layout.projectDirectory) script.set(rootProject.layout.projectDirectory.file("Export-QuestGameKit.ps1")) - androidCpu.set(if (variant.name.startsWith("quest1", ignoreCase = true)) "kryo" else "cortex-a77") + androidCpu.set(headsetCpus.getValue(checkNotNull(variant.flavorName) { "${variant.name} has no headset flavour" })) outputDir.set(layout.buildDirectory.dir("generated/assets/questGameKit/${variant.name}")) llvmStrip.set(sdkComponents.ndkDirectory.map { it.file("toolchains/llvm/prebuilt/$host/bin/llvm-strip" + if (host.startsWith("windows")) ".exe" else "") diff --git a/android/app/src/steamFrame/AndroidManifest.xml b/android/app/src/steamFrame/AndroidManifest.xml new file mode 100644 index 0000000..4979970 --- /dev/null +++ b/android/app/src/steamFrame/AndroidManifest.xml @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/android/app/src/steamFrame/java/org/wiicompiled/quest/FrameEntryActivity.kt b/android/app/src/steamFrame/java/org/wiicompiled/quest/FrameEntryActivity.kt new file mode 100644 index 0000000..6099588 --- /dev/null +++ b/android/app/src/steamFrame/java/org/wiicompiled/quest/FrameEntryActivity.kt @@ -0,0 +1,52 @@ +package org.wiicompiled.quest + +import android.app.Activity +import android.content.Intent +import android.os.Bundle +import android.util.Log +import org.wiicompiled.quest.launcher.LauncherActivity +import org.wiicompiled.quest.launcher.ModLibrary + +/** + * The Steam Frame's library entry. Lepton starts the one real activity that is both MAIN and + * LAUNCHER, in VR when it carries a VR category, and ignores activity-aliases, so neither of the + * Quest flavours' entries works there. This one shows nothing: the setup panel (LauncherActivity) + * always opens, and when the selected game can start as it is, the game opens on top of it, so the + * headset goes straight into VR and quitting the game comes back to the panel. + */ +class FrameEntryActivity : Activity() { + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + val launcher = Intent(this, LauncherActivity::class.java) + if (readyToPlay()) { + Log.i(TAG, "Starting the game over the setup panel") + startActivities(arrayOf(launcher, Intent(this, QuestActivity::class.java))) + } else { + Log.i(TAG, "The game is not ready to start; opening the setup panel") + startActivity(launcher) + } + finish() + } + + /** + * What Home's Play needs (LauncherActivity), short of the steps only the panel performs: the + * game files and the selected game built for this app, Retro Rewind's pack, and no enabled mods + * to copy into the pack's Patches folder first. + */ + private fun readyToPlay(): Boolean = runCatching { + val profile = GameProfile.selected(this) + GameStorage.discStatus(this) == GameStorage.DiscStatus.Ready && + GameLibrary.status(this, profile) == GameLibrary.Status.Ready && + GameStorage.modContentReady(this, profile) && + (!profile.modPack || ModLibrary.load(GameStorage.modsDirectory(this)).none { it.enabled }) + }.getOrElse { + Log.w(TAG, "Could not check the installed game", it) + false + } + + private companion object { + // The launcher's tag, so the usual `adb logcat -s WiiCompiledLauncher` shows the routing. + const val TAG = "WiiCompiledLauncher" + } +} diff --git a/runtime/CMakeLists.txt b/runtime/CMakeLists.txt index 3a2cb1e..5820ffb 100644 --- a/runtime/CMakeLists.txt +++ b/runtime/CMakeLists.txt @@ -37,6 +37,21 @@ option(MKW_BUILD_PRODUCTS "Build translated WiiCompiled product targets" ON) # aurora-main and its third-party tree stay unaffected. set(MKW_PROJECT_COMPILE_DEFINITIONS NOMINMAX) +# The Android headset family the build is for (the Gradle flavour passes it). The +# Steam Frame build runs under SteamVR's Android runtime rather than Horizon OS and +# takes its own defaults (runtime_config.h) through MKW_HEADSET_STEAM_FRAME. The +# definition reaches every target below, so the game kit's recorded compile flags +# carry it too. +if(MKW_PLATFORM_ANDROID) + set(MKW_ANDROID_HEADSET "quest" CACHE STRING "Android headset family: quest (Meta Quest, any flavour) or steam_frame") + set_property(CACHE MKW_ANDROID_HEADSET PROPERTY STRINGS quest steam_frame) + if(MKW_ANDROID_HEADSET STREQUAL "steam_frame") + list(APPEND MKW_PROJECT_COMPILE_DEFINITIONS MKW_HEADSET_STEAM_FRAME=1) + elseif(NOT MKW_ANDROID_HEADSET STREQUAL "quest") + message(FATAL_ERROR "MKW_ANDROID_HEADSET must be quest or steam_frame, not '${MKW_ANDROID_HEADSET}'") + endif() +endif() + set(CMAKE_CXX_STANDARD 17) # Declared before the vendored libraries so the third-party block below can diff --git a/runtime/cmake/PublicProducts.cmake b/runtime/cmake/PublicProducts.cmake index 9c78816..4cc7ba2 100644 --- a/runtime/cmake/PublicProducts.cmake +++ b/runtime/cmake/PublicProducts.cmake @@ -448,6 +448,7 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(AMD64|amd64|x86_64|X86_64)$") elseif(MKW_PLATFORM_ANDROID) # Cross-compiled, so "native" would describe the build host. Cortex-A77 is # the Snapdragon XR2 Gen 1 (Quest 2) core; Quest 3 / Pro are supersets. + # Each Gradle headset flavour passes its own (android/app/build.gradle.kts). set(MKW_ANDROID_CPU "cortex-a77" CACHE STRING "AArch64 -mcpu target for the Android products") set(MKW_BASELINE_ARCH_FLAG -mcpu=${MKW_ANDROID_CPU}) elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64|ARM64)$") diff --git a/runtime/include/vr/openxr_input.h b/runtime/include/vr/openxr_input.h index e1f3b4e..759b40c 100644 --- a/runtime/include/vr/openxr_input.h +++ b/runtime/include/vr/openxr_input.h @@ -193,6 +193,11 @@ private: XrAction m_button_primary = XR_NULL_HANDLE; // A / X XrAction m_button_secondary = XR_NULL_HANDLE; // B / Y XrAction m_menu = XR_NULL_HANDLE; + // The Steam Frame's left D-pad (valve/frame_controller_valve); Touch has none. + XrAction m_dpad_up = XR_NULL_HANDLE; + XrAction m_dpad_down = XR_NULL_HANDLE; + XrAction m_dpad_left = XR_NULL_HANDLE; + XrAction m_dpad_right = XR_NULL_HANDLE; XrAction m_aim_pose = XR_NULL_HANDLE; XrAction m_grip_pose = XR_NULL_HANDLE; XrAction m_haptic = XR_NULL_HANDLE; diff --git a/runtime/include/vr/openxr_wii_remote.h b/runtime/include/vr/openxr_wii_remote.h index 666b2f6..bacbbc0 100644 --- a/runtime/include/vr/openxr_wii_remote.h +++ b/runtime/include/vr/openxr_wii_remote.h @@ -341,6 +341,12 @@ struct HandInputs { float squeeze = 0.0f; float stick_x = 0.0f; float stick_y = 0.0f; // +up + // A D-pad, which only the Steam Frame's left controller has + // (valve/frame_controller_valve). + bool dpad_up = false; + bool dpad_down = false; + bool dpad_left = false; + bool dpad_right = false; }; // Adapted from DolphinXR's default "OpenXR Wii Remote" profile @@ -353,6 +359,8 @@ struct HandInputs { // HOME has no button; left Y opens the settings panel (openxr_settings_panel.h). // The grips press nothing: they take hold of the wheel (openxr_driving.h), and C // is the game's look-behind, which a hand on the wheel would otherwise hold down. +// A controller D-pad (the Steam Frame's left one) is the remote's D-pad; the +// Frame's left View button is the left menu and its left shoulder the left Y. inline uint32_t RemoteButtons(const HandInputs& left, const HandInputs& right) noexcept { uint32_t hold = 0; const auto press = [&hold](bool held, uint32_t bit) { @@ -367,6 +375,10 @@ inline uint32_t RemoteButtons(const HandInputs& left, const HandInputs& right) n press(right.stick_y < -kPressThreshold, kButtonTwo); press(left.primary || left.menu, kButtonPlus); press(left.trigger > kPressThreshold, kButtonZ); + press(left.dpad_up || right.dpad_up, kButtonUp); + press(left.dpad_down || right.dpad_down, kButtonDown); + press(left.dpad_left || right.dpad_left, kButtonLeft); + press(left.dpad_right || right.dpad_right, kButtonRight); return hold; } diff --git a/runtime/src/vr/openxr_input.cpp b/runtime/src/vr/openxr_input.cpp index bc4b3e7..001baf2 100644 --- a/runtime/src/vr/openxr_input.cpp +++ b/runtime/src/vr/openxr_input.cpp @@ -146,7 +146,9 @@ private: (1u << SDL_GAMEPAD_BUTTON_START) | (1u << SDL_GAMEPAD_BUTTON_LEFT_STICK) | (1u << SDL_GAMEPAD_BUTTON_RIGHT_STICK) | (1u << SDL_GAMEPAD_BUTTON_LEFT_SHOULDER) | - (1u << SDL_GAMEPAD_BUTTON_RIGHT_SHOULDER); + (1u << SDL_GAMEPAD_BUTTON_RIGHT_SHOULDER) | + (1u << SDL_GAMEPAD_BUTTON_DPAD_UP) | (1u << SDL_GAMEPAD_BUTTON_DPAD_DOWN) | + (1u << SDL_GAMEPAD_BUTTON_DPAD_LEFT) | (1u << SDL_GAMEPAD_BUTTON_DPAD_RIGHT); desc.axis_mask = (1u << SDL_GAMEPAD_AXIS_LEFTX) | (1u << SDL_GAMEPAD_AXIS_LEFTY) | (1u << SDL_GAMEPAD_AXIS_RIGHTX) | (1u << SDL_GAMEPAD_AXIS_RIGHTY) | (1u << SDL_GAMEPAD_AXIS_LEFT_TRIGGER) | (1u << SDL_GAMEPAD_AXIS_RIGHT_TRIGGER); @@ -394,7 +396,7 @@ bool OpenXRInput::CreateActions() { const char* localized; XrActionType type; }; - const std::array specs{{ + const std::array specs{{ {&m_thumbstick, "thumbstick", "Thumbstick", XR_ACTION_TYPE_VECTOR2F_INPUT}, {&m_thumbstick_click, "thumbstick_click", "Thumbstick Click", XR_ACTION_TYPE_BOOLEAN_INPUT}, {&m_trigger, "trigger", "Trigger", XR_ACTION_TYPE_FLOAT_INPUT}, @@ -402,6 +404,10 @@ bool OpenXRInput::CreateActions() { {&m_button_primary, "button_primary", "A / X", XR_ACTION_TYPE_BOOLEAN_INPUT}, {&m_button_secondary, "button_secondary", "B / Y", XR_ACTION_TYPE_BOOLEAN_INPUT}, {&m_menu, "menu", "Menu", XR_ACTION_TYPE_BOOLEAN_INPUT}, + {&m_dpad_up, "dpad_up", "D-pad Up", XR_ACTION_TYPE_BOOLEAN_INPUT}, + {&m_dpad_down, "dpad_down", "D-pad Down", XR_ACTION_TYPE_BOOLEAN_INPUT}, + {&m_dpad_left, "dpad_left", "D-pad Left", XR_ACTION_TYPE_BOOLEAN_INPUT}, + {&m_dpad_right, "dpad_right", "D-pad Right", XR_ACTION_TYPE_BOOLEAN_INPUT}, {&m_aim_pose, "aim_pose", "Pointer", XR_ACTION_TYPE_POSE_INPUT}, {&m_grip_pose, "grip_pose", "Motion", XR_ACTION_TYPE_POSE_INPUT}, {&m_haptic, "haptic", "Haptic", XR_ACTION_TYPE_VIBRATION_OUTPUT}, @@ -493,6 +499,40 @@ bool OpenXRInput::SuggestBindings() { {&m_haptic, "/user/hand/right/output/haptic"}, }; suggest("/interaction_profiles/khr/simple_controller", simple, false); + + // The Steam Frame's controllers (XR_VALVE_frame_controller_interaction). Without the profile + // SteamVR presents them as Touch controllers, which loses the left D-pad. Its left hand has a + // D-pad where Touch has X and Y, a View button for the menu and a shoulder button, which takes + // left Y's place as the settings panel's button. Right X, Y, menu and shoulder stay free. + const auto& extensions = m_runtime->EnabledExtensions(); + if (std::find(extensions.begin(), extensions.end(), "XR_VALVE_frame_controller_interaction") != + extensions.end()) { + const std::vector frame{ + {&m_thumbstick, "/user/hand/left/input/thumbstick"}, + {&m_thumbstick, "/user/hand/right/input/thumbstick"}, + {&m_thumbstick_click, "/user/hand/left/input/thumbstick/click"}, + {&m_thumbstick_click, "/user/hand/right/input/thumbstick/click"}, + {&m_trigger, "/user/hand/left/input/trigger/value"}, + {&m_trigger, "/user/hand/right/input/trigger/value"}, + {&m_squeeze, "/user/hand/left/input/squeeze/value"}, + {&m_squeeze, "/user/hand/right/input/squeeze/value"}, + {&m_button_primary, "/user/hand/right/input/a/click"}, + {&m_button_secondary, "/user/hand/right/input/b/click"}, + {&m_button_secondary, "/user/hand/left/input/shoulder/click"}, + {&m_menu, "/user/hand/left/input/view/click"}, + {&m_dpad_up, "/user/hand/left/input/dpad_up/click"}, + {&m_dpad_down, "/user/hand/left/input/dpad_down/click"}, + {&m_dpad_left, "/user/hand/left/input/dpad_left/click"}, + {&m_dpad_right, "/user/hand/left/input/dpad_right/click"}, + {&m_aim_pose, "/user/hand/left/input/aim/pose"}, + {&m_aim_pose, "/user/hand/right/input/aim/pose"}, + {&m_grip_pose, "/user/hand/left/input/grip/pose"}, + {&m_grip_pose, "/user/hand/right/input/grip/pose"}, + {&m_haptic, "/user/hand/left/output/haptic"}, + {&m_haptic, "/user/hand/right/output/haptic"}, + }; + suggest("/interaction_profiles/valve/frame_controller_valve", frame, false); + } return true; } @@ -868,6 +908,7 @@ void OpenXRInput::Destroy() { } m_thumbstick = m_thumbstick_click = m_trigger = m_squeeze = XR_NULL_HANDLE; m_button_primary = m_button_secondary = m_menu = m_haptic = XR_NULL_HANDLE; + m_dpad_up = m_dpad_down = m_dpad_left = m_dpad_right = XR_NULL_HANDLE; m_aim_pose = m_grip_pose = XR_NULL_HANDLE; m_hand_paths[0] = m_hand_paths[1] = XR_NULL_PATH; m_convert_now_to_xr_time = nullptr; @@ -983,6 +1024,10 @@ void OpenXRInput::Sync(XrTime predicted_display_time, const OpenXRPointerScreen& inputs.primary = boolean(m_button_primary, hand, &primary_active); inputs.secondary = boolean(m_button_secondary, hand, &secondary_active); inputs.menu = boolean(m_menu, hand); + inputs.dpad_up = boolean(m_dpad_up, hand); + inputs.dpad_down = boolean(m_dpad_down, hand); + inputs.dpad_left = boolean(m_dpad_left, hand); + inputs.dpad_right = boolean(m_dpad_right, hand); inputs.thumbstick_click = boolean(m_thumbstick_click, hand); inputs.trigger = scalar(m_trigger, hand); inputs.squeeze = scalar(m_squeeze, hand, &squeeze_active); @@ -1165,6 +1210,10 @@ void OpenXRInput::Sync(XrTime predicted_display_time, const OpenXRPointerScreen& pad.buttons[SDL_GAMEPAD_BUTTON_RIGHT_STICK] = right.thumbstick_click; pad.buttons[SDL_GAMEPAD_BUTTON_LEFT_SHOULDER] = left.squeeze > 0.5f; pad.buttons[SDL_GAMEPAD_BUTTON_RIGHT_SHOULDER] = right.squeeze > 0.5f; + pad.buttons[SDL_GAMEPAD_BUTTON_DPAD_UP] = left.dpad_up || right.dpad_up; + pad.buttons[SDL_GAMEPAD_BUTTON_DPAD_DOWN] = left.dpad_down || right.dpad_down; + pad.buttons[SDL_GAMEPAD_BUTTON_DPAD_LEFT] = left.dpad_left || right.dpad_left; + pad.buttons[SDL_GAMEPAD_BUTTON_DPAD_RIGHT] = left.dpad_right || right.dpad_right; Relay().Publish(pad); } diff --git a/runtime/src/vr/openxr_integration.cpp b/runtime/src/vr/openxr_integration.cpp index 86b30e3..e91a65e 100644 --- a/runtime/src/vr/openxr_integration.cpp +++ b/runtime/src/vr/openxr_integration.cpp @@ -407,7 +407,8 @@ public: #if defined(_WIN32) config.required_extensions = {kRequiredAuroraBackend == BACKEND_VULKAN ? "XR_KHR_vulkan_enable2" : "XR_KHR_D3D12_enable"}; config.optional_extensions = {"XR_KHR_win32_convert_performance_counter_time", - "XR_FB_display_refresh_rate", "XR_EXT_performance_settings"}; + "XR_FB_display_refresh_rate", "XR_EXT_performance_settings", + "XR_VALVE_frame_controller_interaction"}; AddHandMeshExtensions(config); #else // Either Vulkan binding extension is acceptable; the backend picks @@ -415,11 +416,13 @@ public: config.required_extensions = {"XR_KHR_android_create_instance"}; // XR_FB_passthrough: the room around the virtual screen (OpenXRPassthrough), asked for // whatever [vr] passthrough says, since the setting is live. + // XR_VALVE_frame_controller_interaction: the Steam Frame's controllers, D-pad included + // (OpenXRInput::SuggestBindings), under SteamVR here or streamed from a PC. config.optional_extensions = {"XR_KHR_vulkan_enable2", "XR_KHR_vulkan_enable", "XR_KHR_convert_timespec_time", "XR_KHR_android_thread_settings", "XR_FB_display_refresh_rate", "XR_EXT_performance_settings", - "XR_FB_passthrough"}; + "XR_FB_passthrough", "XR_VALVE_frame_controller_interaction"}; AddHandMeshExtensions(config); config.instance_create_next = OpenXRAndroidInstanceCreateNext(); #endif diff --git a/runtime/tests/vr_wii_remote_tests.cpp b/runtime/tests/vr_wii_remote_tests.cpp index 7ffe66d..1feed26 100644 --- a/runtime/tests/vr_wii_remote_tests.cpp +++ b/runtime/tests/vr_wii_remote_tests.cpp @@ -282,6 +282,19 @@ void TestButtons() { left.menu = true; Check((RemoteButtons(left, right) & kButtonHome) == 0, "left menu is no longer HOME"); + // The Steam Frame's left D-pad is the remote's, one bit per direction. + left = {}; + left.dpad_up = true; + Check(RemoteButtons(left, right) == kButtonUp, "D-pad up is the remote's up"); + left = {}; + left.dpad_down = true; + left.dpad_left = true; + Check(RemoteButtons(left, right) == (kButtonDown | kButtonLeft), "D-pad down and left together"); + left = {}; + right.dpad_right = true; + Check(RemoteButtons(left, right) == kButtonRight, "a D-pad on either hand counts"); + right = {}; + left.stick_x = 1.0f; left.stick_y = 1.0f; const std::array stick = NunchukStick(left); From 8b983304856caa78feeb4324ba049a465ff9d504 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 08:32:33 +0000 Subject: [PATCH 02/14] Ask the headset for a configured refresh rate, 120 Hz on the Steam Frame [vr] refresh_rate (Hz, 0 = the headset's own) is requested through XR_FB_display_refresh_rate each time the session starts running and whenever the setting changes, matched against the runtime's own list within half a hertz. Setting it back to 0 restores the rate the session started at. A rate that is not offered, or one the runtime declines, is logged and changes nothing. The game renders 60 frames a second, so at 120 Hz every frame shows for exactly two refreshes, where 72 or 90 Hz hold some frames longer than others. The Steam Frame build defaults to 120; every other build keeps the headset's own rate unless the player picks one. The F10/headset panel and the Android launcher's Headset section offer the choice; the interpolation tooltip points at it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_019HBRGKTE1GnN2ah8gcZKr3 --- .../quest/launcher/SettingsPage.kt | 15 ++++ android/app/src/main/res/values/strings.xml | 3 + runtime/include/runtime_config.h | 41 +++++++++ .../include/vr/frame_interpolation_pacing.h | 15 ++++ runtime/src/settings_overlay.cpp | 28 +++++- runtime/src/vr/openxr_integration.cpp | 89 +++++++++++++++++++ .../frame_interpolation_pacing_tests.cpp | 7 ++ runtime/tests/vr_config_tests.cpp | 18 ++++ 8 files changed, 215 insertions(+), 1 deletion(-) diff --git a/android/app/src/main/java/org/wiicompiled/quest/launcher/SettingsPage.kt b/android/app/src/main/java/org/wiicompiled/quest/launcher/SettingsPage.kt index f218f9b..3789507 100644 --- a/android/app/src/main/java/org/wiicompiled/quest/launcher/SettingsPage.kt +++ b/android/app/src/main/java/org/wiicompiled/quest/launcher/SettingsPage.kt @@ -243,6 +243,13 @@ class SettingsPage( read = { stringIndex(it, "vr", "performance_level", PERFORMANCE_LEVELS) }, write = { c, index -> c.setString("vr", "performance_level", PERFORMANCE_LEVELS[index]) }, ) + choice( + R.string.vr_refresh_rate, R.string.vr_refresh_rate_helper, + listOf(activity.getString(R.string.vr_refresh_rate_headset), "72 Hz", "90 Hz", "120 Hz", "144 Hz"), + read = { REFRESH_RATES.indexOf(vrRefreshRate(it)) }, + write = { c, index -> c.setInteger("vr", "refresh_rate", REFRESH_RATES[index]) }, + custom = { "${vrRefreshRate(it)} Hz" }, + ) choice( R.string.vr_foveation, R.string.vr_foveation_helper, listOf( @@ -784,6 +791,10 @@ class SettingsPage( val FOVEATION_DEFAULT = FOVEATION_LEVELS.indexOf("medium") val CONTROLLER_MODES = listOf("wii_remote", "gamepad", "none") val INTERPOLATION_FPS = listOf(0L, 1L, 72L, 90L, 120L) + // [vr] refresh_rate choices, 0 being the headset's own rate. + val REFRESH_RATES = listOf(0L, 72L, 90L, 120L, 144L) + /** runtime_config.h's kVrRefreshRateDefault: 120 on the Steam Frame, the headset's own elsewhere. */ + val REFRESH_RATE_DEFAULT = if (BuildConfig.STEAM_FRAME) 120L else 0L val RESOLUTIONS = listOf(1.0, 1.5, 2.0, 3.0, 4.0) val SUPPORTED_RESOLUTIONS = listOf(0.0, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0, 8.0) const val BLOOM_PATH = 0x10L @@ -813,6 +824,10 @@ class SettingsPage( config.integer("video", "disabled_post_processing_paths") ?.takeIf { it in 0L..UINT32_MAX && (it and BLOOM_PATH.inv()) == 0L } ?: BLOOM_PATH + /** vr.refresh_rate where IsSupportedVrRefreshRate accepts it, else the runtime's default. */ + fun vrRefreshRate(config: TomlConfig): Long = + config.integer("vr", "refresh_rate")?.takeIf { it == 0L || it in 60L..240L } ?: REFRESH_RATE_DEFAULT + /** vr.frame_interpolation_fps, with the legacy frame_interpolation switch and NormalizeFrameInterpolationFps. */ fun vrInterpolationFps(config: TomlConfig): Long { val value = config.integer("vr", "frame_interpolation_fps")?.takeIf { it in 0L..UINT32_MAX } diff --git a/android/app/src/main/res/values/strings.xml b/android/app/src/main/res/values/strings.xml index 87e2e12..7a9d218 100644 --- a/android/app/src/main/res/values/strings.xml +++ b/android/app/src/main/res/values/strings.xml @@ -559,6 +559,9 @@ Sustained low Power savings Headset default + Headset refresh rate + The display rate asked of the headset. The game runs at 60 FPS, so 120 Hz shows every frame for exactly two refreshes, where 72 and 90 Hz hold some frames longer than others. A rate the headset does not offer leaves its own. + Headset\'s own Foveated rendering Shades the edges of the race view in coarser blocks, where the lenses blur the picture anyway, to free GPU time for a higher render scale or a steadier frame rate. Higher levels start closer to the centre; High also coarsens the corners of the HUD. Menus are never foveated. Off diff --git a/runtime/include/runtime_config.h b/runtime/include/runtime_config.h index 1193c64..f2e45b4 100644 --- a/runtime/include/runtime_config.h +++ b/runtime/include/runtime_config.h @@ -63,6 +63,7 @@ struct RuntimeUserConfig { std::optional vrMirrorView; std::optional vrControllerMode; std::optional vrFrameInterpolationFps; + std::optional vrRefreshRate; std::optional vrFirstPerson; std::optional vrFirstPersonToggleClick; std::optional vrFirstPersonUnitsPerMeter; @@ -281,6 +282,25 @@ inline bool IsSupportedVrPerformanceLevel(std::string_view value) { return value == "default" || value == "power_savings" || value == "sustained_low" || value == "sustained_high" || value == "boost"; } +// The display refresh rate asked of the OpenXR runtime (XR_FB_display_refresh_rate), in Hz, each +// time the session starts and whenever the setting changes; 0 leaves the headset's own rate. The +// game renders 60 frames a second, so at 120 Hz every frame shows for exactly two refreshes, where +// 72 or 90 Hz hold some frames longer than others. The Steam Frame starts at 120; elsewhere the +// headset's own setting stays in charge. A runtime without the extension, or one that declines the +// rate, keeps its own. +#if defined(MKW_HEADSET_STEAM_FRAME) +inline constexpr uint32_t kVrRefreshRateDefault = 120; +#define MKW_VR_REFRESH_RATE_DEFAULT_TEXT "120" +#else +inline constexpr uint32_t kVrRefreshRateDefault = 0; +#define MKW_VR_REFRESH_RATE_DEFAULT_TEXT "0" +#endif +inline constexpr uint32_t kVrRefreshRateMin = 60; +inline constexpr uint32_t kVrRefreshRateMax = 240; + +inline bool IsSupportedVrRefreshRate(uint32_t value) { + return value == 0 || (value >= kVrRefreshRateMin && value <= kVrRefreshRateMax); +} // Fixed foveated rendering of the immersive eyes on the Quest, in the order of // aurora_set_stereo_foveation's levels: the periphery is shaded in 2x2, then // 4x4 pixel blocks, the higher the level the closer to the centre. Whether the @@ -545,6 +565,11 @@ inline void EnsureConfigFile() { "controller_mode = \"wii_remote\"\n" "# VR interpolation: 0 = Off, 1 = Auto, or 72/90/120 FPS. Live.\n" "frame_interpolation_fps = 0\n" + "# Display refresh rate asked of the headset, in Hz (72, 90, 120,\n" + "# 144, ...), or 0 to leave the headset's own setting. The game runs\n" + "# at 60, so 120 shows every frame twice. Only runtimes that let apps\n" + "# choose (XR_FB_display_refresh_rate) take it. Live.\n" + "refresh_rate = " MKW_VR_REFRESH_RATE_DEFAULT_TEXT "\n" "render_scale = " MKW_VR_RENDER_SCALE_DEFAULT_TEXT "\n" "world_units_per_meter = 500.0\n" "hud_distance_meters = 2.0\n" @@ -872,6 +897,9 @@ inline RuntimeUserConfig ParseConfigDocument(const toml::value& document) { value && IsSupportedVrControllerMode(*value)) { config.vrControllerMode = *value; } + if (auto value = FindConfigUint(document, "vr", "refresh_rate"); value && IsSupportedVrRefreshRate(*value)) { + config.vrRefreshRate = *value; + } config.vrFrameInterpolationFps = FindConfigValue(document, "vr", "frame_interpolation_fps"); if (!config.vrFrameInterpolationFps) { // Migrate the initial experimental checkbox to Auto. @@ -1241,6 +1269,14 @@ inline bool SetVrFrameInterpolationFps(uint32_t value) { return WriteSetting("vr", "frame_interpolation_fps", std::to_string(value)); } +inline bool SetVrRefreshRate(uint32_t value) { + if (!IsSupportedVrRefreshRate(value)) { + return false; + } + Mutable().vrRefreshRate = value; + return WriteSetting("vr", "refresh_rate", std::to_string(value)); +} + inline bool SetVrFirstPersonRotation(std::string value) { if (!IsSupportedVrFirstPersonRotation(value)) { return false; @@ -1743,6 +1779,11 @@ inline uint32_t VrFrameInterpolationFps() { return mkw::vr::NormalizeFrameInterpolationFps(Get().vrFrameInterpolationFps.value_or(0)); } +inline uint32_t VrRefreshRate(uint32_t fallback = kVrRefreshRateDefault) { + const uint32_t value = Get().vrRefreshRate.value_or(fallback); + return IsSupportedVrRefreshRate(value) ? value : 0; +} + inline bool DiagnosticsOpenXRLogging(bool fallback = false) { return Get().diagnosticsOpenXRLogging.value_or(fallback); } diff --git a/runtime/include/vr/frame_interpolation_pacing.h b/runtime/include/vr/frame_interpolation_pacing.h index e4efaab..52232fb 100644 --- a/runtime/include/vr/frame_interpolation_pacing.h +++ b/runtime/include/vr/frame_interpolation_pacing.h @@ -9,6 +9,21 @@ inline uint32_t NormalizeFrameInterpolationFps(uint32_t value) noexcept { return value == 1 || value == 72 || value == 90 || value == 120 ? value : 0; } +// The display refresh rate to ask the runtime for ([vr] refresh_rate): the one it lists within half +// a hertz of `requested` (runtimes report 119.9 or 120.00001 for 120), or 0 when none is close. +inline float MatchDisplayRefreshRate(const float* rates, uint32_t count, float requested) noexcept { + float best = 0.0f; + float best_distance = 0.5f; + for (uint32_t i = 0; i < count; ++i) { + const float distance = rates[i] > requested ? rates[i] - requested : requested - rates[i]; + if (rates[i] > 0.0f && distance <= best_distance) { + best = rates[i]; + best_distance = distance; + } + } + return best; +} + // A render-rate ceiling on the compositor's own display-time grid. Auto (1) // renders every tick. Fixed targets cannot increase the physical refresh rate. class FrameInterpolationPacing { diff --git a/runtime/src/settings_overlay.cpp b/runtime/src/settings_overlay.cpp index 2e568f6..4492c01 100644 --- a/runtime/src/settings_overlay.cpp +++ b/runtime/src/settings_overlay.cpp @@ -187,6 +187,9 @@ int g_vrFrameInterpolationMode = [] { return static_cast(std::find(kVrInterpolationFps.begin(), kVrInterpolationFps.end(), value) - kVrInterpolationFps.begin()); }(); +// [vr] refresh_rate choices; a value set in Config.toml outside them is shown as it is. +constexpr std::array kVrRefreshRates{0, 72, 90, 120, 144}; +uint32_t g_vrRefreshRate = RuntimeConfigFile::VrRefreshRate(); int g_vrFirstPersonHiddenModel = RuntimeConfigFile::VrFirstPersonHiddenModel(); bool g_openxrDiagnosticsLogging = RuntimeConfigFile::DiagnosticsOpenXRLogging(false); bool g_firstPersonDiagnosticsLogging = RuntimeConfigFile::DiagnosticsFirstPersonLogging(false); @@ -1517,6 +1520,29 @@ void DrawVrSettings() { "off, or without a GPU that supports it: a new level applies after a restart."); } #endif + { + const auto rateLabel = [](uint32_t hz) { + return hz == 0 ? std::string("Headset's own") : std::to_string(hz) + " Hz"; + }; + if (ImGui::BeginCombo("Headset refresh rate", rateLabel(g_vrRefreshRate).c_str())) { + for (const uint32_t hz : kVrRefreshRates) { + if (ImGui::Selectable(rateLabel(hz).c_str(), hz == g_vrRefreshRate) && hz != g_vrRefreshRate) { + g_vrRefreshRate = hz; + // The XR thread follows the setting and asks the runtime for it. + RuntimeConfigFile::SetVrRefreshRate(hz); + } + } + ImGui::EndCombo(); + } + if (ImGui::IsItemHovered()) { + ImGui::SetTooltip( + "Asks the headset for this display rate. The game runs at 60 FPS, so 120 Hz shows every " + "frame for exactly two refreshes, where 72 and 90 Hz hold some frames longer than others. " + "Only runtimes that let apps choose (XR_FB_display_refresh_rate: the Quest, the Steam " + "Frame, Virtual Desktop) take it; the Headset line below shows the rate in use. Applies " + "immediately."); + } + } if (ImGui::Combo("VR frame interpolation (experimental)", &g_vrFrameInterpolationMode, kVrInterpolationLabels.data(), static_cast(kVrInterpolationLabels.size()))) { const auto target = kVrInterpolationFps[static_cast(g_vrFrameInterpolationMode)]; @@ -1526,7 +1552,7 @@ void DrawVrSettings() { if (ImGui::IsItemHovered()) { ImGui::SetTooltip( "Auto matches the headset refresh rate. 72, 90 and 120 cap the scene rendering rate; " - "set the headset's refresh rate in Virtual Desktop or your VR runtime. " + "set the headset's refresh rate above, or in Virtual Desktop or your VR runtime. " "The game stays at 60 Hz. Adds one game frame of scene latency; head tracking stays current. " "Needs GPU headroom and may show interpolation artifacts. Applies immediately."); } diff --git a/runtime/src/vr/openxr_integration.cpp b/runtime/src/vr/openxr_integration.cpp index e91a65e..81e5651 100644 --- a/runtime/src/vr/openxr_integration.cpp +++ b/runtime/src/vr/openxr_integration.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #if defined(MKW_ENABLE_OPENXR) #include "vr/openxr_backend.h" @@ -446,6 +447,8 @@ public: #endif if (has_extension("XR_FB_display_refresh_rate")) { runtime_->LoadFunction("xrGetDisplayRefreshRateFB", &get_display_refresh_rate_); + runtime_->LoadFunction("xrEnumerateDisplayRefreshRatesFB", &enumerate_refresh_rates_); + runtime_->LoadFunction("xrRequestDisplayRefreshRateFB", &request_refresh_rate_); } if (has_extension("XR_EXT_performance_settings")) { runtime_->LoadFunction("xrPerfSettingsSetPerformanceLevelEXT", &set_performance_level_); @@ -576,6 +579,8 @@ public: prepared_ = false; convert_display_time_ = nullptr; get_display_refresh_rate_ = nullptr; + enumerate_refresh_rates_ = nullptr; + request_refresh_rate_ = nullptr; set_performance_level_ = nullptr; headset_hz_.store(0, std::memory_order_relaxed); rendered_fps_.store(0, std::memory_order_relaxed); @@ -590,6 +595,10 @@ public: ResetTrackingOrigin(); applied_session_run_serial_ = 0; session_was_active_ = false; + refresh_rate_session_serial_ = 0; + requested_refresh_rate_ = 0; + session_start_refresh_rate_ = 0.0f; + refresh_rate_changed_ = false; } bool IsRunning() const noexcept { return running_.load(std::memory_order_acquire); } @@ -797,6 +806,70 @@ private: << (XR_SUCCEEDED(gpu) ? "set" : "refused") << " (" << gpu << ")" << std::endl; } + // [vr] refresh_rate (XR_FB_display_refresh_rate), asked of the runtime each time the session + // starts running and whenever the setting changes. The game renders 60 frames a second, so a + // display at 120 Hz shows each one for exactly two refreshes. 0 gives back the rate the session + // started at, if this changed it. A rate the runtime does not list, or declines (SteamVR may), + // is logged and the runtime keeps its own. + void ApplyRefreshRate(uint32_t requested, bool session_started) { + if (runtime_ == nullptr || !runtime_->HasSession()) { + return; + } + if (session_started) { + session_start_refresh_rate_ = 0.0f; + refresh_rate_changed_ = false; + if (get_display_refresh_rate_ != nullptr && + XR_FAILED(get_display_refresh_rate_(runtime_->Session(), &session_start_refresh_rate_))) { + session_start_refresh_rate_ = 0.0f; + } + } + float target = static_cast(requested); + if (requested == 0) { + if (!refresh_rate_changed_ || !(session_start_refresh_rate_ > 0.0f)) { + return; + } + target = session_start_refresh_rate_; + } + if (enumerate_refresh_rates_ == nullptr || request_refresh_rate_ == nullptr) { + RT_LOG(RT_TAG_RUNTIME) << "OpenXR: display refresh rate " << target + << " Hz not requested: the runtime does not offer XR_FB_display_refresh_rate" + << std::endl; + return; + } + uint32_t count = 0; + XrResult result = enumerate_refresh_rates_(runtime_->Session(), 0, &count, nullptr); + std::vector rates; + if (XR_SUCCEEDED(result) && count > 0) { + rates.resize(count); + result = enumerate_refresh_rates_(runtime_->Session(), count, &count, rates.data()); + rates.resize(XR_SUCCEEDED(result) ? std::min(count, rates.size()) : 0); + } + if (XR_FAILED(result) || rates.empty()) { + RT_LOG(RT_TAG_RUNTIME) << "OpenXR: display refresh rate " << target + << " Hz not requested: the runtime lists no rates (" << result << ")" << std::endl; + return; + } + std::ostringstream available; + for (size_t i = 0; i < rates.size(); ++i) { + available << (i == 0 ? "" : "/") << rates[i]; + } + const float rate = MatchDisplayRefreshRate(rates.data(), static_cast(rates.size()), target); + if (rate == 0.0f) { + RT_LOG(RT_TAG_RUNTIME) << "OpenXR: display refresh rate " << target + << " Hz is not offered (available " << available.str() + << " Hz); keeping the runtime's" << std::endl; + return; + } + const XrResult set = request_refresh_rate_(runtime_->Session(), rate); + if (XR_SUCCEEDED(set)) { + refresh_rate_changed_ = requested != 0; + } + RT_LOG(RT_TAG_RUNTIME) << "OpenXR: display refresh rate " << rate << " Hz " + << (XR_SUCCEEDED(set) ? "requested" : "refused") << " (" << set << "; available " + << available.str() << " Hz, session started at " << session_start_refresh_rate_ + << " Hz)" << std::endl; + } + static bool ProvideStereoFrame(uint32_t, AuroraStereoFrame* output, void* userdata) { auto* self = static_cast(userdata); if (self == nullptr || output == nullptr) { @@ -894,6 +967,14 @@ private: WaitForStopOrDelay(std::chrono::milliseconds(5)); continue; } + // The configured refresh rate, at each session start and whenever it changes. + if (const uint32_t refresh_rate = RuntimeConfigFile::VrRefreshRate(); + session_run_serial != refresh_rate_session_serial_ || refresh_rate != requested_refresh_rate_) { + const bool session_started = session_run_serial != refresh_rate_session_serial_; + refresh_rate_session_serial_ = session_run_serial; + requested_refresh_rate_ = refresh_rate; + ApplyRefreshRate(refresh_rate, session_started); + } const MkwVRPolicySnapshot policy = MkwVRPolicyGetSnapshot(); aurora_set_stereo_motion_logging(diagnostics::Enabled()); @@ -1874,7 +1955,15 @@ private: std::chrono::steady_clock::time_point timing_start_ = std::chrono::steady_clock::now(); uint32_t timing_submissions_ = 0; PFN_xrGetDisplayRefreshRateFB get_display_refresh_rate_ = nullptr; + PFN_xrEnumerateDisplayRefreshRatesFB enumerate_refresh_rates_ = nullptr; + PFN_xrRequestDisplayRefreshRateFB request_refresh_rate_ = nullptr; PFN_xrPerfSettingsSetPerformanceLevelEXT set_performance_level_ = nullptr; + // [vr] refresh_rate as last applied, the session run it was applied in, and the rate that run + // started at (pacing thread). + uint64_t refresh_rate_session_serial_ = 0; + uint32_t requested_refresh_rate_ = 0; + float session_start_refresh_rate_ = 0.0f; + bool refresh_rate_changed_ = false; #if defined(_WIN32) using ConvertDisplayTime = XrResult (XRAPI_PTR*)(XrInstance, XrTime, LARGE_INTEGER*); #else diff --git a/runtime/tests/frame_interpolation_pacing_tests.cpp b/runtime/tests/frame_interpolation_pacing_tests.cpp index af99498..1f58283 100644 --- a/runtime/tests/frame_interpolation_pacing_tests.cpp +++ b/runtime/tests/frame_interpolation_pacing_tests.cpp @@ -12,6 +12,13 @@ static void Require(bool condition) { } int main() { + // [vr] refresh_rate picks the runtime's own value for the rate asked. + const float frame_rates[] = {72.0f, 90.0f, 119.98f, 144.0f}; + Require(mkw::vr::MatchDisplayRefreshRate(frame_rates, 4, 120.0f) == 119.98f); + Require(mkw::vr::MatchDisplayRefreshRate(frame_rates, 4, 90.0f) == 90.0f); + Require(mkw::vr::MatchDisplayRefreshRate(frame_rates, 4, 60.0f) == 0.0f); + Require(mkw::vr::MatchDisplayRefreshRate(frame_rates, 4, 100.0f) == 0.0f); + Require(mkw::vr::MatchDisplayRefreshRate(frame_rates, 0, 120.0f) == 0.0f); for (uint32_t target : {0u, 1u, 72u, 90u, 120u}) { std::istringstream input("[vr]\nframe_interpolation_fps = " + std::to_string(target) + "\n"); const auto config = RuntimeConfigFile::ParseConfig(input); diff --git a/runtime/tests/vr_config_tests.cpp b/runtime/tests/vr_config_tests.cpp index 2dc3fa6..4cb5e6f 100644 --- a/runtime/tests/vr_config_tests.cpp +++ b/runtime/tests/vr_config_tests.cpp @@ -50,6 +50,24 @@ int main() { Require(RuntimeConfigFile::VrFoveationLevelIndex("high") == 3); Require(RuntimeConfigFile::VrFoveationLevelIndex("ultra") == 0); + // [vr] refresh_rate: Hz asked of the runtime, 0 leaving its own. The Steam Frame + // starts at 120, twice the game's 60. + Require(Parse("[vr]\nrefresh_rate = 120\n").vrRefreshRate == 120u); + Require(Parse("[vr]\nrefresh_rate = 0\n").vrRefreshRate == 0u); + Require(Parse("[vr]\nrefresh_rate = 144\n").vrRefreshRate == 144u); + Require(!Parse("[vr]\nrefresh_rate = 30\n").vrRefreshRate.has_value()); + Require(!Parse("[vr]\nrefresh_rate = 500\n").vrRefreshRate.has_value()); + Require(!Parse("[vr]\nrefresh_rate = -1\n").vrRefreshRate.has_value()); + Require(!Parse("[vr]\nrefresh_rate = \"120\"\n").vrRefreshRate.has_value()); + Require(!Parse("[vr]\n").vrRefreshRate.has_value()); +#if defined(MKW_HEADSET_STEAM_FRAME) + Require(RuntimeConfigFile::kVrRefreshRateDefault == 120u); + Require(std::string_view(MKW_VR_REFRESH_RATE_DEFAULT_TEXT) == "120"); +#else + Require(RuntimeConfigFile::kVrRefreshRateDefault == 0u); + Require(std::string_view(MKW_VR_REFRESH_RATE_DEFAULT_TEXT) == "0"); +#endif + // [vr] hand_tracking: the cockpit hands follow the headset's hand tracking. Require(Parse("[vr]\nhand_tracking = true\n").vrHandTracking == true); Require(Parse("[vr]\nhand_tracking = false\n").vrHandTracking == false); From e626e0ecebb45cf9815f458e87eb81ecec90ee76 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 08:34:04 +0000 Subject: [PATCH 03/14] Give the Steam Frame build its own passthrough default SteamVR has no XR_FB_passthrough, so the Steam Frame build neither asks for it nor offers the setting: [vr] passthrough defaults to false there, and the F10/headset panel and the launcher hide the switch. The Frame keeps the Android defaults that suit it (render scale 0.8, foveation medium, the GX thread, object culling, performance level boost) and starts at 120 Hz (previous commit). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_019HBRGKTE1GnN2ah8gcZKr3 --- .../quest/launcher/SettingsPage.kt | 13 +++++++----- runtime/include/runtime_config.h | 21 +++++++++++++------ runtime/src/settings_overlay.cpp | 4 ++++ runtime/src/vr/openxr_integration.cpp | 6 +++++- runtime/tests/vr_config_tests.cpp | 9 ++++++++ 5 files changed, 41 insertions(+), 12 deletions(-) diff --git a/android/app/src/main/java/org/wiicompiled/quest/launcher/SettingsPage.kt b/android/app/src/main/java/org/wiicompiled/quest/launcher/SettingsPage.kt index 3789507..297dfeb 100644 --- a/android/app/src/main/java/org/wiicompiled/quest/launcher/SettingsPage.kt +++ b/android/app/src/main/java/org/wiicompiled/quest/launcher/SettingsPage.kt @@ -286,11 +286,14 @@ class SettingsPage( format = { "%.1f m".format(it) }, write = { c, value -> c.setFloat("vr", "hud_width_meters", value) }, ) - toggle( - R.string.vr_passthrough, R.string.vr_passthrough_helper, - read = { it.bool("vr", "passthrough") ?: true }, - write = { c, value -> c.setBool("vr", "passthrough", value) }, - ) + // Horizon OS's XR_FB_passthrough; SteamVR on the Steam Frame has no such layer. + if (!BuildConfig.STEAM_FRAME) { + toggle( + R.string.vr_passthrough, R.string.vr_passthrough_helper, + read = { it.bool("vr", "passthrough") ?: true }, + write = { c, value -> c.setBool("vr", "passthrough", value) }, + ) + } } } diff --git a/runtime/include/runtime_config.h b/runtime/include/runtime_config.h index f2e45b4..7dc44a0 100644 --- a/runtime/include/runtime_config.h +++ b/runtime/include/runtime_config.h @@ -189,8 +189,9 @@ inline constexpr uint32_t kPostProcessingBloomPath = 0x10u; inline constexpr uint32_t kDisabledPostProcessingPathsDefault = kPostProcessingBloomPath; // Headset eye size as a fraction of what the OpenXR runtime recommends. A -// standalone headset renders on a mobile GPU, so the Quest starts below it; -// the launcher's first Config.toml (GameStorage.kt) writes the same value. +// standalone headset renders on a mobile GPU, so the Quest and the Steam Frame +// start below it; the launcher's first Config.toml (GameStorage.kt) writes the +// same value. #if defined(__ANDROID__) inline constexpr float kVrRenderScaleDefault = 0.8f; #define MKW_VR_RENDER_SCALE_DEFAULT_TEXT "0.8" @@ -306,7 +307,8 @@ inline bool IsSupportedVrRefreshRate(uint32_t value) { // 4x4 pixel blocks, the higher the level the closer to the centre. Whether the // GPU device gets fragment density maps at all is decided at launch, so going // from "off" to a level takes a restart; between levels and back to "off" it is -// live. The Quest starts at "medium"; elsewhere it does nothing. +// live. The standalone headsets (Quest, Steam Frame) start at "medium"; +// elsewhere it does nothing. #if defined(__ANDROID__) inline constexpr const char* kVrFoveationDefault = "medium"; #else @@ -1719,9 +1721,16 @@ inline bool SetVrRaceView(VrRaceView view) { // The room, through the headset's cameras, around the menu screen and every // other virtual screen, a Flat Screen race included, and around the immersive -// window (never a fully immersive race). Only the Quest offers it; the -// launcher's Settings page shows the same default. -inline bool VrPassthrough(bool fallback = true) { +// window (never a fully immersive race). Only the Quest offers it +// (XR_FB_passthrough, which SteamVR does not have); the launcher's Settings +// page shows the same default. +#if defined(MKW_HEADSET_STEAM_FRAME) +inline constexpr bool kVrPassthroughDefault = false; +#else +inline constexpr bool kVrPassthroughDefault = true; +#endif + +inline bool VrPassthrough(bool fallback = kVrPassthroughDefault) { return Get().vrPassthrough.value_or(fallback); } diff --git a/runtime/src/settings_overlay.cpp b/runtime/src/settings_overlay.cpp index 4492c01..8983d58 100644 --- a/runtime/src/settings_overlay.cpp +++ b/runtime/src/settings_overlay.cpp @@ -147,7 +147,9 @@ int g_vrRaceView = static_cast(RuntimeConfigFile::GetVrRaceView()); bool g_vrFlatScreen = g_vrRaceView == static_cast(RuntimeConfigFile::VrRaceView::FlatScreen); constexpr std::array kVrRaceViewLabels{"Immersive", "Immersive window", "Flat screen"}; #if defined(__ANDROID__) +#if !defined(MKW_HEADSET_STEAM_FRAME) bool g_vrPassthrough = RuntimeConfigFile::VrPassthrough(); +#endif bool g_vrHandTracking = RuntimeConfigFile::VrHandTracking(); // Menu labels for the foveation levels, index-matched to RuntimeConfigFile::kVrFoveationLevels and to // aurora_set_stereo_foveation. @@ -1489,6 +1491,7 @@ void DrawVrSettings() { } } #if defined(__ANDROID__) +#if !defined(MKW_HEADSET_STEAM_FRAME) if (ImGui::Checkbox("Passthrough around the menu screen", &g_vrPassthrough)) { mkw::vr::OpenXRSetPassthrough(g_vrPassthrough); RuntimeConfigFile::SetVrPassthrough(g_vrPassthrough); @@ -1500,6 +1503,7 @@ void DrawVrSettings() { "fully virtual; the immersive window and the Flat Screen race have the room " "around them too. Applies immediately."); } +#endif // Shows the live level Aurora holds. g_vrFoveation = static_cast(aurora_get_stereo_foveation()); if (ImGui::Combo("Foveated rendering", &g_vrFoveation, kVrFoveationLabels.data(), diff --git a/runtime/src/vr/openxr_integration.cpp b/runtime/src/vr/openxr_integration.cpp index 81e5651..8da12af 100644 --- a/runtime/src/vr/openxr_integration.cpp +++ b/runtime/src/vr/openxr_integration.cpp @@ -423,7 +423,11 @@ public: "XR_KHR_convert_timespec_time", "XR_KHR_android_thread_settings", "XR_FB_display_refresh_rate", "XR_EXT_performance_settings", - "XR_FB_passthrough", "XR_VALVE_frame_controller_interaction"}; + "XR_VALVE_frame_controller_interaction"}; +#if !defined(MKW_HEADSET_STEAM_FRAME) + // Horizon OS's room view; the Steam Frame build neither asks for it nor offers the setting. + config.optional_extensions.push_back("XR_FB_passthrough"); +#endif AddHandMeshExtensions(config); config.instance_create_next = OpenXRAndroidInstanceCreateNext(); #endif diff --git a/runtime/tests/vr_config_tests.cpp b/runtime/tests/vr_config_tests.cpp index 4cb5e6f..409a4aa 100644 --- a/runtime/tests/vr_config_tests.cpp +++ b/runtime/tests/vr_config_tests.cpp @@ -68,6 +68,15 @@ int main() { Require(std::string_view(MKW_VR_REFRESH_RATE_DEFAULT_TEXT) == "0"); #endif + // [vr] passthrough: Horizon OS's room view, which the Steam Frame build does not offer. + Require(Parse("[vr]\npassthrough = false\n").vrPassthrough == false); + Require(!Parse("[vr]\n").vrPassthrough.has_value()); +#if defined(MKW_HEADSET_STEAM_FRAME) + Require(!RuntimeConfigFile::kVrPassthroughDefault); +#else + Require(RuntimeConfigFile::kVrPassthroughDefault); +#endif + // [vr] hand_tracking: the cockpit hands follow the headset's hand tracking. Require(Parse("[vr]\nhand_tracking = true\n").vrHandTracking == true); Require(Parse("[vr]\nhand_tracking = false\n").vrHandTracking == false); From b1a8b034d9f358560fb77a0dc89e2e68fa4fa32a Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 08:44:30 +0000 Subject: [PATCH 04/14] Centre foveation on the player's gaze on headsets with eye tracking With [vr] eye_tracked_foveation (on by default on the Steam Frame, off elsewhere) the runtime asks for XR_EXT_eye_gaze_interaction. When the system reports an eye tracker, OpenXRInput binds the gaze pose and locates it for each packet's display time, in the space the eye views are located in; vr/eye_gaze.h turns it into tangents of each eye's own view, which AuroraStereoFrame now carries (appended, after the existing prefix). Aurora centres the eye's fragment density map on the gaze snapped to a cell of two map texels (about 3 degrees). Each eye keeps up to 32 maps, one per cell looked at, so a glance back reuses its map; a new map is bound once its upload completes, and until then the eye keeps the map it had. Without a tracked gaze (a blink, no tracker, the setting off) foveation centres on the forward direction exactly as before: the forward maps are byte-identical. Also logs every extension the OpenXR runtime offers at startup, so the first Steam Frame session shows what SteamVR's Android runtime has. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_019HBRGKTE1GnN2ah8gcZKr3 --- .../quest/launcher/SettingsPage.kt | 8 + android/app/src/main/res/values/strings.xml | 2 + aurora-main/include/aurora/aurora.h | 5 + aurora-main/lib/aurora.cpp | 146 +++++++++++++----- aurora-main/lib/gfx/foveation.hpp | 75 ++++++++- aurora-main/tests/foveation_test.cpp | 107 +++++++++++++ runtime/CMakeLists.txt | 5 + runtime/include/runtime_config.h | 21 +++ runtime/include/vr/eye_gaze.h | 60 +++++++ runtime/include/vr/openxr_input.h | 18 +++ runtime/src/settings_overlay.cpp | 10 ++ runtime/src/vr/openxr_input.cpp | 86 +++++++++++ runtime/src/vr/openxr_integration.cpp | 26 ++++ runtime/src/vr/openxr_runtime.cpp | 8 + runtime/tests/vr_config_tests.cpp | 11 ++ runtime/tests/vr_eye_gaze_tests.cpp | 75 +++++++++ 16 files changed, 616 insertions(+), 47 deletions(-) create mode 100644 runtime/include/vr/eye_gaze.h create mode 100644 runtime/tests/vr_eye_gaze_tests.cpp diff --git a/android/app/src/main/java/org/wiicompiled/quest/launcher/SettingsPage.kt b/android/app/src/main/java/org/wiicompiled/quest/launcher/SettingsPage.kt index 297dfeb..efdfd52 100644 --- a/android/app/src/main/java/org/wiicompiled/quest/launcher/SettingsPage.kt +++ b/android/app/src/main/java/org/wiicompiled/quest/launcher/SettingsPage.kt @@ -259,6 +259,14 @@ class SettingsPage( read = { stringIndex(it, "vr", "foveation", FOVEATION_LEVELS, FOVEATION_DEFAULT) }, write = { c, index -> c.setString("vr", "foveation", FOVEATION_LEVELS[index]) }, ) + // runtime_config.h's kVrEyeTrackedFoveationDefault: on for the Steam Frame, whose eyes are + // tracked; off elsewhere, since Horizon OS asks for an eye tracking permission. + toggle( + R.string.vr_eye_tracked_foveation, R.string.vr_eye_tracked_foveation_helper, + read = { it.bool("vr", "eye_tracked_foveation") ?: BuildConfig.STEAM_FRAME }, + write = { c, value -> c.setBool("vr", "eye_tracked_foveation", value) }, + enabledIf = { stringIndex(it, "vr", "foveation", FOVEATION_LEVELS, FOVEATION_DEFAULT) != 0 }, + ) choice( R.string.vr_interpolation, R.string.vr_interpolation_helper, listOf(activity.getString(R.string.vr_interpolation_off), activity.getString(R.string.vr_interpolation_auto), "72 FPS", "90 FPS", "120 FPS"), diff --git a/android/app/src/main/res/values/strings.xml b/android/app/src/main/res/values/strings.xml index 7a9d218..b9423a1 100644 --- a/android/app/src/main/res/values/strings.xml +++ b/android/app/src/main/res/values/strings.xml @@ -564,6 +564,8 @@ Headset\'s own Foveated rendering Shades the edges of the race view in coarser blocks, where the lenses blur the picture anyway, to free GPU time for a higher render scale or a steadier frame rate. Higher levels start closer to the centre; High also coarsens the corners of the HUD. Menus are never foveated. + Foveation follows the eyes + With eye tracking (the Steam Frame), the sharp centre of the foveated race view moves to where you look instead of staying straight ahead. Off Low Medium diff --git a/aurora-main/include/aurora/aurora.h b/aurora-main/include/aurora/aurora.h index cb5ee48..46bec06 100644 --- a/aurora-main/include/aurora/aurora.h +++ b/aurora-main/include/aurora/aurora.h @@ -209,6 +209,11 @@ typedef struct { // everywhere else, for the host's compositor to show its own background // (the room, on a headset with passthrough) around it. bool window; + // Eye-tracked foveation (aurora_set_stereo_foveation), while gazeValid: where the player looks, in + // tangents of each eye's view (x right, y up, as in the projection's frustum). The immersive eyes' + // full-density region then centres there instead of on each eye's forward direction. + float gaze[AURORA_STEREO_EYE_COUNT][2]; + bool gazeValid; } AuroraStereoFrame; /** diff --git a/aurora-main/lib/aurora.cpp b/aurora-main/lib/aurora.cpp index 9a6cdba..961cfae 100644 --- a/aurora-main/lib/aurora.cpp +++ b/aurora-main/lib/aurora.cpp @@ -607,6 +607,17 @@ std::mutex g_surfaceMutex; std::atomic g_surfaceReconfigurePending{false}; std::atomic g_surfaceRecreatePending{false}; +// One fragment density map of an eye (see StereoEyeTarget): centred on the eye's forward direction, +// or on a gaze cell (gfx/foveation.hpp) with eye-tracked foveation. +struct EyeDensityMap { + gfx::foveation::GazeCell cell; + bool forward = true; + uint64_t map = 0; + uint64_t lastUse = 0; +}; +// The gaze cells' maps an eye keeps: a few glances' worth, each 2 bytes per 32x32 pixels. +constexpr size_t kEyeDensityMapCacheSize = 32; + struct StereoEyeTarget { webgpu::TextureWithSampler color; webgpu::TextureWithSampler resolvedColor; @@ -620,24 +631,31 @@ struct StereoEyeTarget { // the target when ensure_stereo_eye_target replaces the textures. wgpu::BindGroup copyBindGroup; // Foveated rendering: a second view of `color` for the immersive eye passes, - // which the patched Dawn binds to this eye's fragment density map - // (webgpu/fdm.hpp), and what that map was built for. + // which the patched Dawn binds to one of this eye's fragment density maps + // (webgpu/fdm.hpp). The maps share what densityBase records (the eye's size, + // level and field of view); with eye tracking there is one per gaze cell + // looked at, the least recently used dropped beyond kEyeDensityMapCacheSize. wgpu::TextureView foveatedView; - uint64_t densityMap = 0; - std::array densityKey{}; + std::array densityBase{}; + std::vector densityMaps; + uint64_t boundDensityMap = 0; + uint64_t densityUses = 0; const webgpu::TextureWithSampler& output() const noexcept { return resolvedColor.texture ? resolvedColor : color; } }; std::array g_stereoEyeTargets; stereo::MirrorState g_stereoMirrorState; -// The map's binding holds the foveated view, and with it the eye texture, until it is released. +// A map's binding holds the foveated view, and with it the eye texture, until it is released. void release_eye_density_map(StereoEyeTarget& target) noexcept { - if (target.densityMap != 0) { - webgpu::fdm::release_map(target.densityMap); - target.densityMap = 0; + for (const EyeDensityMap& entry : target.densityMaps) { + if (entry.map != 0) { + webgpu::fdm::release_map(entry.map); + } } - target.densityKey = {}; + target.densityMaps.clear(); + target.boundDensityMap = 0; + target.densityBase = {}; } // The eye targets outlive a frame, so the mirror samples them through a bind @@ -689,9 +707,12 @@ void ensure_stereo_eye_target(uint32_t eyeIndex, uint32_t width, uint32_t height } // The view an immersive eye's passes render through while foveated, or none. The eye's fragment -// density map is rebuilt whenever its size, field of view or level changes (a map is immutable), and -// is used once its upload has completed. -wgpu::TextureView foveated_eye_view(uint32_t eyeIndex, const AuroraStereoEye& input) { +// density maps are rebuilt whenever its size, field of view or level changes (a map is immutable). +// `gaze`, the tangents the player looks at when eye tracking provides them, picks the map centred on +// the gaze cell it falls in, built on first use; without it the map is centred on the eye's forward +// direction. A map is bound once its upload has completed, and until then the eye keeps the map it +// had, so a glance never leaves the eye unfoveated. +wgpu::TextureView foveated_eye_view(uint32_t eyeIndex, const AuroraStereoEye& input, const float* gaze) { auto& target = g_stereoEyeTargets[eyeIndex]; const auto level = static_cast(gfx::get_stereo_foveation()); if (level == gfx::foveation::Level::Off || target.samples > 1 || !webgpu::fdm::available()) { @@ -700,42 +721,84 @@ wgpu::TextureView foveated_eye_view(uint32_t eyeIndex, const AuroraStereoEye& in const auto fov = gfx::foveation::fov_from_projection(input.projection); // Hundredths of a tangent: finer than a map texel, coarse enough to ignore pose noise. const auto hundredths = [](float value) { return static_cast(std::lround(value * 100.0f)); }; - const std::array key{static_cast(target.color.size.width), - static_cast(target.color.size.height), - static_cast(level), - hundredths(fov.tanLeft), - hundredths(fov.tanRight), - hundredths(fov.tanDown), - hundredths(fov.tanUp)}; - if (key != target.densityKey) { + const std::array base{static_cast(target.color.size.width), + static_cast(target.color.size.height), + static_cast(level), + hundredths(fov.tanLeft), + hundredths(fov.tanRight), + hundredths(fov.tanDown), + hundredths(fov.tanUp)}; + if (base != target.densityBase) { release_eye_density_map(target); - target.densityKey = key; - if (!target.foveatedView) { - const wgpu::TextureViewDescriptor descriptor{ - .label = eyeIndex == 0 ? "Foveated left eye" : "Foveated right eye", - .usage = wgpu::TextureUsage::RenderAttachment, - }; - target.foveatedView = target.color.texture.CreateView(&descriptor); + target.densityBase = base; + } + if (!target.foveatedView) { + const wgpu::TextureViewDescriptor descriptor{ + .label = eyeIndex == 0 ? "Foveated left eye" : "Foveated right eye", + .usage = wgpu::TextureUsage::RenderAttachment, + }; + target.foveatedView = target.color.texture.CreateView(&descriptor); + } + + const uint32_t width = target.color.size.width; + const uint32_t height = target.color.size.height; + const uint32_t texel = webgpu::fdm::texel_size(); + const bool forward = gaze == nullptr; + const gfx::foveation::GazeCell cell = + forward ? gfx::foveation::GazeCell{} + : gfx::foveation::gaze_cell(width, height, texel, fov, {.tanX = gaze[0], .tanY = gaze[1]}); + auto& maps = target.densityMaps; + auto entry = std::find_if(maps.begin(), maps.end(), [&](const EyeDensityMap& candidate) { + return candidate.forward == forward && (forward || candidate.cell == cell); + }); + if (entry == maps.end()) { + if (maps.size() >= kEyeDensityMapCacheSize) { + // The least recently used map, never the one the eye renders with. + auto oldest = maps.end(); + for (auto it = maps.begin(); it != maps.end(); ++it) { + if (it->map != target.boundDensityMap && (oldest == maps.end() || it->lastUse < oldest->lastUse)) { + oldest = it; + } + } + if (oldest != maps.end()) { + if (oldest->map != 0) { + webgpu::fdm::release_map(oldest->map); + } + maps.erase(oldest); + } } + const bool firstOfKind = + std::none_of(maps.begin(), maps.end(), [&](const EyeDensityMap& other) { return other.forward == forward; }); gfx::foveation::Map map; - gfx::foveation::build(target.color.size.width, target.color.size.height, webgpu::fdm::texel_size(), fov, level, - map); - target.densityMap = webgpu::fdm::create_map(map.width, map.height, map.rg8.data()); - if (target.densityMap != 0 && !webgpu::fdm::bind(target.foveatedView, target.densityMap)) { - webgpu::fdm::release_map(target.densityMap); - target.densityMap = 0; - } + gfx::foveation::build(width, height, texel, fov, level, map, + forward ? gfx::foveation::Gaze{} + : gfx::foveation::cell_gaze(width, height, texel, fov, cell)); + EyeDensityMap created{.cell = cell, .forward = forward}; + created.map = webgpu::fdm::create_map(map.width, map.height, map.rg8.data()); static constexpr std::array kLevelNames{"off", "low", "medium", "high"}; - if (target.densityMap != 0) { - Log.info("{} eye foveation {}: {}x{} density map, {} pixels per texel", eyeIndex == 0 ? "Left" : "Right", - kLevelNames[static_cast(level)], map.width, map.height, webgpu::fdm::texel_size()); - } else { + if (created.map == 0) { Log.warn("{} eye foveation {}: the {}x{} density map could not be created", eyeIndex == 0 ? "Left" : "Right", kLevelNames[static_cast(level)], map.width, map.height); + } else if (firstOfKind) { + // Gaze maps come and go with the player's glances; the first says the eye follows the gaze. + Log.info("{} eye foveation {}{}: {}x{} density map, {} pixels per texel", eyeIndex == 0 ? "Left" : "Right", + kLevelNames[static_cast(level)], forward ? "" : " following the gaze", map.width, map.height, + texel); + } + maps.push_back(created); + entry = std::prev(maps.end()); + } + entry->lastUse = ++target.densityUses; + if (entry->map != 0 && entry->map != target.boundDensityMap && webgpu::fdm::map_ready(entry->map)) { + if (webgpu::fdm::bind(target.foveatedView, entry->map)) { + target.boundDensityMap = entry->map; + } else { + Log.warn("{} eye foveation: a density map could not be bound to the eye", eyeIndex == 0 ? "Left" : "Right"); + webgpu::fdm::release_map(entry->map); + entry->map = 0; } } - return target.densityMap != 0 && webgpu::fdm::map_ready(target.densityMap) ? target.foveatedView - : wgpu::TextureView{}; + return target.boundDensityMap != 0 ? target.foveatedView : wgpu::TextureView{}; } std::optional request_stereo_frame(uint32_t logicalFrame, uint64_t contentTag) noexcept { @@ -864,7 +927,8 @@ gfx::StereoReplayFrame make_stereo_replay_frame(const AuroraStereoFrame& input, // Not the immersive window's eyes: the host may aim them through the window, whose field of // view then changes with every head movement and would rebuild the density map each frame. if (input.mode == AURORA_STEREO_FRAME_IMMERSIVE_REPLAY && !input.window) { - view.target.foveatedColorView = foveated_eye_view(eye, input.eyes[eye]); + view.target.foveatedColorView = + foveated_eye_view(eye, input.eyes[eye], input.gazeValid ? input.gaze[eye] : nullptr); } std::memcpy(&view.projection, input.eyes[eye].projection, sizeof(view.projection)); std::memcpy(&view.viewFromCenter, input.eyes[eye].viewFromCenter, sizeof(view.viewFromCenter)); diff --git a/aurora-main/lib/gfx/foveation.hpp b/aurora-main/lib/gfx/foveation.hpp index 48ed5ba..e749e78 100644 --- a/aurora-main/lib/gfx/foveation.hpp +++ b/aurora-main/lib/gfx/foveation.hpp @@ -5,10 +5,10 @@ #include #include -// Fixed foveated rendering for the immersive eyes: the fragment density map an eye's render pass -// runs under (webgpu/fdm.hpp). Each texel says how finely the framebuffer area it covers is shaded: +// Foveated rendering for the immersive eyes: the fragment density map an eye's render pass runs +// under (webgpu/fdm.hpp). Each texel says how finely the framebuffer area it covers is shaded: // fully at the centre of the view, in 2x2 then 4x4 pixel blocks towards the edges, where the -// headset's lenses blur the picture anyway. +// headset's lenses blur the picture anyway. With eye tracking the centre is where the player looks. namespace aurora::gfx::foveation { enum class Level : uint32_t { @@ -79,6 +79,65 @@ inline float eccentricity_degrees(float tanX, float tanY) noexcept { return std::atan(std::sqrt(tanX * tanX + tanY * tanY)) * (180.0f / 3.14159265358979f); } +// Where the map's full density is centred, in tangents of the eye's view like EyeFov's (x right, +// y up): the forward direction, or the point the player looks at. +struct Gaze { + float tanX = 0.0f; + float tanY = 0.0f; +}; + +// The angle between the rays through tangents (x, y) and through the gaze. +inline float angle_from_gaze_degrees(float tanX, float tanY, const Gaze& gaze) noexcept { + const float dot = tanX * gaze.tanX + tanY * gaze.tanY + 1.0f; + const float norms = std::sqrt((tanX * tanX + tanY * tanY + 1.0f) * (gaze.tanX * gaze.tanX + gaze.tanY * gaze.tanY + 1.0f)); + return std::acos(std::clamp(dot / norms, -1.0f, 1.0f)) * (180.0f / 3.14159265358979f); +} + +// Eye-tracked maps are built for the gaze snapped to cells of this many map texels square, so an +// eye's map changes only when the gaze moves that far (about 3 degrees with 32-pixel texels), and a +// few maps serve a whole session's glances. +inline constexpr uint32_t kGazeCellTexels = 2; + +struct GazeCell { + int32_t x = 0; + int32_t y = 0; + bool operator==(const GazeCell&) const = default; +}; + +// The cell of an eye of `eyeWidth` by `eyeHeight` pixels the gaze falls in, counted from the top +// left and clamped to the eye. A gaze that is not a number counts as the forward direction. +inline GazeCell gaze_cell(uint32_t eyeWidth, uint32_t eyeHeight, uint32_t texel, const EyeFov& fov, + Gaze gaze) noexcept { + const float cellPixels = static_cast(std::max(texel, 1u) * kGazeCellTexels); + if (!std::isfinite(gaze.tanX) || !std::isfinite(gaze.tanY)) { + gaze = {}; + } + const float spanX = fov.tanRight - fov.tanLeft; + const float spanY = fov.tanDown - fov.tanUp; + const float u = spanX != 0.0f ? (gaze.tanX - fov.tanLeft) / spanX : 0.5f; + const float v = spanY != 0.0f ? (gaze.tanY - fov.tanUp) / spanY : 0.5f; + const auto cell = [cellPixels](float fraction, uint32_t pixels) { + const int32_t count = std::max(1, static_cast(std::ceil(static_cast(pixels) / cellPixels))); + const float position = std::clamp(fraction, 0.0f, 1.0f) * static_cast(pixels) / cellPixels; + return std::clamp(static_cast(std::floor(position)), 0, count - 1); + }; + return {cell(u, eyeWidth), cell(v, eyeHeight)}; +} + +// The gaze through the centre of a cell, clamped to the eye for an overhanging last row or column. +inline Gaze cell_gaze(uint32_t eyeWidth, uint32_t eyeHeight, uint32_t texel, const EyeFov& fov, + GazeCell cell) noexcept { + const float cellPixels = static_cast(std::max(texel, 1u) * kGazeCellTexels); + const float u = eyeWidth > 0 ? std::min((static_cast(cell.x) + 0.5f) * cellPixels, static_cast(eyeWidth)) / + static_cast(eyeWidth) + : 0.5f; + const float v = eyeHeight > 0 ? std::min((static_cast(cell.y) + 0.5f) * cellPixels, static_cast(eyeHeight)) / + static_cast(eyeHeight) + : 0.5f; + return Gaze{.tanX = fov.tanLeft + (fov.tanRight - fov.tanLeft) * u, + .tanY = fov.tanUp + (fov.tanDown - fov.tanUp) * v}; +} + inline uint8_t density(Level level, float eccentricity) noexcept { const Rings ring = rings(level); if (eccentricity < ring.full) { @@ -95,15 +154,18 @@ struct Map { }; // The map for an eye of `eyeWidth` by `eyeHeight` pixels whose field of view is `fov`, `texel` pixels -// per map texel. The map covers the whole eye, its last row and column possibly overhanging it. +// per map texel, centred on `gaze` (the forward direction by default). The map covers the whole eye, +// its last row and column possibly overhanging it. inline void build(uint32_t eyeWidth, uint32_t eyeHeight, uint32_t texel, const EyeFov& fov, Level level, - Map& map) { + Map& map, const Gaze& gaze = {}) { map.width = texel > 0 ? (eyeWidth + texel - 1) / texel : 0; map.height = texel > 0 ? (eyeHeight + texel - 1) / texel : 0; map.rg8.assign(static_cast(map.width) * map.height * 2, kFullDensity); if (level == Level::Off || eyeWidth == 0 || eyeHeight == 0) { return; } + // The forward direction keeps its own, exact formula, so the fixed maps do not change. + const bool forward = gaze.tanX == 0.0f && gaze.tanY == 0.0f; for (uint32_t y = 0; y < map.height; ++y) { // Texel centres, clamped to the eye for an overhanging last row or column. const float v = std::min((static_cast(y) + 0.5f) * static_cast(texel), static_cast(eyeHeight)) / @@ -113,7 +175,8 @@ inline void build(uint32_t eyeWidth, uint32_t eyeHeight, uint32_t texel, const E const float u = std::min((static_cast(x) + 0.5f) * static_cast(texel), static_cast(eyeWidth)) / static_cast(eyeWidth); const float tanX = fov.tanLeft + (fov.tanRight - fov.tanLeft) * u; - const uint8_t value = density(level, eccentricity_degrees(tanX, tanY)); + const uint8_t value = + density(level, forward ? eccentricity_degrees(tanX, tanY) : angle_from_gaze_degrees(tanX, tanY, gaze)); uint8_t* texelBytes = &map.rg8[(static_cast(y) * map.width + x) * 2]; texelBytes[0] = value; texelBytes[1] = value; diff --git a/aurora-main/tests/foveation_test.cpp b/aurora-main/tests/foveation_test.cpp index 6d0662f..3d7012c 100644 --- a/aurora-main/tests/foveation_test.cpp +++ b/aurora-main/tests/foveation_test.cpp @@ -175,5 +175,112 @@ TEST(Foveation, ReadsTheFieldOfViewBackFromTheEyeProjection) { EXPECT_EQ(fallback.tanUp, 1.0f); } +// Eye-tracked foveation: the full-density centre follows the gaze. + +// The pixel a gaze lands on, as build lays the eye out. +std::pair gaze_pixel(const EyeFov& fov, const Gaze& gaze, uint32_t width, uint32_t height) { + return {(gaze.tanX - fov.tanLeft) / (fov.tanRight - fov.tanLeft) * width, + (gaze.tanY - fov.tanUp) / (fov.tanDown - fov.tanUp) * height}; +} + +TEST(Foveation, TheForwardGazeKeepsTheFixedMap) { + for (Level level : {Level::Low, Level::Medium, Level::High}) { + Map gazed; + foveation::build(1344, 1408, 32, left_eye(), level, gazed, Gaze{}); + EXPECT_TRUE(gazed.rg8 == build_map(level).rg8) << "level " << int(level); + } + // The general angle agrees with the forward one. + for (float tanX : {-1.2f, -0.3f, 0.0f, 0.4f, 0.9f}) { + for (float tanY : {-1.0f, 0.0f, 0.7f}) { + EXPECT_NEAR(angle_from_gaze_degrees(tanX, tanY, Gaze{}), eccentricity_degrees(tanX, tanY), 0.01f); + } + } +} + +TEST(Foveation, TheFullDensityRegionFollowsTheGaze) { + const EyeFov fov = left_eye(); + // Down and to the right, well off the forward direction. + const Gaze gaze{.tanX = std::tan(20.0f * kDegrees), .tanY = std::tan(-15.0f * kDegrees)}; + Map map; + foveation::build(1344, 1408, 32, fov, Level::High, map, gaze); + double sumX = 0.0; + double sumY = 0.0; + uint32_t count = 0; + for (uint32_t y = 0; y < map.height; ++y) { + for (uint32_t x = 0; x < map.width; ++x) { + if (at(map, x, y) == kFullDensity) { + sumX += x + 0.5; + sumY += y + 0.5; + ++count; + } + } + } + ASSERT_GT(count, 0u); + const auto [pixelX, pixelY] = gaze_pixel(fov, gaze, 1344, 1408); + EXPECT_NEAR(sumX / count, pixelX / 32.0, 1.5); + EXPECT_NEAR(sumY / count, pixelY / 32.0, 1.5); + // Where the forward map was sharpest, the far side of the gaze is now coarse. + const Map fixed = build_map(Level::High, fov); + EXPECT_EQ(at(map, static_cast(pixelX / 32.0f), static_cast(pixelY / 32.0f)), kFullDensity); + EXPECT_EQ(at(fixed, 0, 0), kQuarterDensity); + EXPECT_EQ(at(map, 0, 0), kQuarterDensity); +} + +TEST(Foveation, DensityNeverRisesAwayFromTheGaze) { + const EyeFov fov = left_eye(); + const Gaze gaze{.tanX = -0.35f, .tanY = 0.2f}; + for (Level level : {Level::Low, Level::Medium, Level::High}) { + Map map; + foveation::build(1344, 1408, 32, fov, level, map, gaze); + std::vector> texels; + for (uint32_t y = 0; y < map.height; ++y) { + for (uint32_t x = 0; x < map.width; ++x) { + const auto [tanX, tanY] = tangents(map, fov, x, y, 1344, 1408, 32); + texels.emplace_back(angle_from_gaze_degrees(tanX, tanY, gaze), at(map, x, y)); + } + } + std::sort(texels.begin(), texels.end()); + for (size_t i = 1; i < texels.size(); ++i) { + EXPECT_LE(texels[i].second, texels[i - 1].second); + } + EXPECT_EQ(texels.front().second, kFullDensity); + } +} + +TEST(Foveation, GazeCellsSnapTheGazeAndStayInsideTheEye) { + const EyeFov fov = left_eye(); + constexpr uint32_t kWidth = 1344, kHeight = 1408, kTexel = 32; + constexpr float kCellPixels = kTexel * kGazeCellTexels; + // A cell's own gaze lies within half a cell of every gaze that falls in it. + for (float tanX : {-0.9f, -0.2f, 0.0f, 0.31f, 0.8f}) { + for (float tanY : {-0.8f, 0.0f, 0.45f}) { + const Gaze gaze{.tanX = tanX, .tanY = tanY}; + const GazeCell cell = gaze_cell(kWidth, kHeight, kTexel, fov, gaze); + const Gaze centre = cell_gaze(kWidth, kHeight, kTexel, fov, cell); + const auto [gx, gy] = gaze_pixel(fov, gaze, kWidth, kHeight); + const auto [cx, cy] = gaze_pixel(fov, centre, kWidth, kHeight); + EXPECT_LE(std::abs(gx - cx), kCellPixels / 2.0f + 0.01f); + EXPECT_LE(std::abs(gy - cy), kCellPixels / 2.0f + 0.01f); + EXPECT_TRUE(gaze_cell(kWidth, kHeight, kTexel, fov, centre) == cell); + } + } + // Gazes a few pixels apart share a cell; the forward direction has one of its own. + const Gaze forward{}; + const GazeCell forwardCell = gaze_cell(kWidth, kHeight, kTexel, fov, forward); + const auto [fx, fy] = gaze_pixel(fov, forward, kWidth, kHeight); + EXPECT_EQ(forwardCell.x, static_cast(fx / kCellPixels)); + EXPECT_EQ(forwardCell.y, static_cast(fy / kCellPixels)); + // Beyond the eye, and not a number at all. + const int32_t lastColumn = static_cast(std::ceil(kWidth / kCellPixels)) - 1; + const int32_t lastRow = static_cast(std::ceil(kHeight / kCellPixels)) - 1; + const GazeCell far = gaze_cell(kWidth, kHeight, kTexel, fov, Gaze{.tanX = 10.0f, .tanY = -10.0f}); + EXPECT_EQ(far.x, lastColumn); + EXPECT_EQ(far.y, lastRow); + const GazeCell farOther = gaze_cell(kWidth, kHeight, kTexel, fov, Gaze{.tanX = -10.0f, .tanY = 10.0f}); + EXPECT_EQ(farOther.x, 0); + EXPECT_EQ(farOther.y, 0); + EXPECT_TRUE(gaze_cell(kWidth, kHeight, kTexel, fov, Gaze{.tanX = NAN, .tanY = 0.2f}) == forwardCell); +} + } // namespace } // namespace aurora::gfx::foveation diff --git a/runtime/CMakeLists.txt b/runtime/CMakeLists.txt index 5820ffb..bbda761 100644 --- a/runtime/CMakeLists.txt +++ b/runtime/CMakeLists.txt @@ -456,6 +456,11 @@ add_executable(mkw_vr_wii_remote_tests "${CMAKE_CURRENT_LIST_DIR}/tests/vr_wii_r target_include_directories(mkw_vr_wii_remote_tests PRIVATE "${CMAKE_CURRENT_LIST_DIR}/include") target_compile_features(mkw_vr_wii_remote_tests PRIVATE cxx_std_17) add_test(NAME mkw_vr_wii_remote_tests COMMAND mkw_vr_wii_remote_tests) +# Eye-tracked foveation: the gaze in each eye's view (vr/eye_gaze.h). +add_executable(mkw_vr_eye_gaze_tests "${CMAKE_CURRENT_LIST_DIR}/tests/vr_eye_gaze_tests.cpp") +target_include_directories(mkw_vr_eye_gaze_tests PRIVATE "${CMAKE_CURRENT_LIST_DIR}/include") +target_compile_features(mkw_vr_eye_gaze_tests PRIVATE cxx_std_17) +add_test(NAME mkw_vr_eye_gaze_tests COMMAND mkw_vr_eye_gaze_tests) # The in-headset settings panel's controller chord, release latch, selection, # scrolling and canvas mapping, plus the thread bridge they publish through. diff --git a/runtime/include/runtime_config.h b/runtime/include/runtime_config.h index 7dc44a0..7f9466d 100644 --- a/runtime/include/runtime_config.h +++ b/runtime/include/runtime_config.h @@ -92,6 +92,7 @@ struct RuntimeUserConfig { std::optional vrWheelHaptics; std::optional vrPerformanceLevel; std::optional vrFoveation; + std::optional vrEyeTrackedFoveation; std::optional vrRecenterKey; std::optional vrLeanBackDegrees; // F10 > Diagnostics: OpenXR pacing and presentation logging in console.log. @@ -320,6 +321,16 @@ inline bool IsSupportedVrFoveation(std::string_view value) { return std::find(kVrFoveationLevels.begin(), kVrFoveationLevels.end(), value) != kVrFoveationLevels.end(); } +// Eye-tracked foveation: with a headset that tracks the eyes (XR_EXT_eye_gaze_interaction, the Steam +// Frame's), the foveation level's full-density region follows the gaze instead of staying on each +// eye's forward direction. Live while the session's runtime offered the gaze at launch. On by +// default on the Steam Frame; elsewhere off, since Horizon OS asks for an eye tracking permission. +#if defined(MKW_HEADSET_STEAM_FRAME) +inline constexpr bool kVrEyeTrackedFoveationDefault = true; +#else +inline constexpr bool kVrEyeTrackedFoveationDefault = false; +#endif + // The level aurora_set_stereo_foveation takes; anything unknown is off. inline uint32_t VrFoveationLevelIndex(std::string_view value) { const auto it = std::find(kVrFoveationLevels.begin(), kVrFoveationLevels.end(), value); @@ -891,6 +902,7 @@ inline RuntimeUserConfig ParseConfigDocument(const toml::value& document) { value && IsSupportedVrFoveation(*value)) { config.vrFoveation = *value; } + config.vrEyeTrackedFoveation = FindConfigValue(document, "vr", "eye_tracked_foveation"); if (auto value = FindConfigValue(document, "vr", "mirror_view"); value && IsSupportedVrMirrorView(*value)) { config.vrMirrorView = *value; @@ -1303,6 +1315,11 @@ inline bool SetVrFoveation(std::string value) { return WriteSetting("vr", "foveation", FormatString(value)); } +inline bool SetVrEyeTrackedFoveation(bool value) { + Mutable().vrEyeTrackedFoveation = value; + return WriteSetting("vr", "eye_tracked_foveation", value ? "true" : "false"); +} + inline bool SetVrFirstPersonSeat(std::string value) { if (!IsSupportedVrFirstPersonSeat(value)) { return false; @@ -1828,6 +1845,10 @@ inline std::string VrFoveation(std::string fallback = kVrFoveationDefault) { return value && IsSupportedVrFoveation(*value) ? *value : std::move(fallback); } +inline bool VrEyeTrackedFoveation(bool fallback = kVrEyeTrackedFoveationDefault) { + return Get().vrEyeTrackedFoveation.value_or(fallback); +} + inline int32_t VrFirstPersonHiddenModel(int32_t fallback = kVrFirstPersonHiddenModelDefault) { return std::clamp(Get().vrFirstPersonHiddenModel.value_or(fallback), -1, 31); } diff --git a/runtime/include/vr/eye_gaze.h b/runtime/include/vr/eye_gaze.h new file mode 100644 index 0000000..669f4d5 --- /dev/null +++ b/runtime/include/vr/eye_gaze.h @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +#pragma once + +#include + +// Eye-tracked foveation ([vr] eye_tracked_foveation): where the player looks, as each eye's image +// measures it. Kept free of OpenXR types so it can be checked headlessly (tests/vr_eye_gaze_tests.cpp). +// +// Conventions are OpenXR's: right-handed, +Y up, and a pose looks down its -Z axis. +namespace mkw::vr::eye_gaze { + +struct Quaternion { + float x = 0.0f; + float y = 0.0f; + float z = 0.0f; + float w = 1.0f; +}; + +// Tangents of an eye's view, x right and y up, as its frustum (XrFovf) measures them. +struct Tangents { + float x = 0.0f; + float y = 0.0f; + bool valid = false; +}; + +// Beyond this angle from an eye's forward direction a gaze is no point of its image (cos 80 deg). +inline constexpr float kMinForwardCosine = 0.17364818f; + +// The gaze pose's look direction in one eye's view, from both orientations in the same space. The +// eyes' views can be canted outwards, so each eye gets its own tangents. +inline Tangents InEye(Quaternion gaze, Quaternion eye) noexcept { + const auto normalized = [](Quaternion q) { + const float length = std::sqrt(q.x * q.x + q.y * q.y + q.z * q.z + q.w * q.w); + if (!(length > 1.0e-6f)) { + return Quaternion{}; + } + return Quaternion{q.x / length, q.y / length, q.z / length, q.w / length}; + }; + // q * v * conjugate(q). + const auto rotate = [](const Quaternion& q, float vx, float vy, float vz, float out[3]) { + const float tx = 2.0f * (q.y * vz - q.z * vy); + const float ty = 2.0f * (q.z * vx - q.x * vz); + const float tz = 2.0f * (q.x * vy - q.y * vx); + out[0] = vx + q.w * tx + (q.y * tz - q.z * ty); + out[1] = vy + q.w * ty + (q.z * tx - q.x * tz); + out[2] = vz + q.w * tz + (q.x * ty - q.y * tx); + }; + gaze = normalized(gaze); + eye = normalized(eye); + float look[3]; + rotate(gaze, 0.0f, 0.0f, -1.0f, look); + float seen[3]; + rotate(Quaternion{-eye.x, -eye.y, -eye.z, eye.w}, look[0], look[1], look[2], seen); + if (!(-seen[2] > kMinForwardCosine)) { + return {}; + } + return {seen[0] / -seen[2], seen[1] / -seen[2], true}; +} + +} // namespace mkw::vr::eye_gaze diff --git a/runtime/include/vr/openxr_input.h b/runtime/include/vr/openxr_input.h index 759b40c..27613f5 100644 --- a/runtime/include/vr/openxr_input.h +++ b/runtime/include/vr/openxr_input.h @@ -130,6 +130,15 @@ public: // The tracked hands' joints in the seated frame, as the last Sync located // them; read on the pacing thread only. const hand_tracking::HandJointFrame& HandJoints() const noexcept { return m_joint_frame; } + // Eye-tracked foveation: the gaze pose's orientation in the app space, as the last Sync located + // it for the frame's display time. False while there is no tracked gaze (no eye tracker, a + // blink, or the runtime not offering XR_EXT_eye_gaze_interaction). + bool EyeGaze(XrQuaternionf* orientation) const noexcept { + if (m_gaze_valid && orientation != nullptr) { + *orientation = m_gaze_orientation; + } + return m_gaze_valid; + } // A hand's tracker while tracked hands keep one, else XR_NULL_HANDLE. XrHandTrackerEXT HandTracker(uint32_t hand) const noexcept { return hand < kHands ? m_hand_trackers[hand] : XR_NULL_HANDLE; @@ -150,6 +159,9 @@ private: bool CreateActions(); bool SuggestBindings(); + // XR_EXT_eye_gaze_interaction is enabled and the system has an eye tracker. + bool EyeGazeOffered(); + void LocateEyeGaze(XrTime time); void CreatePoseSpaces(); void DestroyPoseSpaces(); void LoadInputClock(); @@ -201,6 +213,12 @@ private: XrAction m_aim_pose = XR_NULL_HANDLE; XrAction m_grip_pose = XR_NULL_HANDLE; XrAction m_haptic = XR_NULL_HANDLE; + // The eyes' gaze (XR_EXT_eye_gaze_interaction), one pose for both, and where it was last located. + XrAction m_gaze_pose = XR_NULL_HANDLE; + XrSpace m_gaze_space = XR_NULL_HANDLE; + XrQuaternionf m_gaze_orientation{0.0f, 0.0f, 0.0f, 1.0f}; + bool m_gaze_valid = false; + bool m_gaze_logged = false; XrPath m_hand_paths[kHands]{}; XrSpace m_aim_spaces[kHands]{}; XrSpace m_grip_spaces[kHands]{}; diff --git a/runtime/src/settings_overlay.cpp b/runtime/src/settings_overlay.cpp index 8983d58..60123d6 100644 --- a/runtime/src/settings_overlay.cpp +++ b/runtime/src/settings_overlay.cpp @@ -156,6 +156,7 @@ bool g_vrHandTracking = RuntimeConfigFile::VrHandTracking(); constexpr std::array kVrFoveationLabels{"Off", "Low", "Medium", "High"}; static_assert(kVrFoveationLabels.size() == RuntimeConfigFile::kVrFoveationLevels.size()); int g_vrFoveation = static_cast(RuntimeConfigFile::VrFoveationLevelIndex(RuntimeConfigFile::VrFoveation())); +bool g_vrEyeTrackedFoveation = RuntimeConfigFile::VrEyeTrackedFoveation(); #endif bool g_vrFirstPerson = RuntimeConfigFile::VrFirstPerson(false); bool g_vrFirstPersonToggleClick = RuntimeConfigFile::VrFirstPersonToggleClick(); @@ -1523,6 +1524,15 @@ void DrawVrSettings() { "lenses blur the picture anyway, to free GPU time. This session started with it " "off, or without a GPU that supports it: a new level applies after a restart."); } + if (ImGui::Checkbox("Foveation follows the eyes", &g_vrEyeTrackedFoveation)) { + RuntimeConfigFile::SetVrEyeTrackedFoveation(g_vrEyeTrackedFoveation); + } + if (ImGui::IsItemHovered()) { + ImGui::SetTooltip( + "With a headset that tracks the eyes (the Steam Frame), the sharp centre of the foveated " + "race view moves to where you look instead of staying straight ahead. Turning it off " + "applies immediately; turning it on needs a restart if the session started without it."); + } #endif { const auto rateLabel = [](uint32_t hz) { diff --git a/runtime/src/vr/openxr_input.cpp b/runtime/src/vr/openxr_input.cpp index 001baf2..3436ba3 100644 --- a/runtime/src/vr/openxr_input.cpp +++ b/runtime/src/vr/openxr_input.cpp @@ -423,6 +423,36 @@ bool OpenXRInput::CreateActions() { return false; } } + // Eye-tracked foveation: one gaze pose for both eyes, with no hand to name. + if (EyeGazeOffered()) { + XrActionCreateInfo info{XR_TYPE_ACTION_CREATE_INFO}; + info.actionType = XR_ACTION_TYPE_POSE_INPUT; + std::strncpy(info.actionName, "eye_gaze", XR_MAX_ACTION_NAME_SIZE - 1); + std::strncpy(info.localizedActionName, "Eye Gaze", XR_MAX_LOCALIZED_ACTION_NAME_SIZE - 1); + if (!Check(xrCreateAction(m_action_set, &info, &m_gaze_pose), "eye_gaze")) { + // Not worth the controllers: carry on with the fixed foveation centre. + m_gaze_pose = XR_NULL_HANDLE; + } + } + return true; +} + +bool OpenXRInput::EyeGazeOffered() { + const auto& extensions = m_runtime->EnabledExtensions(); + if (std::find(extensions.begin(), extensions.end(), "XR_EXT_eye_gaze_interaction") == extensions.end()) { + return false; + } + XrSystemEyeGazeInteractionPropertiesEXT gaze{XR_TYPE_SYSTEM_EYE_GAZE_INTERACTION_PROPERTIES_EXT}; + XrSystemProperties properties{XR_TYPE_SYSTEM_PROPERTIES, &gaze}; + const XrResult result = xrGetSystemProperties(m_runtime->Instance(), m_runtime->SystemId(), &properties); + if (XR_FAILED(result) || gaze.supportsEyeGazeInteraction != XR_TRUE) { + std::ostringstream message; + message << "OpenXR eye gaze: the runtime reports no eye tracker (" << result + << "); foveation stays on each eye's forward direction"; + Log(OpenXRLogLevel::Info, message.str()); + return false; + } + Log(OpenXRLogLevel::Info, "OpenXR eye gaze: available; foveation follows the gaze ([vr] eye_tracked_foveation)"); return true; } @@ -533,6 +563,10 @@ bool OpenXRInput::SuggestBindings() { }; suggest("/interaction_profiles/valve/frame_controller_valve", frame, false); } + if (m_gaze_pose != XR_NULL_HANDLE) { + suggest("/interaction_profiles/ext/eye_gaze_interaction", + {{&m_gaze_pose, "/user/eyes_ext/input/gaze_ext/pose"}}, false); + } return true; } @@ -558,9 +592,56 @@ void OpenXRInput::CreatePoseSpaces() { } } } + if (m_gaze_pose != XR_NULL_HANDLE) { + XrActionSpaceCreateInfo info{XR_TYPE_ACTION_SPACE_CREATE_INFO}; + info.action = m_gaze_pose; + info.poseInActionSpace.orientation.w = 1.0f; + const XrResult result = xrCreateActionSpace(m_runtime->Session(), &info, &m_gaze_space); + m_runtime->ObserveResult(result); + if (XR_FAILED(result)) { + m_gaze_space = XR_NULL_HANDLE; + std::ostringstream message; + message << "xrCreateActionSpace(eye gaze) failed (" << result + << "); foveation stays on each eye's forward direction"; + Log(OpenXRLogLevel::Warning, message.str()); + } + } +} + +// Eye-tracked foveation: the gaze for the display time the eyes are rendered for, the time their +// views are located at, so the full-density region lands where the eyes look in that frame. Only a +// tracked orientation counts; the runtime reports an untracked one through blinks. +void OpenXRInput::LocateEyeGaze(XrTime time) { + m_gaze_valid = false; + if (m_gaze_space == XR_NULL_HANDLE) { + return; + } + XrActionStateGetInfo info{XR_TYPE_ACTION_STATE_GET_INFO}; + info.action = m_gaze_pose; + XrActionStatePose state{XR_TYPE_ACTION_STATE_POSE}; + if (XR_FAILED(xrGetActionStatePose(m_runtime->Session(), &info, &state)) || state.isActive != XR_TRUE) { + return; + } + constexpr XrSpaceLocationFlags kTracked = + XR_SPACE_LOCATION_ORIENTATION_VALID_BIT | XR_SPACE_LOCATION_ORIENTATION_TRACKED_BIT; + XrSpaceLocation location{XR_TYPE_SPACE_LOCATION}; + if (XR_FAILED(xrLocateSpace(m_gaze_space, m_runtime->AppSpace(), time, &location)) || + (location.locationFlags & kTracked) != kTracked) { + return; + } + m_gaze_orientation = location.pose.orientation; + m_gaze_valid = true; + if (!m_gaze_logged) { + m_gaze_logged = true; + Log(OpenXRLogLevel::Info, "OpenXR eye gaze: tracking"); + } } void OpenXRInput::DestroyPoseSpaces() { + if (m_gaze_space != XR_NULL_HANDLE) { + xrDestroySpace(m_gaze_space); + m_gaze_space = XR_NULL_HANDLE; + } for (uint32_t hand = 0; hand < kHandCount; ++hand) { for (XrSpace* space : {&m_aim_spaces[hand], &m_grip_spaces[hand]}) { if (*space != XR_NULL_HANDLE) { @@ -909,6 +990,9 @@ void OpenXRInput::Destroy() { m_thumbstick = m_thumbstick_click = m_trigger = m_squeeze = XR_NULL_HANDLE; m_button_primary = m_button_secondary = m_menu = m_haptic = XR_NULL_HANDLE; m_dpad_up = m_dpad_down = m_dpad_left = m_dpad_right = XR_NULL_HANDLE; + m_gaze_pose = XR_NULL_HANDLE; + m_gaze_valid = false; + m_gaze_logged = false; m_aim_pose = m_grip_pose = XR_NULL_HANDLE; m_hand_paths[0] = m_hand_paths[1] = XR_NULL_PATH; m_convert_now_to_xr_time = nullptr; @@ -933,6 +1017,7 @@ void OpenXRInput::Idle() { } m_pointer.Reset(); m_horizon = {1.0f, 0.0f}; + m_gaze_valid = false; OpenXRPublishWiiRemote(Relay().JoystickId(), OpenXRWiiRemoteSample{}); // The panel stays as it was; only what the controllers were holding is forgotten. m_panel_controls.Reset(); @@ -976,6 +1061,7 @@ void OpenXRInput::Sync(XrTime predicted_display_time, const OpenXRPointerScreen& Idle(); return; } + LocateEyeGaze(predicted_display_time); // `active`, when given, says whether the action is bound to a source the // runtime has right now (a controller, or a tracked hand). diff --git a/runtime/src/vr/openxr_integration.cpp b/runtime/src/vr/openxr_integration.cpp index 8da12af..be5f9c9 100644 --- a/runtime/src/vr/openxr_integration.cpp +++ b/runtime/src/vr/openxr_integration.cpp @@ -9,6 +9,7 @@ #include "runtime_config.h" #include "gx_thread.h" #include "runtime_log.h" +#include "vr/eye_gaze.h" #include "vr/mkw_vr_culling.h" #include "vr/mkw_vr_first_person.h" #include "vr/mkw_vr_policy.h" @@ -428,6 +429,10 @@ public: // Horizon OS's room view; the Steam Frame build neither asks for it nor offers the setting. config.optional_extensions.push_back("XR_FB_passthrough"); #endif + // Eye-tracked foveation: the gaze the density maps centre on (OpenXRInput::EyeGaze). + if (RuntimeConfigFile::VrEyeTrackedFoveation()) { + config.optional_extensions.push_back("XR_EXT_eye_gaze_interaction"); + } AddHandMeshExtensions(config); config.instance_create_next = OpenXRAndroidInstanceCreateNext(); #endif @@ -1465,9 +1470,30 @@ private: position_valid && base_position_valid_, units_per_meter, lean_back_radians, destination.eyes[eye].viewFromCenter); } + BuildEyeGaze(source, destination); BuildCockpit(source, position_valid, units_per_meter, lean_back_radians, destination.cockpit); } + // Eye-tracked foveation: where the eyes look, in each eye's own view (the views may be canted), + // from the gaze the input located for this packet's display time. Without a tracked gaze, or + // with the setting off, Aurora centres foveation on each eye's forward direction. + void BuildEyeGaze(const OpenXRBackendFrame& source, AuroraStereoFrame& destination) const noexcept { + XrQuaternionf gaze{}; + if (input_ == nullptr || !RuntimeConfigFile::VrEyeTrackedFoveation() || !input_->EyeGaze(&gaze)) { + return; + } + bool valid = true; + for (uint32_t eye = 0; eye < kOpenXREyeCount; ++eye) { + const XrQuaternionf& view = source.xr_frame.views[eye].pose.orientation; + const eye_gaze::Tangents seen = + eye_gaze::InEye({gaze.x, gaze.y, gaze.z, gaze.w}, {view.x, view.y, view.z, view.w}); + valid = valid && seen.valid; + destination.gaze[eye][0] = seen.x; + destination.gaze[eye][1] = seen.y; + } + destination.gazeValid = valid; + } + // The first-person cockpit's hands and separate wheel, in the seated frame // the eye transforms place at base + lean * seat (metres). Always carries // the packet's world scale, which Aurora rescales to the sealed frame's. diff --git a/runtime/src/vr/openxr_runtime.cpp b/runtime/src/vr/openxr_runtime.cpp index 384e40f..dd48fdf 100644 --- a/runtime/src/vr/openxr_runtime.cpp +++ b/runtime/src/vr/openxr_runtime.cpp @@ -151,6 +151,14 @@ bool OpenXRRuntime::EnumerateInstanceCapabilities() { for (const XrExtensionProperties& extension : extension_properties) { m_available_extensions.emplace_back(extension.extensionName); } + // Everything the runtime offers, once per instance: what a new headset's runtime can do (the + // Steam Frame's SteamVR, say) is in the first session log rather than behind a debugger. + std::ostringstream offered; + offered << "OpenXR runtime offers " << m_available_extensions.size() << " extensions:"; + for (const std::string& name : m_available_extensions) { + offered << ' ' << name; + } + Log(OpenXRLogLevel::Info, offered.str()); uint32_t layer_count = 0; if (!Check(xrEnumerateApiLayerProperties(0, &layer_count, nullptr), diff --git a/runtime/tests/vr_config_tests.cpp b/runtime/tests/vr_config_tests.cpp index 409a4aa..ef3a79c 100644 --- a/runtime/tests/vr_config_tests.cpp +++ b/runtime/tests/vr_config_tests.cpp @@ -68,6 +68,17 @@ int main() { Require(std::string_view(MKW_VR_REFRESH_RATE_DEFAULT_TEXT) == "0"); #endif + // [vr] eye_tracked_foveation: the foveation centre follows the gaze, on the Steam Frame by default. + Require(Parse("[vr]\neye_tracked_foveation = true\n").vrEyeTrackedFoveation == true); + Require(Parse("[vr]\neye_tracked_foveation = false\n").vrEyeTrackedFoveation == false); + Require(!Parse("[vr]\neye_tracked_foveation = 1\n").vrEyeTrackedFoveation.has_value()); + Require(!Parse("[vr]\n").vrEyeTrackedFoveation.has_value()); +#if defined(MKW_HEADSET_STEAM_FRAME) + Require(RuntimeConfigFile::kVrEyeTrackedFoveationDefault); +#else + Require(!RuntimeConfigFile::kVrEyeTrackedFoveationDefault); +#endif + // [vr] passthrough: Horizon OS's room view, which the Steam Frame build does not offer. Require(Parse("[vr]\npassthrough = false\n").vrPassthrough == false); Require(!Parse("[vr]\n").vrPassthrough.has_value()); diff --git a/runtime/tests/vr_eye_gaze_tests.cpp b/runtime/tests/vr_eye_gaze_tests.cpp new file mode 100644 index 0000000..b8d7c39 --- /dev/null +++ b/runtime/tests/vr_eye_gaze_tests.cpp @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +#include "vr/eye_gaze.h" + +#include +#include +#include + +using mkw::vr::eye_gaze::InEye; +using mkw::vr::eye_gaze::Quaternion; +using mkw::vr::eye_gaze::Tangents; + +namespace { + +void Check(bool condition, const char* what) { + if (!condition) { + std::fprintf(stderr, "vr_eye_gaze_tests: %s\n", what); + std::exit(1); + } +} + +void CheckNear(float value, float expected, const char* what) { + Check(std::fabs(value - expected) <= 1.0e-4f, what); +} + +constexpr float kDegrees = 3.14159265358979f / 180.0f; + +// A turn by `degrees` about the unit axis (x, y, z). +Quaternion Turn(float degrees, float x, float y, float z) { + const float half = 0.5f * degrees * kDegrees; + return {x * std::sin(half), y * std::sin(half), z * std::sin(half), std::cos(half)}; +} + +} // namespace + +int main() { + // Looking straight ahead through an eye looking straight ahead: its forward direction. + Tangents t = InEye({}, {}); + Check(t.valid, "straight ahead is valid"); + CheckNear(t.x, 0.0f, "straight ahead: x"); + CheckNear(t.y, 0.0f, "straight ahead: y"); + + // A turn about +Y by a positive angle looks left (-X), about +X looks up (+Y). + t = InEye(Turn(20.0f, 0.0f, 1.0f, 0.0f), {}); + Check(t.valid, "20 degrees left is valid"); + CheckNear(t.x, -std::tan(20.0f * kDegrees), "20 degrees left: x"); + CheckNear(t.y, 0.0f, "20 degrees left: y"); + t = InEye(Turn(15.0f, 1.0f, 0.0f, 0.0f), {}); + CheckNear(t.x, 0.0f, "15 degrees up: x"); + CheckNear(t.y, std::tan(15.0f * kDegrees), "15 degrees up: y"); + + // An eye canted outwards sees the same gaze off its own centre. + t = InEye({}, Turn(10.0f, 0.0f, 1.0f, 0.0f)); + CheckNear(t.x, std::tan(10.0f * kDegrees), "a left-canted eye sees straight ahead to its right"); + t = InEye(Turn(10.0f, 0.0f, 1.0f, 0.0f), Turn(10.0f, 0.0f, 1.0f, 0.0f)); + CheckNear(t.x, 0.0f, "gaze along the canted eye: x"); + CheckNear(t.y, 0.0f, "gaze along the canted eye: y"); + + // The head's own turn cancels out: only the gaze relative to the eye counts. + const Quaternion head = Turn(70.0f, 0.0f, 1.0f, 0.0f); + const Quaternion look = Turn(70.0f + 12.0f, 0.0f, 1.0f, 0.0f); + t = InEye(look, head); + CheckNear(t.x, -std::tan(12.0f * kDegrees), "a turned head: x"); + + // Sideways or behind is no point of the image; an unnormalised quaternion still works. + Check(!InEye(Turn(85.0f, 0.0f, 1.0f, 0.0f), {}).valid, "85 degrees off is invalid"); + Check(!InEye(Turn(180.0f, 0.0f, 1.0f, 0.0f), {}).valid, "behind is invalid"); + Quaternion scaled = Turn(15.0f, 1.0f, 0.0f, 0.0f); + scaled = {scaled.x * 3.0f, scaled.y * 3.0f, scaled.z * 3.0f, scaled.w * 3.0f}; + t = InEye(scaled, {}); + CheckNear(t.y, std::tan(15.0f * kDegrees), "unnormalised quaternion"); + Check(InEye({0.0f, 0.0f, 0.0f, 0.0f}, {}).valid, "a zero quaternion reads as identity"); + + std::printf("vr_eye_gaze_tests: all checks passed\n"); + return 0; +} From 6815b101222186d3ed8adacc4bd81a948cc4be40 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 08:46:49 +0000 Subject: [PATCH 05/14] Document the Steam Frame build docs/steam-frame.md covers the steamFrame flavour: the CPU target and why it excludes SVE, the Lepton entry activity and manifest, the Frame controller map, the refresh rate and eye-tracked foveation, building and installing, the device checklist (what to collect from the headset, the log lines a first session should show, the open questions), and what a native SteamOS build would need. OPENXR.md documents refresh_rate and eye_tracked_foveation and the Frame controller profile; the Quest doc, the Android README and the README point to it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_019HBRGKTE1GnN2ah8gcZKr3 --- OPENXR.md | 31 +++++- README.md | 3 + android/README.md | 6 +- docs/quest-port.md | 9 +- docs/steam-frame.md | 240 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 282 insertions(+), 7 deletions(-) create mode 100644 docs/steam-frame.md diff --git a/OPENXR.md b/OPENXR.md index 981b422..3622845 100644 --- a/OPENXR.md +++ b/OPENXR.md @@ -40,6 +40,7 @@ required = false mirror_view = "normal" controller_mode = "wii_remote" frame_interpolation_fps = 0 +refresh_rate = 0 render_scale = 1.0 world_units_per_meter = 500.0 hud_distance_meters = 2.0 @@ -217,11 +218,19 @@ its CPU and GPU domains: `boost`, `sustained_high`, `sustained_low`, `power_savi request (see `docs/quest-port.md`); desktop runtimes rarely offer the extension, and the setting then does nothing. It is read at launch, and the session log records whether the runtime accepted it and any later performance notification (a thermal or rendering warning). -`foveation` (Quest only, default `medium`) shades the edges of the immersive race view more coarsely: +`foveation` (Quest and Steam Frame only, default `medium`) shades the edges of the immersive race view more coarsely: `off`, `low`, `medium` or `high`, see [Foveated rendering](#foveated-rendering). A session launched with it off runs without fragment density maps, so going from `off` to a level takes a restart; between levels, and back to `off`, it is live from the headset panel's VR tab. The launcher's -Settings page has it too. +Settings page has it too. `eye_tracked_foveation` (default on for the Steam Frame, off elsewhere) +centres it on the player's gaze where the runtime offers `XR_EXT_eye_gaze_interaction` with an eye +tracker; see [Eye-tracked foveation](#eye-tracked-foveation). +`refresh_rate` is the display rate in Hz asked of the runtime through `XR_FB_display_refresh_rate` +each time the session starts and whenever it changes, or `0` (the default, `120` on the Steam +Frame) to leave the headset's own. The game renders 60 frames a second, so 120 Hz shows each frame +for exactly two refreshes. A rate the runtime does not list, or declines, is logged and leaves its +own; setting `0` again restores the rate the session started at. It is live from F10 / the headset +panel (*Headset refresh rate*) and the Quest launcher; runtimes without the extension ignore it. ## Controllers @@ -269,6 +278,12 @@ because hand steering holds a grip down for a whole corner, and C is the game's game's Wii Remote rumble vibrates both controllers, subject to the ordinary controller-vibration switch. +The Steam Frame's controllers get their own profile where the runtime offers it +(`XR_VALVE_frame_controller_interaction`, `/interaction_profiles/valve/frame_controller_valve`): +right A, B, trigger and stick as above, left View as the left menu (+), the left shoulder as left Y +(the settings panel), and the left D-pad as the Wii Remote's D-pad (the gamepad's D-pad in +`"gamepad"` mode). The table is in `docs/steam-frame.md`. + **Motion.** Each XR frame the aim and grip poses are located at the measured current time (`XR_KHR_win32_convert_performance_counter_time`, `XR_KHR_convert_timespec_time` on Android), not the predicted display time, whose extrapolation sprays fast wrist motion. The grip's linear @@ -1068,6 +1083,17 @@ ripples give way to a smoother look. It is no fix for a heavy track: on Retro Re 2 at 1.0, GPU-bound at about 40 FPS, no level raised the frame rate, while merging the eye passes did (39 to 41.5 FPS). +### Eye-tracked foveation + +With `eye_tracked_foveation`, a runtime that offers `XR_EXT_eye_gaze_interaction` and reports an eye +tracker (the Steam Frame's SteamVR) has its gaze pose located for each packet's display time and +turned into tangents of each eye's view (`vr/eye_gaze.h`), which `AuroraStereoFrame` carries as +`gaze`/`gazeValid`. Aurora centres the level's rings on the gaze snapped to a cell of two map texels +(about 3 degrees), keeping up to 32 maps per eye, one per cell looked at, and binds a new one once +its upload completes, the previous map staying bound meanwhile. Without a tracked gaze (a blink, no +tracker, the setting off) the map is the forward one above, unchanged. Details and the Steam Frame +checks are in `docs/steam-frame.md`. + ## Diagnostics **F10 > Diagnostics** holds three bug-report aids. @@ -1250,6 +1276,7 @@ custom DLL's ABI through three borrowed-image copy/readback cycles; run it with | Windows D3D12 | Implemented: same-adapter, same-device asynchronous OpenXR submission. | | Windows Vulkan | Implemented, opt-in (`video.graphics_api = "vulkan"`): the runtime creates Dawn's Vulkan instance and device through `XR_KHR_vulkan_enable2`, eyes are copied on the same queue, and Dawn's device guard is held around the four queue-touching OpenXR calls. Needs the custom Dawn from `Launcher/Build-DawnVulkan.ps1`. Raced on SteamVR/PSVR2 at the headset's full rate; other runtimes unexercised. See [Windows Vulkan](#windows-vulkan). | | Android Vulkan (Meta Quest) | Implemented and running on a Quest 3: the OpenXR side owns its own Vulkan device (`XR_KHR_vulkan_enable2`, `XR_KHR_vulkan_enable` fallback) and shares eyes with Dawn through `AHardwareBuffer`s ordered by sync-fd fences. Controllers arrive through OpenXR actions as a virtual SDL gamepad. See `docs/quest-port.md`. | +| Android Vulkan (Steam Frame) | The same backend in the `steamFrame` flavour, for SteamVR's Android runtime under Lepton: Frame controller profile, 120 Hz request, eye-tracked foveation. Built and unit-tested, not yet run on the headset. See `docs/steam-frame.md`. | | Linux Vulkan | Not wired. The pinned Dawn package does not expose a native Vulkan device, and the AHardwareBuffer bridge is Android-only; a dma-buf/opaque-fd variant of the same design would cover desktop Linux. | | Other platforms | Not wired yet. | diff --git a/README.md b/README.md index 509c1a7..25db3ea 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,9 @@ to immersive stereo rendering. VR is opt-in and falls back to the normal desktop runtime or headset is unavailable. In first person you sit in the cockpit, where the steering wheel or handlebar turns with your steering, and hand steering by heurazy lets you grab it with the tracked controllers and turn it. On a Quest the hands can follow the headset's own hand tracking. +A Steam Frame build of the Android app (not yet tested on the headset) adds the Frame controllers' +D-pad, a 120 Hz display for the game's 60 FPS, and foveation that follows your eyes; see +[`docs/steam-frame.md`](docs/steam-frame.md). See [`OPENXR.md`](OPENXR.md) for setup, configuration, and the current limitations. **Music ducking.** diff --git a/android/README.md b/android/README.md index 8c3b3fc..fc85b0e 100644 --- a/android/README.md +++ b/android/README.md @@ -1,8 +1,9 @@ # WiiCompiled VR for Meta Quest (Android) Standalone Android/OpenXR build of the Mario Kart Wii recompilation for Quest 1, -Quest 2, Quest 3, Quest 3S and Quest Pro. The full design, build walkthrough and current -status live in [docs/quest-port.md](../docs/quest-port.md); this directory only +Quest 2, Quest 3, Quest 3S and Quest Pro, and, as the `steamFrame` flavour, for Valve's Steam +Frame under Lepton ([docs/steam-frame.md](../docs/steam-frame.md)). The full design, build +walkthrough and current status live in [docs/quest-port.md](../docs/quest-port.md); this directory only holds the Gradle project, its helper scripts, the game kit tooling (`QuestGameKit.psm1`, `Build-QuestGame.ps1`), the on-headset build toolchain (`Prepare-QuestToolchain.ps1`, `toolchain/`) and `nod-jni`. @@ -10,6 +11,7 @@ holds the Gradle project, its helper scripts, the game kit tooling ```powershell powershell -ExecutionPolicy Bypass -File android/Prepare-QuestDependencies.ps1 # stages the SDL3 3.4.4 AAR once powershell -ExecutionPolicy Bypass -File android/Build-Quest.ps1 -Install # the app, debug-signed, installs over adb +powershell -ExecutionPolicy Bypass -File android/Build-Quest.ps1 -Headset frame # Steam Frame flavour powershell -ExecutionPolicy Bypass -File android/Build-Quest.ps1 -Headset quest1 -Install # Quest 1 flavour ``` diff --git a/docs/quest-port.md b/docs/quest-port.md index 34f98c1..074fb78 100644 --- a/docs/quest-port.md +++ b/docs/quest-port.md @@ -819,9 +819,11 @@ Android facts this design rests on, all measured on a Quest 3: `generate-data-init` and `translate-mod`, for pipelines that generate on another host. - `android/`: the Gradle project, with `modernQuest` (the default script - target) and `quest1` headset flavours. They share the application ID and - storage, but select the appropriate CPU baseline, supported-device manifest, - and launcher behavior. + target), `quest1` and `steamFrame` headset flavours. They share the + application ID and storage, but select the appropriate CPU baseline (one + `headsetCpus` map in `app/build.gradle.kts`), manifest and launcher + behavior. `steamFrame` is Valve's Steam Frame under Lepton, SteamOS's + Android layer; see `docs/steam-frame.md`. `app/src/main/cpp/CMakeLists.txt` adds the repository's `runtime/` as a subdirectory with those Android choices and builds both game kit probes (the Retro Rewind one only when the translation includes the mod), which @@ -850,6 +852,7 @@ powershell -ExecutionPolicy Bypass -File android/Build-Quest.ps1 -Headset quest1 powershell -ExecutionPolicy Bypass -File android/Build-QuestGame.ps1 -Install # your game, against that kit, into Import (or WheelWizard VR's Build for Quest) powershell -ExecutionPolicy Bypass -File android/Build-QuestGame.ps1 -Product retro_rewind -Mod -Install # the mod and its pack (needs translate-mod output with --retro-wfc-payload) powershell -ExecutionPolicy Bypass -File android/Build-QuestGame.ps1 -Headset quest1 -Install # game package from the Quest 1 kit +powershell -ExecutionPolicy Bypass -File android/Build-Quest.ps1 -Headset frame # Steam Frame flavour (docs/steam-frame.md) adb push MarioKart.iso /sdcard/Download/ # then Select disc image in the launcher ``` diff --git a/docs/steam-frame.md b/docs/steam-frame.md new file mode 100644 index 0000000..701f631 --- /dev/null +++ b/docs/steam-frame.md @@ -0,0 +1,240 @@ +# WiiCompiled VR on the Steam Frame + +Valve's Steam Frame runs SteamOS on a Snapdragon 8 Gen 3 (Cortex-X4, A720 and A520 cores, Adreno 750), +with 2160x2160 panels per eye at 72 to 144 Hz, eye tracking, and SteamVR as its OpenXR runtime. It runs +Android apps through Lepton, SteamOS's Android layer, where SteamVR provides an Android OpenXR runtime +(OpenXR 1.0, through the Khronos loader's runtime broker). The Steam Frame build is therefore a third +flavour of the Quest app, `steamFrame`: everything in `docs/quest-port.md` below the app shell (the +Vulkan backend, the game kit, `.wcgame` packages, the on-headset build) applies unchanged, and this +document covers what differs. + +A native SteamOS ARM64 build is a separate, later piece of work: Linux has no OpenXR graphics backend +yet (see [A native SteamOS build](#a-native-steamos-build)). + +**Status: not yet run on a Steam Frame.** Everything below compiles and is unit-tested, but the +device checks at the end are still to do. + +## What the flavour changes + +| | Quest flavours | `steamFrame` | +| --- | --- | --- | +| CPU target (`kit.json` `androidCpu`) | `cortex-a77` (`kryo` on Quest 1) | `cortex-x4+nosve` | +| `MKW_ANDROID_HEADSET` | `quest` | `steam_frame` (defines `MKW_HEADSET_STEAM_FRAME`) | +| Library entry | `LauncherActivity` (Quest 1: `QuestActivity`) | `FrameEntryActivity` | +| Horizon OS manifest entries | present | removed | +| `[vr] refresh_rate` default | `0` (the headset's own) | `120` | +| `[vr] passthrough` | default on (`XR_FB_passthrough`) | not asked for, default off, setting hidden | +| `[vr] eye_tracked_foveation` default | off | on | + +The application ID stays `org.wiicompiled.quest`, so the storage paths in `docs/quest-port.md` hold +as they are. The kit's CPU string differs from the Quest ones, which gives the Frame its own kit +fingerprint: a game built for a Quest is refused on the Frame and the other way round, by the same +checks that keep Quest 1 and modern Quest games apart. + +**CPU.** Every core of the 8 Gen 3 implements ARMv9.2, so the products are tuned for the Cortex-X4. +`+nosve` matters: clang auto-vectorises with SVE for a `cortex-x4` (a simple loop compiled with +`-O3` used SVE registers ten times), and Qualcomm's firmware does not expose SVE on this chip, so +those instructions would end the game with `SIGILL`. With `+nosve` the target features read +`-sve -sve2 -sve2-bitperm` and the same loop uses NEON only. The flavour-to-CPU map lives once in +`android/app/build.gradle.kts` (`headsetCpus`), which the kit export also reads now instead of +guessing from the variant name. + +**Launch under Lepton.** Lepton starts the one real activity that is both `MAIN` and `LAUNCHER`, and +runs the app in VR when that activity carries a VR category; it ignores `activity-alias` entries. +Quest builds put `LAUNCHER` on the 2D panel (or, on Quest 1, add an alias), so neither works there. +`src/steamFrame/AndroidManifest.xml` makes `FrameEntryActivity` the only `MAIN`/`LAUNCHER` activity, +with `org.khronos.openxr.intent.category.IMMERSIVE_HMD` and `com.oculus.intent.category.VR`. It +shows nothing: it always opens the setup panel (`LauncherActivity`), and when the selected game can +start as it is (game files, a game built for this kit, Retro Rewind's pack, and no enabled mods +still to copy into the pack), it opens `QuestActivity` on top of it. The headset therefore goes +straight into VR, and quitting the game returns to the panel for setup, imports and mods. +`adb logcat -s WiiCompiledLauncher` shows which way it went. + +The manifest also removes Horizon OS's own entries (`com.oculus.supportedDevices`, `focusaware`, +`trade_cpu_for_gpu_amount`, the passthrough feature and the hand tracking permissions and feature) +and keeps the Khronos broker queries and the `OPENXR_SYSTEM` permission, which any Android OpenXR +runtime needs. + +## Controllers + +With `XR_VALVE_frame_controller_interaction` the runtime offers the Frame controller's own profile, +`/interaction_profiles/valve/frame_controller_valve`. Without it SteamVR presents the controllers as +Touch controllers, which loses the left D-pad. `openxr_input.cpp` suggests it after Touch and the +simple controller. Each hand has a thumbstick, trigger, grip and shoulder button. The right hand has +A, B, X, Y and a menu button; the left hand has a D-pad and a View button. Binding paths follow +DolphinXR's port (iChris4/dolphinXR#9). Windows asks for the same extension, so a Frame streaming +from a PC through SteamVR gets the D-pad too. + +| Frame controller | Wii Remote mode | Gamepad mode | +| --- | --- | --- | +| Right A | A | South (A) | +| Right B | C (look behind) | East (B) | +| Right trigger | B | Right trigger | +| Right stick up / down | 1 / 2 | Right stick | +| Left View | + (pause) | Start | +| Left shoulder | Settings panel (Touch's left Y) | North (Y) | +| Left D-pad | Wii Remote D-pad | D-pad | +| Left stick, left trigger | Nunchuk stick, Z | Left stick, left trigger | +| Grips, stick clicks, motion, aim | as on Touch (`OPENXR.md`, Controllers) | as on Touch | +| Right X, Y, menu and shoulder | unbound | unbound | + +The four D-pad actions are new and also reach the virtual gamepad's D-pad; Touch leaves them unbound, +so nothing changes on a Quest. + +## Refresh rate + +`[vr] refresh_rate` (Hz, `0` = the headset's own rate) is asked of the runtime through +`XR_FB_display_refresh_rate` each time the session starts running and whenever the setting changes. +The request uses the runtime's own value within half a hertz of the setting (runtimes report 119.98 +for 120). Setting it back to `0` restores the rate the session started at. The game renders 60 frames +a second, so at 120 Hz each frame shows for exactly two refreshes. At 72 or 90 Hz some frames show +for one refresh and others for two, which judders. The Frame starts at 120. Render-first pacing +(`docs/quest-port.md`) already waits for each sealed game frame, so on the Frame the pacing summary +should read about 60 `skipped-slots` a second with no `late` cycles. + +Lepton may decline the request (frame-control found SteamVR keeping its own rate there). The session +log then says `display refresh rate 120 Hz refused` with the rates it offers, and nothing else +changes. The setting is in the headset panel and in the launcher (VR → Headset); other headsets +offer it as well, at their own default of `0`. + +## Eye-tracked foveation + +`[vr] eye_tracked_foveation` (default on for the Frame) moves the foveation level's full-density +region to where the player looks: + +1. At launch the runtime asks for `XR_EXT_eye_gaze_interaction`. If the system reports an eye tracker, + `OpenXRInput` binds the gaze pose (`/user/eyes_ext/input/gaze_ext/pose`) and locates it for each + packet's display time, in the space the eye views are located in. +2. `vr/eye_gaze.h` turns the gaze into tangents of each eye's own view, which may be canted. + `AuroraStereoFrame` carries them as `gaze` and `gazeValid`, appended after its existing fields. +3. Aurora snaps the gaze to a cell of two map texels (64 pixels, about 3 degrees) and builds that + cell's density map with the level's rings centred on the gaze (`gfx/foveation.hpp`). + +Each eye keeps up to 32 maps, one per cell looked at, so a glance back reuses its map instead of +uploading a new one. A new map is bound once its upload has completed, and until then the eye keeps +the map it had. A blink, lost tracking or the setting turned off returns to the map centred on the +forward direction, which is byte-identical to the fixed foveation map. No change to the Dawn patch +was needed: maps stay immutable, and the patch already lets a view be rebound to another map. + +The session log reports `OpenXR eye gaze: available` (or that the runtime has no eye tracker), +`OpenXR eye gaze: tracking` at the first tracked sample, and `eye foveation medium following the +gaze` for each eye's first gaze map. Foveation pays only when an eye is pixel-bound +(`OPENXR.md`, Foveated rendering), which the Frame's larger eyes make more likely. + +If the Frame's driver offers `VK_QCOM_fragment_density_map_offset` or +`VK_EXT_fragment_density_map_offset`, shifting one map per pass would replace switching between +maps. That needs the Dawn patch to create the eye textures with the offset flag, so it waits for +the device's extension list. + +## Building and installing + +On the Windows build host described in `docs/quest-port.md`: + +```powershell +powershell -ExecutionPolicy Bypass -File android/Build-Quest.ps1 -Headset frame # the steamFrame APK; checks kit.json says cortex-x4+nosve +powershell -ExecutionPolicy Bypass -File android/Build-QuestGame.ps1 -Headset frame -Product base -Data # a .wcgame for the Frame's kit +``` + +The APK lands in `android/app/build/outputs/apk/steamFrame/`. WheelWizard VR's +"Build for Quest" builds a Frame game unchanged once it is given the Frame APK (Setup takes the +headset from the APK's kit). WheelWizard itself still needs a Steam Frame choice that fetches that +APK from the release, published as `…-SteamFrame.apk`. + +Lepton opens an adb port (5555 and up) for each running Android instance, reachable over the +network. With an Android app running on the Frame, `adb connect :5555` reaches it from the +build PC. How the APK reaches the Steam library (adb into a Lepton instance, frame-control, or +Steam's own sideloading) is to be confirmed on the device. + +## Device checklist + +Information to collect first, from the build PC over Lepton's adb port, with an Android app running +on the Frame: + +```powershell +adb connect :5555 +$f = ":5555" +adb -s $f shell "getprop ro.build.version.release; getprop ro.build.version.sdk; getprop ro.product.manufacturer; getprop ro.product.model; getprop ro.product.device; getprop ro.hardware.vulkan; getprop ro.board.platform" +adb -s $f shell "uname -a; getconf PAGE_SIZE; grep -m1 Features /proc/cpuinfo; grep -c processor /proc/cpuinfo" +adb -s $f shell cmd gpu vkjson > frame-vkjson.json +adb -s $f shell "pm list packages | grep -i -E 'xr|valve|steam|khronos|openxr'" +``` + +and on the Frame itself (Desktop Mode, Konsole): + +```bash +uname -a; getconf PAGESIZE; grep -m1 Features /proc/cpuinfo; head -5 /etc/os-release +cat ~/.config/openxr/1/active_runtime.json 2>/dev/null; ls /usr/share/openxr/1/ /etc/xdg/openxr/1/ 2>/dev/null +``` + +What each answers: + +- `vkjson`: whether Lepton's Vulkan driver has what the Android bridge needs: + - `VK_ANDROID_external_memory_android_hardware_buffer`; + - `VK_KHR_external_semaphore_fd` and `VK_KHR_external_fence_fd` with sync fd handles. + + Without these the APK route cannot present at all, and the native route becomes the way. It also + shows `VK_EXT_fragment_density_map` (foveation), the density map offset extensions, and which + driver Lepton uses. +- The CPU features line: no `sve`, as the CPU target assumes. +- The page size: on a 16 KB-page kernel the runtime finds out at launch and routes translated memory + accesses through the checked path (`guest_flat_memory.h`, `RequiresCheckedAccess`), which works but + is slower. That is worth knowing before measuring. +- The Android version: the app needs API 29 (Android 10) or newer. + +Then, on the first launch, the session log (`Logs/__pid/console.log` next to +`DATA`) and `adb logcat -s SDL WiiCompiledQuest WiiCompiledLauncher` should show, in order: + +1. `OpenXR Android loader initialized`; +2. `OpenXR runtime offers N extensions: ...`, the whole list SteamVR's Android runtime has; +3. `OpenXR initialized: runtime '...'` with SteamVR's name; +4. the negotiated Vulkan binding, then `OpenXR Vulkan swapchains ready`; +5. `display refresh rate 120 Hz requested` (or `refused`, with the rates on offer); +6. the session reaching `FOCUSED`; +7. `OpenXR interaction profiles: left /interaction_profiles/valve/frame_controller_valve, right ...`; +8. `OpenXR eye gaze: available`, then `tracking`; +9. `presentation=virtual-screen` for the menus, and `first immersive packet consumed` on race entry. + +Then, in the game: + +- the D-pad does tricks, the left shoulder opens the settings panel, and View pauses; +- `debug.wiicompiled.fpslog 1` on a race start shows the pacing summary and the GPU time per pass + with foveation off, fixed and following the gaze; +- `render_scale` starts at 0.8, the Quest's value. The runtime's recommended eye size (logged at + startup) and those measurements decide whether the Frame keeps it. + +A black headset with a working mirror points at the AHardwareBuffer copy, as on the Quest +(`docs/quest-port.md`, Validation status). + +Open questions only the device can answer: + +- Whether Lepton's driver supports the AHardwareBuffer and sync fd bridge. +- Whether Lepton shows the 2D setup panel while the app runs in VR mode. If it does not, the game + still starts directly once a `.wcgame` with the game files has been imported, but importing needs + the panel. +- Whether SteamVR grants 120 Hz. +- Whether the gaze needs an Android permission under Lepton. +- Whether "Build on this headset" can run its toolchain through `/system/bin/linker64` inside + Lepton. A game built on the PC does not depend on it. + +## A native SteamOS build + +Valve recommends native Linux ARM64 builds for the Frame, and one would avoid Lepton and the +two-device copy. The pieces: + +- **Backend.** The Windows Vulkan backend (`openxr_vulkan_win32.cpp`), where OpenXR creates Dawn's + own device and eyes are copied on Dawn's queue, ports almost as it is. Only its `_WIN32` guards are + platform-specific. +- **Interop.** Aurora's `vulkan_win32_interop.cpp` finds the patched Dawn's exports with + `GetModuleHandleW`. A static Linux Dawn would reference them directly, and `aurora_core.cmake` + compiles that file on Windows only. +- **Dawn.** A linux-aarch64 Dawn built with `aurora-main/patches/dawn` (the hook ABI and density + maps), as `android/Build-QuestDawn.ps1` already does for Android. +- **`openxr_integration.cpp`.** It needs a Linux branch asking for `XR_KHR_vulkan_enable2` and + `XR_KHR_convert_timespec_time`. Today its `#else` is Android's and requires + `XR_KHR_android_create_instance`. +- **Controller timing.** `openxr_input.cpp` needs a `__linux__` branch for the input clock. +- **CPU target.** An `MKW_LINUX_CPU` knob in place of `-mcpu=native`, for cross-builds. +- **Android-gated fixes.** The Adreno vertex padding, `headset_owns_display` and the + `last_pass_feeding_replay` saving are gated on `__ANDROID__`. They would follow the GPU or the + headset instead. +- **Packaging.** SteamOS packaging, and a way to build the player's game for it. From 1639e7f3c66be8bfc06f204afcd9d4522ef7cb14 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 09:00:48 +0000 Subject: [PATCH 06/14] Write the Steam Frame device checklist for any POSIX shell The adb commands named the device through a PowerShell variable, which fish and bash do not read; each line now names it, and two grep lines pull the driver and the relevant extensions out of the vkjson dump. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_019HBRGKTE1GnN2ah8gcZKr3 --- docs/steam-frame.md | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/steam-frame.md b/docs/steam-frame.md index 701f631..eb1f082 100644 --- a/docs/steam-frame.md +++ b/docs/steam-frame.md @@ -147,16 +147,18 @@ Steam's own sideloading) is to be confirmed on the device. ## Device checklist -Information to collect first, from the build PC over Lepton's adb port, with an Android app running -on the Frame: +Information to collect first, from any PC over Lepton's adb port (the commands work in bash, zsh and +fish; in PowerShell only the adb lines do), with an Android app running on the Frame: -```powershell +```sh adb connect :5555 -$f = ":5555" -adb -s $f shell "getprop ro.build.version.release; getprop ro.build.version.sdk; getprop ro.product.manufacturer; getprop ro.product.model; getprop ro.product.device; getprop ro.hardware.vulkan; getprop ro.board.platform" -adb -s $f shell "uname -a; getconf PAGE_SIZE; grep -m1 Features /proc/cpuinfo; grep -c processor /proc/cpuinfo" -adb -s $f shell cmd gpu vkjson > frame-vkjson.json -adb -s $f shell "pm list packages | grep -i -E 'xr|valve|steam|khronos|openxr'" +adb -s :5555 shell 'getprop ro.build.version.release; getprop ro.build.version.sdk; getprop ro.product.manufacturer; getprop ro.product.model; getprop ro.product.device; getprop ro.hardware.vulkan; getprop ro.board.platform' +adb -s :5555 shell 'uname -a; getconf PAGE_SIZE; grep -m1 Features /proc/cpuinfo; grep -c processor /proc/cpuinfo' +adb -s :5555 shell cmd gpu vkjson > frame-vkjson.json +adb -s :5555 shell "pm list packages | grep -i -E 'xr|valve|steam|khronos|openxr'" +# The parts of frame-vkjson.json that matter: +grep -oE '"(deviceName|driverName|driverInfo|apiVersion|driverVersion)": *[^,]*' frame-vkjson.json | sort -u +grep -oE '"extensionName": *"[^"]*"' frame-vkjson.json | grep -iE 'hardware_buffer|external_semaphore|external_fence|external_memory|fragment_density|shading_rate|dynamic_rendering' | sort -u ``` and on the Frame itself (Desktop Mode, Konsole): From 07e96a39138cc571afd12ef3251740765b44b0f7 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 09:07:54 +0000 Subject: [PATCH 07/14] Target the whole Cortex-X4 on the Steam Frame, which exposes SVE The Frame's kernel reports sve, sve2 and the SVE2 extensions in its HWCAP, on SteamOS and inside Lepton alike, unlike phones with the same Snapdragon 8 Gen 3. The steamFrame flavour now builds with -mcpu=cortex-x4 instead of cortex-x4+nosve. docs/steam-frame.md records what the headset reported: 4 KB pages, Android 11 (API 30) in a Waydroid-based Lepton, Mesa's Turnip as its Vulkan driver, and SteamVR's native linuxarm64 OpenXR runtime on the host, with the questions still open. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_019HBRGKTE1GnN2ah8gcZKr3 --- android/Build-Quest.ps1 | 2 +- android/app/build.gradle.kts | 6 +++--- docs/steam-frame.md | 42 ++++++++++++++++++++++++++++++------ 3 files changed, 39 insertions(+), 11 deletions(-) diff --git a/android/Build-Quest.ps1 b/android/Build-Quest.ps1 index 340796e..432a109 100644 --- a/android/Build-Quest.ps1 +++ b/android/Build-Quest.ps1 @@ -86,7 +86,7 @@ $variant = (Get-Culture).TextInfo.ToTitleCase($Configuration) # The Gradle flavour and the -mcpu target its kit must record (headsetCpus in app/build.gradle.kts). $flavourDir, $expectedCpu = switch ($Headset) { 'quest1' { 'quest1', 'kryo' } - 'frame' { 'steamFrame', 'cortex-x4+nosve' } + 'frame' { 'steamFrame', 'cortex-x4' } default { 'modernQuest', 'cortex-a77' } } $flavour = $flavourDir.Substring(0, 1).ToUpperInvariant() + $flavourDir.Substring(1) diff --git a/android/app/build.gradle.kts b/android/app/build.gradle.kts index e71493b..b2c42d6 100644 --- a/android/app/build.gradle.kts +++ b/android/app/build.gradle.kts @@ -61,9 +61,9 @@ val headsetCpus = mapOf( "modernQuest" to "cortex-a77", // Snapdragon 835. cortex-a77 binaries terminate with SIGILL on Quest 1. "quest1" to "kryo", - // Snapdragon 8 Gen 3 (Cortex-X4, A720 and A520, all ARMv9.2). Its firmware does not expose SVE, - // which clang would otherwise auto-vectorise with for a cortex-x4. - "steamFrame" to "cortex-x4+nosve", + // Steam Frame: Snapdragon 8 Gen 3 (Cortex-X4, A720 and A520, all ARMv9.2). Its kernel exposes + // SVE and SVE2 (HWCAP, inside Lepton too), so the whole cortex-x4 feature set is safe. + "steamFrame" to "cortex-x4", ) // android/nod-jni: nod, the disc image library the PC installer runs as nodtool, diff --git a/docs/steam-frame.md b/docs/steam-frame.md index eb1f082..1a093b6 100644 --- a/docs/steam-frame.md +++ b/docs/steam-frame.md @@ -18,7 +18,7 @@ device checks at the end are still to do. | | Quest flavours | `steamFrame` | | --- | --- | --- | -| CPU target (`kit.json` `androidCpu`) | `cortex-a77` (`kryo` on Quest 1) | `cortex-x4+nosve` | +| CPU target (`kit.json` `androidCpu`) | `cortex-a77` (`kryo` on Quest 1) | `cortex-x4` | | `MKW_ANDROID_HEADSET` | `quest` | `steam_frame` (defines `MKW_HEADSET_STEAM_FRAME`) | | Library entry | `LauncherActivity` (Quest 1: `QuestActivity`) | `FrameEntryActivity` | | Horizon OS manifest entries | present | removed | @@ -31,11 +31,12 @@ as they are. The kit's CPU string differs from the Quest ones, which gives the F fingerprint: a game built for a Quest is refused on the Frame and the other way round, by the same checks that keep Quest 1 and modern Quest games apart. -**CPU.** Every core of the 8 Gen 3 implements ARMv9.2, so the products are tuned for the Cortex-X4. -`+nosve` matters: clang auto-vectorises with SVE for a `cortex-x4` (a simple loop compiled with -`-O3` used SVE registers ten times), and Qualcomm's firmware does not expose SVE on this chip, so -those instructions would end the game with `SIGILL`. With `+nosve` the target features read -`-sve -sve2 -sve2-bitperm` and the same loop uses NEON only. The flavour-to-CPU map lives once in +**CPU.** Every core of the 8 Gen 3 implements ARMv9.2, so the products target the Cortex-X4 with +its whole feature set. That includes SVE and SVE2, which clang auto-vectorises with (a simple loop +compiled with `-O3` used SVE registers ten times). Phones with this chip do not expose SVE, but the +Frame's kernel does: `/proc/cpuinfo` lists `sve`, `sve2`, `svei8mm`, `svebf16` and the SVE2 crypto +extensions, on SteamOS and inside Lepton alike. A build for a device without SVE would need +`cortex-x4+nosve`. The flavour-to-CPU map lives once in `android/app/build.gradle.kts` (`headsetCpus`), which the kit export also reads now instead of guessing from the variant name. @@ -131,7 +132,7 @@ the device's extension list. On the Windows build host described in `docs/quest-port.md`: ```powershell -powershell -ExecutionPolicy Bypass -File android/Build-Quest.ps1 -Headset frame # the steamFrame APK; checks kit.json says cortex-x4+nosve +powershell -ExecutionPolicy Bypass -File android/Build-Quest.ps1 -Headset frame # the steamFrame APK; checks kit.json says cortex-x4 powershell -ExecutionPolicy Bypass -File android/Build-QuestGame.ps1 -Headset frame -Product base -Data # a .wcgame for the Frame's kit ``` @@ -145,6 +146,33 @@ network. With an Android app running on the Frame, `adb connect :5555` build PC. How the APK reaches the Steam library (adb into a Lepton instance, frame-control, or Steam's own sideloading) is to be confirmed on the device. +## What the Frame reported + +Read on 2026-10-04 from a Steam Frame running SteamOS (`holo`), kernel 6.18.0, with the commands +below: + +| Reading | SteamOS | Lepton | +| --- | --- | --- | +| Page size | 4096 | 4096 | +| CPU | 8 cores; `sve sve2 svei8mm svebf16 sveaes svepmull svebitperm svesha3 svesm4 i8mm bf16 bti paca pacg ...` | the same | +| Android | — | 11 (API 30), `ro.product.model` Lepton, device `lepton_arm64_only`, platform `waydroid` | +| Vulkan driver | — | `ro.hardware.vulkan=freedreno`: Mesa's Turnip, not Qualcomm's driver | +| OpenXR runtime | SteamVR, `bin/linuxarm64/vrclient.so` (`~/.config/openxr/1/active_runtime.json`) | no package named for XR, Valve, Steam or Khronos | + +What follows from them: + +- **Fast memory path.** 4 KB pages keep the translated code's flat memory path. A 16 KB kernel + would have sent it through the checked path. +- **CPU target.** The Frame exposes SVE, so the build targets the whole `cortex-x4` (above). +- **Android version.** API 30 meets the app's minimum of 29. +- **Driver workarounds.** Lepton is a Waydroid container, and its Vulkan driver is Turnip. The + Adreno workarounds in `docs/quest-port.md` were found on Qualcomm's own driver. The vertex padding + stays on (it is correct either way); `debug.wiicompiled.vtxpad 0` can check whether Turnip needs it. +- **Open:** whether Turnip in Lepton imports AHardwareBuffers and sync fds is in the `vkjson` + extension list, still to read. +- **Open:** how an app inside Lepton reaches SteamVR's OpenXR runtime. No broker package shows up, + so the Khronos loader's runtime broker may be provided some other way. + ## Device checklist Information to collect first, from any PC over Lepton's adb port (the commands work in bash, zsh and From 5c3e459a0dc7edd329bea603c16ef7eed632cd63 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 09:10:26 +0000 Subject: [PATCH 08/14] Record how Lepton apps find SteamVR's OpenXR runtime Lepton ships /vendor/etc/openxr/1/active_runtime.json and Valve's implicit fdm_injection layer instead of a runtime broker; the statically linked Khronos loader falls back to that file, so the app needs no change. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_019HBRGKTE1GnN2ah8gcZKr3 --- docs/steam-frame.md | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/docs/steam-frame.md b/docs/steam-frame.md index 1a093b6..fd1182d 100644 --- a/docs/steam-frame.md +++ b/docs/steam-frame.md @@ -155,9 +155,10 @@ below: | --- | --- | --- | | Page size | 4096 | 4096 | | CPU | 8 cores; `sve sve2 svei8mm svebf16 sveaes svepmull svebitperm svesha3 svesm4 i8mm bf16 bti paca pacg ...` | the same | -| Android | — | 11 (API 30), `ro.product.model` Lepton, device `lepton_arm64_only`, platform `waydroid` | +| Android | — | 11 (API 30, LineageOS), `ro.product.model` Lepton, device `lepton_arm64_only`, platform `waydroid`, `ro.steam.running_in_app_container=true` | | Vulkan driver | — | `ro.hardware.vulkan=freedreno`: Mesa's Turnip, not Qualcomm's driver | -| OpenXR runtime | SteamVR, `bin/linuxarm64/vrclient.so` (`~/.config/openxr/1/active_runtime.json`) | no package named for XR, Valve, Steam or Khronos | +| OpenXR runtime | SteamVR, `bin/linuxarm64/vrclient.so` (`~/.config/openxr/1/active_runtime.json`); SteamOS ships the SDK headers, `libopenxr_loader.a` and `openxr.pc` | `/vendor/etc/openxr/1/active_runtime.json`, from the host's `/usr/share/guestos/android/vendor/etc/openxr`; no runtime broker package | +| Implicit OpenXR layers | `XrApiLayer_VALVE_fdm_injection` (also listed as explicit) | `XrApiLayer_VALVE_fdm_injection` | What follows from them: @@ -170,8 +171,17 @@ What follows from them: stays on (it is correct either way); `debug.wiicompiled.vtxpad 0` can check whether Turnip needs it. - **Open:** whether Turnip in Lepton imports AHardwareBuffers and sync fds is in the `vkjson` extension list, still to read. -- **Open:** how an app inside Lepton reaches SteamVR's OpenXR runtime. No broker package shows up, - so the Khronos loader's runtime broker may be provided some other way. +- **Finding the runtime.** An app inside Lepton reaches SteamVR's OpenXR runtime through the system + runtime file, not a broker. The Khronos loader the game links statically (`DYNAMIC_LOADER OFF`) + tries the runtime brokers first, then reads `/{product,odm,oem,vendor,system}/etc/openxr/1/active_runtime.json`, + so no app change is needed; the manifest's broker queries are simply unused here. Walkabout Mini + Golf, an Android VR game, runs in the same Lepton. +- **Valve's foveation layer.** `XrApiLayer_VALVE_fdm_injection` is implicit, so it wraps every + Android OpenXR app. By its name it adds fragment density maps to apps' own render passes. This + game draws its eyes on Dawn's device and only copies them into the swapchain on the OpenXR + device, so the layer has no render pass of the game's to change; the game's own maps + (eye-tracked foveation, above) do that. If the layer gets in the way, its JSON names the + environment variable that disables it. ## Device checklist From cb0be5810c58c2df7188f4cb8bfa4ffb298c919c Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 09:15:34 +0000 Subject: [PATCH 09/14] Record the Steam Frame's Vulkan driver capabilities Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_019HBRGKTE1GnN2ah8gcZKr3 --- docs/steam-frame.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/steam-frame.md b/docs/steam-frame.md index fd1182d..d537822 100644 --- a/docs/steam-frame.md +++ b/docs/steam-frame.md @@ -156,7 +156,7 @@ below: | Page size | 4096 | 4096 | | CPU | 8 cores; `sve sve2 svei8mm svebf16 sveaes svepmull svebitperm svesha3 svesm4 i8mm bf16 bti paca pacg ...` | the same | | Android | — | 11 (API 30, LineageOS), `ro.product.model` Lepton, device `lepton_arm64_only`, platform `waydroid`, `ro.steam.running_in_app_container=true` | -| Vulkan driver | — | `ro.hardware.vulkan=freedreno`: Mesa's Turnip, not Qualcomm's driver | +| Vulkan driver | Turnip, Mesa 26.3.0-devel, Vulkan 1.4.362: `VK_EXT_fragment_density_map` (non-subsampled images, not dynamic), `VK_EXT_fragment_density_map_offset` and `VK_QCOM_fragment_density_map_offset`, `VK_KHR_external_{memory,semaphore,fence}_fd`, `VK_EXT_external_memory_dma_buf`, `VK_EXT_queue_family_foreign`, `VK_KHR_dynamic_rendering`; Valve's `fdm_injection` and `rpo` Vulkan layers | `ro.hardware.vulkan=freedreno`: the same Mesa 26.3.0-devel Turnip built for Android (`vulkan.pastel.so` is also present, not selected). `/dev/kgsl-3d0` is the DRM render node | | OpenXR runtime | SteamVR, `bin/linuxarm64/vrclient.so` (`~/.config/openxr/1/active_runtime.json`); SteamOS ships the SDK headers, `libopenxr_loader.a` and `openxr.pc` | `/vendor/etc/openxr/1/active_runtime.json`, from the host's `/usr/share/guestos/android/vendor/etc/openxr`; no runtime broker package | | Implicit OpenXR layers | `XrApiLayer_VALVE_fdm_injection` (also listed as explicit) | `XrApiLayer_VALVE_fdm_injection` | @@ -169,8 +169,12 @@ What follows from them: - **Driver workarounds.** Lepton is a Waydroid container, and its Vulkan driver is Turnip. The Adreno workarounds in `docs/quest-port.md` were found on Qualcomm's own driver. The vertex padding stays on (it is correct either way); `debug.wiicompiled.vtxpad 0` can check whether Turnip needs it. -- **Open:** whether Turnip in Lepton imports AHardwareBuffers and sync fds is in the `vkjson` - extension list, still to read. +- **Foveation.** The host's Turnip has density maps for non-subsampled images through dynamic + rendering, what the Dawn patch needs, and both density map offset extensions, which would let + eye-tracked foveation shift one map instead of switching maps. +- **Open:** whether Lepton's Android build of Turnip also imports AHardwareBuffers. `cmd gpu + vkjson` from the shell lists only instance extensions there (its device entry comes back empty), + and Mesa's binaries carry every extension name, so it takes an app inside Lepton to tell. - **Finding the runtime.** An app inside Lepton reaches SteamVR's OpenXR runtime through the system runtime file, not a broker. The Khronos loader the game links statically (`DYNAMIC_LOADER OFF`) tries the runtime brokers first, then reads `/{product,odm,oem,vendor,system}/etc/openxr/1/active_runtime.json`, From 79cd6e0478233d97173a7ddac771f1a3cb721213 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 09:37:08 +0000 Subject: [PATCH 10/14] Record the OpenXR extensions SteamVR offers Lepton apps Read from Walkabout Mini Golf's Unity log on the Frame: the Frame controller profile, display refresh rate, eye gaze and performance settings are all there; XR_KHR_convert_timespec_time is not, so VR frame interpolation is unavailable on the Frame. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_019HBRGKTE1GnN2ah8gcZKr3 --- docs/steam-frame.md | 49 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/docs/steam-frame.md b/docs/steam-frame.md index d537822..838d9d9 100644 --- a/docs/steam-frame.md +++ b/docs/steam-frame.md @@ -184,8 +184,53 @@ What follows from them: Android OpenXR app. By its name it adds fragment density maps to apps' own render passes. This game draws its eyes on Dawn's device and only copies them into the swapchain on the OpenXR device, so the layer has no render pass of the game's to change; the game's own maps - (eye-tracked foveation, above) do that. If the layer gets in the way, its JSON names the - environment variable that disables it. + (eye-tracked foveation, above) do that. If the layer gets in the way, + `DISABLE_VULKAN_FDM_INJECTION_LAYER` turns it off (its manifest loads + `libVkLayer_VALVE_fdm_injection.so`; the runtime itself is + `/data/steamvr/runtime/bin/androidarm64/vrclient.so`). + +### SteamVR's Android OpenXR extensions + +Walkabout Mini Golf, a Unity game in the same Lepton, logs what the runtime offers (`adb logcat -d | +grep -F '[XR]'`). Its extensions: + +``` +XR_EXT_active_action_set_priority XR_EXT_debug_utils XR_EXT_dpad_binding XR_EXT_eye_gaze_interaction +XR_EXT_frame_composition_report XR_EXT_frame_synthesis XR_EXT_hand_interaction XR_EXT_hand_joints_motion_range +XR_EXT_hand_tracking XR_EXT_hand_tracking_data_source XR_EXT_hp_mixed_reality_controller +XR_EXT_interaction_profile_battery_state_display XR_EXT_interaction_render_model XR_EXT_local_floor +XR_EXT_palm_pose XR_EXT_performance_settings XR_EXT_render_model XR_EXT_user_presence XR_EXT_uuid +XR_EXT_view_configuration_views_change XR_FB_display_refresh_rate XR_FB_foveation +XR_FB_foveation_configuration XR_FB_foveation_vulkan XR_FB_space_warp XR_FB_swapchain_update_state +XR_HTC_vive_cosmos_controller_interaction XR_HTC_vive_focus3_controller_interaction +XR_HTC_vive_wrist_tracker_interaction XR_HTCX_vive_tracker_interaction XR_KHR_android_create_instance +XR_KHR_binding_modification XR_KHR_composition_layer_depth XR_KHR_generic_controller XR_KHR_locate_spaces +XR_KHR_opengl_enable XR_KHR_opengl_es_enable XR_KHR_visibility_mask XR_KHR_vulkan_enable +XR_KHR_vulkan_enable2 XR_META_foveation_eye_tracked XR_META_performance_metrics +XR_META_recommended_layer_resolution XR_META_vulkan_swapchain_create_info XR_MND_headless +XR_MNDX_egl_enable XR_VALVE_analog_threshold XR_VALVE_app_space_delta_pose +XR_VALVE_frame_controller_interaction XR_VALVE_timing_utils +``` + +Environment blend modes `OPAQUE` and `ALPHA_BLEND`; reference spaces `LOCAL`, `STAGE` and `VIEW`. +What this build asks for and gets: + +| Extension | Offered | What it means here | +| --- | --- | --- | +| `XR_KHR_android_create_instance`, `XR_KHR_vulkan_enable2` | yes | The Android backend's instance and Vulkan binding | +| `XR_VALVE_frame_controller_interaction` | yes | The Frame controller profile and its D-pad | +| `XR_FB_display_refresh_rate` | yes | `[vr] refresh_rate` (120 Hz) can be requested | +| `XR_EXT_eye_gaze_interaction` | yes | Eye-tracked foveation | +| `XR_EXT_performance_settings` | yes | `performance_level` | +| `XR_EXT_hand_tracking`, `XR_EXT_hand_tracking_data_source` | yes | Tracked hands; `XR_FB_hand_tracking_mesh` and `_aim` are not offered, so the cockpit draws its procedural gloves and bare hands get no pinch gestures | +| `XR_KHR_convert_timespec_time` | **no** | VR frame interpolation is unavailable, and controller motion is sampled at the frame's display time rather than the current time | +| `XR_KHR_android_thread_settings` | no | Thread hints are skipped (logged as refused) | +| `XR_FB_passthrough` | no | As expected; the build does not ask for it | + +Later candidates the runtime offers: `ALPHA_BLEND` could bring back the room around the menu screen +without `XR_FB_passthrough`, `XR_KHR_visibility_mask` would skip the pixels the lenses never show, +and `XR_FB_foveation` with `XR_META_foveation_eye_tracked` only shapes render passes into the +runtime's swapchain images, which this game's eyes reach by copy, so it does not apply. ## Device checklist From da0d13bdd27b064a210df00a2f475468eddec613 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 09:41:58 +0000 Subject: [PATCH 11/14] Record that Lepton's Vulkan driver cannot share buffers between devices The Capability Viewer inside Lepton shows no AHardwareBuffer, external memory fd, semaphore fd or fence fd extension, so the Quest backend's two-device eye handoff cannot present there; the eyes need to be copied on Dawn's own device bound to the session, as on Windows Vulkan. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_019HBRGKTE1GnN2ah8gcZKr3 --- docs/steam-frame.md | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/docs/steam-frame.md b/docs/steam-frame.md index 838d9d9..eb92dcf 100644 --- a/docs/steam-frame.md +++ b/docs/steam-frame.md @@ -11,8 +11,11 @@ document covers what differs. A native SteamOS ARM64 build is a separate, later piece of work: Linux has no OpenXR graphics backend yet (see [A native SteamOS build](#a-native-steamos-build)). -**Status: not yet run on a Steam Frame.** Everything below compiles and is unit-tested, but the -device checks at the end are still to do. +**Status: not yet run on a Steam Frame, and the Android backend as it stands cannot present under +Lepton**: Lepton's Vulkan driver has no external memory or sync fd extensions, which the Quest +backend's two-device design needs (see [What the Frame reported](#what-the-frame-reported)). The +flavour, controller, refresh rate and foveation work below stays valid; the eye handoff has to move +to a single shared device first. ## What the flavour changes @@ -172,9 +175,17 @@ What follows from them: - **Foveation.** The host's Turnip has density maps for non-subsampled images through dynamic rendering, what the Dawn patch needs, and both density map offset extensions, which would let eye-tracked foveation shift one map instead of switching maps. -- **Open:** whether Lepton's Android build of Turnip also imports AHardwareBuffers. `cmd gpu - vkjson` from the shell lists only instance extensions there (its device entry comes back empty), - and Mesa's binaries carry every extension name, so it takes an app inside Lepton to tell. +- **No buffer sharing between devices in Lepton.** The Vulkan Hardware Capability Viewer (4.03, + the last release for Android 11), run inside Lepton, reports Turnip `26.2.99` (Vulkan 1.4.362, + display name "Valve Lepton") with `VK_EXT_fragment_density_map`, both density map offset + extensions, `VK_VALVE_fragment_density_map_layered`, `VK_KHR_timeline_semaphore` and the + maintenance extensions, but **no** `VK_ANDROID_external_memory_android_hardware_buffer`, + `VK_KHR_external_memory_fd`, `VK_KHR_external_semaphore_fd` or `VK_KHR_external_fence_fd`. The + Quest backend (`openxr_vulkan.cpp`) hands each eye from Dawn's device to its own OpenXR device + through exactly those, so it cannot present under Lepton. What can: binding Dawn's own device to + the session, as the Windows Vulkan backend (`openxr_vulkan_win32.cpp`) does, so the eyes are + copied into the swapchain on Dawn's queue with no sharing at all. That backend is also the core + of a native SteamOS build ([below](#a-native-steamos-build)). - **Finding the runtime.** An app inside Lepton reaches SteamVR's OpenXR runtime through the system runtime file, not a broker. The Khronos loader the game links statically (`DYNAMIC_LOADER OFF`) tries the runtime brokers first, then reads `/{product,odm,oem,vendor,system}/etc/openxr/1/active_runtime.json`, From 28a25108c6d2f58f76a0cc7f8f31886723000e41 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 09:48:48 +0000 Subject: [PATCH 12/14] Run the same-device Vulkan OpenXR backend on desktop Linux Lepton's Vulkan driver has no external memory or sync fd extensions, so the Steam Frame gets a native SteamOS build instead. Its backend is the PC's: OpenXR creates Dawn's own Vulkan instance and device, and the eyes are copied into the swapchain on Dawn's queue, with nothing shared between devices. - openxr_vulkan_win32 and Aurora's vulkan_win32_interop now compile on desktop Linux. Dawn links statically there, so the bridge calls the patched package's hooks directly, and exists only when the package's aurora-dawn.json declares the Vulkan hook ABI (AuroraDawnProvider turns AuroraVulkanAbi into AURORA_DAWN_VULKAN_HOOKS); a stock Dawn keeps the stubs and VR falls back to the desktop. - openxr_integration.cpp gains a Linux branch: XR_KHR_vulkan_enable2, the timespec clock, refresh rate, performance level, the Frame controllers and eye gaze; fragment density maps are requested on Linux as on the Quest (fdm.cpp and gpu.cpp, AURORA_FDM=0/1 overrides). - Controller motion uses XR_KHR_convert_timespec_time on any Linux. - CMake: Vulkan headers are fetched for Linux OpenXR builds, the headset option is MKW_HEADSET on Android and Linux, and Linux aarch64 builds take MKW_LINUX_CPU (cortex-x4 for the Steam Frame, else native). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_019HBRGKTE1GnN2ah8gcZKr3 --- android/app/build.gradle.kts | 4 +- aurora-main/cmake/AuroraDawnProvider.cmake | 9 +++++ aurora-main/cmake/aurora_core.cmake | 8 ++++ aurora-main/lib/webgpu/fdm.cpp | 8 ++-- aurora-main/lib/webgpu/gpu.cpp | 7 ++++ .../lib/webgpu/vulkan_win32_interop.cpp | 38 +++++++++++++++++-- docs/steam-frame.md | 2 +- runtime/CMakeLists.txt | 35 +++++++++-------- runtime/cmake/PublicProducts.cmake | 9 ++++- runtime/include/vr/openxr_vulkan_win32.h | 6 ++- runtime/src/vr/openxr_input.cpp | 9 ++++- runtime/src/vr/openxr_integration.cpp | 30 +++++++++++++-- runtime/src/vr/openxr_vulkan_win32.cpp | 8 +++- 13 files changed, 137 insertions(+), 36 deletions(-) diff --git a/android/app/build.gradle.kts b/android/app/build.gradle.kts index b2c42d6..beccbc4 100644 --- a/android/app/build.gradle.kts +++ b/android/app/build.gradle.kts @@ -187,7 +187,7 @@ android { } // Valve's Steam Frame, through Lepton (SteamOS's Android layer) and SteamVR's Android // OpenXR runtime. Its manifest (src/steamFrame) drops the Horizon OS entries, and - // MKW_ANDROID_HEADSET gives the runtime the Frame's defaults (runtime_config.h). + // MKW_HEADSET gives the runtime the Frame's defaults (runtime_config.h). create("steamFrame") { dimension = "headset" // Horizon OS only; src/steamFrame/AndroidManifest.xml removes the entry. @@ -199,7 +199,7 @@ android { cmake { arguments += listOf( "-DMKW_ANDROID_CPU=${headsetCpus.getValue("steamFrame")}", - "-DMKW_ANDROID_HEADSET=steam_frame", + "-DMKW_HEADSET=steam_frame", ) } } diff --git a/aurora-main/cmake/AuroraDawnProvider.cmake b/aurora-main/cmake/AuroraDawnProvider.cmake index 0a649dc..35214a0 100644 --- a/aurora-main/cmake/AuroraDawnProvider.cmake +++ b/aurora-main/cmake/AuroraDawnProvider.cmake @@ -268,7 +268,10 @@ elseif (_aurora_dawn_provider STREQUAL "package") # A package built with Aurora's patches (android/Build-QuestDawn.ps1) describes them in # aurora-dawn.json. Only such a package has the fragment density map ABI # (include/aurora/dawn_fdm_abi.h); aurora_core compiles its callers against it. + # The Vulkan hook ABI (include/aurora/dawn_vulkan_abi.h) is the other half: a package built by + # Launcher/build-dawn-linux.sh carries it, and the Linux OpenXR backend needs it. set(AURORA_DAWN_FDM_ABI 0 PARENT_SCOPE) + set(AURORA_DAWN_VULKAN_HOOKS_ABI 0 PARENT_SCOPE) if (EXISTS "${_dawn_pkg_dir}/aurora-dawn.json") file(READ "${_dawn_pkg_dir}/aurora-dawn.json" _aurora_dawn_manifest) string(JSON _aurora_dawn_fdm_abi ERROR_VARIABLE _aurora_dawn_manifest_error @@ -277,6 +280,12 @@ elseif (_aurora_dawn_provider STREQUAL "package") set(AURORA_DAWN_FDM_ABI ${_aurora_dawn_fdm_abi} PARENT_SCOPE) message(STATUS "aurora: Dawn package carries the fragment density map ABI ${_aurora_dawn_fdm_abi}") endif () + string(JSON _aurora_dawn_vulkan_abi ERROR_VARIABLE _aurora_dawn_manifest_error + GET "${_aurora_dawn_manifest}" AuroraVulkanAbi) + if (NOT _aurora_dawn_manifest_error AND _aurora_dawn_vulkan_abi GREATER 0) + set(AURORA_DAWN_VULKAN_HOOKS_ABI ${_aurora_dawn_vulkan_abi} PARENT_SCOPE) + message(STATUS "aurora: Dawn package carries the Vulkan hook ABI ${_aurora_dawn_vulkan_abi}") + endif () endif () _aurora_dawn_set_platform_backends() diff --git a/aurora-main/cmake/aurora_core.cmake b/aurora-main/cmake/aurora_core.cmake index 5ad4c73..f8f979d 100644 --- a/aurora-main/cmake/aurora_core.cmake +++ b/aurora-main/cmake/aurora_core.cmake @@ -39,6 +39,14 @@ if (AURORA_ENABLE_GX) # of producing unresolved interop references. if (CMAKE_SYSTEM_NAME STREQUAL Windows) target_sources(aurora_core PRIVATE lib/webgpu/d3d12_interop.cpp lib/webgpu/vulkan_win32_interop.cpp) + elseif (CMAKE_SYSTEM_NAME STREQUAL Linux) + # The same-device Vulkan bridge, for the desktop Linux OpenXR backend (SteamOS on the Steam + # Frame). Static Dawn on Linux is linked directly, so the bridge is compiled in only against + # a package with Aurora's Vulkan hooks; otherwise the file is its C ABI stubs. + target_sources(aurora_core PRIVATE lib/webgpu/vulkan_win32_interop.cpp) + if (AURORA_DAWN_VULKAN_HOOKS_ABI) + target_compile_definitions(aurora_core PRIVATE AURORA_DAWN_VULKAN_HOOKS=${AURORA_DAWN_VULKAN_HOOKS_ABI}) + endif () endif () # Android/Vulkan counterpart: the AHardwareBuffer stereo bridge. The file # compiles to C ABI stubs on every other platform so the runtime's OpenXR diff --git a/aurora-main/lib/webgpu/fdm.cpp b/aurora-main/lib/webgpu/fdm.cpp index 0e647c9..f48c5a0 100644 --- a/aurora-main/lib/webgpu/fdm.cpp +++ b/aurora-main/lib/webgpu/fdm.cpp @@ -6,8 +6,10 @@ #include #include -// AURORA_DAWN_FDM carries the ABI version of the Dawn package's patches (AuroraDawnProvider.cmake). -#if defined(__ANDROID__) && defined(AURORA_DAWN_FDM) +// AURORA_DAWN_FDM carries the ABI version of the Dawn package's patches (AuroraDawnProvider.cmake): +// the Quest's (android/Build-QuestDawn.ps1) or desktop Linux's (Launcher/build-dawn-linux.sh), both +// linked statically. +#if (defined(__ANDROID__) || defined(__linux__)) && defined(AURORA_DAWN_FDM) #include static_assert(AURORA_DAWN_FDM == AURORA_DAWN_FDM_ABI, "The Dawn package's fragment density map ABI does not match include/aurora/dawn_fdm_abi.h"); @@ -53,7 +55,7 @@ void device_created() noexcept { #else if (g_requested) { Log.warn("Fragment density maps: unavailable; this build links a Dawn without Aurora's patches " - "(android/Build-QuestDawn.ps1)"); + "(android/Build-QuestDawn.ps1, Launcher/build-dawn-linux.sh)"); } #endif } diff --git a/aurora-main/lib/webgpu/gpu.cpp b/aurora-main/lib/webgpu/gpu.cpp index 02c0690..0876bb6 100644 --- a/aurora-main/lib/webgpu/gpu.cpp +++ b/aurora-main/lib/webgpu/gpu.cpp @@ -830,6 +830,13 @@ bool initialize(AuroraBackend auroraBackend) { const int fdmOverride = android_debug::property_int("debug.wiicompiled.fdm", -1); fdm::request(g_backendType == wgpu::BackendType::Vulkan && (fdmOverride >= 0 ? fdmOverride == 1 : g_config.xrFragmentDensityMap)); +#elif defined(__linux__) + // Desktop Linux (SteamOS on the Steam Frame), with a Dawn built by Launcher/build-dawn-linux.sh. + // AURORA_FDM=0 or 1 overrides the settings, as debug.wiicompiled.fdm does on the Quest. + const char* fdmOverride = std::getenv("AURORA_FDM"); + fdm::request(g_backendType == wgpu::BackendType::Vulkan && + (fdmOverride != nullptr && *fdmOverride != '\0' ? std::strcmp(fdmOverride, "1") == 0 + : g_config.xrFragmentDensityMap)); #endif const auto future = g_adapter.RequestDevice(&deviceDescriptor, wgpu::CallbackMode::WaitAnyOnly, diff --git a/aurora-main/lib/webgpu/vulkan_win32_interop.cpp b/aurora-main/lib/webgpu/vulkan_win32_interop.cpp index 45dba21..93f44f6 100644 --- a/aurora-main/lib/webgpu/vulkan_win32_interop.cpp +++ b/aurora-main/lib/webgpu/vulkan_win32_interop.cpp @@ -4,8 +4,27 @@ #include "../stereo.hpp" #include "../stereo_overlay.hpp" #include "gpu.hpp" -#if defined(_WIN32) && defined(WEBGPU_DAWN) && defined(DAWN_ENABLE_BACKEND_VULKAN) +// Windows loads the patched webgpu_dawn.dll's exports at run time. Desktop Linux (SteamOS on the +// Steam Frame) links Dawn statically, so there the bridge exists only against a package built with +// the hooks (AURORA_DAWN_VULKAN_HOOKS, from its aurora-dawn.json) and calls them directly. +#if defined(WEBGPU_DAWN) && defined(DAWN_ENABLE_BACKEND_VULKAN) && \ + (defined(_WIN32) || (defined(__linux__) && !defined(__ANDROID__) && defined(AURORA_DAWN_VULKAN_HOOKS))) +#if defined(_WIN32) #include +#else +static_assert(AURORA_DAWN_VULKAN_HOOKS == AURORA_DAWN_VULKAN_ABI, + "The Dawn package's Vulkan hook ABI does not match include/aurora/dawn_vulkan_abi.h"); +extern "C" { +uint32_t AuroraDawnVulkanVersion(void); +int AuroraDawnVulkanConfigure(const AuroraDawnVulkanHooks* hooks); +int AuroraDawnVulkanGetHandles(void* device, AuroraDawnVulkanHandles* handles); +void* AuroraDawnVulkanWrap(void* device, const void* textureDescriptor, uint64_t image); +int AuroraDawnVulkanRelease(void* device, void* const* textures, uint32_t count); +void* AuroraDawnVulkanLock(void* device); +void AuroraDawnVulkanUnlock(void* guard); +int AuroraDawnVulkanDrain(void* device); +} +#endif #include #include #include @@ -23,6 +42,17 @@ struct Api { AuroraDawnVulkanUnlockFn unlock = nullptr; AuroraDawnVulkanDrainFn drain = nullptr; bool Load() { +#if !defined(_WIN32) + if (AuroraDawnVulkanVersion() != AURORA_DAWN_VULKAN_ABI) return false; + configure = &AuroraDawnVulkanConfigure; + handles = &AuroraDawnVulkanGetHandles; + wrap = &AuroraDawnVulkanWrap; + release = &AuroraDawnVulkanRelease; + lock = &AuroraDawnVulkanLock; + unlock = &AuroraDawnVulkanUnlock; + drain = &AuroraDawnVulkanDrain; + return true; +#else HMODULE module = GetModuleHandleW(L"webgpu_dawn.dll"); if (!module) return false; auto version = reinterpret_cast(GetProcAddress(module, "AuroraDawnVulkanVersion")); @@ -37,6 +67,7 @@ struct Api { LOAD(drain, AuroraDawnVulkanDrainFn, "AuroraDawnVulkanDrain"); #undef LOAD return true; +#endif } } api; int64_t VkFormat(wgpu::TextureFormat format) { @@ -222,8 +253,9 @@ void* aurora_vulkan_win32_lock_queue() { void aurora_vulkan_win32_unlock_queue(void* guard) { aurora::vulkan_win32::api.unlock(guard); } #else // C ABI stubs keep the runtime's OpenXR integration linkable on Windows GX -// builds whose Dawn has no Vulkan backend; the backend then reports that the -// bridge is unavailable and the game falls back to the desktop renderer. +// builds whose Dawn has no Vulkan backend, and on Linux against a Dawn without +// Aurora's hooks; the backend then reports that the bridge is unavailable and +// the game falls back to the desktop renderer. bool aurora_vulkan_win32_configure(const AuroraDawnVulkanHooks*) { return false; } bool aurora_vulkan_win32_get_handles(AuroraDawnVulkanHandles* handles, int64_t* format) { if (handles) *handles = {}; diff --git a/docs/steam-frame.md b/docs/steam-frame.md index eb92dcf..c12bfce 100644 --- a/docs/steam-frame.md +++ b/docs/steam-frame.md @@ -22,7 +22,7 @@ to a single shared device first. | | Quest flavours | `steamFrame` | | --- | --- | --- | | CPU target (`kit.json` `androidCpu`) | `cortex-a77` (`kryo` on Quest 1) | `cortex-x4` | -| `MKW_ANDROID_HEADSET` | `quest` | `steam_frame` (defines `MKW_HEADSET_STEAM_FRAME`) | +| `MKW_HEADSET` | empty | `steam_frame` (defines `MKW_HEADSET_STEAM_FRAME`) | | Library entry | `LauncherActivity` (Quest 1: `QuestActivity`) | `FrameEntryActivity` | | Horizon OS manifest entries | present | removed | | `[vr] refresh_rate` default | `0` (the headset's own) | `120` | diff --git a/runtime/CMakeLists.txt b/runtime/CMakeLists.txt index bbda761..e977a35 100644 --- a/runtime/CMakeLists.txt +++ b/runtime/CMakeLists.txt @@ -37,18 +37,19 @@ option(MKW_BUILD_PRODUCTS "Build translated WiiCompiled product targets" ON) # aurora-main and its third-party tree stay unaffected. set(MKW_PROJECT_COMPILE_DEFINITIONS NOMINMAX) -# The Android headset family the build is for (the Gradle flavour passes it). The -# Steam Frame build runs under SteamVR's Android runtime rather than Horizon OS and -# takes its own defaults (runtime_config.h) through MKW_HEADSET_STEAM_FRAME. The -# definition reaches every target below, so the game kit's recorded compile flags -# carry it too. -if(MKW_PLATFORM_ANDROID) - set(MKW_ANDROID_HEADSET "quest" CACHE STRING "Android headset family: quest (Meta Quest, any flavour) or steam_frame") - set_property(CACHE MKW_ANDROID_HEADSET PROPERTY STRINGS quest steam_frame) - if(MKW_ANDROID_HEADSET STREQUAL "steam_frame") +# The headset a standalone build is for: the Android app's Gradle flavour passes +# it, and so does a native SteamOS build for the Steam Frame +# (Launcher/local-build.sh --headset steam_frame). The Steam Frame takes its own +# defaults (runtime_config.h) through MKW_HEADSET_STEAM_FRAME. The definition +# reaches every target below, so the Quest game kit's recorded compile flags carry +# it too. +if(MKW_PLATFORM_ANDROID OR MKW_PLATFORM_LINUX) + set(MKW_HEADSET "" CACHE STRING "Headset the build targets: empty (a Meta Quest on Android, any PC headset elsewhere) or steam_frame") + set_property(CACHE MKW_HEADSET PROPERTY STRINGS "" steam_frame) + if(MKW_HEADSET STREQUAL "steam_frame") list(APPEND MKW_PROJECT_COMPILE_DEFINITIONS MKW_HEADSET_STEAM_FRAME=1) - elseif(NOT MKW_ANDROID_HEADSET STREQUAL "quest") - message(FATAL_ERROR "MKW_ANDROID_HEADSET must be quest or steam_frame, not '${MKW_ANDROID_HEADSET}'") + elseif(NOT MKW_HEADSET STREQUAL "") + message(FATAL_ERROR "MKW_HEADSET must be empty or steam_frame, not '${MKW_HEADSET}'") endif() endif() @@ -148,10 +149,10 @@ set(MKW_TRANSLATED_COMPILE_JOBS 0 CACHE STRING Scheduling only - never affects output bytes, so it is deliberately outside the canonical flag fingerprint.") set(MKW_CPPWINRT_INCLUDE_DIR "" CACHE PATH "Optional self-contained C++/WinRT include directory") -# The functional backend is currently Windows D3D12, so only Windows defaults -# to carrying the loader. Linux keeps the capability-gated Vulkan scaffold -# available to explicit developer builds without making a nonfunctional SDK -# fetch part of every normal build. +# Windows and Android carry the loader by default. Desktop Linux has the +# same-device Vulkan backend too, but it needs a Dawn built with Aurora's patches +# (Launcher/build-dawn-linux.sh), so it stays opt-in: Launcher/local-build.sh +# --openxr, as the Steam Frame's native SteamOS build uses it. set(MKW_ENABLE_OPENXR_DEFAULT OFF) if(MKW_PLATFORM_WINDOWS OR MKW_PLATFORM_ANDROID) set(MKW_ENABLE_OPENXR_DEFAULT ON) @@ -301,7 +302,9 @@ if(MKW_PROJECT_COMPILE_DEFINITIONS) add_compile_definitions(${MKW_PROJECT_COMPILE_DEFINITIONS}) endif() -if(MKW_PLATFORM_WINDOWS AND MKW_ENABLE_OPENXR) +# The same-device Vulkan backend (openxr_vulkan_win32.cpp, on Windows and desktop +# Linux) includes vulkan.h; the pin keeps it independent of the build machine. +if((MKW_PLATFORM_WINDOWS OR MKW_PLATFORM_LINUX) AND MKW_ENABLE_OPENXR) include(FetchContent) FetchContent_Declare(vulkan_headers URL "https://github.com/KhronosGroup/Vulkan-Headers/archive/015e25c3c91b70eb1a754d36fb14c4ba6ad9b0b9.tar.gz" diff --git a/runtime/cmake/PublicProducts.cmake b/runtime/cmake/PublicProducts.cmake index 4cc7ba2..3769473 100644 --- a/runtime/cmake/PublicProducts.cmake +++ b/runtime/cmake/PublicProducts.cmake @@ -452,7 +452,14 @@ elseif(MKW_PLATFORM_ANDROID) set(MKW_ANDROID_CPU "cortex-a77" CACHE STRING "AArch64 -mcpu target for the Android products") set(MKW_BASELINE_ARCH_FLAG -mcpu=${MKW_ANDROID_CPU}) elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64|ARM64)$") - set(MKW_BASELINE_ARCH_FLAG -mcpu=native) + # A cross-build names its CPU; the Steam Frame's Snapdragon 8 Gen 3 exposes SVE (cortex-x4). + if(MKW_HEADSET STREQUAL "steam_frame") + set(_mkw_linux_cpu_default "cortex-x4") + else() + set(_mkw_linux_cpu_default "native") + endif() + set(MKW_LINUX_CPU "${_mkw_linux_cpu_default}" CACHE STRING "AArch64 -mcpu target for the Linux products") + set(MKW_BASELINE_ARCH_FLAG -mcpu=${MKW_LINUX_CPU}) else() set(MKW_BASELINE_ARCH_FLAG "") endif() diff --git a/runtime/include/vr/openxr_vulkan_win32.h b/runtime/include/vr/openxr_vulkan_win32.h index 638290e..98c4be1 100644 --- a/runtime/include/vr/openxr_vulkan_win32.h +++ b/runtime/include/vr/openxr_vulkan_win32.h @@ -2,7 +2,9 @@ #pragma once -#if defined(MKW_ENABLE_OPENXR) && defined(_WIN32) +// Despite the name, also the desktop Linux backend (SteamOS on the Steam Frame): nothing in it is +// Windows-specific. The Quest has its own two-device backend (openxr_vulkan.h). +#if defined(MKW_ENABLE_OPENXR) && (defined(_WIN32) || (defined(__linux__) && !defined(__ANDROID__))) #include "vr/openxr_backend.h" #include "vr/openxr_runtime.h" @@ -112,4 +114,4 @@ private: } // namespace mkw::vr -#endif // defined(MKW_ENABLE_OPENXR) && defined(_WIN32) +#endif // defined(MKW_ENABLE_OPENXR) && (defined(_WIN32) || (defined(__linux__) && !defined(__ANDROID__))) diff --git a/runtime/src/vr/openxr_input.cpp b/runtime/src/vr/openxr_input.cpp index 3436ba3..8929226 100644 --- a/runtime/src/vr/openxr_input.cpp +++ b/runtime/src/vr/openxr_input.cpp @@ -35,6 +35,8 @@ #if defined(__ANDROID__) #include +#endif +#if defined(__linux__) #include #endif @@ -275,6 +277,9 @@ bool Injected(const char*) { return false; } #if defined(_WIN32) using ConvertNowToXrTime = XrResult(XRAPI_PTR*)(XrInstance, const LARGE_INTEGER*, XrTime*); +#elif defined(__linux__) +// Desktop Linux (SteamOS on the Steam Frame); Android declares it with its injection above. +using ConvertNowToXrTime = XrResult(XRAPI_PTR*)(XrInstance, const struct timespec*, XrTime*); #endif #endif @@ -666,7 +671,7 @@ void OpenXRInput::LoadInputClock() { if (enabled("XR_KHR_win32_convert_performance_counter_time")) { m_runtime->GetInstanceProcAddress("xrConvertWin32PerformanceCounterToTimeKHR", &function); } -#elif defined(__ANDROID__) +#elif defined(__linux__) if (enabled("XR_KHR_convert_timespec_time")) { m_runtime->GetInstanceProcAddress("xrConvertTimespecTimeToTimeKHR", &function); } @@ -692,7 +697,7 @@ XrTime OpenXRInput::InputSampleTime(XrTime predicted_display_time) const { &counter, &now))) { return predicted_display_time; } -#elif defined(__ANDROID__) +#elif defined(__linux__) timespec spec{}; if (clock_gettime(CLOCK_MONOTONIC, &spec) != 0 || XR_FAILED(reinterpret_cast(m_convert_now_to_xr_time)(m_runtime->Instance(), diff --git a/runtime/src/vr/openxr_integration.cpp b/runtime/src/vr/openxr_integration.cpp index be5f9c9..695b73b 100644 --- a/runtime/src/vr/openxr_integration.cpp +++ b/runtime/src/vr/openxr_integration.cpp @@ -48,6 +48,14 @@ #define XR_USE_TIMESPEC #include #define MKW_OPENXR_GRAPHICS_BACKEND 1 +#elif defined(__linux__) +// Desktop Linux, SteamOS on the Steam Frame above all: the PC's same-device Vulkan backend, with +// Dawn's own device bound to the session. +#include "vr/openxr_vulkan_win32.h" +#include +#define XR_USE_TIMESPEC +#include +#define MKW_OPENXR_GRAPHICS_BACKEND 1 #else #define MKW_OPENXR_GRAPHICS_BACKEND 0 #endif @@ -85,9 +93,12 @@ void ConfigurePolicy(bool enabled) noexcept { #if defined(_WIN32) using GraphicsBackend = OpenXRWindowsBackend; -#else +#elif defined(__ANDROID__) using GraphicsBackend = OpenXRVulkanBackend; inline constexpr const char* kGraphicsBackendName = "Vulkan"; +#else +using GraphicsBackend = OpenXRWindowsVulkanBackend; +inline constexpr const char* kGraphicsBackendName = "Vulkan"; #endif // Whether the immersive window's eyes can be aimed through the window, so that only the window is @@ -412,6 +423,16 @@ public: "XR_FB_display_refresh_rate", "XR_EXT_performance_settings", "XR_VALVE_frame_controller_interaction"}; AddHandMeshExtensions(config); +#elif defined(__linux__) && !defined(__ANDROID__) + // As the PC's Vulkan binding, plus the clock conversion Linux uses and the Steam Frame's + // eye gaze for eye-tracked foveation. + config.required_extensions = {"XR_KHR_vulkan_enable2"}; + config.optional_extensions = {"XR_KHR_convert_timespec_time", "XR_FB_display_refresh_rate", + "XR_EXT_performance_settings", "XR_VALVE_frame_controller_interaction"}; + if (RuntimeConfigFile::VrEyeTrackedFoveation()) { + config.optional_extensions.push_back("XR_EXT_eye_gaze_interaction"); + } + AddHandMeshExtensions(config); #else // Either Vulkan binding extension is acceptable; the backend picks // whichever the runtime enabled, preferring enable2. @@ -718,9 +739,10 @@ private: void ApplyGraphicsRequirements(AuroraConfig& aurora_config) { aurora_config.desiredBackend = kRequiredAuroraBackend; aurora_config.xrInterop = true; -#if defined(__ANDROID__) - // Foveated rendering: fragment density maps are decided with the device. They put a flag on - // every render pipeline, so a session launched with foveation off does without them. +#if !defined(_WIN32) + // Foveated rendering (the Quest's and desktop Linux's patched Dawn): fragment density maps + // are decided with the device. They put a flag on every render pipeline, so a session + // launched with foveation off does without them. aurora_config.xrFragmentDensityMap = RuntimeConfigFile::VrFoveation() != "off"; #endif #if defined(_WIN32) diff --git a/runtime/src/vr/openxr_vulkan_win32.cpp b/runtime/src/vr/openxr_vulkan_win32.cpp index 1515df8..bd6da99 100644 --- a/runtime/src/vr/openxr_vulkan_win32.cpp +++ b/runtime/src/vr/openxr_vulkan_win32.cpp @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-3.0-or-later -#if defined(MKW_ENABLE_OPENXR) && defined(_WIN32) +#if defined(MKW_ENABLE_OPENXR) && (defined(_WIN32) || (defined(__linux__) && !defined(__ANDROID__))) // OpenXR's Vulkan structures are selected when openxr_platform.h is parsed. #define XR_USE_GRAPHICS_API_VULKAN @@ -180,7 +180,11 @@ public: if (XR_FAILED(get(runtime.Instance(), runtime.SystemId(), &requirements))) return Fail("Vulkan requirements query failed"); requirements_ = {requirements.minApiVersionSupported, requirements.maxApiVersionSupported}; AuroraDawnVulkanHooks hooks{this, CreateInstance, CreateDevice, GetPhysical}; +#if defined(_WIN32) if (!aurora_vulkan_win32_configure(&hooks)) return Fail("PC Vulkan OpenXR requires the custom Dawn library with Aurora Vulkan ABI 1; rebuild/install it with Launcher/Build-DawnVulkan.ps1"); +#else + if (!aurora_vulkan_win32_configure(&hooks)) return Fail("Linux Vulkan OpenXR requires a Dawn built with Aurora's patches (Aurora Vulkan ABI 1); build it with Launcher/build-dawn-linux.sh"); +#endif hooks_installed_ = true; { std::lock_guard lock(submission_mutex_); @@ -1315,4 +1319,4 @@ const std::string& OpenXRWindowsVulkanBackend::LastError() const { return m_impl } // namespace mkw::vr -#endif // defined(MKW_ENABLE_OPENXR) && defined(_WIN32) +#endif // defined(MKW_ENABLE_OPENXR) && (defined(_WIN32) || (defined(__linux__) && !defined(__ANDROID__))) From 0edf10b22010fd311e8d729e50472b05baa5a153 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 09:51:08 +0000 Subject: [PATCH 13/14] Give the Steam Frame's native build the standalone headset defaults MKW_VR_STANDALONE (runtime_config.h) names the standalone headsets: the Quest, and the Steam Frame in its Android flavour and its native SteamOS build. They share render scale 0.8, foveation medium, the GX thread, the game's object culling, and the headset panel's foveation and tracked-hands rows; passthrough stays Quest-only. The Frame's native build also sets AuroraConfig::xrHeadsetOnly, so Aurora neither presents the desktop window nobody watches nor renders it past the last pass the eyes sample, which Android does always (4 to 6 ms of a 12 ms GPU frame on a Quest 3). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_019HBRGKTE1GnN2ah8gcZKr3 --- aurora-main/include/aurora/aurora.h | 4 ++++ aurora-main/lib/aurora.cpp | 2 +- runtime/include/runtime_config.h | 18 +++++++++++++----- runtime/src/settings_overlay.cpp | 14 +++++++------- runtime/src/vr/openxr_integration.cpp | 5 +++++ runtime/tests/vr_config_tests.cpp | 4 ++-- 6 files changed, 32 insertions(+), 15 deletions(-) diff --git a/aurora-main/include/aurora/aurora.h b/aurora-main/include/aurora/aurora.h index 46bec06..ff9120f 100644 --- a/aurora-main/include/aurora/aurora.h +++ b/aurora-main/include/aurora/aurora.h @@ -289,6 +289,10 @@ typedef struct { bool hasD3D12AdapterLuid; uint32_t d3d12AdapterLuidLow; int32_t d3d12AdapterLuidHigh; + // Nobody watches the desktop window while the headset runs (a standalone + // headset such as the Steam Frame's native build): skip presenting it and stop + // the mono render after the last pass the eyes sample, as Android always does. + bool xrHeadsetOnly; } AuroraConfig; typedef struct { diff --git a/aurora-main/lib/aurora.cpp b/aurora-main/lib/aurora.cpp index 961cfae..3ab35d1 100644 --- a/aurora-main/lib/aurora.cpp +++ b/aurora-main/lib/aurora.cpp @@ -1416,7 +1416,7 @@ bool headset_owns_display() noexcept { #if defined(__ANDROID__) return stereo_frame_provider_active(); #else - return false; + return g_config.xrHeadsetOnly && stereo_frame_provider_active(); #endif } diff --git a/runtime/include/runtime_config.h b/runtime/include/runtime_config.h index 7f9466d..068c124 100644 --- a/runtime/include/runtime_config.h +++ b/runtime/include/runtime_config.h @@ -1,5 +1,12 @@ #pragma once +// The standalone headsets: the Quest (Android) and the Steam Frame, in its Android flavour and its +// native SteamOS build. They render on a mobile GPU with a game thread that is the bottleneck, so +// they share the defaults below and the headset panel's foveation and tracked-hands rows. +#if defined(__ANDROID__) || defined(MKW_HEADSET_STEAM_FRAME) +#define MKW_VR_STANDALONE 1 +#endif + #include #include #include @@ -193,7 +200,7 @@ inline constexpr uint32_t kDisabledPostProcessingPathsDefault = kPostProcessingB // standalone headset renders on a mobile GPU, so the Quest and the Steam Frame // start below it; the launcher's first Config.toml (GameStorage.kt) writes the // same value. -#if defined(__ANDROID__) +#if defined(MKW_VR_STANDALONE) inline constexpr float kVrRenderScaleDefault = 0.8f; #define MKW_VR_RENDER_SCALE_DEFAULT_TEXT "0.8" #else @@ -256,7 +263,7 @@ inline constexpr bool kVrCockpitItemThrowDefault = true; // draws them by default; the Quest keeps the game's culling, since every // extra model costs its GPU twice. The macro is the same default for the // config file written on first launch. -#if defined(__ANDROID__) +#if defined(MKW_VR_STANDALONE) inline constexpr bool kVrObjectCullingDefault = true; #define MKW_VR_OBJECT_CULLING_DEFAULT_TOML "true" #else @@ -310,7 +317,7 @@ inline bool IsSupportedVrRefreshRate(uint32_t value) { // from "off" to a level takes a restart; between levels and back to "off" it is // live. The standalone headsets (Quest, Steam Frame) start at "medium"; // elsewhere it does nothing. -#if defined(__ANDROID__) +#if defined(MKW_VR_STANDALONE) inline constexpr const char* kVrFoveationDefault = "medium"; #else inline constexpr const char* kVrFoveationDefault = "off"; @@ -1646,9 +1653,10 @@ inline bool ShowFps(bool fallback = false) { } // Runs the host side of the GX pipeline on its own thread (gx_thread.h). On by -// default on the Quest, where the game thread is the bottleneck; opt-in elsewhere. +// default on the standalone headsets, where the game thread is the bottleneck; +// opt-in elsewhere. inline bool GxThread() { -#if defined(__ANDROID__) +#if defined(MKW_VR_STANDALONE) return Get().gxThread.value_or(true); #else return Get().gxThread.value_or(false); diff --git a/runtime/src/settings_overlay.cpp b/runtime/src/settings_overlay.cpp index 60123d6..a6b430a 100644 --- a/runtime/src/settings_overlay.cpp +++ b/runtime/src/settings_overlay.cpp @@ -146,8 +146,8 @@ bool g_vrHudVirtualScreen = RuntimeConfigFile::VrHudVirtualScreen(true); int g_vrRaceView = static_cast(RuntimeConfigFile::GetVrRaceView()); bool g_vrFlatScreen = g_vrRaceView == static_cast(RuntimeConfigFile::VrRaceView::FlatScreen); constexpr std::array kVrRaceViewLabels{"Immersive", "Immersive window", "Flat screen"}; -#if defined(__ANDROID__) -#if !defined(MKW_HEADSET_STEAM_FRAME) +#if defined(MKW_VR_STANDALONE) +#if defined(__ANDROID__) && !defined(MKW_HEADSET_STEAM_FRAME) bool g_vrPassthrough = RuntimeConfigFile::VrPassthrough(); #endif bool g_vrHandTracking = RuntimeConfigFile::VrHandTracking(); @@ -1260,7 +1260,7 @@ void DrawVrSteeringWheelSettings() { "trailed item."); } ImGui::EndDisabled(); -#if defined(__ANDROID__) +#if defined(MKW_VR_STANDALONE) ImGui::BeginDisabled(!g_vrHandSteering && g_vrCockpitItemHand == 2); if (ImGui::Checkbox("Tracked hands", &g_vrHandTracking)) { RuntimeConfigFile::SetVrHandTracking(g_vrHandTracking); @@ -1491,8 +1491,8 @@ void DrawVrSettings() { eyes.scaled_height, g_vrRenderScalePercent); } } -#if defined(__ANDROID__) -#if !defined(MKW_HEADSET_STEAM_FRAME) +#if defined(MKW_VR_STANDALONE) +#if defined(__ANDROID__) && !defined(MKW_HEADSET_STEAM_FRAME) if (ImGui::Checkbox("Passthrough around the menu screen", &g_vrPassthrough)) { mkw::vr::OpenXRSetPassthrough(g_vrPassthrough); RuntimeConfigFile::SetVrPassthrough(g_vrPassthrough); @@ -1829,7 +1829,7 @@ void DrawVrCameraSettings() { RuntimeConfigFile::SetVrHandSteering(g_vrHandSteering); RuntimeConfigFile::SetVrCockpitItemHand(RuntimeConfigFile::kVrCockpitItemHandDefault); RuntimeConfigFile::SetVrCockpitItemThrow(g_vrCockpitItemThrow); -#if defined(__ANDROID__) +#if defined(MKW_VR_STANDALONE) g_vrHandTracking = RuntimeConfigFile::kVrHandTrackingDefault; RuntimeConfigFile::SetVrHandTracking(g_vrHandTracking); #endif @@ -2531,7 +2531,7 @@ void InitializeRuntimeSettings() noexcept { aurora_set_display_mode(static_cast(g_displayMode)); g_displayMode = static_cast(aurora_get_display_mode()); aurora_set_disable_copy_filter(g_disableCopyFilter); -#if defined(__ANDROID__) +#if defined(MKW_VR_STANDALONE) aurora_set_stereo_foveation(static_cast(g_vrFoveation)); #endif aurora_set_stereo_mirror_view(static_cast(g_vrMirrorView)); diff --git a/runtime/src/vr/openxr_integration.cpp b/runtime/src/vr/openxr_integration.cpp index 695b73b..9129939 100644 --- a/runtime/src/vr/openxr_integration.cpp +++ b/runtime/src/vr/openxr_integration.cpp @@ -745,6 +745,11 @@ private: // launched with foveation off does without them. aurora_config.xrFragmentDensityMap = RuntimeConfigFile::VrFoveation() != "off"; #endif +#if defined(MKW_HEADSET_STEAM_FRAME) && !defined(__ANDROID__) + // The Frame's native build: the headset is the only display anyone looks at, so the + // desktop window is neither presented nor fully rendered, as on Android. + aurora_config.xrHeadsetOnly = true; +#endif #if defined(_WIN32) if (kRequiredAuroraBackend != BACKEND_D3D12) return; const auto& requirements = backend_->GraphicsRequirements(); diff --git a/runtime/tests/vr_config_tests.cpp b/runtime/tests/vr_config_tests.cpp index ef3a79c..4e77da5 100644 --- a/runtime/tests/vr_config_tests.cpp +++ b/runtime/tests/vr_config_tests.cpp @@ -39,7 +39,7 @@ int main() { Require(!Parse("[vr]\nfoveation = \"ultra\"\n").vrFoveation.has_value()); Require(!Parse("[vr]\nfoveation = 2\n").vrFoveation.has_value()); Require(!Parse("[vr]\n").vrFoveation.has_value()); -#if defined(__ANDROID__) +#if defined(MKW_VR_STANDALONE) Require(std::string_view(RuntimeConfigFile::kVrFoveationDefault) == "medium"); #else Require(std::string_view(RuntimeConfigFile::kVrFoveationDefault) == "off"); @@ -101,7 +101,7 @@ int main() { Require(Parse("[vr]\nobject_culling = false\n").vrObjectCulling == false); Require(!Parse("[vr]\n").vrObjectCulling.has_value()); Require(!Parse("[vr]\nobject_culling = 0\n").vrObjectCulling.has_value()); -#if defined(__ANDROID__) +#if defined(MKW_VR_STANDALONE) Require(RuntimeConfigFile::kVrObjectCullingDefault); Require(std::string_view(MKW_VR_OBJECT_CULLING_DEFAULT_TOML) == "true"); #else From d8fc2a996cdea4eaa354bb15b2ebfc0ee05c2eb6 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 09:56:46 +0000 Subject: [PATCH 14/14] Steam Frame native build: patched Dawn for Linux, local-build VR options, docs Launcher/build-dawn-linux.sh builds the pinned Dawn with Aurora's Vulkan hook and density map patches into a package local-build.sh takes through the new --dawn-package option, together with --openxr, --headset steam_frame and --cpu. docs/steam-frame.md now leads with the native SteamOS build (building in a podman container on the Frame, running it, the log lines to expect); OPENXR.md's backend table records the Linux Vulkan backend and why the Lepton flavour cannot present. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_019HBRGKTE1GnN2ah8gcZKr3 --- Launcher/build-dawn-linux.sh | 174 +++++++++++++++++++++++++++++++++++ Launcher/local-build.sh | 46 +++++++++ OPENXR.md | 4 +- README.md | 2 +- docs/steam-frame.md | 142 ++++++++++++++++++++-------- 5 files changed, 327 insertions(+), 41 deletions(-) create mode 100755 Launcher/build-dawn-linux.sh diff --git a/Launcher/build-dawn-linux.sh b/Launcher/build-dawn-linux.sh new file mode 100755 index 0000000..4885c96 --- /dev/null +++ b/Launcher/build-dawn-linux.sh @@ -0,0 +1,174 @@ +#!/usr/bin/env bash +# Builds the pinned Dawn for desktop Linux with Aurora's patches (aurora-main/patches/dawn): the +# Vulkan hooks through which the OpenXR runtime creates Dawn's own instance and device (the +# same-device OpenXR backend, runtime/src/vr/openxr_vulkan_win32.cpp) and fragment density maps for +# foveated rendering. The stock prebuilt package has neither. This is the Linux counterpart of +# android/Build-QuestDawn.ps1, for the Steam Frame's native SteamOS build (docs/steam-frame.md). +# +# Launcher/build-dawn-linux.sh [--work-dir DIR] [--cc PATH --cxx PATH] [--cmake PATH] +# [--ninja PATH] [--python PATH] [--jobs N] [--force] +# +# It builds for the machine it runs on (the Frame's aarch64, in a container there). Pass the same +# --cc/--cxx as Launcher/local-build.sh gets: the archive is static and carries C++ objects, so it +# must be built against the same C++ standard library as the game. +# +# The result is an install tree in the stock package's layout under WORK_DIR/package, with an +# aurora-dawn.json declaring the two ABIs (AuroraVulkanAbi, AuroraFdmAbi) that +# aurora-main/cmake/AuroraDawnProvider.cmake reads. local-build.sh --dawn-package takes it. A later +# run with the same inputs reuses it. +# +# Prerequisites: git, python3, curl, tar, CMake 3.25+, Ninja, a C/C++ compiler, and the X11, XCB and +# Wayland development headers Dawn's Vulkan surfaces need. The first build compiles all of Dawn and +# Tint and takes a while. +set -euo pipefail + +fail() { + echo "build-dawn-linux.sh: error: $*" >&2 + exit 1 +} + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +repo=$(cd "$script_dir/.." && pwd) +work_dir="$repo/.scratch/linux-dawn" +cc_bin=${CC:-cc} +cxx_bin=${CXX:-c++} +cmake_bin=cmake +ninja_bin=ninja +python_bin=python3 +jobs=$(nproc) +force=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --work-dir) work_dir=$2; shift 2 ;; + --cc) cc_bin=$2; shift 2 ;; + --cxx) cxx_bin=$2; shift 2 ;; + --cmake) cmake_bin=$2; shift 2 ;; + --ninja) ninja_bin=$2; shift 2 ;; + --python) python_bin=$2; shift 2 ;; + --jobs) jobs=$2; shift 2 ;; + --force) force=1; shift ;; + -h|--help) sed -n '2,24p' "$0"; exit 0 ;; + *) fail "unknown argument: $1" ;; + esac +done + +for tool in "$cc_bin" "$cxx_bin" "$cmake_bin" "$ninja_bin" "$python_bin" git curl tar sha256sum; do + command -v "$tool" >/dev/null 2>&1 || fail "required tool '$tool' was not found" +done +mkdir -p "$work_dir" +work_dir=$(cd "$work_dir" && pwd) +cc_path=$(command -v "$cc_bin") +cxx_path=$(command -v "$cxx_bin") + +# The revision the stock packages, the Windows Vulkan DLL (Launcher/Build-DawnVulkan.ps1) and the +# Quest's Dawn (android/Build-QuestDawn.ps1) are built from. +revision=13abc3bc8ea2d3c2050f9e77a12d012108ceee24 +archive_hash=713bea5b92d4f6c5175752fd7cbf1c3c5ce36598ff5dd98685d8a1216614ebba +flags=( + -DCMAKE_BUILD_TYPE=Release + -DCMAKE_POSITION_INDEPENDENT_CODE=ON + -DDAWN_FETCH_DEPENDENCIES=ON + -DDAWN_BUILD_MONOLITHIC_LIBRARY=STATIC + -DBUILD_SHARED_LIBS=OFF + -DDAWN_ENABLE_INSTALL=ON + -DDAWN_BUILD_SAMPLES=OFF + -DDAWN_BUILD_TESTS=OFF + -DDAWN_BUILD_BENCHMARKS=OFF + -DDAWN_USE_GLFW=OFF + -DDAWN_USE_WAYLAND=ON + -DDAWN_ENABLE_DESKTOP_GL=OFF + -DDAWN_ENABLE_OPENGLES=OFF + -DTINT_BUILD_TESTS=OFF + -DTINT_BUILD_CMD_TOOLS=OFF + -DTINT_BUILD_IR_BINARY=OFF + -DDAWN_BUILD_PROTOBUF=OFF +) + +patch_dir="$repo/aurora-main/patches/dawn" +patch_files=( + "$patch_dir/apply.py" + "$patch_dir/aurora_vulkan_hooks.h" + "$patch_dir/aurora_vulkan_interop.inc" + "$patch_dir/aurora_fdm.h" + "$patch_dir/aurora_fdm.inc" + "$repo/aurora-main/include/aurora/dawn_vulkan_abi.h" + "$repo/aurora-main/include/aurora/dawn_fdm_abi.h" +) +# Hashed with LF line endings and each file's name, as Build-QuestDawn.ps1 does. +patch_hash=$(for file in "${patch_files[@]}"; do + printf '%s\n' "$(basename "$file")" + tr -d '\r' < "$file" +done | sha256sum | awk '{print $1}') +compiler_id=$("$cxx_path" --version 2>/dev/null | head -n 1 | tr -d '"\\') +cache_key=$(printf '%s|%s|%s|%s|%s|%s' "$revision" "$archive_hash" "$patch_hash" "$(uname -m)" \ + "$compiler_id" "${flags[*]}" | sha256sum | awk '{print $1}') + +package="$work_dir/package" +manifest="$package/aurora-dawn.json" +library="$package/lib/libwebgpu_dawn.a" +[[ -f "$library" ]] || library="$package/lib64/libwebgpu_dawn.a" +if [[ "$force" -eq 0 && -f "$manifest" && -f "$library" ]]; then + library_hash=$(sha256sum "$library" | awk '{print $1}') + if grep -q "\"CacheKey\": \"$cache_key\"" "$manifest" && + grep -q "\"ArchiveSha256\": \"$library_hash\"" "$manifest"; then + echo "Patched Dawn for Linux is up to date: $package" + exit 0 + fi +fi + +archive="$work_dir/dawn-source.tar.gz" +if [[ ! -f "$archive" ]]; then + curl -fL --retry 3 -o "$archive.partial" "https://github.com/google/dawn/archive/$revision.tar.gz" + mv "$archive.partial" "$archive" +fi +[[ "$(sha256sum "$archive" | awk '{print $1}')" == "$archive_hash" ]] || + fail "Dawn source archive does not match the pinned SHA-256; delete $archive and try again" + +# The patches are applied to pristine sources: src/ is extracted again whenever the patches changed, +# while third_party/, which DAWN_FETCH_DEPENDENCIES fills, is kept. +source="$work_dir/dawn-$revision" +patch_marker="$source/aurora-patches.sha256" +if [[ ! -f "$patch_marker" || "$(cat "$patch_marker")" != "$patch_hash" ]]; then + if [[ -d "$source" ]]; then + rm -rf "$source/src" + tar -xzf "$archive" -C "$work_dir" "dawn-$revision/src" + else + tar -xzf "$archive" -C "$work_dir" + fi + "$python_bin" "$patch_dir/apply.py" "$source" + printf '%s' "$patch_hash" > "$patch_marker" +fi + +build="$work_dir/build" +"$cmake_bin" -S "$source" -B "$build" -G Ninja \ + -DCMAKE_MAKE_PROGRAM="$(command -v "$ninja_bin")" \ + -DCMAKE_C_COMPILER="$cc_path" -DCMAKE_CXX_COMPILER="$cxx_path" \ + -DPython3_EXECUTABLE="$(command -v "$python_bin")" \ + "${flags[@]}" -DCMAKE_INSTALL_PREFIX="$package" +"$cmake_bin" --build "$build" --parallel "$jobs" +rm -rf "$package" +"$cmake_bin" --install "$build" +library="$package/lib/libwebgpu_dawn.a" +[[ -f "$library" ]] || library="$package/lib64/libwebgpu_dawn.a" +[[ -f "$library" ]] || fail "Dawn archive missing under $package" +# As the stock package: debug info would only make the archive several times larger. +strip_bin=$(dirname "$cc_path")/llvm-strip +[[ -x "$strip_bin" ]] || strip_bin=strip +"$strip_bin" --strip-debug "$library" + +cat > "$manifest" < --headset steam_frame`. See `docs/steam-frame.md`. | | Other platforms | Not wired yet. | Both bindings share `openxr_integration.cpp`: the pacing thread, policy evaluation, the diff --git a/README.md b/README.md index 25db3ea..a7142a7 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ to immersive stereo rendering. VR is opt-in and falls back to the normal desktop runtime or headset is unavailable. In first person you sit in the cockpit, where the steering wheel or handlebar turns with your steering, and hand steering by heurazy lets you grab it with the tracked controllers and turn it. On a Quest the hands can follow the headset's own hand tracking. -A Steam Frame build of the Android app (not yet tested on the headset) adds the Frame controllers' +A native SteamOS build for the Steam Frame (not yet tested on the headset) adds the Frame controllers' D-pad, a 120 Hz display for the game's 60 FPS, and foveation that follows your eyes; see [`docs/steam-frame.md`](docs/steam-frame.md). See [`OPENXR.md`](OPENXR.md) for setup, configuration, and the current limitations. diff --git a/docs/steam-frame.md b/docs/steam-frame.md index c12bfce..6261b1e 100644 --- a/docs/steam-frame.md +++ b/docs/steam-frame.md @@ -1,23 +1,112 @@ # WiiCompiled VR on the Steam Frame Valve's Steam Frame runs SteamOS on a Snapdragon 8 Gen 3 (Cortex-X4, A720 and A520 cores, Adreno 750), -with 2160x2160 panels per eye at 72 to 144 Hz, eye tracking, and SteamVR as its OpenXR runtime. It runs -Android apps through Lepton, SteamOS's Android layer, where SteamVR provides an Android OpenXR runtime -(OpenXR 1.0, through the Khronos loader's runtime broker). The Steam Frame build is therefore a third -flavour of the Quest app, `steamFrame`: everything in `docs/quest-port.md` below the app shell (the -Vulkan backend, the game kit, `.wcgame` packages, the on-headset build) applies unchanged, and this -document covers what differs. +with 2160x2160 panels per eye at 72 to 144 Hz, eye tracking, and SteamVR as its OpenXR runtime. A game +can run on it two ways, and this project has both: -A native SteamOS ARM64 build is a separate, later piece of work: Linux has no OpenXR graphics backend -yet (see [A native SteamOS build](#a-native-steamos-build)). +- **Natively on SteamOS** (Linux ARM64), with SteamVR's own OpenXR runtime. This is the Frame's build: + [The native SteamOS build](#the-native-steamos-build) says how to make and run it. +- **As an Android app in Lepton**, SteamOS's Android layer, as a third flavour of the Quest app + (`steamFrame`). It is built, but it cannot show a picture there: Lepton's Vulkan driver has no + external memory or sync fd extensions, and the Quest backend's two-device eye handoff needs them + (see [What the Frame reported](#what-the-frame-reported)). -**Status: not yet run on a Steam Frame, and the Android backend as it stands cannot present under -Lepton**: Lepton's Vulkan driver has no external memory or sync fd extensions, which the Quest -backend's two-device design needs (see [What the Frame reported](#what-the-frame-reported)). The -flavour, controller, refresh rate and foveation work below stays valid; the eye handoff has to move -to a single shared device first. +Most of what this document describes is shared by both: the Frame controller profile, the 120 Hz +request, eye-tracked foveation and the Frame's defaults. The native build gets them through +`MKW_HEADSET=steam_frame` (`MKW_HEADSET_STEAM_FRAME`), as the Android flavour does. -## What the flavour changes +**Status: not yet run on a Steam Frame.** The native build's VR code compiles and the unit tests +pass; building it on the Frame and the device checks are still to do. + +## The native SteamOS build + +The backend is the PC's same-device Vulkan backend (`openxr_vulkan_win32.cpp`, on Linux too): the +OpenXR runtime creates Dawn's own Vulkan instance and device through Aurora's patches to Dawn, and +each eye is copied into SteamVR's swapchain on Dawn's queue, so nothing is shared between devices. +On Linux, Dawn links statically, so the patched Dawn is built once on the build machine +(`Launcher/build-dawn-linux.sh`); `aurora-dawn.json` in its package declares the Vulkan hook and +density map ABIs, and only against such a package does Aurora compile the bridge +(`AURORA_DAWN_VULKAN_HOOKS`) and the density maps (`AURORA_DAWN_FDM`). Against a stock Dawn the +build still links, and VR falls back to the desktop. + +What the Frame build changes, beyond the Android flavour's settings: + +- `-mcpu=cortex-x4` (`MKW_LINUX_CPU`, which `--cpu` overrides). +- `AuroraConfig::xrHeadsetOnly`: Aurora neither presents the desktop window nor renders it past the + last pass the eyes sample, as on Android (4 to 6 ms of a 12 ms GPU frame on a Quest 3). +- Fragment density maps are asked for on Linux as on the Quest; `AURORA_FDM=0` or `1` overrides the + settings, as `debug.wiicompiled.fdm` does there. +- Controller motion uses `XR_KHR_convert_timespec_time` when SteamVR offers it. + +### Building it on the Frame + +SteamOS's root file system is read-only, so the build runs in a Debian container on the Frame, +started with the `podman` SteamOS already ships. Over SSH (`ssh steamos@`): + +```bash +mkdir -p ~/wiicompiled && cd ~/wiicompiled +git clone -b claude/peaceful-keller-2ek99b https://github.com/mitch030504/Wiicompiled_VR_Frame.git +podman run -it --name wiicompiled-build -v ~/wiicompiled:/work:Z docker.io/library/debian:trixie bash +``` + +Inside the container (`podman start -ai wiicompiled-build` gets back into it later): + +```bash +apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates curl git python3 xz-utils unzip file pkg-config g++ binutils libicu-dev zlib1g-dev \ + libvulkan-dev libx11-dev libx11-xcb-dev libxcb1-dev libxext-dev libxrandr-dev libxinerama-dev \ + libxcursor-dev libxi-dev libxss-dev libxtst-dev libxkbcommon-dev libwayland-dev wayland-protocols \ + libdecor-0-dev libegl-dev libgl-dev libgles-dev libdrm-dev libgbm-dev libasound2-dev libpulse-dev \ + libpipewire-0.3-dev libudev-dev libdbus-1-dev libusb-1.0-0-dev +cd /work/Wiicompiled_VR_Frame +Launcher/prepare-portable-tools.sh --arch aarch64 --destination /work/tools # clang 22, CMake, Ninja +T=/work/tools/toolchain-aarch64/bin +curl -fsSL https://dot.net/v1/dotnet-install.sh | bash -s -- --channel 8.0 --install-dir /work/dotnet +Launcher/build-dawn-linux.sh --work-dir /work/dawn --cc $T/clang --cxx $T/clang++ --cmake $T/cmake --ninja $T/ninja +``` + +Then the game. `local-build.sh` translates your own disc, so it needs `main.dol` and `StaticR.rel` from +your extracted PAL `RMCP01` disc in `Assets/` (the extracted disc's `sys/main.dol` and +`files/rel/StaticR.rel`; `translator/README.md` explains): + +```bash +mkdir -p Assets && cp /sys/main.dol /files/rel/StaticR.rel Assets/ +Launcher/local-build.sh --output-dir /work/out --cc $T/clang --cxx $T/clang++ --fuse-ld lld \ + --cmake $T/cmake --ninja $T/ninja --dotnet /work/dotnet/dotnet \ + --openxr --dawn-package /work/dawn/package --headset steam_frame +``` + +The game lands in `~/wiicompiled/out` on the Frame. Debian trixie's C library is not newer than +SteamOS's, so the binary runs on SteamOS outside the container. If CMake reports a missing package, +install its `-dev` package in the container and run the same command again; both scripts resume +where they stopped. + +### Running it + +The game reads its `Config.toml` from `~/.local/share/WiiCompiled/` on SteamOS (it is created on the first start): set +`[paths] dvd_root` there to your extracted disc (the directory holding `sys/` and `files/`). Start +SteamVR on the Frame, then start `~/wiicompiled/out/WiiCompiled`, from Desktop Mode or as a +non-Steam game added to the library. The run log is in `Logs/` next to `Config.toml`; it should show, +in order: + +1. `OpenXR runtime offers N extensions: ...`, and `OpenXR initialized: runtime 'SteamVR/OpenXR'`; +2. `OpenXR Vulkan requirements: ... Dawn will create its device through the runtime`; +3. `Fragment density maps: enabled` (the patched Dawn and Turnip's density maps); +4. `OpenXR Vulkan swapchains ready ... same-queue native eye copies`; +5. `display refresh rate 120 Hz requested`, the session reaching `FOCUSED`, and + `OpenXR interaction profiles: left /interaction_profiles/valve/frame_controller_valve`; +6. `OpenXR eye gaze: available`, then `tracking`. + +`Linux Vulkan OpenXR requires a Dawn built with Aurora's patches` means the build used a stock Dawn: +check that `--dawn-package` pointed at `build-dawn-linux.sh`'s `package` directory. + +## The Android flavour in Lepton + +Everything from here to [Building and installing](#building-and-installing) is the `steamFrame` +flavour of the Quest app. Its controller, refresh rate and foveation work is shared with the native +build; its launch and manifest are Lepton's. + +### What the flavour changes | | Quest flavours | `steamFrame` | | --- | --- | --- | @@ -185,7 +274,7 @@ What follows from them: through exactly those, so it cannot present under Lepton. What can: binding Dawn's own device to the session, as the Windows Vulkan backend (`openxr_vulkan_win32.cpp`) does, so the eyes are copied into the swapchain on Dawn's queue with no sharing at all. That backend is also the core - of a native SteamOS build ([below](#a-native-steamos-build)). + of a native SteamOS build ([above](#the-native-steamos-build)). - **Finding the runtime.** An app inside Lepton reaches SteamVR's OpenXR runtime through the system runtime file, not a broker. The Khronos loader the game links statically (`DYNAMIC_LOADER OFF`) tries the runtime brokers first, then reads `/{product,odm,oem,vendor,system}/etc/openxr/1/active_runtime.json`, @@ -315,26 +404,3 @@ Open questions only the device can answer: - Whether the gaze needs an Android permission under Lepton. - Whether "Build on this headset" can run its toolchain through `/system/bin/linker64` inside Lepton. A game built on the PC does not depend on it. - -## A native SteamOS build - -Valve recommends native Linux ARM64 builds for the Frame, and one would avoid Lepton and the -two-device copy. The pieces: - -- **Backend.** The Windows Vulkan backend (`openxr_vulkan_win32.cpp`), where OpenXR creates Dawn's - own device and eyes are copied on Dawn's queue, ports almost as it is. Only its `_WIN32` guards are - platform-specific. -- **Interop.** Aurora's `vulkan_win32_interop.cpp` finds the patched Dawn's exports with - `GetModuleHandleW`. A static Linux Dawn would reference them directly, and `aurora_core.cmake` - compiles that file on Windows only. -- **Dawn.** A linux-aarch64 Dawn built with `aurora-main/patches/dawn` (the hook ABI and density - maps), as `android/Build-QuestDawn.ps1` already does for Android. -- **`openxr_integration.cpp`.** It needs a Linux branch asking for `XR_KHR_vulkan_enable2` and - `XR_KHR_convert_timespec_time`. Today its `#else` is Android's and requires - `XR_KHR_android_create_instance`. -- **Controller timing.** `openxr_input.cpp` needs a `__linux__` branch for the input clock. -- **CPU target.** An `MKW_LINUX_CPU` knob in place of `-mcpu=native`, for cross-builds. -- **Android-gated fixes.** The Adreno vertex padding, `headset_owns_display` and the - `last_pass_feeding_replay` saving are gated on `__ANDROID__`. They would follow the GPU or the - headset instead. -- **Packaging.** SteamOS packaging, and a way to build the player's game for it.