Centre foveation on the player's gaze on headsets with eye tracking

With [vr] eye_tracked_foveation (on by default on the Steam Frame, off
elsewhere) the runtime asks for XR_EXT_eye_gaze_interaction. When the
system reports an eye tracker, OpenXRInput binds the gaze pose and
locates it for each packet's display time, in the space the eye views
are located in; vr/eye_gaze.h turns it into tangents of each eye's own
view, which AuroraStereoFrame now carries (appended, after the existing
prefix).

Aurora centres the eye's fragment density map on the gaze snapped to a
cell of two map texels (about 3 degrees). Each eye keeps up to 32 maps,
one per cell looked at, so a glance back reuses its map; a new map is
bound once its upload completes, and until then the eye keeps the map
it had. Without a tracked gaze (a blink, no tracker, the setting off)
foveation centres on the forward direction exactly as before: the
forward maps are byte-identical.

Also logs every extension the OpenXR runtime offers at startup, so the
first Steam Frame session shows what SteamVR's Android runtime has.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_019HBRGKTE1GnN2ah8gcZKr3

runtime/src/settings_overlay.cpp

@@ -156,6 +156,7 @@ bool g_vrHandTracking = RuntimeConfigFile::VrHandTracking();
 constexpr std::array<const char*, 4> kVrFoveationLabels{"Off", "Low", "Medium", "High"};
 static_assert(kVrFoveationLabels.size() == RuntimeConfigFile::kVrFoveationLevels.size());
 int g_vrFoveation = static_cast<int>(RuntimeConfigFile::VrFoveationLevelIndex(RuntimeConfigFile::VrFoveation()));
+bool g_vrEyeTrackedFoveation = RuntimeConfigFile::VrEyeTrackedFoveation();
 #endif
 bool g_vrFirstPerson = RuntimeConfigFile::VrFirstPerson(false);
 bool g_vrFirstPersonToggleClick = RuntimeConfigFile::VrFirstPersonToggleClick();
@@ -1523,6 +1524,15 @@ void DrawVrSettings() {
                         "lenses blur the picture anyway, to free GPU time. This session started with it "
                         "off, or without a GPU that supports it: a new level applies after a restart.");
     }
+    if (ImGui::Checkbox("Foveation follows the eyes", &g_vrEyeTrackedFoveation)) {
+        RuntimeConfigFile::SetVrEyeTrackedFoveation(g_vrEyeTrackedFoveation);
+    }
+    if (ImGui::IsItemHovered()) {
+        ImGui::SetTooltip(
+            "With a headset that tracks the eyes (the Steam Frame), the sharp centre of the foveated "
+            "race view moves to where you look instead of staying straight ahead. Turning it off "
+            "applies immediately; turning it on needs a restart if the session started without it.");
+    }
 #endif
     {
         const auto rateLabel = [](uint32_t hz) {

runtime/src/vr/openxr_input.cpp

@@ -423,6 +423,36 @@ bool OpenXRInput::CreateActions() {
             return false;
         }
     }
+    // Eye-tracked foveation: one gaze pose for both eyes, with no hand to name.
+    if (EyeGazeOffered()) {
+        XrActionCreateInfo info{XR_TYPE_ACTION_CREATE_INFO};
+        info.actionType = XR_ACTION_TYPE_POSE_INPUT;
+        std::strncpy(info.actionName, "eye_gaze", XR_MAX_ACTION_NAME_SIZE - 1);
+        std::strncpy(info.localizedActionName, "Eye Gaze", XR_MAX_LOCALIZED_ACTION_NAME_SIZE - 1);
+        if (!Check(xrCreateAction(m_action_set, &info, &m_gaze_pose), "eye_gaze")) {
+            // Not worth the controllers: carry on with the fixed foveation centre.
+            m_gaze_pose = XR_NULL_HANDLE;
+        }
+    }
+    return true;
+}
+
+bool OpenXRInput::EyeGazeOffered() {
+    const auto& extensions = m_runtime->EnabledExtensions();
+    if (std::find(extensions.begin(), extensions.end(), "XR_EXT_eye_gaze_interaction") == extensions.end()) {
+        return false;
+    }
+    XrSystemEyeGazeInteractionPropertiesEXT gaze{XR_TYPE_SYSTEM_EYE_GAZE_INTERACTION_PROPERTIES_EXT};
+    XrSystemProperties properties{XR_TYPE_SYSTEM_PROPERTIES, &gaze};
+    const XrResult result = xrGetSystemProperties(m_runtime->Instance(), m_runtime->SystemId(), &properties);
+    if (XR_FAILED(result) || gaze.supportsEyeGazeInteraction != XR_TRUE) {
+        std::ostringstream message;
+        message << "OpenXR eye gaze: the runtime reports no eye tracker (" << result
+                << "); foveation stays on each eye's forward direction";
+        Log(OpenXRLogLevel::Info, message.str());
+        return false;
+    }
+    Log(OpenXRLogLevel::Info, "OpenXR eye gaze: available; foveation follows the gaze ([vr] eye_tracked_foveation)");
     return true;
 }
 
@@ -533,6 +563,10 @@ bool OpenXRInput::SuggestBindings() {
         };
         suggest("/interaction_profiles/valve/frame_controller_valve", frame, false);
     }
+    if (m_gaze_pose != XR_NULL_HANDLE) {
+        suggest("/interaction_profiles/ext/eye_gaze_interaction",
+                {{&m_gaze_pose, "/user/eyes_ext/input/gaze_ext/pose"}}, false);
+    }
     return true;
 }
 
@@ -558,9 +592,56 @@ void OpenXRInput::CreatePoseSpaces() {
             }
         }
     }
+    if (m_gaze_pose != XR_NULL_HANDLE) {
+        XrActionSpaceCreateInfo info{XR_TYPE_ACTION_SPACE_CREATE_INFO};
+        info.action = m_gaze_pose;
+        info.poseInActionSpace.orientation.w = 1.0f;
+        const XrResult result = xrCreateActionSpace(m_runtime->Session(), &info, &m_gaze_space);
+        m_runtime->ObserveResult(result);
+        if (XR_FAILED(result)) {
+            m_gaze_space = XR_NULL_HANDLE;
+            std::ostringstream message;
+            message << "xrCreateActionSpace(eye gaze) failed (" << result
+                    << "); foveation stays on each eye's forward direction";
+            Log(OpenXRLogLevel::Warning, message.str());
+        }
+    }
+}
+
+// Eye-tracked foveation: the gaze for the display time the eyes are rendered for, the time their
+// views are located at, so the full-density region lands where the eyes look in that frame. Only a
+// tracked orientation counts; the runtime reports an untracked one through blinks.
+void OpenXRInput::LocateEyeGaze(XrTime time) {
+    m_gaze_valid = false;
+    if (m_gaze_space == XR_NULL_HANDLE) {
+        return;
+    }
+    XrActionStateGetInfo info{XR_TYPE_ACTION_STATE_GET_INFO};
+    info.action = m_gaze_pose;
+    XrActionStatePose state{XR_TYPE_ACTION_STATE_POSE};
+    if (XR_FAILED(xrGetActionStatePose(m_runtime->Session(), &info, &state)) || state.isActive != XR_TRUE) {
+        return;
+    }
+    constexpr XrSpaceLocationFlags kTracked =
+        XR_SPACE_LOCATION_ORIENTATION_VALID_BIT | XR_SPACE_LOCATION_ORIENTATION_TRACKED_BIT;
+    XrSpaceLocation location{XR_TYPE_SPACE_LOCATION};
+    if (XR_FAILED(xrLocateSpace(m_gaze_space, m_runtime->AppSpace(), time, &location)) ||
+        (location.locationFlags & kTracked) != kTracked) {
+        return;
+    }
+    m_gaze_orientation = location.pose.orientation;
+    m_gaze_valid = true;
+    if (!m_gaze_logged) {
+        m_gaze_logged = true;
+        Log(OpenXRLogLevel::Info, "OpenXR eye gaze: tracking");
+    }
 }
 
 void OpenXRInput::DestroyPoseSpaces() {
+    if (m_gaze_space != XR_NULL_HANDLE) {
+        xrDestroySpace(m_gaze_space);
+        m_gaze_space = XR_NULL_HANDLE;
+    }
     for (uint32_t hand = 0; hand < kHandCount; ++hand) {
         for (XrSpace* space : {&m_aim_spaces[hand], &m_grip_spaces[hand]}) {
             if (*space != XR_NULL_HANDLE) {
@@ -909,6 +990,9 @@ void OpenXRInput::Destroy() {
     m_thumbstick = m_thumbstick_click = m_trigger = m_squeeze = XR_NULL_HANDLE;
     m_button_primary = m_button_secondary = m_menu = m_haptic = XR_NULL_HANDLE;
     m_dpad_up = m_dpad_down = m_dpad_left = m_dpad_right = XR_NULL_HANDLE;
+    m_gaze_pose = XR_NULL_HANDLE;
+    m_gaze_valid = false;
+    m_gaze_logged = false;
     m_aim_pose = m_grip_pose = XR_NULL_HANDLE;
     m_hand_paths[0] = m_hand_paths[1] = XR_NULL_PATH;
     m_convert_now_to_xr_time = nullptr;
@@ -933,6 +1017,7 @@ void OpenXRInput::Idle() {
     }
     m_pointer.Reset();
     m_horizon = {1.0f, 0.0f};
+    m_gaze_valid = false;
     OpenXRPublishWiiRemote(Relay().JoystickId(), OpenXRWiiRemoteSample{});
     // The panel stays as it was; only what the controllers were holding is forgotten.
     m_panel_controls.Reset();
@@ -976,6 +1061,7 @@ void OpenXRInput::Sync(XrTime predicted_display_time, const OpenXRPointerScreen&
         Idle();
         return;
     }
+    LocateEyeGaze(predicted_display_time);
 
     // `active`, when given, says whether the action is bound to a source the
     // runtime has right now (a controller, or a tracked hand).

runtime/src/vr/openxr_integration.cpp

@@ -9,6 +9,7 @@
 #include "runtime_config.h"
 #include "gx_thread.h"
 #include "runtime_log.h"
+#include "vr/eye_gaze.h"
 #include "vr/mkw_vr_culling.h"
 #include "vr/mkw_vr_first_person.h"
 #include "vr/mkw_vr_policy.h"
@@ -428,6 +429,10 @@ public:
         // Horizon OS's room view; the Steam Frame build neither asks for it nor offers the setting.
         config.optional_extensions.push_back("XR_FB_passthrough");
 #endif
+        // Eye-tracked foveation: the gaze the density maps centre on (OpenXRInput::EyeGaze).
+        if (RuntimeConfigFile::VrEyeTrackedFoveation()) {
+            config.optional_extensions.push_back("XR_EXT_eye_gaze_interaction");
+        }
         AddHandMeshExtensions(config);
         config.instance_create_next = OpenXRAndroidInstanceCreateNext();
 #endif
@@ -1465,9 +1470,30 @@ private:
                          position_valid && base_position_valid_, units_per_meter,
                          lean_back_radians, destination.eyes[eye].viewFromCenter);
         }
+        BuildEyeGaze(source, destination);
         BuildCockpit(source, position_valid, units_per_meter, lean_back_radians, destination.cockpit);
     }
 
+    // Eye-tracked foveation: where the eyes look, in each eye's own view (the views may be canted),
+    // from the gaze the input located for this packet's display time. Without a tracked gaze, or
+    // with the setting off, Aurora centres foveation on each eye's forward direction.
+    void BuildEyeGaze(const OpenXRBackendFrame& source, AuroraStereoFrame& destination) const noexcept {
+        XrQuaternionf gaze{};
+        if (input_ == nullptr || !RuntimeConfigFile::VrEyeTrackedFoveation() || !input_->EyeGaze(&gaze)) {
+            return;
+        }
+        bool valid = true;
+        for (uint32_t eye = 0; eye < kOpenXREyeCount; ++eye) {
+            const XrQuaternionf& view = source.xr_frame.views[eye].pose.orientation;
+            const eye_gaze::Tangents seen =
+                eye_gaze::InEye({gaze.x, gaze.y, gaze.z, gaze.w}, {view.x, view.y, view.z, view.w});
+            valid = valid && seen.valid;
+            destination.gaze[eye][0] = seen.x;
+            destination.gaze[eye][1] = seen.y;
+        }
+        destination.gazeValid = valid;
+    }
+
     // The first-person cockpit's hands and separate wheel, in the seated frame
     // the eye transforms place at base + lean * seat (metres). Always carries
     // the packet's world scale, which Aurora rescales to the sealed frame's.

runtime/src/vr/openxr_runtime.cpp

@@ -151,6 +151,14 @@ bool OpenXRRuntime::EnumerateInstanceCapabilities() {
     for (const XrExtensionProperties& extension : extension_properties) {
         m_available_extensions.emplace_back(extension.extensionName);
     }
+    // Everything the runtime offers, once per instance: what a new headset's runtime can do (the
+    // Steam Frame's SteamVR, say) is in the first session log rather than behind a debugger.
+    std::ostringstream offered;
+    offered << "OpenXR runtime offers " << m_available_extensions.size() << " extensions:";
+    for (const std::string& name : m_available_extensions) {
+        offered << ' ' << name;
+    }
+    Log(OpenXRLogLevel::Info, offered.str());
 
     uint32_t layer_count = 0;
     if (!Check(xrEnumerateApiLayerProperties(0, &layer_count, nullptr),